IDI_Master/projet_INSEE
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add a timet class to check performance

Browse source
This commit is contained in:
Yorick Barbanneau 2022-04-19 21:55:27 +02:00
parent f84b6a1367
commit 77456db238
2 changed files with 82 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

27
create_db.py
View file

@ -8,6 +8,7 @@ import re
import time
import logging
import argparse as arg
from timer.Timer import Timer
class CustomFormatter(logging.Formatter):
@ -174,31 +175,43 @@ if __name__ == '__main__':
logger.setLevel(logging.DEBUG)
logger.debug('DEBUG mode activated')
t = Timer(logger=logger.info)
t.start('Import_CSV')
if not os.path.exists(args.source + '/' + args.states):
logger.critical('can\'t find source file for states')
sys.exit(1)
states = import_states_csv(args.source + '/' + args.states)
t.stop()
logger.debug(states)
t.start(name='Import_CSV')
if not os.path.exists(args.source + '/' + args.departments):
logger.critical('can\'t find source file for departments')
sys.exit(1)
departments = import_department_csv(args.source + '/' + args.departments)
t.stop()
logger.debug(departments)
t.start('Import_CSV')
if not os.path.exists(args.source + '/' + args.towns):
logger.critical('can\'t find source file for departments')
sys.exit(1)
towns = import_towns_csv(args.source + '/' + args.towns)
t.stop()
logger.debug(towns)
t.start('Import_CSV')
if not os.path.exists(args.source + '/' + args.statistics):
logger.critical('can\'t find source file for statistics')
sys.exit(1)
statistics = import_statistics_csv(args.source + '/' + args.statistics)
t.stop()
logger.debug(statistics)
t.get_time_by_tag('Import_CSV')
# Create missing table : indicators
indicators = pd.DataFrame({'indicateur': [
'population',
@ -228,9 +241,8 @@ if __name__ == '__main__':
## create statistics dataframes
#
# We need to first iterate on statistics
if args.verbose or arg.debug:
t_begin = time.time()
logger.info('BEGIN - import stats')
if args.verbose or args.debug:
t.start()
c_stats = pd.DataFrame(columns = ['com','id_indicateur','date_debut',
'date_fin','valeur']
@ -277,8 +289,7 @@ if __name__ == '__main__':
temp['date_fin'].append(end)
temp['valeur'].append(value)
if args.verbose or arg.debug:
t_end = time.time()
logger.info('END stats import, time: {} seconds'.format(t_end - t_begin))
t.stop()
t.get_total_time()
sys.exit()