diff --git a/create_db.py b/create_db.py index e13071d..08a6973 100755 --- a/create_db.py +++ b/create_db.py @@ -8,6 +8,7 @@ import re import time import logging import argparse as arg +from timer.Timer import Timer class CustomFormatter(logging.Formatter): @@ -174,31 +175,43 @@ if __name__ == '__main__': logger.setLevel(logging.DEBUG) logger.debug('DEBUG mode activated') + t = Timer(logger=logger.info) + + t.start('Import_CSV') if not os.path.exists(args.source + '/' + args.states): logger.critical('can\'t find source file for states') sys.exit(1) states = import_states_csv(args.source + '/' + args.states) + t.stop() logger.debug(states) + t.start(name='Import_CSV') if not os.path.exists(args.source + '/' + args.departments): logger.critical('can\'t find source file for departments') sys.exit(1) departments = import_department_csv(args.source + '/' + args.departments) + t.stop() logger.debug(departments) - + + t.start('Import_CSV') if not os.path.exists(args.source + '/' + args.towns): logger.critical('can\'t find source file for departments') sys.exit(1) towns = import_towns_csv(args.source + '/' + args.towns) + t.stop() logger.debug(towns) - + + + t.start('Import_CSV') if not os.path.exists(args.source + '/' + args.statistics): logger.critical('can\'t find source file for statistics') sys.exit(1) statistics = import_statistics_csv(args.source + '/' + args.statistics) + t.stop() logger.debug(statistics) + t.get_time_by_tag('Import_CSV') # Create missing table : indicators indicators = pd.DataFrame({'indicateur': [ 'population', @@ -228,9 +241,8 @@ if __name__ == '__main__': ## create statistics dataframes # # We need to first iterate on statistics - if args.verbose or arg.debug: - t_begin = time.time() - logger.info('BEGIN - import stats') + if args.verbose or args.debug: + t.start() c_stats = pd.DataFrame(columns = ['com','id_indicateur','date_debut', 'date_fin','valeur'] @@ -277,8 +289,7 @@ if __name__ == '__main__': temp['date_fin'].append(end) temp['valeur'].append(value) - if args.verbose or arg.debug: - t_end = time.time() - logger.info('END stats import, time: {} seconds'.format(t_end - t_begin)) + t.stop() + t.get_total_time() sys.exit() diff --git a/timer/Timer.py b/timer/Timer.py new file mode 100644 index 0000000..6d0beb5 --- /dev/null +++ b/timer/Timer.py @@ -0,0 +1,63 @@ +import time + + +class TimerError(Exception): + """ Manage timer errors """ + +class Timer: + + def __init__( + self, + text_out = '{}: Elapsed time: {:0.4f} seconds', + text_in = 'Start {}', + text_tag = 'Time elapsed for {}: {:0.4f} seconds', + text_total = 'Total recorded time: {:0.4f} seconds', + logger=print, + ): + self._start_time = None + self.name = None + self.text_in = text_in + self.text_out = text_out + self.text_tag = text_tag + self.text_total = text_total + self.logger = logger + self.timers = {} + + def start(self, name='General'): + """Start a new timer""" + + self.name = name + # if timer does not exists, initiate it + if not name in self.timers: + self.timers[self.name] = 0 + + if self._start_time is not None: + raise TimerError('Timer is running. Use .stop() to stop it') + + self.logger(self.text_in.format(self.name)) + self._start_time = time.perf_counter() + + def stop(self): + """Stop the timer, and report the elapsed time""" + if self._start_time is None: + raise TimerError('Timer is not running. Use .start() to start it') + + elapsed_time = time.perf_counter() - self._start_time + self._start_time = None + + if self.logger: + self.logger(self.text_out.format(self.name, elapsed_time)) + + if self.name: + self.timers[self.name] += elapsed_time + + def get_time_by_tag(self, tag): + if tag in self.timers: + self.logger(self.text_tag.format(tag,self.timers[tag])) + + def get_total_time(self): + total = 0 + for v in self.timers.values(): + total += v + self.logger(self.text_total.format(total)) +