Add a timet class to check performance

create_db.py

@@ -8,6 +8,7 @@ import re
 import time
 import logging
 import argparse as arg
+from timer.Timer import Timer
 
 class CustomFormatter(logging.Formatter):
 
@@ -174,31 +175,43 @@ if __name__ == '__main__':
         logger.setLevel(logging.DEBUG)
         logger.debug('DEBUG mode activated')
     
+    t = Timer(logger=logger.info)
+    
+    t.start('Import_CSV')
     if not os.path.exists(args.source + '/' + args.states):
         logger.critical('can\'t find source file for states')
         sys.exit(1)
 
     states = import_states_csv(args.source + '/' + args.states)
+    t.stop()
     logger.debug(states)
     
+    t.start(name='Import_CSV')
     if not os.path.exists(args.source + '/' + args.departments):
         logger.critical('can\'t find source file for departments')
         sys.exit(1)
     departments = import_department_csv(args.source + '/' + args.departments)
+    t.stop()
     logger.debug(departments)
-    
+
+    t.start('Import_CSV')
     if not os.path.exists(args.source + '/' + args.towns):
         logger.critical('can\'t find source file for departments')
         sys.exit(1)
     towns = import_towns_csv(args.source + '/' + args.towns)
+    t.stop()
     logger.debug(towns)
-    
+
+
+    t.start('Import_CSV')
     if not os.path.exists(args.source + '/' + args.statistics):
         logger.critical('can\'t find source file for statistics')
         sys.exit(1)
     statistics = import_statistics_csv(args.source + '/' + args.statistics)
+    t.stop()
     logger.debug(statistics)
     
+    t.get_time_by_tag('Import_CSV')
     # Create missing table : indicators
     indicators = pd.DataFrame({'indicateur': [
         'population', 
@@ -228,9 +241,8 @@ if __name__ == '__main__':
     ## create statistics dataframes
     # 
     # We need to first iterate on statistics
-    if args.verbose or arg.debug:
+    if args.verbose or args.debug:
-        t_begin = time.time()
+        t.start()
-        logger.info('BEGIN - import stats')
         
     c_stats = pd.DataFrame(columns = ['com','id_indicateur','date_debut',
         'date_fin','valeur']
@@ -277,8 +289,7 @@ if __name__ == '__main__':
                 temp['date_fin'].append(end)
                 temp['valeur'].append(value)
 
-    if args.verbose or arg.debug:
+    t.stop()
-        t_end = time.time()
+    t.get_total_time()
-        logger.info('END stats import, time: {} seconds'.format(t_end - t_begin))
 
     sys.exit()

timer/Timer.py

@@ -0,0 +1,63 @@
+import time
+
+
+class TimerError(Exception):
+    """ Manage timer errors """
+
+class Timer:
+
+    def __init__(
+        self,
+        text_out = '{}: Elapsed time: {:0.4f} seconds',
+        text_in = 'Start {}',
+        text_tag = 'Time elapsed for {}: {:0.4f} seconds',
+        text_total = 'Total recorded time: {:0.4f} seconds',
+        logger=print,
+    ):
+        self._start_time = None
+        self.name = None
+        self.text_in = text_in
+        self.text_out = text_out
+        self.text_tag = text_tag
+        self.text_total = text_total
+        self.logger = logger
+        self.timers = {}
+
+    def start(self, name='General'):
+        """Start a new timer"""
+        
+        self.name = name
+        # if timer does not exists, initiate it
+        if not name in self.timers:
+            self.timers[self.name] = 0
+
+        if self._start_time is not None:
+            raise TimerError('Timer is running. Use .stop() to stop it')
+
+        self.logger(self.text_in.format(self.name))
+        self._start_time = time.perf_counter()
+
+    def stop(self):
+        """Stop the timer, and report the elapsed time"""
+        if self._start_time is None:
+            raise TimerError('Timer is not running. Use .start() to start it')
+
+        elapsed_time = time.perf_counter() - self._start_time
+        self._start_time = None
+        
+        if self.logger:
+            self.logger(self.text_out.format(self.name, elapsed_time))
+        
+        if self.name:
+            self.timers[self.name] += elapsed_time
+
+    def get_time_by_tag(self, tag):
+        if tag in self.timers:
+            self.logger(self.text_tag.format(tag,self.timers[tag]))
+
+    def get_total_time(self):
+        total = 0
+        for v in self.timers.values():
+            total += v
+        self.logger(self.text_total.format(total))
+