Add debug / verbose information

create_db.py

@@ -9,7 +9,7 @@ import argparse as arg
 
 class CustomFormatter(logging.Formatter):
 
-    grey = "\x1b[38;20m"
+    grey = "\x1b[0;35m"
     blue = "\x1b[34;20m"
     yellow = "\x1b[33;20m"
     red = "\x1b[31;20m"
@@ -44,23 +44,29 @@ def parse_args():
             default='exports')
     parser.add_argument('--towns',
             help='town raw csv file (inside source follder)',
-            default='communes2021.csv')
+            default='commune2021.csv')
     parser.add_argument('--departments',
             help='departments raw csv file (inside source follder)',
             default='departement2021.csv')
     parser.add_argument('--states',
             help='states raw csv file (inside source follder)',
             default='region2021.csv')
-    parser.add_argument('--verbose', '-V',
+    debug_group = parser.add_mutually_exclusive_group()
+    debug_group.add_argument('--verbose', '-V',
             help='Verbose output',
             action='store_true')
+    debug_group.add_argument('--debug', '-d',
+            help='Activate debug mode',
+            action='store_true')
     return parser.parse_args()
 
 
-def create_states_csv(raw_file):
+def import_states_csv(raw_file):
     """
     Process states raw file
     """
+
+    logger.info('import states from {}'.format(raw_file))
     reg_convert= lambda x: x if len(str(x)) == 2 else f'0{x}'
     states = pd.read_csv(raw_file,
         usecols=["REG","NCC","LIBELLE","CHEFLIEU"],
@@ -68,10 +74,12 @@ def create_states_csv(raw_file):
     return states
 
 
-def create_department_csv(raw_file):
+def import_department_csv(raw_file):
     """
     Process department files
     """
+
+    logger.info('import departments from {}'.format(raw_file))
     reg_convert= lambda x: x if len(str(x)) == 2 else f'0{x}'
     dep = pd.read_csv(raw_file,
         usecols=["DEP","NCC","LIBELLE","REG","CHEFLIEU"],
@@ -79,29 +87,51 @@ def create_department_csv(raw_file):
     return dep
 
 
+def import_towns_csv(raw_file):
+    """
+    Process department files
+    """
+
+    logger.info('import town from {}'.format(raw_file))
+    towns = pd.read_csv(raw_file,
+        usecols=["COM","TYPECOM","NCC","LIBELLE","DEP"])
+    return towns.loc[towns['TYPECOM'] == 'COM', ['COM','NCC', 'LIBELLE', 'DEP']]
+
+
 if __name__ == '__main__':
     args = parse_args()
+    
     #logging.basicConfig(level=logging.DEBUG)
     logger = logging.getLogger()
-    if args.verbose is True:
-        logger.setLevel(logging.DEBUG)
-        logger.debug('Debug mode activated')
-
     tty_handler = logging.StreamHandler()
 
     # create console handler with a higher log level
     tty_handler.setFormatter(CustomFormatter())
     logger.addHandler(tty_handler)
     
+    if args.verbose is True:
+        logger.setLevel(logging.INFO)
+        logger.info('VERBOSE mode activated')
+    
+    if args.debug is True:
+        logger.setLevel(logging.DEBUG)
+        logger.debug('DEBUG mode activated')
+    
+    
     if not os.path.exists(args.source + '/' + args.states):
         logger.critical('can\'t find source file for states')
 
-    states = create_states_csv(args.source + '/' + args.states)
+    states = import_states_csv(args.source + '/' + args.states)
-    print(states)
+    logger.debug(states)
     
     if not os.path.exists(args.source + '/' + args.states):
         logger.critical('can\'t find source file for departments')
-    departments = create_department_csv(args.source + '/' + args.departments)
+    departments = import_department_csv(args.source + '/' + args.departments)
-    print(departments)
+    logger.debug(departments)
+    
+    if not os.path.exists(args.source + '/' + args.towns):
+        logger.critical('can\'t find source file for departments')
+    towns = import_towns_csv(args.source + '/' + args.towns)
+    logger.debug(towns)
     
     sys.exit()