From 3ec1a6963fd093217c328ad7ecda2f11eea26931 Mon Sep 17 00:00:00 2001 From: Yorick Barbanneau Date: Sun, 17 Apr 2022 23:28:47 +0200 Subject: [PATCH 1/3] Import towns csv --- create_db.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/create_db.py b/create_db.py index 625fbb8..bfc7bb1 100755 --- a/create_db.py +++ b/create_db.py @@ -44,7 +44,7 @@ def parse_args(): default='exports') parser.add_argument('--towns', help='town raw csv file (inside source follder)', - default='communes2021.csv') + default='commune2021.csv') parser.add_argument('--departments', help='departments raw csv file (inside source follder)', default='departement2021.csv') @@ -79,6 +79,15 @@ def create_department_csv(raw_file): return dep +def create_towns_csv(raw_file): + """ + Process department files + """ + towns = pd.read_csv(raw_file, + usecols=["COM","TYPECOM","NCC","LIBELLE","DEP"]) + return towns.loc[towns['TYPECOM'] == 'COM', ['COM','NCC', 'LIBELLE', 'DEP']] + + if __name__ == '__main__': args = parse_args() #logging.basicConfig(level=logging.DEBUG) @@ -104,4 +113,9 @@ if __name__ == '__main__': departments = create_department_csv(args.source + '/' + args.departments) print(departments) + if not os.path.exists(args.source + '/' + args.towns): + logger.critical('can\'t find source file for departments') + towns = create_towns_csv(args.source + '/' + args.towns) + print(towns) + sys.exit() From 608392ad06e38d4ac542e0e315fddef751a6385d Mon Sep 17 00:00:00 2001 From: Yorick Barbanneau Date: Sun, 17 Apr 2022 23:30:09 +0200 Subject: [PATCH 2/3] Change create_* functions name to import* --- create_db.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/create_db.py b/create_db.py index bfc7bb1..7cb8e09 100755 --- a/create_db.py +++ b/create_db.py @@ -57,7 +57,7 @@ def parse_args(): return parser.parse_args() -def create_states_csv(raw_file): +def import_states_csv(raw_file): """ Process states raw file """ @@ -68,7 +68,7 @@ def create_states_csv(raw_file): return states -def create_department_csv(raw_file): +def import_department_csv(raw_file): """ Process department files """ @@ -79,7 +79,7 @@ def create_department_csv(raw_file): return dep -def create_towns_csv(raw_file): +def import_towns_csv(raw_file): """ Process department files """ @@ -105,17 +105,17 @@ if __name__ == '__main__': if not os.path.exists(args.source + '/' + args.states): logger.critical('can\'t find source file for states') - states = create_states_csv(args.source + '/' + args.states) + states = import_states_csv(args.source + '/' + args.states) print(states) if not os.path.exists(args.source + '/' + args.states): logger.critical('can\'t find source file for departments') - departments = create_department_csv(args.source + '/' + args.departments) + departments = import_department_csv(args.source + '/' + args.departments) print(departments) if not os.path.exists(args.source + '/' + args.towns): logger.critical('can\'t find source file for departments') - towns = create_towns_csv(args.source + '/' + args.towns) + towns = import_towns_csv(args.source + '/' + args.towns) print(towns) sys.exit() From 18c246ee9276c41e0a14ce61c6de4584888e8ad8 Mon Sep 17 00:00:00 2001 From: Yorick Barbanneau Date: Sun, 17 Apr 2022 23:53:07 +0200 Subject: [PATCH 3/3] Add debug / verbose information --- create_db.py | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/create_db.py b/create_db.py index 7cb8e09..485b312 100755 --- a/create_db.py +++ b/create_db.py @@ -9,7 +9,7 @@ import argparse as arg class CustomFormatter(logging.Formatter): - grey = "\x1b[38;20m" + grey = "\x1b[0;35m" blue = "\x1b[34;20m" yellow = "\x1b[33;20m" red = "\x1b[31;20m" @@ -51,9 +51,13 @@ def parse_args(): parser.add_argument('--states', help='states raw csv file (inside source follder)', default='region2021.csv') - parser.add_argument('--verbose', '-V', + debug_group = parser.add_mutually_exclusive_group() + debug_group.add_argument('--verbose', '-V', help='Verbose output', action='store_true') + debug_group.add_argument('--debug', '-d', + help='Activate debug mode', + action='store_true') return parser.parse_args() @@ -61,6 +65,8 @@ def import_states_csv(raw_file): """ Process states raw file """ + + logger.info('import states from {}'.format(raw_file)) reg_convert= lambda x: x if len(str(x)) == 2 else f'0{x}' states = pd.read_csv(raw_file, usecols=["REG","NCC","LIBELLE","CHEFLIEU"], @@ -72,6 +78,8 @@ def import_department_csv(raw_file): """ Process department files """ + + logger.info('import departments from {}'.format(raw_file)) reg_convert= lambda x: x if len(str(x)) == 2 else f'0{x}' dep = pd.read_csv(raw_file, usecols=["DEP","NCC","LIBELLE","REG","CHEFLIEU"], @@ -83,6 +91,8 @@ def import_towns_csv(raw_file): """ Process department files """ + + logger.info('import town from {}'.format(raw_file)) towns = pd.read_csv(raw_file, usecols=["COM","TYPECOM","NCC","LIBELLE","DEP"]) return towns.loc[towns['TYPECOM'] == 'COM', ['COM','NCC', 'LIBELLE', 'DEP']] @@ -90,32 +100,38 @@ def import_towns_csv(raw_file): if __name__ == '__main__': args = parse_args() + #logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger() - if args.verbose is True: - logger.setLevel(logging.DEBUG) - logger.debug('Debug mode activated') - tty_handler = logging.StreamHandler() # create console handler with a higher log level tty_handler.setFormatter(CustomFormatter()) logger.addHandler(tty_handler) + if args.verbose is True: + logger.setLevel(logging.INFO) + logger.info('VERBOSE mode activated') + + if args.debug is True: + logger.setLevel(logging.DEBUG) + logger.debug('DEBUG mode activated') + + if not os.path.exists(args.source + '/' + args.states): logger.critical('can\'t find source file for states') states = import_states_csv(args.source + '/' + args.states) - print(states) + logger.debug(states) if not os.path.exists(args.source + '/' + args.states): logger.critical('can\'t find source file for departments') departments = import_department_csv(args.source + '/' + args.departments) - print(departments) + logger.debug(departments) if not os.path.exists(args.source + '/' + args.towns): logger.critical('can\'t find source file for departments') towns = import_towns_csv(args.source + '/' + args.towns) - print(towns) + logger.debug(towns) sys.exit()