diff --git a/create_db.py b/create_db.py index 625fbb8..485b312 100755 --- a/create_db.py +++ b/create_db.py @@ -9,7 +9,7 @@ import argparse as arg class CustomFormatter(logging.Formatter): - grey = "\x1b[38;20m" + grey = "\x1b[0;35m" blue = "\x1b[34;20m" yellow = "\x1b[33;20m" red = "\x1b[31;20m" @@ -44,23 +44,29 @@ def parse_args(): default='exports') parser.add_argument('--towns', help='town raw csv file (inside source follder)', - default='communes2021.csv') + default='commune2021.csv') parser.add_argument('--departments', help='departments raw csv file (inside source follder)', default='departement2021.csv') parser.add_argument('--states', help='states raw csv file (inside source follder)', default='region2021.csv') - parser.add_argument('--verbose', '-V', + debug_group = parser.add_mutually_exclusive_group() + debug_group.add_argument('--verbose', '-V', help='Verbose output', action='store_true') + debug_group.add_argument('--debug', '-d', + help='Activate debug mode', + action='store_true') return parser.parse_args() -def create_states_csv(raw_file): +def import_states_csv(raw_file): """ Process states raw file """ + + logger.info('import states from {}'.format(raw_file)) reg_convert= lambda x: x if len(str(x)) == 2 else f'0{x}' states = pd.read_csv(raw_file, usecols=["REG","NCC","LIBELLE","CHEFLIEU"], @@ -68,10 +74,12 @@ def create_states_csv(raw_file): return states -def create_department_csv(raw_file): +def import_department_csv(raw_file): """ Process department files """ + + logger.info('import departments from {}'.format(raw_file)) reg_convert= lambda x: x if len(str(x)) == 2 else f'0{x}' dep = pd.read_csv(raw_file, usecols=["DEP","NCC","LIBELLE","REG","CHEFLIEU"], @@ -79,29 +87,51 @@ def create_department_csv(raw_file): return dep +def import_towns_csv(raw_file): + """ + Process department files + """ + + logger.info('import town from {}'.format(raw_file)) + towns = pd.read_csv(raw_file, + usecols=["COM","TYPECOM","NCC","LIBELLE","DEP"]) + return towns.loc[towns['TYPECOM'] == 'COM', ['COM','NCC', 'LIBELLE', 'DEP']] + + if __name__ == '__main__': args = parse_args() + #logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger() - if args.verbose is True: - logger.setLevel(logging.DEBUG) - logger.debug('Debug mode activated') - tty_handler = logging.StreamHandler() # create console handler with a higher log level tty_handler.setFormatter(CustomFormatter()) logger.addHandler(tty_handler) + if args.verbose is True: + logger.setLevel(logging.INFO) + logger.info('VERBOSE mode activated') + + if args.debug is True: + logger.setLevel(logging.DEBUG) + logger.debug('DEBUG mode activated') + + if not os.path.exists(args.source + '/' + args.states): logger.critical('can\'t find source file for states') - states = create_states_csv(args.source + '/' + args.states) - print(states) + states = import_states_csv(args.source + '/' + args.states) + logger.debug(states) if not os.path.exists(args.source + '/' + args.states): logger.critical('can\'t find source file for departments') - departments = create_department_csv(args.source + '/' + args.departments) - print(departments) + departments = import_department_csv(args.source + '/' + args.departments) + logger.debug(departments) + + if not os.path.exists(args.source + '/' + args.towns): + logger.critical('can\'t find source file for departments') + towns = import_towns_csv(args.source + '/' + args.towns) + logger.debug(towns) sys.exit()