IDI_Master/projet_INSEE
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add debug / verbose information

Browse source
This commit is contained in:
Yorick Barbanneau 2022-04-17 23:53:07 +02:00
parent 608392ad06
commit 18c246ee92
1 changed files with 25 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

34
create_db.py
View file

@ -9,7 +9,7 @@ import argparse as arg
class CustomFormatter(logging.Formatter): class CustomFormatter(logging.Formatter):
grey = "\x1b[38;20m" grey = "\x1b[0;35m"
blue = "\x1b[34;20m" blue = "\x1b[34;20m"
yellow = "\x1b[33;20m" yellow = "\x1b[33;20m"
red = "\x1b[31;20m" red = "\x1b[31;20m"
@ -51,9 +51,13 @@ def parse_args():
parser.add_argument('--states', parser.add_argument('--states',
help='states raw csv file (inside source follder)', help='states raw csv file (inside source follder)',
default='region2021.csv') default='region2021.csv')
parser.add_argument('--verbose', '-V', debug_group = parser.add_mutually_exclusive_group()
debug_group.add_argument('--verbose', '-V',
help='Verbose output', help='Verbose output',
action='store_true') action='store_true')
debug_group.add_argument('--debug', '-d',
help='Activate debug mode',
action='store_true')
return parser.parse_args() return parser.parse_args()
@ -61,6 +65,8 @@ def import_states_csv(raw_file):
""" """
Process states raw file Process states raw file
""" """
logger.info('import states from {}'.format(raw_file))
reg_convert= lambda x: x if len(str(x)) == 2 else f'0{x}' reg_convert= lambda x: x if len(str(x)) == 2 else f'0{x}'
states = pd.read_csv(raw_file, states = pd.read_csv(raw_file,
usecols=["REG","NCC","LIBELLE","CHEFLIEU"], usecols=["REG","NCC","LIBELLE","CHEFLIEU"],
@ -72,6 +78,8 @@ def import_department_csv(raw_file):
""" """
Process department files Process department files
""" """
logger.info('import departments from {}'.format(raw_file))
reg_convert= lambda x: x if len(str(x)) == 2 else f'0{x}' reg_convert= lambda x: x if len(str(x)) == 2 else f'0{x}'
dep = pd.read_csv(raw_file, dep = pd.read_csv(raw_file,
usecols=["DEP","NCC","LIBELLE","REG","CHEFLIEU"], usecols=["DEP","NCC","LIBELLE","REG","CHEFLIEU"],
@ -83,6 +91,8 @@ def import_towns_csv(raw_file):
""" """
Process department files Process department files
""" """
logger.info('import town from {}'.format(raw_file))
towns = pd.read_csv(raw_file, towns = pd.read_csv(raw_file,
usecols=["COM","TYPECOM","NCC","LIBELLE","DEP"]) usecols=["COM","TYPECOM","NCC","LIBELLE","DEP"])
return towns.loc[towns['TYPECOM'] == 'COM', ['COM','NCC', 'LIBELLE', 'DEP']] return towns.loc[towns['TYPECOM'] == 'COM', ['COM','NCC', 'LIBELLE', 'DEP']]
@ -90,32 +100,38 @@ def import_towns_csv(raw_file):
if __name__ == '__main__': if __name__ == '__main__':
args = parse_args() args = parse_args()
#logging.basicConfig(level=logging.DEBUG) #logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger() logger = logging.getLogger()
if args.verbose is True:
logger.setLevel(logging.DEBUG)
logger.debug('Debug mode activated')
tty_handler = logging.StreamHandler() tty_handler = logging.StreamHandler()
# create console handler with a higher log level # create console handler with a higher log level
tty_handler.setFormatter(CustomFormatter()) tty_handler.setFormatter(CustomFormatter())
logger.addHandler(tty_handler) logger.addHandler(tty_handler)
if args.verbose is True:
logger.setLevel(logging.INFO)
logger.info('VERBOSE mode activated')
if args.debug is True:
logger.setLevel(logging.DEBUG)
logger.debug('DEBUG mode activated')
if not os.path.exists(args.source + '/' + args.states): if not os.path.exists(args.source + '/' + args.states):
logger.critical('can\'t find source file for states') logger.critical('can\'t find source file for states')
states = import_states_csv(args.source + '/' + args.states) states = import_states_csv(args.source + '/' + args.states)
print(states) logger.debug(states)
if not os.path.exists(args.source + '/' + args.states): if not os.path.exists(args.source + '/' + args.states):
logger.critical('can\'t find source file for departments') logger.critical('can\'t find source file for departments')
departments = import_department_csv(args.source + '/' + args.departments) departments = import_department_csv(args.source + '/' + args.departments)
print(departments) logger.debug(departments)
if not os.path.exists(args.source + '/' + args.towns): if not os.path.exists(args.source + '/' + args.towns):
logger.critical('can\'t find source file for departments') logger.critical('can\'t find source file for departments')
towns = import_towns_csv(args.source + '/' + args.towns) towns = import_towns_csv(args.source + '/' + args.towns)
print(towns) logger.debug(towns)
sys.exit() sys.exit()