.gitignore

@@ -1,3 +1,2 @@
 .venv/
 .ipynb_checkpoints/
-__pycache__

create_db.py

@@ -111,7 +111,6 @@ def import_statistics_csv(raw_file):
     """
 
     logger.info('import town from {}'.format(raw_file))
-    stats_convert= lambda x: x if len(str(x)) == 5 else f'0{x}'
     stats = pd.read_csv(raw_file,
         usecols=["CODGEO","SUPERF","P18_POP","P13_POP","P08_POP","D99_POP",
             "NAIS1318","NAIS0813","NAIS9908","NAIS9099","NAIS8290","DECE1318",
@@ -121,9 +120,7 @@ def import_statistics_csv(raw_file):
             "P13_RP","P08_RP","D99_RP","D90_RP","D82_RP", "P18_RSECOCC",
             "P13_RSECOCC","P08_RSECOCC","D99_RSECOCC","D90_RSECOCC",
             "D82_RSECOCC"],
-            sep=';',
+            sep=';')
-            converters={'CODGEO':stats_convert}
-            )
     return stats
 
 def get_single_date(attr):
@@ -159,18 +156,6 @@ def get_range_date(attr):
 
     return date[1], date[2]
 
-def export_csv(dataframe, path):
-    logger.debug('export csv from panda dataframe')
-    try:
-        dataframe.to_csv(path ,header = False, index= False)
-    except Exception as e:
-        logger.error(
-                'Erro when exporting Dataframe to csvfile {}. \n{}'.format(
-                    path,
-                    e)
-                )
-        sys.exit(1)
-
 if __name__ == '__main__':
     args = parse_args()
     
@@ -256,7 +241,8 @@ if __name__ == '__main__':
     ## create statistics dataframes
     # 
     # We need to first iterate on statistics
-    t.start('Process_Statistics')
+    if args.verbose or args.debug:
+        t.start()
         
     c_stats = pd.DataFrame(columns = ['com','id_indicateur','date_debut',
         'date_fin','valeur']
@@ -303,31 +289,7 @@ if __name__ == '__main__':
                 temp['date_fin'].append(end)
                 temp['valeur'].append(value)
 
-    consolidated_stats = pd.DataFrame.from_dict(temp)
     t.stop()
-
-    t.start('Process_town')
-    towns = pd.merge(towns, 
-            statistics[['CODGEO', 'SUPERF']], 
-            left_on=['COM'],
-            right_on=['CODGEO'],
-            how = 'left'
-    )[['COM','NCC','LIBELLE', 'DEP', 'SUPERF']]
-    t.stop()
-    logger.debug(towns)
-
-    # Export all dataframes to csv
-    t.start('Export_CSV')
-    export_csv(towns, args.export + '/commune.csv')
-    export_csv(departments, args.export + '/departement.csv')
-    export_csv(states, args.export + '/region.csv')
-    export_csv(dep_capitals, args.export + '/cheflieudepartement.csv')
-    export_csv(states_capitals, args.export + '/cheflieuregion.csv')
-    export_csv(indicators, args.export + '/indicateur.csv')
-    export_csv(consolidated_stats, args.export + '/statistiques.csv')
-    t.stop()
-    
-    t.get_times_by_tag()
     t.get_total_time()
 
     sys.exit()

csv_propre/cheflieuRegion.csv

@@ -0,0 +1,18 @@
+97105,01
+97209,02
+97302,03
+97411,04
+97608,06
+75056,11
+45234,24
+21231,27
+76540,28
+59350,32
+67482,44
+44109,52
+35238,53
+33063,75
+31555,76
+69123,84
+13055,93
+2A004,94

csv_propre/cheflieudepartement.csv

@@ -0,0 +1,101 @@
+01053,01
+02408,02
+03190,03
+04070,04
+05061,05
+06088,06
+07186,07
+08105,08
+09122,09
+10387,10
+11069,11
+12202,12
+13055,13
+14118,14
+15014,15
+16015,16
+17300,17
+18033,18
+19272,19
+21231,21
+22278,22
+23096,23
+24322,24
+25056,25
+26362,26
+27229,27
+28085,28
+29232,29
+2A004,2A
+2B033,2B
+30189,30
+31555,31
+32013,32
+33063,33
+34172,34
+35238,35
+36044,36
+37261,37
+38185,38
+39300,39
+40192,40
+41018,41
+42218,42
+43157,43
+44109,44
+45234,45
+46042,46
+47001,47
+48095,48
+49007,49
+50502,50
+51108,51
+52121,52
+53130,53
+54395,54
+55029,55
+56260,56
+57463,57
+58194,58
+59350,59
+60057,60
+61001,61
+62041,62
+63113,63
+64445,64
+65440,65
+66136,66
+67482,67
+68066,68
+69123,69
+70550,70
+71270,71
+72181,72
+73065,73
+74010,74
+75056,75
+76540,76
+77288,77
+78646,78
+79191,79
+80021,80
+81004,81
+82121,82
+83137,83
+84007,84
+85191,85
+86194,86
+87085,87
+88160,88
+89024,89
+90010,90
+91228,91
+92050,92
+93008,93
+94028,94
+95500,95
+97105,971
+97209,972
+97302,973
+97411,974
+97608,976

csv_propre/departement.csv

@@ -0,0 +1,101 @@
+01,AIN,Ain,84
+02,AISNE,Aisne,32
+03,ALLIER,Allier,84
+04,ALPES DE HAUTE PROVENCE,Alpes-de-Haute-Provence,93
+05,HAUTES ALPES,Hautes-Alpes,93
+06,ALPES MARITIMES,Alpes-Maritimes,93
+07,ARDECHE,Ardèche,84
+08,ARDENNES,Ardennes,44
+09,ARIEGE,Ariège,76
+10,AUBE,Aube,44
+11,AUDE,Aude,76
+12,AVEYRON,Aveyron,76
+13,BOUCHES DU RHONE,Bouches-du-Rhône,93
+14,CALVADOS,Calvados,28
+15,CANTAL,Cantal,84
+16,CHARENTE,Charente,75
+17,CHARENTE MARITIME,Charente-Maritime,75
+18,CHER,Cher,24
+19,CORREZE,Corrèze,75
+21,COTE D OR,Côte-d'Or,27
+22,COTES D ARMOR,Côtes-d'Armor,53
+23,CREUSE,Creuse,75
+24,DORDOGNE,Dordogne,75
+25,DOUBS,Doubs,27
+26,DROME,Drôme,84
+27,EURE,Eure,28
+28,EURE ET LOIR,Eure-et-Loir,24
+29,FINISTERE,Finistère,53
+2A,CORSE DU SUD,Corse-du-Sud,94
+2B,HAUTE CORSE,Haute-Corse,94
+30,GARD,Gard,76
+31,HAUTE GARONNE,Haute-Garonne,76
+32,GERS,Gers,76
+33,GIRONDE,Gironde,75
+34,HERAULT,Hérault,76
+35,ILLE ET VILAINE,Ille-et-Vilaine,53
+36,INDRE,Indre,24
+37,INDRE ET LOIRE,Indre-et-Loire,24
+38,ISERE,Isère,84
+39,JURA,Jura,27
+40,LANDES,Landes,75
+41,LOIR ET CHER,Loir-et-Cher,24
+42,LOIRE,Loire,84
+43,HAUTE LOIRE,Haute-Loire,84
+44,LOIRE ATLANTIQUE,Loire-Atlantique,52
+45,LOIRET,Loiret,24
+46,LOT,Lot,76
+47,LOT ET GARONNE,Lot-et-Garonne,75
+48,LOZERE,Lozère,76
+49,MAINE ET LOIRE,Maine-et-Loire,52
+50,MANCHE,Manche,28
+51,MARNE,Marne,44
+52,HAUTE MARNE,Haute-Marne,44
+53,MAYENNE,Mayenne,52
+54,MEURTHE ET MOSELLE,Meurthe-et-Moselle,44
+55,MEUSE,Meuse,44
+56,MORBIHAN,Morbihan,53
+57,MOSELLE,Moselle,44
+58,NIEVRE,Nièvre,27
+59,NORD,Nord,32
+60,OISE,Oise,32
+61,ORNE,Orne,28
+62,PAS DE CALAIS,Pas-de-Calais,32
+63,PUY DE DOME,Puy-de-Dôme,84
+64,PYRENEES ATLANTIQUES,Pyrénées-Atlantiques,75
+65,HAUTES PYRENEES,Hautes-Pyrénées,76
+66,PYRENEES ORIENTALES,Pyrénées-Orientales,76
+67,BAS RHIN,Bas-Rhin,44
+68,HAUT RHIN,Haut-Rhin,44
+69,RHONE,Rhône,84
+70,HAUTE SAONE,Haute-Saône,27
+71,SAONE ET LOIRE,Saône-et-Loire,27
+72,SARTHE,Sarthe,52
+73,SAVOIE,Savoie,84
+74,HAUTE SAVOIE,Haute-Savoie,84
+75,PARIS,Paris,11
+76,SEINE MARITIME,Seine-Maritime,28
+77,SEINE ET MARNE,Seine-et-Marne,11
+78,YVELINES,Yvelines,11
+79,DEUX SEVRES,Deux-Sèvres,75
+80,SOMME,Somme,32
+81,TARN,Tarn,76
+82,TARN ET GARONNE,Tarn-et-Garonne,76
+83,VAR,Var,93
+84,VAUCLUSE,Vaucluse,93
+85,VENDEE,Vendée,52
+86,VIENNE,Vienne,75
+87,HAUTE VIENNE,Haute-Vienne,75
+88,VOSGES,Vosges,44
+89,YONNE,Yonne,27
+90,TERRITOIRE DE BELFORT,Territoire de Belfort,27
+91,ESSONNE,Essonne,11
+92,HAUTS DE SEINE,Hauts-de-Seine,11
+93,SEINE SAINT DENIS,Seine-Saint-Denis,11
+94,VAL DE MARNE,Val-de-Marne,11
+95,VAL D OISE,Val-d'Oise,11
+971,GUADELOUPE,Guadeloupe,01
+972,MARTINIQUE,Martinique,02
+973,GUYANE,Guyane,03
+974,LA REUNION,La Réunion,04
+976,MAYOTTE,Mayotte,06

csv_propre/indicateur.csv

@@ -0,0 +1,7 @@
+1,population
+2,naissances
+3,deces
+4,logements
+5,logements vacants
+6,residences principales
+7,residences secondaires et logements occasionnels

csv_propre/region.csv

@@ -0,0 +1,18 @@
+01,GUADELOUPE,Guadeloupe
+02,MARTINIQUE,Martinique
+03,GUYANE,Guyane
+04,LA REUNION,La Réunion
+06,MAYOTTE,Mayotte
+11,ILE DE FRANCE,Île-de-France
+24,CENTRE VAL DE LOIRE,Centre-Val de Loire
+27,BOURGOGNE FRANCHE COMTE,Bourgogne-Franche-Comté
+28,NORMANDIE,Normandie
+32,HAUTS DE FRANCE,Hauts-de-France
+44,GRAND EST,Grand Est
+52,PAYS DE LA LOIRE,Pays de la Loire
+53,BRETAGNE,Bretagne
+75,NOUVELLE AQUITAINE,Nouvelle-Aquitaine
+76,OCCITANIE,Occitanie
+84,AUVERGNE RHONE ALPES,Auvergne-Rhône-Alpes
+93,PROVENCE ALPES COTE D AZUR,Provence-Alpes-Côte d'Azur
+94,CORSE,Corse

timer/Timer.py

@@ -61,6 +61,3 @@ class Timer:
             total += v
         self.logger(self.text_total.format(total))
 
-    def get_times_by_tag(self):
-        for k in self.timers.keys():
-            self.get_time_by_tag(k)