From 1699aa747f7d6bea39c45f270ab5abc4f9d43c6e Mon Sep 17 00:00:00 2001 From: Yorick Barbanneau Date: Tue, 3 May 2022 21:59:25 +0200 Subject: [PATCH 1/3] Rename script + remove triggers, view, procedure All theses part will be moving on another script --- createdatabase.py => create_database.py | 98 ------------------------- 1 file changed, 98 deletions(-) rename createdatabase.py => create_database.py (64%) diff --git a/createdatabase.py b/create_database.py similarity index 64% rename from createdatabase.py rename to create_database.py index f0317c9..c59c58b 100755 --- a/createdatabase.py +++ b/create_database.py @@ -142,72 +142,6 @@ if __name__ == '__main__': PRIMARY KEY (com,id_indicateur,date_debut) );""", {'schema':AsIs(args.schema_name)}) - - curs.execute(""" - CREATE OR REPLACE VIEW %(schema)s.view_indicateur_dep AS - SELECT id_departement, d.dep, d.ncc, s.id_indicateur, s.date_debut, SUM(s.VALEUR) - FROM %(schema)s.departement d - INNER JOIN %(schema)s.commune c ON c.dep = d.dep - INNER JOIN %(schema)s.statistique s ON s.com = c.com - GROUP BY id_departement,d.dep, s.date_debut, d.ncc, s.id_indicateur - ORDER BY id_indicateur;""", - {'schema':AsIs(args.schema_name)}) - - curs.execute(""" - CREATE OR REPLACE VIEW %(schema)s.view_indicateur_reg AS - SELECT id_region, r.reg, r.ncc, s.id_indicateur, s.date_debut, SUM(s.valeur) - FROM %(schema)s.region r - INNER JOIN %(schema)s.departement d ON d.reg = r.reg - INNER JOIN %(schema)s.commune c ON c.dep = d.dep - INNER JOIN %(schema)s.statistique s ON s.com = c.com - GROUP BY id_region, r.reg, s.date_debut, r.ncc, s.id_indicateur - ORDER BY id_indicateur;""", - {'schema':AsIs(args.schema_name)}) - - curs.execute(""" - ALTER TABLE %(schema)s.region - ADD COLUMN IF NOT EXISTS population INT; - """, - {'schema':AsIs(args.schema_name)}) - - curs.execute(""" - ALTER TABLE %(schema)s.departement - ADD COLUMN IF NOT EXISTS population INT; - """, - {'schema':AsIs(args.schema_name)}) - - curs.execute(""" - CREATE OR REPLACE PROCEDURE %(schema)s.PRC_POP_REG_DEP() - LANGUAGE plpgsql - AS $$ - DECLARE - REC RECORD; - BEGIN - - FOR REC IN (SELECT id_departement, ncc, SUM valeur - FROM %(schema)s.view_indicateur_dep v - WHERE id_indicateur = 1 - AND date_debut = '2018') LOOP - - UPDATE %(schema)s.departement - SET population = REC.valeur - WHERE id_departement = REC.id_departement; - - END LOOP; - - FOR REC IN (SELECT reg, SUM(population) valeur - FROM DEPARTEMENT d - GROUP BY REG) LOOP - - UPDATE %(schema)s.region - SET population = REC.VALEUR - WHERE reg = REC.reg; - - END LOOP; - END; - $$;""", - {'schema':AsIs(args.schema_name)}) - conn.commit() t.stop() t.start('Import data from csv files') @@ -245,36 +179,4 @@ if __name__ == '__main__': conn.commit() t.stop() - t.start('Call procedure') - with conn.cursor() as curs: - curs.execute("CALL %(schema)s.PRC_POP_REG_DEP()", - {'schema':AsIs(args.schema_name)} - ) - t.stop() - - t.start('Add Modify Trigger') - with conn.cursor() as curs: - curs.execute(""" - CREATE OR REPLACE FUNCTION %(schema)s.block_maj_reg_dep() - RETURNS TRIGGER AS $bloquage$ - BEGIN - RAISE EXCEPTION - 'Mise à jour non autorisé'; - END; - $bloquage$ language plpgsql; - - CREATE TRIGGER TRG_BLOQ_MAJ_REG - BEFORE INSERT OR UPDATE OR DELETE ON %(schema)s.region - - FOR EACH ROW EXECUTE PROCEDURE %(schema)s.block_maj_reg_dep(); - - CREATE TRIGGER TRG_BLOQ_MAJ_DEP - BEFORE INSERT OR UPDATE OR DELETE ON %(schema)s.departement - - FOR EACH ROW EXECUTE PROCEDURE %(schema)s.block_maj_reg_dep(); - """, - {'schema':AsIs(args.schema_name)}) - conn.commit() - - t.stop() conn.close() From a75b24f2d844cf4359a2a612d9d48361bce25f35 Mon Sep 17 00:00:00 2001 From: Yorick Barbanneau Date: Tue, 3 May 2022 22:01:26 +0200 Subject: [PATCH 2/3] Add script to delete database schema --- delete_database.py | 90 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100755 delete_database.py diff --git a/delete_database.py b/delete_database.py new file mode 100755 index 0000000..c32834f --- /dev/null +++ b/delete_database.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +import sys +import os +import re +# import time +import logging +import argparse as arg +import psycopg2 as pg +from psycopg2.extensions import AsIs + +from classes.CustomFormater import CustomFormatter +from classes.Timer import Timer + +# Schema name is NAMEDATALEN-1 (PGSQL source code) +# -> src/include/pg_config_manual.h +def check_schema_name(arg_value, pat=re.compile(r"^[a-z0-9A-Z]{1,63}$")): + if not pat.match(arg_value): + raise ValueError + return arg_value + + +def parse_args(): + """ + Parse arguments + """ + parser = arg.ArgumentParser('Process csv files from INSEE') + + parser.add_argument('--connection-file', '-f', + help='Postgresql connexion file', + default='.pgconn' + ) + + # As whe use AsIs function to not include '' in our sql queries for schema + # name, me mus ensure that it is not contains SQL special characters + # like comments --, final coma etc and avoid SQL injection + parser.add_argument('--schema-name', + help='Database schema name', + type=check_schema_name, + default='insee' + ) + + debug_group = parser.add_mutually_exclusive_group() + debug_group.add_argument('--verbose', '-V', + help='Verbose output', + action='store_true') + debug_group.add_argument('--debug', '-d', + help='Activate debug mode', + action='store_true') + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + #logging.basicConfig(level=logging.DEBUG) + logger = logging.getLogger() + tty_handler = logging.StreamHandler() + + # create console handler with a higher log level + tty_handler.setFormatter(CustomFormatter()) + logger.addHandler(tty_handler) + + if args.verbose is True: + logger.setLevel(logging.INFO) + logger.info('VERBOSE mode activated') + + if args.debug is True: + logger.setLevel(logging.DEBUG) + logger.debug('DEBUG mode activated') + + t = Timer(logger=logger.info) + logging.debug('Import pgsql connection file {}'.format(args.connection_file)) + with open(args.connection_file) as cf: + pg_conn = cf.read() + + t.start('Delete schema') + logger.debug('Delete schema {}'.format(args.schema_name)) + conn = pg.connect(pg_conn) + with conn.cursor() as curs: + try: + curs.execute('DROP SCHEMA IF EXISTS %s CASCADE', + (AsIs(args.schema_name),)) + conn.commit() + except Exception as e: + logger.error('Can\'t drop schema {}: {}'.format( + args.schema_name, + e + ) + ) + + t.stop() From 9cd33c4818549f2d5bf0e66eefa39ca1c1f0ee16 Mon Sep 17 00:00:00 2001 From: Yorick Barbanneau Date: Tue, 3 May 2022 22:37:34 +0200 Subject: [PATCH 3/3] Add script that add trigger --- create_view.py | 134 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100755 create_view.py diff --git a/create_view.py b/create_view.py new file mode 100755 index 0000000..5bcad4a --- /dev/null +++ b/create_view.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +import sys +import os +import re +# import time +import logging +import argparse as arg +import psycopg2 as pg +from psycopg2.extensions import AsIs + +from classes.CustomFormater import CustomFormatter +from classes.Timer import Timer + + +import locale +locale.setlocale(locale.LC_ALL, 'fr_FR.UTF-8') + +# Schema name is NAMEDATALEN-1 (PGSQL source code) +# -> src/include/pg_config_manual.h +def check_schema_name(arg_value, pat=re.compile(r"^[a-z0-9A-Z]{1,63}$")): + if not pat.match(arg_value): + raise ValueError + return arg_value + + +def parse_args(): + """ + Parse arguments + """ + parser = arg.ArgumentParser('Process csv files from INSEE') + + parser.add_argument('--source', '-s', + help='csv source directory', + default='exports') + + parser.add_argument('--connection-file', '-f', + help='Postgresql connexion file', + default='.pgconn' + ) + + # As whe use AsIs function to not include '' in our sql queries for schema + # name, me mus ensure that it is not contains SQL special characters + # like comments --, final coma etc and avoid SQL injection + parser.add_argument('--schema-name', + help='Database schema name', + type=check_schema_name, + default='insee' + ) + + parser.add_argument('--test', '-t', + help='test view, get 10 record of each', + action='store_true' + ) + debug_group = parser.add_mutually_exclusive_group() + debug_group.add_argument('--verbose', '-V', + help='Verbose output', + action='store_true') + debug_group.add_argument('--debug', '-d', + help='Activate debug mode', + action='store_true') + return parser.parse_args() + +if __name__ == '__main__': + args = parse_args() + t = Timer() + #logging.basicConfig(level=logging.DEBUG) + logger = logging.getLogger() + tty_handler = logging.StreamHandler() + + # create console handler with a higher log level + tty_handler.setFormatter(CustomFormatter()) + logger.addHandler(tty_handler) + + if args.verbose is True: + logger.setLevel(logging.INFO) + logger.info('VERBOSE mode activated') + + if args.debug is True: + logger.setLevel(logging.DEBUG) + logger.debug('DEBUG mode activated') + + logging.debug('Import pgsql connection file {}'.format(args.connection_file)) + with open(args.connection_file) as cf: + pg_conn = cf.read() + + t.start('Add Views') + conn = pg.connect(pg_conn) + with conn.cursor() as curs: + + logger.debug('Add view_indicateur_reg view') + try: + curs.execute(""" + CREATE OR REPLACE VIEW %(schema)s.view_indicateur_dep AS + SELECT id_departement, d.dep, d.ncc, s.id_indicateur, s.date_debut, SUM(s.VALEUR) + FROM %(schema)s.departement d + INNER JOIN %(schema)s.commune c ON c.dep = d.dep + INNER JOIN %(schema)s.statistique s ON s.com = c.com + GROUP BY id_departement,d.dep, s.date_debut, d.ncc, s.id_indicateur + ORDER BY id_indicateur;""", + {'schema':AsIs(args.schema_name)}) + conn.commit() + except Exception as e: + logger.error('Can\'t add view view_indicateur_dep: {}'.format(e)) + sys.exit(1) + + logger.debug('Add view_indicateur_reg view') + try: + curs.execute(""" + CREATE OR REPLACE VIEW %(schema)s.view_indicateur_reg AS + SELECT id_region, r.reg, r.ncc, s.id_indicateur, s.date_debut, SUM(s.valeur) + FROM %(schema)s.region r + INNER JOIN %(schema)s.departement d ON d.reg = r.reg + INNER JOIN %(schema)s.commune c ON c.dep = d.dep + INNER JOIN %(schema)s.statistique s ON s.com = c.com + GROUP BY id_region, r.reg, s.date_debut, r.ncc, s.id_indicateur + ORDER BY id_indicateur;""", + {'schema':AsIs(args.schema_name)}) + conn.commit() + except Exception as e: + logger.error('Can\'t add view view_indicateur_reg: {}'.format(e)) + sys.exit(1) + t.stop() + + if args.test: + for view in ('view_indicateur_dep','view_indicateur_reg'): + curs.execute(""" + SELECT v.ncc, v.date_debut, v.sum + FROM %(schema)s.%(view)s v LIMIT 10; + """, + {'schema':AsIs(args.schema_name), 'view': AsIs(view)}) + data = curs.fetchall() + print('\nCheck {}:'.format(view)) + for row in data: + print('\t{:.<40}{}: {:>7n}'.format(*row))