Merge pull request #167 from betagouv/history-at-import

L'import CSV log dans les tables historical<model>

data/admin/__init__.py

@@ -7,17 +7,33 @@
 from .plant import PlantAdmin  # noqa
 
 from data.models import Ingredient, Plant, PlantPart, PlantFamily, Microorganism  # noqa
+from data.admin.abstract_admin import IngredientAdminWithHistoryChangedFields
 
 
-def get_admin_header():
-    return "Compl'Alim"
+@admin.register(Ingredient)
+class IngredientAdmin(IngredientAdminWithHistoryChangedFields):
+    pass
+
+
+@admin.register(PlantPart)
+class PlantPartAdmin(IngredientAdminWithHistoryChangedFields):
+    pass
+
+
+@admin.register(PlantFamily)
+class PlantFamilyAdmin(IngredientAdminWithHistoryChangedFields):
+    pass
 
 
-admin.site.register(Ingredient)
-admin.site.register(PlantPart)
-admin.site.register(PlantFamily)
-admin.site.register(Microorganism)
+@admin.register(Microorganism)
+class MicroorganismAdmin(IngredientAdminWithHistoryChangedFields):
+    pass
+
+
+def get_admin_header():
+    return "Compl'Alim"
 
 
 admin.site.site_header = get_admin_header()
 admin.site.site_title = get_admin_header()
+

data/admin/abstract_admin.py

@@ -0,0 +1,11 @@
+from simple_history.admin import SimpleHistoryAdmin
+
+
+class IngredientAdminWithHistoryChangedFields(SimpleHistoryAdmin):
+    history_list_display = ['changed_fields']
+
+    def changed_fields(self, obj):
+        if obj.prev_record:
+            delta = obj.diff_against(obj.prev_record)
+            return delta.changed_fields
+        return None

data/admin/plant.py

@@ -1,7 +1,9 @@
 from django import forms
 from django.contrib import admin
 from django.db import models
+
 from data.models import Plant, PlantSynonym
+from data.admin.abstract_admin import IngredientAdminWithHistoryChangedFields
 
 
 class PlantSynonymInline(admin.TabularInline):
@@ -29,7 +31,7 @@ class Meta:
 
 
 @admin.register(Plant)
-class PlantAdmin(admin.ModelAdmin):
+class PlantAdmin(IngredientAdminWithHistoryChangedFields):
     form = PlantForm
     fieldsets = [
         (
@@ -61,3 +63,10 @@ class PlantAdmin(admin.ModelAdmin):
         "family",
     )
     list_filter = ("is_obsolete", "family")
+    history_list_display = ['changed_fields']
+
+    def changed_fields(self, obj):
+        if obj.prev_record:
+            delta = obj.diff_against(obj.prev_record)
+            return delta.changed_fields
+        return None

data/admin/substance.py

@@ -4,6 +4,7 @@
 from django.urls import reverse
 
 from data.models import Substance
+from data.admin.abstract_admin import IngredientAdminWithHistoryChangedFields
 
 
 class SubstanceForm(forms.ModelForm):
@@ -18,7 +19,7 @@ class Meta:
 
 
 @admin.register(Substance)
-class SubstanceAdmin(admin.ModelAdmin):
+class SubstanceAdmin(IngredientAdminWithHistoryChangedFields):
     @classmethod
     def links_to_objects(cls, object_name, objects):
         rel_list = "<ul>"

data/csv_importer.py

@@ -2,8 +2,15 @@
 import logging
 import os
 
-from django.db.models import ForeignKey, ManyToManyField, TextField, CharField, FloatField, IntegerField
-
+from datetime import datetime
+from django.db.models import (
+    ForeignKey,
+    ManyToManyField,
+    TextField,
+    CharField,
+    FloatField,
+    IntegerField,
+)
 from .exceptions import CSVFileError
 
 # Import the model
@@ -14,7 +21,7 @@
 from .models.substance import Substance, SubstanceSynonym
 
 logger = logging.getLogger(__name__)
-
+from simple_history.utils import update_change_reason
 
 # Modèles pour recevoir l'import des données des fichier csv
 CSV_TO_MODEL_MAPPING = {
@@ -39,7 +46,10 @@
 }
 
 # Le fichier REF_ICA_PARTIE_PL_A_SURVEILLER n'est pas traité comme une relation car il correspond à un model à part entière
-RELATION_CSV = ["REF_ICA_AUTREING_SUBSTACTIVE.csv", "REF_ICA_PLANTE_SUBSTANCE.csv"]
+RELATION_CSV = [
+    "REF_ICA_AUTREING_SUBSTACTIVE.csv",
+    "REF_ICA_PLANTE_SUBSTANCE.csv",
+]
 
 # Établi le préfix des champs du csv
 CSV_TO_TABLE_PREFIX_MAPPING = {
@@ -104,7 +114,13 @@
 }
 
 # Ces champs sont remplis automatiquement et ne sont pas recherchés dans les fichiers csv
-AUTOMATICALLY_FILLED = ["id", "siccrf_id", "creation_date", "modification_date", "missing_import_data"]
+AUTOMATICALLY_FILLED = [
+    "id",
+    "siccrf_id",
+    "creation_date",
+    "modification_date",
+    "missing_import_data",
+]
 
 
 def import_csv(csv_filepath):
@@ -130,10 +146,15 @@ def import_csv(csv_filepath):
 
         csvreader = csv.DictReader(csv_lines, dialect=dialect)
 
-        logger.info(f"Import de {csv_filename} dans le modèle {model.__name__} en cours.")
+        logger.info(
+            f"Import de {csv_filename} dans le modèle {model.__name__} en cours."
+        )
         is_relation = True if csv_filename in RELATION_CSV else False
         nb_row, nb_created, updated_models = _import_csv_to_model(
-            csv_reader=csvreader, csv_filename=csv_filename, model=model, is_relation=is_relation
+            csv_reader=csvreader,
+            csv_filename=csv_filename,
+            model=model,
+            is_relation=is_relation,
         )
         logger.info(
             f"Import de {csv_filename} dans le modèle {model.__name__} terminé : {nb_row} objets importés, {nb_created} objets créés."
@@ -152,11 +173,15 @@ def _import_csv_to_model(csv_reader, csv_filename, model, is_relation=False):
     nb_objects_created = 0
     linked_models = set()
     csv_fieldnames = csv_reader.fieldnames
-    django_fields_to_column_names = _create_django_fields_to_column_names_mapping(model, csv_fieldnames, csv_filename)
+    django_fields_to_column_names = _create_django_fields_to_column_names_mapping(
+        model, csv_fieldnames, csv_filename
+    )
     for row in csv_reader:
         object_definition = {}
         for field, column_name in django_fields_to_column_names.items():
-            if not isinstance(field, ForeignKey) and not isinstance(field, ManyToManyField):
+            if not isinstance(field, ForeignKey) and not isinstance(
+                field, ManyToManyField
+            ):
                 # cas d'un champ simple avec une valeur
                 value = row.get(column_name)
                 object_definition[field.name] = _clean_value(value, field)
@@ -166,9 +191,15 @@ def _import_csv_to_model(csv_reader, csv_filename, model, is_relation=False):
                 try:
                     linked_model = _get_linked_model(column_name)
                     linked_models.add(linked_model)
-                    object_definition[field.name] = _get_update_or_create_related_object(linked_model, foreign_key_id)
+                    object_definition[field.name] = (
+                        _get_update_or_create_related_object(
+                            linked_model, foreign_key_id, csv_filename
+                        )
+                    )
                 except KeyError as e:
-                    logger.warning(f"Il n'y a pas de modèle défini pour cette table : {e}")
+                    logger.warning(
+                        f"Il n'y a pas de modèle défini pour cette table : {e}"
+                    )
 
         # ici, c'est un csv correspondant à une relation complexe (stockée dans un Model spécifique) qui est importée
         if model == Part:
@@ -180,13 +211,16 @@ def _import_csv_to_model(csv_reader, csv_filename, model, is_relation=False):
             object_with_history, created = model.objects.update_or_create(
                 **object_definition, defaults=default_extra_fields
             )
+            update_change_reason(object_with_history, f"Import csv {csv_filename}.")
         else:
             primary_key = _get_primary_key_label(csv_filename)
             if is_relation:
                 # seul le champ correspondant à la relation est mis à jour
                 # il n'y a que ce champ dans object_definition
                 field_name = list(object_definition)[0]
-                instance = _get_update_or_create_related_object(model, row.get(primary_key))
+                instance = _get_update_or_create_related_object(
+                    model, row.get(primary_key), csv_filename
+                )
                 field_to_update = getattr(instance, field_name)
                 nb_elem_in_field = len(field_to_update.all())
                 field_to_update.add(object_definition[field_name])
@@ -195,9 +229,11 @@ def _import_csv_to_model(csv_reader, csv_filename, model, is_relation=False):
                 # c'est le csv d'un Model qui est importé
                 # le champ `missing_import_data` devient False
                 object_definition["missing_import_data"] = False
+                # object_definition["_history_user"] = False
                 object_with_history, created = model.objects.update_or_create(
                     siccrf_id=row.get(primary_key), defaults=object_definition
                 )
+                update_change_reason(object_with_history, f"Import csv {csv_filename}.")
 
         nb_objects_created += created
         nb_line_in_success += 1
@@ -208,14 +244,20 @@ def _get_model_fields_to_complete(model):
     "Returns all fields(including many-to-many and foreign key) except non editable fields"
     model_fields = model._meta.get_fields()
     # le flag concrete indique les champs qui ont une colonne associée
-    return [field for field in model_fields if field.concrete and field.name not in AUTOMATICALLY_FILLED]
+    return [
+        field
+        for field in model_fields
+        if field.concrete and field.name not in AUTOMATICALLY_FILLED
+    ]
 
 
 def _get_column_name(field_name, csv_fields_in_header, csv_filename, prefixed=True):
     csv_field_names = DJANGO_FIELD_NAME_TO_CSV_FIELD_NAME_MAPPING[field_name]
     if prefixed:
         prefix = CSV_TO_TABLE_PREFIX_MAPPING[csv_filename]
-        csv_field_names = [f"{prefix}_{csv_field_name}" for csv_field_name in csv_field_names]
+        csv_field_names = [
+            f"{prefix}_{csv_field_name}" for csv_field_name in csv_field_names
+        ]
         csv_field_names = [name.removeprefix("_") for name in csv_field_names]
     try:
         csv_field_name = list(set(csv_field_names) & set(csv_fields_in_header))[0]
@@ -229,7 +271,9 @@ def _get_linked_model(column_name):
     Récupération du modèle correspondante au nom de colonne d'une clé étrangère
     """
     if not column_name.endswith("IDENT"):
-        logger.error(f"{column_name} n'est pas une colonne contenant une clé étrangère.")
+        logger.error(
+            f"{column_name} n'est pas une colonne contenant une clé étrangère."
+        )
         return
     else:
         foreign_key_prefix = column_name.split("_")[0]
@@ -255,9 +299,15 @@ def _create_django_fields_to_column_names_mapping(model, csv_fieldnames, csv_fil
         if model == Part and field.name in ["must_be_monitored", "is_useful"]:
             continue
         # le nom des colonnes contenant les clés étrangères ne sont pas préfixées par le nom de la table
-        prefixed = False if isinstance(field, ForeignKey) or isinstance(field, ManyToManyField) else True
+        prefixed = (
+            False
+            if isinstance(field, ForeignKey) or isinstance(field, ManyToManyField)
+            else True
+        )
         try:
-            column_name = _get_column_name(field.name, csv_fieldnames, csv_filename, prefixed=prefixed)
+            column_name = _get_column_name(
+                field.name, csv_fieldnames, csv_filename, prefixed=prefixed
+            )
             django_fields_to_column_names[field] = column_name
         except NameError:
             missing_fields.append(field.name)
@@ -290,7 +340,7 @@ def _clean_value(value, field):
     return value
 
 
-def _get_update_or_create_related_object(model, id):
+def _get_update_or_create_related_object(model, id, csv_filename):
     """
     Indépendamment de l'ordre dans lequel les fichiers sont importés,
     les objets sont créés avec seulement leur id s'ils existent dans un fichier relation
@@ -300,5 +350,10 @@ def _get_update_or_create_related_object(model, id):
         return model.objects.get(siccrf_id=id)
     except model.DoesNotExist as e:
         logger.warning(f"Création de l'id {id}, qui n'existait pas encore dans {e}.")
-        linked_obj, _ = model.objects.update_or_create(siccrf_id=id, defaults={"name": ""})
+
+        linked_obj, _ = model.objects.update_or_create(
+            siccrf_id=id, defaults={"name": ""}
+        )
+        update_change_reason(linked_obj, f"Import csv {csv_filename}.")
+
         return linked_obj

data/management/commands/load_ingredients.py

@@ -22,7 +22,11 @@ class Command(BaseCommand):
     def add_arguments(self, parser):
         # argument optionnel
         parser.add_argument(
-            "-d", "--directory", type=str, help="Indicates where the files are located.", default="files"
+            "-d",
+            "--directory",
+            type=str,
+            help="Indicates where the files are located.",
+            default="files",
         )
 
     def handle(self, *args, **options):