Merge branch 'feat/add_status_file' into 'main'

Feat/add status file

Closes #12

See merge request !19

rlt_migration/core.py

@@ -6,6 +6,7 @@
 # -- IMPORTS
 
 # standard lib
+import datetime
 import logging
 from pathlib import Path
 
@@ -154,6 +155,21 @@ def dataset_migration(
     return dataset
 
 
+def _add_status(
+    status_file: Path, dataset_identifier: str, valid: bool, message: str
+) -> None:
+    """Add dataset status to file
+
+    Args:
+        status_file (Path): Status file
+        dataset_identifier (str): dataset identifier
+        valid (bool): True if migration valid, False otherwise
+        message (str): Message for invalid migration
+    """
+    with status_file.open("a") as f:
+        f.write(f"{dataset_identifier};{'OK' if valid else 'KO'};{message}\n")
+
+
 @benchmark.timing_decorator()
 def datasets_migration(
     input_dataset: [Dataset],
@@ -162,12 +178,18 @@ def datasets_migration(
     input_color_csv: Path,
     input_svc_couches_database_config: DatabaseConfig,
     db_config: DatabaseConfig,
+    status_file: Path,
 ) -> [Dataset]:
     """Get valid dataset from input dataset csv and input kml folder
 
     Args:
         input_dataset_csv (Path): Input dataset csv
         input_kml_folder (Path): Input kml folder
+        input_ta_num_scan_csv (Path): Input TA NumScan csv. File must exist.
+        input_color_csv (Path): Input color csv. File must exist.
+        input_svc_couches_database_config (DatabaseConfig) : Input svc couches database configuration
+        db_config  (DatabaseConfig) : Output database configuration,
+        status_file (Path) : Status file
 
     Returns:
         [Dataset]: List of valid dataset
@@ -191,6 +213,12 @@ def datasets_migration(
                 f"Fichier kml root '{root_kml_path}' non présent. La mission {dataset.dataset_identifier} ne sera pas intégrée."
             )
             nb_invalid_data_set = nb_invalid_data_set + 1
+            _add_status(
+                dataset_identifier=dataset.dataset_identifier,
+                status_file=status_file,
+                valid=False,
+                message="No root kml",
+            )
             continue
         try:
             dataset_migration(
@@ -207,31 +235,67 @@ def datasets_migration(
                 f"Fichier kml root '{root_kml_path}' est invalide : {exc}. La mission {dataset.dataset_identifier} ne sera pas intégrée."
             )
             nb_invalid_data_set = nb_invalid_data_set + 1
+            _add_status(
+                dataset_identifier=dataset.dataset_identifier,
+                status_file=status_file,
+                valid=False,
+                message=f"Invalid root kml : {exc}",
+            )
             continue
         except InvalidLeafKml as exc:
             logger.info(
                 f"Un des fichiers feuilles est invalide : {exc}. La mission {dataset.dataset_identifier} ne sera pas intégrée."
             )
             nb_invalid_data_set = nb_invalid_data_set + 1
+            _add_status(
+                dataset_identifier=dataset.dataset_identifier,
+                status_file=status_file,
+                valid=False,
+                message=f"Invalid leaf kml : {exc}",
+            )
             continue
         except InvalidPlacemarkElement as exc:
             logger.info(
                 f"Un des placemark d'un fichier feuille est invalide : {exc}. La mission {dataset.dataset_identifier} ne sera pas intégrée."
             )
             nb_invalid_data_set = nb_invalid_data_set + 1
+            _add_status(
+                dataset_identifier=dataset.dataset_identifier,
+                status_file=status_file,
+                valid=False,
+                message=f"Invalid placemark in kml : {exc}",
+            )
             continue
         except MissingImage as exc:
             logger.info(
                 f"Une image est manquante : {exc}. La mission {dataset.dataset_identifier} ne sera pas intégrée."
             )
             nb_invalid_data_set = nb_invalid_data_set + 1
+            _add_status(
+                dataset_identifier=dataset.dataset_identifier,
+                status_file=status_file,
+                valid=False,
+                message=f"Missing image : {exc}",
+            )
             continue
         except Exception as exc:
             logger.info(
                 f"Un des fichiers feuilles est invalide : {exc}. La mission {dataset.dataset_identifier} ne sera pas intégrée."
             )
             nb_invalid_data_set = nb_invalid_data_set + 1
+            _add_status(
+                dataset_identifier=dataset.dataset_identifier,
+                status_file=status_file,
+                valid=False,
+                message=f"Unknown exception : {exc}",
+            )
             continue
+        _add_status(
+            dataset_identifier=dataset.dataset_identifier,
+            status_file=status_file,
+            valid=True,
+            message="",
+        )
         added_dataset.append(dataset)
     if nb_invalid_data_set:
         logger.warning(f"{nb_invalid_data_set} missions ne sont pas valides.")
@@ -300,6 +364,13 @@ def run(
     input_dataset: [Dataset] = read_dataset_from_csv(input_dataset_csv)
     logger.info(f"{len(input_dataset)} missions à intégrer")
 
+    logger.info("Création du .csv des statuts de migration.")
+    status_file = (
+        Path(work_dir)
+        / f"rlt-migration-status-{datetime.datetime.now().strftime('%Y-%m-%d-%H:%M-%S')}.csv"
+    )
+    status_file.write_text("Dataset identifier;Status;Message\n")
+
     valid_dataset: [Dataset] = datasets_migration(
         input_dataset=input_dataset,
         input_kml_folder=input_kml_folder,
@@ -307,6 +378,7 @@ def run(
         input_color_csv=input_color_csv,
         input_svc_couches_database_config=input_svc_couches_database_config,
         db_config=output_database_config,
+        status_file=status_file,
     )
     nb_dataset = len(valid_dataset)
     if nb_dataset:

[sonar-scanner]

SonarQube analysis indicates that quality gate is passed.

• Reliability Rating on New Code is passed: Actual value 1
• Security Rating on New Code is passed: Actual value 1
• Maintainability Rating on New Code is passed: Actual value 1
• Coverage on New Code is passed: Actual value 84.40265486725664
• Duplicated Lines (%) on New Code is passed: Actual value 0.0

SonarQube analysis reported no issues.