.gitlab-ci.yml

@@ -24,7 +24,6 @@ variables:
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   PROJECT_FOLDER: "src"
   WITH_DOCKER_JOBS: "false"
-  # DOCKER_IMG_NAME: "gpf-md5-checker"
 
 # Pip's cache doesn't store the python packages
 # https://pip.pypa.io/en/stable/reference/pip_install/#caching
@@ -62,7 +61,7 @@ git-hooks:
 
 flake8:
   stage: lint
-  image: python:3.9-slim-buster
+  image: python:3.9-slim-bullseye
   only:
     changes:
       - "**/*.py"
@@ -79,7 +78,7 @@ sast:
 # -- TEST JOBS --------------------------------------------------------------------------
 test:
   stage: test
-  image: python:3.9-slim-buster
+  image: python:3.9-slim-bullseye
   # filter disabled because sonar job requires it and always runs...
   # only:
   #   changes:
@@ -99,38 +98,19 @@ test:
   coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
   artifacts:
     when: always
+    paths:
+      - coverage.xml
+      - junit/test-results.xml
     reports:
       junit: junit/test-results.xml
       coverage_report:
         coverage_format: cobertura
         path: coverage.xml
 
-test:documentation:
-  stage: test
-  image: python:3.9-slim-buster
-  only:
-    changes:
-      - "docs/"
-      - requirements/documentation.txt
-      - ".gitlab-ci.yml"
-    # refs:
-    #   - merge_requests
-  before_script:
-    - python -m pip install -U -r requirements/base.txt
-    - python -m pip install -U -r requirements/documentation.txt
-  script:
-    - sphinx-build -b html -d docs/_build/cache -j auto -q docs build/docs
-  artifacts:
-    name: documentation
-    expose_as: "Built documentation static website"
-    paths:
-      - build/docs
-    when: always
-
 # -- BUILD JOBS -------------------------------------------------------------------------
 build:pip-wheel:
   stage: build
-  image: python:3.9-slim-buster
+  image: python:3.9-slim-bullseye
   only:
     refs:
       - main
@@ -147,7 +127,7 @@ build:pip-wheel:
 
 build:documentation:
   stage: build
-  image: python:3.9-slim-buster
+  image: python:3.9-slim-bullseye
   only:
     refs:
       - main
@@ -172,7 +152,7 @@ gitlab:container:
 
 gitlab:pypi:
   stage: deploy
-  image: python:3.9-slim-buster
+  image: python:3.9-slim-bullseye
   variables:
     GIT_STRATEGY: none
     TWINE_PASSWORD: "${CI_JOB_TOKEN}"

CHANGELOG.md

@@ -16,6 +16,13 @@ Unreleased
 
 -->
 
+## 0.5.0 - 2022-12-16
+
+- Fix Status returned
+- CI: add artifact for test report and coverage
+- Replace (TECHNICAL) ?ERROR by TECHNICAL_ERROR
+- Fix typo
+
 ## 0.4.0 - 2022-12-06
 
 - Make package much more generic

docs/conf.py

@@ -126,7 +126,7 @@ html_theme_options = {
 # Sphinx supports the following languages:
 #   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
 #   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
-html_search_language = "en"
+html_search_language = "fr"
 
 
 # Example configuration for intersphinx: refer to the Python standard library.

docs/development/setup.md

@@ -3,7 +3,7 @@
 ## Prérequis système
 
 - Python >= 3.9
-- Accès réseau sur:
+- Accès réseau sur :
   - l'instance GitLab : <https://gitlab.gpf-tech.ign.fr>
   - le dépôt officiel de paquets Python : <https://pypi.org/>
 - un [jeton d'accès personnel ou Personal Access Token (PAT)](https://gitlab.gpf-tech.ign.fr/-/profile/personal_access_tokens) avec le scope `read_api`

src/__about__.py

@@ -40,7 +40,7 @@ __uri_repository__ = (
 __uri_tracker__ = f"{__uri_repository__}issues/"
 __uri__ = __uri_repository__
 
-__version__ = "0.4.0"
+__version__ = "0.5.0"
 __version_info__ = tuple(
     [
         int(num) if num.isdigit() else num

src/cli.py

@@ -148,7 +148,7 @@ def main(argv: List[str] = None):
                 "CI_JOB_ID",
                 f"{__title_clean__}-{__version__}-{datetime.now():%Y-%m-%d_%H%M%s}",
             ),
-            status=Status.ERROR,
+            status=Status.TECHNICAL_ERROR,
             failures=["Bad upload (livraison) structure"],
             trace=error_message,
         )
@@ -181,7 +181,7 @@ def main(argv: List[str] = None):
                 "CI_JOB_ID",
                 f"{__title_clean__}-{__version__}-{datetime.now():%Y-%m-%d_%H%M%s}",
             ),
-            status=Status.ERROR,
+            status=Status.TECHNICAL_ERROR,
             failures=["Bad input configuration file."],
             trace=error_message,
         )
@@ -202,14 +202,16 @@ def main(argv: List[str] = None):
             chunk_size=args.chunk_size,
         )
         if run_result[0] != 0:
-            result_output.status = Status(run_result[0] % 2)
+            result_output.status = (
+                Status.TECHNICAL_ERROR if run_result[0] >= 3 else Status(run_result[0])
+            )
             result_output.failures = run_result[1]
     except Exception as error:
         logger.error(
             f"Running {__title__} (version {__version__}) failed. Trace: {error}"
         )
         result_output.failures = run_result[1]
-        result_output.status = Status.ERROR
+        result_output.status = Status.TECHNICAL_ERROR
         result_output.trace = error
 
     # write result into output file
@@ -221,7 +223,7 @@ def main(argv: List[str] = None):
     ct_success, ct_failure, ct_error = (
         count_dict_values(run_result[1], Status.SUCCESS.name),
         count_dict_values(run_result[1], Status.FAILURE.name),
-        count_dict_values(run_result[1], Status.ERROR.name),
+        count_dict_values(run_result[1], Status.TECHNICAL_ERROR.name),
     )
 
     # TODO: make this kind of output a generic function

src/constants.py

@@ -17,7 +17,7 @@ class Status(Enum):
 
     SUCCESS = 0
     FAILURE = 1
-    ERROR = 2
+    TECHNICAL_ERROR = 2
 
 
 def arg_type_path_folder(input_path: Union[Path, str]) -> Path:

src/md5sum.py

@@ -54,7 +54,7 @@ def validate(filename: str, md5digest: str, chunksize: int = 8192) -> Status:
     La fonction va comparer le hash md5 en entrée avec celui calculé sur le fichier.
 
     Returns:
-        Status.ERROR en cas d'erreur (lecture de fichier)
+        Status.TECHNICAL_ERROR en cas d'erreur (lecture de fichier)
         Status.FAILURE si le hash ne correspond pas avec celui calculé
         Status.SUCCESS autrement
     """
@@ -64,7 +64,7 @@ def validate(filename: str, md5digest: str, chunksize: int = 8192) -> Status:
         logger.error(
             f"{filename}: TECHNICAL ERROR: le fichier {filename} n'existe pas."
         )
-        return Status.ERROR
+        return Status.TECHNICAL_ERROR
 
     result = generate_md5_sum(filename, chunksize) == md5digest
     status_return = Status.SUCCESS if result is True else Status.FAILURE
@@ -101,7 +101,7 @@ def check_md5_file(filename: Path, status: dict, chunksize: int = 8192) -> int:
                     logger.error(
                         f"TECHNICAL ERROR: la ligne {line} n'est pas conforme."
                     )
-                    result |= Status.ERROR.value
+                    result |= Status.TECHNICAL_ERROR.value
                     continue
 
                 checksum = line[:32]

tests/fixtures/fix_702/parameters.json

+{
+    "executionId": "1231544456-1546546-164565",
+    "userId": "1546546231-1231544-164565",
+    "inputParameters": {
+        "parameter1_str": "string_value",
+        "parameter2_int": 10000
+    },
+    "targetParameters": {
+        "databaseInfo": {
+            "url": "",
+            "user": "gpf-ro-user",
+            "password": "use-me-i-m-read-only",
+            "schema": ""
+        },
+        "objectStorageInfo": {
+            "url": "s3://gpf/datasets/",
+            "token": "AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4IgRmpRV3zZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE"
+        }
+    },
+    "technicalParameters": {
+        "databaseInfo": {
+            "url": "",
+            "user": "",
+            "password": "",
+            "schema": ""
+        }
+    }
+}

tests/fixtures/fix_702/upload/fichier.md5

+c292924b183497010f77e1d5bfb4c4b1 file.md5
\ No newline at end of file

tests/test_cli.py

@@ -8,6 +8,8 @@
 """
 
 
+import json
+
 # standard
 from os import environ
 from pathlib import Path
@@ -58,6 +60,33 @@ def test_cli_run_env_vars(capsys):
     assert err == ""
 
 
+def test_consistent_output_level(capsys):
+    """Test bugfix https://jira.worldline.com/browse/IGNGPF-702"""
+
+    test_path = Path(__file__).parent.absolute()
+    result = {
+        "executionId": "1231544456-1546546-164565",
+        "failures": {
+            f"{test_path}/fixtures/fix_702/upload/fichier.md5": [
+                [f"{test_path}/fixtures/fix_702/upload/file.md5", "TECHNICAL_ERROR"]
+            ]
+        },
+        "status": "TECHNICAL_ERROR",
+        "trace": None,
+    }
+    # '{"executionId": "1231544456-1546546-164565", "status": "TECHNICAL_ERROR", "failures": {"tests/fixtures/fix_702/upload/fichier.md5": [["tests/fixtures/fix_702/upload/file.md5", "TECHNICAL_ERROR"]]}, "trace": None}'
+
+    environ["GPF_WORK_DIR"] = str(Path("./tests/fixtures/fix_702").resolve())
+    environ["GPF_UPLOAD_DIR"] = "upload"
+    environ["GPF_INPUT_CONFIGURATION_FILENAME"] = "parameters.json"
+    cli.main([])
+    out, err = capsys.readouterr()
+
+    with open(Path(environ["GPF_WORK_DIR"]) / "output.json") as output_json:
+        data = json.load(output_json)
+        assert data == result
+
+
 def test_cli_help(capsys):
     """Test CLI help."""
     with pytest.raises(SystemExit):

tests/test_md5_checker.py

@@ -64,7 +64,7 @@ class TestMD5(unittest.TestCase):
                 Path("./tests/fixtures/livraisons/good/default/upload/oslandia.tx"),
                 "b5871a318190397c5878ff2bd9f326d2",
             )
-            == md5sum.Status.ERROR
+            == md5sum.Status.TECHNICAL_ERROR
         )
 
     def test_check_md5_file(self):

tests/test_model_output.py

@@ -125,7 +125,7 @@ class TestOutputObject(unittest.TestCase):
         """Test object."""
         output = OutputDataStructure(
             executionId="1231544456-1546546-164565",
-            status=Status.ERROR,
+            status=Status.TECHNICAL_ERROR,
             failures=("adresses.gpkg", "sirene.csv"),
             trace="Fichier illisible",
         )
@@ -156,7 +156,7 @@ class TestOutputObject(unittest.TestCase):
 
         # values
         self.assertEqual(data.get("executionId"), "1231544456-1546546-164565")
-        self.assertEqual(data.get("status"), Status.ERROR.name)
+        self.assertEqual(data.get("status"), Status.TECHNICAL_ERROR.name)
         self.assertEqual(data.get("failures"), ["adresses.gpkg", "sirene.csv"])
         self.assertEqual(data.get("trace"), "Fichier illisible")
 
@@ -167,7 +167,7 @@ class TestOutputObject(unittest.TestCase):
         """Test object."""
         output = OutputDataStructure(
             executionId="1231544456-1546546-164565",
-            status=Status.ERROR,
+            status=Status.TECHNICAL_ERROR,
             failures=("adresses.gpkg", "sirene.csv"),
             trace=FileNotFoundError("Fichier illisible"),
         )
@@ -198,7 +198,7 @@ class TestOutputObject(unittest.TestCase):
 
         # values
         self.assertEqual(data.get("executionId"), "1231544456-1546546-164565")
-        self.assertEqual(data.get("status"), Status.ERROR.value)
+        self.assertEqual(data.get("status"), Status.TECHNICAL_ERROR.value)
         self.assertEqual(data.get("failures"), ["adresses.gpkg", "sirene.csv"])
         self.assertEqual(data.get("trace"), "Fichier illisible")