diff --git a/CHANGELOG.md b/CHANGELOG.md
index cc10a748a41fc5828d8f78a2d885586e37b5ad67..a1c70b67a373dbbcb9aa7508f88334ebf914c21a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,8 @@
The format is based on [Keep a Changelog](https://keepachangelog.com/), and this project adheres to [Semantic Versioning](https://semver.org/).
## [Unreleased]
+### Changed
+- IGNGPF-4527: delete pdal_merge and add pdal_merge_using_tile, delete pdal_convert_las_to_copc and add untwine_convert_las_to_copc
## [1.9.0] - 2025-03-05
### Fixed
diff --git a/gpf_common_point_cloud/models/pdal_tile_parameters.py b/gpf_common_point_cloud/models/pdal_tile_parameters.py
index b606545ae9606eaf447f03957a55c976281a30bb..2b92e0a4edf46c271f97c730ea12421205381313 100644
--- a/gpf_common_point_cloud/models/pdal_tile_parameters.py
+++ b/gpf_common_point_cloud/models/pdal_tile_parameters.py
@@ -22,3 +22,16 @@ class PdalTileParameters:
self.origin_x = origin_x if origin_x is not None else 0
self.origin_y = origin_y if origin_y is not None else 0
self.tile_size = tile_size if tile_size is not None else 1000
+
+ def __eq__(self, other):
+ """
+ Override __eq__ to avoid comparing objects memory id, usefull for unittest
+ """
+ if not isinstance(other, PdalTileParameters):
+ return NotImplemented
+ return (
+ self.out_srs == other.out_srs
+ and self.origin_x == other.origin_x
+ and self.origin_y == other.origin_y
+ and self.tile_size == other.tile_size
+ )
diff --git a/gpf_common_point_cloud/utils/pdal_utils.py b/gpf_common_point_cloud/utils/pdal_utils.py
index fc8add62daca218bc45e0e196d294be082e8cbc1..27bb32cbf418611cdcad20e75350ff17f9534ba9 100644
--- a/gpf_common_point_cloud/utils/pdal_utils.py
+++ b/gpf_common_point_cloud/utils/pdal_utils.py
@@ -7,12 +7,14 @@
# standard lib
import logging
import re
+import shutil
import subprocess
from pathlib import Path
# 3rd party
import pdal
from gpf_entrepot_toolbelt.orchestrator.status import Status
+from gpf_entrepot_toolbelt.utils.os_utils import create_symlink
# project
from gpf_common_point_cloud.models import PdalTileParameters
@@ -117,38 +119,47 @@ def pdal_info_from_ept(ept_head_file: Path) -> Status:
return Status.TECHNICAL_ERROR
-def pdal_merge(files_to_merge: list[Path], output_file_path: Path) -> Status:
+def pdal_merge_using_tile(
+ files_to_merge: list[Path], output_file_path: Path, out_srs: str
+) -> Status:
"""
- Uses pdal merge to combine input files into a single output file
+ Uses pdal tile (with tile_size=0) to combine input files into a single output file
Args:
files_to_merge (list[Path]): input files paths
output_file_path (Path): output file path
+ out_srs (str): output SRS
Returns:
Status: SUCCESS if the job is ok, FAILURE otherwise
"""
- logger.info(f"pdal merge vers le fichier {output_file_path}")
+ logger.info(
+ f"Lancement du processus de fusion des dalles en utilisant pdal tile vers le fichier {output_file_path}"
+ )
+ if not files_to_merge:
+ logger.warning("La liste des fichiers à fusionner est vide, rien à faire")
+ return Status.SUCCESS
try:
- if not files_to_merge:
- logger.warning("La liste des fichiers à fusionner est vide, rien à faire")
- return Status.SUCCESS
+ temp_merge_input_folder = files_to_merge[0].parent / "merge_input"
+ create_symlink(
+ files_to_merge, temp_merge_input_folder, add_parent_as_suffix=True
+ )
- files_to_process: str = " ".join([str(tile) for tile in files_to_merge])
- command: str = f"pdal merge --verbose=2 {files_to_process} {output_file_path}"
- logger.info("Lancement de la commande pdal merge")
- logger.debug(command)
- run_result = subprocess.run(command, shell=True, capture_output=True, text=True)
+ pdal_tile_params = PdalTileParameters(out_srs, 0, 0, tile_size=int(1e12))
+ pdal_tile_las_result: Status = pdal_tile_las(
+ pdal_tile_params, temp_merge_input_folder.resolve(), output_file_path.parent
+ )
- if run_result.returncode != 0:
- logger.user_error("Erreur lors de l'execution de pdal merge")
- logger.error(f"Code retour : {run_result.returncode}")
- logger.error(f"Stdout : {run_result.stdout}")
- logger.error(f"Stderr : {run_result.stderr}")
- return Status.FAILURE
+ if pdal_tile_las_result != Status.SUCCESS:
+ return pdal_tile_las_result
+ shutil.rmtree(temp_merge_input_folder)
+ outfile: Path = output_file_path.parent / "tile_0_0.laz"
+ outfile.rename(
+ output_file_path.parent / f"{output_file_path.name.split('.')[0]}.laz"
+ )
return Status.SUCCESS
except Exception as error:
- logger.user_error("Erreur lors de l'execution de pdal merge")
+ logger.user_error("Une erreur rencontré lors du processus de fusion des dalles")
logger.error(error, stack_info=True)
return Status.FAILURE
@@ -207,25 +218,41 @@ def pdal_tile_las(
return Status.FAILURE
-def pdal_convert_las_to_copc(file_path: Path, output_file_path: Path) -> Status:
+def untwine_convert_las_to_copc(
+ files_paths: list[Path], output_file_path: Path, out_srs: str
+) -> Status:
"""
Conversion of files from las/laz to COPC
Args:
- file_path (Path): las/laz file path to convert
+ files_paths (list[Path]): las/laz files paths to merge and convert
output_file_path (Path): output file path copc.laz
+ out_srs (str): output SRS
Returns:
Status: SUCCESS if the job is ok, FAILURE otherwise
"""
- logger.info(f"Conversion en COPC de {file_path}")
+ logger.info(
+ f"Conversion en COPC de {files_paths} vers le fichier {output_file_path}"
+ )
try:
- pipeline = pdal.Reader.las(filename=str(file_path)).pipeline()
- pipeline |= pdal.Writer.copc(filename=str(output_file_path))
- pipeline.execute()
+ files = ",".join([str(file.resolve()) for file in files_paths])
+
+ command: str = (
+ f"untwine --files={files} --output_file={output_file_path.resolve()} --a_srs={out_srs}"
+ )
+ logger.info(command)
+
+ run_result = subprocess.run(command, shell=True, capture_output=True, text=True)
+ logger.info(f"Stdout : {run_result.stdout}")
+
+ if run_result.returncode != 0:
+ logger.user_error("Erreur lors de la conversion en COPC")
+ logger.error(f"Code retour : {run_result.returncode}")
+ logger.error(f"Stderr : {run_result.stderr}")
+ return Status.FAILURE
+
return Status.SUCCESS
except Exception as error:
- logger.user_error(
- f"Une erreur rencontré lors de la conversion en COPC de {file_path.name}"
- )
+ logger.user_error("Erreur lors de la conversion en COPC")
logger.error(error, stack_info=True)
return Status.FAILURE
diff --git a/tests/test_pdal_utils.py b/tests/test_pdal_utils.py
index b4a4228ecff35ed3444238ae2ca2b141b430caee..4dcb069d05d880f2b01a353aaf3d128327fdde69 100644
--- a/tests/test_pdal_utils.py
+++ b/tests/test_pdal_utils.py
@@ -21,11 +21,11 @@ from gpf_common_point_cloud.utils.laslaz_utils import (
READER_NAME,
)
from gpf_common_point_cloud.utils.pdal_utils import (
- pdal_convert_las_to_copc,
pdal_info_from_ept,
pdal_info_from_las,
- pdal_merge,
+ pdal_merge_using_tile,
pdal_tile_las,
+ untwine_convert_las_to_copc,
)
logger = gpf_logger_script(verbosity=0, title="TestPdalUtils")
@@ -98,78 +98,94 @@ class TestPdalUtils(unittest.TestCase):
self.assertEqual(result[READER_NAME][MAXY_FIELD], 6288550)
self.assertEqual(result[READER_NAME]["maxz"], 381.9)
- @patch("subprocess.run")
- def test_merge_tiles(self, mock_run):
+ @patch("gpf_common_point_cloud.utils.pdal_utils.create_symlink")
+ @patch("gpf_common_point_cloud.utils.pdal_utils.pdal_tile_las")
+ @patch("shutil.rmtree")
+ @patch("gpf_common_point_cloud.utils.pdal_utils.Path.rename")
+ def test_merge_tiles(
+ self, mock_path_rename, mock_rmtree, mock_pdal_tile_las, mock_create_symlink
+ ):
# Given
- mock_proc = MagicMock(returncode=0)
- mock_proc.communicate.return_value = (b"standard output", b"")
- mock_run.return_value = mock_proc
-
+ mock_pdal_tile_las.return_value = Status.SUCCESS
tiles_to_merge = [Path("/fake/path/tile1.laz"), Path("/fake/path/tile2.laz")]
output_file_path = Path("/fake/output/merged.laz")
+ out_srs = "EPSG:2154"
# When
- result = pdal_merge(tiles_to_merge, output_file_path)
+ result = pdal_merge_using_tile(tiles_to_merge, output_file_path, out_srs)
# Then
- files_to_process = " ".join([str(tile) for tile in tiles_to_merge])
- command_to_run = f"pdal merge --verbose=2 {files_to_process} {output_file_path}"
+ pdal_tile_params = PdalTileParameters(out_srs, 0, 0, tile_size=int(1e12))
- mock_run.assert_called_once_with(
- command_to_run, shell=True, capture_output=True, text=True
+ mock_create_symlink.assert_called_once_with(
+ tiles_to_merge, Path("/fake/path/merge_input"), add_parent_as_suffix=True
+ )
+ mock_pdal_tile_las.assert_called_once_with(
+ pdal_tile_params, Path("/fake/path/merge_input"), Path("/fake/output")
)
+ mock_path_rename.assert_called_once_with(Path("/fake/output/merged.laz"))
+ mock_rmtree.assert_any_call(Path("/fake/path/merge_input"))
+
self.assertEqual(Status.SUCCESS, result)
def test_merge_tiles_no_files(self):
# Given
tiles_to_merge = []
output_file_path = Path("/fake/output/merged.laz")
+ out_srs = "EPSG:2154"
# When
- result = pdal_merge(tiles_to_merge, output_file_path)
+ result = pdal_merge_using_tile(tiles_to_merge, output_file_path, out_srs)
# Then
self.assertEqual(Status.SUCCESS, result)
- @patch("subprocess.run")
- def test_merge_tiles_error(self, mock_run):
+ @patch("gpf_common_point_cloud.utils.pdal_utils.create_symlink")
+ @patch("gpf_common_point_cloud.utils.pdal_utils.pdal_tile_las")
+ @patch("shutil.rmtree")
+ @patch("gpf_common_point_cloud.utils.pdal_utils.Path.rename")
+ def test_merge_tiles_error(
+ self, mock_path_rename, mock_rmtree, mock_pdal_tile_las, mock_create_symlink
+ ):
# Given
- mock_proc = MagicMock(returncode=3)
- mock_proc.communicate.return_value = (b"standard output", b"standard error")
- mock_run.return_value = mock_proc
-
+ mock_pdal_tile_las.return_value = Status.FAILURE
tiles_to_merge = [Path("/fake/path/tile1.laz"), Path("/fake/path/tile2.laz")]
output_file_path = Path("/fake/output/merged.laz")
+ out_srs = "EPSG:2154"
# When
- result = pdal_merge(tiles_to_merge, output_file_path)
+ result = pdal_merge_using_tile(tiles_to_merge, output_file_path, out_srs)
# Then
- files_to_process = " ".join([str(tile) for tile in tiles_to_merge])
- command_to_run = f"pdal merge --verbose=2 {files_to_process} {output_file_path}"
+ pdal_tile_params = PdalTileParameters(out_srs, 0, 0, tile_size=int(1e12))
- mock_run.assert_called_once_with(
- command_to_run, shell=True, capture_output=True, text=True
+ mock_create_symlink.assert_called_once_with(
+ tiles_to_merge, Path("/fake/path/merge_input"), add_parent_as_suffix=True
+ )
+ mock_pdal_tile_las.assert_called_once_with(
+ pdal_tile_params, Path("/fake/path/merge_input"), Path("/fake/output")
)
+ mock_path_rename.assert_not_called()
+ mock_rmtree.assert_not_called()
+
self.assertEqual(Status.FAILURE, result)
- @patch("subprocess.run")
- def test_merge_tiles_exception(self, mock_run):
+ @patch("gpf_common_point_cloud.utils.pdal_utils.create_symlink")
+ def test_merge_tiles_exception(self, mock_create_symlink):
# Given
- mock_run.side_effect = Exception("Test exception")
+ mock_create_symlink.side_effect = Exception("Test exception")
tiles_to_merge = [Path("/fake/path/tile1.laz"), Path("/fake/path/tile2.laz")]
output_file_path = Path("/fake/output/merged.laz")
+ out_srs = "EPSG:2154"
# When
- result = pdal_merge(tiles_to_merge, output_file_path)
+ result = pdal_merge_using_tile(tiles_to_merge, output_file_path, out_srs)
# Then
- files_to_process = " ".join([str(tile) for tile in tiles_to_merge])
- command_to_run = f"pdal merge --verbose=2 {files_to_process} {output_file_path}"
-
- mock_run.assert_called_once_with(
- command_to_run, shell=True, capture_output=True, text=True
+ mock_create_symlink.assert_called_once_with(
+ tiles_to_merge, Path("/fake/path/merge_input"), add_parent_as_suffix=True
)
+
self.assertEqual(Status.FAILURE, result)
@patch("subprocess.run")
@@ -294,30 +310,99 @@ class TestPdalUtils(unittest.TestCase):
)
self.assertEqual(Status.FAILURE, result)
- def test_pdal_convert_las_to_copc(self):
+ @patch("subprocess.run")
+ def test_untwine_convert_las_to_copc_one_file(self, mock_run):
# Given
- input_file_path = Path("./tests/fixtures/laslaz/tile_4860_67320.copc.laz")
- output_file_path = Path("./tests/fixtures/laslaz/converted.copc.laz")
+ input_file = Path("/path/to/input/file.laz")
+ output_file = Path("/path/to/output/file.laz")
+ out_srs = "EPSG:2154"
+
+ mock_proc = MagicMock(returncode=0)
+ mock_proc.communicate.return_value = (b"standard output", b"")
+ mock_run.return_value = mock_proc
# When
- result = pdal_convert_las_to_copc(input_file_path, output_file_path)
+ result = untwine_convert_las_to_copc([input_file], output_file, out_srs)
# Then
+ command: str = (
+ f"untwine --files={input_file.resolve()} --output_file={output_file.resolve()} --a_srs={out_srs}"
+ )
+
+ mock_run.assert_called_once_with(
+ command, shell=True, capture_output=True, text=True
+ )
self.assertEqual(Status.SUCCESS, result)
- self.assertTrue(output_file_path.exists())
- output_file_path.unlink()
- @patch("pdal.Reader.las")
- def test_pdal_convert_las_to_copc_exception(self, mock_pdal_reader):
+ @patch("subprocess.run")
+ def test_untwine_convert_las_to_copc_multiple_files(self, mock_run):
# Given
- input_file_path = Path("./tests/fixtures/laslaz/tile_4860_67320.copc.laz")
- output_file_path = Path("./tests/fixtures/laslaz/converted.copc.laz")
- mock_pdal_reader.side_effect = Exception("Test exception")
+ input_file = Path("/path/to/input/file.laz")
+ output_file = Path("/path/to/output/file.laz")
+ out_srs = "EPSG:2154"
+
+ mock_proc = MagicMock(returncode=0)
+ mock_proc.communicate.return_value = (b"standard output", b"")
+ mock_run.return_value = mock_proc
# When
- result = pdal_convert_las_to_copc(input_file_path, output_file_path)
+ result = untwine_convert_las_to_copc(
+ [input_file, input_file, input_file], output_file, out_srs
+ )
# Then
+ command: str = (
+ f"untwine --files={input_file.resolve()},{input_file.resolve()},{input_file.resolve()} --output_file={output_file.resolve()} --a_srs={out_srs}"
+ )
+
+ mock_run.assert_called_once_with(
+ command, shell=True, capture_output=True, text=True
+ )
+ self.assertEqual(Status.SUCCESS, result)
+
+ @patch("subprocess.run")
+ def test_untwine_convert_las_to_copc_error(self, mock_run):
+ # Given
+ input_file = Path("/path/to/input/file.laz")
+ output_file = Path("/path/to/output/file.laz")
+ out_srs = "EPSG:2154"
+
+ mock_proc = MagicMock(returncode=3)
+ mock_proc.communicate.return_value = (b"standard output", b"standard error")
+ mock_run.return_value = mock_proc
+
+ # When
+ result = untwine_convert_las_to_copc([input_file], output_file, out_srs)
+
+ # Then
+ command: str = (
+ f"untwine --files={input_file.resolve()} --output_file={output_file.resolve()} --a_srs={out_srs}"
+ )
+
+ mock_run.assert_called_once_with(
+ command, shell=True, capture_output=True, text=True
+ )
+ self.assertEqual(Status.FAILURE, result)
+
+ @patch("subprocess.run")
+ def test_untwine_convert_las_to_copc_exception(self, mock_run):
+ # Given
+ input_file = Path("/path/to/input/file.laz")
+ output_file = Path("/path/to/output/file.laz")
+ out_srs = "EPSG:2154"
+ mock_run.side_effect = Exception("Test exception")
+
+ # When
+ result = untwine_convert_las_to_copc([input_file], output_file, out_srs)
+
+ # Then
+ command: str = (
+ f"untwine --files={input_file.resolve()} --output_file={output_file.resolve()} --a_srs={out_srs}"
+ )
+
+ mock_run.assert_called_once_with(
+ command, shell=True, capture_output=True, text=True
+ )
self.assertEqual(Status.FAILURE, result)
def test_pdal_info_from_ept_success(self):