Update test data with dataformat_id >=6 to check for classifications …

…> 31

test/test_las_add_buffer.py

@@ -2,10 +2,14 @@
 import os
 import shutil
 import test.utils as tu
+from test.utils import assert_header_info_are_similar
 
 import laspy
 import numpy as np
 
+from pdaltools.count_occurences.count_occurences_for_attribute import (
+    compute_count_one_file,
+)
 from pdaltools.las_add_buffer import create_las_with_buffer
 
 TEST_PATH = os.path.dirname(os.path.abspath(__file__))
@@ -43,18 +47,16 @@ def get_2d_bounding_box(path):
 # Tests
 def test_create_las_with_buffer():
     output_file = os.path.join(TMP_PATH, "buffer.las")
-
-    coord_x = 77055
-    coord_y = 627760
-    # Note: neighbor tile 77050_627760 is cropped to simulate missing data in neighbors during merge
-    input_file = os.path.join(INPUT_DIR, f"test_data_{coord_x}_{coord_y}_LA93_IGN69_ground.las")
+    # Note: this tile does not have a tile at its bottom
+    # And its left-side tile has been crop to have no data in the buffer area. This case must not generate any error
+    input_file = os.path.join(INPUT_DIR, "test_data_77055_627755_LA93_IGN69.laz")
     tile_width = 50
     tile_coord_scale = 10
-    expected_output_nb_points = 40177
-    expected_out_mins = [770540.01, 6277540.0]
-    expected_out_maxs = [770610.0, 6277600.0]
+    input_nb_points = 72770
+    expected_out_mins = [770545.0, 6277500.0]
+    expected_out_maxs = [770605.0, 6277555.0]
 
-    buffer_width = 10
+    buffer_width = 5
     create_las_with_buffer(
         INPUT_DIR,
         input_file,
@@ -77,15 +79,20 @@ def test_create_las_with_buffer():
     tu.allclose_absolute(out_maxs, in_maxs + buffer_width, 1e-3)
 
     # check number of points
-    assert get_nb_points(output_file) == expected_output_nb_points
+    assert get_nb_points(output_file) > input_nb_points
 
-    # Check contre valeur attendue
+    # Check boundaries
     assert np.all(out_mins == expected_out_mins)
     assert np.all(out_maxs == expected_out_maxs)
 
-    # Check output las version (input is 1.4)
-    json_info = tu.get_pdal_infos_summary(output_file)
-    assert json_info["summary"]["metadata"]["minor_version"] == 4
+    # Check the input header infos are preserved in the output
+    assert_header_info_are_similar(output_file, input_file)
+
+    # Check that classes are preserved (in particular classes over 31)
+    # Warning: classification values > 31 exist only for las 1.4 with dataformat_id >= 6
+    classes_counts = compute_count_one_file(output_file)
+
+    assert set(classes_counts.keys()) == {"1", "2", "3", "4", "5", "6", "64"}
 
 
 if __name__ == "__main__":

test/test_las_clip.py

@@ -1,36 +1,25 @@
 import logging
 import os
 import shutil
+from test.utils import assert_header_info_are_similar
 
 import laspy
 import numpy as np
 
 from pdaltools.las_clip import las_crop
 
-test_path = os.path.dirname(os.path.abspath(__file__))
-tmp_path = os.path.join(test_path, "tmp")
-input_dir = os.path.join(test_path, "data")
-output_file = os.path.join(tmp_path, "cropped.las")
-
-coord_x = 77055
-coord_y = 627760
-input_file = os.path.join(input_dir, f"test_data_{coord_x}_{coord_y}_LA93_IGN69_ground.las")
-
-input_nb_points = 22343
-expected_output_nb_points = 6578
-input_mins = [770550.0, 6277550.0]
-input_maxs = [770600.0, 6277600.0]
-expected_out_mins = [770560.0, 6277560.0]
-expected_out_maxs = [770590.0, 6277590.0]
+TEST_PATH = os.path.dirname(os.path.abspath(__file__))
+TMP_PATH = os.path.join(TEST_PATH, "tmp")
+INPUT_DIR = os.path.join(TEST_PATH, "data")
 
 
 def setup_module(module):
     try:
-        shutil.rmtree(tmp_path)
+        shutil.rmtree(TMP_PATH)
 
     except FileNotFoundError:
         pass
-    os.mkdir(tmp_path)
+    os.mkdir(TMP_PATH)
 
 
 # Utils functions
@@ -53,14 +42,22 @@ def get_2d_bounding_box(path):
 
 # Tests
 def test_las_crop():
+    output_file = os.path.join(TMP_PATH, "cropped.las")
+
+    input_file = os.path.join(INPUT_DIR, "test_data_77055_627760_LA93_IGN69.laz")
+    # FYI input min/max are: input_mins = [770550.0, 6277550.0] and input_maxs = [770600.0, 6277600.0]
+
+    expected_output_nb_points = 20862
+    expected_out_mins = [770560.0, 6277560.0]
+    expected_out_maxs = [770590.0, 6277590.0]
     bounds = ([expected_out_mins[0], expected_out_maxs[0]], [expected_out_mins[1], expected_out_maxs[1]])
+
     las_crop(input_file, output_file, bounds)
 
     # check file exists
     assert os.path.isfile(output_file)
 
     # check difference in bbox
-    in_mins, in_maxs = get_2d_bounding_box(input_file)
     out_mins, out_maxs = get_2d_bounding_box(output_file)
 
     assert np.all(out_mins == expected_out_mins)
@@ -69,6 +66,9 @@ def test_las_crop():
     # check number of points
     assert get_nb_points(output_file) == expected_output_nb_points
 
+    # check that the las format is preserved
+    assert_header_info_are_similar(output_file, input_file)
+
 
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)

test/test_las_info.py

@@ -13,7 +13,7 @@
 COORD_X = 77055
 COORD_Y = 627760
 
-INPUT_FILE = os.path.join(DATA_PATH, f"test_data_{COORD_X}_{COORD_Y}_LA93_IGN69_ground.las")
+INPUT_FILE = os.path.join(DATA_PATH, f"test_data_{COORD_X}_{COORD_Y}_LA93_IGN69.laz")
 
 TILE_WIDTH = 50
 TILE_COORD_SCALE = 10
@@ -70,7 +70,7 @@ def test_las_get_xy_bounds_with_buffer():
 def test_parse_filename():
     prefix, parsed_coord_x, parsed_coord_y, suffix = las_info.parse_filename(INPUT_FILE)
     assert prefix == "test_data"
-    assert suffix == "LA93_IGN69_ground.las"
+    assert suffix == "LA93_IGN69.laz"
     assert parsed_coord_x == COORD_X
     assert parsed_coord_y == COORD_Y
 

test/test_las_merge.py

@@ -1,26 +1,15 @@
 import logging
 import os
+from test.utils import assert_header_info_are_similar
 
 import laspy
 import numpy as np
 
 from pdaltools.las_merge import las_merge
 
-test_path = os.path.dirname(os.path.abspath(__file__))
-tmp_path = os.path.join(test_path, "tmp")
-input_dir = os.path.join(test_path, "data")
-output_file = os.path.join(tmp_path, "merged.las")
-
-coord_x = 77055
-coord_y = 627760
-input_file = os.path.join(input_dir, f"test_data_{coord_x}_{coord_y}_LA93_IGN69_ground.las")
-tile_width = 50
-tile_coord_scale = 10
-
-input_nb_points = 22343
-expected_output_nb_points = 154134
-expected_out_mins = [770500.0, 6277500.0]
-expected_out_maxs = [770650.0, 6277600.0]
+TEST_PATH = os.path.dirname(os.path.abspath(__file__))
+TMP_PATH = os.path.join(TEST_PATH, "tmp")
+INPUT_DIR = os.path.join(TEST_PATH, "data")
 
 
 # def setup_module(module):
@@ -52,7 +41,20 @@ def get_2d_bounding_box(path):
 
 # Tests
 def test_las_merge():
-    las_merge(input_dir, input_file, output_file, tile_width=tile_width, tile_coord_scale=tile_coord_scale)
+    coord_x = 77055
+    coord_y = 627760
+    input_file = os.path.join(INPUT_DIR, f"test_data_{coord_x}_{coord_y}_LA93_IGN69.laz")
+    output_file = os.path.join(TMP_PATH, "merged.las")
+    tile_width = 50
+    tile_coord_scale = 10
+    input_mins = [coord_x * tile_coord_scale, coord_y * tile_coord_scale - tile_width]
+    input_maxs = [coord_x * tile_coord_scale + tile_width, coord_y * tile_coord_scale]
+    expected_out_mins = [input_mins[0] - tile_width, input_mins[1] - tile_width]
+    expected_out_maxs = [input_maxs[0] + tile_width, input_maxs[1]]  # There is no tile above the tile to merge
+
+    expected_output_nb_points = 405937
+
+    las_merge(INPUT_DIR, input_file, output_file, tile_width=tile_width, tile_coord_scale=tile_coord_scale)
 
     # check file exists
     assert os.path.isfile(output_file)
@@ -64,10 +66,13 @@ def test_las_merge():
     # check number of points
     assert get_nb_points(output_file) == expected_output_nb_points
 
-    # Check contre valeur attendue
+    # check bounds are the expected ones
     assert np.all(out_mins == expected_out_mins)
     assert np.all(out_maxs == expected_out_maxs)
 
+    # check that the las format is preserved
+    assert_header_info_are_similar(output_file, input_file)
+
 
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)

test/test_standardize_format.py

@@ -9,31 +9,26 @@
 
 from pdaltools.standardize_format import exec_las2las, rewrite_with_pdal, standardize
 
-# Note: tile 77050_627760 is cropped to simulate missing data in neighbors during merge
-test_path = os.path.dirname(os.path.abspath(__file__))
-tmp_path = os.path.join(test_path, "tmp")
-input_dir = os.path.join(test_path, "data")
-
-coord_x = 77055
-coord_y = 627760
-input_file = os.path.join(input_dir, f"test_data_{coord_x}_{coord_y}_LA93_IGN69_ground.las")
-output_file = os.path.join(tmp_path, "formatted.laz")
-multiple_params = [
+TEST_PATH = os.path.dirname(os.path.abspath(__file__))
+TMP_PATH = os.path.join(TEST_PATH, "tmp")
+INPUT_DIR = os.path.join(TEST_PATH, "data")
+
+MUTLIPLE_PARAMS = [
     {"dataformat_id": 6, "a_srs": "EPSG:2154"},
     {"dataformat_id": 8, "a_srs": "EPSG:4326"},
 ]
 
 
 def setup_module(module):
     try:
-        shutil.rmtree(tmp_path)
+        shutil.rmtree(TMP_PATH)
 
     except FileNotFoundError:
         pass
-    os.mkdir(tmp_path)
+    os.mkdir(TMP_PATH)
 
 
-def _test_standardize_format_one_params_set(params):
+def _test_standardize_format_one_params_set(input_file, output_file, params):
     rewrite_with_pdal(input_file, output_file, params)
     # check file exists
     assert os.path.isfile(output_file)
@@ -58,8 +53,10 @@ def _test_standardize_format_one_params_set(params):
 
 
 def test_standardize_format():
-    for params in multiple_params:
-        _test_standardize_format_one_params_set(params)
+    input_file = os.path.join(INPUT_DIR, "test_data_77055_627760_LA93_IGN69.laz")
+    output_file = os.path.join(TMP_PATH, "formatted.laz")
+    for params in MUTLIPLE_PARAMS:
+        _test_standardize_format_one_params_set(input_file, output_file, params)
 
 
 def exec_lasinfo(input_file: str):
@@ -93,20 +90,20 @@ def test_standardize_does_NOT_produce_any_warning_with_Lasinfo():
     #     "/var/data/store-lidarhd/developpement/standaLAS/demo_standardization/Semis_2022_0584_6880_LA93_IGN69.laz"
     # )
 
-    input_file = os.path.join(test_path, "data/classified_laz/test_data_77050_627755_LA93_IGN69.laz")
-    output_file = os.path.join(tmp_path, "test_standardize_produce_no_warning_with_lasinfo.las")
+    input_file = os.path.join(TEST_PATH, "data/classified_laz/test_data_77050_627755_LA93_IGN69.laz")
+    output_file = os.path.join(TMP_PATH, "test_standardize_produce_no_warning_with_lasinfo.las")
 
     # if you want to see input_file warnings
     # assert_lasinfo_no_warning(input_file)
 
-    standardize(input_file, output_file, multiple_params[0])
+    standardize(input_file, output_file, MUTLIPLE_PARAMS[0])
     assert_lasinfo_no_warning(output_file)
 
 
 def test_standardize_malformed_laz():
-    input_file = os.path.join(test_path, "data/test_pdalfail_0643_6319_LA93_IGN69.laz")
-    output_file = os.path.join(tmp_path, "standardize_pdalfail_0643_6319_LA93_IGN69.laz")
-    standardize(input_file, output_file, multiple_params[0])
+    input_file = os.path.join(TEST_PATH, "data/test_pdalfail_0643_6319_LA93_IGN69.laz")
+    output_file = os.path.join(TMP_PATH, "standardize_pdalfail_0643_6319_LA93_IGN69.laz")
+    standardize(input_file, output_file, MUTLIPLE_PARAMS[0])
     assert os.path.isfile(output_file)
 
 

test/utils.py

@@ -34,6 +34,14 @@ def get_pdal_infos_summary(f: str):
     return json_info
 
 
+def assert_header_info_are_similar(file_to_compare, expected):
+    expected_metadata = get_pdal_infos_summary(expected)["summary"]["metadata"]
+    file_to_compare_metadata = get_pdal_infos_summary(file_to_compare)["summary"]["metadata"]
+    assert file_to_compare_metadata["minor_version"] == expected_metadata["minor_version"]
+    assert file_to_compare_metadata["global_encoding"] == expected_metadata["global_encoding"]
+    assert file_to_compare_metadata["dataformat_id"] == expected_metadata["dataformat_id"]
+
+
 EXPECTED_DIMS_BY_DATAFORMAT = {
     6: set(
         [