Back to report index

Step 3 segmentationtrajectory 5d20310: AI3D-382 Guard the run3 NPZ schema against drift from iolabs.common.segment_points_io

Miroslav Simko <ms@iolabs.ch> 2026-09-01T15:11:49+02:00

Commit #101 · 5 snippets

 CLAUDE.md                       |  9 +++++++-
 tests/test_number_of_returns.py | 51 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+), 1 deletion(-)
Importance #1: tests/test_number_of_returns.py @@ -1,6 +1,7 @@
1import dataclasses1import dataclasses
2import logging2import logging
3import types
3from pathlib import Path4from pathlib import Path
45
5import laspy6import laspy
6import numpy as np7import numpy as np
Importance #2: tests/test_number_of_returns.py @@ -160,4 +161,54 @@
160 np.array([1, 2, 3], dtype=sm.NUMBER_OF_RETURNS_DTYPE),161 np.array([1, 2, 3], dtype=sm.NUMBER_OF_RETURNS_DTYPE),
161 )162 )
162 else:163 else:
163 assert not hasattr(data, sm.NUMBER_OF_RETURNS_KEY)164 assert not hasattr(data, sm.NUMBER_OF_RETURNS_KEY)
165
166
167def test_record_field_or_zeros_is_generic_over_key_and_dtype() -> None:
168 """The fallback helper must be parameterised, not hard-wired to one field."""
169 record = _Chunk(np.array([1, 2, 3], dtype=np.uint8))
170
171 present = sm._record_field_or_zeros(
172 record,
173 key="intensity",
174 dtype=np.uint16,
175 point_count=3,
176 source="test",
177 )
178 np.testing.assert_array_equal(present, np.array([10, 20, 30], dtype=np.uint16))
179
180 missing = sm._record_field_or_zeros(
181 record,
182 key="classification",
183 dtype=np.int32,
184 point_count=3,
185 source="test",
186 )
187 assert missing.dtype == np.int32
188 np.testing.assert_array_equal(missing, np.zeros(3, dtype=np.int32))
189
190
191def test_default_field_dtypes_is_immutable() -> None:
192 """The public schema mapping must not be mutable process-wide."""
193 assert isinstance(sm.DEFAULT_FIELD_DTYPES, types.MappingProxyType)
194 with pytest.raises(TypeError):
195 sm.DEFAULT_FIELD_DTYPES["points"] = np.dtype(np.float32) # type: ignore[index]
196
197
198def test_npz_schema_matches_common_segment_points_io() -> None:
199 """The duplicated run3 NPZ schema must not drift from the SSOT in common.
200
201 Skips against an `iolabs-common` that predates `segment_points_io`, and
202 activates by itself once the floor is raised to a release that has it.
203 """
204 segment_points_io = pytest.importorskip(
205 "iolabs.common.segment_points_io",
206 reason="installed iolabs-common predates the segment_points_io SSOT",
207 )
208
209 assert set(sm.SEGMENT_NPZ_FIELD_NAMES) == set(segment_points_io.POINT_RECORD_KEYS)
210 assert sm.NUMBER_OF_RETURNS_KEY == segment_points_io.NUMBER_OF_RETURNS_KEY
211 assert (
212 np.dtype(sm.NUMBER_OF_RETURNS_DTYPE)
213 == np.dtype(segment_points_io.NUMBER_OF_RETURNS_DTYPE)
214 )
Importance #3: CLAUDE.md @@ -34,9 +34,16 @@
34 6. If `save_points_between_planes`, call `divide_las_file_by_planes` for each LAS via a `ThreadPoolExecutor(max_workers=max_parallel_las_files)` (capped at `os.cpu_count()`). Per-segment `.npz` files land under `<segments_base_dir>/segment_NNN/` (3-digit zero-padded index, matching the `point{i:03d}`/`normal{i:03d}` keys in `run3_planes.npz`) and `save_version_json` drops a `run3_versions.json` next to them.34 6. If `save_points_between_planes`, call `divide_las_file_by_planes` for each LAS via a `ThreadPoolExecutor(max_workers=max_parallel_las_files)` (capped at `os.cpu_count()`). Per-segment `.npz` files land under `<segments_base_dir>/segment_NNN/` (3-digit zero-padded index, matching the `point{i:03d}`/`normal{i:03d}` keys in `run3_planes.npz`) and `save_version_json` drops a `run3_versions.json` next to them.
3535
36- **`divide_las_file_by_planes`** streams the LAS via `laspy.open(...).chunk_iterator(las_points_per_chunk)`, never loading the whole file. Per chunk: optional `|scan_angle| < angle_limit` filter, then a sign-count mask against all selected planes assigns each point to a segment bucket. Scan-angle field is auto-detected (`scan_angle_rank` legacy vs `scan_angle` newer); RGB and intensity are required and the code raises if missing. Each per-segment `.npz` also carries `number_of_returns` (uint8, AI3D-382); LAS files without that field degrade to zeros, which the run3 NPZ contract reads as "unknown" (0 is not a legal LAS return count). Output points are written **geoshift-relative**.36- **`divide_las_file_by_planes`** streams the LAS via `laspy.open(...).chunk_iterator(las_points_per_chunk)`, never loading the whole file. Per chunk: optional `|scan_angle| < angle_limit` filter, then a sign-count mask against all selected planes assigns each point to a segment bucket. Scan-angle field is auto-detected (`scan_angle_rank` legacy vs `scan_angle` newer); RGB and intensity are required and the code raises if missing. Each per-segment `.npz` also carries `number_of_returns` (uint8, AI3D-382); LAS files without that field degrade to zeros, which the run3 NPZ contract reads as "unknown" (0 is not a legal LAS return count). Output points are written **geoshift-relative**.
3737
38 The per-point ancillary arrays travel as a single `dict[str, np.ndarray]` keyed by the module-level `ANCILLARY_FIELD_NAMES` tuple (`points` stays separate as the `(N, 3)` geometry array; `SEGMENT_NPZ_FIELD_NAMES = ("points", *ANCILLARY_FIELD_NAMES)` fixes the NPZ member order). Extraction, the angle-limit mask, dtype registration, `_SegmentSplitWriter.write`, the memmap allocation and the archive member list all iterate that tuple, so **adding an NPZ field = one entry in `ANCILLARY_FIELD_NAMES` + one extraction line in `SegmentMapper._chunk_field_arrays`** (plus a default in `DEFAULT_FIELD_DTYPES` if the field can be absent from the source LAS).38 The per-point ancillary arrays travel as a single `dict[str, np.ndarray]` keyed by the module-level `ANCILLARY_FIELD_NAMES` tuple (`points` stays separate as the `(N, 3)` geometry array; `SEGMENT_NPZ_FIELD_NAMES = ("points", *ANCILLARY_FIELD_NAMES)` fixes the NPZ member order). Extraction, the angle-limit mask, dtype registration, `_SegmentSplitWriter.write`, the memmap allocation and the archive member list all iterate that tuple. Adding an NPZ field therefore costs, in full:
39
40 1. one entry in `ANCILLARY_FIELD_NAMES`;
41 2. one extraction line in `SegmentMapper._chunk_field_arrays`;
42 3. an entry in `DEFAULT_FIELD_DTYPES` **if the LAS may lack the field** — the mapping is a `types.MappingProxyType`, so extend the literal rather than mutating it at runtime. Optional fields are read via `_record_field_or_zeros(chunk, key=..., dtype=...)`, which is generic: a new fallback field needs no new helper;
43 4. an entry in `required_chunk_fields` inside `divide_las_file_by_planes` **if the field is mandatory** — that tuple is what turns a missing field into an upfront `ValueError` instead of a later `AttributeError`.
44
45 `SEGMENT_NPZ_FIELD_NAMES` / `NUMBER_OF_RETURNS_KEY` / `NUMBER_OF_RETURNS_DTYPE` duplicate the schema owned by `iolabs.common.segment_points_io` (mirrored, not imported, so this module stays importable against older `iolabs-common`). `tests/test_number_of_returns.py::test_npz_schema_matches_common_segment_points_io` guards the duplication; it `importorskip`s the consumer module, so it is inert until the `iolabs-common` floor is raised to a release that ships it, and then activates on its own. For the same cross-version reason, `SegmentMapper.load_color_intensity_data` builds its `ColorIntensityData` kwargs filtered by `dataclasses.fields(...)`: passing a kwarg the installed dataclass does not declare is a `TypeError`, so a field the installed `iolabs-common` predates is dropped rather than forced.
3946
40- **`_config.py`** — strict whitelist validation. `ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS` and `ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS` are enforced; unknown keys raise `SegmentMapperConfigError`. `normalize_segment_mapper_config` fills defaults; `build_segment_mapper_config(overrides=..., config_path=...)` does deep-merge over `segment_mapper.default.json`. **When adding a new config key you must update both the whitelist set and `normalize_segment_mapper_config`'s `setdefault` block, and add a default in `segment_mapper.default.json`.**47- **`_config.py`** — strict whitelist validation. `ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS` and `ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS` are enforced; unknown keys raise `SegmentMapperConfigError`. `normalize_segment_mapper_config` fills defaults; `build_segment_mapper_config(overrides=..., config_path=...)` does deep-merge over `segment_mapper.default.json`. **When adding a new config key you must update both the whitelist set and `normalize_segment_mapper_config`'s `setdefault` block, and add a default in `segment_mapper.default.json`.**
4148
42- **`segment_mapper.default.json`** — bundled defaults. It is force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`; if you rename or move it, update that mapping or the installed package will be missing the file at runtime.49- **`segment_mapper.default.json`** — bundled defaults. It is force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`; if you rename or move it, update that mapping or the installed package will be missing the file at runtime.
Importance #4: tests/test_number_of_returns.py @@ -1,6 +1,7 @@
1import dataclasses1import dataclasses
2import logging2import logging
3import types
3from pathlib import Path4from pathlib import Path
45
5import laspy6import laspy
6import numpy as np7import numpy as np
Importance #5: tests/test_number_of_returns.py @@ -160,4 +161,54 @@
160 np.array([1, 2, 3], dtype=sm.NUMBER_OF_RETURNS_DTYPE),161 np.array([1, 2, 3], dtype=sm.NUMBER_OF_RETURNS_DTYPE),
161 )162 )
162 else:163 else:
163 assert not hasattr(data, sm.NUMBER_OF_RETURNS_KEY)164 assert not hasattr(data, sm.NUMBER_OF_RETURNS_KEY)
165
166
167def test_record_field_or_zeros_is_generic_over_key_and_dtype() -> None:
168 """The fallback helper must be parameterised, not hard-wired to one field."""
169 record = _Chunk(np.array([1, 2, 3], dtype=np.uint8))
170
171 present = sm._record_field_or_zeros(
172 record,
173 key="intensity",
174 dtype=np.uint16,
175 point_count=3,
176 source="test",
177 )
178 np.testing.assert_array_equal(present, np.array([10, 20, 30], dtype=np.uint16))
179
180 missing = sm._record_field_or_zeros(
181 record,
182 key="classification",
183 dtype=np.int32,
184 point_count=3,
185 source="test",
186 )
187 assert missing.dtype == np.int32
188 np.testing.assert_array_equal(missing, np.zeros(3, dtype=np.int32))
189
190
191def test_default_field_dtypes_is_immutable() -> None:
192 """The public schema mapping must not be mutable process-wide."""
193 assert isinstance(sm.DEFAULT_FIELD_DTYPES, types.MappingProxyType)
194 with pytest.raises(TypeError):
195 sm.DEFAULT_FIELD_DTYPES["points"] = np.dtype(np.float32) # type: ignore[index]
196
197
198def test_npz_schema_matches_common_segment_points_io() -> None:
199 """The duplicated run3 NPZ schema must not drift from the SSOT in common.
200
201 Skips against an `iolabs-common` that predates `segment_points_io`, and
202 activates by itself once the floor is raised to a release that has it.
203 """
204 segment_points_io = pytest.importorskip(
205 "iolabs.common.segment_points_io",
206 reason="installed iolabs-common predates the segment_points_io SSOT",
207 )
208
209 assert set(sm.SEGMENT_NPZ_FIELD_NAMES) == set(segment_points_io.POINT_RECORD_KEYS)
210 assert sm.NUMBER_OF_RETURNS_KEY == segment_points_io.NUMBER_OF_RETURNS_KEY
211 assert (
212 np.dtype(sm.NUMBER_OF_RETURNS_DTYPE)
213 == np.dtype(segment_points_io.NUMBER_OF_RETURNS_DTYPE)
214 )