Back to report index

Step 3 segmentationtrajectory f081ed9: AI3D-379 Pydantic config models via iolabs-common ConfigModel

Miroslav Simko <ms@iolabs.ch> 2026-09-02T08:34:28+02:00

Commit #17 · 25 snippets

 AGENTS.md                                          |   2 +-
 CLAUDE.md                                          |   2 +-
 README.md                                          |   2 +-
 knowledge.md                                       |   8 +-
 pyproject.toml                                     |   3 +-
 .../_config.py                                     | 321 ++++++---------------
 tests/conftest.py                                  |   6 +-
 tests/test_config.py                               |  59 +++-
 8 files changed, 159 insertions(+), 244 deletions(-)
Importance #1: src/iolabs_point_cloud_segmentation_trajectory/_config.py @@ -1,247 +1,118 @@
1"""Segment mapper config: pydantic model tree over the packaged default JSON."""
2
1from __future__ import annotations3from __future__ import annotations
24
3import copy
4import json
5from importlib import resources as importlib_resources
6from pathlib import Path5from pathlib import Path
7from typing import Any6from typing import Any
87
9ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS = frozenset(8import pydantic
10 {9from iolabs.common import config_loader
11 "n_segments",10
12 "segment_length_m",11_PACKAGE = "iolabs_point_cloud_segmentation_trajectory"
13 "max_distance_to_plane",12_DEFAULT_FILENAME = "segment_mapper.default.json"
14 "save_planes",13_CONTEXT = "segment mapper config"
15 "device",14
16 "visualize",15
17 "visualize_las_segment_coloring",16class SegmentMapperConfigError(config_loader.ConfigError):
18 "save_points_between_planes",17 """Raised when segment mapper config contains unsupported keys or values."""
19 "las_points_per_chunk",18
20 "max_parallel_las_files",19
21 "angle_limit",20class SegmentMapperFileNamingConfig(config_loader.ConfigModel):
22 "n_extra_planes",21 """Output filenames under the ``file_naming`` section."""
23 "segments_base_dir_name",22
24 "spline_pcd_extension",23 planes_filename: str = "run3_planes.npz"
25 "reuse_existing_planes",24 segment_points_suffix: str = "_run3_points"
26 "reuse_existing_geoshift",25 geoshift_filename: str = "run3_geoshift.json"
27 "enable_longitudinal_limit_planes",26 longitudinal_limit_planes_filename: str = "run3_longitudinal_limit_planes.npz"
28 "longitudinal_limit_distance_m",27
29 "save_longitudinal_limit_planes",28
30 "write_only_segments",29class SegmentMapperVisualizationColorsConfig(config_loader.ConfigModel):
31 "visualization_colors",30 """RGB visualization colors; each value is a list of three numbers."""
32 "file_naming",31
33 }32 angle_limit_rejected: list[float] = pydantic.Field(
34)33 default=[0.45, 0.45, 0.45], min_length=3, max_length=3
35
36ALLOWED_SEGMENT_MAPPER_VISUALIZATION_COLOR_KEYS = frozenset(
37 {
38 "angle_limit_rejected",
39 "segmentation_plane",
40 "longitudinal_left_plane",
41 "longitudinal_right_plane",
42 }
43)
44
45ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS = frozenset(
46 {
47 "planes_filename",
48 "segment_points_suffix",
49 "geoshift_filename",
50 "longitudinal_limit_planes_filename",
51 }
52)
53
54class SegmentMapperConfigError(ValueError):
55 """Raised when segment mapper config contains unsupported keys."""
56
57
58def _default_config_path() -> Path:
59 if __package__ in {None, ""}:
60 return Path(__file__).resolve().with_name("segment_mapper.default.json")
61 return Path(str(importlib_resources.files(__package__).joinpath("segment_mapper.default.json")))
62
63
64def _read_config_json(config_path: str | Path) -> dict[str, Any]:
65 with Path(config_path).open("r", encoding="utf-8") as handle:
66 return json.load(handle)
67
68
69def _load_default_config() -> dict[str, Any]:
70 return _read_config_json(_default_config_path())
71
72
73def _deep_merge_dicts(
74 base: dict[str, Any],
75 overrides: dict[str, Any],
76) -> dict[str, Any]:
77 for key, value in overrides.items():
78 if isinstance(value, dict) and isinstance(base.get(key), dict):
79 base[key] = _deep_merge_dicts(dict(base[key]), value)
80 else:
81 base[key] = value
82 return base
83
84
85def _validate_segment_mapper_config_keys(config: dict[str, Any]) -> None:
86 unknown_keys = sorted(set(config) - ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS)
87 if unknown_keys:
88 allowed_keys = ", ".join(sorted(ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS))
89 raise SegmentMapperConfigError(
90 "Unknown segment mapper config key(s): "
91 f"{', '.join(unknown_keys)}. Allowed keys: {allowed_keys}"
92 )
93
94
95def _normalize_file_naming(
96 raw_file_naming: Any,
97 *,
98 default_file_naming: dict[str, Any],
99) -> dict[str, Any]:
100 if raw_file_naming is None:
101 file_naming = dict(default_file_naming)
102 elif isinstance(raw_file_naming, dict):
103 file_naming = _deep_merge_dicts(dict(default_file_naming), raw_file_naming)
104 else:
105 raise SegmentMapperConfigError(
106 "segment mapper config field 'file_naming' must be a mapping"
107 )
108
109 unknown_keys = sorted(set(file_naming) - ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS)
110 if unknown_keys:
111 allowed_keys = ", ".join(sorted(ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS))
112 raise SegmentMapperConfigError(
113 "Unknown segment mapper file_naming key(s): "
114 f"{', '.join(unknown_keys)}. Allowed keys: {allowed_keys}"
115 )
116
117 missing_keys = sorted(ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS - set(file_naming))
118 if missing_keys:
119 raise SegmentMapperConfigError(
120 "Bundled segment mapper default config is missing file_naming key(s): "
121 f"{', '.join(missing_keys)}"
122 )
123 return file_naming
124
125
126def _normalize_rgb_color(raw_color: Any, *, key: str) -> list[float]:
127 if (
128 not isinstance(raw_color, list | tuple)
129 or len(raw_color) != 3
130 or any(not isinstance(value, int | float) for value in raw_color)
131 ):
132 raise SegmentMapperConfigError(
133 f"segment mapper visualization color {key!r} must be an RGB list of 3 numbers"
134 )
135 return [float(value) for value in raw_color]
136
137
138def _normalize_visualization_colors(
139 raw_visualization_colors: Any,
140 *,
141 default_visualization_colors: dict[str, Any],
142) -> dict[str, Any]:
143 if raw_visualization_colors is None:
144 visualization_colors = dict(default_visualization_colors)
145 elif isinstance(raw_visualization_colors, dict):
146 visualization_colors = _deep_merge_dicts(
147 dict(default_visualization_colors),
148 raw_visualization_colors,
149 )
150 else:
151 raise SegmentMapperConfigError(
152 "segment mapper config field 'visualization_colors' must be a mapping"
153 )
154
155 unknown_keys = sorted(
156 set(visualization_colors) - ALLOWED_SEGMENT_MAPPER_VISUALIZATION_COLOR_KEYS
157 )34 )
158 if unknown_keys:35 segmentation_plane: list[float] = pydantic.Field(
159 allowed_keys = ", ".join(sorted(ALLOWED_SEGMENT_MAPPER_VISUALIZATION_COLOR_KEYS))36 default=[0.1, 0.35, 1.0], min_length=3, max_length=3
160 raise SegmentMapperConfigError(
161 "Unknown segment mapper visualization_colors key(s): "
162 f"{', '.join(unknown_keys)}. Allowed keys: {allowed_keys}"
163 )
164
165 missing_keys = sorted(
166 ALLOWED_SEGMENT_MAPPER_VISUALIZATION_COLOR_KEYS - set(visualization_colors)
167 )37 )
168 if missing_keys:38 longitudinal_left_plane: list[float] = pydantic.Field(
169 raise SegmentMapperConfigError(39 default=[1.0, 0.25, 0.0], min_length=3, max_length=3
170 "Bundled segment mapper default config is missing visualization_colors key(s): "
171 f"{', '.join(missing_keys)}"
172 )
173 return {
174 key: _normalize_rgb_color(value, key=key)
175 for key, value in visualization_colors.items()
176 }
177
178
179def _normalize_segment_mapper_config_values(
180 raw_config: dict[str, Any],
181 *,
182 default_config: dict[str, Any],
183) -> dict[str, Any]:
184 config = dict(raw_config)
185 _validate_segment_mapper_config_keys(config)
186
187 missing_keys = sorted(ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS - set(config))
188 if missing_keys:
189 raise SegmentMapperConfigError(
190 "Bundled segment mapper default config is missing key(s): "
191 f"{', '.join(missing_keys)}"
192 )
193
194 longitudinal_limit_distance_m = float(config["longitudinal_limit_distance_m"])
195 if longitudinal_limit_distance_m <= 0.0:
196 raise SegmentMapperConfigError(
197 "segment mapper config field 'longitudinal_limit_distance_m' must be > 0"
198 )
199 config["longitudinal_limit_distance_m"] = longitudinal_limit_distance_m
200 config["file_naming"] = _normalize_file_naming(
201 config.get("file_naming"),
202 default_file_naming=dict(default_config.get("file_naming", {})),
203 )40 )
204 config["visualization_colors"] = _normalize_visualization_colors(41 longitudinal_right_plane: list[float] = pydantic.Field(
205 config.get("visualization_colors"),42 default=[1.0, 0.55, 0.0], min_length=3, max_length=3
206 default_visualization_colors=dict(default_config.get("visualization_colors", {})),
207 )43 )
208 return config
20944
21045
211def normalize_segment_mapper_config(raw_config: dict[str, Any]) -> dict[str, Any]:46class SegmentMapperConfig(config_loader.ConfigModel):
212 default_config = _load_default_config()47 """Segment mapper config; field names and nesting match the packaged JSON."""
213 config = _deep_merge_dicts(copy.deepcopy(default_config), dict(raw_config))48
214 return _normalize_segment_mapper_config_values(49 n_segments: int = 100
215 config,50 segment_length_m: float | None = 50.0
216 default_config=default_config,51 max_distance_to_plane: float = 200.0
52 save_planes: bool = True
53 device: str = "CPU:0"
54 visualize: bool = False
55 visualize_las_segment_coloring: bool = False
56 save_points_between_planes: bool = True
57 las_points_per_chunk: int = 500000
58 max_parallel_las_files: int = 1
59 angle_limit: int | None = 80
60 n_extra_planes: int = 4
61 segments_base_dir_name: str = "lane_points"
62 spline_pcd_extension: str = "_run1_spline_points"
63 reuse_existing_planes: bool = False
64 reuse_existing_geoshift: bool = False
65 enable_longitudinal_limit_planes: bool = True
66 longitudinal_limit_distance_m: float = pydantic.Field(default=100.0, gt=0)
67 write_only_segments: list[int] = []
68 save_longitudinal_limit_planes: bool = True
69 visualization_colors: SegmentMapperVisualizationColorsConfig = (
70 SegmentMapperVisualizationColorsConfig()
217 )71 )
72 file_naming: SegmentMapperFileNamingConfig = SegmentMapperFileNamingConfig()
21873
74 @pydantic.field_validator("file_naming", "visualization_colors", mode="before")
75 @classmethod
76 def _none_section_uses_defaults(cls, value: Any) -> Any:
77 """Treat a JSON ``null`` section as 'use nested defaults'."""
78 if value is None:
79 return {}
80 return value
21981
220def load_segment_mapper_config(config_path: str | Path | None = None) -> dict[str, Any]:82
221 default_config = _load_default_config()83def _load_segment_mapper_model(
222 if config_path is None:84 *,
223 raw_config = copy.deepcopy(default_config)85 overrides: dict[str, Any] | None = None,
224 else:86 config_path: str | Path | None = None,
225 raw_config = _deep_merge_dicts(87) -> SegmentMapperConfig:
226 copy.deepcopy(default_config),88 return config_loader.load_config(
227 _read_config_json(config_path),89 SegmentMapperConfig,
228 )90 package=_PACKAGE,
229 return _normalize_segment_mapper_config_values(91 filename=_DEFAULT_FILENAME,
230 raw_config,92 overrides=overrides,
231 default_config=default_config,93 config_path=config_path,
94 context=_CONTEXT,
95 error_cls=SegmentMapperConfigError,
232 )96 )
23397
23498
99def normalize_segment_mapper_config(raw_config: dict[str, Any]) -> dict[str, Any]:
100 """Merge *raw_config* onto the packaged defaults and return the validated dict."""
101 return _load_segment_mapper_model(overrides=dict(raw_config)).model_dump()
102
103
104def load_segment_mapper_config(config_path: str | Path | None = None) -> dict[str, Any]:
105 """Return the validated config from *config_path*, or the packaged defaults."""
106 return _load_segment_mapper_model(config_path=config_path).model_dump()
107
108
235def build_segment_mapper_config(109def build_segment_mapper_config(
236 *,110 *,
237 overrides: dict[str, Any] | None = None,111 overrides: dict[str, Any] | None = None,
238 config_path: str | Path | None = None,112 config_path: str | Path | None = None,
239) -> dict[str, Any]:113) -> dict[str, Any]:
240 default_config = _load_default_config()114 """Return the validated config with *overrides* merged onto the defaults."""
241 config = load_segment_mapper_config(config_path)115 return _load_segment_mapper_model(
242 if overrides:116 overrides=overrides,
243 config = _deep_merge_dicts(config, dict(overrides))117 config_path=config_path,
244 return _normalize_segment_mapper_config_values(118 ).model_dump()
245 config,
246 default_config=default_config,
247 )
Importance #2: tests/conftest.py @@ -10,11 +10,11 @@
10with a real `SourceFileLoader` but *without* executing `__init__.py`, so:10with a real `SourceFileLoader` but *without* executing `__init__.py`, so:
1111
12- submodule imports like `from iolabs_...._config import ...` resolve via12- submodule imports like `from iolabs_...._config import ...` resolve via
13 the spec's `submodule_search_locations`, and13 the spec's `submodule_search_locations`, and
14- `importlib.resources.files(<pkg>)` (used by `_config._default_config_path`)14- `importlib.resources.files(<pkg>)` (used by `config_loader.load_config` via
15 has a loader with `get_resource_reader` so bundled `segment_mapper.default.json`15 the packaged default JSON) has a loader with `get_resource_reader` so bundled
16 is findable,16 `segment_mapper.default.json` is findable,
1717
18while the heavy runtime deps remain untouched.18while the heavy runtime deps remain untouched.
19"""19"""
2020
Importance #3: tests/test_config.py @@ -6,12 +6,14 @@
6from pathlib import Path6from pathlib import Path
77
8import pytest8import pytest
99
10from iolabs_point_cloud_segmentation_trajectory import _config
10from iolabs_point_cloud_segmentation_trajectory._config import (11from iolabs_point_cloud_segmentation_trajectory._config import (
11 ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS,12 SegmentMapperConfig,
12 ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS,
13 SegmentMapperConfigError,13 SegmentMapperConfigError,
14 SegmentMapperFileNamingConfig,
15 SegmentMapperVisualizationColorsConfig,
14 build_segment_mapper_config,16 build_segment_mapper_config,
15 load_segment_mapper_config,17 load_segment_mapper_config,
16 normalize_segment_mapper_config,18 normalize_segment_mapper_config,
17)19)
Importance #4: tests/test_config.py @@ -23,8 +25,9 @@
23 "max_distance_to_plane": 200.0,25 "max_distance_to_plane": 200.0,
24 "save_planes": True,26 "save_planes": True,
25 "device": "CPU:0",27 "device": "CPU:0",
26 "visualize": False,28 "visualize": False,
29 "visualize_las_segment_coloring": False,
27 "save_points_between_planes": True,30 "save_points_between_planes": True,
28 "las_points_per_chunk": 500_000,31 "las_points_per_chunk": 500_000,
29 "max_parallel_las_files": 1,32 "max_parallel_las_files": 1,
30 "angle_limit": 80,33 "angle_limit": 80,
Importance #5: tests/test_config.py @@ -68,13 +71,28 @@
68 config = normalize_segment_mapper_config({})71 config = normalize_segment_mapper_config({})
69 assert config["file_naming"] == EXPECTED_FILE_NAMING_DEFAULTS72 assert config["file_naming"] == EXPECTED_FILE_NAMING_DEFAULTS
7073
7174
72def test_defaults_cover_every_allowed_key() -> None:75def test_defaults_cover_every_model_field() -> None:
73 """Guard against adding a new whitelisted key without a matching default."""76 """Guard against adding a model field without a matching packaged default."""
74 config = normalize_segment_mapper_config({})77 config = normalize_segment_mapper_config({})
75 assert set(config.keys()) == ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS78 assert set(config.keys()) == set(SegmentMapperConfig.model_fields)
76 assert set(config["file_naming"].keys()) == ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS79 assert set(config["file_naming"].keys()) == set(
80 SegmentMapperFileNamingConfig.model_fields
81 )
82 assert set(config["visualization_colors"].keys()) == set(
83 SegmentMapperVisualizationColorsConfig.model_fields
84 )
85
86
87def test_packaged_json_matches_model_defaults() -> None:
88 """The packaged JSON must stay in sync with the model defaults, key and value."""
89 packaged = json.loads(
90 (
91 Path(_config.__file__).with_name("segment_mapper.default.json")
92 ).read_text(encoding="utf-8")
93 )
94 assert packaged == SegmentMapperConfig().model_dump()
7795
7896
79def test_normalize_is_idempotent() -> None:97def test_normalize_is_idempotent() -> None:
80 once = normalize_segment_mapper_config({})98 once = normalize_segment_mapper_config({})
Importance #6: tests/test_config.py @@ -217,8 +235,12 @@
217 assert config["file_naming"]["planes_filename"] == "x.npz"235 assert config["file_naming"]["planes_filename"] == "x.npz"
218 # file_naming still deep-merged with defaults236 # file_naming still deep-merged with defaults
219 assert config["file_naming"]["segment_points_suffix"] == "_run3_points"237 assert config["file_naming"]["segment_points_suffix"] == "_run3_points"
220 assert config["file_naming"]["geoshift_filename"] == "run3_geoshift.json"238 assert config["file_naming"]["geoshift_filename"] == "run3_geoshift.json"
239 # A partial file still yields every top-level default, and the whole config.
240 assert config["device"] == "CPU:0"
241 assert config["angle_limit"] == 80
242 assert set(config) == set(SegmentMapperConfig.model_fields)
221243
222244
223def test_load_from_custom_path_validates_keys(tmp_path: Path) -> None:245def test_load_from_custom_path_validates_keys(tmp_path: Path) -> None:
224 cfg_file = tmp_path / "c.json"246 cfg_file = tmp_path / "c.json"
Importance #7: tests/test_config.py @@ -246,8 +268,33 @@
246def test_build_without_overrides_equals_load_default() -> None:268def test_build_without_overrides_equals_load_default() -> None:
247 assert build_segment_mapper_config() == load_segment_mapper_config()269 assert build_segment_mapper_config() == load_segment_mapper_config()
248270
249271
272def test_build_overrides_win_over_config_path(tmp_path: Path) -> None:
273 """`overrides` is merged on top of `config_path`, not the other way round."""
274 cfg_file = tmp_path / "c.json"
275 cfg_file.write_text(
276 json.dumps(
277 {
278 "n_segments": 5,
279 "device": "CUDA:1",
280 "file_naming": {"planes_filename": "from_file.npz"},
281 }
282 )
283 )
284 config = build_segment_mapper_config(
285 overrides={"n_segments": 42, "file_naming": {"geoshift_filename": "from_ovr.json"}},
286 config_path=cfg_file,
287 )
288 assert config["n_segments"] == 42
289 # Keys only the file sets survive, at both levels.
290 assert config["device"] == "CUDA:1"
291 assert config["file_naming"]["planes_filename"] == "from_file.npz"
292 assert config["file_naming"]["geoshift_filename"] == "from_ovr.json"
293 # Keys neither sets still come from the packaged defaults.
294 assert config["file_naming"]["segment_points_suffix"] == "_run3_points"
295
296
250def test_build_rejects_unknown_override_keys() -> None:297def test_build_rejects_unknown_override_keys() -> None:
251 with pytest.raises(SegmentMapperConfigError):298 with pytest.raises(SegmentMapperConfigError):
252 build_segment_mapper_config(overrides={"not_allowed": True})299 build_segment_mapper_config(overrides={"not_allowed": True})
253300
Importance #8: pyproject.toml @@ -1,13 +1,14 @@
1[project]1[project]
2name = "iolabs-point-cloud-segmentation-trajectory"2name = "iolabs-point-cloud-segmentation-trajectory"
3version = "0.7.3"3version = "0.7.4"
4description = "Trajectory-based segment mapping for LIDAR highway scans"4description = "Trajectory-based segment mapping for LIDAR highway scans"
5requires-python = ">=3.11,<3.13"5requires-python = ">=3.11,<3.13"
6dependencies = [6dependencies = [
7 "numpy>=1.20.0",7 "numpy>=1.20.0",
8 "open3d>=0.19.0",8 "open3d>=0.19.0",
9 "laspy>=2.0.0",9 "laspy>=2.0.0",
10 "pydantic>=2.7",
10 "iolabs-logstash>=0.5.1",11 "iolabs-logstash>=0.5.1",
11 "iolabs-common>=0.8.0",12 "iolabs-common>=0.8.0",
12 "iolabs-geometry-geometry",13 "iolabs-geometry-geometry",
13 "iolabs-point-cloud-filtering-intensity>=0.5.0",14 "iolabs-point-cloud-filtering-intensity>=0.5.0",
Importance #9: AGENTS.md @@ -35,9 +35,9 @@
35 6. If `save_points_between_planes`, call `divide_las_file_by_planes` for each LAS — single-threaded on CUDA, otherwise a `ThreadPoolExecutor(max_workers=max_parallel_las_files)`. Per-segment `.npz` files land under `<segments_base_dir>/segment_NNN/` (3-digit zero-padded index, matching the `point{i:03d}`/`normal{i:03d}` keys in `run3_planes.npz`) and `save_version_json` drops a `run3_versions.json` next to them.35 6. If `save_points_between_planes`, call `divide_las_file_by_planes` for each LAS — single-threaded on CUDA, otherwise a `ThreadPoolExecutor(max_workers=max_parallel_las_files)`. Per-segment `.npz` files land under `<segments_base_dir>/segment_NNN/` (3-digit zero-padded index, matching the `point{i:03d}`/`normal{i:03d}` keys in `run3_planes.npz`) and `save_version_json` drops a `run3_versions.json` next to them.
3636
37- **`divide_las_file_by_planes`** streams the LAS via `laspy.open(...).chunk_iterator(las_points_per_chunk)`, never loading the whole file. Per chunk: optional `|scan_angle| < angle_limit` filter, then a sign-count mask against all selected planes assigns each point to a segment bucket. Scan-angle field is auto-detected (`scan_angle_rank` legacy vs `scan_angle` newer); RGB and intensity are required and the code raises if missing. Each per-segment `.npz` also carries `number_of_returns` (uint8, AI3D-382); LAS files without that field degrade to zeros, which the run3 NPZ contract reads as "unknown" (0 is not a legal LAS return count). Output points are written **geoshift-relative**.37- **`divide_las_file_by_planes`** streams the LAS via `laspy.open(...).chunk_iterator(las_points_per_chunk)`, never loading the whole file. Per chunk: optional `|scan_angle| < angle_limit` filter, then a sign-count mask against all selected planes assigns each point to a segment bucket. Scan-angle field is auto-detected (`scan_angle_rank` legacy vs `scan_angle` newer); RGB and intensity are required and the code raises if missing. Each per-segment `.npz` also carries `number_of_returns` (uint8, AI3D-382); LAS files without that field degrade to zeros, which the run3 NPZ contract reads as "unknown" (0 is not a legal LAS return count). Output points are written **geoshift-relative**.
3838
39- **`_config.py`** — strict whitelist validation. `ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS` and `ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS` are enforced; unknown keys raise `SegmentMapperConfigError`. `normalize_segment_mapper_config` fills defaults; `build_segment_mapper_config(overrides=..., config_path=...)` does deep-merge over `segment_mapper.default.json`. **When adding a new config key you must update both the whitelist set and `normalize_segment_mapper_config`'s `setdefault` block, and add a default in `segment_mapper.default.json`.**39- **`_config.py`** — pydantic `config_loader.ConfigModel` tree (`SegmentMapperConfig` plus nested section models) validated by `iolabs.common.config_loader.load_config`. Unknown keys raise `SegmentMapperConfigError` (a `config_loader.ConfigError`). `normalize_segment_mapper_config` / `load_segment_mapper_config` / `build_segment_mapper_config` return plain dicts (`model.model_dump()`); overrides deep-merge onto the packaged JSON. **When adding a new config key, add a field to the model and a matching default in `segment_mapper.default.json`. Nothing else.**
4040
41- **`segment_mapper.default.json`** — bundled defaults. It is force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`; if you rename or move it, update that mapping or the installed package will be missing the file at runtime.41- **`segment_mapper.default.json`** — bundled defaults. It is force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`; if you rename or move it, update that mapping or the installed package will be missing the file at runtime.
4242
43- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS-file work is wrapped in `las_file_scope(las_file)` so log records carry the LAS filename context — keep new per-file code paths inside that scope.43- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS-file work is wrapped in `las_file_scope(las_file)` so log records carry the LAS filename context — keep new per-file code paths inside that scope.
Importance #10: CLAUDE.md @@ -43,9 +43,9 @@
43 4. an entry in `required_chunk_fields` inside `divide_las_file_by_planes` **if the field is mandatory** — that tuple is what turns a missing field into an upfront `ValueError` instead of a later `AttributeError`.43 4. an entry in `required_chunk_fields` inside `divide_las_file_by_planes` **if the field is mandatory** — that tuple is what turns a missing field into an upfront `ValueError` instead of a later `AttributeError`.
4444
45 `SEGMENT_NPZ_FIELD_NAMES` / `NUMBER_OF_RETURNS_KEY` / `NUMBER_OF_RETURNS_DTYPE` duplicate the schema owned by `iolabs.common.segment_points_io` (mirrored, not imported, so this module stays importable against older `iolabs-common`). `tests/test_number_of_returns.py::test_npz_schema_matches_common_segment_points_io` guards the duplication; it `importorskip`s the consumer module, so it is inert until the `iolabs-common` floor is raised to a release that ships it, and then activates on its own. For the same cross-version reason, `SegmentMapper.load_color_intensity_data` builds its `ColorIntensityData` kwargs filtered by `dataclasses.fields(...)`: passing a kwarg the installed dataclass does not declare is a `TypeError`, so a field the installed `iolabs-common` predates is dropped rather than forced.45 `SEGMENT_NPZ_FIELD_NAMES` / `NUMBER_OF_RETURNS_KEY` / `NUMBER_OF_RETURNS_DTYPE` duplicate the schema owned by `iolabs.common.segment_points_io` (mirrored, not imported, so this module stays importable against older `iolabs-common`). `tests/test_number_of_returns.py::test_npz_schema_matches_common_segment_points_io` guards the duplication; it `importorskip`s the consumer module, so it is inert until the `iolabs-common` floor is raised to a release that ships it, and then activates on its own. For the same cross-version reason, `SegmentMapper.load_color_intensity_data` builds its `ColorIntensityData` kwargs filtered by `dataclasses.fields(...)`: passing a kwarg the installed dataclass does not declare is a `TypeError`, so a field the installed `iolabs-common` predates is dropped rather than forced.
4646
47- **`_config.py`** — strict whitelist validation. `ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS` and `ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS` are enforced; unknown keys raise `SegmentMapperConfigError`. `normalize_segment_mapper_config` fills defaults; `build_segment_mapper_config(overrides=..., config_path=...)` does deep-merge over `segment_mapper.default.json`. **When adding a new config key you must update both the whitelist set and `normalize_segment_mapper_config`'s `setdefault` block, and add a default in `segment_mapper.default.json`.**47- **`_config.py`** — pydantic `config_loader.ConfigModel` tree (`SegmentMapperConfig` plus nested section models) validated by `iolabs.common.config_loader.load_config`. Unknown keys raise `SegmentMapperConfigError` (a `config_loader.ConfigError`). `normalize_segment_mapper_config` / `load_segment_mapper_config` / `build_segment_mapper_config` return plain dicts (`model.model_dump()`); overrides deep-merge onto the packaged JSON. **When adding a new config key, add a field to the model and a matching default in `segment_mapper.default.json`. Nothing else.**
4848
49- **`segment_mapper.default.json`** — bundled defaults. It is force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`; if you rename or move it, update that mapping or the installed package will be missing the file at runtime.49- **`segment_mapper.default.json`** — bundled defaults. It is force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`; if you rename or move it, update that mapping or the installed package will be missing the file at runtime.
5050
51- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS-file work is wrapped in `las_file_scope(las_file)` so log records carry the LAS filename context — keep new per-file code paths inside that scope.51- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS-file work is wrapped in `las_file_scope(las_file)` so log records carry the LAS filename context — keep new per-file code paths inside that scope.
Importance #11: README.md @@ -16,9 +16,9 @@
1616
17## Requirements17## Requirements
1818
19- Python ≥3.11, <3.1319- Python ≥3.11, <3.13
20- numpy, open3d, laspy, iolabs-common, iolabs-geometry-geometry, iolabs-point-cloud-filtering-intensity, iolabs-point-cloud-las-tools20- numpy, open3d, laspy, pydantic, iolabs-common, iolabs-geometry-geometry, iolabs-point-cloud-filtering-intensity, iolabs-point-cloud-las-tools
2121
22## Usage22## Usage
2323
24Maps segments along trajectories for highway LIDAR scans, building on trajectory detection and intensity filtering.24Maps segments along trajectories for highway LIDAR scans, building on trajectory detection and intensity filtering.
Importance #12: knowledge.md @@ -36,9 +36,9 @@
36 7. If `save_points_between_planes`: call `divide_las_file_by_planes` per LAS. **Single-threaded on CUDA**; otherwise `max_workers = max(1, min(max_parallel_las_files, os.cpu_count()))`, and a serial path is taken when `max_workers == 1` or only one LAS file is queued — else a `ThreadPoolExecutor` runs the splits in parallel. Per-segment `.npz` files land under `<segments_base_dir>/segment_NNN/` (3-digit zero-padded, matching the `point{i:03d}`/`normal{i:03d}` keys in `run3_planes.npz`). `save_version_json` then drops `run3_versions.json` in each segment dir.36 7. If `save_points_between_planes`: call `divide_las_file_by_planes` per LAS. **Single-threaded on CUDA**; otherwise `max_workers = max(1, min(max_parallel_las_files, os.cpu_count()))`, and a serial path is taken when `max_workers == 1` or only one LAS file is queued — else a `ThreadPoolExecutor` runs the splits in parallel. Per-segment `.npz` files land under `<segments_base_dir>/segment_NNN/` (3-digit zero-padded, matching the `point{i:03d}`/`normal{i:03d}` keys in `run3_planes.npz`). `save_version_json` then drops `run3_versions.json` in each segment dir.
3737
38- **`divide_las_file_by_planes`** streams the LAS via `laspy.open(...).chunk_iterator(las_points_per_chunk)` — never loads the whole file. Per chunk: optional `|scan_angle| < angle_limit` filter, then a sign-count mask against all selected planes assigns each point to a segment bucket. Scan-angle field is auto-detected (`scan_angle_rank` legacy vs `scan_angle` newer). RGB and intensity are **required** — missing fields raise. Each per-segment `.npz` also carries `number_of_returns` (uint8, AI3D-382); LAS files without that field degrade to zeros, which the run3 NPZ contract reads as "unknown" (0 is not a legal LAS return count). Output points are written **geoshift-relative**.38- **`divide_las_file_by_planes`** streams the LAS via `laspy.open(...).chunk_iterator(las_points_per_chunk)` — never loads the whole file. Per chunk: optional `|scan_angle| < angle_limit` filter, then a sign-count mask against all selected planes assigns each point to a segment bucket. Scan-angle field is auto-detected (`scan_angle_rank` legacy vs `scan_angle` newer). RGB and intensity are **required** — missing fields raise. Each per-segment `.npz` also carries `number_of_returns` (uint8, AI3D-382); LAS files without that field degrade to zeros, which the run3 NPZ contract reads as "unknown" (0 is not a legal LAS return count). Output points are written **geoshift-relative**.
3939
40- **`_config.py`** — strict whitelist validation. `ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS` and `ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS` are enforced; unknown keys raise `SegmentMapperConfigError`. `normalize_segment_mapper_config` fills defaults. `build_segment_mapper_config(overrides=..., config_path=...)` deep-merges overrides over `segment_mapper.default.json`.40- **`_config.py`** — pydantic `config_loader.ConfigModel` tree (`SegmentMapperConfig` plus nested section models) validated by `iolabs.common.config_loader.load_config`. Unknown keys raise `SegmentMapperConfigError` (a `config_loader.ConfigError`). `normalize_segment_mapper_config` / `load_segment_mapper_config` / `build_segment_mapper_config` return plain dicts (`model.model_dump()`); overrides deep-merge onto the packaged JSON.
4141
42- **`segment_mapper.default.json`** — bundled defaults. Force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`. If you rename/move it, update that mapping or the installed package will be missing it at runtime.42- **`segment_mapper.default.json`** — bundled defaults. Force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`. If you rename/move it, update that mapping or the installed package will be missing it at runtime.
4343
44- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS work is wrapped in `las_file_scope(las_file)` so log records carry LAS filename context.44- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS work is wrapped in `las_file_scope(las_file)` so log records carry LAS filename context.
Importance #13: knowledge.md @@ -82,13 +82,9 @@
82- `segment_points_suffix` (default `_run3_points`) — appended to `<las_stem>` for per-segment `.npz` files.82- `segment_points_suffix` (default `_run3_points`) — appended to `<las_stem>` for per-segment `.npz` files.
8383
84## Conventions & gotchas84## Conventions & gotchas
8585
86- **Adding a new config key requires three edits:**86- **Adding a new config key requires two edits:** add a field to the pydantic model in `_config.py` and a matching default in `segment_mapper.default.json`. Nothing else. Missing either crashes validation or leaves the packaged JSON out of sync with the model.
87 1. Add it to the whitelist in `_config.py` (`ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS` or `ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS`).
88 2. Add a `setdefault(...)` in `normalize_segment_mapper_config` (or `_normalize_file_naming`).
89 3. Add the default value in `segment_mapper.default.json`.
90 Missing any of these either crashes validation or leaves the field undefined at runtime.
91- **Never rename/move `segment_mapper.default.json`** without updating `[tool.hatch.build.targets.wheel.force-include]` — it won't ship in the wheel.87- **Never rename/move `segment_mapper.default.json`** without updating `[tool.hatch.build.targets.wheel.force-include]` — it won't ship in the wheel.
92- **Plane probe uses `n=10`, not `n=2`** (see rationale above). Do not "optimize" this.88- **Plane probe uses `n=10`, not `n=2`** (see rationale above). Do not "optimize" this.
93- **All saved points are geoshift-relative** (planes' `point`, segment `.npz` `points`). Downstream code must add `geoshift` back to get world coords.89- **All saved points are geoshift-relative** (planes' `point`, segment `.npz` `points`). Downstream code must add `geoshift` back to get world coords.
94- **Per-LAS logging must stay inside `las_file_scope(...)`** so log records keep the LAS filename context.90- **Per-LAS logging must stay inside `las_file_scope(...)`** so log records keep the LAS filename context.
Importance #14: CLAUDE.md @@ -43,9 +43,9 @@
43 4. an entry in `required_chunk_fields` inside `divide_las_file_by_planes` **if the field is mandatory** — that tuple is what turns a missing field into an upfront `ValueError` instead of a later `AttributeError`.43 4. an entry in `required_chunk_fields` inside `divide_las_file_by_planes` **if the field is mandatory** — that tuple is what turns a missing field into an upfront `ValueError` instead of a later `AttributeError`.
4444
45 `SEGMENT_NPZ_FIELD_NAMES` / `NUMBER_OF_RETURNS_KEY` / `NUMBER_OF_RETURNS_DTYPE` duplicate the schema owned by `iolabs.common.segment_points_io` (mirrored, not imported, so this module stays importable against older `iolabs-common`). `tests/test_number_of_returns.py::test_npz_schema_matches_common_segment_points_io` guards the duplication; it `importorskip`s the consumer module, so it is inert until the `iolabs-common` floor is raised to a release that ships it, and then activates on its own. For the same cross-version reason, `SegmentMapper.load_color_intensity_data` builds its `ColorIntensityData` kwargs filtered by `dataclasses.fields(...)`: passing a kwarg the installed dataclass does not declare is a `TypeError`, so a field the installed `iolabs-common` predates is dropped rather than forced.45 `SEGMENT_NPZ_FIELD_NAMES` / `NUMBER_OF_RETURNS_KEY` / `NUMBER_OF_RETURNS_DTYPE` duplicate the schema owned by `iolabs.common.segment_points_io` (mirrored, not imported, so this module stays importable against older `iolabs-common`). `tests/test_number_of_returns.py::test_npz_schema_matches_common_segment_points_io` guards the duplication; it `importorskip`s the consumer module, so it is inert until the `iolabs-common` floor is raised to a release that ships it, and then activates on its own. For the same cross-version reason, `SegmentMapper.load_color_intensity_data` builds its `ColorIntensityData` kwargs filtered by `dataclasses.fields(...)`: passing a kwarg the installed dataclass does not declare is a `TypeError`, so a field the installed `iolabs-common` predates is dropped rather than forced.
4646
47- **`_config.py`** — strict whitelist validation. `ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS` and `ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS` are enforced; unknown keys raise `SegmentMapperConfigError`. `normalize_segment_mapper_config` fills defaults; `build_segment_mapper_config(overrides=..., config_path=...)` does deep-merge over `segment_mapper.default.json`. **When adding a new config key you must update both the whitelist set and `normalize_segment_mapper_config`'s `setdefault` block, and add a default in `segment_mapper.default.json`.**47- **`_config.py`** — pydantic `config_loader.ConfigModel` tree (`SegmentMapperConfig` plus nested section models) validated by `iolabs.common.config_loader.load_config`. Unknown keys raise `SegmentMapperConfigError` (a `config_loader.ConfigError`). `normalize_segment_mapper_config` / `load_segment_mapper_config` / `build_segment_mapper_config` return plain dicts (`model.model_dump()`); overrides deep-merge onto the packaged JSON. **When adding a new config key, add a field to the model and a matching default in `segment_mapper.default.json`. Nothing else.**
4848
49- **`segment_mapper.default.json`** — bundled defaults. It is force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`; if you rename or move it, update that mapping or the installed package will be missing the file at runtime.49- **`segment_mapper.default.json`** — bundled defaults. It is force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`; if you rename or move it, update that mapping or the installed package will be missing the file at runtime.
5050
51- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS-file work is wrapped in `las_file_scope(las_file)` so log records carry the LAS filename context — keep new per-file code paths inside that scope.51- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS-file work is wrapped in `las_file_scope(las_file)` so log records carry the LAS filename context — keep new per-file code paths inside that scope.
Importance #15: README.md @@ -16,9 +16,9 @@
1616
17## Requirements17## Requirements
1818
19- Python ≥3.11, <3.1319- Python ≥3.11, <3.13
20- numpy, open3d, laspy, iolabs-common, iolabs-geometry-geometry, iolabs-point-cloud-filtering-intensity, iolabs-point-cloud-las-tools20- numpy, open3d, laspy, pydantic, iolabs-common, iolabs-geometry-geometry, iolabs-point-cloud-filtering-intensity, iolabs-point-cloud-las-tools
2121
22## Usage22## Usage
2323
24Maps segments along trajectories for highway LIDAR scans, building on trajectory detection and intensity filtering.24Maps segments along trajectories for highway LIDAR scans, building on trajectory detection and intensity filtering.
Importance #16: knowledge.md @@ -36,9 +36,9 @@
36 7. If `save_points_between_planes`: call `divide_las_file_by_planes` per LAS. **Single-threaded on CUDA**; otherwise `max_workers = max(1, min(max_parallel_las_files, os.cpu_count()))`, and a serial path is taken when `max_workers == 1` or only one LAS file is queued — else a `ThreadPoolExecutor` runs the splits in parallel. Per-segment `.npz` files land under `<segments_base_dir>/segment_NNN/` (3-digit zero-padded, matching the `point{i:03d}`/`normal{i:03d}` keys in `run3_planes.npz`). `save_version_json` then drops `run3_versions.json` in each segment dir.36 7. If `save_points_between_planes`: call `divide_las_file_by_planes` per LAS. **Single-threaded on CUDA**; otherwise `max_workers = max(1, min(max_parallel_las_files, os.cpu_count()))`, and a serial path is taken when `max_workers == 1` or only one LAS file is queued — else a `ThreadPoolExecutor` runs the splits in parallel. Per-segment `.npz` files land under `<segments_base_dir>/segment_NNN/` (3-digit zero-padded, matching the `point{i:03d}`/`normal{i:03d}` keys in `run3_planes.npz`). `save_version_json` then drops `run3_versions.json` in each segment dir.
3737
38- **`divide_las_file_by_planes`** streams the LAS via `laspy.open(...).chunk_iterator(las_points_per_chunk)` — never loads the whole file. Per chunk: optional `|scan_angle| < angle_limit` filter, then a sign-count mask against all selected planes assigns each point to a segment bucket. Scan-angle field is auto-detected (`scan_angle_rank` legacy vs `scan_angle` newer). RGB and intensity are **required** — missing fields raise. Each per-segment `.npz` also carries `number_of_returns` (uint8, AI3D-382); LAS files without that field degrade to zeros, which the run3 NPZ contract reads as "unknown" (0 is not a legal LAS return count). Output points are written **geoshift-relative**.38- **`divide_las_file_by_planes`** streams the LAS via `laspy.open(...).chunk_iterator(las_points_per_chunk)` — never loads the whole file. Per chunk: optional `|scan_angle| < angle_limit` filter, then a sign-count mask against all selected planes assigns each point to a segment bucket. Scan-angle field is auto-detected (`scan_angle_rank` legacy vs `scan_angle` newer). RGB and intensity are **required** — missing fields raise. Each per-segment `.npz` also carries `number_of_returns` (uint8, AI3D-382); LAS files without that field degrade to zeros, which the run3 NPZ contract reads as "unknown" (0 is not a legal LAS return count). Output points are written **geoshift-relative**.
3939
40- **`_config.py`** — strict whitelist validation. `ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS` and `ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS` are enforced; unknown keys raise `SegmentMapperConfigError`. `normalize_segment_mapper_config` fills defaults. `build_segment_mapper_config(overrides=..., config_path=...)` deep-merges overrides over `segment_mapper.default.json`.40- **`_config.py`** — pydantic `config_loader.ConfigModel` tree (`SegmentMapperConfig` plus nested section models) validated by `iolabs.common.config_loader.load_config`. Unknown keys raise `SegmentMapperConfigError` (a `config_loader.ConfigError`). `normalize_segment_mapper_config` / `load_segment_mapper_config` / `build_segment_mapper_config` return plain dicts (`model.model_dump()`); overrides deep-merge onto the packaged JSON.
4141
42- **`segment_mapper.default.json`** — bundled defaults. Force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`. If you rename/move it, update that mapping or the installed package will be missing it at runtime.42- **`segment_mapper.default.json`** — bundled defaults. Force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`. If you rename/move it, update that mapping or the installed package will be missing it at runtime.
4343
44- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS work is wrapped in `las_file_scope(las_file)` so log records carry LAS filename context.44- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS work is wrapped in `las_file_scope(las_file)` so log records carry LAS filename context.
Importance #17: knowledge.md @@ -82,13 +82,9 @@
82- `segment_points_suffix` (default `_run3_points`) — appended to `<las_stem>` for per-segment `.npz` files.82- `segment_points_suffix` (default `_run3_points`) — appended to `<las_stem>` for per-segment `.npz` files.
8383
84## Conventions & gotchas84## Conventions & gotchas
8585
86- **Adding a new config key requires three edits:**86- **Adding a new config key requires two edits:** add a field to the pydantic model in `_config.py` and a matching default in `segment_mapper.default.json`. Nothing else. Missing either crashes validation or leaves the packaged JSON out of sync with the model.
87 1. Add it to the whitelist in `_config.py` (`ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS` or `ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS`).
88 2. Add a `setdefault(...)` in `normalize_segment_mapper_config` (or `_normalize_file_naming`).
89 3. Add the default value in `segment_mapper.default.json`.
90 Missing any of these either crashes validation or leaves the field undefined at runtime.
91- **Never rename/move `segment_mapper.default.json`** without updating `[tool.hatch.build.targets.wheel.force-include]` — it won't ship in the wheel.87- **Never rename/move `segment_mapper.default.json`** without updating `[tool.hatch.build.targets.wheel.force-include]` — it won't ship in the wheel.
92- **Plane probe uses `n=10`, not `n=2`** (see rationale above). Do not "optimize" this.88- **Plane probe uses `n=10`, not `n=2`** (see rationale above). Do not "optimize" this.
93- **All saved points are geoshift-relative** (planes' `point`, segment `.npz` `points`). Downstream code must add `geoshift` back to get world coords.89- **All saved points are geoshift-relative** (planes' `point`, segment `.npz` `points`). Downstream code must add `geoshift` back to get world coords.
94- **Per-LAS logging must stay inside `las_file_scope(...)`** so log records keep the LAS filename context.90- **Per-LAS logging must stay inside `las_file_scope(...)`** so log records keep the LAS filename context.
Importance #18: pyproject.toml @@ -1,13 +1,14 @@
1[project]1[project]
2name = "iolabs-point-cloud-segmentation-trajectory"2name = "iolabs-point-cloud-segmentation-trajectory"
3version = "0.7.3"3version = "0.7.4"
4description = "Trajectory-based segment mapping for LIDAR highway scans"4description = "Trajectory-based segment mapping for LIDAR highway scans"
5requires-python = ">=3.11,<3.13"5requires-python = ">=3.11,<3.13"
6dependencies = [6dependencies = [
7 "numpy>=1.20.0",7 "numpy>=1.20.0",
8 "open3d>=0.19.0",8 "open3d>=0.19.0",
9 "laspy>=2.0.0",9 "laspy>=2.0.0",
10 "pydantic>=2.7",
10 "iolabs-logstash>=0.5.1",11 "iolabs-logstash>=0.5.1",
11 "iolabs-common>=0.8.0",12 "iolabs-common>=0.8.0",
12 "iolabs-geometry-geometry",13 "iolabs-geometry-geometry",
13 "iolabs-point-cloud-filtering-intensity>=0.5.0",14 "iolabs-point-cloud-filtering-intensity>=0.5.0",
Importance #19: src/iolabs_point_cloud_segmentation_trajectory/_config.py @@ -1,247 +1,118 @@
1"""Segment mapper config: pydantic model tree over the packaged default JSON."""
2
1from __future__ import annotations3from __future__ import annotations
24
3import copy
4import json
5from importlib import resources as importlib_resources
6from pathlib import Path5from pathlib import Path
7from typing import Any6from typing import Any
87
9ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS = frozenset(8import pydantic
10 {9from iolabs.common import config_loader
11 "n_segments",10
12 "segment_length_m",11_PACKAGE = "iolabs_point_cloud_segmentation_trajectory"
13 "max_distance_to_plane",12_DEFAULT_FILENAME = "segment_mapper.default.json"
14 "save_planes",13_CONTEXT = "segment mapper config"
15 "device",14
16 "visualize",15
17 "visualize_las_segment_coloring",16class SegmentMapperConfigError(config_loader.ConfigError):
18 "save_points_between_planes",17 """Raised when segment mapper config contains unsupported keys or values."""
19 "las_points_per_chunk",18
20 "max_parallel_las_files",19
21 "angle_limit",20class SegmentMapperFileNamingConfig(config_loader.ConfigModel):
22 "n_extra_planes",21 """Output filenames under the ``file_naming`` section."""
23 "segments_base_dir_name",22
24 "spline_pcd_extension",23 planes_filename: str = "run3_planes.npz"
25 "reuse_existing_planes",24 segment_points_suffix: str = "_run3_points"
26 "reuse_existing_geoshift",25 geoshift_filename: str = "run3_geoshift.json"
27 "enable_longitudinal_limit_planes",26 longitudinal_limit_planes_filename: str = "run3_longitudinal_limit_planes.npz"
28 "longitudinal_limit_distance_m",27
29 "save_longitudinal_limit_planes",28
30 "write_only_segments",29class SegmentMapperVisualizationColorsConfig(config_loader.ConfigModel):
31 "visualization_colors",30 """RGB visualization colors; each value is a list of three numbers."""
32 "file_naming",31
33 }32 angle_limit_rejected: list[float] = pydantic.Field(
34)33 default=[0.45, 0.45, 0.45], min_length=3, max_length=3
35
36ALLOWED_SEGMENT_MAPPER_VISUALIZATION_COLOR_KEYS = frozenset(
37 {
38 "angle_limit_rejected",
39 "segmentation_plane",
40 "longitudinal_left_plane",
41 "longitudinal_right_plane",
42 }
43)
44
45ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS = frozenset(
46 {
47 "planes_filename",
48 "segment_points_suffix",
49 "geoshift_filename",
50 "longitudinal_limit_planes_filename",
51 }
52)
53
54class SegmentMapperConfigError(ValueError):
55 """Raised when segment mapper config contains unsupported keys."""
56
57
58def _default_config_path() -> Path:
59 if __package__ in {None, ""}:
60 return Path(__file__).resolve().with_name("segment_mapper.default.json")
61 return Path(str(importlib_resources.files(__package__).joinpath("segment_mapper.default.json")))
62
63
64def _read_config_json(config_path: str | Path) -> dict[str, Any]:
65 with Path(config_path).open("r", encoding="utf-8") as handle:
66 return json.load(handle)
67
68
69def _load_default_config() -> dict[str, Any]:
70 return _read_config_json(_default_config_path())
71
72
73def _deep_merge_dicts(
74 base: dict[str, Any],
75 overrides: dict[str, Any],
76) -> dict[str, Any]:
77 for key, value in overrides.items():
78 if isinstance(value, dict) and isinstance(base.get(key), dict):
79 base[key] = _deep_merge_dicts(dict(base[key]), value)
80 else:
81 base[key] = value
82 return base
83
84
85def _validate_segment_mapper_config_keys(config: dict[str, Any]) -> None:
86 unknown_keys = sorted(set(config) - ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS)
87 if unknown_keys:
88 allowed_keys = ", ".join(sorted(ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS))
89 raise SegmentMapperConfigError(
90 "Unknown segment mapper config key(s): "
91 f"{', '.join(unknown_keys)}. Allowed keys: {allowed_keys}"
92 )
93
94
95def _normalize_file_naming(
96 raw_file_naming: Any,
97 *,
98 default_file_naming: dict[str, Any],
99) -> dict[str, Any]:
100 if raw_file_naming is None:
101 file_naming = dict(default_file_naming)
102 elif isinstance(raw_file_naming, dict):
103 file_naming = _deep_merge_dicts(dict(default_file_naming), raw_file_naming)
104 else:
105 raise SegmentMapperConfigError(
106 "segment mapper config field 'file_naming' must be a mapping"
107 )
108
109 unknown_keys = sorted(set(file_naming) - ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS)
110 if unknown_keys:
111 allowed_keys = ", ".join(sorted(ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS))
112 raise SegmentMapperConfigError(
113 "Unknown segment mapper file_naming key(s): "
114 f"{', '.join(unknown_keys)}. Allowed keys: {allowed_keys}"
115 )
116
117 missing_keys = sorted(ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS - set(file_naming))
118 if missing_keys:
119 raise SegmentMapperConfigError(
120 "Bundled segment mapper default config is missing file_naming key(s): "
121 f"{', '.join(missing_keys)}"
122 )
123 return file_naming
124
125
126def _normalize_rgb_color(raw_color: Any, *, key: str) -> list[float]:
127 if (
128 not isinstance(raw_color, list | tuple)
129 or len(raw_color) != 3
130 or any(not isinstance(value, int | float) for value in raw_color)
131 ):
132 raise SegmentMapperConfigError(
133 f"segment mapper visualization color {key!r} must be an RGB list of 3 numbers"
134 )
135 return [float(value) for value in raw_color]
136
137
138def _normalize_visualization_colors(
139 raw_visualization_colors: Any,
140 *,
141 default_visualization_colors: dict[str, Any],
142) -> dict[str, Any]:
143 if raw_visualization_colors is None:
144 visualization_colors = dict(default_visualization_colors)
145 elif isinstance(raw_visualization_colors, dict):
146 visualization_colors = _deep_merge_dicts(
147 dict(default_visualization_colors),
148 raw_visualization_colors,
149 )
150 else:
151 raise SegmentMapperConfigError(
152 "segment mapper config field 'visualization_colors' must be a mapping"
153 )
154
155 unknown_keys = sorted(
156 set(visualization_colors) - ALLOWED_SEGMENT_MAPPER_VISUALIZATION_COLOR_KEYS
157 )34 )
158 if unknown_keys:35 segmentation_plane: list[float] = pydantic.Field(
159 allowed_keys = ", ".join(sorted(ALLOWED_SEGMENT_MAPPER_VISUALIZATION_COLOR_KEYS))36 default=[0.1, 0.35, 1.0], min_length=3, max_length=3
160 raise SegmentMapperConfigError(
161 "Unknown segment mapper visualization_colors key(s): "
162 f"{', '.join(unknown_keys)}. Allowed keys: {allowed_keys}"
163 )
164
165 missing_keys = sorted(
166 ALLOWED_SEGMENT_MAPPER_VISUALIZATION_COLOR_KEYS - set(visualization_colors)
167 )37 )
168 if missing_keys:38 longitudinal_left_plane: list[float] = pydantic.Field(
169 raise SegmentMapperConfigError(39 default=[1.0, 0.25, 0.0], min_length=3, max_length=3
170 "Bundled segment mapper default config is missing visualization_colors key(s): "
171 f"{', '.join(missing_keys)}"
172 )
173 return {
174 key: _normalize_rgb_color(value, key=key)
175 for key, value in visualization_colors.items()
176 }
177
178
179def _normalize_segment_mapper_config_values(
180 raw_config: dict[str, Any],
181 *,
182 default_config: dict[str, Any],
183) -> dict[str, Any]:
184 config = dict(raw_config)
185 _validate_segment_mapper_config_keys(config)
186
187 missing_keys = sorted(ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS - set(config))
188 if missing_keys:
189 raise SegmentMapperConfigError(
190 "Bundled segment mapper default config is missing key(s): "
191 f"{', '.join(missing_keys)}"
192 )
193
194 longitudinal_limit_distance_m = float(config["longitudinal_limit_distance_m"])
195 if longitudinal_limit_distance_m <= 0.0:
196 raise SegmentMapperConfigError(
197 "segment mapper config field 'longitudinal_limit_distance_m' must be > 0"
198 )
199 config["longitudinal_limit_distance_m"] = longitudinal_limit_distance_m
200 config["file_naming"] = _normalize_file_naming(
201 config.get("file_naming"),
202 default_file_naming=dict(default_config.get("file_naming", {})),
203 )40 )
204 config["visualization_colors"] = _normalize_visualization_colors(41 longitudinal_right_plane: list[float] = pydantic.Field(
205 config.get("visualization_colors"),42 default=[1.0, 0.55, 0.0], min_length=3, max_length=3
206 default_visualization_colors=dict(default_config.get("visualization_colors", {})),
207 )43 )
208 return config
20944
21045
211def normalize_segment_mapper_config(raw_config: dict[str, Any]) -> dict[str, Any]:46class SegmentMapperConfig(config_loader.ConfigModel):
212 default_config = _load_default_config()47 """Segment mapper config; field names and nesting match the packaged JSON."""
213 config = _deep_merge_dicts(copy.deepcopy(default_config), dict(raw_config))48
214 return _normalize_segment_mapper_config_values(49 n_segments: int = 100
215 config,50 segment_length_m: float | None = 50.0
216 default_config=default_config,51 max_distance_to_plane: float = 200.0
52 save_planes: bool = True
53 device: str = "CPU:0"
54 visualize: bool = False
55 visualize_las_segment_coloring: bool = False
56 save_points_between_planes: bool = True
57 las_points_per_chunk: int = 500000
58 max_parallel_las_files: int = 1
59 angle_limit: int | None = 80
60 n_extra_planes: int = 4
61 segments_base_dir_name: str = "lane_points"
62 spline_pcd_extension: str = "_run1_spline_points"
63 reuse_existing_planes: bool = False
64 reuse_existing_geoshift: bool = False
65 enable_longitudinal_limit_planes: bool = True
66 longitudinal_limit_distance_m: float = pydantic.Field(default=100.0, gt=0)
67 write_only_segments: list[int] = []
68 save_longitudinal_limit_planes: bool = True
69 visualization_colors: SegmentMapperVisualizationColorsConfig = (
70 SegmentMapperVisualizationColorsConfig()
217 )71 )
72 file_naming: SegmentMapperFileNamingConfig = SegmentMapperFileNamingConfig()
21873
74 @pydantic.field_validator("file_naming", "visualization_colors", mode="before")
75 @classmethod
76 def _none_section_uses_defaults(cls, value: Any) -> Any:
77 """Treat a JSON ``null`` section as 'use nested defaults'."""
78 if value is None:
79 return {}
80 return value
21981
220def load_segment_mapper_config(config_path: str | Path | None = None) -> dict[str, Any]:82
221 default_config = _load_default_config()83def _load_segment_mapper_model(
222 if config_path is None:84 *,
223 raw_config = copy.deepcopy(default_config)85 overrides: dict[str, Any] | None = None,
224 else:86 config_path: str | Path | None = None,
225 raw_config = _deep_merge_dicts(87) -> SegmentMapperConfig:
226 copy.deepcopy(default_config),88 return config_loader.load_config(
227 _read_config_json(config_path),89 SegmentMapperConfig,
228 )90 package=_PACKAGE,
229 return _normalize_segment_mapper_config_values(91 filename=_DEFAULT_FILENAME,
230 raw_config,92 overrides=overrides,
231 default_config=default_config,93 config_path=config_path,
94 context=_CONTEXT,
95 error_cls=SegmentMapperConfigError,
232 )96 )
23397
23498
99def normalize_segment_mapper_config(raw_config: dict[str, Any]) -> dict[str, Any]:
100 """Merge *raw_config* onto the packaged defaults and return the validated dict."""
101 return _load_segment_mapper_model(overrides=dict(raw_config)).model_dump()
102
103
104def load_segment_mapper_config(config_path: str | Path | None = None) -> dict[str, Any]:
105 """Return the validated config from *config_path*, or the packaged defaults."""
106 return _load_segment_mapper_model(config_path=config_path).model_dump()
107
108
235def build_segment_mapper_config(109def build_segment_mapper_config(
236 *,110 *,
237 overrides: dict[str, Any] | None = None,111 overrides: dict[str, Any] | None = None,
238 config_path: str | Path | None = None,112 config_path: str | Path | None = None,
239) -> dict[str, Any]:113) -> dict[str, Any]:
240 default_config = _load_default_config()114 """Return the validated config with *overrides* merged onto the defaults."""
241 config = load_segment_mapper_config(config_path)115 return _load_segment_mapper_model(
242 if overrides:116 overrides=overrides,
243 config = _deep_merge_dicts(config, dict(overrides))117 config_path=config_path,
244 return _normalize_segment_mapper_config_values(118 ).model_dump()
245 config,
246 default_config=default_config,
247 )
Importance #20: tests/conftest.py @@ -10,11 +10,11 @@
10with a real `SourceFileLoader` but *without* executing `__init__.py`, so:10with a real `SourceFileLoader` but *without* executing `__init__.py`, so:
1111
12- submodule imports like `from iolabs_...._config import ...` resolve via12- submodule imports like `from iolabs_...._config import ...` resolve via
13 the spec's `submodule_search_locations`, and13 the spec's `submodule_search_locations`, and
14- `importlib.resources.files(<pkg>)` (used by `_config._default_config_path`)14- `importlib.resources.files(<pkg>)` (used by `config_loader.load_config` via
15 has a loader with `get_resource_reader` so bundled `segment_mapper.default.json`15 the packaged default JSON) has a loader with `get_resource_reader` so bundled
16 is findable,16 `segment_mapper.default.json` is findable,
1717
18while the heavy runtime deps remain untouched.18while the heavy runtime deps remain untouched.
19"""19"""
2020
Importance #21: tests/test_config.py @@ -6,12 +6,14 @@
6from pathlib import Path6from pathlib import Path
77
8import pytest8import pytest
99
10from iolabs_point_cloud_segmentation_trajectory import _config
10from iolabs_point_cloud_segmentation_trajectory._config import (11from iolabs_point_cloud_segmentation_trajectory._config import (
11 ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS,12 SegmentMapperConfig,
12 ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS,
13 SegmentMapperConfigError,13 SegmentMapperConfigError,
14 SegmentMapperFileNamingConfig,
15 SegmentMapperVisualizationColorsConfig,
14 build_segment_mapper_config,16 build_segment_mapper_config,
15 load_segment_mapper_config,17 load_segment_mapper_config,
16 normalize_segment_mapper_config,18 normalize_segment_mapper_config,
17)19)
Importance #22: tests/test_config.py @@ -23,8 +25,9 @@
23 "max_distance_to_plane": 200.0,25 "max_distance_to_plane": 200.0,
24 "save_planes": True,26 "save_planes": True,
25 "device": "CPU:0",27 "device": "CPU:0",
26 "visualize": False,28 "visualize": False,
29 "visualize_las_segment_coloring": False,
27 "save_points_between_planes": True,30 "save_points_between_planes": True,
28 "las_points_per_chunk": 500_000,31 "las_points_per_chunk": 500_000,
29 "max_parallel_las_files": 1,32 "max_parallel_las_files": 1,
30 "angle_limit": 80,33 "angle_limit": 80,
Importance #23: tests/test_config.py @@ -68,13 +71,28 @@
68 config = normalize_segment_mapper_config({})71 config = normalize_segment_mapper_config({})
69 assert config["file_naming"] == EXPECTED_FILE_NAMING_DEFAULTS72 assert config["file_naming"] == EXPECTED_FILE_NAMING_DEFAULTS
7073
7174
72def test_defaults_cover_every_allowed_key() -> None:75def test_defaults_cover_every_model_field() -> None:
73 """Guard against adding a new whitelisted key without a matching default."""76 """Guard against adding a model field without a matching packaged default."""
74 config = normalize_segment_mapper_config({})77 config = normalize_segment_mapper_config({})
75 assert set(config.keys()) == ALLOWED_SEGMENT_MAPPER_CONFIG_KEYS78 assert set(config.keys()) == set(SegmentMapperConfig.model_fields)
76 assert set(config["file_naming"].keys()) == ALLOWED_SEGMENT_MAPPER_FILE_NAMING_KEYS79 assert set(config["file_naming"].keys()) == set(
80 SegmentMapperFileNamingConfig.model_fields
81 )
82 assert set(config["visualization_colors"].keys()) == set(
83 SegmentMapperVisualizationColorsConfig.model_fields
84 )
85
86
87def test_packaged_json_matches_model_defaults() -> None:
88 """The packaged JSON must stay in sync with the model defaults, key and value."""
89 packaged = json.loads(
90 (
91 Path(_config.__file__).with_name("segment_mapper.default.json")
92 ).read_text(encoding="utf-8")
93 )
94 assert packaged == SegmentMapperConfig().model_dump()
7795
7896
79def test_normalize_is_idempotent() -> None:97def test_normalize_is_idempotent() -> None:
80 once = normalize_segment_mapper_config({})98 once = normalize_segment_mapper_config({})
Importance #24: tests/test_config.py @@ -217,8 +235,12 @@
217 assert config["file_naming"]["planes_filename"] == "x.npz"235 assert config["file_naming"]["planes_filename"] == "x.npz"
218 # file_naming still deep-merged with defaults236 # file_naming still deep-merged with defaults
219 assert config["file_naming"]["segment_points_suffix"] == "_run3_points"237 assert config["file_naming"]["segment_points_suffix"] == "_run3_points"
220 assert config["file_naming"]["geoshift_filename"] == "run3_geoshift.json"238 assert config["file_naming"]["geoshift_filename"] == "run3_geoshift.json"
239 # A partial file still yields every top-level default, and the whole config.
240 assert config["device"] == "CPU:0"
241 assert config["angle_limit"] == 80
242 assert set(config) == set(SegmentMapperConfig.model_fields)
221243
222244
223def test_load_from_custom_path_validates_keys(tmp_path: Path) -> None:245def test_load_from_custom_path_validates_keys(tmp_path: Path) -> None:
224 cfg_file = tmp_path / "c.json"246 cfg_file = tmp_path / "c.json"
Importance #25: tests/test_config.py @@ -246,8 +268,33 @@
246def test_build_without_overrides_equals_load_default() -> None:268def test_build_without_overrides_equals_load_default() -> None:
247 assert build_segment_mapper_config() == load_segment_mapper_config()269 assert build_segment_mapper_config() == load_segment_mapper_config()
248270
249271
272def test_build_overrides_win_over_config_path(tmp_path: Path) -> None:
273 """`overrides` is merged on top of `config_path`, not the other way round."""
274 cfg_file = tmp_path / "c.json"
275 cfg_file.write_text(
276 json.dumps(
277 {
278 "n_segments": 5,
279 "device": "CUDA:1",
280 "file_naming": {"planes_filename": "from_file.npz"},
281 }
282 )
283 )
284 config = build_segment_mapper_config(
285 overrides={"n_segments": 42, "file_naming": {"geoshift_filename": "from_ovr.json"}},
286 config_path=cfg_file,
287 )
288 assert config["n_segments"] == 42
289 # Keys only the file sets survive, at both levels.
290 assert config["device"] == "CUDA:1"
291 assert config["file_naming"]["planes_filename"] == "from_file.npz"
292 assert config["file_naming"]["geoshift_filename"] == "from_ovr.json"
293 # Keys neither sets still come from the packaged defaults.
294 assert config["file_naming"]["segment_points_suffix"] == "_run3_points"
295
296
250def test_build_rejects_unknown_override_keys() -> None:297def test_build_rejects_unknown_override_keys() -> None:
251 with pytest.raises(SegmentMapperConfigError):298 with pytest.raises(SegmentMapperConfigError):
252 build_segment_mapper_config(overrides={"not_allowed": True})299 build_segment_mapper_config(overrides={"not_allowed": True})
253300