Back to report index

Step 3 segmentationtrajectory 64cc82f: AI3D-379 Align config module with fleet pattern

Miroslav Simko <ms@iolabs.ch> 2026-09-02T09:37:55+02:00

Commit #19 · 31 snippets

 README.md                                          | 12 ++++
 docs/configuration.md                              | 12 ++--
 .../_config.py                                     | 70 +++++++++++++---------
 tests/test_config.py                               | 58 +++++++++++++-----
 4 files changed, 104 insertions(+), 48 deletions(-)
Importance #1: src/iolabs_point_cloud_segmentation_trajectory/_config.py @@ -26,22 +39,14 @@
26 longitudinal_limit_planes_filename: str = "run3_longitudinal_limit_planes.npz"39 longitudinal_limit_planes_filename: str = "run3_longitudinal_limit_planes.npz"
2740
2841
29class SegmentMapperVisualizationColorsConfig(config_loader.ConfigModel):42class SegmentMapperVisualizationColorsConfig(config_loader.ConfigModel):
30 """RGB visualization colors; each value is a list of three numbers."""43 """RGB visualization colors; each value is a triple of numbers."""
3144
32 angle_limit_rejected: list[float] = pydantic.Field(45 angle_limit_rejected: tuple[float, float, float] = (0.45, 0.45, 0.45)
33 default=[0.45, 0.45, 0.45], min_length=3, max_length=346 segmentation_plane: tuple[float, float, float] = (0.1, 0.35, 1.0)
34 )47 longitudinal_left_plane: tuple[float, float, float] = (1.0, 0.25, 0.0)
35 segmentation_plane: list[float] = pydantic.Field(48 longitudinal_right_plane: tuple[float, float, float] = (1.0, 0.55, 0.0)
36 default=[0.1, 0.35, 1.0], min_length=3, max_length=3
37 )
38 longitudinal_left_plane: list[float] = pydantic.Field(
39 default=[1.0, 0.25, 0.0], min_length=3, max_length=3
40 )
41 longitudinal_right_plane: list[float] = pydantic.Field(
42 default=[1.0, 0.55, 0.0], min_length=3, max_length=3
43 )
4449
4550
46class SegmentMapperConfig(config_loader.ConfigModel):51class SegmentMapperConfig(config_loader.ConfigModel):
47 """Segment mapper config; field names and nesting match the packaged JSON."""52 """Segment mapper config; field names and nesting match the packaged JSON."""
Importance #2: src/iolabs_point_cloud_segmentation_trajectory/_config.py @@ -83,44 +88,51 @@
83 @classmethod88 @classmethod
84 def _none_write_only_segments_is_empty(cls, value: Any) -> Any:89 def _none_write_only_segments_is_empty(cls, value: Any) -> Any:
85 """Treat a JSON ``null`` as 'no restriction', as the pre-pydantic code did."""90 """Treat a JSON ``null`` as 'no restriction', as the pre-pydantic code did."""
86 if value is None:91 if value is None:
87 return []92 return ()
88 return value93 return value
8994
9095
91def _load_segment_mapper_model(96def _load_model(
92 *,97 *,
93 overrides: dict[str, Any] | None = None,98 overrides: Mapping[str, Any] | None = None,
94 config_path: str | Path | None = None,99 config_path: str | Path | None = None,
95) -> SegmentMapperConfig:100) -> SegmentMapperConfig:
101 """Load the packaged defaults (or *config_path*) with *overrides* merged on top."""
102 if overrides:
103 logger.info("Config overrides applied: %s", ", ".join(sorted(overrides)))
104 if config_path is not None:
105 logger.info("Config file applied: %s", config_path)
96 return config_loader.load_config(106 return config_loader.load_config(
97 SegmentMapperConfig,107 SegmentMapperConfig,
98 package=_PACKAGE,108 package=_PACKAGE_NAME,
99 filename=_DEFAULT_FILENAME,109 filename=_DEFAULT_FILENAME,
100 overrides=overrides,110 overrides=overrides,
101 config_path=config_path,111 config_path=config_path,
102 context=_CONTEXT,112 context=_CONTEXT,
103 error_cls=SegmentMapperConfigError,113 error_cls=SegmentMapperConfigError,
104 )114 )
105115
106116
107def normalize_segment_mapper_config(raw_config: dict[str, Any]) -> dict[str, Any]:117def normalize_segment_mapper_config(raw_config: Mapping[str, Any]) -> dict[str, Any]:
108 """Merge *raw_config* onto the packaged defaults and return the validated dict."""118 """Validate *raw_config*, filling every unset key with its model default."""
109 return _load_segment_mapper_model(overrides=dict(raw_config)).model_dump()119 return config_loader.validate_config(
120 SegmentMapperConfig,
121 raw_config,
122 context=_CONTEXT,
123 error_cls=SegmentMapperConfigError,
124 ).model_dump()
110125
111126
112def load_segment_mapper_config(config_path: str | Path | None = None) -> dict[str, Any]:127def load_segment_mapper_config(config_path: str | Path | None = None) -> dict[str, Any]:
113 """Return the validated config from *config_path*, or the packaged defaults."""128 """Return the validated config from *config_path*, or the packaged defaults."""
114 return _load_segment_mapper_model(config_path=config_path).model_dump()129 return _load_model(config_path=config_path).model_dump()
115130
116131
117def build_segment_mapper_config(132def build_segment_mapper_config(
118 *,133 *,
119 overrides: dict[str, Any] | None = None,134 overrides: Mapping[str, Any] | None = None,
120 config_path: str | Path | None = None,135 config_path: str | Path | None = None,
121) -> dict[str, Any]:136) -> dict[str, Any]:
122 """Return the validated config with *overrides* merged onto the defaults."""137 """Return the validated config with *overrides* merged onto the defaults."""
123 return _load_segment_mapper_model(138 return _load_model(overrides=overrides, config_path=config_path).model_dump()
124 overrides=overrides,
125 config_path=config_path,
126 ).model_dump()
Importance #3: src/iolabs_point_cloud_segmentation_trajectory/_config.py @@ -1,15 +1,28 @@
1"""Segment mapper config: pydantic model tree over the packaged default JSON."""1"""Configuration for the trajectory-based segment mapper (pipeline step 3).
2
3The schema is `SegmentMapperConfig` (a `config_loader.ConfigModel`), mirroring
4`segment_mapper.default.json` key for key.
5
6Adding a config key means adding the field to the model and the same key to
7`segment_mapper.default.json` — nothing else. Unknown keys are rejected.
8
9The entry points return a plain `dict[str, Any]` (the validated `model_dump()`).
10"""
211
3from __future__ import annotations12from __future__ import annotations
413
14import logging
15from collections.abc import Mapping
5from pathlib import Path16from pathlib import Path
6from typing import Any17from typing import Any
718
8import pydantic19import pydantic
9from iolabs.common import config_loader20from iolabs.common import config_loader
1021
11_PACKAGE = "iolabs_point_cloud_segmentation_trajectory"22logger = logging.getLogger(__name__)
23
24_PACKAGE_NAME = "iolabs_point_cloud_segmentation_trajectory"
12_DEFAULT_FILENAME = "segment_mapper.default.json"25_DEFAULT_FILENAME = "segment_mapper.default.json"
13_CONTEXT = "segment mapper config"26_CONTEXT = "segment mapper config"
1427
1528
Importance #4: src/iolabs_point_cloud_segmentation_trajectory/_config.py @@ -63,9 +68,9 @@
63 reuse_existing_planes: bool = False68 reuse_existing_planes: bool = False
64 reuse_existing_geoshift: bool = False69 reuse_existing_geoshift: bool = False
65 enable_longitudinal_limit_planes: bool = True70 enable_longitudinal_limit_planes: bool = True
66 longitudinal_limit_distance_m: float = pydantic.Field(default=100.0, gt=0)71 longitudinal_limit_distance_m: float = pydantic.Field(default=100.0, gt=0)
67 write_only_segments: list[int] = []72 write_only_segments: tuple[int, ...] = ()
68 save_longitudinal_limit_planes: bool = True73 save_longitudinal_limit_planes: bool = True
69 visualization_colors: SegmentMapperVisualizationColorsConfig = (74 visualization_colors: SegmentMapperVisualizationColorsConfig = (
70 SegmentMapperVisualizationColorsConfig()75 SegmentMapperVisualizationColorsConfig()
71 )76 )
Importance #5: tests/test_config.py @@ -5,8 +5,9 @@
5import json5import json
6from pathlib import Path6from pathlib import Path
77
8import pytest8import pytest
9from iolabs.common import config_loader
910
10from iolabs_point_cloud_segmentation_trajectory import _config11from iolabs_point_cloud_segmentation_trajectory import _config
11from iolabs_point_cloud_segmentation_trajectory._config import (12from iolabs_point_cloud_segmentation_trajectory._config import (
12 SegmentMapperConfig,13 SegmentMapperConfig,
Importance #6: tests/test_config.py @@ -37,15 +38,15 @@
37 "reuse_existing_planes": False,38 "reuse_existing_planes": False,
38 "reuse_existing_geoshift": False,39 "reuse_existing_geoshift": False,
39 "enable_longitudinal_limit_planes": True,40 "enable_longitudinal_limit_planes": True,
40 "longitudinal_limit_distance_m": 100.0,41 "longitudinal_limit_distance_m": 100.0,
41 "write_only_segments": [],42 "write_only_segments": (),
42 "save_longitudinal_limit_planes": True,43 "save_longitudinal_limit_planes": True,
43 "visualization_colors": {44 "visualization_colors": {
44 "angle_limit_rejected": [0.45, 0.45, 0.45],45 "angle_limit_rejected": (0.45, 0.45, 0.45),
45 "segmentation_plane": [0.1, 0.35, 1.0],46 "segmentation_plane": (0.1, 0.35, 1.0),
46 "longitudinal_left_plane": [1.0, 0.25, 0.0],47 "longitudinal_left_plane": (1.0, 0.25, 0.0),
47 "longitudinal_right_plane": [1.0, 0.55, 0.0],48 "longitudinal_right_plane": (1.0, 0.55, 0.0),
48 },49 },
49}50}
5051
51EXPECTED_FILE_NAMING_DEFAULTS: dict[str, str] = {52EXPECTED_FILE_NAMING_DEFAULTS: dict[str, str] = {
Importance #7: tests/test_config.py @@ -83,16 +84,16 @@
83 SegmentMapperVisualizationColorsConfig.model_fields84 SegmentMapperVisualizationColorsConfig.model_fields
84 )85 )
8586
8687
87def test_packaged_json_matches_model_defaults() -> None:88def test_model_defaults_match_packaged_json() -> None:
88 """The packaged JSON must stay in sync with the model defaults, key and value."""89 """The packaged JSON must stay in sync with the model defaults, key and value."""
89 packaged = json.loads(90 packaged = json.loads(
90 (91 (
91 Path(_config.__file__).with_name("segment_mapper.default.json")92 Path(_config.__file__).with_name("segment_mapper.default.json")
92 ).read_text(encoding="utf-8")93 ).read_text(encoding="utf-8")
93 )94 )
94 assert packaged == SegmentMapperConfig().model_dump()95 assert packaged == SegmentMapperConfig().model_dump(mode="json")
9596
9697
97def test_normalize_is_idempotent() -> None:98def test_normalize_is_idempotent() -> None:
98 once = normalize_segment_mapper_config({})99 once = normalize_segment_mapper_config({})
Importance #8: tests/test_config.py @@ -111,9 +112,9 @@
111# Whitelist validation112# Whitelist validation
112# ---------------------------------------------------------------------------113# ---------------------------------------------------------------------------
113114
114115
115def test_unknown_top_level_key_raises() -> None:116def test_unknown_top_level_key_is_rejected() -> None:
116 with pytest.raises(SegmentMapperConfigError) as excinfo:117 with pytest.raises(SegmentMapperConfigError) as excinfo:
117 normalize_segment_mapper_config({"bogus_key": 1})118 normalize_segment_mapper_config({"bogus_key": 1})
118 message = str(excinfo.value)119 message = str(excinfo.value)
119 assert "bogus_key" in message120 assert "bogus_key" in message
Importance #9: tests/test_config.py @@ -128,9 +129,9 @@
128 assert "aaa_unknown" in message129 assert "aaa_unknown" in message
129 assert "zzz_unknown" in message130 assert "zzz_unknown" in message
130131
131132
132def test_unknown_file_naming_key_raises() -> None:133def test_unknown_nested_key_is_rejected() -> None:
133 with pytest.raises(SegmentMapperConfigError) as excinfo:134 with pytest.raises(SegmentMapperConfigError) as excinfo:
134 normalize_segment_mapper_config({"file_naming": {"bogus_fn": "x"}})135 normalize_segment_mapper_config({"file_naming": {"bogus_fn": "x"}})
135 assert "bogus_fn" in str(excinfo.value)136 assert "bogus_fn" in str(excinfo.value)
136137
Importance #10: tests/test_config.py @@ -174,14 +175,14 @@
174175
175def test_write_only_segments_none_means_no_restriction() -> None:176def test_write_only_segments_none_means_no_restriction() -> None:
176 """A JSON ``null`` keeps the pre-pydantic 'write every segment' behaviour."""177 """A JSON ``null`` keeps the pre-pydantic 'write every segment' behaviour."""
177 config = normalize_segment_mapper_config({"write_only_segments": None})178 config = normalize_segment_mapper_config({"write_only_segments": None})
178 assert config["write_only_segments"] == []179 assert config["write_only_segments"] == ()
179180
180181
181def test_write_only_segments_accepts_int_list() -> None:182def test_write_only_segments_accepts_int_list() -> None:
182 config = normalize_segment_mapper_config({"write_only_segments": [3, 7]})183 config = normalize_segment_mapper_config({"write_only_segments": [3, 7]})
183 assert config["write_only_segments"] == [3, 7]184 assert config["write_only_segments"] == (3, 7)
184185
185186
186def test_las_points_per_chunk_must_be_positive() -> None:187def test_las_points_per_chunk_must_be_positive() -> None:
187 for bad in (0, -1):188 for bad in (0, -1):
Importance #11: tests/test_config.py @@ -225,18 +226,18 @@
225def test_partial_visualization_color_override_keeps_other_defaults() -> None:226def test_partial_visualization_color_override_keeps_other_defaults() -> None:
226 config = normalize_segment_mapper_config(227 config = normalize_segment_mapper_config(
227 {"visualization_colors": {"angle_limit_rejected": [0.5, 0.5, 0.5]}}228 {"visualization_colors": {"angle_limit_rejected": [0.5, 0.5, 0.5]}}
228 )229 )
229 assert config["visualization_colors"]["angle_limit_rejected"] == [0.5, 0.5, 0.5]230 assert config["visualization_colors"]["angle_limit_rejected"] == (0.5, 0.5, 0.5)
230 assert config["visualization_colors"]["segmentation_plane"] == [0.1, 0.35, 1.0]231 assert config["visualization_colors"]["segmentation_plane"] == (0.1, 0.35, 1.0)
231232
232233
233# ---------------------------------------------------------------------------234# ---------------------------------------------------------------------------
234# load_segment_mapper_config / build_segment_mapper_config235# load_segment_mapper_config / build_segment_mapper_config
235# ---------------------------------------------------------------------------236# ---------------------------------------------------------------------------
236237
237238
238def test_load_default_bundled_config() -> None:239def test_load_segment_mapper_config_returns_packaged_defaults() -> None:
239 """The bundled default JSON must produce the same defaults as normalizing `{}`."""240 """The bundled default JSON must produce the same defaults as normalizing `{}`."""
240 from_disk = load_segment_mapper_config()241 from_disk = load_segment_mapper_config()
241 from_empty = normalize_segment_mapper_config({})242 from_empty = normalize_segment_mapper_config({})
242 assert from_disk == from_empty243 assert from_disk == from_empty
Importance #12: tests/test_config.py @@ -272,9 +273,9 @@
272 with pytest.raises(SegmentMapperConfigError):273 with pytest.raises(SegmentMapperConfigError):
273 load_segment_mapper_config(cfg_file)274 load_segment_mapper_config(cfg_file)
274275
275276
276def test_build_deep_merges_overrides_over_default_json() -> None:277def test_overrides_deep_merge_onto_defaults() -> None:
277 config = build_segment_mapper_config(278 config = build_segment_mapper_config(
278 overrides={279 overrides={
279 "n_segments": 42,280 "n_segments": 42,
280 "file_naming": {"planes_filename": "custom.npz"},281 "file_naming": {"planes_filename": "custom.npz"},
Importance #13: tests/test_config.py @@ -327,4 +328,31 @@
327 with pytest.raises(SegmentMapperConfigError):328 with pytest.raises(SegmentMapperConfigError):
328 build_segment_mapper_config(329 build_segment_mapper_config(
329 overrides={"file_naming": {"not_allowed_fn": "x"}}330 overrides={"file_naming": {"not_allowed_fn": "x"}}
330 )331 )
332
333
334# ---------------------------------------------------------------------------
335# Error class / --set overrides
336# ---------------------------------------------------------------------------
337
338
339def test_error_class_is_config_error() -> None:
340 assert issubclass(SegmentMapperConfigError, config_loader.ConfigError)
341 assert issubclass(SegmentMapperConfigError, ValueError)
342
343
344def test_set_override_coercion_and_rejection() -> None:
345 overrides = config_loader.parse_set_overrides(
346 ["las_points_per_chunk=1e3", "save_planes=on"],
347 error_cls=SegmentMapperConfigError,
348 )
349 config = build_segment_mapper_config(overrides=overrides)
350 assert config["las_points_per_chunk"] == 1000
351 assert config["save_planes"] is True
352
353 with pytest.raises(SegmentMapperConfigError):
354 build_segment_mapper_config(
355 overrides=config_loader.parse_set_overrides(
356 ["save_planes=flase"], error_cls=SegmentMapperConfigError
357 )
358 )
Importance #14: README.md @@ -24,8 +24,20 @@
24Maps segments along trajectories for highway LIDAR scans, building on trajectory detection and intensity filtering.24Maps segments along trajectories for highway LIDAR scans, building on trajectory detection and intensity filtering.
2525
26For the full configuration reference, see [docs/configuration.md](docs/configuration.md).26For the full configuration reference, see [docs/configuration.md](docs/configuration.md).
2727
28## Configuration
29
30Defaults live in `src/iolabs_point_cloud_segmentation_trajectory/segment_mapper.default.json`.
31The schema is `SegmentMapperConfig` in `iolabs_point_cloud_segmentation_trajectory._config`
32(a `config_loader.ConfigModel`); nested JSON sections (`visualization_colors`,
33`file_naming`) are nested models and unknown keys are rejected. **To add a config
34key: add the field (with its type, default and any `Field` range) to the model and
35the same key with the same default to the JSON — nothing else.**
36`normalize_segment_mapper_config`, `load_segment_mapper_config` and
37`build_segment_mapper_config` return a plain `dict`. Runtime overrides come from
38repeatable `--set KEY=VALUE`, never repo-local JSON.
39
28### Segment count configuration40### Segment count configuration
2941
30The configured `n_segments` value is not a hard cap. By default, segmentation is42The configured `n_segments` value is not a hard cap. By default, segmentation is
31length-based because `segment_length_m` is set in the bundled config. In that43length-based because `segment_length_m` is set in the bundled config. In that
Importance #15: docs/configuration.md @@ -3,11 +3,15 @@
3`SegmentMapper` accepts a strict JSON-compatible configuration mapping. Unknown3`SegmentMapper` accepts a strict JSON-compatible configuration mapping. Unknown
4top-level keys, unknown `file_naming` keys, and unknown `visualization_colors`4top-level keys, unknown `file_naming` keys, and unknown `visualization_colors`
5keys raise `SegmentMapperConfigError`.5keys raise `SegmentMapperConfigError`.
66
7Defaults are loaded from7The schema is `SegmentMapperConfig` in
8`src/iolabs_point_cloud_segmentation_trajectory/segment_mapper.default.json`.8`iolabs_point_cloud_segmentation_trajectory._config`, mirroring
9Partial nested overrides are deep-merged with the bundled defaults. A config9`src/iolabs_point_cloud_segmentation_trajectory/segment_mapper.default.json`
10key for key. Keys a caller omits fall back to the model defaults, which are
11identical to the bundled JSON, so partial nested overrides keep the remaining
12defaults. Sequence values (`write_only_segments`, the RGB colors) come back as
13tuples in the validated dict. A config
10file passed explicitly by path replaces the bundled JSON rather than extending14file passed explicitly by path replaces the bundled JSON rather than extending
11it; keys it omits fall back to the same values the bundled JSON carries.15it; keys it omits fall back to the same values the bundled JSON carries.
1216
13## Example17## Example
Importance #16: docs/configuration.md @@ -55,9 +59,9 @@
55| `enable_longitudinal_limit_planes` | `true` | Builds side limit planes for each segment and drops saved points outside the left/right corridor. This affects LAS splitting and the LAS coloring visualization. |59| `enable_longitudinal_limit_planes` | `true` | Builds side limit planes for each segment and drops saved points outside the left/right corridor. This affects LAS splitting and the LAS coloring visualization. |
56| `longitudinal_limit_distance_m` | `100.0` | Left/right offset distance, in metres, used to build longitudinal limit planes. Must be greater than zero. |60| `longitudinal_limit_distance_m` | `100.0` | Left/right offset distance, in metres, used to build longitudinal limit planes. Must be greater than zero. |
57| `write_only_segments` | `[]` | Restricts LAS splitting to the listed segment indices; an empty list (or `null`) writes every segment. Useful for re-running a few failed segments without redoing the whole split. |61| `write_only_segments` | `[]` | Restricts LAS splitting to the listed segment indices; an empty list (or `null`) writes every segment. Useful for re-running a few failed segments without redoing the whole split. |
58| `save_longitudinal_limit_planes` | `true` | Writes longitudinal limit planes to `<output_dir>/lane_points/<longitudinal_limit_planes_filename>` when longitudinal limits are enabled. |62| `save_longitudinal_limit_planes` | `true` | Writes longitudinal limit planes to `<output_dir>/lane_points/<longitudinal_limit_planes_filename>` when longitudinal limits are enabled. |
59| `visualization_colors` | See below | RGB colors used by visualization-only code. Values are lists of three numbers. |63| `visualization_colors` | See below | RGB colors used by visualization-only code. Values are triples of numbers. |
60| `file_naming` | See below | Output filename overrides. Partial overrides keep unspecified defaults. |64| `file_naming` | See below | Output filename overrides. Partial overrides keep unspecified defaults. |
6165
62## `file_naming`66## `file_naming`
6367
Importance #17: docs/configuration.md @@ -3,11 +3,15 @@
3`SegmentMapper` accepts a strict JSON-compatible configuration mapping. Unknown3`SegmentMapper` accepts a strict JSON-compatible configuration mapping. Unknown
4top-level keys, unknown `file_naming` keys, and unknown `visualization_colors`4top-level keys, unknown `file_naming` keys, and unknown `visualization_colors`
5keys raise `SegmentMapperConfigError`.5keys raise `SegmentMapperConfigError`.
66
7Defaults are loaded from7The schema is `SegmentMapperConfig` in
8`src/iolabs_point_cloud_segmentation_trajectory/segment_mapper.default.json`.8`iolabs_point_cloud_segmentation_trajectory._config`, mirroring
9Partial nested overrides are deep-merged with the bundled defaults. A config9`src/iolabs_point_cloud_segmentation_trajectory/segment_mapper.default.json`
10key for key. Keys a caller omits fall back to the model defaults, which are
11identical to the bundled JSON, so partial nested overrides keep the remaining
12defaults. Sequence values (`write_only_segments`, the RGB colors) come back as
13tuples in the validated dict. A config
10file passed explicitly by path replaces the bundled JSON rather than extending14file passed explicitly by path replaces the bundled JSON rather than extending
11it; keys it omits fall back to the same values the bundled JSON carries.15it; keys it omits fall back to the same values the bundled JSON carries.
1216
13## Example17## Example
Importance #18: docs/configuration.md @@ -55,9 +59,9 @@
55| `enable_longitudinal_limit_planes` | `true` | Builds side limit planes for each segment and drops saved points outside the left/right corridor. This affects LAS splitting and the LAS coloring visualization. |59| `enable_longitudinal_limit_planes` | `true` | Builds side limit planes for each segment and drops saved points outside the left/right corridor. This affects LAS splitting and the LAS coloring visualization. |
56| `longitudinal_limit_distance_m` | `100.0` | Left/right offset distance, in metres, used to build longitudinal limit planes. Must be greater than zero. |60| `longitudinal_limit_distance_m` | `100.0` | Left/right offset distance, in metres, used to build longitudinal limit planes. Must be greater than zero. |
57| `write_only_segments` | `[]` | Restricts LAS splitting to the listed segment indices; an empty list (or `null`) writes every segment. Useful for re-running a few failed segments without redoing the whole split. |61| `write_only_segments` | `[]` | Restricts LAS splitting to the listed segment indices; an empty list (or `null`) writes every segment. Useful for re-running a few failed segments without redoing the whole split. |
58| `save_longitudinal_limit_planes` | `true` | Writes longitudinal limit planes to `<output_dir>/lane_points/<longitudinal_limit_planes_filename>` when longitudinal limits are enabled. |62| `save_longitudinal_limit_planes` | `true` | Writes longitudinal limit planes to `<output_dir>/lane_points/<longitudinal_limit_planes_filename>` when longitudinal limits are enabled. |
59| `visualization_colors` | See below | RGB colors used by visualization-only code. Values are lists of three numbers. |63| `visualization_colors` | See below | RGB colors used by visualization-only code. Values are triples of numbers. |
60| `file_naming` | See below | Output filename overrides. Partial overrides keep unspecified defaults. |64| `file_naming` | See below | Output filename overrides. Partial overrides keep unspecified defaults. |
6165
62## `file_naming`66## `file_naming`
6367
Importance #19: src/iolabs_point_cloud_segmentation_trajectory/_config.py @@ -1,15 +1,28 @@
1"""Segment mapper config: pydantic model tree over the packaged default JSON."""1"""Configuration for the trajectory-based segment mapper (pipeline step 3).
2
3The schema is `SegmentMapperConfig` (a `config_loader.ConfigModel`), mirroring
4`segment_mapper.default.json` key for key.
5
6Adding a config key means adding the field to the model and the same key to
7`segment_mapper.default.json` — nothing else. Unknown keys are rejected.
8
9The entry points return a plain `dict[str, Any]` (the validated `model_dump()`).
10"""
211
3from __future__ import annotations12from __future__ import annotations
413
14import logging
15from collections.abc import Mapping
5from pathlib import Path16from pathlib import Path
6from typing import Any17from typing import Any
718
8import pydantic19import pydantic
9from iolabs.common import config_loader20from iolabs.common import config_loader
1021
11_PACKAGE = "iolabs_point_cloud_segmentation_trajectory"22logger = logging.getLogger(__name__)
23
24_PACKAGE_NAME = "iolabs_point_cloud_segmentation_trajectory"
12_DEFAULT_FILENAME = "segment_mapper.default.json"25_DEFAULT_FILENAME = "segment_mapper.default.json"
13_CONTEXT = "segment mapper config"26_CONTEXT = "segment mapper config"
1427
1528
Importance #20: src/iolabs_point_cloud_segmentation_trajectory/_config.py @@ -26,22 +39,14 @@
26 longitudinal_limit_planes_filename: str = "run3_longitudinal_limit_planes.npz"39 longitudinal_limit_planes_filename: str = "run3_longitudinal_limit_planes.npz"
2740
2841
29class SegmentMapperVisualizationColorsConfig(config_loader.ConfigModel):42class SegmentMapperVisualizationColorsConfig(config_loader.ConfigModel):
30 """RGB visualization colors; each value is a list of three numbers."""43 """RGB visualization colors; each value is a triple of numbers."""
3144
32 angle_limit_rejected: list[float] = pydantic.Field(45 angle_limit_rejected: tuple[float, float, float] = (0.45, 0.45, 0.45)
33 default=[0.45, 0.45, 0.45], min_length=3, max_length=346 segmentation_plane: tuple[float, float, float] = (0.1, 0.35, 1.0)
34 )47 longitudinal_left_plane: tuple[float, float, float] = (1.0, 0.25, 0.0)
35 segmentation_plane: list[float] = pydantic.Field(48 longitudinal_right_plane: tuple[float, float, float] = (1.0, 0.55, 0.0)
36 default=[0.1, 0.35, 1.0], min_length=3, max_length=3
37 )
38 longitudinal_left_plane: list[float] = pydantic.Field(
39 default=[1.0, 0.25, 0.0], min_length=3, max_length=3
40 )
41 longitudinal_right_plane: list[float] = pydantic.Field(
42 default=[1.0, 0.55, 0.0], min_length=3, max_length=3
43 )
4449
4550
46class SegmentMapperConfig(config_loader.ConfigModel):51class SegmentMapperConfig(config_loader.ConfigModel):
47 """Segment mapper config; field names and nesting match the packaged JSON."""52 """Segment mapper config; field names and nesting match the packaged JSON."""
Importance #21: src/iolabs_point_cloud_segmentation_trajectory/_config.py @@ -63,9 +68,9 @@
63 reuse_existing_planes: bool = False68 reuse_existing_planes: bool = False
64 reuse_existing_geoshift: bool = False69 reuse_existing_geoshift: bool = False
65 enable_longitudinal_limit_planes: bool = True70 enable_longitudinal_limit_planes: bool = True
66 longitudinal_limit_distance_m: float = pydantic.Field(default=100.0, gt=0)71 longitudinal_limit_distance_m: float = pydantic.Field(default=100.0, gt=0)
67 write_only_segments: list[int] = []72 write_only_segments: tuple[int, ...] = ()
68 save_longitudinal_limit_planes: bool = True73 save_longitudinal_limit_planes: bool = True
69 visualization_colors: SegmentMapperVisualizationColorsConfig = (74 visualization_colors: SegmentMapperVisualizationColorsConfig = (
70 SegmentMapperVisualizationColorsConfig()75 SegmentMapperVisualizationColorsConfig()
71 )76 )
Importance #22: src/iolabs_point_cloud_segmentation_trajectory/_config.py @@ -83,44 +88,51 @@
83 @classmethod88 @classmethod
84 def _none_write_only_segments_is_empty(cls, value: Any) -> Any:89 def _none_write_only_segments_is_empty(cls, value: Any) -> Any:
85 """Treat a JSON ``null`` as 'no restriction', as the pre-pydantic code did."""90 """Treat a JSON ``null`` as 'no restriction', as the pre-pydantic code did."""
86 if value is None:91 if value is None:
87 return []92 return ()
88 return value93 return value
8994
9095
91def _load_segment_mapper_model(96def _load_model(
92 *,97 *,
93 overrides: dict[str, Any] | None = None,98 overrides: Mapping[str, Any] | None = None,
94 config_path: str | Path | None = None,99 config_path: str | Path | None = None,
95) -> SegmentMapperConfig:100) -> SegmentMapperConfig:
101 """Load the packaged defaults (or *config_path*) with *overrides* merged on top."""
102 if overrides:
103 logger.info("Config overrides applied: %s", ", ".join(sorted(overrides)))
104 if config_path is not None:
105 logger.info("Config file applied: %s", config_path)
96 return config_loader.load_config(106 return config_loader.load_config(
97 SegmentMapperConfig,107 SegmentMapperConfig,
98 package=_PACKAGE,108 package=_PACKAGE_NAME,
99 filename=_DEFAULT_FILENAME,109 filename=_DEFAULT_FILENAME,
100 overrides=overrides,110 overrides=overrides,
101 config_path=config_path,111 config_path=config_path,
102 context=_CONTEXT,112 context=_CONTEXT,
103 error_cls=SegmentMapperConfigError,113 error_cls=SegmentMapperConfigError,
104 )114 )
105115
106116
107def normalize_segment_mapper_config(raw_config: dict[str, Any]) -> dict[str, Any]:117def normalize_segment_mapper_config(raw_config: Mapping[str, Any]) -> dict[str, Any]:
108 """Merge *raw_config* onto the packaged defaults and return the validated dict."""118 """Validate *raw_config*, filling every unset key with its model default."""
109 return _load_segment_mapper_model(overrides=dict(raw_config)).model_dump()119 return config_loader.validate_config(
120 SegmentMapperConfig,
121 raw_config,
122 context=_CONTEXT,
123 error_cls=SegmentMapperConfigError,
124 ).model_dump()
110125
111126
112def load_segment_mapper_config(config_path: str | Path | None = None) -> dict[str, Any]:127def load_segment_mapper_config(config_path: str | Path | None = None) -> dict[str, Any]:
113 """Return the validated config from *config_path*, or the packaged defaults."""128 """Return the validated config from *config_path*, or the packaged defaults."""
114 return _load_segment_mapper_model(config_path=config_path).model_dump()129 return _load_model(config_path=config_path).model_dump()
115130
116131
117def build_segment_mapper_config(132def build_segment_mapper_config(
118 *,133 *,
119 overrides: dict[str, Any] | None = None,134 overrides: Mapping[str, Any] | None = None,
120 config_path: str | Path | None = None,135 config_path: str | Path | None = None,
121) -> dict[str, Any]:136) -> dict[str, Any]:
122 """Return the validated config with *overrides* merged onto the defaults."""137 """Return the validated config with *overrides* merged onto the defaults."""
123 return _load_segment_mapper_model(138 return _load_model(overrides=overrides, config_path=config_path).model_dump()
124 overrides=overrides,
125 config_path=config_path,
126 ).model_dump()
Importance #23: tests/test_config.py @@ -5,8 +5,9 @@
5import json5import json
6from pathlib import Path6from pathlib import Path
77
8import pytest8import pytest
9from iolabs.common import config_loader
910
10from iolabs_point_cloud_segmentation_trajectory import _config11from iolabs_point_cloud_segmentation_trajectory import _config
11from iolabs_point_cloud_segmentation_trajectory._config import (12from iolabs_point_cloud_segmentation_trajectory._config import (
12 SegmentMapperConfig,13 SegmentMapperConfig,
Importance #24: tests/test_config.py @@ -37,15 +38,15 @@
37 "reuse_existing_planes": False,38 "reuse_existing_planes": False,
38 "reuse_existing_geoshift": False,39 "reuse_existing_geoshift": False,
39 "enable_longitudinal_limit_planes": True,40 "enable_longitudinal_limit_planes": True,
40 "longitudinal_limit_distance_m": 100.0,41 "longitudinal_limit_distance_m": 100.0,
41 "write_only_segments": [],42 "write_only_segments": (),
42 "save_longitudinal_limit_planes": True,43 "save_longitudinal_limit_planes": True,
43 "visualization_colors": {44 "visualization_colors": {
44 "angle_limit_rejected": [0.45, 0.45, 0.45],45 "angle_limit_rejected": (0.45, 0.45, 0.45),
45 "segmentation_plane": [0.1, 0.35, 1.0],46 "segmentation_plane": (0.1, 0.35, 1.0),
46 "longitudinal_left_plane": [1.0, 0.25, 0.0],47 "longitudinal_left_plane": (1.0, 0.25, 0.0),
47 "longitudinal_right_plane": [1.0, 0.55, 0.0],48 "longitudinal_right_plane": (1.0, 0.55, 0.0),
48 },49 },
49}50}
5051
51EXPECTED_FILE_NAMING_DEFAULTS: dict[str, str] = {52EXPECTED_FILE_NAMING_DEFAULTS: dict[str, str] = {
Importance #25: tests/test_config.py @@ -83,16 +84,16 @@
83 SegmentMapperVisualizationColorsConfig.model_fields84 SegmentMapperVisualizationColorsConfig.model_fields
84 )85 )
8586
8687
87def test_packaged_json_matches_model_defaults() -> None:88def test_model_defaults_match_packaged_json() -> None:
88 """The packaged JSON must stay in sync with the model defaults, key and value."""89 """The packaged JSON must stay in sync with the model defaults, key and value."""
89 packaged = json.loads(90 packaged = json.loads(
90 (91 (
91 Path(_config.__file__).with_name("segment_mapper.default.json")92 Path(_config.__file__).with_name("segment_mapper.default.json")
92 ).read_text(encoding="utf-8")93 ).read_text(encoding="utf-8")
93 )94 )
94 assert packaged == SegmentMapperConfig().model_dump()95 assert packaged == SegmentMapperConfig().model_dump(mode="json")
9596
9697
97def test_normalize_is_idempotent() -> None:98def test_normalize_is_idempotent() -> None:
98 once = normalize_segment_mapper_config({})99 once = normalize_segment_mapper_config({})
Importance #26: tests/test_config.py @@ -111,9 +112,9 @@
111# Whitelist validation112# Whitelist validation
112# ---------------------------------------------------------------------------113# ---------------------------------------------------------------------------
113114
114115
115def test_unknown_top_level_key_raises() -> None:116def test_unknown_top_level_key_is_rejected() -> None:
116 with pytest.raises(SegmentMapperConfigError) as excinfo:117 with pytest.raises(SegmentMapperConfigError) as excinfo:
117 normalize_segment_mapper_config({"bogus_key": 1})118 normalize_segment_mapper_config({"bogus_key": 1})
118 message = str(excinfo.value)119 message = str(excinfo.value)
119 assert "bogus_key" in message120 assert "bogus_key" in message
Importance #27: tests/test_config.py @@ -128,9 +129,9 @@
128 assert "aaa_unknown" in message129 assert "aaa_unknown" in message
129 assert "zzz_unknown" in message130 assert "zzz_unknown" in message
130131
131132
132def test_unknown_file_naming_key_raises() -> None:133def test_unknown_nested_key_is_rejected() -> None:
133 with pytest.raises(SegmentMapperConfigError) as excinfo:134 with pytest.raises(SegmentMapperConfigError) as excinfo:
134 normalize_segment_mapper_config({"file_naming": {"bogus_fn": "x"}})135 normalize_segment_mapper_config({"file_naming": {"bogus_fn": "x"}})
135 assert "bogus_fn" in str(excinfo.value)136 assert "bogus_fn" in str(excinfo.value)
136137
Importance #28: tests/test_config.py @@ -174,14 +175,14 @@
174175
175def test_write_only_segments_none_means_no_restriction() -> None:176def test_write_only_segments_none_means_no_restriction() -> None:
176 """A JSON ``null`` keeps the pre-pydantic 'write every segment' behaviour."""177 """A JSON ``null`` keeps the pre-pydantic 'write every segment' behaviour."""
177 config = normalize_segment_mapper_config({"write_only_segments": None})178 config = normalize_segment_mapper_config({"write_only_segments": None})
178 assert config["write_only_segments"] == []179 assert config["write_only_segments"] == ()
179180
180181
181def test_write_only_segments_accepts_int_list() -> None:182def test_write_only_segments_accepts_int_list() -> None:
182 config = normalize_segment_mapper_config({"write_only_segments": [3, 7]})183 config = normalize_segment_mapper_config({"write_only_segments": [3, 7]})
183 assert config["write_only_segments"] == [3, 7]184 assert config["write_only_segments"] == (3, 7)
184185
185186
186def test_las_points_per_chunk_must_be_positive() -> None:187def test_las_points_per_chunk_must_be_positive() -> None:
187 for bad in (0, -1):188 for bad in (0, -1):
Importance #29: tests/test_config.py @@ -225,18 +226,18 @@
225def test_partial_visualization_color_override_keeps_other_defaults() -> None:226def test_partial_visualization_color_override_keeps_other_defaults() -> None:
226 config = normalize_segment_mapper_config(227 config = normalize_segment_mapper_config(
227 {"visualization_colors": {"angle_limit_rejected": [0.5, 0.5, 0.5]}}228 {"visualization_colors": {"angle_limit_rejected": [0.5, 0.5, 0.5]}}
228 )229 )
229 assert config["visualization_colors"]["angle_limit_rejected"] == [0.5, 0.5, 0.5]230 assert config["visualization_colors"]["angle_limit_rejected"] == (0.5, 0.5, 0.5)
230 assert config["visualization_colors"]["segmentation_plane"] == [0.1, 0.35, 1.0]231 assert config["visualization_colors"]["segmentation_plane"] == (0.1, 0.35, 1.0)
231232
232233
233# ---------------------------------------------------------------------------234# ---------------------------------------------------------------------------
234# load_segment_mapper_config / build_segment_mapper_config235# load_segment_mapper_config / build_segment_mapper_config
235# ---------------------------------------------------------------------------236# ---------------------------------------------------------------------------
236237
237238
238def test_load_default_bundled_config() -> None:239def test_load_segment_mapper_config_returns_packaged_defaults() -> None:
239 """The bundled default JSON must produce the same defaults as normalizing `{}`."""240 """The bundled default JSON must produce the same defaults as normalizing `{}`."""
240 from_disk = load_segment_mapper_config()241 from_disk = load_segment_mapper_config()
241 from_empty = normalize_segment_mapper_config({})242 from_empty = normalize_segment_mapper_config({})
242 assert from_disk == from_empty243 assert from_disk == from_empty
Importance #30: tests/test_config.py @@ -272,9 +273,9 @@
272 with pytest.raises(SegmentMapperConfigError):273 with pytest.raises(SegmentMapperConfigError):
273 load_segment_mapper_config(cfg_file)274 load_segment_mapper_config(cfg_file)
274275
275276
276def test_build_deep_merges_overrides_over_default_json() -> None:277def test_overrides_deep_merge_onto_defaults() -> None:
277 config = build_segment_mapper_config(278 config = build_segment_mapper_config(
278 overrides={279 overrides={
279 "n_segments": 42,280 "n_segments": 42,
280 "file_naming": {"planes_filename": "custom.npz"},281 "file_naming": {"planes_filename": "custom.npz"},
Importance #31: tests/test_config.py @@ -327,4 +328,31 @@
327 with pytest.raises(SegmentMapperConfigError):328 with pytest.raises(SegmentMapperConfigError):
328 build_segment_mapper_config(329 build_segment_mapper_config(
329 overrides={"file_naming": {"not_allowed_fn": "x"}}330 overrides={"file_naming": {"not_allowed_fn": "x"}}
330 )331 )
332
333
334# ---------------------------------------------------------------------------
335# Error class / --set overrides
336# ---------------------------------------------------------------------------
337
338
339def test_error_class_is_config_error() -> None:
340 assert issubclass(SegmentMapperConfigError, config_loader.ConfigError)
341 assert issubclass(SegmentMapperConfigError, ValueError)
342
343
344def test_set_override_coercion_and_rejection() -> None:
345 overrides = config_loader.parse_set_overrides(
346 ["las_points_per_chunk=1e3", "save_planes=on"],
347 error_cls=SegmentMapperConfigError,
348 )
349 config = build_segment_mapper_config(overrides=overrides)
350 assert config["las_points_per_chunk"] == 1000
351 assert config["save_planes"] is True
352
353 with pytest.raises(SegmentMapperConfigError):
354 build_segment_mapper_config(
355 overrides=config_loader.parse_set_overrides(
356 ["save_planes=flase"], error_cls=SegmentMapperConfigError
357 )
358 )