Back to report index

Step 6 maskclustering 3bc1b7b: AI3D-379 Align config module with fleet pattern

Miroslav Simko <ms@iolabs.ch> 2026-09-02T09:40:09+02:00

Commit #39 · 31 snippets

 AGENTS.md                                          |   8 +-
 README.md                                          |  19 +--
 src/iolabs_point_cloud_mask_clustering/__init__.py |  12 ++
 src/iolabs_point_cloud_mask_clustering/_config.py  | 157 +++++++++------------
 src/iolabs_point_cloud_mask_clustering/cli.py      |   6 +-
 tests/test_config.py                               | 111 +++++++++------
 6 files changed, 168 insertions(+), 145 deletions(-)
Importance #1: src/iolabs_point_cloud_mask_clustering/_config.py @@ -27,17 +26,18 @@
27from ._log_props import LOG_PROPS26from ._log_props import LOG_PROPS
2827
29logger = get_props_logger(__name__, LOG_PROPS)28logger = get_props_logger(__name__, LOG_PROPS)
3029
31PACKAGE = "iolabs_point_cloud_mask_clustering"30_PACKAGE_NAME = "iolabs_point_cloud_mask_clustering"
32DEFAULT_CONFIG_FILENAME = "mask_clustering.default.json"31_DEFAULT_FILENAME = "mask_clustering.default.json"
32_CONTEXT = "mask clustering config"
3333
3434
35class MaskClusteringConfigError(config_loader.ConfigError):35class MaskClusteringConfigError(config_loader.ConfigError):
36 """Raised when mask-clustering configuration is invalid."""36 """Raised when mask clustering config contains unsupported keys or values."""
3737
3838
39class MaskConfig(config_loader.ConfigModel):39class MaskClusteringMaskConfig(config_loader.ConfigModel):
40 """Mask rasterisation and labelling settings."""40 """Mask rasterisation and labelling settings."""
4141
42 background_class: int = 042 background_class: int = 0
43 solid_class: int = 143 solid_class: int = 1
Importance #2: src/iolabs_point_cloud_mask_clustering/_config.py @@ -53,16 +53,16 @@
53 raise ValueError("mask.connectivity must be 4 or 8")53 raise ValueError("mask.connectivity must be 4 or 8")
54 return value54 return value
5555
5656
57class ClustersConfig(config_loader.ConfigModel):57class MaskClusteringClustersConfig(config_loader.ConfigModel):
58 """Sparse-cluster thresholds."""58 """Sparse-cluster thresholds."""
5959
60 min_points_per_cluster: int = pydantic.Field(default=20, ge=0)60 min_points_per_cluster: int = pydantic.Field(default=20, ge=0)
61 warn_below_points: int = pydantic.Field(default=200, ge=0)61 warn_below_points: int = pydantic.Field(default=200, ge=0)
6262
63 @pydantic.model_validator(mode="after")63 @pydantic.model_validator(mode="after")
64 def _check_thresholds(self) -> "ClustersConfig":64 def _check_thresholds(self) -> MaskClusteringClustersConfig:
65 """Reject a minimum above the warning threshold."""65 """Reject a minimum above the warning threshold."""
66 if self.min_points_per_cluster > self.warn_below_points:66 if self.min_points_per_cluster > self.warn_below_points:
67 raise ValueError(67 raise ValueError(
68 "clusters thresholds must satisfy "68 "clusters thresholds must satisfy "
Importance #3: src/iolabs_point_cloud_mask_clustering/_config.py @@ -70,9 +70,9 @@
70 )70 )
71 return self71 return self
7272
7373
74class IntensitySeparationConfig(config_loader.ConfigModel):74class MaskClusteringIntensitySeparationConfig(config_loader.ConfigModel):
75 """Paint/asphalt intensity-separation settings."""75 """Paint/asphalt intensity-separation settings."""
7676
77 enabled: bool = True77 enabled: bool = True
78 apply_filter: bool = True78 apply_filter: bool = True
Importance #4: src/iolabs_point_cloud_mask_clustering/_config.py @@ -93,33 +93,33 @@
93 clusters_per_page: int = pydantic.Field(default=3, ge=1)93 clusters_per_page: int = pydantic.Field(default=3, ge=1)
94 pdf_filename: str = "intensity_separation.pdf"94 pdf_filename: str = "intensity_separation.pdf"
9595
9696
97class RasterFrameConfig(config_loader.ConfigModel):97class MaskClusteringRasterFrameConfig(config_loader.ConfigModel):
98 """Tolerances used when reconstructing the raster frame."""98 """Tolerances used when reconstructing the raster frame."""
9999
100 margin_pixels: float = pydantic.Field(default=1.0, ge=0.0)100 margin_pixels: float = pydantic.Field(default=1.0, ge=0.0)
101 metadata_origin_tolerance_pixels: float = pydantic.Field(default=0.25, ge=0.0)101 metadata_origin_tolerance_pixels: float = pydantic.Field(default=0.25, ge=0.0)
102102
103103
104class GeometryConfig(config_loader.ConfigModel):104class MaskClusteringGeometryConfig(config_loader.ConfigModel):
105 """Which diagnostic geometry artifacts to write."""105 """Which diagnostic geometry artifacts to write."""
106106
107 write_geojson: bool = True107 write_geojson: bool = True
108 write_ply: bool = True108 write_ply: bool = True
109 simplify_tolerance_px: float = 0.0109 simplify_tolerance_px: float = 0.0
110110
111111
112class OutputConfig(config_loader.ConfigModel):112class MaskClusteringOutputConfig(config_loader.ConfigModel):
113 """Output directory and file-name layout."""113 """Output directory and file-name layout."""
114114
115 cluster_dir: str = "clusters_mask"115 cluster_dir: str = "clusters_mask"
116 cluster_prefix: str = "run6_cluster_"116 cluster_prefix: str = "run6_cluster_"
117 geometry_dir: str = "mask_geometry"117 geometry_dir: str = "mask_geometry"
118 manifest_filename: str = "mask_clustering_manifest.json"118 manifest_filename: str = "mask_clustering_manifest.json"
119119
120120
121class FileNamingConfig(config_loader.ConfigModel):121class MaskClusteringFileNamingConfig(config_loader.ConfigModel):
122 """How Step 3 inputs are discovered inside a segment directory."""122 """How Step 3 inputs are discovered inside a segment directory."""
123123
124 segment_points_suffix: str = "_run3_points.npz"124 segment_points_suffix: str = "_run3_points.npz"
125125
Importance #5: src/iolabs_point_cloud_mask_clustering/_config.py @@ -174,107 +176,86 @@
174176
175 Raises:177 Raises:
176 MaskClusteringConfigError: The mapping is not a valid configuration.178 MaskClusteringConfigError: The mapping is not a valid configuration.
177 """179 """
178 return _load_model(overrides=_as_mapping(config))180 return config_loader.validate_config(
179181 cls, config, context=_CONTEXT, error_cls=MaskClusteringConfigError
180
181def _as_mapping(config: Any) -> dict[str, Any]:
182 """Return *config* as a dict, rejecting values that are not mappings.
183
184 Raises:
185 MaskClusteringConfigError: *config* is not a mapping (``None`` included).
186 """
187 if not isinstance(config, Mapping):
188 raise MaskClusteringConfigError(
189 f"config must be a mapping, got {type(config).__name__}"
190 )182 )
191 return dict(config)
192183
193184
194def _load_model(overrides: dict[str, Any] | None = None) -> MaskClusteringConfig:185def _load_model(
195 """Merge *overrides* onto the packaged defaults and validate the result."""186 *,
187 overrides: Mapping[str, Any] | None = None,
188 config_path: str | Path | None = None,
189) -> MaskClusteringConfig:
190 """Merge a config file and *overrides* onto the packaged defaults and validate."""
191 merged: dict[str, Any] = dict(overrides or {})
192 if config_path is not None:
193 file_overrides = config_loader.load_json_overrides(
194 config_path, error_cls=MaskClusteringConfigError
195 )
196 merged = config_loader.deep_merge_dicts(file_overrides, merged)
197 logger.info("Config file applied: %s", config_path)
198 if merged:
199 logger.info("Config overrides applied: %s", ", ".join(sorted(merged)))
196 return config_loader.load_config(200 return config_loader.load_config(
197 MaskClusteringConfig,201 MaskClusteringConfig,
198 package=PACKAGE,202 package=_PACKAGE_NAME,
199 filename=DEFAULT_CONFIG_FILENAME,203 filename=_DEFAULT_FILENAME,
200 overrides=overrides,204 overrides=merged,
201 context="config",205 context=_CONTEXT,
202 error_cls=MaskClusteringConfigError,206 error_cls=MaskClusteringConfigError,
203 )207 )
204208
205209
206def _read_overrides(config_path: str | Path | None) -> dict[str, Any]:210def normalize_config(raw: Mapping[str, Any]) -> dict[str, Any]:
207 """Read a JSON override file, or return an empty mapping when there is none.211 """Validate *raw* and fill in the model defaults.
208
209 Raises:
210 MaskClusteringConfigError: The file is not valid JSON, or does not hold
211 a JSON object.
212 """
213 if config_path is None:
214 return {}
215 path = Path(config_path)
216 try:
217 with path.open(encoding="utf-8") as handle:
218 loaded = json.load(handle)
219 except json.JSONDecodeError as exc:
220 raise MaskClusteringConfigError(f"Invalid JSON in config file {path}: {exc}") from exc
221 if not isinstance(loaded, dict):
222 raise MaskClusteringConfigError(
223 f"Config file {path} must hold a JSON object, got {type(loaded).__name__}"
224 )
225 return loaded
226
227
228def normalize_config(raw: dict[str, Any]) -> dict[str, Any]:
229 """Merge *raw* onto the packaged defaults and validate the result.
230212
231 Args:213 Args:
232 raw: Partial configuration mapping.214 raw: Partial configuration mapping.
233215
234 Returns:216 Returns:
235 The merged, validated configuration.217 The validated configuration.
236218
237 Raises:219 Raises:
238 MaskClusteringConfigError: An unknown key or an out-of-range value.220 MaskClusteringConfigError: An unknown key or an out-of-range value.
239 """221 """
240 return _load_model(overrides=_as_mapping(raw)).model_dump()222 return MaskClusteringConfig.from_mapping(raw).model_dump()
241223
242224
243def load_config(config_path: str | Path | None = None) -> dict[str, Any]:225def load_config(config_path: str | Path | None = None) -> dict[str, Any]:
244 """Load a configuration JSON, or the packaged defaults when *config_path* is None.226 """Load the packaged defaults, merging a config file onto them when given.
245227
246 Args:228 Args:
247 config_path: Path to a JSON file holding partial overrides.229 config_path: Path to a JSON file holding partial overrides. It is
230 merged onto the packaged defaults rather than replacing them.
248231
249 Returns:232 Returns:
250 The merged, validated configuration.233 The merged, validated configuration.
251234
252 Raises:235 Raises:
253 MaskClusteringConfigError: A malformed config file, an unknown key or an236 MaskClusteringConfigError: A malformed config file, an unknown key or an
254 out-of-range value.237 out-of-range value.
255 """238 """
256 return _load_model(overrides=_read_overrides(config_path)).model_dump()239 return _load_model(config_path=config_path).model_dump()
257240
258241
259def build_config(242def build_config(
260 *,243 *,
261 overrides: dict[str, Any] | None = None,244 overrides: Mapping[str, Any] | None = None,
262 config_path: str | Path | None = None,245 config_path: str | Path | None = None,
263) -> dict[str, Any]:246) -> dict[str, Any]:
264 """Load a configuration file and apply in-memory overrides on top of it.247 """Load a configuration file and apply in-memory overrides on top of it.
265248
266 Args:249 Args:
267 overrides: Nested override mapping, e.g. from CLI ``--set`` flags.250 overrides: Nested override mapping, e.g. from CLI ``--set`` flags.
268 config_path: Path to a JSON file holding partial overrides.251 config_path: Path to a JSON file holding partial overrides. It is
252 merged onto the packaged defaults rather than replacing them.
269253
270 Returns:254 Returns:
271 The merged, validated configuration.255 The merged, validated configuration.
272256
273 Raises:257 Raises:
274 MaskClusteringConfigError: A malformed config file, an unknown key or an258 MaskClusteringConfigError: A malformed config file, an unknown key or an
275 out-of-range value.259 out-of-range value.
276 """260 """
277 merged = config_loader.deep_merge_dicts(261 return _load_model(overrides=overrides, config_path=config_path).model_dump()
278 _read_overrides(config_path), dict(overrides or {})
279 )
280 return _load_model(overrides=merged).model_dump()
Importance #6: src/iolabs_point_cloud_mask_clustering/_config.py @@ -1,22 +1,21 @@
1"""Load, merge, validate and type the mask-clustering configuration.1"""Mask-clustering configuration: packaged defaults, overrides and validation.
22
3The pydantic model tree below is the schema and mirrors the packaged JSON3The schema is `MaskClusteringConfig` (a `config_loader.ConfigModel`), mirroring
4default exactly: unknown keys fail, and every value is range-checked here rather4`mask_clustering.default.json` key for key.
5than at the point of use. Loading, deep-merging and validation are delegated to
6:mod:`iolabs.common.config_loader`.
75
8:func:`load_config` / :func:`build_config` keep returning plain dicts, because6Adding a config key means adding the field to the model and the same key to
9callers pass ``--set``-style overrides around as dicts and the run manifest7`mask_clustering.default.json` — nothing else. Unknown keys are rejected.
10embeds the normalized mapping verbatim. :class:`MaskClusteringConfig` is the
11typed view the pipeline and the CLI actually read, so no production code path
12indexes nested config dicts by string.
138
14Adding a config key means adding the field to the model here and the same key to9`normalize_config`, `load_config` and `build_config` return a plain ``dict``,
15``mask_clustering.default.json`` — nothing else.10because callers pass ``--set``-style overrides around as dicts and the run
11manifest embeds the normalized mapping verbatim; `MaskClusteringConfig.coerce`
12is the typed view the pipeline, the CLI and the overlay actually read, so no
13production code path indexes nested config dicts by string.
16"""14"""
1715
18import json16from __future__ import annotations
17
19from collections.abc import Mapping18from collections.abc import Mapping
20from pathlib import Path19from pathlib import Path
21from typing import Any, Literal20from typing import Any, Literal
2221
Importance #7: src/iolabs_point_cloud_mask_clustering/_config.py @@ -136,18 +136,20 @@
136 output: Output directory and file-name layout.136 output: Output directory and file-name layout.
137 file_naming: Step 3 input discovery settings.137 file_naming: Step 3 input discovery settings.
138 """138 """
139139
140 mask: MaskConfig = MaskConfig()140 mask: MaskClusteringMaskConfig = MaskClusteringMaskConfig()
141 clusters: ClustersConfig = ClustersConfig()141 clusters: MaskClusteringClustersConfig = MaskClusteringClustersConfig()
142 intensity_separation: IntensitySeparationConfig = IntensitySeparationConfig()142 intensity_separation: MaskClusteringIntensitySeparationConfig = (
143 raster_frame: RasterFrameConfig = RasterFrameConfig()143 MaskClusteringIntensitySeparationConfig()
144 geometry: GeometryConfig = GeometryConfig()144 )
145 output: OutputConfig = OutputConfig()145 raster_frame: MaskClusteringRasterFrameConfig = MaskClusteringRasterFrameConfig()
146 file_naming: FileNamingConfig = FileNamingConfig()146 geometry: MaskClusteringGeometryConfig = MaskClusteringGeometryConfig()
147 output: MaskClusteringOutputConfig = MaskClusteringOutputConfig()
148 file_naming: MaskClusteringFileNamingConfig = MaskClusteringFileNamingConfig()
147149
148 @classmethod150 @classmethod
149 def coerce(cls, config: "MaskClusteringConfig | dict[str, Any]") -> "MaskClusteringConfig":151 def coerce(cls, config: MaskClusteringConfig | Mapping[str, Any]) -> MaskClusteringConfig:
150 """Return *config* as a typed configuration, converting a mapping if needed.152 """Return *config* as a typed configuration, converting a mapping if needed.
151153
152 Args:154 Args:
153 config: An already-typed configuration, or a raw/normalized mapping.155 config: An already-typed configuration, or a raw/normalized mapping.
Importance #8: src/iolabs_point_cloud_mask_clustering/_config.py @@ -162,10 +164,10 @@
162 return config164 return config
163 return cls.from_mapping(config)165 return cls.from_mapping(config)
164166
165 @classmethod167 @classmethod
166 def from_mapping(cls, config: dict[str, Any]) -> "MaskClusteringConfig":168 def from_mapping(cls, config: Mapping[str, Any]) -> MaskClusteringConfig:
167 """Build the typed view, merging *config* onto the packaged defaults.169 """Validate *config* into the typed view, filling in model defaults.
168170
169 Args:171 Args:
170 config: A raw or already-normalized configuration mapping.172 config: A raw or already-normalized configuration mapping.
171173
Importance #9: src/iolabs_point_cloud_mask_clustering/__init__.py @@ -1,8 +1,15 @@
1"""Mask-to-point-cloud clustering for road-marking artifacts."""1"""Mask-to-point-cloud clustering for road-marking artifacts."""
22
3from importlib.metadata import PackageNotFoundError, version3from importlib.metadata import PackageNotFoundError, version
44
5from ._config import (
6 MaskClusteringConfig,
7 MaskClusteringConfigError,
8 build_config,
9 load_config,
10 normalize_config,
11)
5from .pipeline import process_segment12from .pipeline import process_segment
6from .raster_frame import RasterFrame13from .raster_frame import RasterFrame
7from .types import (14from .types import (
8 ComponentResult,15 ComponentResult,
Importance #10: src/iolabs_point_cloud_mask_clustering/__init__.py @@ -13,13 +20,18 @@
13)20)
1421
15__all__ = [22__all__ = [
16 "ComponentResult",23 "ComponentResult",
24 "MaskClusteringConfig",
25 "MaskClusteringConfigError",
17 "MaskComponent",26 "MaskComponent",
18 "PointChannels",27 "PointChannels",
19 "RasterFrame",28 "RasterFrame",
20 "SegmentResult",29 "SegmentResult",
21 "SegmentType",30 "SegmentType",
31 "build_config",
32 "load_config",
33 "normalize_config",
22 "process_segment",34 "process_segment",
23]35]
2436
25try:37try:
Importance #11: src/iolabs_point_cloud_mask_clustering/cli.py @@ -67,13 +67,15 @@
67 Returns:67 Returns:
68 The nested override mapping.68 The nested override mapping.
6969
70 Raises:70 Raises:
71 ValueError: An argument is not ``section.key=value`` (message:71 _config.MaskClusteringConfigError: An argument is not ``section.key=value`` (message:
72 ``Invalid --set override '...'. Expected SECTION.KEY=VALUE.``), or72 ``Invalid --set override '...'. Expected SECTION.KEY=VALUE.``), or
73 two arguments disagree about whether a path segment is a section.73 two arguments disagree about whether a path segment is a section.
74 """74 """
75 return config_loader.parse_set_overrides(values, nested=True, error_cls=ValueError)75 return config_loader.parse_set_overrides(
76 values, nested=True, error_cls=_config.MaskClusteringConfigError
77 )
7678
7779
78def _load_jobs(path: Path) -> list[dict[str, Any]]:80def _load_jobs(path: Path) -> list[dict[str, Any]]:
79 with path.open(encoding="utf-8") as handle:81 with path.open(encoding="utf-8") as handle:
Importance #12: tests/test_config.py @@ -1,52 +1,83 @@
1import json1import json
2from importlib import resources2from importlib import resources
33
4import pytest4import pytest
5from iolabs.common import config_loader
56
7import iolabs_point_cloud_mask_clustering as mask_clustering
6from iolabs_point_cloud_mask_clustering import _config8from iolabs_point_cloud_mask_clustering import _config
7from iolabs_point_cloud_mask_clustering._config import (
8 MaskClusteringConfigError,
9 build_config,
10 load_config,
11)
129
1310
14def test_recursive_overrides_work() -> None:11def _packaged_defaults() -> dict:
15 defaults = load_config()12 return json.loads(
16 config = build_config(overrides={"clusters": {"warn_below_points": 250}})13 resources.files("iolabs_point_cloud_mask_clustering")
14 .joinpath("mask_clustering.default.json")
15 .read_text(encoding="utf-8")
16 )
17
18
19def test_model_defaults_match_packaged_json() -> None:
20 """The model tree and the packaged JSON must stay in lock-step."""
21 assert _config.MaskClusteringConfig().model_dump() == _packaged_defaults()
22
23
24def test_load_config_returns_packaged_defaults() -> None:
25 assert mask_clustering.load_config() == _packaged_defaults()
26
27
28def test_error_class_is_config_error() -> None:
29 assert issubclass(_config.MaskClusteringConfigError, config_loader.ConfigError)
30 assert issubclass(_config.MaskClusteringConfigError, ValueError)
31
32
33def test_unknown_top_level_key_is_rejected() -> None:
34 with pytest.raises(_config.MaskClusteringConfigError, match="unknown"):
35 mask_clustering.build_config(overrides={"unknown": 1})
36
37
38def test_unknown_nested_key_is_rejected() -> None:
39 with pytest.raises(_config.MaskClusteringConfigError, match="unknown"):
40 mask_clustering.build_config(overrides={"mask": {"unknown": 1}})
41
42
43def test_overrides_deep_merge_onto_defaults() -> None:
44 defaults = mask_clustering.load_config()
45 config = mask_clustering.build_config(overrides={"clusters": {"warn_below_points": 250}})
17 assert config["clusters"]["warn_below_points"] == 25046 assert config["clusters"]["warn_below_points"] == 250
18 assert (47 assert (
19 config["clusters"]["min_points_per_cluster"]48 config["clusters"]["min_points_per_cluster"]
20 == defaults["clusters"]["min_points_per_cluster"]49 == defaults["clusters"]["min_points_per_cluster"]
21 )50 )
2251
2352
53def test_set_override_coercion_and_rejection() -> None:
54 overrides = config_loader.parse_set_overrides(
55 ["mask.vector_stroke_px=6", "geometry.write_ply=on"],
56 nested=True,
57 error_cls=_config.MaskClusteringConfigError,
58 )
59 config = mask_clustering.build_config(overrides=overrides)
60 assert config["mask"]["vector_stroke_px"] == 6
61 assert config["geometry"]["write_ply"] is True
62 with pytest.raises(_config.MaskClusteringConfigError):
63 mask_clustering.build_config(overrides={"geometry": {"write_ply": "flase"}})
64 with pytest.raises(_config.MaskClusteringConfigError):
65 mask_clustering.build_config(overrides={"mask": {"vector_stroke_px": True}})
66
67
24@pytest.mark.parametrize(68@pytest.mark.parametrize(
25 "overrides",69 "overrides",
26 [70 [
27 {"unknown": 1},
28 {"mask": {"unknown": 1}},
29 {"clusters": {"min_points_per_cluster": 201, "warn_below_points": 200}},71 {"clusters": {"min_points_per_cluster": 201, "warn_below_points": 200}},
30 {"mask": {"connectivity": 6}},72 {"mask": {"connectivity": 6}},
31 {"mask": {"vector_stroke_px": 0}},73 {"mask": {"vector_stroke_px": 0}},
32 {"raster_frame": {"metadata_origin_tolerance_pixels": -1}},74 {"raster_frame": {"metadata_origin_tolerance_pixels": -1}},
33 ],75 ],
34)76)
35def test_invalid_configuration_fails(overrides: dict) -> None:77def test_invalid_configuration_fails(overrides: dict) -> None:
36 with pytest.raises(MaskClusteringConfigError):78 with pytest.raises(_config.MaskClusteringConfigError):
37 build_config(overrides=overrides)79 mask_clustering.build_config(overrides=overrides)
38
39
40def test_defaults_match_packaged_json() -> None:
41 """The model tree and the packaged JSON must stay in lock-step."""
42 packaged = json.loads(
43 resources.files(_config.PACKAGE)
44 .joinpath(_config.DEFAULT_CONFIG_FILENAME)
45 .read_text(encoding="utf-8")
46 )
47 assert _config.MaskClusteringConfig().model_dump() == packaged
48 assert load_config() == packaged
4980
5081
51@pytest.mark.parametrize(82@pytest.mark.parametrize(
52 "overrides",83 "overrides",
Importance #13: tests/test_config.py @@ -60,30 +91,30 @@
60 {"intensity_separation": {"unknown": 1}},91 {"intensity_separation": {"unknown": 1}},
61 ],92 ],
62)93)
63def test_invalid_intensity_separation_fails(overrides: dict) -> None:94def test_invalid_intensity_separation_fails(overrides: dict) -> None:
64 with pytest.raises(MaskClusteringConfigError):95 with pytest.raises(_config.MaskClusteringConfigError):
65 build_config(overrides=overrides)96 mask_clustering.build_config(overrides=overrides)
6697
6798
68def test_malformed_config_file_fails(tmp_path) -> None:99def test_malformed_config_file_fails(tmp_path) -> None:
69 path = tmp_path / "cfg.json"100 path = tmp_path / "cfg.json"
70 path.write_text("{oops", encoding="utf-8")101 path.write_text("{oops", encoding="utf-8")
71 with pytest.raises(MaskClusteringConfigError):102 with pytest.raises(_config.MaskClusteringConfigError):
72 load_config(path)103 mask_clustering.load_config(path)
73104
74105
75def test_non_object_config_file_fails(tmp_path) -> None:106def test_non_object_config_file_fails(tmp_path) -> None:
76 path = tmp_path / "cfg.json"107 path = tmp_path / "cfg.json"
77 path.write_text("[1, 2]", encoding="utf-8")108 path.write_text("[1, 2]", encoding="utf-8")
78 with pytest.raises(MaskClusteringConfigError):109 with pytest.raises(_config.MaskClusteringConfigError):
79 build_config(config_path=path)110 mask_clustering.build_config(config_path=path)
80111
81112
82def test_config_file_and_overrides_merge_onto_defaults(tmp_path) -> None:113def test_config_file_and_overrides_merge_onto_defaults(tmp_path) -> None:
83 path = tmp_path / "cfg.json"114 path = tmp_path / "cfg.json"
84 path.write_text('{"clusters": {"warn_below_points": 500}}', encoding="utf-8")115 path.write_text('{"clusters": {"warn_below_points": 500}}', encoding="utf-8")
85 config = build_config(116 config = mask_clustering.build_config(
86 config_path=path, overrides={"clusters": {"min_points_per_cluster": 400}}117 config_path=path, overrides={"clusters": {"min_points_per_cluster": 400}}
87 )118 )
88 assert config["clusters"] == {119 assert config["clusters"] == {
89 "min_points_per_cluster": 400,120 "min_points_per_cluster": 400,
Importance #14: tests/test_config.py @@ -91,26 +122,22 @@
91 }122 }
92 assert config["mask"]["connectivity"] == 8123 assert config["mask"]["connectivity"] == 8
93124
94125
95def test_scalar_coercion_and_bool_rejection() -> None:
96 assert build_config(overrides={"mask": {"vector_stroke_px": "6"}})["mask"][
97 "vector_stroke_px"
98 ] == 6
99 with pytest.raises(MaskClusteringConfigError):
100 build_config(overrides={"mask": {"vector_stroke_px": True}})
101
102
103@pytest.mark.parametrize("value", [8, 8.0, "8", 4])126@pytest.mark.parametrize("value", [8, 8.0, "8", 4])
104def test_connectivity_accepts_legacy_int_spellings(value: object) -> None:127def test_connectivity_accepts_legacy_int_spellings(value: object) -> None:
105 """JSON/CLI spellings of an int reach ``mask.connectivity`` as an int."""128 """JSON/CLI spellings of an int reach ``mask.connectivity`` as an int."""
106 config = build_config(overrides={"mask": {"connectivity": value}})129 config = mask_clustering.build_config(overrides={"mask": {"connectivity": value}})
107 assert config["mask"]["connectivity"] == int(value) # type: ignore[arg-type]130 assert config["mask"]["connectivity"] == int(value) # type: ignore[arg-type]
108131
109132
110@pytest.mark.parametrize("config", [None, [], "", 0, False])133@pytest.mark.parametrize("config", [None, [], "", 0, False])
111def test_non_mapping_config_is_rejected(config: object) -> None:134def test_non_mapping_config_is_rejected(config: object) -> None:
112 """A non-mapping is an error, not a silent "use the defaults"."""135 """A non-mapping is an error, not a silent "use the defaults"."""
113 with pytest.raises(MaskClusteringConfigError):136 with pytest.raises(_config.MaskClusteringConfigError):
114 _config.MaskClusteringConfig.coerce(config) # type: ignore[arg-type]137 _config.MaskClusteringConfig.coerce(config) # type: ignore[arg-type]
115 with pytest.raises(MaskClusteringConfigError):138 with pytest.raises(_config.MaskClusteringConfigError):
116 _config.normalize_config(config) # type: ignore[arg-type]139 mask_clustering.normalize_config(config) # type: ignore[arg-type]
140
141
142def test_normalize_config_fills_model_defaults() -> None:
143 assert mask_clustering.normalize_config({}) == _packaged_defaults()
Importance #15: AGENTS.md @@ -8,12 +8,12 @@
8## Conventions8## Conventions
99
10- Library code lives in `src/iolabs_point_cloud_mask_clustering/`.10- Library code lives in `src/iolabs_point_cloud_mask_clustering/`.
11- Runtime wrappers live in `scripts/` and are run from the repository root.11- Runtime wrappers live in `scripts/` and are run from the repository root.
12- Configuration is a pydantic model tree in `_config.py` derived from12- Configuration is the `MaskClusteringConfig` model tree in `_config.py`
13 `iolabs.common.config_loader.ConfigModel`; it is strict, unknown keys fail.13 (`config_loader.ConfigModel`), re-exported from the package root; it is
14 A new config key means one field on the model plus the same key in14 strict, unknown keys fail. A new config key means one field on the model plus
15 `mask_clustering.default.json`.15 the same key in `mask_clustering.default.json`.
16- Preserve Step 3 XYZ unchanged and use `ColorIntensityData` for aligned channels.16- Preserve Step 3 XYZ unchanged and use `ColorIntensityData` for aligned channels.
17- Never modify a segment's existing `clusters/` directory.17- Never modify a segment's existing `clusters/` directory.
18- Use `iolabs.logstash.get_props_logger` with `_log_props.LOG_PROPS`.18- Use `iolabs.logstash.get_props_logger` with `_log_props.LOG_PROPS`.
19- Run `uv run --extra dev pytest -q` and `uv build` before completion.19- Run `uv run --extra dev pytest -q` and `uv build` before completion.
Importance #16: README.md @@ -73,17 +73,18 @@
73`0 <= min_points_per_cluster <= warn_below_points`.73`0 <= min_points_per_cluster <= warn_below_points`.
7474
75## Configuration75## Configuration
7676
77Defaults live in `src/iolabs_point_cloud_mask_clustering/mask_clustering.default.json`77Defaults live in `src/iolabs_point_cloud_mask_clustering/mask_clustering.default.json`.
78and are typed by the pydantic model tree in `_config.py`78The schema is `MaskClusteringConfig` in `_config.py` (a
79(`iolabs.common.config_loader.ConfigModel`). Unknown keys and out-of-range values79`config_loader.ConfigModel`); nested JSON sections are nested models and unknown
80fail loudly; `--config` files and `--set` overrides are deep-merged onto the80keys are rejected. **To add a config key: add the field (with its type, default
81defaults and re-validated.81and any `Field` range) to the model and the same key with the same default to the
8282JSON — nothing else.** `normalize_config`, `load_config` and `build_config`
83Adding a config key: add the field (with its type, default and any range83(re-exported from the package root) return a plain `dict`;
84constraint) to the matching model in `_config.py`, and add the same key to84`MaskClusteringConfig.coerce` is the typed view used inside the package. A
85`mask_clustering.default.json`. Nothing else.85`--config` file is deep-merged onto the packaged defaults, and runtime overrides
86come from repeatable `--set KEY=VALUE`, never repo-local JSON.
8687
87## CLI88## CLI
8889
89Install development dependencies:90Install development dependencies:
Importance #17: README.md @@ -73,17 +73,18 @@
73`0 <= min_points_per_cluster <= warn_below_points`.73`0 <= min_points_per_cluster <= warn_below_points`.
7474
75## Configuration75## Configuration
7676
77Defaults live in `src/iolabs_point_cloud_mask_clustering/mask_clustering.default.json`77Defaults live in `src/iolabs_point_cloud_mask_clustering/mask_clustering.default.json`.
78and are typed by the pydantic model tree in `_config.py`78The schema is `MaskClusteringConfig` in `_config.py` (a
79(`iolabs.common.config_loader.ConfigModel`). Unknown keys and out-of-range values79`config_loader.ConfigModel`); nested JSON sections are nested models and unknown
80fail loudly; `--config` files and `--set` overrides are deep-merged onto the80keys are rejected. **To add a config key: add the field (with its type, default
81defaults and re-validated.81and any `Field` range) to the model and the same key with the same default to the
8282JSON — nothing else.** `normalize_config`, `load_config` and `build_config`
83Adding a config key: add the field (with its type, default and any range83(re-exported from the package root) return a plain `dict`;
84constraint) to the matching model in `_config.py`, and add the same key to84`MaskClusteringConfig.coerce` is the typed view used inside the package. A
85`mask_clustering.default.json`. Nothing else.85`--config` file is deep-merged onto the packaged defaults, and runtime overrides
86come from repeatable `--set KEY=VALUE`, never repo-local JSON.
8687
87## CLI88## CLI
8889
89Install development dependencies:90Install development dependencies:
Importance #18: src/iolabs_point_cloud_mask_clustering/__init__.py @@ -1,8 +1,15 @@
1"""Mask-to-point-cloud clustering for road-marking artifacts."""1"""Mask-to-point-cloud clustering for road-marking artifacts."""
22
3from importlib.metadata import PackageNotFoundError, version3from importlib.metadata import PackageNotFoundError, version
44
5from ._config import (
6 MaskClusteringConfig,
7 MaskClusteringConfigError,
8 build_config,
9 load_config,
10 normalize_config,
11)
5from .pipeline import process_segment12from .pipeline import process_segment
6from .raster_frame import RasterFrame13from .raster_frame import RasterFrame
7from .types import (14from .types import (
8 ComponentResult,15 ComponentResult,
Importance #19: src/iolabs_point_cloud_mask_clustering/__init__.py @@ -13,13 +20,18 @@
13)20)
1421
15__all__ = [22__all__ = [
16 "ComponentResult",23 "ComponentResult",
24 "MaskClusteringConfig",
25 "MaskClusteringConfigError",
17 "MaskComponent",26 "MaskComponent",
18 "PointChannels",27 "PointChannels",
19 "RasterFrame",28 "RasterFrame",
20 "SegmentResult",29 "SegmentResult",
21 "SegmentType",30 "SegmentType",
31 "build_config",
32 "load_config",
33 "normalize_config",
22 "process_segment",34 "process_segment",
23]35]
2436
25try:37try:
Importance #20: src/iolabs_point_cloud_mask_clustering/_config.py @@ -1,22 +1,21 @@
1"""Load, merge, validate and type the mask-clustering configuration.1"""Mask-clustering configuration: packaged defaults, overrides and validation.
22
3The pydantic model tree below is the schema and mirrors the packaged JSON3The schema is `MaskClusteringConfig` (a `config_loader.ConfigModel`), mirroring
4default exactly: unknown keys fail, and every value is range-checked here rather4`mask_clustering.default.json` key for key.
5than at the point of use. Loading, deep-merging and validation are delegated to
6:mod:`iolabs.common.config_loader`.
75
8:func:`load_config` / :func:`build_config` keep returning plain dicts, because6Adding a config key means adding the field to the model and the same key to
9callers pass ``--set``-style overrides around as dicts and the run manifest7`mask_clustering.default.json` — nothing else. Unknown keys are rejected.
10embeds the normalized mapping verbatim. :class:`MaskClusteringConfig` is the
11typed view the pipeline and the CLI actually read, so no production code path
12indexes nested config dicts by string.
138
14Adding a config key means adding the field to the model here and the same key to9`normalize_config`, `load_config` and `build_config` return a plain ``dict``,
15``mask_clustering.default.json`` — nothing else.10because callers pass ``--set``-style overrides around as dicts and the run
11manifest embeds the normalized mapping verbatim; `MaskClusteringConfig.coerce`
12is the typed view the pipeline, the CLI and the overlay actually read, so no
13production code path indexes nested config dicts by string.
16"""14"""
1715
18import json16from __future__ import annotations
17
19from collections.abc import Mapping18from collections.abc import Mapping
20from pathlib import Path19from pathlib import Path
21from typing import Any, Literal20from typing import Any, Literal
2221
Importance #21: src/iolabs_point_cloud_mask_clustering/_config.py @@ -27,17 +26,18 @@
27from ._log_props import LOG_PROPS26from ._log_props import LOG_PROPS
2827
29logger = get_props_logger(__name__, LOG_PROPS)28logger = get_props_logger(__name__, LOG_PROPS)
3029
31PACKAGE = "iolabs_point_cloud_mask_clustering"30_PACKAGE_NAME = "iolabs_point_cloud_mask_clustering"
32DEFAULT_CONFIG_FILENAME = "mask_clustering.default.json"31_DEFAULT_FILENAME = "mask_clustering.default.json"
32_CONTEXT = "mask clustering config"
3333
3434
35class MaskClusteringConfigError(config_loader.ConfigError):35class MaskClusteringConfigError(config_loader.ConfigError):
36 """Raised when mask-clustering configuration is invalid."""36 """Raised when mask clustering config contains unsupported keys or values."""
3737
3838
39class MaskConfig(config_loader.ConfigModel):39class MaskClusteringMaskConfig(config_loader.ConfigModel):
40 """Mask rasterisation and labelling settings."""40 """Mask rasterisation and labelling settings."""
4141
42 background_class: int = 042 background_class: int = 0
43 solid_class: int = 143 solid_class: int = 1
Importance #22: src/iolabs_point_cloud_mask_clustering/_config.py @@ -53,16 +53,16 @@
53 raise ValueError("mask.connectivity must be 4 or 8")53 raise ValueError("mask.connectivity must be 4 or 8")
54 return value54 return value
5555
5656
57class ClustersConfig(config_loader.ConfigModel):57class MaskClusteringClustersConfig(config_loader.ConfigModel):
58 """Sparse-cluster thresholds."""58 """Sparse-cluster thresholds."""
5959
60 min_points_per_cluster: int = pydantic.Field(default=20, ge=0)60 min_points_per_cluster: int = pydantic.Field(default=20, ge=0)
61 warn_below_points: int = pydantic.Field(default=200, ge=0)61 warn_below_points: int = pydantic.Field(default=200, ge=0)
6262
63 @pydantic.model_validator(mode="after")63 @pydantic.model_validator(mode="after")
64 def _check_thresholds(self) -> "ClustersConfig":64 def _check_thresholds(self) -> MaskClusteringClustersConfig:
65 """Reject a minimum above the warning threshold."""65 """Reject a minimum above the warning threshold."""
66 if self.min_points_per_cluster > self.warn_below_points:66 if self.min_points_per_cluster > self.warn_below_points:
67 raise ValueError(67 raise ValueError(
68 "clusters thresholds must satisfy "68 "clusters thresholds must satisfy "
Importance #23: src/iolabs_point_cloud_mask_clustering/_config.py @@ -70,9 +70,9 @@
70 )70 )
71 return self71 return self
7272
7373
74class IntensitySeparationConfig(config_loader.ConfigModel):74class MaskClusteringIntensitySeparationConfig(config_loader.ConfigModel):
75 """Paint/asphalt intensity-separation settings."""75 """Paint/asphalt intensity-separation settings."""
7676
77 enabled: bool = True77 enabled: bool = True
78 apply_filter: bool = True78 apply_filter: bool = True
Importance #24: src/iolabs_point_cloud_mask_clustering/_config.py @@ -93,33 +93,33 @@
93 clusters_per_page: int = pydantic.Field(default=3, ge=1)93 clusters_per_page: int = pydantic.Field(default=3, ge=1)
94 pdf_filename: str = "intensity_separation.pdf"94 pdf_filename: str = "intensity_separation.pdf"
9595
9696
97class RasterFrameConfig(config_loader.ConfigModel):97class MaskClusteringRasterFrameConfig(config_loader.ConfigModel):
98 """Tolerances used when reconstructing the raster frame."""98 """Tolerances used when reconstructing the raster frame."""
9999
100 margin_pixels: float = pydantic.Field(default=1.0, ge=0.0)100 margin_pixels: float = pydantic.Field(default=1.0, ge=0.0)
101 metadata_origin_tolerance_pixels: float = pydantic.Field(default=0.25, ge=0.0)101 metadata_origin_tolerance_pixels: float = pydantic.Field(default=0.25, ge=0.0)
102102
103103
104class GeometryConfig(config_loader.ConfigModel):104class MaskClusteringGeometryConfig(config_loader.ConfigModel):
105 """Which diagnostic geometry artifacts to write."""105 """Which diagnostic geometry artifacts to write."""
106106
107 write_geojson: bool = True107 write_geojson: bool = True
108 write_ply: bool = True108 write_ply: bool = True
109 simplify_tolerance_px: float = 0.0109 simplify_tolerance_px: float = 0.0
110110
111111
112class OutputConfig(config_loader.ConfigModel):112class MaskClusteringOutputConfig(config_loader.ConfigModel):
113 """Output directory and file-name layout."""113 """Output directory and file-name layout."""
114114
115 cluster_dir: str = "clusters_mask"115 cluster_dir: str = "clusters_mask"
116 cluster_prefix: str = "run6_cluster_"116 cluster_prefix: str = "run6_cluster_"
117 geometry_dir: str = "mask_geometry"117 geometry_dir: str = "mask_geometry"
118 manifest_filename: str = "mask_clustering_manifest.json"118 manifest_filename: str = "mask_clustering_manifest.json"
119119
120120
121class FileNamingConfig(config_loader.ConfigModel):121class MaskClusteringFileNamingConfig(config_loader.ConfigModel):
122 """How Step 3 inputs are discovered inside a segment directory."""122 """How Step 3 inputs are discovered inside a segment directory."""
123123
124 segment_points_suffix: str = "_run3_points.npz"124 segment_points_suffix: str = "_run3_points.npz"
125125
Importance #25: src/iolabs_point_cloud_mask_clustering/_config.py @@ -136,18 +136,20 @@
136 output: Output directory and file-name layout.136 output: Output directory and file-name layout.
137 file_naming: Step 3 input discovery settings.137 file_naming: Step 3 input discovery settings.
138 """138 """
139139
140 mask: MaskConfig = MaskConfig()140 mask: MaskClusteringMaskConfig = MaskClusteringMaskConfig()
141 clusters: ClustersConfig = ClustersConfig()141 clusters: MaskClusteringClustersConfig = MaskClusteringClustersConfig()
142 intensity_separation: IntensitySeparationConfig = IntensitySeparationConfig()142 intensity_separation: MaskClusteringIntensitySeparationConfig = (
143 raster_frame: RasterFrameConfig = RasterFrameConfig()143 MaskClusteringIntensitySeparationConfig()
144 geometry: GeometryConfig = GeometryConfig()144 )
145 output: OutputConfig = OutputConfig()145 raster_frame: MaskClusteringRasterFrameConfig = MaskClusteringRasterFrameConfig()
146 file_naming: FileNamingConfig = FileNamingConfig()146 geometry: MaskClusteringGeometryConfig = MaskClusteringGeometryConfig()
147 output: MaskClusteringOutputConfig = MaskClusteringOutputConfig()
148 file_naming: MaskClusteringFileNamingConfig = MaskClusteringFileNamingConfig()
147149
148 @classmethod150 @classmethod
149 def coerce(cls, config: "MaskClusteringConfig | dict[str, Any]") -> "MaskClusteringConfig":151 def coerce(cls, config: MaskClusteringConfig | Mapping[str, Any]) -> MaskClusteringConfig:
150 """Return *config* as a typed configuration, converting a mapping if needed.152 """Return *config* as a typed configuration, converting a mapping if needed.
151153
152 Args:154 Args:
153 config: An already-typed configuration, or a raw/normalized mapping.155 config: An already-typed configuration, or a raw/normalized mapping.
Importance #26: src/iolabs_point_cloud_mask_clustering/_config.py @@ -162,10 +164,10 @@
162 return config164 return config
163 return cls.from_mapping(config)165 return cls.from_mapping(config)
164166
165 @classmethod167 @classmethod
166 def from_mapping(cls, config: dict[str, Any]) -> "MaskClusteringConfig":168 def from_mapping(cls, config: Mapping[str, Any]) -> MaskClusteringConfig:
167 """Build the typed view, merging *config* onto the packaged defaults.169 """Validate *config* into the typed view, filling in model defaults.
168170
169 Args:171 Args:
170 config: A raw or already-normalized configuration mapping.172 config: A raw or already-normalized configuration mapping.
171173
Importance #27: src/iolabs_point_cloud_mask_clustering/_config.py @@ -174,107 +176,86 @@
174176
175 Raises:177 Raises:
176 MaskClusteringConfigError: The mapping is not a valid configuration.178 MaskClusteringConfigError: The mapping is not a valid configuration.
177 """179 """
178 return _load_model(overrides=_as_mapping(config))180 return config_loader.validate_config(
179181 cls, config, context=_CONTEXT, error_cls=MaskClusteringConfigError
180
181def _as_mapping(config: Any) -> dict[str, Any]:
182 """Return *config* as a dict, rejecting values that are not mappings.
183
184 Raises:
185 MaskClusteringConfigError: *config* is not a mapping (``None`` included).
186 """
187 if not isinstance(config, Mapping):
188 raise MaskClusteringConfigError(
189 f"config must be a mapping, got {type(config).__name__}"
190 )182 )
191 return dict(config)
192183
193184
194def _load_model(overrides: dict[str, Any] | None = None) -> MaskClusteringConfig:185def _load_model(
195 """Merge *overrides* onto the packaged defaults and validate the result."""186 *,
187 overrides: Mapping[str, Any] | None = None,
188 config_path: str | Path | None = None,
189) -> MaskClusteringConfig:
190 """Merge a config file and *overrides* onto the packaged defaults and validate."""
191 merged: dict[str, Any] = dict(overrides or {})
192 if config_path is not None:
193 file_overrides = config_loader.load_json_overrides(
194 config_path, error_cls=MaskClusteringConfigError
195 )
196 merged = config_loader.deep_merge_dicts(file_overrides, merged)
197 logger.info("Config file applied: %s", config_path)
198 if merged:
199 logger.info("Config overrides applied: %s", ", ".join(sorted(merged)))
196 return config_loader.load_config(200 return config_loader.load_config(
197 MaskClusteringConfig,201 MaskClusteringConfig,
198 package=PACKAGE,202 package=_PACKAGE_NAME,
199 filename=DEFAULT_CONFIG_FILENAME,203 filename=_DEFAULT_FILENAME,
200 overrides=overrides,204 overrides=merged,
201 context="config",205 context=_CONTEXT,
202 error_cls=MaskClusteringConfigError,206 error_cls=MaskClusteringConfigError,
203 )207 )
204208
205209
206def _read_overrides(config_path: str | Path | None) -> dict[str, Any]:210def normalize_config(raw: Mapping[str, Any]) -> dict[str, Any]:
207 """Read a JSON override file, or return an empty mapping when there is none.211 """Validate *raw* and fill in the model defaults.
208
209 Raises:
210 MaskClusteringConfigError: The file is not valid JSON, or does not hold
211 a JSON object.
212 """
213 if config_path is None:
214 return {}
215 path = Path(config_path)
216 try:
217 with path.open(encoding="utf-8") as handle:
218 loaded = json.load(handle)
219 except json.JSONDecodeError as exc:
220 raise MaskClusteringConfigError(f"Invalid JSON in config file {path}: {exc}") from exc
221 if not isinstance(loaded, dict):
222 raise MaskClusteringConfigError(
223 f"Config file {path} must hold a JSON object, got {type(loaded).__name__}"
224 )
225 return loaded
226
227
228def normalize_config(raw: dict[str, Any]) -> dict[str, Any]:
229 """Merge *raw* onto the packaged defaults and validate the result.
230212
231 Args:213 Args:
232 raw: Partial configuration mapping.214 raw: Partial configuration mapping.
233215
234 Returns:216 Returns:
235 The merged, validated configuration.217 The validated configuration.
236218
237 Raises:219 Raises:
238 MaskClusteringConfigError: An unknown key or an out-of-range value.220 MaskClusteringConfigError: An unknown key or an out-of-range value.
239 """221 """
240 return _load_model(overrides=_as_mapping(raw)).model_dump()222 return MaskClusteringConfig.from_mapping(raw).model_dump()
241223
242224
243def load_config(config_path: str | Path | None = None) -> dict[str, Any]:225def load_config(config_path: str | Path | None = None) -> dict[str, Any]:
244 """Load a configuration JSON, or the packaged defaults when *config_path* is None.226 """Load the packaged defaults, merging a config file onto them when given.
245227
246 Args:228 Args:
247 config_path: Path to a JSON file holding partial overrides.229 config_path: Path to a JSON file holding partial overrides. It is
230 merged onto the packaged defaults rather than replacing them.
248231
249 Returns:232 Returns:
250 The merged, validated configuration.233 The merged, validated configuration.
251234
252 Raises:235 Raises:
253 MaskClusteringConfigError: A malformed config file, an unknown key or an236 MaskClusteringConfigError: A malformed config file, an unknown key or an
254 out-of-range value.237 out-of-range value.
255 """238 """
256 return _load_model(overrides=_read_overrides(config_path)).model_dump()239 return _load_model(config_path=config_path).model_dump()
257240
258241
259def build_config(242def build_config(
260 *,243 *,
261 overrides: dict[str, Any] | None = None,244 overrides: Mapping[str, Any] | None = None,
262 config_path: str | Path | None = None,245 config_path: str | Path | None = None,
263) -> dict[str, Any]:246) -> dict[str, Any]:
264 """Load a configuration file and apply in-memory overrides on top of it.247 """Load a configuration file and apply in-memory overrides on top of it.
265248
266 Args:249 Args:
267 overrides: Nested override mapping, e.g. from CLI ``--set`` flags.250 overrides: Nested override mapping, e.g. from CLI ``--set`` flags.
268 config_path: Path to a JSON file holding partial overrides.251 config_path: Path to a JSON file holding partial overrides. It is
252 merged onto the packaged defaults rather than replacing them.
269253
270 Returns:254 Returns:
271 The merged, validated configuration.255 The merged, validated configuration.
272256
273 Raises:257 Raises:
274 MaskClusteringConfigError: A malformed config file, an unknown key or an258 MaskClusteringConfigError: A malformed config file, an unknown key or an
275 out-of-range value.259 out-of-range value.
276 """260 """
277 merged = config_loader.deep_merge_dicts(261 return _load_model(overrides=overrides, config_path=config_path).model_dump()
278 _read_overrides(config_path), dict(overrides or {})
279 )
280 return _load_model(overrides=merged).model_dump()
Importance #28: src/iolabs_point_cloud_mask_clustering/cli.py @@ -67,13 +67,15 @@
67 Returns:67 Returns:
68 The nested override mapping.68 The nested override mapping.
6969
70 Raises:70 Raises:
71 ValueError: An argument is not ``section.key=value`` (message:71 _config.MaskClusteringConfigError: An argument is not ``section.key=value`` (message:
72 ``Invalid --set override '...'. Expected SECTION.KEY=VALUE.``), or72 ``Invalid --set override '...'. Expected SECTION.KEY=VALUE.``), or
73 two arguments disagree about whether a path segment is a section.73 two arguments disagree about whether a path segment is a section.
74 """74 """
75 return config_loader.parse_set_overrides(values, nested=True, error_cls=ValueError)75 return config_loader.parse_set_overrides(
76 values, nested=True, error_cls=_config.MaskClusteringConfigError
77 )
7678
7779
78def _load_jobs(path: Path) -> list[dict[str, Any]]:80def _load_jobs(path: Path) -> list[dict[str, Any]]:
79 with path.open(encoding="utf-8") as handle:81 with path.open(encoding="utf-8") as handle:
Importance #29: tests/test_config.py @@ -1,52 +1,83 @@
1import json1import json
2from importlib import resources2from importlib import resources
33
4import pytest4import pytest
5from iolabs.common import config_loader
56
7import iolabs_point_cloud_mask_clustering as mask_clustering
6from iolabs_point_cloud_mask_clustering import _config8from iolabs_point_cloud_mask_clustering import _config
7from iolabs_point_cloud_mask_clustering._config import (
8 MaskClusteringConfigError,
9 build_config,
10 load_config,
11)
129
1310
14def test_recursive_overrides_work() -> None:11def _packaged_defaults() -> dict:
15 defaults = load_config()12 return json.loads(
16 config = build_config(overrides={"clusters": {"warn_below_points": 250}})13 resources.files("iolabs_point_cloud_mask_clustering")
14 .joinpath("mask_clustering.default.json")
15 .read_text(encoding="utf-8")
16 )
17
18
19def test_model_defaults_match_packaged_json() -> None:
20 """The model tree and the packaged JSON must stay in lock-step."""
21 assert _config.MaskClusteringConfig().model_dump() == _packaged_defaults()
22
23
24def test_load_config_returns_packaged_defaults() -> None:
25 assert mask_clustering.load_config() == _packaged_defaults()
26
27
28def test_error_class_is_config_error() -> None:
29 assert issubclass(_config.MaskClusteringConfigError, config_loader.ConfigError)
30 assert issubclass(_config.MaskClusteringConfigError, ValueError)
31
32
33def test_unknown_top_level_key_is_rejected() -> None:
34 with pytest.raises(_config.MaskClusteringConfigError, match="unknown"):
35 mask_clustering.build_config(overrides={"unknown": 1})
36
37
38def test_unknown_nested_key_is_rejected() -> None:
39 with pytest.raises(_config.MaskClusteringConfigError, match="unknown"):
40 mask_clustering.build_config(overrides={"mask": {"unknown": 1}})
41
42
43def test_overrides_deep_merge_onto_defaults() -> None:
44 defaults = mask_clustering.load_config()
45 config = mask_clustering.build_config(overrides={"clusters": {"warn_below_points": 250}})
17 assert config["clusters"]["warn_below_points"] == 25046 assert config["clusters"]["warn_below_points"] == 250
18 assert (47 assert (
19 config["clusters"]["min_points_per_cluster"]48 config["clusters"]["min_points_per_cluster"]
20 == defaults["clusters"]["min_points_per_cluster"]49 == defaults["clusters"]["min_points_per_cluster"]
21 )50 )
2251
2352
53def test_set_override_coercion_and_rejection() -> None:
54 overrides = config_loader.parse_set_overrides(
55 ["mask.vector_stroke_px=6", "geometry.write_ply=on"],
56 nested=True,
57 error_cls=_config.MaskClusteringConfigError,
58 )
59 config = mask_clustering.build_config(overrides=overrides)
60 assert config["mask"]["vector_stroke_px"] == 6
61 assert config["geometry"]["write_ply"] is True
62 with pytest.raises(_config.MaskClusteringConfigError):
63 mask_clustering.build_config(overrides={"geometry": {"write_ply": "flase"}})
64 with pytest.raises(_config.MaskClusteringConfigError):
65 mask_clustering.build_config(overrides={"mask": {"vector_stroke_px": True}})
66
67
24@pytest.mark.parametrize(68@pytest.mark.parametrize(
25 "overrides",69 "overrides",
26 [70 [
27 {"unknown": 1},
28 {"mask": {"unknown": 1}},
29 {"clusters": {"min_points_per_cluster": 201, "warn_below_points": 200}},71 {"clusters": {"min_points_per_cluster": 201, "warn_below_points": 200}},
30 {"mask": {"connectivity": 6}},72 {"mask": {"connectivity": 6}},
31 {"mask": {"vector_stroke_px": 0}},73 {"mask": {"vector_stroke_px": 0}},
32 {"raster_frame": {"metadata_origin_tolerance_pixels": -1}},74 {"raster_frame": {"metadata_origin_tolerance_pixels": -1}},
33 ],75 ],
34)76)
35def test_invalid_configuration_fails(overrides: dict) -> None:77def test_invalid_configuration_fails(overrides: dict) -> None:
36 with pytest.raises(MaskClusteringConfigError):78 with pytest.raises(_config.MaskClusteringConfigError):
37 build_config(overrides=overrides)79 mask_clustering.build_config(overrides=overrides)
38
39
40def test_defaults_match_packaged_json() -> None:
41 """The model tree and the packaged JSON must stay in lock-step."""
42 packaged = json.loads(
43 resources.files(_config.PACKAGE)
44 .joinpath(_config.DEFAULT_CONFIG_FILENAME)
45 .read_text(encoding="utf-8")
46 )
47 assert _config.MaskClusteringConfig().model_dump() == packaged
48 assert load_config() == packaged
4980
5081
51@pytest.mark.parametrize(82@pytest.mark.parametrize(
52 "overrides",83 "overrides",
Importance #30: tests/test_config.py @@ -60,30 +91,30 @@
60 {"intensity_separation": {"unknown": 1}},91 {"intensity_separation": {"unknown": 1}},
61 ],92 ],
62)93)
63def test_invalid_intensity_separation_fails(overrides: dict) -> None:94def test_invalid_intensity_separation_fails(overrides: dict) -> None:
64 with pytest.raises(MaskClusteringConfigError):95 with pytest.raises(_config.MaskClusteringConfigError):
65 build_config(overrides=overrides)96 mask_clustering.build_config(overrides=overrides)
6697
6798
68def test_malformed_config_file_fails(tmp_path) -> None:99def test_malformed_config_file_fails(tmp_path) -> None:
69 path = tmp_path / "cfg.json"100 path = tmp_path / "cfg.json"
70 path.write_text("{oops", encoding="utf-8")101 path.write_text("{oops", encoding="utf-8")
71 with pytest.raises(MaskClusteringConfigError):102 with pytest.raises(_config.MaskClusteringConfigError):
72 load_config(path)103 mask_clustering.load_config(path)
73104
74105
75def test_non_object_config_file_fails(tmp_path) -> None:106def test_non_object_config_file_fails(tmp_path) -> None:
76 path = tmp_path / "cfg.json"107 path = tmp_path / "cfg.json"
77 path.write_text("[1, 2]", encoding="utf-8")108 path.write_text("[1, 2]", encoding="utf-8")
78 with pytest.raises(MaskClusteringConfigError):109 with pytest.raises(_config.MaskClusteringConfigError):
79 build_config(config_path=path)110 mask_clustering.build_config(config_path=path)
80111
81112
82def test_config_file_and_overrides_merge_onto_defaults(tmp_path) -> None:113def test_config_file_and_overrides_merge_onto_defaults(tmp_path) -> None:
83 path = tmp_path / "cfg.json"114 path = tmp_path / "cfg.json"
84 path.write_text('{"clusters": {"warn_below_points": 500}}', encoding="utf-8")115 path.write_text('{"clusters": {"warn_below_points": 500}}', encoding="utf-8")
85 config = build_config(116 config = mask_clustering.build_config(
86 config_path=path, overrides={"clusters": {"min_points_per_cluster": 400}}117 config_path=path, overrides={"clusters": {"min_points_per_cluster": 400}}
87 )118 )
88 assert config["clusters"] == {119 assert config["clusters"] == {
89 "min_points_per_cluster": 400,120 "min_points_per_cluster": 400,
Importance #31: tests/test_config.py @@ -91,26 +122,22 @@
91 }122 }
92 assert config["mask"]["connectivity"] == 8123 assert config["mask"]["connectivity"] == 8
93124
94125
95def test_scalar_coercion_and_bool_rejection() -> None:
96 assert build_config(overrides={"mask": {"vector_stroke_px": "6"}})["mask"][
97 "vector_stroke_px"
98 ] == 6
99 with pytest.raises(MaskClusteringConfigError):
100 build_config(overrides={"mask": {"vector_stroke_px": True}})
101
102
103@pytest.mark.parametrize("value", [8, 8.0, "8", 4])126@pytest.mark.parametrize("value", [8, 8.0, "8", 4])
104def test_connectivity_accepts_legacy_int_spellings(value: object) -> None:127def test_connectivity_accepts_legacy_int_spellings(value: object) -> None:
105 """JSON/CLI spellings of an int reach ``mask.connectivity`` as an int."""128 """JSON/CLI spellings of an int reach ``mask.connectivity`` as an int."""
106 config = build_config(overrides={"mask": {"connectivity": value}})129 config = mask_clustering.build_config(overrides={"mask": {"connectivity": value}})
107 assert config["mask"]["connectivity"] == int(value) # type: ignore[arg-type]130 assert config["mask"]["connectivity"] == int(value) # type: ignore[arg-type]
108131
109132
110@pytest.mark.parametrize("config", [None, [], "", 0, False])133@pytest.mark.parametrize("config", [None, [], "", 0, False])
111def test_non_mapping_config_is_rejected(config: object) -> None:134def test_non_mapping_config_is_rejected(config: object) -> None:
112 """A non-mapping is an error, not a silent "use the defaults"."""135 """A non-mapping is an error, not a silent "use the defaults"."""
113 with pytest.raises(MaskClusteringConfigError):136 with pytest.raises(_config.MaskClusteringConfigError):
114 _config.MaskClusteringConfig.coerce(config) # type: ignore[arg-type]137 _config.MaskClusteringConfig.coerce(config) # type: ignore[arg-type]
115 with pytest.raises(MaskClusteringConfigError):138 with pytest.raises(_config.MaskClusteringConfigError):
116 _config.normalize_config(config) # type: ignore[arg-type]139 mask_clustering.normalize_config(config) # type: ignore[arg-type]
140
141
142def test_normalize_config_fills_model_defaults() -> None:
143 assert mask_clustering.normalize_config({}) == _packaged_defaults()