Back to report index

Step 9 filteringsurface e99dbec: AI3D-379 Pydantic config models via iolabs-common ConfigModel

Miroslav Simko <ms@iolabs.ch> 2026-09-02T08:37:52+02:00

Commit #53 · 21 snippets

 CLAUDE.md                                      |   2 +-
 README.md                                      |  14 +-
 pyproject.toml                                 |   5 +-
 src/iolabs_point_cloud_surface_mesh/_config.py | 213 +++++++++----------------
 tests/test_surface_mesh_config.py              |  29 +++-
 5 files changed, 106 insertions(+), 157 deletions(-)
Importance #1: src/iolabs_point_cloud_surface_mesh/_config.py @@ -1,143 +1,93 @@
1"""Strict config loading and validation for the surface mesh builder.1"""Strict config loading and validation for the surface mesh builder.
22
3Generic plumbing (packaged JSON, deep merge, key whitelisting) comes from3The schema is a pydantic model tree derived from
4``iolabs.common.config_loader``; this module owns only the dataclass-derived4``iolabs.common.config_loader.ConfigModel``. Packaged defaults live in
5whitelists and the surface-mesh-specific normalization.5``surface_mesh.default.json``; ``load_config`` / ``validate_config`` handle
6JSON loading, deep-merge, unknown-key rejection and scalar coercion.
6"""7"""
78
8import json
9from dataclasses import asdict, fields
10from pathlib import Path9from pathlib import Path
11from typing import Any10from typing import Any
1211
12import pydantic
13from iolabs.common import config_loader, segment_points_io13from iolabs.common import config_loader, segment_points_io
1414
15from . import params15from . import params
1616
17_DEFAULT_CONFIG_FILENAME = "surface_mesh.default.json"17_DEFAULT_CONFIG_FILENAME = "surface_mesh.default.json"
18_PACKAGE_NAME = "iolabs_point_cloud_surface_mesh"18_PACKAGE_NAME = "iolabs_point_cloud_surface_mesh"
1919_CONFIG_CONTEXT = "surface mesh config"
20ALLOWED_SURFACE_MESH_CONFIG_KEYS = frozenset(
21 {
22 "file_naming",
23 "surface_mesh_parameters",
24 "npz_blacklist_by_segment",
25 }
26)
27
28ALLOWED_SURFACE_MESH_FILE_NAMING_KEYS = frozenset(
29 {
30 "segment_points_suffix",
31 "surface_mesh_suffix",
32 "diagnostics_pdf_suffix",
33 "versions_json_name",
34 }
35)
36
37ALLOWED_SURFACE_MESH_PARAMETER_KEYS = frozenset(
38 field.name for field in fields(params.SurfaceMeshParameters)
39)
4020
4121
42class SurfaceMeshConfigError(config_loader.ConfigError):22class SurfaceMeshConfigError(config_loader.ConfigError):
43 """Raised when a surface mesh config contains unsupported keys or values."""23 """Raised when a surface mesh config contains unsupported keys or values."""
4424
4525
46def _normalize_file_naming(raw_file_naming: Any) -> dict[str, Any]:26class SurfaceMeshFileNamingConfig(config_loader.ConfigModel):
47 if raw_file_naming is None:27 """Output and input filename stems for one surface-mesh run."""
48 file_naming: dict[str, Any] = {}
49 elif isinstance(raw_file_naming, dict):
50 file_naming = dict(raw_file_naming)
51 else:
52 raise SurfaceMeshConfigError("surface mesh config field 'file_naming' must be a mapping")
53
54 config_loader.validate_allowed_keys(
55 file_naming,
56 ALLOWED_SURFACE_MESH_FILE_NAMING_KEYS,
57 context="surface mesh file_naming",
58 error_cls=SurfaceMeshConfigError,
59 )
60
61 file_naming.setdefault("segment_points_suffix", "_run3_points")
62 file_naming.setdefault("surface_mesh_suffix", "_surface_mesh")
63 file_naming.setdefault("diagnostics_pdf_suffix", "_surface_mesh_diagnostics")
64 file_naming.setdefault("versions_json_name", "surface_mesh_versions.json")
65 return file_naming
66
67
68def _normalize_surface_mesh_parameters(raw_parameters: Any) -> dict[str, Any]:
69 if raw_parameters is None:
70 parameters: dict[str, Any] = {}
71 elif isinstance(raw_parameters, dict):
72 parameters = dict(raw_parameters)
73 else:
74 raise SurfaceMeshConfigError(
75 "surface mesh config field 'surface_mesh_parameters' must be a mapping"
76 )
77
78 config_loader.validate_allowed_keys(
79 parameters,
80 ALLOWED_SURFACE_MESH_PARAMETER_KEYS,
81 context="surface mesh surface_mesh_parameters",
82 error_cls=SurfaceMeshConfigError,
83 )
84
85 normalized = asdict(params.SurfaceMeshParameters())
86 normalized.update(parameters)
87 _validate_parameter_values(normalized)
88 return normalized
89
90
91def _is_real(value: Any) -> bool:
92 return isinstance(value, (int, float)) and not isinstance(value, bool)
93
94
95def _is_int(value: Any) -> bool:
96 return isinstance(value, int) and not isinstance(value, bool)
9728
29 segment_points_suffix: str = "_run3_points"
30 surface_mesh_suffix: str = "_surface_mesh"
31 diagnostics_pdf_suffix: str = "_surface_mesh_diagnostics"
32 versions_json_name: str = "surface_mesh_versions.json"
9833
99_PARAMETER_RULES: dict[str, tuple[Any, str]] = {
100 "device": (lambda v: isinstance(v, str) and v != "", "a non-empty string"),
101 "cells_across": (lambda v: _is_int(v) and v >= 1, "an integer >= 1"),
102 "row_spacing_m": (lambda v: _is_real(v) and v > 0, "a number > 0"),
103 "min_points_per_cell": (lambda v: _is_int(v) and v >= 3, "an integer >= 3"),
104 "z_trim_mad_factor": (lambda v: _is_real(v) and v >= 0, "a number >= 0"),
105 "max_extrapolation_m": (lambda v: _is_real(v) and v >= 0, "a number >= 0"),
106 "min_measured_edge_fraction": (
107 lambda v: _is_real(v) and 0.0 <= v <= 1.0,
108 "a number in [0, 1]",
109 ),
110 "edge_smoothing_window_m": (lambda v: _is_real(v) and v >= 0, "a number >= 0"),
111 "worst_cells_count": (lambda v: _is_int(v) and v >= 0, "an integer >= 0"),
112 "save_diagnostics": (lambda v: isinstance(v, bool), "a boolean"),
113}
11434
35class SurfaceMeshParametersConfig(config_loader.ConfigModel):
36 """Tunables for the edge-bounded pavement surface mesh.
11537
116def _validate_parameter_values(normalized: dict[str, Any]) -> None:38 Field names and defaults match ``surface_mesh.default.json`` and
117 """Rejects invalid types/ranges at config time instead of deep in processing.39 ``params.SurfaceMeshParameters``.
118
119 Raises:
120 SurfaceMeshConfigError: Naming the offending parameter and requirement.
121 """40 """
122 for key, (predicate, requirement) in _PARAMETER_RULES.items():41
123 value = normalized.get(key)42 device: str = pydantic.Field(default="CUDA:0", min_length=1)
124 if not predicate(value):43 cells_across: int = pydantic.Field(default=5, ge=1)
125 raise SurfaceMeshConfigError(44 row_spacing_m: float = pydantic.Field(default=3.0, gt=0)
126 f"surface_mesh_parameters.{key} must be {requirement}, "45 min_points_per_cell: int = pydantic.Field(default=20, ge=3)
127 f"got {value!r}"46 z_trim_mad_factor: float = pydantic.Field(default=3.0, ge=0)
128 )47 max_extrapolation_m: float = pydantic.Field(default=15.0, ge=0)
12948 min_measured_edge_fraction: float = pydantic.Field(default=0.25, ge=0, le=1)
13049 edge_smoothing_window_m: float = pydantic.Field(default=0.0, ge=0)
131def _normalize_segment_file_blacklist(raw_blacklist: Any) -> dict[int, list[str]]:50 worst_cells_count: int = pydantic.Field(default=8, ge=0)
132 if raw_blacklist is not None and not isinstance(raw_blacklist, dict):51 save_diagnostics: bool = True
133 raise SurfaceMeshConfigError(52
134 "surface mesh config field 'npz_blacklist_by_segment' must be a mapping"53
135 )54class SurfaceMeshConfig(config_loader.ConfigModel):
136 try:55 """Top-level surface-mesh config mirroring ``surface_mesh.default.json``."""
137 return segment_points_io.normalize_segment_file_blacklist(raw_blacklist)56
138 except ValueError as exc:57 file_naming: SurfaceMeshFileNamingConfig = SurfaceMeshFileNamingConfig()
139 raise SurfaceMeshConfigError(str(exc)) from exc58 surface_mesh_parameters: SurfaceMeshParametersConfig = SurfaceMeshParametersConfig()
59 npz_blacklist_by_segment: dict[int, list[str]] = pydantic.Field(default_factory=dict)
60
61 @pydantic.field_validator("npz_blacklist_by_segment", mode="before")
62 @classmethod
63 def _normalize_blacklist(cls, value: object) -> dict[int, list[str]]:
64 """Accept ``segment_<idx>`` keys and string-or-list pattern values."""
65 if value is None:
66 value = {}
67 if not isinstance(value, dict):
68 raise ValueError("npz_blacklist_by_segment must be a mapping")
69 try:
70 return segment_points_io.normalize_segment_file_blacklist(value)
71 except ValueError as exc:
72 raise ValueError(str(exc)) from exc
73
74
75def _load_model(
76 *,
77 overrides: dict[str, Any] | None = None,
78 config_path: str | Path | None = None,
79) -> SurfaceMeshConfig:
80 """Load packaged (or file) defaults, merge overrides, and validate."""
81 return config_loader.load_config(
82 SurfaceMeshConfig,
83 package=_PACKAGE_NAME,
84 filename=_DEFAULT_CONFIG_FILENAME,
85 overrides=overrides,
86 config_path=config_path,
87 context=_CONFIG_CONTEXT,
88 error_cls=SurfaceMeshConfigError,
89 )
14090
14191
142def normalize_surface_mesh_config(raw_config: dict[str, Any]) -> dict[str, Any]:92def normalize_surface_mesh_config(raw_config: dict[str, Any]) -> dict[str, Any]:
143 """Validates a config mapping strictly and fills in every default.93 """Validates a config mapping strictly and fills in every default.
Importance #2: src/iolabs_point_cloud_surface_mesh/_config.py @@ -194,14 +133,9 @@
194133
195 Returns:134 Returns:
196 The normalized config dict.135 The normalized config dict.
197 """136 """
198 if config_path is None:137 return _load_model(config_path=config_path).model_dump()
199 raw_config = config_loader.load_packaged_json(_PACKAGE_NAME, _DEFAULT_CONFIG_FILENAME)
200 else:
201 with Path(config_path).open("r", encoding="utf-8") as handle:
202 raw_config = json.load(handle)
203 return normalize_surface_mesh_config(raw_config)
204138
205139
206def build_surface_mesh_config(140def build_surface_mesh_config(
207 *,141 *,
Importance #3: src/iolabs_point_cloud_surface_mesh/_config.py @@ -153,25 +103,14 @@
153 level or a section has the wrong type.103 level or a section has the wrong type.
154 """104 """
155 if not isinstance(raw_config, dict):105 if not isinstance(raw_config, dict):
156 raise SurfaceMeshConfigError("surface mesh config must be a mapping")106 raise SurfaceMeshConfigError("surface mesh config must be a mapping")
157107 return config_loader.validate_config(
158 config = dict(raw_config)108 SurfaceMeshConfig,
159 config_loader.validate_allowed_keys(109 raw_config,
160 config,110 context=_CONFIG_CONTEXT,
161 ALLOWED_SURFACE_MESH_CONFIG_KEYS,
162 context="surface mesh config",
163 error_cls=SurfaceMeshConfigError,111 error_cls=SurfaceMeshConfigError,
164 )112 ).model_dump()
165
166 config["file_naming"] = _normalize_file_naming(config.get("file_naming"))
167 config["surface_mesh_parameters"] = _normalize_surface_mesh_parameters(
168 config.get("surface_mesh_parameters")
169 )
170 config["npz_blacklist_by_segment"] = _normalize_segment_file_blacklist(
171 config.get("npz_blacklist_by_segment")
172 )
173 return config
174113
175114
176def surface_mesh_parameters_from_config(config: dict[str, Any]) -> params.SurfaceMeshParameters:115def surface_mesh_parameters_from_config(config: dict[str, Any]) -> params.SurfaceMeshParameters:
177 """Builds the parameters dataclass from a normalized config.116 """Builds the parameters dataclass from a normalized config.
Importance #4: src/iolabs_point_cloud_surface_mesh/_config.py @@ -217,8 +151,5 @@
217151
218 Returns:152 Returns:
219 The normalized, merged config dict.153 The normalized, merged config dict.
220 """154 """
221 config = load_surface_mesh_config(config_path)155 return _load_model(overrides=overrides, config_path=config_path).model_dump()
222 if overrides:
223 config = config_loader.deep_merge_dicts(config, dict(overrides))
224 return normalize_surface_mesh_config(config)
Importance #5: tests/test_surface_mesh_config.py @@ -1,12 +1,13 @@
1import dataclasses
1import json2import json
23
3import pytest4import pytest
45
5from iolabs_point_cloud_surface_mesh import params6from iolabs_point_cloud_surface_mesh import params
6from iolabs_point_cloud_surface_mesh._config import (7from iolabs_point_cloud_surface_mesh._config import (
7 ALLOWED_SURFACE_MESH_PARAMETER_KEYS,
8 SurfaceMeshConfigError,8 SurfaceMeshConfigError,
9 SurfaceMeshParametersConfig,
9 build_surface_mesh_config,10 build_surface_mesh_config,
10 load_surface_mesh_config,11 load_surface_mesh_config,
11 normalize_surface_mesh_config,12 normalize_surface_mesh_config,
12 surface_mesh_parameters_from_config,13 surface_mesh_parameters_from_config,
Importance #6: tests/test_surface_mesh_config.py @@ -43,12 +44,13 @@
43 with pytest.raises(SurfaceMeshConfigError, match="file_naming"):44 with pytest.raises(SurfaceMeshConfigError, match="file_naming"):
44 normalize_surface_mesh_config({"file_naming": {"road_surface_suffix": "_x"}})45 normalize_surface_mesh_config({"file_naming": {"road_surface_suffix": "_x"}})
4546
4647
47def test_parameter_whitelist_tracks_dataclass_fields():48def test_parameter_model_tracks_dataclass_fields():
48 assert "cells_across" in ALLOWED_SURFACE_MESH_PARAMETER_KEYS49 model_keys = set(SurfaceMeshParametersConfig.model_fields)
49 assert "device" in ALLOWED_SURFACE_MESH_PARAMETER_KEYS50 dataclass_keys = {field.name for field in dataclasses.fields(params.SurfaceMeshParameters)}
50 assert "road_surface_suffix" not in ALLOWED_SURFACE_MESH_PARAMETER_KEYS51 assert model_keys == dataclass_keys
52 assert "road_surface_suffix" not in model_keys
5153
5254
53def test_build_config_deep_merges_overrides():55def test_build_config_deep_merges_overrides():
54 config = build_surface_mesh_config(56 config = build_surface_mesh_config(
Importance #7: tests/test_surface_mesh_config.py @@ -57,8 +59,21 @@
57 assert config["surface_mesh_parameters"]["cells_across"] == 759 assert config["surface_mesh_parameters"]["cells_across"] == 7
58 assert config["surface_mesh_parameters"]["row_spacing_m"] == 3.060 assert config["surface_mesh_parameters"]["row_spacing_m"] == 3.0
5961
6062
63def test_build_config_coerces_string_overrides():
64 config = build_surface_mesh_config(
65 overrides={
66 "surface_mesh_parameters": {
67 "row_spacing_m": "4.5",
68 "save_diagnostics": "yes",
69 }
70 }
71 )
72 assert config["surface_mesh_parameters"]["row_spacing_m"] == 4.5
73 assert config["surface_mesh_parameters"]["save_diagnostics"] is True
74
75
61def test_build_config_rejects_unknown_override_keys():76def test_build_config_rejects_unknown_override_keys():
62 with pytest.raises(SurfaceMeshConfigError):77 with pytest.raises(SurfaceMeshConfigError):
63 build_surface_mesh_config(overrides={"surface_parameters": {}})78 build_surface_mesh_config(overrides={"surface_parameters": {}})
6479
Importance #8: tests/test_surface_mesh_config.py @@ -89,14 +104,14 @@
89 ("key", "value"),104 ("key", "value"),
90 [105 [
91 ("cells_across", 0),106 ("cells_across", 0),
92 ("cells_across", 2.5),107 ("cells_across", 2.5),
93 ("row_spacing_m", "3"),108 ("row_spacing_m", "abc"),
94 ("row_spacing_m", 0),109 ("row_spacing_m", 0),
95 ("min_points_per_cell", 1),110 ("min_points_per_cell", 1),
96 ("min_measured_edge_fraction", 1.5),111 ("min_measured_edge_fraction", 1.5),
97 ("z_trim_mad_factor", -1),112 ("z_trim_mad_factor", -1),
98 ("save_diagnostics", "yes"),113 ("save_diagnostics", "maybe"),
99 ("device", ""),114 ("device", ""),
100 ],115 ],
101)116)
102def test_normalize_rejects_invalid_parameter_values(key, value):117def test_normalize_rejects_invalid_parameter_values(key, value):
Importance #9: pyproject.toml @@ -1,16 +1,17 @@
1[project]1[project]
2name = "iolabs-point-cloud-surface-mesh"2name = "iolabs-point-cloud-surface-mesh"
3version = "0.8.0"3version = "0.8.1"
4description = "Edge-bounded pavement surface mesh from asphalt-edge polylines and LIDAR point clouds"4description = "Edge-bounded pavement surface mesh from asphalt-edge polylines and LIDAR point clouds"
5requires-python = ">=3.11,<3.13"5requires-python = ">=3.11,<3.13"
6dependencies = [6dependencies = [
7 "numpy>=1.20.0",7 "numpy>=1.20.0",
8 "torch>=2.0.0",8 "torch>=2.0.0",
9 "scipy>=1.7.0",9 "scipy>=1.7.0",
10 "matplotlib>=3.4.0",10 "matplotlib>=3.4.0",
11 "pydantic>=2.7",
11 "iolabs-logstash>=0.4.0",12 "iolabs-logstash>=0.4.0",
12 "iolabs-common>=0.5.0",13 "iolabs-common>=0.8.0",
13 "iolabs-geometry-geometry>=0.9.0",14 "iolabs-geometry-geometry>=0.9.0",
14]15]
1516
16[tool.uv]17[tool.uv]
Importance #10: CLAUDE.md @@ -36,9 +36,9 @@
36`surface_mesh_versions.json` is written per segment via `iolabs.common.version_info.save_version_json(..., "surface_mesh_builder")`.36`surface_mesh_versions.json` is written per segment via `iolabs.common.version_info.save_version_json(..., "surface_mesh_builder")`.
3737
38## Config model (important)38## Config model (important)
3939
40`_config.py` is authoritative and delegates generic plumbing to `iolabs.common.config_loader`. Whitelists: `ALLOWED_SURFACE_MESH_CONFIG_KEYS` (top level: `file_naming`, `surface_mesh_parameters`, `npz_blacklist_by_segment`), `ALLOWED_SURFACE_MESH_FILE_NAMING_KEYS`, and `ALLOWED_SURFACE_MESH_PARAMETER_KEYS` (derived from the `params.SurfaceMeshParameters` dataclass fields). Unknown keys raise `SurfaceMeshConfigError`. When adding a tunable: add the field to `params.SurfaceMeshParameters` **and** to `surface_mesh.default.json` (wheel force-include). `npz_blacklist_by_segment` maps `segment_<idx>`/int keys to fnmatch patterns (normalization lives in `iolabs.common.segment_points_io`).40`_config.py` is authoritative: a pydantic model tree (`SurfaceMeshConfig` and nested sections) derived from `iolabs.common.config_loader.ConfigModel`, loaded via `config_loader.load_config`. Unknown keys raise `SurfaceMeshConfigError`. When adding a config key: add the field to the model **and** the matching default in `surface_mesh.default.json` (wheel force-include) — nothing else. Algorithm tunables that processing code reads also go on `params.SurfaceMeshParameters` (built from the model). `npz_blacklist_by_segment` is `dict[int, list[str]]`; `segment_<idx>` keys are normalized via `iolabs.common.segment_points_io`.
4141
42## Input contracts42## Input contracts
4343
44- Points: `<scan>_run3_points.npz` with keys `points, red, green, blue, intensity, scan_angle` (schema SSOT: `iolabs.common.segment_points_io.POINT_RECORD_KEYS`), geoshift-relative float64.44- Points: `<scan>_run3_points.npz` with keys `points, red, green, blue, intensity, scan_angle` (schema SSOT: `iolabs.common.segment_points_io.POINT_RECORD_KEYS`), geoshift-relative float64.
Importance #11: README.md @@ -25,17 +25,19 @@
25| `surface_mesh` | Vertex pool (all edge points + interior lattice), Delaunay, centroid-in-polygon clip. |25| `surface_mesh` | Vertex pool (all edge points + interior lattice), Delaunay, centroid-in-polygon clip. |
26| `ply_io` | Self-contained binary little-endian PLY writer (double precision). |26| `ply_io` | Self-contained binary little-endian PLY writer (double precision). |
27| `diagnostics` | Cell-status map, point-count heatmap, residual histograms, per-source-file residual split (misaligned-LAS detector), edge-quality page. |27| `diagnostics` | Cell-status map, point-count heatmap, residual histograms, per-source-file residual split (misaligned-LAS detector), edge-quality page. |
28| `surface_mesh_builder` | Per-segment driver: load + blacklist NPZs, build meshes per carriageway, write PLYs/PDF/versions JSON, return summary. |28| `surface_mesh_builder` | Per-segment driver: load + blacklist NPZs, build meshes per carriageway, write PLYs/PDF/versions JSON, return summary. |
29| `params` / `_config` | `SurfaceMeshParameters` + strict config validation (`surface_mesh.default.json`), delegating plumbing to `iolabs.common.config_loader`. |29| `params` / `_config` | `SurfaceMeshParameters` algorithm object + pydantic `SurfaceMeshConfig` (`surface_mesh.default.json`), loaded via `iolabs.common.config_loader`. |
3030
31## Configuration31## Configuration
3232
33Packaged defaults in `surface_mesh.default.json`; unknown keys raise33Packaged defaults in `surface_mesh.default.json` are validated by the pydantic
34`SurfaceMeshConfigError`. Key tunables: `cells_across` (5), `row_spacing_m`34`SurfaceMeshConfig` tree; unknown keys raise `SurfaceMeshConfigError`. To add a
35(3.0), `device` ("CUDA:0"), `min_points_per_cell`, `z_trim_mad_factor`,35key, add the field to the model and the JSON default. Key tunables:
36`max_extrapolation_m`, `min_measured_edge_fraction`, `edge_smoothing_window_m`.36`cells_across` (5), `row_spacing_m` (3.0), `device` ("CUDA:0"),
37File naming lives under `file_naming`; per-segment NPZ exclusion under37`min_points_per_cell`, `z_trim_mad_factor`, `max_extrapolation_m`,
38`min_measured_edge_fraction`, `edge_smoothing_window_m`. File naming lives
39under `file_naming`; per-segment NPZ exclusion under
38`npz_blacklist_by_segment`.40`npz_blacklist_by_segment`.
3941
40## Commands42## Commands
4143
Importance #12: README.md @@ -25,17 +25,19 @@
25| `surface_mesh` | Vertex pool (all edge points + interior lattice), Delaunay, centroid-in-polygon clip. |25| `surface_mesh` | Vertex pool (all edge points + interior lattice), Delaunay, centroid-in-polygon clip. |
26| `ply_io` | Self-contained binary little-endian PLY writer (double precision). |26| `ply_io` | Self-contained binary little-endian PLY writer (double precision). |
27| `diagnostics` | Cell-status map, point-count heatmap, residual histograms, per-source-file residual split (misaligned-LAS detector), edge-quality page. |27| `diagnostics` | Cell-status map, point-count heatmap, residual histograms, per-source-file residual split (misaligned-LAS detector), edge-quality page. |
28| `surface_mesh_builder` | Per-segment driver: load + blacklist NPZs, build meshes per carriageway, write PLYs/PDF/versions JSON, return summary. |28| `surface_mesh_builder` | Per-segment driver: load + blacklist NPZs, build meshes per carriageway, write PLYs/PDF/versions JSON, return summary. |
29| `params` / `_config` | `SurfaceMeshParameters` + strict config validation (`surface_mesh.default.json`), delegating plumbing to `iolabs.common.config_loader`. |29| `params` / `_config` | `SurfaceMeshParameters` algorithm object + pydantic `SurfaceMeshConfig` (`surface_mesh.default.json`), loaded via `iolabs.common.config_loader`. |
3030
31## Configuration31## Configuration
3232
33Packaged defaults in `surface_mesh.default.json`; unknown keys raise33Packaged defaults in `surface_mesh.default.json` are validated by the pydantic
34`SurfaceMeshConfigError`. Key tunables: `cells_across` (5), `row_spacing_m`34`SurfaceMeshConfig` tree; unknown keys raise `SurfaceMeshConfigError`. To add a
35(3.0), `device` ("CUDA:0"), `min_points_per_cell`, `z_trim_mad_factor`,35key, add the field to the model and the JSON default. Key tunables:
36`max_extrapolation_m`, `min_measured_edge_fraction`, `edge_smoothing_window_m`.36`cells_across` (5), `row_spacing_m` (3.0), `device` ("CUDA:0"),
37File naming lives under `file_naming`; per-segment NPZ exclusion under37`min_points_per_cell`, `z_trim_mad_factor`, `max_extrapolation_m`,
38`min_measured_edge_fraction`, `edge_smoothing_window_m`. File naming lives
39under `file_naming`; per-segment NPZ exclusion under
38`npz_blacklist_by_segment`.40`npz_blacklist_by_segment`.
3941
40## Commands42## Commands
4143
Importance #13: pyproject.toml @@ -1,16 +1,17 @@
1[project]1[project]
2name = "iolabs-point-cloud-surface-mesh"2name = "iolabs-point-cloud-surface-mesh"
3version = "0.8.0"3version = "0.8.1"
4description = "Edge-bounded pavement surface mesh from asphalt-edge polylines and LIDAR point clouds"4description = "Edge-bounded pavement surface mesh from asphalt-edge polylines and LIDAR point clouds"
5requires-python = ">=3.11,<3.13"5requires-python = ">=3.11,<3.13"
6dependencies = [6dependencies = [
7 "numpy>=1.20.0",7 "numpy>=1.20.0",
8 "torch>=2.0.0",8 "torch>=2.0.0",
9 "scipy>=1.7.0",9 "scipy>=1.7.0",
10 "matplotlib>=3.4.0",10 "matplotlib>=3.4.0",
11 "pydantic>=2.7",
11 "iolabs-logstash>=0.4.0",12 "iolabs-logstash>=0.4.0",
12 "iolabs-common>=0.5.0",13 "iolabs-common>=0.8.0",
13 "iolabs-geometry-geometry>=0.9.0",14 "iolabs-geometry-geometry>=0.9.0",
14]15]
1516
16[tool.uv]17[tool.uv]
Importance #14: src/iolabs_point_cloud_surface_mesh/_config.py @@ -1,143 +1,93 @@
1"""Strict config loading and validation for the surface mesh builder.1"""Strict config loading and validation for the surface mesh builder.
22
3Generic plumbing (packaged JSON, deep merge, key whitelisting) comes from3The schema is a pydantic model tree derived from
4``iolabs.common.config_loader``; this module owns only the dataclass-derived4``iolabs.common.config_loader.ConfigModel``. Packaged defaults live in
5whitelists and the surface-mesh-specific normalization.5``surface_mesh.default.json``; ``load_config`` / ``validate_config`` handle
6JSON loading, deep-merge, unknown-key rejection and scalar coercion.
6"""7"""
78
8import json
9from dataclasses import asdict, fields
10from pathlib import Path9from pathlib import Path
11from typing import Any10from typing import Any
1211
12import pydantic
13from iolabs.common import config_loader, segment_points_io13from iolabs.common import config_loader, segment_points_io
1414
15from . import params15from . import params
1616
17_DEFAULT_CONFIG_FILENAME = "surface_mesh.default.json"17_DEFAULT_CONFIG_FILENAME = "surface_mesh.default.json"
18_PACKAGE_NAME = "iolabs_point_cloud_surface_mesh"18_PACKAGE_NAME = "iolabs_point_cloud_surface_mesh"
1919_CONFIG_CONTEXT = "surface mesh config"
20ALLOWED_SURFACE_MESH_CONFIG_KEYS = frozenset(
21 {
22 "file_naming",
23 "surface_mesh_parameters",
24 "npz_blacklist_by_segment",
25 }
26)
27
28ALLOWED_SURFACE_MESH_FILE_NAMING_KEYS = frozenset(
29 {
30 "segment_points_suffix",
31 "surface_mesh_suffix",
32 "diagnostics_pdf_suffix",
33 "versions_json_name",
34 }
35)
36
37ALLOWED_SURFACE_MESH_PARAMETER_KEYS = frozenset(
38 field.name for field in fields(params.SurfaceMeshParameters)
39)
4020
4121
42class SurfaceMeshConfigError(config_loader.ConfigError):22class SurfaceMeshConfigError(config_loader.ConfigError):
43 """Raised when a surface mesh config contains unsupported keys or values."""23 """Raised when a surface mesh config contains unsupported keys or values."""
4424
4525
46def _normalize_file_naming(raw_file_naming: Any) -> dict[str, Any]:26class SurfaceMeshFileNamingConfig(config_loader.ConfigModel):
47 if raw_file_naming is None:27 """Output and input filename stems for one surface-mesh run."""
48 file_naming: dict[str, Any] = {}
49 elif isinstance(raw_file_naming, dict):
50 file_naming = dict(raw_file_naming)
51 else:
52 raise SurfaceMeshConfigError("surface mesh config field 'file_naming' must be a mapping")
53
54 config_loader.validate_allowed_keys(
55 file_naming,
56 ALLOWED_SURFACE_MESH_FILE_NAMING_KEYS,
57 context="surface mesh file_naming",
58 error_cls=SurfaceMeshConfigError,
59 )
60
61 file_naming.setdefault("segment_points_suffix", "_run3_points")
62 file_naming.setdefault("surface_mesh_suffix", "_surface_mesh")
63 file_naming.setdefault("diagnostics_pdf_suffix", "_surface_mesh_diagnostics")
64 file_naming.setdefault("versions_json_name", "surface_mesh_versions.json")
65 return file_naming
66
67
68def _normalize_surface_mesh_parameters(raw_parameters: Any) -> dict[str, Any]:
69 if raw_parameters is None:
70 parameters: dict[str, Any] = {}
71 elif isinstance(raw_parameters, dict):
72 parameters = dict(raw_parameters)
73 else:
74 raise SurfaceMeshConfigError(
75 "surface mesh config field 'surface_mesh_parameters' must be a mapping"
76 )
77
78 config_loader.validate_allowed_keys(
79 parameters,
80 ALLOWED_SURFACE_MESH_PARAMETER_KEYS,
81 context="surface mesh surface_mesh_parameters",
82 error_cls=SurfaceMeshConfigError,
83 )
84
85 normalized = asdict(params.SurfaceMeshParameters())
86 normalized.update(parameters)
87 _validate_parameter_values(normalized)
88 return normalized
89
90
91def _is_real(value: Any) -> bool:
92 return isinstance(value, (int, float)) and not isinstance(value, bool)
93
94
95def _is_int(value: Any) -> bool:
96 return isinstance(value, int) and not isinstance(value, bool)
9728
29 segment_points_suffix: str = "_run3_points"
30 surface_mesh_suffix: str = "_surface_mesh"
31 diagnostics_pdf_suffix: str = "_surface_mesh_diagnostics"
32 versions_json_name: str = "surface_mesh_versions.json"
9833
99_PARAMETER_RULES: dict[str, tuple[Any, str]] = {
100 "device": (lambda v: isinstance(v, str) and v != "", "a non-empty string"),
101 "cells_across": (lambda v: _is_int(v) and v >= 1, "an integer >= 1"),
102 "row_spacing_m": (lambda v: _is_real(v) and v > 0, "a number > 0"),
103 "min_points_per_cell": (lambda v: _is_int(v) and v >= 3, "an integer >= 3"),
104 "z_trim_mad_factor": (lambda v: _is_real(v) and v >= 0, "a number >= 0"),
105 "max_extrapolation_m": (lambda v: _is_real(v) and v >= 0, "a number >= 0"),
106 "min_measured_edge_fraction": (
107 lambda v: _is_real(v) and 0.0 <= v <= 1.0,
108 "a number in [0, 1]",
109 ),
110 "edge_smoothing_window_m": (lambda v: _is_real(v) and v >= 0, "a number >= 0"),
111 "worst_cells_count": (lambda v: _is_int(v) and v >= 0, "an integer >= 0"),
112 "save_diagnostics": (lambda v: isinstance(v, bool), "a boolean"),
113}
11434
35class SurfaceMeshParametersConfig(config_loader.ConfigModel):
36 """Tunables for the edge-bounded pavement surface mesh.
11537
116def _validate_parameter_values(normalized: dict[str, Any]) -> None:38 Field names and defaults match ``surface_mesh.default.json`` and
117 """Rejects invalid types/ranges at config time instead of deep in processing.39 ``params.SurfaceMeshParameters``.
118
119 Raises:
120 SurfaceMeshConfigError: Naming the offending parameter and requirement.
121 """40 """
122 for key, (predicate, requirement) in _PARAMETER_RULES.items():41
123 value = normalized.get(key)42 device: str = pydantic.Field(default="CUDA:0", min_length=1)
124 if not predicate(value):43 cells_across: int = pydantic.Field(default=5, ge=1)
125 raise SurfaceMeshConfigError(44 row_spacing_m: float = pydantic.Field(default=3.0, gt=0)
126 f"surface_mesh_parameters.{key} must be {requirement}, "45 min_points_per_cell: int = pydantic.Field(default=20, ge=3)
127 f"got {value!r}"46 z_trim_mad_factor: float = pydantic.Field(default=3.0, ge=0)
128 )47 max_extrapolation_m: float = pydantic.Field(default=15.0, ge=0)
12948 min_measured_edge_fraction: float = pydantic.Field(default=0.25, ge=0, le=1)
13049 edge_smoothing_window_m: float = pydantic.Field(default=0.0, ge=0)
131def _normalize_segment_file_blacklist(raw_blacklist: Any) -> dict[int, list[str]]:50 worst_cells_count: int = pydantic.Field(default=8, ge=0)
132 if raw_blacklist is not None and not isinstance(raw_blacklist, dict):51 save_diagnostics: bool = True
133 raise SurfaceMeshConfigError(52
134 "surface mesh config field 'npz_blacklist_by_segment' must be a mapping"53
135 )54class SurfaceMeshConfig(config_loader.ConfigModel):
136 try:55 """Top-level surface-mesh config mirroring ``surface_mesh.default.json``."""
137 return segment_points_io.normalize_segment_file_blacklist(raw_blacklist)56
138 except ValueError as exc:57 file_naming: SurfaceMeshFileNamingConfig = SurfaceMeshFileNamingConfig()
139 raise SurfaceMeshConfigError(str(exc)) from exc58 surface_mesh_parameters: SurfaceMeshParametersConfig = SurfaceMeshParametersConfig()
59 npz_blacklist_by_segment: dict[int, list[str]] = pydantic.Field(default_factory=dict)
60
61 @pydantic.field_validator("npz_blacklist_by_segment", mode="before")
62 @classmethod
63 def _normalize_blacklist(cls, value: object) -> dict[int, list[str]]:
64 """Accept ``segment_<idx>`` keys and string-or-list pattern values."""
65 if value is None:
66 value = {}
67 if not isinstance(value, dict):
68 raise ValueError("npz_blacklist_by_segment must be a mapping")
69 try:
70 return segment_points_io.normalize_segment_file_blacklist(value)
71 except ValueError as exc:
72 raise ValueError(str(exc)) from exc
73
74
75def _load_model(
76 *,
77 overrides: dict[str, Any] | None = None,
78 config_path: str | Path | None = None,
79) -> SurfaceMeshConfig:
80 """Load packaged (or file) defaults, merge overrides, and validate."""
81 return config_loader.load_config(
82 SurfaceMeshConfig,
83 package=_PACKAGE_NAME,
84 filename=_DEFAULT_CONFIG_FILENAME,
85 overrides=overrides,
86 config_path=config_path,
87 context=_CONFIG_CONTEXT,
88 error_cls=SurfaceMeshConfigError,
89 )
14090
14191
142def normalize_surface_mesh_config(raw_config: dict[str, Any]) -> dict[str, Any]:92def normalize_surface_mesh_config(raw_config: dict[str, Any]) -> dict[str, Any]:
143 """Validates a config mapping strictly and fills in every default.93 """Validates a config mapping strictly and fills in every default.
Importance #15: src/iolabs_point_cloud_surface_mesh/_config.py @@ -153,25 +103,14 @@
153 level or a section has the wrong type.103 level or a section has the wrong type.
154 """104 """
155 if not isinstance(raw_config, dict):105 if not isinstance(raw_config, dict):
156 raise SurfaceMeshConfigError("surface mesh config must be a mapping")106 raise SurfaceMeshConfigError("surface mesh config must be a mapping")
157107 return config_loader.validate_config(
158 config = dict(raw_config)108 SurfaceMeshConfig,
159 config_loader.validate_allowed_keys(109 raw_config,
160 config,110 context=_CONFIG_CONTEXT,
161 ALLOWED_SURFACE_MESH_CONFIG_KEYS,
162 context="surface mesh config",
163 error_cls=SurfaceMeshConfigError,111 error_cls=SurfaceMeshConfigError,
164 )112 ).model_dump()
165
166 config["file_naming"] = _normalize_file_naming(config.get("file_naming"))
167 config["surface_mesh_parameters"] = _normalize_surface_mesh_parameters(
168 config.get("surface_mesh_parameters")
169 )
170 config["npz_blacklist_by_segment"] = _normalize_segment_file_blacklist(
171 config.get("npz_blacklist_by_segment")
172 )
173 return config
174113
175114
176def surface_mesh_parameters_from_config(config: dict[str, Any]) -> params.SurfaceMeshParameters:115def surface_mesh_parameters_from_config(config: dict[str, Any]) -> params.SurfaceMeshParameters:
177 """Builds the parameters dataclass from a normalized config.116 """Builds the parameters dataclass from a normalized config.
Importance #16: src/iolabs_point_cloud_surface_mesh/_config.py @@ -194,14 +133,9 @@
194133
195 Returns:134 Returns:
196 The normalized config dict.135 The normalized config dict.
197 """136 """
198 if config_path is None:137 return _load_model(config_path=config_path).model_dump()
199 raw_config = config_loader.load_packaged_json(_PACKAGE_NAME, _DEFAULT_CONFIG_FILENAME)
200 else:
201 with Path(config_path).open("r", encoding="utf-8") as handle:
202 raw_config = json.load(handle)
203 return normalize_surface_mesh_config(raw_config)
204138
205139
206def build_surface_mesh_config(140def build_surface_mesh_config(
207 *,141 *,
Importance #17: src/iolabs_point_cloud_surface_mesh/_config.py @@ -217,8 +151,5 @@
217151
218 Returns:152 Returns:
219 The normalized, merged config dict.153 The normalized, merged config dict.
220 """154 """
221 config = load_surface_mesh_config(config_path)155 return _load_model(overrides=overrides, config_path=config_path).model_dump()
222 if overrides:
223 config = config_loader.deep_merge_dicts(config, dict(overrides))
224 return normalize_surface_mesh_config(config)
Importance #18: tests/test_surface_mesh_config.py @@ -1,12 +1,13 @@
1import dataclasses
1import json2import json
23
3import pytest4import pytest
45
5from iolabs_point_cloud_surface_mesh import params6from iolabs_point_cloud_surface_mesh import params
6from iolabs_point_cloud_surface_mesh._config import (7from iolabs_point_cloud_surface_mesh._config import (
7 ALLOWED_SURFACE_MESH_PARAMETER_KEYS,
8 SurfaceMeshConfigError,8 SurfaceMeshConfigError,
9 SurfaceMeshParametersConfig,
9 build_surface_mesh_config,10 build_surface_mesh_config,
10 load_surface_mesh_config,11 load_surface_mesh_config,
11 normalize_surface_mesh_config,12 normalize_surface_mesh_config,
12 surface_mesh_parameters_from_config,13 surface_mesh_parameters_from_config,
Importance #19: tests/test_surface_mesh_config.py @@ -43,12 +44,13 @@
43 with pytest.raises(SurfaceMeshConfigError, match="file_naming"):44 with pytest.raises(SurfaceMeshConfigError, match="file_naming"):
44 normalize_surface_mesh_config({"file_naming": {"road_surface_suffix": "_x"}})45 normalize_surface_mesh_config({"file_naming": {"road_surface_suffix": "_x"}})
4546
4647
47def test_parameter_whitelist_tracks_dataclass_fields():48def test_parameter_model_tracks_dataclass_fields():
48 assert "cells_across" in ALLOWED_SURFACE_MESH_PARAMETER_KEYS49 model_keys = set(SurfaceMeshParametersConfig.model_fields)
49 assert "device" in ALLOWED_SURFACE_MESH_PARAMETER_KEYS50 dataclass_keys = {field.name for field in dataclasses.fields(params.SurfaceMeshParameters)}
50 assert "road_surface_suffix" not in ALLOWED_SURFACE_MESH_PARAMETER_KEYS51 assert model_keys == dataclass_keys
52 assert "road_surface_suffix" not in model_keys
5153
5254
53def test_build_config_deep_merges_overrides():55def test_build_config_deep_merges_overrides():
54 config = build_surface_mesh_config(56 config = build_surface_mesh_config(
Importance #20: tests/test_surface_mesh_config.py @@ -57,8 +59,21 @@
57 assert config["surface_mesh_parameters"]["cells_across"] == 759 assert config["surface_mesh_parameters"]["cells_across"] == 7
58 assert config["surface_mesh_parameters"]["row_spacing_m"] == 3.060 assert config["surface_mesh_parameters"]["row_spacing_m"] == 3.0
5961
6062
63def test_build_config_coerces_string_overrides():
64 config = build_surface_mesh_config(
65 overrides={
66 "surface_mesh_parameters": {
67 "row_spacing_m": "4.5",
68 "save_diagnostics": "yes",
69 }
70 }
71 )
72 assert config["surface_mesh_parameters"]["row_spacing_m"] == 4.5
73 assert config["surface_mesh_parameters"]["save_diagnostics"] is True
74
75
61def test_build_config_rejects_unknown_override_keys():76def test_build_config_rejects_unknown_override_keys():
62 with pytest.raises(SurfaceMeshConfigError):77 with pytest.raises(SurfaceMeshConfigError):
63 build_surface_mesh_config(overrides={"surface_parameters": {}})78 build_surface_mesh_config(overrides={"surface_parameters": {}})
6479
Importance #21: tests/test_surface_mesh_config.py @@ -89,14 +104,14 @@
89 ("key", "value"),104 ("key", "value"),
90 [105 [
91 ("cells_across", 0),106 ("cells_across", 0),
92 ("cells_across", 2.5),107 ("cells_across", 2.5),
93 ("row_spacing_m", "3"),108 ("row_spacing_m", "abc"),
94 ("row_spacing_m", 0),109 ("row_spacing_m", 0),
95 ("min_points_per_cell", 1),110 ("min_points_per_cell", 1),
96 ("min_measured_edge_fraction", 1.5),111 ("min_measured_edge_fraction", 1.5),
97 ("z_trim_mad_factor", -1),112 ("z_trim_mad_factor", -1),
98 ("save_diagnostics", "yes"),113 ("save_diagnostics", "maybe"),
99 ("device", ""),114 ("device", ""),
100 ],115 ],
101)116)
102def test_normalize_rejects_invalid_parameter_values(key, value):117def test_normalize_rejects_invalid_parameter_values(key, value):