Back to report index

guardrails-seg3d 171ba86: AI3D-379 Align config module with fleet pattern

Miroslav Simko <ms@iolabs.ch> 2026-09-02T09:39:31+02:00

Commit #63 · 5 snippets

 README.md            |  30 +++---
 guardrails/config.py | 262 ++++++++++++++++++++++++++++++---------------------
 tests/test_config.py |  79 ++++++++++------
 3 files changed, 215 insertions(+), 156 deletions(-)
Importance #1: guardrails/config.py @@ -1,227 +1,269 @@
1"""Detector configuration.1"""Detector configuration for the guardrails point-cloud detector.
22
3Mirrors the config convention used by the iolabs point-cloud packages3The schema is `DetectorConfig` (a `config_loader.ConfigModel`), mirroring
4(``iolabs_point_cloud_segmentation_trajectory`` etc.): the package owns a4`guardrails.default.json` key for key: unknown keys are rejected and raw JSON /
5``guardrails.default.json`` algorithm config, and a typed params object5`--set` values are coerced to the declared field types by the shared layer.
6(:class:`DetectorConfig`) is loaded from it at CLI start. Runtime overrides are6
7applied through repeatable ``--set KEY=VALUE`` flags, never repo-local JSON.7Adding a config key means adding the field to the model and the same key to
8``config.py`` is the loader/schema: the pydantic model field set is the schema8`guardrails.default.json` — nothing else. Unknown keys are rejected.
9and every field default is kept identical to ``guardrails.default.json``9
10(guarded by a unit test), so ``DetectorConfig()`` and ``load_config()`` agree.10`load_default_config_dict` returns a plain `dict`; `config_from_dict`,
1111`load_config` and `with_overrides` return the frozen `DetectorConfig`.
12To add a config key: add a field on :class:`DetectorConfig` and the matching12Runtime overrides come from repeatable `--set KEY=VALUE`, never repo-local JSON.
13key/value on ``guardrails.default.json``. Nothing else.
14"""13"""
1514
16from __future__ import annotations15from __future__ import annotations
1716
18import json
19import logging17import logging
20from pathlib import Path
21from typing import Any18from typing import Any
2219
20import pydantic
23from iolabs.common import config_loader21from iolabs.common import config_loader
2422
25logger = logging.getLogger(__name__)23logger = logging.getLogger(__name__)
2624
27_PACKAGE_NAME = "guardrails"25_PACKAGE_NAME = "guardrails"
28_DEFAULT_CONFIG_NAME = "guardrails.default.json"26_DEFAULT_FILENAME = "guardrails.default.json"
27_CONTEXT = "guardrails config"
2928
3029
31class DetectorConfig(config_loader.ConfigModel):30class DetectorConfig(config_loader.ConfigModel):
32 """Spatial and geometric thresholds, in metres unless stated otherwise."""31 """Spatial and geometric thresholds, in metres unless stated otherwise."""
3332
34 # Ground model33 # Ground model
35 ground_cell_m: float = 0.7534 ground_cell_m: float = pydantic.Field(0.75, gt=0)
36 ground_percentile: float = 8.035 ground_percentile: float = pydantic.Field(8.0, ge=0, le=100)
3736
38 # Corridor crop (station / offset frame)37 # Corridor crop (station / offset frame)
39 corridor_offset_min_m: float = 1.538 corridor_offset_min_m: float = pydantic.Field(1.5, ge=0)
40 corridor_offset_max_m: float = 10.039 corridor_offset_max_m: float = pydantic.Field(10.0, gt=0)
41 corridor_include_median_zone: bool = True40 corridor_include_median_zone: bool = True
42 median_corridor_offset_min_m: float = 0.841 median_corridor_offset_min_m: float = pydantic.Field(0.8, ge=0)
43 median_corridor_offset_max_m: float = 3.842 median_corridor_offset_max_m: float = pydantic.Field(3.8, gt=0)
44 corridor_max_height_m: float = 2.043 corridor_max_height_m: float = pydantic.Field(2.0, gt=0)
45 station_window_m: float = 5.044 station_window_m: float = pydantic.Field(5.0, gt=0)
46 median_side_max_offset_m: float = 3.545 median_side_max_offset_m: float = pydantic.Field(3.5, ge=0)
4746
48 # Occupancy grid for candidate cells47 # Occupancy grid for candidate cells
49 occupancy_cell_m: float = 0.1048 occupancy_cell_m: float = pydantic.Field(0.10, gt=0)
5049
51 # Height band for initial point candidates (also drives candidates overlay)50 # Height band for initial point candidates (also drives candidates overlay)
52 min_height_m: float = 0.2051 min_height_m: float = pydantic.Field(0.20, ge=0)
53 max_height_m: float = 1.3052 max_height_m: float = pydantic.Field(1.30, gt=0)
5453
55 # Per-cell rail-band fraction and mean-height gates54 # Per-cell rail-band fraction and mean-height gates
56 rail_band_min_m: float = 0.3555 rail_band_min_m: float = pydantic.Field(0.35, ge=0)
57 rail_band_max_m: float = 0.8556 rail_band_max_m: float = pydantic.Field(0.85, gt=0)
58 min_cell_points: int = 357 min_cell_points: int = pydantic.Field(3, ge=1)
59 min_rail_points: int = 258 min_rail_points: int = pydantic.Field(2, ge=1)
60 min_rail_fraction: float = 0.4059 min_rail_fraction: float = pydantic.Field(0.40, ge=0, le=1)
61 min_mean_height_m: float = 0.4260 min_mean_height_m: float = pydantic.Field(0.42, ge=0)
62 max_mean_height_m: float = 0.7861 max_mean_height_m: float = pydantic.Field(0.78, gt=0)
6362
64 # Vegetation rejection: compact height-above-ground spread within a cell63 # Vegetation rejection: compact height-above-ground spread within a cell
65 max_cell_height_spread_m: float = 0.5064 max_cell_height_spread_m: float = pydantic.Field(0.50, ge=0)
6665
67 # Tall-object fraction per cell (trees, poles)66 # Tall-object fraction per cell (trees, poles)
68 tall_min_m: float = 1.3067 tall_min_m: float = pydantic.Field(1.30, ge=0)
69 tall_max_m: float = 4.5068 tall_max_m: float = pydantic.Field(4.50, gt=0)
70 max_tall_fraction: float = 0.1269 max_tall_fraction: float = pydantic.Field(0.12, ge=0, le=1)
7170
72 # Local covariance / eigenvector candidate filter (cell-level)71 # Local covariance / eigenvector candidate filter (cell-level)
73 eigen_neighborhood_radius_m: float = 0.4072 eigen_neighborhood_radius_m: float = pydantic.Field(0.40, gt=0)
74 eigen_min_neighbors: int = 573 eigen_min_neighbors: int = pydantic.Field(5, ge=1)
75 min_linearity: float = 0.3074 min_linearity: float = pydantic.Field(0.30, ge=0, le=1)
76 min_verticality: float = 0.1575 min_verticality: float = pydantic.Field(0.15, ge=0, le=1)
77 use_eigen_cell_filter: bool = False76 use_eigen_cell_filter: bool = False
7877
79 # DBSCAN clustering on selected occupancy cells78 # DBSCAN clustering on selected occupancy cells
80 cluster_eps_m: float = 0.2079 cluster_eps_m: float = pydantic.Field(0.20, gt=0)
81 cluster_min_samples: int = 380 cluster_min_samples: int = pydantic.Field(3, ge=1)
8281
83 # Post-cluster merge of collinear fragments82 # Post-cluster merge of collinear fragments
84 merge_gap_m: float = 4.583 merge_gap_m: float = pydantic.Field(4.5, ge=0)
85 merge_angle_deg: float = 15.084 merge_angle_deg: float = pydantic.Field(15.0, ge=0, le=180)
86 merge_lateral_max_m: float = 0.5085 merge_lateral_max_m: float = pydantic.Field(0.50, ge=0)
8786
88 # Occlusion bridging: join collinear fragments across a parked-vehicle /87 # Occlusion bridging: join collinear fragments across a parked-vehicle /
89 # occlusion shadow when heading and offset stay continuous (defect 4). The88 # occlusion shadow when heading and offset stay continuous (defect 4). The
90 # bridged station interval is recorded in ``gap_spans`` (never interpolated89 # bridged station interval is recorded in ``gap_spans`` (never interpolated
91 # silently).90 # silently).
92 # Default is conservative (8 m) so bridging never fuses two distinct91 # Default is conservative (8 m) so bridging never fuses two distinct
93 # barriers into one instance; raise via --set occlusion_bridge_max_m=15 for92 # barriers into one instance; raise via --set occlusion_bridge_max_m=15 for
94 # datasets with longer occlusion shadows.93 # datasets with longer occlusion shadows.
95 occlusion_bridge_max_m: float = 8.094 occlusion_bridge_max_m: float = pydantic.Field(8.0, ge=0)
96 occlusion_bridge_max_angle_deg: float = 4.095 occlusion_bridge_max_angle_deg: float = pydantic.Field(4.0, ge=0, le=180)
97 occlusion_bridge_max_lateral_m: float = 0.4096 occlusion_bridge_max_lateral_m: float = pydantic.Field(0.40, ge=0)
9897
99 # Parallel-face deduplication (two faces of one physical rail).98 # Parallel-face deduplication (two faces of one physical rail).
100 # ``dedupe_*`` are retained for backward compatibility; the active policy is99 # ``dedupe_*`` are retained for backward compatibility; the active policy is
101 # driven by ``merge_face_*`` (see README "Face / barrier merge policy").100 # driven by ``merge_face_*`` (see README "Face / barrier merge policy").
102 dedupe_face_max_sep_m: float = 1.0101 dedupe_face_max_sep_m: float = pydantic.Field(1.0, ge=0)
103 dedupe_max_angle_deg: float = 12.0102 dedupe_max_angle_deg: float = pydantic.Field(12.0, ge=0, le=180)
104 merge_face_max_spacing_m: float = 1.3103 merge_face_max_spacing_m: float = pydantic.Field(1.3, ge=0)
105 merge_face_max_heading_deg: float = 5.0104 merge_face_max_heading_deg: float = pydantic.Field(5.0, ge=0, le=180)
106 merge_face_min_station_overlap: float = 0.5105 merge_face_min_station_overlap: float = pydantic.Field(0.5, ge=0, le=1)
107 merge_face_max_faces: int = 2106 merge_face_max_faces: int = pydantic.Field(2, ge=1)
108107
109 # Instance acceptance (applied after merge)108 # Instance acceptance (applied after merge)
110 min_length_m: float = 12.0109 min_length_m: float = pydantic.Field(12.0, ge=0)
111 max_local_width_m: float = 0.75110 max_local_width_m: float = pydantic.Field(0.75, gt=0)
112 min_longitudinal_coverage: float = 0.35111 min_longitudinal_coverage: float = pydantic.Field(0.35, ge=0, le=1)
113112
114 # Ordered-walk polyline construction113 # Ordered-walk polyline construction
115 polyline_bin_m: float = 1.0114 polyline_bin_m: float = pydantic.Field(1.0, gt=0)
116 polyline_smooth_window: int = 5115 polyline_smooth_window: int = pydantic.Field(5, ge=1)
117 walk_max_step_m: float = 0.30116 walk_max_step_m: float = pydantic.Field(0.30, gt=0)
118117
119 # Gap recording along station118 # Gap recording along station
120 gap_min_span_m: float = 2.0119 gap_min_span_m: float = pydantic.Field(2.0, ge=0)
121120
122 # Vehicle / occlusion-shadow rejection on cluster height distribution121 # Vehicle / occlusion-shadow rejection on cluster height distribution
123 max_cluster_height_spread_m: float = 0.80122 max_cluster_height_spread_m: float = pydantic.Field(0.80, ge=0)
124 max_cluster_p95_height_m: float = 1.15123 max_cluster_p95_height_m: float = pydantic.Field(1.15, ge=0)
125124
126 # Straightness check along sliding window (short clusters only)125 # Straightness check along sliding window (short clusters only)
127 straightness_window_m: float = 10.0126 straightness_window_m: float = pydantic.Field(10.0, gt=0)
128 max_straightness_deviation_m: float = 0.50127 max_straightness_deviation_m: float = pydantic.Field(0.50, ge=0)
129 straightness_max_length_m: float = 25.0128 straightness_max_length_m: float = pydantic.Field(25.0, ge=0)
130129
131 # Heuristic type classification thresholds130 # Heuristic type classification thresholds
132 w_beam_min_height_m: float = 0.40131 w_beam_min_height_m: float = pydantic.Field(0.40, ge=0)
133 w_beam_max_height_m: float = 0.90132 w_beam_max_height_m: float = pydantic.Field(0.90, gt=0)
134 w_beam_max_height_spread_m: float = 0.55133 w_beam_max_height_spread_m: float = pydantic.Field(0.55, ge=0)
135 concrete_min_height_m: float = 0.80134 concrete_min_height_m: float = pydantic.Field(0.80, ge=0)
136 concrete_max_height_spread_m: float = 0.45135 concrete_max_height_spread_m: float = pydantic.Field(0.45, ge=0)
137 cable_suspect_max_spread_m: float = 0.25136 cable_suspect_max_spread_m: float = pydantic.Field(0.25, ge=0)
138137
139 # Per-run confidence heuristic (0-1); see README "Run confidence".138 # Per-run confidence heuristic (0-1); see README "Run confidence".
140 # confidence = 0.35*support + 0.25*continuity + 0.25*extent + 0.15*height139 # confidence = 0.35*support + 0.25*continuity + 0.25*extent + 0.15*height
141 confidence_density_norm_pts_per_m: float = 500.0140 confidence_density_norm_pts_per_m: float = pydantic.Field(500.0, gt=0)
142 confidence_full_extent_m: float = 40.0141 confidence_full_extent_m: float = pydantic.Field(40.0, gt=0)
143 confidence_max_height_std_m: float = 0.2142 confidence_max_height_std_m: float = pydantic.Field(0.2, gt=0)
144143
145 # Memory hardening (deployment target is a 32 GB RAM Azure node).144 # Memory hardening (deployment target is a 32 GB RAM Azure node).
146 memory_budget_gb: float = 10.0145 memory_budget_gb: float = pydantic.Field(10.0, gt=0)
147 station_process_window_m: float = 5.0146 station_process_window_m: float = pydantic.Field(5.0, gt=0)
148 decimation_enabled: bool = False147 decimation_enabled: bool = False
149 decimation_voxel_m: float = 0.05148 decimation_voxel_m: float = pydantic.Field(0.05, gt=0)
150 decimation_density_cap: int = 400000149 decimation_density_cap: int = pydantic.Field(400000, ge=1)
151 # Records larger than this stream through the corridor crop in chunks of150 # Records larger than this stream through the corridor crop in chunks of
152 # this many points instead of being materialized whole (byte-identical151 # this many points instead of being materialized whole (byte-identical
153 # results for records at or below the threshold, which use the old path).152 # results for records at or below the threshold, which use the old path).
154 record_chunk_points: int = 4000000153 record_chunk_points: int = pydantic.Field(4000000, ge=1)
155 # Exclusion clustering guard: DBSCAN memory scales with the number of154 # Exclusion clustering guard: DBSCAN memory scales with the number of
156 # eps-neighbour pairs. When a cheap grid estimate of that count exceeds155 # eps-neighbour pairs. When a cheap grid estimate of that count exceeds
157 # this cap the exclusion candidates are voxel-decimated first (auto-trigger156 # this cap the exclusion candidates are voxel-decimated first (auto-trigger
158 # only; sparse segments are untouched). segment_134's dense record157 # only; sparse segments are untouched). segment_134's dense record
159 # estimated 4.0e9 pairs (25 GB RSS); curated segments peak at 6.3e8.158 # estimated 4.0e9 pairs (25 GB RSS); curated segments peak at 6.3e8.
160 exclusion_pair_estimate_max: float = 1000000000.0159 exclusion_pair_estimate_max: float = pydantic.Field(1000000000.0, gt=0)
161 exclusion_decimation_cell_m: float = 0.10160 exclusion_decimation_cell_m: float = pydantic.Field(0.10, gt=0)
162 # After the density trigger decimates, the residual DBSCAN runs under the161 # After the density trigger decimates, the residual DBSCAN runs under the
163 # shared iolabs.common.memory_guard watchdog (subprocess + psutil RSS162 # shared iolabs.common.memory_guard watchdog (subprocess + psutil RSS
164 # monitor, hard kill above the limit) as a second line of defense. Mirrors163 # monitor, hard kill above the limit) as a second line of defense. Mirrors
165 # the subcluster_dbscan_memory_guard wiring in164 # the subcluster_dbscan_memory_guard wiring in
166 # iolabs_point_cloud_modelling_lines / iolabs_geometry_geometry.fit_spline.165 # iolabs_point_cloud_modelling_lines / iolabs_geometry_geometry.fit_spline.
167 exclusion_use_shared_watchdog: bool = True166 exclusion_use_shared_watchdog: bool = True
168 exclusion_dbscan_mem_limit_gb: float = 6.0167 exclusion_dbscan_mem_limit_gb: float = pydantic.Field(6.0, gt=0)
169 exclusion_dbscan_timeout_s: float = 120.0168 exclusion_dbscan_timeout_s: float = pydantic.Field(120.0, gt=0)
170169
171170
172class DetectorConfigError(config_loader.ConfigError):171class DetectorConfigError(config_loader.ConfigError):
173 """Raised when the guardrails config holds unknown keys or invalid values."""172 """Raised when guardrails config contains unsupported keys or values."""
174
175
176def _default_config_path() -> Path:
177 if __package__ in {None, ""}:
178 return Path(__file__).resolve().with_name(_DEFAULT_CONFIG_NAME)
179 return config_loader.default_config_path(__package__, _DEFAULT_CONFIG_NAME)
180173
181174
182def load_default_config_dict() -> dict[str, Any]:175def load_default_config_dict() -> dict[str, Any]:
183 """Return the package-owned default config as a plain dict."""176 """Return the package-owned default config as a plain dict.
184 if __package__ in {None, ""}:177
185 with _default_config_path().open("r", encoding="utf-8") as handle:178 Returns:
186 return json.load(handle)179 The decoded ``guardrails.default.json`` object.
187 return config_loader.load_packaged_json(__package__, _DEFAULT_CONFIG_NAME)180 """
181 return config_loader.load_packaged_json(_PACKAGE_NAME, _DEFAULT_FILENAME)
188182
189183
190def config_from_dict(raw: dict[str, Any]) -> DetectorConfig:184def config_from_dict(raw: dict[str, Any]) -> DetectorConfig:
191 """Build a validated :class:`DetectorConfig` from a raw mapping."""185 """Build a validated :class:`DetectorConfig` from a raw mapping.
186
187 Args:
188 raw: Merged config mapping (packaged defaults plus overrides).
189
190 Returns:
191 The validated config.
192
193 Raises:
194 DetectorConfigError: ``raw`` holds an unknown key or a value outside
195 its declared type/range.
196 """
192 return config_loader.validate_config(197 return config_loader.validate_config(
193 DetectorConfig,198 DetectorConfig,
194 raw,199 raw,
195 context="guardrails config",200 context=_CONTEXT,
196 error_cls=DetectorConfigError,201 error_cls=DetectorConfigError,
197 )202 )
198203
199204
200def load_config(overrides: dict[str, Any] | None = None) -> DetectorConfig:205def load_config(overrides: dict[str, Any] | None = None) -> DetectorConfig:
201 """Load the default config and apply flat ``PATH=VALUE`` overrides.206 """Load the packaged default config and apply flat ``KEY=VALUE`` overrides.
202207
203 Overrides come from the CLI ``--set`` flag (already parsed into a dict).208 Overrides come from the CLI ``--set`` flag (already parsed into a dict).
209
210 Args:
211 overrides: Flat mapping of config key to value, or ``None``.
212
213 Returns:
214 The validated config.
215
216 Raises:
217 DetectorConfigError: An override names an unknown key or holds a value
218 outside its declared type/range.
204 """219 """
205 config_path = None
206 if __package__ in {None, ""}:
207 config_path = Path(__file__).resolve().with_name(_DEFAULT_CONFIG_NAME)
208 config = config_loader.load_config(220 config = config_loader.load_config(
209 DetectorConfig,221 DetectorConfig,
210 package=__package__ or _PACKAGE_NAME,222 package=_PACKAGE_NAME,
211 filename=_DEFAULT_CONFIG_NAME,223 filename=_DEFAULT_FILENAME,
212 overrides=overrides,224 overrides=overrides,
213 config_path=config_path,225 context=_CONTEXT,
214 context="guardrails config",
215 error_cls=DetectorConfigError,226 error_cls=DetectorConfigError,
216 )227 )
217 if overrides:228 if overrides:
218 logger.info("Config overrides applied: %s", ", ".join(sorted(overrides)))229 logger.info("Config overrides applied: %s", ", ".join(sorted(overrides)))
219 return config230 return config
220231
221232
233def with_overrides(config: DetectorConfig, updates: dict[str, Any]) -> DetectorConfig:
234 """Return a re-validated copy of *config* with *updates* applied.
235
236 Unlike ``model_copy(update=...)``, which writes raw values straight into the
237 copy, this rebuilds the model, so unknown keys, wrong types and out-of-range
238 values are rejected exactly as they are on load.
239
240 Args:
241 config: The config to derive from; never mutated (frozen model).
242 updates: Field name to new value.
243
244 Returns:
245 A validated copy carrying *updates*.
246
247 Raises:
248 DetectorConfigError: *updates* names an unknown field or holds a value
249 outside its declared type/range.
250 """
251 return config_from_dict({**config.model_dump(), **updates})
252
253
222def parse_set_overrides(raw_overrides: list[str] | None) -> dict[str, Any]:254def parse_set_overrides(raw_overrides: list[str] | None) -> dict[str, Any]:
223 """Parse repeated ``--set KEY=VALUE`` strings, JSON-decoding each value."""255 """Parse repeated ``--set KEY=VALUE`` strings, JSON-decoding each value.
256
257 Args:
258 raw_overrides: Raw ``KEY=VALUE`` strings from the CLI, or ``None``.
259
260 Returns:
261 A flat mapping of config key to decoded value; later duplicates win.
262
263 Raises:
264 DetectorConfigError: An override is missing its ``=``.
265 """
224 return config_loader.parse_set_overrides(266 return config_loader.parse_set_overrides(
225 raw_overrides,267 raw_overrides,
226 error_cls=DetectorConfigError,268 error_cls=DetectorConfigError,
227 )269 )
Importance #2: tests/test_config.py @@ -1,59 +1,78 @@
1"""Config schema/loader tests: JSON parity, rejection, overrides, coercion."""
2
1import pydantic3import pydantic
2import pytest4import pytest
5from iolabs.common import config_loader
36
4from guardrails import config as config_module7from guardrails import config as config_module
58
69
7def test_default_json_matches_model_defaults() -> None:10def test_model_defaults_match_packaged_json() -> None:
8 """guardrails.default.json is the schema source of truth; keep it in sync."""11 """guardrails.default.json is the schema mirror; compare the whole dict."""
9 defaults = config_module.DetectorConfig().model_dump()12 assert config_module.load_default_config_dict() == (
10 json_config = config_module.load_default_config_dict()13 config_module.DetectorConfig().model_dump()
11 assert set(json_config) == set(defaults)14 )
12 for key, value in defaults.items():
13 assert json_config[key] == value, key
1415
1516
16def test_load_config_without_overrides_equals_defaults() -> None:17def test_load_config_returns_packaged_defaults() -> None:
17 assert config_module.load_config() == config_module.DetectorConfig()18 assert config_module.load_config() == config_module.DetectorConfig()
1819
1920
20def test_parse_set_overrides_json_decodes_values() -> None:21def test_error_class_is_config_error() -> None:
21 parsed = config_module.parse_set_overrides(22 assert issubclass(config_module.DetectorConfigError, config_loader.ConfigError)
22 ["merge_face_max_spacing_m=1.5", "decimation_enabled=true", "memory_budget_gb=8"]23 assert issubclass(config_module.DetectorConfigError, ValueError)
23 )
24 assert parsed == {
25 "merge_face_max_spacing_m": 1.5,
26 "decimation_enabled": True,
27 "memory_budget_gb": 8,
28 }
2924
3025
31def test_load_config_applies_overrides_with_type_coercion() -> None:26def test_unknown_top_level_key_is_rejected() -> None:
27 with pytest.raises(config_module.DetectorConfigError, match="not_a_key"):
28 config_module.config_from_dict(
29 {**config_module.load_default_config_dict(), "not_a_key": 1}
30 )
31
32
33def test_out_of_range_value_is_rejected() -> None:
34 with pytest.raises(config_module.DetectorConfigError, match="min_rail_fraction"):
35 config_module.load_config({"min_rail_fraction": 1.5})
36
37
38def test_overrides_apply_onto_defaults() -> None:
32 config = config_module.load_config(39 config = config_module.load_config(
33 {"decimation_enabled": "true", "merge_face_max_faces": 3}40 {"decimation_enabled": "true", "merge_face_max_faces": 3}
34 )41 )
35 assert config.decimation_enabled is True42 assert config.decimation_enabled is True
36 assert config.merge_face_max_faces == 343 assert config.merge_face_max_faces == 3
44 assert config.merge_gap_m == config_module.DetectorConfig().merge_gap_m
3745
3846
39def test_unknown_key_rejected() -> None:47def test_set_override_coercion_and_rejection() -> None:
48 parsed = config_module.parse_set_overrides(
49 [
50 "merge_face_max_spacing_m=1.5",
51 "decimation_enabled=true",
52 "memory_budget_gb=8",
53 ]
54 )
55 assert parsed == {
56 "merge_face_max_spacing_m": 1.5,
57 "decimation_enabled": True,
58 "memory_budget_gb": 8,
59 }
40 with pytest.raises(config_module.DetectorConfigError):60 with pytest.raises(config_module.DetectorConfigError):
41 config_module.config_from_dict(61 config_module.parse_set_overrides(["missing_equals_sign"])
42 {**config_module.load_default_config_dict(), "not_a_key": 1}
43 )
44
45
46def test_invalid_value_rejected() -> None:
47 with pytest.raises(config_module.DetectorConfigError, match="merge_face_max_faces"):62 with pytest.raises(config_module.DetectorConfigError, match="merge_face_max_faces"):
48 config_module.load_config({"merge_face_max_faces": "abc"})63 config_module.load_config({"merge_face_max_faces": "abc"})
4964
5065
66def test_with_overrides_revalidates_updates() -> None:
67 """with_overrides() rejects what load_config() rejects, unlike model_copy()."""
68 config = config_module.load_config()
69 assert config_module.with_overrides(config, {"merge_gap_m": 3.0}).merge_gap_m == 3.0
70 assert config.merge_gap_m != 3.0
71 with pytest.raises(config_module.DetectorConfigError):
72 config_module.with_overrides(config, {"not_a_key": 1})
73
74
51def test_config_is_frozen() -> None:75def test_config_is_frozen() -> None:
52 config = config_module.DetectorConfig()76 config = config_module.DetectorConfig()
53 with pytest.raises(pydantic.ValidationError):77 with pytest.raises(pydantic.ValidationError):
54 config.merge_gap_m = 1.078 config.merge_gap_m = 1.0
55
56
57def test_invalid_override_string_rejected() -> None:
58 with pytest.raises(config_module.DetectorConfigError):
59 config_module.parse_set_overrides(["missing_equals_sign"])
Importance #3: README.md @@ -18,29 +18,27 @@
18Each segment directory contains `guardrails.json`, RGB and intensity overlays,18Each segment directory contains `guardrails.json`, RGB and intensity overlays,
19and a red candidate-mask diagnostic. `out/run_summary.json` records per-segment19and a red candidate-mask diagnostic. `out/run_summary.json` records per-segment
20timings, alignment checks and peak RSS.20timings, alignment checks and peak RSS.
2121
22## Configuration (iolabs convention)22## Configuration
2323
24Following the other iolabs point-cloud packages24Defaults live in `guardrails/guardrails.default.json`. The schema is
25(`iolabs_point_cloud_segmentation_trajectory` etc.), the package owns an25`DetectorConfig` in `guardrails/config.py` (a `config_loader.ConfigModel`);
26algorithm config `guardrails/guardrails.default.json`. `guardrails/config.py` is26unknown keys are rejected and field ranges are declared with `Field(ge=..., ...)`.
27the loader/schema: the frozen pydantic `DetectorConfig` model (derived from27**To add a config key: add the field (with its type, default and any `Field`
28`iolabs.common.config_loader.ConfigModel`) is the typed params object and its28range) to the model and the same key with the same default to the JSON — nothing
29field set is the schema. Every model default is kept identical to the JSON29else.** `load_default_config_dict()` returns a plain `dict`; `config_from_dict()`,
30(asserted by `tests/test_config.py`). To add a config key, add a field on30`load_config()` and `with_overrides()` return the frozen `DetectorConfig`.
31`DetectorConfig` and the matching default in the JSON; nothing else.31Runtime overrides come from repeatable `--set KEY=VALUE` (values are
3232JSON-decoded), never repo-local JSON:
33Runtime overrides use the repeatable `--set KEY=VALUE` CLI flag (values are
34JSON-decoded), never repo-local JSON files:
3533
36```bash34```bash
37--set decimation_enabled=true --set memory_budget_gb=8 --set occlusion_bridge_max_m=1535--set decimation_enabled=true --set memory_budget_gb=8 --set occlusion_bridge_max_m=15
38```36```
3937
40Logging mirrors those packages: `logging.getLogger(__name__)` with INFO progress38Logging mirrors the other iolabs packages: `logging.getLogger(__name__)` with
41per stage (ground DEM, per-record corridor candidates, occupancy clustering, XML39INFO progress per stage (ground DEM, per-record corridor candidates, occupancy
42export, per-segment completion with peak RSS).40clustering, XML export, per-segment completion with peak RSS).
4341
44## Face / barrier merge policy42## Face / barrier merge policy
4543
46A single physical rail (e.g. a W-beam) presents up to two near-parallel faces44A single physical rail (e.g. a W-beam) presents up to two near-parallel faces
Importance #4: guardrails/config.py @@ -1,227 +1,269 @@
1"""Detector configuration.1"""Detector configuration for the guardrails point-cloud detector.
22
3Mirrors the config convention used by the iolabs point-cloud packages3The schema is `DetectorConfig` (a `config_loader.ConfigModel`), mirroring
4(``iolabs_point_cloud_segmentation_trajectory`` etc.): the package owns a4`guardrails.default.json` key for key: unknown keys are rejected and raw JSON /
5``guardrails.default.json`` algorithm config, and a typed params object5`--set` values are coerced to the declared field types by the shared layer.
6(:class:`DetectorConfig`) is loaded from it at CLI start. Runtime overrides are6
7applied through repeatable ``--set KEY=VALUE`` flags, never repo-local JSON.7Adding a config key means adding the field to the model and the same key to
8``config.py`` is the loader/schema: the pydantic model field set is the schema8`guardrails.default.json` — nothing else. Unknown keys are rejected.
9and every field default is kept identical to ``guardrails.default.json``9
10(guarded by a unit test), so ``DetectorConfig()`` and ``load_config()`` agree.10`load_default_config_dict` returns a plain `dict`; `config_from_dict`,
1111`load_config` and `with_overrides` return the frozen `DetectorConfig`.
12To add a config key: add a field on :class:`DetectorConfig` and the matching12Runtime overrides come from repeatable `--set KEY=VALUE`, never repo-local JSON.
13key/value on ``guardrails.default.json``. Nothing else.
14"""13"""
1514
16from __future__ import annotations15from __future__ import annotations
1716
18import json
19import logging17import logging
20from pathlib import Path
21from typing import Any18from typing import Any
2219
20import pydantic
23from iolabs.common import config_loader21from iolabs.common import config_loader
2422
25logger = logging.getLogger(__name__)23logger = logging.getLogger(__name__)
2624
27_PACKAGE_NAME = "guardrails"25_PACKAGE_NAME = "guardrails"
28_DEFAULT_CONFIG_NAME = "guardrails.default.json"26_DEFAULT_FILENAME = "guardrails.default.json"
27_CONTEXT = "guardrails config"
2928
3029
31class DetectorConfig(config_loader.ConfigModel):30class DetectorConfig(config_loader.ConfigModel):
32 """Spatial and geometric thresholds, in metres unless stated otherwise."""31 """Spatial and geometric thresholds, in metres unless stated otherwise."""
3332
34 # Ground model33 # Ground model
35 ground_cell_m: float = 0.7534 ground_cell_m: float = pydantic.Field(0.75, gt=0)
36 ground_percentile: float = 8.035 ground_percentile: float = pydantic.Field(8.0, ge=0, le=100)
3736
38 # Corridor crop (station / offset frame)37 # Corridor crop (station / offset frame)
39 corridor_offset_min_m: float = 1.538 corridor_offset_min_m: float = pydantic.Field(1.5, ge=0)
40 corridor_offset_max_m: float = 10.039 corridor_offset_max_m: float = pydantic.Field(10.0, gt=0)
41 corridor_include_median_zone: bool = True40 corridor_include_median_zone: bool = True
42 median_corridor_offset_min_m: float = 0.841 median_corridor_offset_min_m: float = pydantic.Field(0.8, ge=0)
43 median_corridor_offset_max_m: float = 3.842 median_corridor_offset_max_m: float = pydantic.Field(3.8, gt=0)
44 corridor_max_height_m: float = 2.043 corridor_max_height_m: float = pydantic.Field(2.0, gt=0)
45 station_window_m: float = 5.044 station_window_m: float = pydantic.Field(5.0, gt=0)
46 median_side_max_offset_m: float = 3.545 median_side_max_offset_m: float = pydantic.Field(3.5, ge=0)
4746
48 # Occupancy grid for candidate cells47 # Occupancy grid for candidate cells
49 occupancy_cell_m: float = 0.1048 occupancy_cell_m: float = pydantic.Field(0.10, gt=0)
5049
51 # Height band for initial point candidates (also drives candidates overlay)50 # Height band for initial point candidates (also drives candidates overlay)
52 min_height_m: float = 0.2051 min_height_m: float = pydantic.Field(0.20, ge=0)
53 max_height_m: float = 1.3052 max_height_m: float = pydantic.Field(1.30, gt=0)
5453
55 # Per-cell rail-band fraction and mean-height gates54 # Per-cell rail-band fraction and mean-height gates
56 rail_band_min_m: float = 0.3555 rail_band_min_m: float = pydantic.Field(0.35, ge=0)
57 rail_band_max_m: float = 0.8556 rail_band_max_m: float = pydantic.Field(0.85, gt=0)
58 min_cell_points: int = 357 min_cell_points: int = pydantic.Field(3, ge=1)
59 min_rail_points: int = 258 min_rail_points: int = pydantic.Field(2, ge=1)
60 min_rail_fraction: float = 0.4059 min_rail_fraction: float = pydantic.Field(0.40, ge=0, le=1)
61 min_mean_height_m: float = 0.4260 min_mean_height_m: float = pydantic.Field(0.42, ge=0)
62 max_mean_height_m: float = 0.7861 max_mean_height_m: float = pydantic.Field(0.78, gt=0)
6362
64 # Vegetation rejection: compact height-above-ground spread within a cell63 # Vegetation rejection: compact height-above-ground spread within a cell
65 max_cell_height_spread_m: float = 0.5064 max_cell_height_spread_m: float = pydantic.Field(0.50, ge=0)
6665
67 # Tall-object fraction per cell (trees, poles)66 # Tall-object fraction per cell (trees, poles)
68 tall_min_m: float = 1.3067 tall_min_m: float = pydantic.Field(1.30, ge=0)
69 tall_max_m: float = 4.5068 tall_max_m: float = pydantic.Field(4.50, gt=0)
70 max_tall_fraction: float = 0.1269 max_tall_fraction: float = pydantic.Field(0.12, ge=0, le=1)
7170
72 # Local covariance / eigenvector candidate filter (cell-level)71 # Local covariance / eigenvector candidate filter (cell-level)
73 eigen_neighborhood_radius_m: float = 0.4072 eigen_neighborhood_radius_m: float = pydantic.Field(0.40, gt=0)
74 eigen_min_neighbors: int = 573 eigen_min_neighbors: int = pydantic.Field(5, ge=1)
75 min_linearity: float = 0.3074 min_linearity: float = pydantic.Field(0.30, ge=0, le=1)
76 min_verticality: float = 0.1575 min_verticality: float = pydantic.Field(0.15, ge=0, le=1)
77 use_eigen_cell_filter: bool = False76 use_eigen_cell_filter: bool = False
7877
79 # DBSCAN clustering on selected occupancy cells78 # DBSCAN clustering on selected occupancy cells
80 cluster_eps_m: float = 0.2079 cluster_eps_m: float = pydantic.Field(0.20, gt=0)
81 cluster_min_samples: int = 380 cluster_min_samples: int = pydantic.Field(3, ge=1)
8281
83 # Post-cluster merge of collinear fragments82 # Post-cluster merge of collinear fragments
84 merge_gap_m: float = 4.583 merge_gap_m: float = pydantic.Field(4.5, ge=0)
85 merge_angle_deg: float = 15.084 merge_angle_deg: float = pydantic.Field(15.0, ge=0, le=180)
86 merge_lateral_max_m: float = 0.5085 merge_lateral_max_m: float = pydantic.Field(0.50, ge=0)
8786
88 # Occlusion bridging: join collinear fragments across a parked-vehicle /87 # Occlusion bridging: join collinear fragments across a parked-vehicle /
89 # occlusion shadow when heading and offset stay continuous (defect 4). The88 # occlusion shadow when heading and offset stay continuous (defect 4). The
90 # bridged station interval is recorded in ``gap_spans`` (never interpolated89 # bridged station interval is recorded in ``gap_spans`` (never interpolated
91 # silently).90 # silently).
92 # Default is conservative (8 m) so bridging never fuses two distinct91 # Default is conservative (8 m) so bridging never fuses two distinct
93 # barriers into one instance; raise via --set occlusion_bridge_max_m=15 for92 # barriers into one instance; raise via --set occlusion_bridge_max_m=15 for
94 # datasets with longer occlusion shadows.93 # datasets with longer occlusion shadows.
95 occlusion_bridge_max_m: float = 8.094 occlusion_bridge_max_m: float = pydantic.Field(8.0, ge=0)
96 occlusion_bridge_max_angle_deg: float = 4.095 occlusion_bridge_max_angle_deg: float = pydantic.Field(4.0, ge=0, le=180)
97 occlusion_bridge_max_lateral_m: float = 0.4096 occlusion_bridge_max_lateral_m: float = pydantic.Field(0.40, ge=0)
9897
99 # Parallel-face deduplication (two faces of one physical rail).98 # Parallel-face deduplication (two faces of one physical rail).
100 # ``dedupe_*`` are retained for backward compatibility; the active policy is99 # ``dedupe_*`` are retained for backward compatibility; the active policy is
101 # driven by ``merge_face_*`` (see README "Face / barrier merge policy").100 # driven by ``merge_face_*`` (see README "Face / barrier merge policy").
102 dedupe_face_max_sep_m: float = 1.0101 dedupe_face_max_sep_m: float = pydantic.Field(1.0, ge=0)
103 dedupe_max_angle_deg: float = 12.0102 dedupe_max_angle_deg: float = pydantic.Field(12.0, ge=0, le=180)
104 merge_face_max_spacing_m: float = 1.3103 merge_face_max_spacing_m: float = pydantic.Field(1.3, ge=0)
105 merge_face_max_heading_deg: float = 5.0104 merge_face_max_heading_deg: float = pydantic.Field(5.0, ge=0, le=180)
106 merge_face_min_station_overlap: float = 0.5105 merge_face_min_station_overlap: float = pydantic.Field(0.5, ge=0, le=1)
107 merge_face_max_faces: int = 2106 merge_face_max_faces: int = pydantic.Field(2, ge=1)
108107
109 # Instance acceptance (applied after merge)108 # Instance acceptance (applied after merge)
110 min_length_m: float = 12.0109 min_length_m: float = pydantic.Field(12.0, ge=0)
111 max_local_width_m: float = 0.75110 max_local_width_m: float = pydantic.Field(0.75, gt=0)
112 min_longitudinal_coverage: float = 0.35111 min_longitudinal_coverage: float = pydantic.Field(0.35, ge=0, le=1)
113112
114 # Ordered-walk polyline construction113 # Ordered-walk polyline construction
115 polyline_bin_m: float = 1.0114 polyline_bin_m: float = pydantic.Field(1.0, gt=0)
116 polyline_smooth_window: int = 5115 polyline_smooth_window: int = pydantic.Field(5, ge=1)
117 walk_max_step_m: float = 0.30116 walk_max_step_m: float = pydantic.Field(0.30, gt=0)
118117
119 # Gap recording along station118 # Gap recording along station
120 gap_min_span_m: float = 2.0119 gap_min_span_m: float = pydantic.Field(2.0, ge=0)
121120
122 # Vehicle / occlusion-shadow rejection on cluster height distribution121 # Vehicle / occlusion-shadow rejection on cluster height distribution
123 max_cluster_height_spread_m: float = 0.80122 max_cluster_height_spread_m: float = pydantic.Field(0.80, ge=0)
124 max_cluster_p95_height_m: float = 1.15123 max_cluster_p95_height_m: float = pydantic.Field(1.15, ge=0)
125124
126 # Straightness check along sliding window (short clusters only)125 # Straightness check along sliding window (short clusters only)
127 straightness_window_m: float = 10.0126 straightness_window_m: float = pydantic.Field(10.0, gt=0)
128 max_straightness_deviation_m: float = 0.50127 max_straightness_deviation_m: float = pydantic.Field(0.50, ge=0)
129 straightness_max_length_m: float = 25.0128 straightness_max_length_m: float = pydantic.Field(25.0, ge=0)
130129
131 # Heuristic type classification thresholds130 # Heuristic type classification thresholds
132 w_beam_min_height_m: float = 0.40131 w_beam_min_height_m: float = pydantic.Field(0.40, ge=0)
133 w_beam_max_height_m: float = 0.90132 w_beam_max_height_m: float = pydantic.Field(0.90, gt=0)
134 w_beam_max_height_spread_m: float = 0.55133 w_beam_max_height_spread_m: float = pydantic.Field(0.55, ge=0)
135 concrete_min_height_m: float = 0.80134 concrete_min_height_m: float = pydantic.Field(0.80, ge=0)
136 concrete_max_height_spread_m: float = 0.45135 concrete_max_height_spread_m: float = pydantic.Field(0.45, ge=0)
137 cable_suspect_max_spread_m: float = 0.25136 cable_suspect_max_spread_m: float = pydantic.Field(0.25, ge=0)
138137
139 # Per-run confidence heuristic (0-1); see README "Run confidence".138 # Per-run confidence heuristic (0-1); see README "Run confidence".
140 # confidence = 0.35*support + 0.25*continuity + 0.25*extent + 0.15*height139 # confidence = 0.35*support + 0.25*continuity + 0.25*extent + 0.15*height
141 confidence_density_norm_pts_per_m: float = 500.0140 confidence_density_norm_pts_per_m: float = pydantic.Field(500.0, gt=0)
142 confidence_full_extent_m: float = 40.0141 confidence_full_extent_m: float = pydantic.Field(40.0, gt=0)
143 confidence_max_height_std_m: float = 0.2142 confidence_max_height_std_m: float = pydantic.Field(0.2, gt=0)
144143
145 # Memory hardening (deployment target is a 32 GB RAM Azure node).144 # Memory hardening (deployment target is a 32 GB RAM Azure node).
146 memory_budget_gb: float = 10.0145 memory_budget_gb: float = pydantic.Field(10.0, gt=0)
147 station_process_window_m: float = 5.0146 station_process_window_m: float = pydantic.Field(5.0, gt=0)
148 decimation_enabled: bool = False147 decimation_enabled: bool = False
149 decimation_voxel_m: float = 0.05148 decimation_voxel_m: float = pydantic.Field(0.05, gt=0)
150 decimation_density_cap: int = 400000149 decimation_density_cap: int = pydantic.Field(400000, ge=1)
151 # Records larger than this stream through the corridor crop in chunks of150 # Records larger than this stream through the corridor crop in chunks of
152 # this many points instead of being materialized whole (byte-identical151 # this many points instead of being materialized whole (byte-identical
153 # results for records at or below the threshold, which use the old path).152 # results for records at or below the threshold, which use the old path).
154 record_chunk_points: int = 4000000153 record_chunk_points: int = pydantic.Field(4000000, ge=1)
155 # Exclusion clustering guard: DBSCAN memory scales with the number of154 # Exclusion clustering guard: DBSCAN memory scales with the number of
156 # eps-neighbour pairs. When a cheap grid estimate of that count exceeds155 # eps-neighbour pairs. When a cheap grid estimate of that count exceeds
157 # this cap the exclusion candidates are voxel-decimated first (auto-trigger156 # this cap the exclusion candidates are voxel-decimated first (auto-trigger
158 # only; sparse segments are untouched). segment_134's dense record157 # only; sparse segments are untouched). segment_134's dense record
159 # estimated 4.0e9 pairs (25 GB RSS); curated segments peak at 6.3e8.158 # estimated 4.0e9 pairs (25 GB RSS); curated segments peak at 6.3e8.
160 exclusion_pair_estimate_max: float = 1000000000.0159 exclusion_pair_estimate_max: float = pydantic.Field(1000000000.0, gt=0)
161 exclusion_decimation_cell_m: float = 0.10160 exclusion_decimation_cell_m: float = pydantic.Field(0.10, gt=0)
162 # After the density trigger decimates, the residual DBSCAN runs under the161 # After the density trigger decimates, the residual DBSCAN runs under the
163 # shared iolabs.common.memory_guard watchdog (subprocess + psutil RSS162 # shared iolabs.common.memory_guard watchdog (subprocess + psutil RSS
164 # monitor, hard kill above the limit) as a second line of defense. Mirrors163 # monitor, hard kill above the limit) as a second line of defense. Mirrors
165 # the subcluster_dbscan_memory_guard wiring in164 # the subcluster_dbscan_memory_guard wiring in
166 # iolabs_point_cloud_modelling_lines / iolabs_geometry_geometry.fit_spline.165 # iolabs_point_cloud_modelling_lines / iolabs_geometry_geometry.fit_spline.
167 exclusion_use_shared_watchdog: bool = True166 exclusion_use_shared_watchdog: bool = True
168 exclusion_dbscan_mem_limit_gb: float = 6.0167 exclusion_dbscan_mem_limit_gb: float = pydantic.Field(6.0, gt=0)
169 exclusion_dbscan_timeout_s: float = 120.0168 exclusion_dbscan_timeout_s: float = pydantic.Field(120.0, gt=0)
170169
171170
172class DetectorConfigError(config_loader.ConfigError):171class DetectorConfigError(config_loader.ConfigError):
173 """Raised when the guardrails config holds unknown keys or invalid values."""172 """Raised when guardrails config contains unsupported keys or values."""
174
175
176def _default_config_path() -> Path:
177 if __package__ in {None, ""}:
178 return Path(__file__).resolve().with_name(_DEFAULT_CONFIG_NAME)
179 return config_loader.default_config_path(__package__, _DEFAULT_CONFIG_NAME)
180173
181174
182def load_default_config_dict() -> dict[str, Any]:175def load_default_config_dict() -> dict[str, Any]:
183 """Return the package-owned default config as a plain dict."""176 """Return the package-owned default config as a plain dict.
184 if __package__ in {None, ""}:177
185 with _default_config_path().open("r", encoding="utf-8") as handle:178 Returns:
186 return json.load(handle)179 The decoded ``guardrails.default.json`` object.
187 return config_loader.load_packaged_json(__package__, _DEFAULT_CONFIG_NAME)180 """
181 return config_loader.load_packaged_json(_PACKAGE_NAME, _DEFAULT_FILENAME)
188182
189183
190def config_from_dict(raw: dict[str, Any]) -> DetectorConfig:184def config_from_dict(raw: dict[str, Any]) -> DetectorConfig:
191 """Build a validated :class:`DetectorConfig` from a raw mapping."""185 """Build a validated :class:`DetectorConfig` from a raw mapping.
186
187 Args:
188 raw: Merged config mapping (packaged defaults plus overrides).
189
190 Returns:
191 The validated config.
192
193 Raises:
194 DetectorConfigError: ``raw`` holds an unknown key or a value outside
195 its declared type/range.
196 """
192 return config_loader.validate_config(197 return config_loader.validate_config(
193 DetectorConfig,198 DetectorConfig,
194 raw,199 raw,
195 context="guardrails config",200 context=_CONTEXT,
196 error_cls=DetectorConfigError,201 error_cls=DetectorConfigError,
197 )202 )
198203
199204
200def load_config(overrides: dict[str, Any] | None = None) -> DetectorConfig:205def load_config(overrides: dict[str, Any] | None = None) -> DetectorConfig:
201 """Load the default config and apply flat ``PATH=VALUE`` overrides.206 """Load the packaged default config and apply flat ``KEY=VALUE`` overrides.
202207
203 Overrides come from the CLI ``--set`` flag (already parsed into a dict).208 Overrides come from the CLI ``--set`` flag (already parsed into a dict).
209
210 Args:
211 overrides: Flat mapping of config key to value, or ``None``.
212
213 Returns:
214 The validated config.
215
216 Raises:
217 DetectorConfigError: An override names an unknown key or holds a value
218 outside its declared type/range.
204 """219 """
205 config_path = None
206 if __package__ in {None, ""}:
207 config_path = Path(__file__).resolve().with_name(_DEFAULT_CONFIG_NAME)
208 config = config_loader.load_config(220 config = config_loader.load_config(
209 DetectorConfig,221 DetectorConfig,
210 package=__package__ or _PACKAGE_NAME,222 package=_PACKAGE_NAME,
211 filename=_DEFAULT_CONFIG_NAME,223 filename=_DEFAULT_FILENAME,
212 overrides=overrides,224 overrides=overrides,
213 config_path=config_path,225 context=_CONTEXT,
214 context="guardrails config",
215 error_cls=DetectorConfigError,226 error_cls=DetectorConfigError,
216 )227 )
217 if overrides:228 if overrides:
218 logger.info("Config overrides applied: %s", ", ".join(sorted(overrides)))229 logger.info("Config overrides applied: %s", ", ".join(sorted(overrides)))
219 return config230 return config
220231
221232
233def with_overrides(config: DetectorConfig, updates: dict[str, Any]) -> DetectorConfig:
234 """Return a re-validated copy of *config* with *updates* applied.
235
236 Unlike ``model_copy(update=...)``, which writes raw values straight into the
237 copy, this rebuilds the model, so unknown keys, wrong types and out-of-range
238 values are rejected exactly as they are on load.
239
240 Args:
241 config: The config to derive from; never mutated (frozen model).
242 updates: Field name to new value.
243
244 Returns:
245 A validated copy carrying *updates*.
246
247 Raises:
248 DetectorConfigError: *updates* names an unknown field or holds a value
249 outside its declared type/range.
250 """
251 return config_from_dict({**config.model_dump(), **updates})
252
253
222def parse_set_overrides(raw_overrides: list[str] | None) -> dict[str, Any]:254def parse_set_overrides(raw_overrides: list[str] | None) -> dict[str, Any]:
223 """Parse repeated ``--set KEY=VALUE`` strings, JSON-decoding each value."""255 """Parse repeated ``--set KEY=VALUE`` strings, JSON-decoding each value.
256
257 Args:
258 raw_overrides: Raw ``KEY=VALUE`` strings from the CLI, or ``None``.
259
260 Returns:
261 A flat mapping of config key to decoded value; later duplicates win.
262
263 Raises:
264 DetectorConfigError: An override is missing its ``=``.
265 """
224 return config_loader.parse_set_overrides(266 return config_loader.parse_set_overrides(
225 raw_overrides,267 raw_overrides,
226 error_cls=DetectorConfigError,268 error_cls=DetectorConfigError,
227 )269 )
Importance #5: tests/test_config.py @@ -1,59 +1,78 @@
1"""Config schema/loader tests: JSON parity, rejection, overrides, coercion."""
2
1import pydantic3import pydantic
2import pytest4import pytest
5from iolabs.common import config_loader
36
4from guardrails import config as config_module7from guardrails import config as config_module
58
69
7def test_default_json_matches_model_defaults() -> None:10def test_model_defaults_match_packaged_json() -> None:
8 """guardrails.default.json is the schema source of truth; keep it in sync."""11 """guardrails.default.json is the schema mirror; compare the whole dict."""
9 defaults = config_module.DetectorConfig().model_dump()12 assert config_module.load_default_config_dict() == (
10 json_config = config_module.load_default_config_dict()13 config_module.DetectorConfig().model_dump()
11 assert set(json_config) == set(defaults)14 )
12 for key, value in defaults.items():
13 assert json_config[key] == value, key
1415
1516
16def test_load_config_without_overrides_equals_defaults() -> None:17def test_load_config_returns_packaged_defaults() -> None:
17 assert config_module.load_config() == config_module.DetectorConfig()18 assert config_module.load_config() == config_module.DetectorConfig()
1819
1920
20def test_parse_set_overrides_json_decodes_values() -> None:21def test_error_class_is_config_error() -> None:
21 parsed = config_module.parse_set_overrides(22 assert issubclass(config_module.DetectorConfigError, config_loader.ConfigError)
22 ["merge_face_max_spacing_m=1.5", "decimation_enabled=true", "memory_budget_gb=8"]23 assert issubclass(config_module.DetectorConfigError, ValueError)
23 )
24 assert parsed == {
25 "merge_face_max_spacing_m": 1.5,
26 "decimation_enabled": True,
27 "memory_budget_gb": 8,
28 }
2924
3025
31def test_load_config_applies_overrides_with_type_coercion() -> None:26def test_unknown_top_level_key_is_rejected() -> None:
27 with pytest.raises(config_module.DetectorConfigError, match="not_a_key"):
28 config_module.config_from_dict(
29 {**config_module.load_default_config_dict(), "not_a_key": 1}
30 )
31
32
33def test_out_of_range_value_is_rejected() -> None:
34 with pytest.raises(config_module.DetectorConfigError, match="min_rail_fraction"):
35 config_module.load_config({"min_rail_fraction": 1.5})
36
37
38def test_overrides_apply_onto_defaults() -> None:
32 config = config_module.load_config(39 config = config_module.load_config(
33 {"decimation_enabled": "true", "merge_face_max_faces": 3}40 {"decimation_enabled": "true", "merge_face_max_faces": 3}
34 )41 )
35 assert config.decimation_enabled is True42 assert config.decimation_enabled is True
36 assert config.merge_face_max_faces == 343 assert config.merge_face_max_faces == 3
44 assert config.merge_gap_m == config_module.DetectorConfig().merge_gap_m
3745
3846
39def test_unknown_key_rejected() -> None:47def test_set_override_coercion_and_rejection() -> None:
48 parsed = config_module.parse_set_overrides(
49 [
50 "merge_face_max_spacing_m=1.5",
51 "decimation_enabled=true",
52 "memory_budget_gb=8",
53 ]
54 )
55 assert parsed == {
56 "merge_face_max_spacing_m": 1.5,
57 "decimation_enabled": True,
58 "memory_budget_gb": 8,
59 }
40 with pytest.raises(config_module.DetectorConfigError):60 with pytest.raises(config_module.DetectorConfigError):
41 config_module.config_from_dict(61 config_module.parse_set_overrides(["missing_equals_sign"])
42 {**config_module.load_default_config_dict(), "not_a_key": 1}
43 )
44
45
46def test_invalid_value_rejected() -> None:
47 with pytest.raises(config_module.DetectorConfigError, match="merge_face_max_faces"):62 with pytest.raises(config_module.DetectorConfigError, match="merge_face_max_faces"):
48 config_module.load_config({"merge_face_max_faces": "abc"})63 config_module.load_config({"merge_face_max_faces": "abc"})
4964
5065
66def test_with_overrides_revalidates_updates() -> None:
67 """with_overrides() rejects what load_config() rejects, unlike model_copy()."""
68 config = config_module.load_config()
69 assert config_module.with_overrides(config, {"merge_gap_m": 3.0}).merge_gap_m == 3.0
70 assert config.merge_gap_m != 3.0
71 with pytest.raises(config_module.DetectorConfigError):
72 config_module.with_overrides(config, {"not_a_key": 1})
73
74
51def test_config_is_frozen() -> None:75def test_config_is_frozen() -> None:
52 config = config_module.DetectorConfig()76 config = config_module.DetectorConfig()
53 with pytest.raises(pydantic.ValidationError):77 with pytest.raises(pydantic.ValidationError):
54 config.merge_gap_m = 1.078 config.merge_gap_m = 1.0
55
56
57def test_invalid_override_string_rejected() -> None:
58 with pytest.raises(config_module.DetectorConfigError):
59 config_module.parse_set_overrides(["missing_equals_sign"])