Back to report index

Step 7 modellinglines 1d31948: AI3D-379 Align config module with fleet pattern

Miroslav Simko <ms@iolabs.ch> 2026-09-02T09:39:14+02:00

Commit #42 · 17 snippets

 README.md                                          |  11 +
 src/iolabs_point_cloud_modelling_lines/__init__.py |  12 +-
 src/iolabs_point_cloud_modelling_lines/_config.py  | 468 +++++----------------
 .../_config_model.py                               | 399 ++++++++++++++++++
 tests/test_config.py                               |  29 +-
 5 files changed, 542 insertions(+), 377 deletions(-)
Importance #1: src/iolabs_point_cloud_modelling_lines/_config.py @@ -412,9 +136,9 @@
412 config_path: JSON file read instead of the packaged defaults. Keys it136 config_path: JSON file read instead of the packaged defaults. Keys it
413 omits fall back to the model defaults, which mirror137 omits fall back to the model defaults, which mirror
414 ``cluster_stepper.default.json``.138 ``cluster_stepper.default.json``.
415 """139 """
416 return _load_model(config_path=config_path).model_dump()140 return build_cluster_stepper_config(config_path=config_path)
417141
418142
419def build_cluster_stepper_config(143def build_cluster_stepper_config(
420 *,144 *,
Importance #2: src/iolabs_point_cloud_modelling_lines/_config_model.py @@ -0,0 +1,399 @@
1"""Field declarations for the cluster-stepper config.
2
3The schema is `ClusterStepperConfig` (a `config_loader.ConfigModel`), mirroring
4`cluster_stepper.default.json` key for key.
5
6Adding a config key means adding the field to the model and the same key to
7`cluster_stepper.default.json` — nothing else. Unknown keys are rejected.
8
9Loading, merging and the public entry points live in `_config`, which
10re-exports every name declared here. Sequence fields stay `list[...]` because
11the entry points return a plain `dict` that callers are free to mutate.
12"""
13
14from __future__ import annotations
15
16import pydantic
17from iolabs.common import config_loader
18
19
20class ClusterStepperInitialOutlierRemovalConfig(config_loader.ConfigModel):
21 """Statistical outlier removal before clustering."""
22
23 nb_neighbors: int = pydantic.Field(10, ge=1)
24 std_ratio: float = pydantic.Field(4.5, gt=0.0)
25 max_points_per_segment: int = pydantic.Field(800000, ge=1)
26 n_times_max_points2kill_clustering: float = pydantic.Field(2.0, gt=0.0)
27
28
29class ClusterStepperClusteringConfig(config_loader.ConfigModel):
30 """DBSCAN clustering and cache/visualization toggles."""
31
32 cluster_dir: str = "clusters"
33 dashed_min_length: float = pydantic.Field(0.7, ge=0.0)
34 use_cache: bool = False
35 save_cache: bool = True
36 visualize: bool = False
37 enable_density_filtering: bool = True
38 enable_intensity_trimming: bool = False
39 enable_intensity_peak_split: bool = False
40 dbscan_eps: float = pydantic.Field(1.0, gt=0.0)
41 dbscan_min_points: int = pydantic.Field(50, ge=1)
42 min_cluster_size: int = pydantic.Field(200, ge=1)
43 dbscan_guard_threshold_points: int = pydantic.Field(15000, ge=1)
44
45
46class ClusterStepperClusterDensityFilteringConfig(config_loader.ConfigModel):
47 """Density-cutoff filters applied to clusters."""
48
49 density_cutoffs: list[float] = [0.001, 0.002, 0.005, 0.012]
50 cutoff_portions: list[float] = [0.85, 0.85, 0.15, 0.001]
51
52
53class ClusterStepperClusterIntensityFilteringConfig(config_loader.ConfigModel):
54 """Intensity-histogram filters applied to clusters."""
55
56 n_bins: int = pydantic.Field(50, ge=1)
57 intensity_range: list[int] = [0, 255]
58 peak_distance: int = pydantic.Field(5, ge=1)
59 peak_prominence: int = pydantic.Field(10, ge=1)
60 sigma_estimate: float = pydantic.Field(3.0, gt=0.0)
61 sigma_estimate_peak_distance_fraction: float = pydantic.Field(0.125, ge=0.0, le=1.0)
62 n_sigma_intensity_cutoff: float = pydantic.Field(5.0, gt=0.0)
63 min_sigma: float = pydantic.Field(0.5, gt=0.0)
64
65
66class ClusterStepperSubclusterDbscanMemoryGuardConfig(config_loader.ConfigModel):
67 """Memory/time guard around subcluster DBSCAN."""
68
69 mem_limit_bytes: int = pydantic.Field(16106127360, ge=1)
70 mem_poll_interval_s: float = pydantic.Field(0.5, gt=0.0)
71 mem_hard_kill: bool = False
72 timeout_s: float = pydantic.Field(120.0, gt=0.0)
73 progress_in_guarded: bool = False
74 dbscan_guard_threshold_points: int = pydantic.Field(15000, ge=1)
75 dbscan_stats_enabled: bool = True
76 dbscan_stats_output_path: str = (
77 "data/00_external/250812_Color scans/inference/dbscan_stats.csv"
78 )
79 dbscan_stats_format: str = "csv"
80
81
82class ClusterStepperSubclusterVisualizationConfig(config_loader.ConfigModel):
83 """Optional subcluster histogram visualization."""
84
85 visualize_every_cluster: bool = False
86 save_histograms: bool = False
87 histogram_dir: str = "histograms"
88 n_bins: int = pydantic.Field(30, ge=1)
89
90
91class ClusterStepperEdgeLinesSplineFittingConfig(config_loader.ConfigModel):
92 """Spline fitting of edge-line clusters."""
93
94 max_sharp_bend_degrees: float = pydantic.Field(7.5, ge=0.0, le=180.0)
95 max_sharp_bend_rejection_rate: float = pydantic.Field(0.2, ge=0.0, le=1.0)
96 fit_line_length: float = pydantic.Field(3.0, gt=0.0)
97 spline_point_count: int = pydantic.Field(1000, ge=1)
98 target_distance: float = pydantic.Field(0.3, gt=0.0)
99 n_parts_start_line: int = pydantic.Field(6, ge=1)
100 min_width: float = pydantic.Field(0.1, ge=0.0)
101 subcluster_dbscan_eps: float = pydantic.Field(0.2, gt=0.0)
102 subcluster_dbscan_min_samples: int = pydantic.Field(5, ge=1)
103 subcluster_dbscan_n_jobs: int = -1
104 subcluster_dbscan_memory_guard: ClusterStepperSubclusterDbscanMemoryGuardConfig = (
105 ClusterStepperSubclusterDbscanMemoryGuardConfig()
106 )
107 subcluster_visualization: ClusterStepperSubclusterVisualizationConfig = (
108 ClusterStepperSubclusterVisualizationConfig()
109 )
110
111
112class ClusterStepperEdgeLinesExtrapolationConfig(config_loader.ConfigModel):
113 """How far edge-line splines are extrapolated."""
114
115 extrapolate_to: float = pydantic.Field(10.0, ge=0.0)
116 extrapolation_points: int = pydantic.Field(30, ge=1)
117
118
119class ClusterStepperEdgeLinesSegmentPropertiesConfig(config_loader.ConfigModel):
120 """Curvature thresholds that split edge-line splines into segments."""
121
122 angle_change_rate: float = pydantic.Field(5.0, ge=0.0)
123 angle_change_threshold: float = pydantic.Field(45.0, ge=0.0, le=180.0)
124 angle_change_rate_last_segment: float = pydantic.Field(5.0, ge=0.0)
125
126
127class ClusterStepperEdgeLinesConfig(config_loader.ConfigModel):
128 """Edge-lane spline fitting, extrapolation, and segmentation."""
129
130 spline_fitting: ClusterStepperEdgeLinesSplineFittingConfig = (
131 ClusterStepperEdgeLinesSplineFittingConfig()
132 )
133 extrapolation: ClusterStepperEdgeLinesExtrapolationConfig = (
134 ClusterStepperEdgeLinesExtrapolationConfig()
135 )
136 segment_properties: ClusterStepperEdgeLinesSegmentPropertiesConfig = (
137 ClusterStepperEdgeLinesSegmentPropertiesConfig()
138 )
139
140
141class ClusterStepperLineConnectionConfig(config_loader.ConfigModel):
142 """Legacy 3D line-connection gates."""
143
144 perpendicular_distance_threshold: float = pydantic.Field(1.0, ge=0.0)
145 longitudinal_distance_factor: float = pydantic.Field(0.2, ge=0.0)
146 negative_offset: float = 0.5
147
148
149class ClusterStepperCrossSectionConfig(config_loader.ConfigModel):
150 """Cross-section sampling along the road axis."""
151
152 interval: float = pydantic.Field(5.0, gt=0.0)
153 visualize_crosssections: bool = False
154
155
156class ClusterStepperXmlMetadataConfig(config_loader.ConfigModel):
157 """XML export metadata."""
158
159 software_version: str = "0.3.1"
160
161
162class ClusterStepperStartSectionConfig(config_loader.ConfigModel):
163 """Where the walk starts relative to the first plane."""
164
165 distance_from_plane: float = pydantic.Field(10.0, ge=0.0)
166
167
168class ClusterStepperSectionConfig(config_loader.ConfigModel):
169 """Segment walk, visualization, and plane-association settings."""
170
171 segment_dir_basename: str = "segment"
172 planes_filename: str = "run3_planes.npz"
173 surface_filenames_base: str = "run4_road_surface_road_extension_*.ply"
174 step_length: float = pydantic.Field(10.0, gt=0.0)
175 visualize_each_step: bool = False
176 visualize_each_segment: bool = False
177 visualize_road_state_plane_intersections: bool = False
178 visualize_road_side_state_points: bool = False
179 road_axis_color: list[float] = [0.0, 0.0, 1.0]
180 plane_distance_from_lane_threshold: float = pydantic.Field(50.0, ge=0.0)
181 lane_distance_from_road_surface: float = pydantic.Field(0.03, ge=0.0)
182
183
184class ClusterStepperLaneConnection2dConfig(config_loader.ConfigModel):
185 """2D continuation matching used to assemble lanes."""
186
187 lateral_threshold: float = pydantic.Field(0.5, ge=0.0)
188 lateral_slack_per_meter: float = pydantic.Field(0.02, ge=0.0)
189 max_lateral: float = pydantic.Field(1.5, ge=0.0)
190 heading_threshold_degrees: float = pydantic.Field(25.0, ge=0.0, le=180.0)
191 heading_weight: float = pydantic.Field(0.02, ge=0.0)
192 gap_weight: float = pydantic.Field(0.01, ge=0.0)
193 max_along_gap: float = pydantic.Field(40.0, ge=0.0)
194 max_overlap: float = pydantic.Field(1.0, ge=0.0)
195 same_type_only: bool = True
196 merge_max_along_gap: float = pydantic.Field(80.0, ge=0.0)
197 merge_lateral_threshold: float = pydantic.Field(0.6, ge=0.0)
198 merge_heading_threshold_degrees: float = pydantic.Field(20.0, ge=0.0, le=180.0)
199 duplicate_lateral_threshold: float = pydantic.Field(0.7, ge=0.0)
200 merge_max_duplicate_overlap: float = pydantic.Field(40.0, ge=0.0)
201 refinement_max_station_gap: float = pydantic.Field(200.0, ge=0.0)
202 line_chain_max_gap: float = pydantic.Field(2000.0, ge=0.0)
203 chain_max_endpoint_gap: float = pydantic.Field(250.0, ge=0.0)
204 chain_max_lateral_jump: float = pydantic.Field(2.0, ge=0.0)
205
206
207class ClusterStepperLaneRolesConfig(config_loader.ConfigModel):
208 """Line grouping and edge-role dedupe thresholds."""
209
210 line_group_lateral_threshold: float = pydantic.Field(1.0, ge=0.0)
211 min_line_length: float = pydantic.Field(10.0, ge=0.0)
212 min_profile_overlap: float = pydantic.Field(10.0, ge=0.0)
213 edge_dedupe_min_overlap: float = pydantic.Field(30.0, ge=0.0)
214 edge_dedupe_min_separation: float = pydantic.Field(1.5, ge=0.0)
215 edge_dedupe_max_covered_fraction: float = pydantic.Field(0.7, ge=0.0, le=1.0)
216
217
218class ClusterStepperRoadAxisConfig(config_loader.ConfigModel):
219 """Road-axis derivation from completed sides."""
220
221 max_angle_degrees: float = pydantic.Field(5.0, ge=0.0, le=180.0)
222 both_sides_min_coverage: float = pydantic.Field(0.5, ge=0.0, le=1.0)
223 both_sides_sample_spacing: float = pydantic.Field(10.0, gt=0.0)
224 both_sides_max_station_gap: float = pydantic.Field(30.0, ge=0.0)
225 station_merge_tolerance: float = pydantic.Field(2.0, ge=0.0)
226 median_width_min: float = pydantic.Field(1.0, ge=0.0)
227 median_width_max: float = pydantic.Field(25.0, ge=0.0)
228 max_lateral_deviation: float = pydantic.Field(6.0, ge=0.0)
229 max_extension_length: float = pydantic.Field(200.0, ge=0.0)
230 both_sides_merge_lateral_tol: float = pydantic.Field(6.0, ge=0.0)
231 max_blend_length: float = pydantic.Field(300.0, ge=0.0)
232
233
234class ClusterStepperControlPlotConfig(config_loader.ConfigModel):
235 """Control-plot PDF output."""
236
237 enabled: bool = True
238 filename: str = "run7_control_plot.pdf"
239
240
241class ClusterStepperLaneStateConfig(config_loader.ConfigModel):
242 """Lane-state snapshot persistence."""
243
244 enabled: bool = True
245 debug_subdir: str = "run7_lane_state"
246 filename: str = "lane_state_snapshot.json"
247 lazy_load: bool = True
248 incremental_intersection_rebuild: bool = True
249
250
251class ClusterStepperTwoStageConfig(config_loader.ConfigModel):
252 """Two-stage walk and boundary-reconnect settings."""
253
254 partial_snapshot_filename: str = "lane_state_partial.json"
255 partial_glob: str = "*lane_state_partial.json"
256 boundary_reconnect: bool = False
257
258
259class ClusterStepperVisualizationLabelsConfig(config_loader.ConfigModel):
260 """On-geometry text labels."""
261
262 enabled: bool = True
263 show_lane_labels: bool = True
264 show_lane_segment_labels: bool = False
265 show_road_surface_labels: bool = True
266 text_depth: float = pydantic.Field(0.02, ge=0.0)
267 label_offset: float = pydantic.Field(0.6, ge=0.0)
268 z_lift: float = pydantic.Field(0.08, ge=0.0)
269 lane_text_scale: float = pydantic.Field(0.18, ge=0.0)
270 lane_segment_text_scale: float = pydantic.Field(0.12, ge=0.0)
271 road_surface_text_scale: float = pydantic.Field(0.14, ge=0.0)
272 lane_label_color: list[float] = [0.12, 0.12, 0.12]
273 lane_segment_label_color: list[float] = [0.28, 0.36, 0.44]
274
275
276class ClusterStepperRoadSidesConfig(config_loader.ConfigModel):
277 """Road-side assignment and median geometry."""
278
279 visualize_every_intersection: bool = False
280 visualize_every_new_intersection: bool = False
281 visualize_every_road_side_plane: bool = False
282 middle_lane_color: list[float] = [0.0, 1.0, 1.0]
283 edge_lane_color: list[float] = [1.0, 0.0, 1.0]
284 extra_lane_color: list[float] = [0.0, 1.0, 0.0]
285 intersection_extend_search_by_n_segments: int = pydantic.Field(1, ge=0)
286 lane_distance: float = pydantic.Field(3.5, ge=0.0)
287 lane_distance_limits: list[float] = [3.3, 3.9]
288 side_split_lateral_gap: float = pydantic.Field(3.0, ge=0.0)
289 through_line_fraction: float = pydantic.Field(0.3, ge=0.0, le=1.0)
290 median_min_gap: float = pydantic.Field(1.5, ge=0.0)
291 median_max_gap: float = pydantic.Field(20.0, ge=0.0)
292 side_band_margin: float = pydantic.Field(3.5, ge=0.0)
293 left_side_id: str = "B"
294 right_side_id: str = "A"
295 side_window_length: float = pydantic.Field(2000.0, gt=0.0)
296 side_window_stride: float = pydantic.Field(1000.0, gt=0.0)
297 side_max_median_offset: float = pydantic.Field(30.0, ge=0.0)
298 median_max_slope: float = pydantic.Field(0.04, ge=0.0)
299 median_max_extrapolation: float = pydantic.Field(3000.0, ge=0.0)
300
301
302class ClusterStepperLaneSegmentEndsConfig(config_loader.ConfigModel):
303 """Paint-marking end detection along a lane-segment axis."""
304
305 n_bins: int = pydantic.Field(40, ge=1)
306 max_distance: float = pydantic.Field(1.0, ge=0.0)
307 start_from_spline: float = pydantic.Field(0.3, ge=0.0)
308 start_from_line: float = pydantic.Field(0.5, ge=0.0)
309 show_histograms: bool = False
310 histogram_dir: str | None = None
311 axis_sample_spacing: float = pydantic.Field(0.5, gt=0.0)
312 lateral_gate: float = pydantic.Field(0.5, ge=0.0)
313 min_amplitude: float = pydantic.Field(3.0, ge=0.0)
314 min_points: int = pydantic.Field(30, ge=1)
315
316
317class ClusterStepperLaneSegmentWidthConfig(config_loader.ConfigModel):
318 """Paint-marking width measurement along a lane-segment axis."""
319
320 bin_width: float = pydantic.Field(0.01, gt=0.0)
321 lateral_max: float = pydantic.Field(0.5, ge=0.0)
322 axis_sample_spacing: float = pydantic.Field(0.5, gt=0.0)
323 end_margin: float = pydantic.Field(0.15, ge=0.0)
324 min_points: int = pydantic.Field(50, ge=1)
325 min_amplitude: float = pydantic.Field(3.0, ge=0.0)
326 width_min: float = pydantic.Field(0.04, ge=0.0)
327 width_max: float = pydantic.Field(0.6, ge=0.0)
328 flag_width_min: float = pydantic.Field(0.08, ge=0.0)
329 flag_width_max: float = pydantic.Field(0.35, ge=0.0)
330 integration_bin_length: float = pydantic.Field(0.25, gt=0.0)
331 min_bin_points: int = pydantic.Field(3, ge=1)
332 show_histograms: bool = False
333 histogram_dir: str | None = None
334
335
336class ClusterStepperLanePointsExportConfig(config_loader.ConfigModel):
337 """Optional NPZ dumps of lane points by marking type."""
338
339 save_all: bool = True
340 save_dashed: bool = True
341 save_solid: bool = True
342 output_dir: str = "lane_points_exports"
343 filename_all: str = "lane_points_all.npz"
344 filename_dashed: str = "lane_points_dashed.npz"
345 filename_solid: str = "lane_points_solid.npz"
346
347
348class ClusterStepperFileNamingConfig(config_loader.ConfigModel):
349 """Step 6/3 input discovery and Step 7 output stems."""
350
351 cluster_prefix: str = "run6_cluster_"
352 geoshift_filename: str = "run3_geoshift.json"
353 step7_prefix: str = "run7_"
354 lanes_stem: str = "lanes"
355
356
357class ClusterStepperConfig(config_loader.ConfigModel):
358 """Root cluster-stepper config mirroring ``cluster_stepper.default.json``."""
359
360 initial_outlier_removal: ClusterStepperInitialOutlierRemovalConfig = (
361 ClusterStepperInitialOutlierRemovalConfig()
362 )
363 clustering: ClusterStepperClusteringConfig = ClusterStepperClusteringConfig()
364 cluster_density_filtering: ClusterStepperClusterDensityFilteringConfig = (
365 ClusterStepperClusterDensityFilteringConfig()
366 )
367 cluster_intensity_filtering: ClusterStepperClusterIntensityFilteringConfig = (
368 ClusterStepperClusterIntensityFilteringConfig()
369 )
370 edge_lines: ClusterStepperEdgeLinesConfig = ClusterStepperEdgeLinesConfig()
371 line_connection: ClusterStepperLineConnectionConfig = (
372 ClusterStepperLineConnectionConfig()
373 )
374 cross_section: ClusterStepperCrossSectionConfig = ClusterStepperCrossSectionConfig()
375 xml_metadata: ClusterStepperXmlMetadataConfig = ClusterStepperXmlMetadataConfig()
376 start_section: ClusterStepperStartSectionConfig = ClusterStepperStartSectionConfig()
377 cluster_stepper: ClusterStepperSectionConfig = ClusterStepperSectionConfig()
378 lane_connection_2d: ClusterStepperLaneConnection2dConfig = (
379 ClusterStepperLaneConnection2dConfig()
380 )
381 lane_roles: ClusterStepperLaneRolesConfig = ClusterStepperLaneRolesConfig()
382 road_axis: ClusterStepperRoadAxisConfig = ClusterStepperRoadAxisConfig()
383 control_plot: ClusterStepperControlPlotConfig = ClusterStepperControlPlotConfig()
384 lane_state: ClusterStepperLaneStateConfig = ClusterStepperLaneStateConfig()
385 two_stage: ClusterStepperTwoStageConfig = ClusterStepperTwoStageConfig()
386 visualization_labels: ClusterStepperVisualizationLabelsConfig = (
387 ClusterStepperVisualizationLabelsConfig()
388 )
389 road_sides: ClusterStepperRoadSidesConfig = ClusterStepperRoadSidesConfig()
390 lane_segment_ends: ClusterStepperLaneSegmentEndsConfig = (
391 ClusterStepperLaneSegmentEndsConfig()
392 )
393 lane_segment_width: ClusterStepperLaneSegmentWidthConfig = (
394 ClusterStepperLaneSegmentWidthConfig()
395 )
396 lane_points_export: ClusterStepperLanePointsExportConfig = (
397 ClusterStepperLanePointsExportConfig()
398 )
399 file_naming: ClusterStepperFileNamingConfig = ClusterStepperFileNamingConfig()
0
Importance #3: src/iolabs_point_cloud_modelling_lines/_config.py @@ -1,407 +1,131 @@
1"""Cluster-stepper config: packaged JSON defaults validated by a ConfigModel tree."""1"""Cluster-stepper config: packaged JSON defaults validated by a ConfigModel tree.
2
3The schema is `ClusterStepperConfig` (a `config_loader.ConfigModel`), mirroring
4`cluster_stepper.default.json` key for key; the field declarations live in
5`_config_model` and are re-exported here.
6
7Adding a config key means adding the field to the model and the same key to
8`cluster_stepper.default.json` — nothing else. Unknown keys are rejected.
9
10The entry points return a plain `dict` (the validated model dumped).
11"""
212
3from __future__ import annotations13from __future__ import annotations
414
15import logging
5from collections.abc import Mapping16from collections.abc import Mapping
6from pathlib import Path17from pathlib import Path
7from typing import Any18from typing import Any
819
9from iolabs.common import config_loader20from iolabs.common import config_loader
1021
22from ._config_model import (
23 ClusterStepperClusterDensityFilteringConfig,
24 ClusterStepperClusterIntensityFilteringConfig,
25 ClusterStepperClusteringConfig,
26 ClusterStepperConfig,
27 ClusterStepperControlPlotConfig,
28 ClusterStepperCrossSectionConfig,
29 ClusterStepperEdgeLinesConfig,
30 ClusterStepperEdgeLinesExtrapolationConfig,
31 ClusterStepperEdgeLinesSegmentPropertiesConfig,
32 ClusterStepperEdgeLinesSplineFittingConfig,
33 ClusterStepperFileNamingConfig,
34 ClusterStepperInitialOutlierRemovalConfig,
35 ClusterStepperLaneConnection2dConfig,
36 ClusterStepperLanePointsExportConfig,
37 ClusterStepperLaneRolesConfig,
38 ClusterStepperLaneSegmentEndsConfig,
39 ClusterStepperLaneSegmentWidthConfig,
40 ClusterStepperLaneStateConfig,
41 ClusterStepperLineConnectionConfig,
42 ClusterStepperRoadAxisConfig,
43 ClusterStepperRoadSidesConfig,
44 ClusterStepperSectionConfig,
45 ClusterStepperStartSectionConfig,
46 ClusterStepperSubclusterDbscanMemoryGuardConfig,
47 ClusterStepperSubclusterVisualizationConfig,
48 ClusterStepperTwoStageConfig,
49 ClusterStepperVisualizationLabelsConfig,
50 ClusterStepperXmlMetadataConfig,
51)
52
53logger = logging.getLogger(__name__)
54
11_PACKAGE_NAME = "iolabs_point_cloud_modelling_lines"55_PACKAGE_NAME = "iolabs_point_cloud_modelling_lines"
12_DEFAULT_CONFIG_NAME = "cluster_stepper.default.json"56_DEFAULT_FILENAME = "cluster_stepper.default.json"
13_CONTEXT = "cluster-stepper config"57_CONTEXT = "cluster-stepper config"
1458
59__all__ = [
60 "ClusterStepperClusterDensityFilteringConfig",
61 "ClusterStepperClusterIntensityFilteringConfig",
62 "ClusterStepperClusteringConfig",
63 "ClusterStepperConfig",
64 "ClusterStepperConfigError",
65 "ClusterStepperControlPlotConfig",
66 "ClusterStepperCrossSectionConfig",
67 "ClusterStepperEdgeLinesConfig",
68 "ClusterStepperEdgeLinesExtrapolationConfig",
69 "ClusterStepperEdgeLinesSegmentPropertiesConfig",
70 "ClusterStepperEdgeLinesSplineFittingConfig",
71 "ClusterStepperFileNamingConfig",
72 "ClusterStepperInitialOutlierRemovalConfig",
73 "ClusterStepperLaneConnection2dConfig",
74 "ClusterStepperLanePointsExportConfig",
75 "ClusterStepperLaneRolesConfig",
76 "ClusterStepperLaneSegmentEndsConfig",
77 "ClusterStepperLaneSegmentWidthConfig",
78 "ClusterStepperLaneStateConfig",
79 "ClusterStepperLineConnectionConfig",
80 "ClusterStepperRoadAxisConfig",
81 "ClusterStepperRoadSidesConfig",
82 "ClusterStepperSectionConfig",
83 "ClusterStepperStartSectionConfig",
84 "ClusterStepperSubclusterDbscanMemoryGuardConfig",
85 "ClusterStepperSubclusterVisualizationConfig",
86 "ClusterStepperTwoStageConfig",
87 "ClusterStepperVisualizationLabelsConfig",
88 "ClusterStepperXmlMetadataConfig",
89 "build_cluster_stepper_config",
90 "load_cluster_stepper_config",
91 "normalize_cluster_stepper_config",
92]
93
1594
16class ClusterStepperConfigError(config_loader.ConfigError):95class ClusterStepperConfigError(config_loader.ConfigError):
17 """Raised when cluster-stepper config contains unsupported keys or values."""96 """Raised when cluster-stepper config contains unsupported keys or values."""
1897
1998
20class InitialOutlierRemovalConfig(config_loader.ConfigModel):
21 """Statistical outlier removal before clustering."""
22
23 nb_neighbors: int = 10
24 std_ratio: float = 4.5
25 max_points_per_segment: int = 800000
26 n_times_max_points2kill_clustering: float = 2.0
27
28
29class ClusteringConfig(config_loader.ConfigModel):
30 """DBSCAN clustering and cache/visualization toggles."""
31
32 cluster_dir: str = "clusters"
33 dashed_min_length: float = 0.7
34 use_cache: bool = False
35 save_cache: bool = True
36 visualize: bool = False
37 enable_density_filtering: bool = True
38 enable_intensity_trimming: bool = False
39 enable_intensity_peak_split: bool = False
40 dbscan_eps: float = 1.0
41 dbscan_min_points: int = 50
42 min_cluster_size: int = 200
43 dbscan_guard_threshold_points: int = 15000
44
45
46class ClusterDensityFilteringConfig(config_loader.ConfigModel):
47 """Density-cutoff filters applied to clusters."""
48
49 density_cutoffs: list[float] = [0.001, 0.002, 0.005, 0.012]
50 cutoff_portions: list[float] = [0.85, 0.85, 0.15, 0.001]
51
52
53class ClusterIntensityFilteringConfig(config_loader.ConfigModel):
54 """Intensity-histogram filters applied to clusters."""
55
56 n_bins: int = 50
57 intensity_range: list[int] = [0, 255]
58 peak_distance: int = 5
59 peak_prominence: int = 10
60 sigma_estimate: float = 3.0
61 sigma_estimate_peak_distance_fraction: float = 0.125
62 n_sigma_intensity_cutoff: float = 5.0
63 min_sigma: float = 0.5
64
65
66class SubclusterDbscanMemoryGuardConfig(config_loader.ConfigModel):
67 """Memory/time guard around subcluster DBSCAN."""
68
69 mem_limit_bytes: int = 16106127360
70 mem_poll_interval_s: float = 0.5
71 mem_hard_kill: bool = False
72 timeout_s: float = 120.0
73 progress_in_guarded: bool = False
74 dbscan_guard_threshold_points: int = 15000
75 dbscan_stats_enabled: bool = True
76 dbscan_stats_output_path: str = (
77 "data/00_external/250812_Color scans/inference/dbscan_stats.csv"
78 )
79 dbscan_stats_format: str = "csv"
80
81
82class SubclusterVisualizationConfig(config_loader.ConfigModel):
83 """Optional subcluster histogram visualization."""
84
85 visualize_every_cluster: bool = False
86 save_histograms: bool = False
87 histogram_dir: str = "histograms"
88 n_bins: int = 30
89
90
91class EdgeLinesSplineFittingConfig(config_loader.ConfigModel):
92 """Spline fitting of edge-line clusters."""
93
94 max_sharp_bend_degrees: float = 7.5
95 max_sharp_bend_rejection_rate: float = 0.2
96 fit_line_length: float = 3.0
97 spline_point_count: int = 1000
98 target_distance: float = 0.3
99 n_parts_start_line: int = 6
100 min_width: float = 0.1
101 subcluster_dbscan_eps: float = 0.2
102 subcluster_dbscan_min_samples: int = 5
103 subcluster_dbscan_n_jobs: int = -1
104 subcluster_dbscan_memory_guard: SubclusterDbscanMemoryGuardConfig = (
105 SubclusterDbscanMemoryGuardConfig()
106 )
107 subcluster_visualization: SubclusterVisualizationConfig = (
108 SubclusterVisualizationConfig()
109 )
110
111
112class EdgeLinesExtrapolationConfig(config_loader.ConfigModel):
113 """How far edge-line splines are extrapolated."""
114
115 extrapolate_to: float = 10.0
116 extrapolation_points: int = 30
117
118
119class EdgeLinesSegmentPropertiesConfig(config_loader.ConfigModel):
120 """Curvature thresholds that split edge-line splines into segments."""
121
122 angle_change_rate: float = 5.0
123 angle_change_threshold: float = 45.0
124 angle_change_rate_last_segment: float = 5.0
125
126
127class EdgeLinesConfig(config_loader.ConfigModel):
128 """Edge-lane spline fitting, extrapolation, and segmentation."""
129
130 spline_fitting: EdgeLinesSplineFittingConfig = EdgeLinesSplineFittingConfig()
131 extrapolation: EdgeLinesExtrapolationConfig = EdgeLinesExtrapolationConfig()
132 segment_properties: EdgeLinesSegmentPropertiesConfig = (
133 EdgeLinesSegmentPropertiesConfig()
134 )
135
136
137class LineConnectionConfig(config_loader.ConfigModel):
138 """Legacy 3D line-connection gates."""
139
140 perpendicular_distance_threshold: float = 1.0
141 longitudinal_distance_factor: float = 0.2
142 negative_offset: float = 0.5
143
144
145class CrossSectionConfig(config_loader.ConfigModel):
146 """Cross-section sampling along the road axis."""
147
148 interval: float = 5.0
149 visualize_crosssections: bool = False
150
151
152class XmlMetadataConfig(config_loader.ConfigModel):
153 """XML export metadata."""
154
155 software_version: str = "0.3.1"
156
157
158class StartSectionConfig(config_loader.ConfigModel):
159 """Where the walk starts relative to the first plane."""
160
161 distance_from_plane: float = 10.0
162
163
164class ClusterStepperSectionConfig(config_loader.ConfigModel):
165 """Segment walk, visualization, and plane-association settings."""
166
167 segment_dir_basename: str = "segment"
168 planes_filename: str = "run3_planes.npz"
169 surface_filenames_base: str = "run4_road_surface_road_extension_*.ply"
170 step_length: float = 10.0
171 visualize_each_step: bool = False
172 visualize_each_segment: bool = False
173 visualize_road_state_plane_intersections: bool = False
174 visualize_road_side_state_points: bool = False
175 road_axis_color: list[float] = [0.0, 0.0, 1.0]
176 plane_distance_from_lane_threshold: float = 50.0
177 lane_distance_from_road_surface: float = 0.03
178
179
180class LaneConnection2dConfig(config_loader.ConfigModel):
181 """2D continuation matching used to assemble lanes."""
182
183 lateral_threshold: float = 0.5
184 lateral_slack_per_meter: float = 0.02
185 max_lateral: float = 1.5
186 heading_threshold_degrees: float = 25.0
187 heading_weight: float = 0.02
188 gap_weight: float = 0.01
189 max_along_gap: float = 40.0
190 max_overlap: float = 1.0
191 same_type_only: bool = True
192 merge_max_along_gap: float = 80.0
193 merge_lateral_threshold: float = 0.6
194 merge_heading_threshold_degrees: float = 20.0
195 duplicate_lateral_threshold: float = 0.7
196 merge_max_duplicate_overlap: float = 40.0
197 refinement_max_station_gap: float = 200.0
198 line_chain_max_gap: float = 2000.0
199 chain_max_endpoint_gap: float = 250.0
200 chain_max_lateral_jump: float = 2.0
201
202
203class LaneRolesConfig(config_loader.ConfigModel):
204 """Line grouping and edge-role dedupe thresholds."""
205
206 line_group_lateral_threshold: float = 1.0
207 min_line_length: float = 10.0
208 min_profile_overlap: float = 10.0
209 edge_dedupe_min_overlap: float = 30.0
210 edge_dedupe_min_separation: float = 1.5
211 edge_dedupe_max_covered_fraction: float = 0.7
212
213
214class RoadAxisConfig(config_loader.ConfigModel):
215 """Road-axis derivation from completed sides."""
216
217 max_angle_degrees: float = 5.0
218 both_sides_min_coverage: float = 0.5
219 both_sides_sample_spacing: float = 10.0
220 both_sides_max_station_gap: float = 30.0
221 station_merge_tolerance: float = 2.0
222 median_width_min: float = 1.0
223 median_width_max: float = 25.0
224 max_lateral_deviation: float = 6.0
225 max_extension_length: float = 200.0
226 both_sides_merge_lateral_tol: float = 6.0
227 max_blend_length: float = 300.0
228
229
230class ControlPlotConfig(config_loader.ConfigModel):
231 """Control-plot PDF output."""
232
233 enabled: bool = True
234 filename: str = "run7_control_plot.pdf"
235
236
237class LaneStateConfig(config_loader.ConfigModel):
238 """Lane-state snapshot persistence."""
239
240 enabled: bool = True
241 debug_subdir: str = "run7_lane_state"
242 filename: str = "lane_state_snapshot.json"
243 lazy_load: bool = True
244 incremental_intersection_rebuild: bool = True
245
246
247class TwoStageConfig(config_loader.ConfigModel):
248 """Two-stage walk and boundary-reconnect settings."""
249
250 partial_snapshot_filename: str = "lane_state_partial.json"
251 partial_glob: str = "*lane_state_partial.json"
252 boundary_reconnect: bool = False
253
254
255class VisualizationLabelsConfig(config_loader.ConfigModel):
256 """On-geometry text labels."""
257
258 enabled: bool = True
259 show_lane_labels: bool = True
260 show_lane_segment_labels: bool = False
261 show_road_surface_labels: bool = True
262 text_depth: float = 0.02
263 label_offset: float = 0.6
264 z_lift: float = 0.08
265 lane_text_scale: float = 0.18
266 lane_segment_text_scale: float = 0.12
267 road_surface_text_scale: float = 0.14
268 lane_label_color: list[float] = [0.12, 0.12, 0.12]
269 lane_segment_label_color: list[float] = [0.28, 0.36, 0.44]
270
271
272class RoadSidesConfig(config_loader.ConfigModel):
273 """Road-side assignment and median geometry."""
274
275 visualize_every_intersection: bool = False
276 visualize_every_new_intersection: bool = False
277 visualize_every_road_side_plane: bool = False
278 middle_lane_color: list[float] = [0.0, 1.0, 1.0]
279 edge_lane_color: list[float] = [1.0, 0.0, 1.0]
280 extra_lane_color: list[float] = [0.0, 1.0, 0.0]
281 intersection_extend_search_by_n_segments: int = 1
282 lane_distance: float = 3.5
283 lane_distance_limits: list[float] = [3.3, 3.9]
284 side_split_lateral_gap: float = 3.0
285 through_line_fraction: float = 0.3
286 median_min_gap: float = 1.5
287 median_max_gap: float = 20.0
288 side_band_margin: float = 3.5
289 left_side_id: str = "B"
290 right_side_id: str = "A"
291 side_window_length: float = 2000.0
292 side_window_stride: float = 1000.0
293 side_max_median_offset: float = 30.0
294 median_max_slope: float = 0.04
295 median_max_extrapolation: float = 3000.0
296
297
298class LaneSegmentEndsConfig(config_loader.ConfigModel):
299 """Paint-marking end detection along a lane-segment axis."""
300
301 n_bins: int = 40
302 max_distance: float = 1.0
303 start_from_spline: float = 0.3
304 start_from_line: float = 0.5
305 show_histograms: bool = False
306 histogram_dir: str | None = None
307 axis_sample_spacing: float = 0.5
308 lateral_gate: float = 0.5
309 min_amplitude: float = 3.0
310 min_points: int = 30
311
312
313class LaneSegmentWidthConfig(config_loader.ConfigModel):
314 """Paint-marking width measurement along a lane-segment axis."""
315
316 bin_width: float = 0.01
317 lateral_max: float = 0.5
318 axis_sample_spacing: float = 0.5
319 end_margin: float = 0.15
320 min_points: int = 50
321 min_amplitude: float = 3.0
322 width_min: float = 0.04
323 width_max: float = 0.6
324 flag_width_min: float = 0.08
325 flag_width_max: float = 0.35
326 integration_bin_length: float = 0.25
327 min_bin_points: int = 3
328 show_histograms: bool = False
329 histogram_dir: str | None = None
330
331
332class LanePointsExportConfig(config_loader.ConfigModel):
333 """Optional NPZ dumps of lane points by marking type."""
334
335 save_all: bool = True
336 save_dashed: bool = True
337 save_solid: bool = True
338 output_dir: str = "lane_points_exports"
339 filename_all: str = "lane_points_all.npz"
340 filename_dashed: str = "lane_points_dashed.npz"
341 filename_solid: str = "lane_points_solid.npz"
342
343
344class FileNamingConfig(config_loader.ConfigModel):
345 """Step 6/3 input discovery and Step 7 output stems."""
346
347 cluster_prefix: str = "run6_cluster_"
348 geoshift_filename: str = "run3_geoshift.json"
349 step7_prefix: str = "run7_"
350 lanes_stem: str = "lanes"
351
352
353class ClusterStepperConfig(config_loader.ConfigModel):
354 """Root cluster-stepper config mirroring ``cluster_stepper.default.json``."""
355
356 initial_outlier_removal: InitialOutlierRemovalConfig = InitialOutlierRemovalConfig()
357 clustering: ClusteringConfig = ClusteringConfig()
358 cluster_density_filtering: ClusterDensityFilteringConfig = (
359 ClusterDensityFilteringConfig()
360 )
361 cluster_intensity_filtering: ClusterIntensityFilteringConfig = (
362 ClusterIntensityFilteringConfig()
363 )
364 edge_lines: EdgeLinesConfig = EdgeLinesConfig()
365 line_connection: LineConnectionConfig = LineConnectionConfig()
366 cross_section: CrossSectionConfig = CrossSectionConfig()
367 xml_metadata: XmlMetadataConfig = XmlMetadataConfig()
368 start_section: StartSectionConfig = StartSectionConfig()
369 cluster_stepper: ClusterStepperSectionConfig = ClusterStepperSectionConfig()
370 lane_connection_2d: LaneConnection2dConfig = LaneConnection2dConfig()
371 lane_roles: LaneRolesConfig = LaneRolesConfig()
372 road_axis: RoadAxisConfig = RoadAxisConfig()
373 control_plot: ControlPlotConfig = ControlPlotConfig()
374 lane_state: LaneStateConfig = LaneStateConfig()
375 two_stage: TwoStageConfig = TwoStageConfig()
376 visualization_labels: VisualizationLabelsConfig = VisualizationLabelsConfig()
377 road_sides: RoadSidesConfig = RoadSidesConfig()
378 lane_segment_ends: LaneSegmentEndsConfig = LaneSegmentEndsConfig()
379 lane_segment_width: LaneSegmentWidthConfig = LaneSegmentWidthConfig()
380 lane_points_export: LanePointsExportConfig = LanePointsExportConfig()
381 file_naming: FileNamingConfig = FileNamingConfig()
382
383
384def _load_model(99def _load_model(
385 *,100 *,
386 overrides: Mapping[str, Any] | None = None,101 overrides: Mapping[str, Any] | None = None,
387 config_path: str | Path | None = None,102 config_path: str | Path | None = None,
388) -> ClusterStepperConfig:103) -> ClusterStepperConfig:
389 """Load packaged defaults, merge overrides, and validate the model."""104 """Load packaged defaults (or *config_path*), merge overrides, and validate."""
105 if config_path is not None:
106 logger.info("Config file applied: %s", config_path)
107 if overrides:
108 logger.info("Config overrides applied: %s", ", ".join(sorted(overrides)))
390 return config_loader.load_config(109 return config_loader.load_config(
391 ClusterStepperConfig,110 ClusterStepperConfig,
392 package=__package__ or _PACKAGE_NAME,111 package=_PACKAGE_NAME,
393 filename=_DEFAULT_CONFIG_NAME,112 filename=_DEFAULT_FILENAME,
394 overrides=overrides,113 overrides=overrides,
395 config_path=config_path,114 config_path=config_path,
396 context=_CONTEXT,115 context=_CONTEXT,
397 error_cls=ClusterStepperConfigError,116 error_cls=ClusterStepperConfigError,
398 )117 )
399118
400119
401def normalize_cluster_stepper_config(raw_config: Mapping[str, Any]) -> dict[str, Any]:120def normalize_cluster_stepper_config(raw_config: Mapping[str, Any]) -> dict[str, Any]:
402 """Deep-merge *raw_config* onto the packaged defaults and validate it."""121 """Validate *raw_config* and fill in the model defaults, returning a dict."""
403 return _load_model(overrides=raw_config).model_dump()122 return config_loader.validate_config(
123 ClusterStepperConfig,
124 raw_config,
125 context=_CONTEXT,
126 error_cls=ClusterStepperConfigError,
127 ).model_dump()
404128
405129
406def load_cluster_stepper_config(130def load_cluster_stepper_config(
407 config_path: str | Path | None = None,131 config_path: str | Path | None = None,
Importance #4: src/iolabs_point_cloud_modelling_lines/__init__.py @@ -1,7 +1,13 @@
1"""Lane assembly from LIDAR clusters, road-side assignment, road-axis derivation, and XML export."""1"""Lane assembly from LIDAR clusters, road-side assignment, road-axis derivation, and XML export."""
22
3from ._config import load_cluster_stepper_config3from ._config import (
4 ClusterStepperConfig,
5 ClusterStepperConfigError,
6 build_cluster_stepper_config,
7 load_cluster_stepper_config,
8 normalize_cluster_stepper_config,
9)
4from .cluster_stepper import ClusterStepper10from .cluster_stepper import ClusterStepper
5from . import (11from . import (
6 cluster_stepper,12 cluster_stepper,
7 edge_lanes,13 edge_lanes,
Importance #5: src/iolabs_point_cloud_modelling_lines/__init__.py @@ -18,9 +24,13 @@
18)24)
1925
20__all__ = [26__all__ = [
21 "ClusterStepper",27 "ClusterStepper",
28 "ClusterStepperConfig",
29 "ClusterStepperConfigError",
30 "build_cluster_stepper_config",
22 "load_cluster_stepper_config",31 "load_cluster_stepper_config",
32 "normalize_cluster_stepper_config",
23 "cluster_stepper",33 "cluster_stepper",
24 "edge_lanes",34 "edge_lanes",
25 "lane",35 "lane",
26 "lane_state",36 "lane_state",
Importance #6: tests/test_config.py @@ -19,17 +19,18 @@
19def test_model_defaults_match_packaged_json():19def test_model_defaults_match_packaged_json():
20 assert _config.ClusterStepperConfig().model_dump() == _packaged_defaults()20 assert _config.ClusterStepperConfig().model_dump() == _packaged_defaults()
2121
2222
23def test_load_defaults_matches_packaged_json():23def test_load_cluster_stepper_config_returns_packaged_defaults():
24 assert _config.load_cluster_stepper_config() == _packaged_defaults()24 assert _config.load_cluster_stepper_config() == _packaged_defaults()
2525
2626
27def test_error_class_is_a_config_error():27def test_error_class_is_config_error():
28 assert issubclass(_config.ClusterStepperConfigError, config_loader.ConfigError)28 assert issubclass(_config.ClusterStepperConfigError, config_loader.ConfigError)
29 assert issubclass(_config.ClusterStepperConfigError, ValueError)
2930
3031
31def test_unknown_root_key_is_rejected():32def test_unknown_top_level_key_is_rejected():
32 with pytest.raises(_config.ClusterStepperConfigError, match="bogus"):33 with pytest.raises(_config.ClusterStepperConfigError, match="bogus"):
33 _config.normalize_cluster_stepper_config({"bogus": 1})34 _config.normalize_cluster_stepper_config({"bogus": 1})
3435
3536
Importance #7: tests/test_config.py @@ -37,9 +38,9 @@
37 with pytest.raises(_config.ClusterStepperConfigError, match="nope"):38 with pytest.raises(_config.ClusterStepperConfigError, match="nope"):
38 _config.normalize_cluster_stepper_config({"clustering": {"nope": 1}})39 _config.normalize_cluster_stepper_config({"clustering": {"nope": 1}})
3940
4041
41def test_overrides_are_deep_merged_and_coerced():42def test_overrides_deep_merge_onto_defaults():
42 config = _config.build_cluster_stepper_config(43 config = _config.build_cluster_stepper_config(
43 overrides={"clustering": {"dbscan_eps": "2.5"}}44 overrides={"clustering": {"dbscan_eps": "2.5"}}
44 )45 )
45 defaults = _packaged_defaults()46 defaults = _packaged_defaults()
Importance #8: tests/test_config.py @@ -47,8 +48,28 @@
47 assert config["clustering"]["use_cache"] == defaults["clustering"]["use_cache"]48 assert config["clustering"]["use_cache"] == defaults["clustering"]["use_cache"]
48 assert config["road_axis"] == defaults["road_axis"]49 assert config["road_axis"] == defaults["road_axis"]
4950
5051
52def test_set_override_coercion_and_rejection():
53 overrides = config_loader.parse_set_overrides(
54 ["clustering.min_cluster_size=1e3", "clustering.use_cache=on"],
55 error_cls=_config.ClusterStepperConfigError,
56 nested=True,
57 )
58 config = _config.build_cluster_stepper_config(overrides=overrides)
59 assert config["clustering"]["min_cluster_size"] == 1000
60 assert config["clustering"]["use_cache"] is True
61 with pytest.raises(_config.ClusterStepperConfigError, match="use_cache"):
62 _config.build_cluster_stepper_config(
63 overrides={"clustering": {"use_cache": "flase"}}
64 )
65
66
67def test_out_of_range_value_is_rejected():
68 with pytest.raises(_config.ClusterStepperConfigError, match="dbscan_eps"):
69 _config.normalize_cluster_stepper_config({"clustering": {"dbscan_eps": -1.0}})
70
71
51def test_bool_is_not_accepted_for_an_int_field():72def test_bool_is_not_accepted_for_an_int_field():
52 with pytest.raises(_config.ClusterStepperConfigError, match="dbscan_min_points"):73 with pytest.raises(_config.ClusterStepperConfigError, match="dbscan_min_points"):
53 _config.normalize_cluster_stepper_config(74 _config.normalize_cluster_stepper_config(
54 {"clustering": {"dbscan_min_points": True}}75 {"clustering": {"dbscan_min_points": True}}
Importance #9: README.md @@ -22,8 +22,19 @@
22## Usage22## Usage
2323
24Builds lane models from Step 6 `run6_cluster_*.npz` outputs: spline-fit clusters, assemble lanes by 2D continuation matching, assign completed lanes to road sides, derive the road axis, and export XML. Consumes the GPU Step 6 cluster metadata API and preserves geoshift metadata in XML output.24Builds lane models from Step 6 `run6_cluster_*.npz` outputs: spline-fit clusters, assemble lanes by 2D continuation matching, assign completed lanes to road sides, derive the road axis, and export XML. Consumes the GPU Step 6 cluster metadata API and preserves geoshift metadata in XML output.
2525
26## Configuration
27
28Defaults live in `src/iolabs_point_cloud_modelling_lines/cluster_stepper.default.json`.
29The schema is `ClusterStepperConfig` in `iolabs_point_cloud_modelling_lines._config_model`
30(a `config_loader.ConfigModel`), re-exported from `_config`; nested JSON sections are
31nested models and unknown keys are rejected. **To add a config key: add the field (with
32its type, default and any `Field` range) to the model and the same key with the same
33default to the JSON — nothing else.** `load_cluster_stepper_config`,
34`build_cluster_stepper_config` and `normalize_cluster_stepper_config` return a plain
35`dict`. Runtime overrides come from repeatable `--set KEY=VALUE`, never repo-local JSON.
36
26## Develop locally (Nexus)37## Develop locally (Nexus)
2738
28Internal `iolabs-*` dependencies resolve through the private Nexus index declared in `pyproject.toml`. Export Nexus credentials before any `uv` command that touches private deps — e.g. by sourcing `../3dai.lanefinder/scripts/nexus_credentials.sh` from your shell rc — then:39Internal `iolabs-*` dependencies resolve through the private Nexus index declared in `pyproject.toml`. Export Nexus credentials before any `uv` command that touches private deps — e.g. by sourcing `../3dai.lanefinder/scripts/nexus_credentials.sh` from your shell rc — then:
2940
Importance #10: src/iolabs_point_cloud_modelling_lines/__init__.py @@ -1,7 +1,13 @@
1"""Lane assembly from LIDAR clusters, road-side assignment, road-axis derivation, and XML export."""1"""Lane assembly from LIDAR clusters, road-side assignment, road-axis derivation, and XML export."""
22
3from ._config import load_cluster_stepper_config3from ._config import (
4 ClusterStepperConfig,
5 ClusterStepperConfigError,
6 build_cluster_stepper_config,
7 load_cluster_stepper_config,
8 normalize_cluster_stepper_config,
9)
4from .cluster_stepper import ClusterStepper10from .cluster_stepper import ClusterStepper
5from . import (11from . import (
6 cluster_stepper,12 cluster_stepper,
7 edge_lanes,13 edge_lanes,
Importance #11: src/iolabs_point_cloud_modelling_lines/__init__.py @@ -18,9 +24,13 @@
18)24)
1925
20__all__ = [26__all__ = [
21 "ClusterStepper",27 "ClusterStepper",
28 "ClusterStepperConfig",
29 "ClusterStepperConfigError",
30 "build_cluster_stepper_config",
22 "load_cluster_stepper_config",31 "load_cluster_stepper_config",
32 "normalize_cluster_stepper_config",
23 "cluster_stepper",33 "cluster_stepper",
24 "edge_lanes",34 "edge_lanes",
25 "lane",35 "lane",
26 "lane_state",36 "lane_state",
Importance #12: src/iolabs_point_cloud_modelling_lines/_config.py @@ -1,407 +1,131 @@
1"""Cluster-stepper config: packaged JSON defaults validated by a ConfigModel tree."""1"""Cluster-stepper config: packaged JSON defaults validated by a ConfigModel tree.
2
3The schema is `ClusterStepperConfig` (a `config_loader.ConfigModel`), mirroring
4`cluster_stepper.default.json` key for key; the field declarations live in
5`_config_model` and are re-exported here.
6
7Adding a config key means adding the field to the model and the same key to
8`cluster_stepper.default.json` — nothing else. Unknown keys are rejected.
9
10The entry points return a plain `dict` (the validated model dumped).
11"""
212
3from __future__ import annotations13from __future__ import annotations
414
15import logging
5from collections.abc import Mapping16from collections.abc import Mapping
6from pathlib import Path17from pathlib import Path
7from typing import Any18from typing import Any
819
9from iolabs.common import config_loader20from iolabs.common import config_loader
1021
22from ._config_model import (
23 ClusterStepperClusterDensityFilteringConfig,
24 ClusterStepperClusterIntensityFilteringConfig,
25 ClusterStepperClusteringConfig,
26 ClusterStepperConfig,
27 ClusterStepperControlPlotConfig,
28 ClusterStepperCrossSectionConfig,
29 ClusterStepperEdgeLinesConfig,
30 ClusterStepperEdgeLinesExtrapolationConfig,
31 ClusterStepperEdgeLinesSegmentPropertiesConfig,
32 ClusterStepperEdgeLinesSplineFittingConfig,
33 ClusterStepperFileNamingConfig,
34 ClusterStepperInitialOutlierRemovalConfig,
35 ClusterStepperLaneConnection2dConfig,
36 ClusterStepperLanePointsExportConfig,
37 ClusterStepperLaneRolesConfig,
38 ClusterStepperLaneSegmentEndsConfig,
39 ClusterStepperLaneSegmentWidthConfig,
40 ClusterStepperLaneStateConfig,
41 ClusterStepperLineConnectionConfig,
42 ClusterStepperRoadAxisConfig,
43 ClusterStepperRoadSidesConfig,
44 ClusterStepperSectionConfig,
45 ClusterStepperStartSectionConfig,
46 ClusterStepperSubclusterDbscanMemoryGuardConfig,
47 ClusterStepperSubclusterVisualizationConfig,
48 ClusterStepperTwoStageConfig,
49 ClusterStepperVisualizationLabelsConfig,
50 ClusterStepperXmlMetadataConfig,
51)
52
53logger = logging.getLogger(__name__)
54
11_PACKAGE_NAME = "iolabs_point_cloud_modelling_lines"55_PACKAGE_NAME = "iolabs_point_cloud_modelling_lines"
12_DEFAULT_CONFIG_NAME = "cluster_stepper.default.json"56_DEFAULT_FILENAME = "cluster_stepper.default.json"
13_CONTEXT = "cluster-stepper config"57_CONTEXT = "cluster-stepper config"
1458
59__all__ = [
60 "ClusterStepperClusterDensityFilteringConfig",
61 "ClusterStepperClusterIntensityFilteringConfig",
62 "ClusterStepperClusteringConfig",
63 "ClusterStepperConfig",
64 "ClusterStepperConfigError",
65 "ClusterStepperControlPlotConfig",
66 "ClusterStepperCrossSectionConfig",
67 "ClusterStepperEdgeLinesConfig",
68 "ClusterStepperEdgeLinesExtrapolationConfig",
69 "ClusterStepperEdgeLinesSegmentPropertiesConfig",
70 "ClusterStepperEdgeLinesSplineFittingConfig",
71 "ClusterStepperFileNamingConfig",
72 "ClusterStepperInitialOutlierRemovalConfig",
73 "ClusterStepperLaneConnection2dConfig",
74 "ClusterStepperLanePointsExportConfig",
75 "ClusterStepperLaneRolesConfig",
76 "ClusterStepperLaneSegmentEndsConfig",
77 "ClusterStepperLaneSegmentWidthConfig",
78 "ClusterStepperLaneStateConfig",
79 "ClusterStepperLineConnectionConfig",
80 "ClusterStepperRoadAxisConfig",
81 "ClusterStepperRoadSidesConfig",
82 "ClusterStepperSectionConfig",
83 "ClusterStepperStartSectionConfig",
84 "ClusterStepperSubclusterDbscanMemoryGuardConfig",
85 "ClusterStepperSubclusterVisualizationConfig",
86 "ClusterStepperTwoStageConfig",
87 "ClusterStepperVisualizationLabelsConfig",
88 "ClusterStepperXmlMetadataConfig",
89 "build_cluster_stepper_config",
90 "load_cluster_stepper_config",
91 "normalize_cluster_stepper_config",
92]
93
1594
16class ClusterStepperConfigError(config_loader.ConfigError):95class ClusterStepperConfigError(config_loader.ConfigError):
17 """Raised when cluster-stepper config contains unsupported keys or values."""96 """Raised when cluster-stepper config contains unsupported keys or values."""
1897
1998
20class InitialOutlierRemovalConfig(config_loader.ConfigModel):
21 """Statistical outlier removal before clustering."""
22
23 nb_neighbors: int = 10
24 std_ratio: float = 4.5
25 max_points_per_segment: int = 800000
26 n_times_max_points2kill_clustering: float = 2.0
27
28
29class ClusteringConfig(config_loader.ConfigModel):
30 """DBSCAN clustering and cache/visualization toggles."""
31
32 cluster_dir: str = "clusters"
33 dashed_min_length: float = 0.7
34 use_cache: bool = False
35 save_cache: bool = True
36 visualize: bool = False
37 enable_density_filtering: bool = True
38 enable_intensity_trimming: bool = False
39 enable_intensity_peak_split: bool = False
40 dbscan_eps: float = 1.0
41 dbscan_min_points: int = 50
42 min_cluster_size: int = 200
43 dbscan_guard_threshold_points: int = 15000
44
45
46class ClusterDensityFilteringConfig(config_loader.ConfigModel):
47 """Density-cutoff filters applied to clusters."""
48
49 density_cutoffs: list[float] = [0.001, 0.002, 0.005, 0.012]
50 cutoff_portions: list[float] = [0.85, 0.85, 0.15, 0.001]
51
52
53class ClusterIntensityFilteringConfig(config_loader.ConfigModel):
54 """Intensity-histogram filters applied to clusters."""
55
56 n_bins: int = 50
57 intensity_range: list[int] = [0, 255]
58 peak_distance: int = 5
59 peak_prominence: int = 10
60 sigma_estimate: float = 3.0
61 sigma_estimate_peak_distance_fraction: float = 0.125
62 n_sigma_intensity_cutoff: float = 5.0
63 min_sigma: float = 0.5
64
65
66class SubclusterDbscanMemoryGuardConfig(config_loader.ConfigModel):
67 """Memory/time guard around subcluster DBSCAN."""
68
69 mem_limit_bytes: int = 16106127360
70 mem_poll_interval_s: float = 0.5
71 mem_hard_kill: bool = False
72 timeout_s: float = 120.0
73 progress_in_guarded: bool = False
74 dbscan_guard_threshold_points: int = 15000
75 dbscan_stats_enabled: bool = True
76 dbscan_stats_output_path: str = (
77 "data/00_external/250812_Color scans/inference/dbscan_stats.csv"
78 )
79 dbscan_stats_format: str = "csv"
80
81
82class SubclusterVisualizationConfig(config_loader.ConfigModel):
83 """Optional subcluster histogram visualization."""
84
85 visualize_every_cluster: bool = False
86 save_histograms: bool = False
87 histogram_dir: str = "histograms"
88 n_bins: int = 30
89
90
91class EdgeLinesSplineFittingConfig(config_loader.ConfigModel):
92 """Spline fitting of edge-line clusters."""
93
94 max_sharp_bend_degrees: float = 7.5
95 max_sharp_bend_rejection_rate: float = 0.2
96 fit_line_length: float = 3.0
97 spline_point_count: int = 1000
98 target_distance: float = 0.3
99 n_parts_start_line: int = 6
100 min_width: float = 0.1
101 subcluster_dbscan_eps: float = 0.2
102 subcluster_dbscan_min_samples: int = 5
103 subcluster_dbscan_n_jobs: int = -1
104 subcluster_dbscan_memory_guard: SubclusterDbscanMemoryGuardConfig = (
105 SubclusterDbscanMemoryGuardConfig()
106 )
107 subcluster_visualization: SubclusterVisualizationConfig = (
108 SubclusterVisualizationConfig()
109 )
110
111
112class EdgeLinesExtrapolationConfig(config_loader.ConfigModel):
113 """How far edge-line splines are extrapolated."""
114
115 extrapolate_to: float = 10.0
116 extrapolation_points: int = 30
117
118
119class EdgeLinesSegmentPropertiesConfig(config_loader.ConfigModel):
120 """Curvature thresholds that split edge-line splines into segments."""
121
122 angle_change_rate: float = 5.0
123 angle_change_threshold: float = 45.0
124 angle_change_rate_last_segment: float = 5.0
125
126
127class EdgeLinesConfig(config_loader.ConfigModel):
128 """Edge-lane spline fitting, extrapolation, and segmentation."""
129
130 spline_fitting: EdgeLinesSplineFittingConfig = EdgeLinesSplineFittingConfig()
131 extrapolation: EdgeLinesExtrapolationConfig = EdgeLinesExtrapolationConfig()
132 segment_properties: EdgeLinesSegmentPropertiesConfig = (
133 EdgeLinesSegmentPropertiesConfig()
134 )
135
136
137class LineConnectionConfig(config_loader.ConfigModel):
138 """Legacy 3D line-connection gates."""
139
140 perpendicular_distance_threshold: float = 1.0
141 longitudinal_distance_factor: float = 0.2
142 negative_offset: float = 0.5
143
144
145class CrossSectionConfig(config_loader.ConfigModel):
146 """Cross-section sampling along the road axis."""
147
148 interval: float = 5.0
149 visualize_crosssections: bool = False
150
151
152class XmlMetadataConfig(config_loader.ConfigModel):
153 """XML export metadata."""
154
155 software_version: str = "0.3.1"
156
157
158class StartSectionConfig(config_loader.ConfigModel):
159 """Where the walk starts relative to the first plane."""
160
161 distance_from_plane: float = 10.0
162
163
164class ClusterStepperSectionConfig(config_loader.ConfigModel):
165 """Segment walk, visualization, and plane-association settings."""
166
167 segment_dir_basename: str = "segment"
168 planes_filename: str = "run3_planes.npz"
169 surface_filenames_base: str = "run4_road_surface_road_extension_*.ply"
170 step_length: float = 10.0
171 visualize_each_step: bool = False
172 visualize_each_segment: bool = False
173 visualize_road_state_plane_intersections: bool = False
174 visualize_road_side_state_points: bool = False
175 road_axis_color: list[float] = [0.0, 0.0, 1.0]
176 plane_distance_from_lane_threshold: float = 50.0
177 lane_distance_from_road_surface: float = 0.03
178
179
180class LaneConnection2dConfig(config_loader.ConfigModel):
181 """2D continuation matching used to assemble lanes."""
182
183 lateral_threshold: float = 0.5
184 lateral_slack_per_meter: float = 0.02
185 max_lateral: float = 1.5
186 heading_threshold_degrees: float = 25.0
187 heading_weight: float = 0.02
188 gap_weight: float = 0.01
189 max_along_gap: float = 40.0
190 max_overlap: float = 1.0
191 same_type_only: bool = True
192 merge_max_along_gap: float = 80.0
193 merge_lateral_threshold: float = 0.6
194 merge_heading_threshold_degrees: float = 20.0
195 duplicate_lateral_threshold: float = 0.7
196 merge_max_duplicate_overlap: float = 40.0
197 refinement_max_station_gap: float = 200.0
198 line_chain_max_gap: float = 2000.0
199 chain_max_endpoint_gap: float = 250.0
200 chain_max_lateral_jump: float = 2.0
201
202
203class LaneRolesConfig(config_loader.ConfigModel):
204 """Line grouping and edge-role dedupe thresholds."""
205
206 line_group_lateral_threshold: float = 1.0
207 min_line_length: float = 10.0
208 min_profile_overlap: float = 10.0
209 edge_dedupe_min_overlap: float = 30.0
210 edge_dedupe_min_separation: float = 1.5
211 edge_dedupe_max_covered_fraction: float = 0.7
212
213
214class RoadAxisConfig(config_loader.ConfigModel):
215 """Road-axis derivation from completed sides."""
216
217 max_angle_degrees: float = 5.0
218 both_sides_min_coverage: float = 0.5
219 both_sides_sample_spacing: float = 10.0
220 both_sides_max_station_gap: float = 30.0
221 station_merge_tolerance: float = 2.0
222 median_width_min: float = 1.0
223 median_width_max: float = 25.0
224 max_lateral_deviation: float = 6.0
225 max_extension_length: float = 200.0
226 both_sides_merge_lateral_tol: float = 6.0
227 max_blend_length: float = 300.0
228
229
230class ControlPlotConfig(config_loader.ConfigModel):
231 """Control-plot PDF output."""
232
233 enabled: bool = True
234 filename: str = "run7_control_plot.pdf"
235
236
237class LaneStateConfig(config_loader.ConfigModel):
238 """Lane-state snapshot persistence."""
239
240 enabled: bool = True
241 debug_subdir: str = "run7_lane_state"
242 filename: str = "lane_state_snapshot.json"
243 lazy_load: bool = True
244 incremental_intersection_rebuild: bool = True
245
246
247class TwoStageConfig(config_loader.ConfigModel):
248 """Two-stage walk and boundary-reconnect settings."""
249
250 partial_snapshot_filename: str = "lane_state_partial.json"
251 partial_glob: str = "*lane_state_partial.json"
252 boundary_reconnect: bool = False
253
254
255class VisualizationLabelsConfig(config_loader.ConfigModel):
256 """On-geometry text labels."""
257
258 enabled: bool = True
259 show_lane_labels: bool = True
260 show_lane_segment_labels: bool = False
261 show_road_surface_labels: bool = True
262 text_depth: float = 0.02
263 label_offset: float = 0.6
264 z_lift: float = 0.08
265 lane_text_scale: float = 0.18
266 lane_segment_text_scale: float = 0.12
267 road_surface_text_scale: float = 0.14
268 lane_label_color: list[float] = [0.12, 0.12, 0.12]
269 lane_segment_label_color: list[float] = [0.28, 0.36, 0.44]
270
271
272class RoadSidesConfig(config_loader.ConfigModel):
273 """Road-side assignment and median geometry."""
274
275 visualize_every_intersection: bool = False
276 visualize_every_new_intersection: bool = False
277 visualize_every_road_side_plane: bool = False
278 middle_lane_color: list[float] = [0.0, 1.0, 1.0]
279 edge_lane_color: list[float] = [1.0, 0.0, 1.0]
280 extra_lane_color: list[float] = [0.0, 1.0, 0.0]
281 intersection_extend_search_by_n_segments: int = 1
282 lane_distance: float = 3.5
283 lane_distance_limits: list[float] = [3.3, 3.9]
284 side_split_lateral_gap: float = 3.0
285 through_line_fraction: float = 0.3
286 median_min_gap: float = 1.5
287 median_max_gap: float = 20.0
288 side_band_margin: float = 3.5
289 left_side_id: str = "B"
290 right_side_id: str = "A"
291 side_window_length: float = 2000.0
292 side_window_stride: float = 1000.0
293 side_max_median_offset: float = 30.0
294 median_max_slope: float = 0.04
295 median_max_extrapolation: float = 3000.0
296
297
298class LaneSegmentEndsConfig(config_loader.ConfigModel):
299 """Paint-marking end detection along a lane-segment axis."""
300
301 n_bins: int = 40
302 max_distance: float = 1.0
303 start_from_spline: float = 0.3
304 start_from_line: float = 0.5
305 show_histograms: bool = False
306 histogram_dir: str | None = None
307 axis_sample_spacing: float = 0.5
308 lateral_gate: float = 0.5
309 min_amplitude: float = 3.0
310 min_points: int = 30
311
312
313class LaneSegmentWidthConfig(config_loader.ConfigModel):
314 """Paint-marking width measurement along a lane-segment axis."""
315
316 bin_width: float = 0.01
317 lateral_max: float = 0.5
318 axis_sample_spacing: float = 0.5
319 end_margin: float = 0.15
320 min_points: int = 50
321 min_amplitude: float = 3.0
322 width_min: float = 0.04
323 width_max: float = 0.6
324 flag_width_min: float = 0.08
325 flag_width_max: float = 0.35
326 integration_bin_length: float = 0.25
327 min_bin_points: int = 3
328 show_histograms: bool = False
329 histogram_dir: str | None = None
330
331
332class LanePointsExportConfig(config_loader.ConfigModel):
333 """Optional NPZ dumps of lane points by marking type."""
334
335 save_all: bool = True
336 save_dashed: bool = True
337 save_solid: bool = True
338 output_dir: str = "lane_points_exports"
339 filename_all: str = "lane_points_all.npz"
340 filename_dashed: str = "lane_points_dashed.npz"
341 filename_solid: str = "lane_points_solid.npz"
342
343
344class FileNamingConfig(config_loader.ConfigModel):
345 """Step 6/3 input discovery and Step 7 output stems."""
346
347 cluster_prefix: str = "run6_cluster_"
348 geoshift_filename: str = "run3_geoshift.json"
349 step7_prefix: str = "run7_"
350 lanes_stem: str = "lanes"
351
352
353class ClusterStepperConfig(config_loader.ConfigModel):
354 """Root cluster-stepper config mirroring ``cluster_stepper.default.json``."""
355
356 initial_outlier_removal: InitialOutlierRemovalConfig = InitialOutlierRemovalConfig()
357 clustering: ClusteringConfig = ClusteringConfig()
358 cluster_density_filtering: ClusterDensityFilteringConfig = (
359 ClusterDensityFilteringConfig()
360 )
361 cluster_intensity_filtering: ClusterIntensityFilteringConfig = (
362 ClusterIntensityFilteringConfig()
363 )
364 edge_lines: EdgeLinesConfig = EdgeLinesConfig()
365 line_connection: LineConnectionConfig = LineConnectionConfig()
366 cross_section: CrossSectionConfig = CrossSectionConfig()
367 xml_metadata: XmlMetadataConfig = XmlMetadataConfig()
368 start_section: StartSectionConfig = StartSectionConfig()
369 cluster_stepper: ClusterStepperSectionConfig = ClusterStepperSectionConfig()
370 lane_connection_2d: LaneConnection2dConfig = LaneConnection2dConfig()
371 lane_roles: LaneRolesConfig = LaneRolesConfig()
372 road_axis: RoadAxisConfig = RoadAxisConfig()
373 control_plot: ControlPlotConfig = ControlPlotConfig()
374 lane_state: LaneStateConfig = LaneStateConfig()
375 two_stage: TwoStageConfig = TwoStageConfig()
376 visualization_labels: VisualizationLabelsConfig = VisualizationLabelsConfig()
377 road_sides: RoadSidesConfig = RoadSidesConfig()
378 lane_segment_ends: LaneSegmentEndsConfig = LaneSegmentEndsConfig()
379 lane_segment_width: LaneSegmentWidthConfig = LaneSegmentWidthConfig()
380 lane_points_export: LanePointsExportConfig = LanePointsExportConfig()
381 file_naming: FileNamingConfig = FileNamingConfig()
382
383
384def _load_model(99def _load_model(
385 *,100 *,
386 overrides: Mapping[str, Any] | None = None,101 overrides: Mapping[str, Any] | None = None,
387 config_path: str | Path | None = None,102 config_path: str | Path | None = None,
388) -> ClusterStepperConfig:103) -> ClusterStepperConfig:
389 """Load packaged defaults, merge overrides, and validate the model."""104 """Load packaged defaults (or *config_path*), merge overrides, and validate."""
105 if config_path is not None:
106 logger.info("Config file applied: %s", config_path)
107 if overrides:
108 logger.info("Config overrides applied: %s", ", ".join(sorted(overrides)))
390 return config_loader.load_config(109 return config_loader.load_config(
391 ClusterStepperConfig,110 ClusterStepperConfig,
392 package=__package__ or _PACKAGE_NAME,111 package=_PACKAGE_NAME,
393 filename=_DEFAULT_CONFIG_NAME,112 filename=_DEFAULT_FILENAME,
394 overrides=overrides,113 overrides=overrides,
395 config_path=config_path,114 config_path=config_path,
396 context=_CONTEXT,115 context=_CONTEXT,
397 error_cls=ClusterStepperConfigError,116 error_cls=ClusterStepperConfigError,
398 )117 )
399118
400119
401def normalize_cluster_stepper_config(raw_config: Mapping[str, Any]) -> dict[str, Any]:120def normalize_cluster_stepper_config(raw_config: Mapping[str, Any]) -> dict[str, Any]:
402 """Deep-merge *raw_config* onto the packaged defaults and validate it."""121 """Validate *raw_config* and fill in the model defaults, returning a dict."""
403 return _load_model(overrides=raw_config).model_dump()122 return config_loader.validate_config(
123 ClusterStepperConfig,
124 raw_config,
125 context=_CONTEXT,
126 error_cls=ClusterStepperConfigError,
127 ).model_dump()
404128
405129
406def load_cluster_stepper_config(130def load_cluster_stepper_config(
407 config_path: str | Path | None = None,131 config_path: str | Path | None = None,
Importance #13: src/iolabs_point_cloud_modelling_lines/_config.py @@ -412,9 +136,9 @@
412 config_path: JSON file read instead of the packaged defaults. Keys it136 config_path: JSON file read instead of the packaged defaults. Keys it
413 omits fall back to the model defaults, which mirror137 omits fall back to the model defaults, which mirror
414 ``cluster_stepper.default.json``.138 ``cluster_stepper.default.json``.
415 """139 """
416 return _load_model(config_path=config_path).model_dump()140 return build_cluster_stepper_config(config_path=config_path)
417141
418142
419def build_cluster_stepper_config(143def build_cluster_stepper_config(
420 *,144 *,
Importance #14: src/iolabs_point_cloud_modelling_lines/_config_model.py @@ -0,0 +1,399 @@
1"""Field declarations for the cluster-stepper config.
2
3The schema is `ClusterStepperConfig` (a `config_loader.ConfigModel`), mirroring
4`cluster_stepper.default.json` key for key.
5
6Adding a config key means adding the field to the model and the same key to
7`cluster_stepper.default.json` — nothing else. Unknown keys are rejected.
8
9Loading, merging and the public entry points live in `_config`, which
10re-exports every name declared here. Sequence fields stay `list[...]` because
11the entry points return a plain `dict` that callers are free to mutate.
12"""
13
14from __future__ import annotations
15
16import pydantic
17from iolabs.common import config_loader
18
19
20class ClusterStepperInitialOutlierRemovalConfig(config_loader.ConfigModel):
21 """Statistical outlier removal before clustering."""
22
23 nb_neighbors: int = pydantic.Field(10, ge=1)
24 std_ratio: float = pydantic.Field(4.5, gt=0.0)
25 max_points_per_segment: int = pydantic.Field(800000, ge=1)
26 n_times_max_points2kill_clustering: float = pydantic.Field(2.0, gt=0.0)
27
28
29class ClusterStepperClusteringConfig(config_loader.ConfigModel):
30 """DBSCAN clustering and cache/visualization toggles."""
31
32 cluster_dir: str = "clusters"
33 dashed_min_length: float = pydantic.Field(0.7, ge=0.0)
34 use_cache: bool = False
35 save_cache: bool = True
36 visualize: bool = False
37 enable_density_filtering: bool = True
38 enable_intensity_trimming: bool = False
39 enable_intensity_peak_split: bool = False
40 dbscan_eps: float = pydantic.Field(1.0, gt=0.0)
41 dbscan_min_points: int = pydantic.Field(50, ge=1)
42 min_cluster_size: int = pydantic.Field(200, ge=1)
43 dbscan_guard_threshold_points: int = pydantic.Field(15000, ge=1)
44
45
46class ClusterStepperClusterDensityFilteringConfig(config_loader.ConfigModel):
47 """Density-cutoff filters applied to clusters."""
48
49 density_cutoffs: list[float] = [0.001, 0.002, 0.005, 0.012]
50 cutoff_portions: list[float] = [0.85, 0.85, 0.15, 0.001]
51
52
53class ClusterStepperClusterIntensityFilteringConfig(config_loader.ConfigModel):
54 """Intensity-histogram filters applied to clusters."""
55
56 n_bins: int = pydantic.Field(50, ge=1)
57 intensity_range: list[int] = [0, 255]
58 peak_distance: int = pydantic.Field(5, ge=1)
59 peak_prominence: int = pydantic.Field(10, ge=1)
60 sigma_estimate: float = pydantic.Field(3.0, gt=0.0)
61 sigma_estimate_peak_distance_fraction: float = pydantic.Field(0.125, ge=0.0, le=1.0)
62 n_sigma_intensity_cutoff: float = pydantic.Field(5.0, gt=0.0)
63 min_sigma: float = pydantic.Field(0.5, gt=0.0)
64
65
66class ClusterStepperSubclusterDbscanMemoryGuardConfig(config_loader.ConfigModel):
67 """Memory/time guard around subcluster DBSCAN."""
68
69 mem_limit_bytes: int = pydantic.Field(16106127360, ge=1)
70 mem_poll_interval_s: float = pydantic.Field(0.5, gt=0.0)
71 mem_hard_kill: bool = False
72 timeout_s: float = pydantic.Field(120.0, gt=0.0)
73 progress_in_guarded: bool = False
74 dbscan_guard_threshold_points: int = pydantic.Field(15000, ge=1)
75 dbscan_stats_enabled: bool = True
76 dbscan_stats_output_path: str = (
77 "data/00_external/250812_Color scans/inference/dbscan_stats.csv"
78 )
79 dbscan_stats_format: str = "csv"
80
81
82class ClusterStepperSubclusterVisualizationConfig(config_loader.ConfigModel):
83 """Optional subcluster histogram visualization."""
84
85 visualize_every_cluster: bool = False
86 save_histograms: bool = False
87 histogram_dir: str = "histograms"
88 n_bins: int = pydantic.Field(30, ge=1)
89
90
91class ClusterStepperEdgeLinesSplineFittingConfig(config_loader.ConfigModel):
92 """Spline fitting of edge-line clusters."""
93
94 max_sharp_bend_degrees: float = pydantic.Field(7.5, ge=0.0, le=180.0)
95 max_sharp_bend_rejection_rate: float = pydantic.Field(0.2, ge=0.0, le=1.0)
96 fit_line_length: float = pydantic.Field(3.0, gt=0.0)
97 spline_point_count: int = pydantic.Field(1000, ge=1)
98 target_distance: float = pydantic.Field(0.3, gt=0.0)
99 n_parts_start_line: int = pydantic.Field(6, ge=1)
100 min_width: float = pydantic.Field(0.1, ge=0.0)
101 subcluster_dbscan_eps: float = pydantic.Field(0.2, gt=0.0)
102 subcluster_dbscan_min_samples: int = pydantic.Field(5, ge=1)
103 subcluster_dbscan_n_jobs: int = -1
104 subcluster_dbscan_memory_guard: ClusterStepperSubclusterDbscanMemoryGuardConfig = (
105 ClusterStepperSubclusterDbscanMemoryGuardConfig()
106 )
107 subcluster_visualization: ClusterStepperSubclusterVisualizationConfig = (
108 ClusterStepperSubclusterVisualizationConfig()
109 )
110
111
112class ClusterStepperEdgeLinesExtrapolationConfig(config_loader.ConfigModel):
113 """How far edge-line splines are extrapolated."""
114
115 extrapolate_to: float = pydantic.Field(10.0, ge=0.0)
116 extrapolation_points: int = pydantic.Field(30, ge=1)
117
118
119class ClusterStepperEdgeLinesSegmentPropertiesConfig(config_loader.ConfigModel):
120 """Curvature thresholds that split edge-line splines into segments."""
121
122 angle_change_rate: float = pydantic.Field(5.0, ge=0.0)
123 angle_change_threshold: float = pydantic.Field(45.0, ge=0.0, le=180.0)
124 angle_change_rate_last_segment: float = pydantic.Field(5.0, ge=0.0)
125
126
127class ClusterStepperEdgeLinesConfig(config_loader.ConfigModel):
128 """Edge-lane spline fitting, extrapolation, and segmentation."""
129
130 spline_fitting: ClusterStepperEdgeLinesSplineFittingConfig = (
131 ClusterStepperEdgeLinesSplineFittingConfig()
132 )
133 extrapolation: ClusterStepperEdgeLinesExtrapolationConfig = (
134 ClusterStepperEdgeLinesExtrapolationConfig()
135 )
136 segment_properties: ClusterStepperEdgeLinesSegmentPropertiesConfig = (
137 ClusterStepperEdgeLinesSegmentPropertiesConfig()
138 )
139
140
141class ClusterStepperLineConnectionConfig(config_loader.ConfigModel):
142 """Legacy 3D line-connection gates."""
143
144 perpendicular_distance_threshold: float = pydantic.Field(1.0, ge=0.0)
145 longitudinal_distance_factor: float = pydantic.Field(0.2, ge=0.0)
146 negative_offset: float = 0.5
147
148
149class ClusterStepperCrossSectionConfig(config_loader.ConfigModel):
150 """Cross-section sampling along the road axis."""
151
152 interval: float = pydantic.Field(5.0, gt=0.0)
153 visualize_crosssections: bool = False
154
155
156class ClusterStepperXmlMetadataConfig(config_loader.ConfigModel):
157 """XML export metadata."""
158
159 software_version: str = "0.3.1"
160
161
162class ClusterStepperStartSectionConfig(config_loader.ConfigModel):
163 """Where the walk starts relative to the first plane."""
164
165 distance_from_plane: float = pydantic.Field(10.0, ge=0.0)
166
167
168class ClusterStepperSectionConfig(config_loader.ConfigModel):
169 """Segment walk, visualization, and plane-association settings."""
170
171 segment_dir_basename: str = "segment"
172 planes_filename: str = "run3_planes.npz"
173 surface_filenames_base: str = "run4_road_surface_road_extension_*.ply"
174 step_length: float = pydantic.Field(10.0, gt=0.0)
175 visualize_each_step: bool = False
176 visualize_each_segment: bool = False
177 visualize_road_state_plane_intersections: bool = False
178 visualize_road_side_state_points: bool = False
179 road_axis_color: list[float] = [0.0, 0.0, 1.0]
180 plane_distance_from_lane_threshold: float = pydantic.Field(50.0, ge=0.0)
181 lane_distance_from_road_surface: float = pydantic.Field(0.03, ge=0.0)
182
183
184class ClusterStepperLaneConnection2dConfig(config_loader.ConfigModel):
185 """2D continuation matching used to assemble lanes."""
186
187 lateral_threshold: float = pydantic.Field(0.5, ge=0.0)
188 lateral_slack_per_meter: float = pydantic.Field(0.02, ge=0.0)
189 max_lateral: float = pydantic.Field(1.5, ge=0.0)
190 heading_threshold_degrees: float = pydantic.Field(25.0, ge=0.0, le=180.0)
191 heading_weight: float = pydantic.Field(0.02, ge=0.0)
192 gap_weight: float = pydantic.Field(0.01, ge=0.0)
193 max_along_gap: float = pydantic.Field(40.0, ge=0.0)
194 max_overlap: float = pydantic.Field(1.0, ge=0.0)
195 same_type_only: bool = True
196 merge_max_along_gap: float = pydantic.Field(80.0, ge=0.0)
197 merge_lateral_threshold: float = pydantic.Field(0.6, ge=0.0)
198 merge_heading_threshold_degrees: float = pydantic.Field(20.0, ge=0.0, le=180.0)
199 duplicate_lateral_threshold: float = pydantic.Field(0.7, ge=0.0)
200 merge_max_duplicate_overlap: float = pydantic.Field(40.0, ge=0.0)
201 refinement_max_station_gap: float = pydantic.Field(200.0, ge=0.0)
202 line_chain_max_gap: float = pydantic.Field(2000.0, ge=0.0)
203 chain_max_endpoint_gap: float = pydantic.Field(250.0, ge=0.0)
204 chain_max_lateral_jump: float = pydantic.Field(2.0, ge=0.0)
205
206
207class ClusterStepperLaneRolesConfig(config_loader.ConfigModel):
208 """Line grouping and edge-role dedupe thresholds."""
209
210 line_group_lateral_threshold: float = pydantic.Field(1.0, ge=0.0)
211 min_line_length: float = pydantic.Field(10.0, ge=0.0)
212 min_profile_overlap: float = pydantic.Field(10.0, ge=0.0)
213 edge_dedupe_min_overlap: float = pydantic.Field(30.0, ge=0.0)
214 edge_dedupe_min_separation: float = pydantic.Field(1.5, ge=0.0)
215 edge_dedupe_max_covered_fraction: float = pydantic.Field(0.7, ge=0.0, le=1.0)
216
217
218class ClusterStepperRoadAxisConfig(config_loader.ConfigModel):
219 """Road-axis derivation from completed sides."""
220
221 max_angle_degrees: float = pydantic.Field(5.0, ge=0.0, le=180.0)
222 both_sides_min_coverage: float = pydantic.Field(0.5, ge=0.0, le=1.0)
223 both_sides_sample_spacing: float = pydantic.Field(10.0, gt=0.0)
224 both_sides_max_station_gap: float = pydantic.Field(30.0, ge=0.0)
225 station_merge_tolerance: float = pydantic.Field(2.0, ge=0.0)
226 median_width_min: float = pydantic.Field(1.0, ge=0.0)
227 median_width_max: float = pydantic.Field(25.0, ge=0.0)
228 max_lateral_deviation: float = pydantic.Field(6.0, ge=0.0)
229 max_extension_length: float = pydantic.Field(200.0, ge=0.0)
230 both_sides_merge_lateral_tol: float = pydantic.Field(6.0, ge=0.0)
231 max_blend_length: float = pydantic.Field(300.0, ge=0.0)
232
233
234class ClusterStepperControlPlotConfig(config_loader.ConfigModel):
235 """Control-plot PDF output."""
236
237 enabled: bool = True
238 filename: str = "run7_control_plot.pdf"
239
240
241class ClusterStepperLaneStateConfig(config_loader.ConfigModel):
242 """Lane-state snapshot persistence."""
243
244 enabled: bool = True
245 debug_subdir: str = "run7_lane_state"
246 filename: str = "lane_state_snapshot.json"
247 lazy_load: bool = True
248 incremental_intersection_rebuild: bool = True
249
250
251class ClusterStepperTwoStageConfig(config_loader.ConfigModel):
252 """Two-stage walk and boundary-reconnect settings."""
253
254 partial_snapshot_filename: str = "lane_state_partial.json"
255 partial_glob: str = "*lane_state_partial.json"
256 boundary_reconnect: bool = False
257
258
259class ClusterStepperVisualizationLabelsConfig(config_loader.ConfigModel):
260 """On-geometry text labels."""
261
262 enabled: bool = True
263 show_lane_labels: bool = True
264 show_lane_segment_labels: bool = False
265 show_road_surface_labels: bool = True
266 text_depth: float = pydantic.Field(0.02, ge=0.0)
267 label_offset: float = pydantic.Field(0.6, ge=0.0)
268 z_lift: float = pydantic.Field(0.08, ge=0.0)
269 lane_text_scale: float = pydantic.Field(0.18, ge=0.0)
270 lane_segment_text_scale: float = pydantic.Field(0.12, ge=0.0)
271 road_surface_text_scale: float = pydantic.Field(0.14, ge=0.0)
272 lane_label_color: list[float] = [0.12, 0.12, 0.12]
273 lane_segment_label_color: list[float] = [0.28, 0.36, 0.44]
274
275
276class ClusterStepperRoadSidesConfig(config_loader.ConfigModel):
277 """Road-side assignment and median geometry."""
278
279 visualize_every_intersection: bool = False
280 visualize_every_new_intersection: bool = False
281 visualize_every_road_side_plane: bool = False
282 middle_lane_color: list[float] = [0.0, 1.0, 1.0]
283 edge_lane_color: list[float] = [1.0, 0.0, 1.0]
284 extra_lane_color: list[float] = [0.0, 1.0, 0.0]
285 intersection_extend_search_by_n_segments: int = pydantic.Field(1, ge=0)
286 lane_distance: float = pydantic.Field(3.5, ge=0.0)
287 lane_distance_limits: list[float] = [3.3, 3.9]
288 side_split_lateral_gap: float = pydantic.Field(3.0, ge=0.0)
289 through_line_fraction: float = pydantic.Field(0.3, ge=0.0, le=1.0)
290 median_min_gap: float = pydantic.Field(1.5, ge=0.0)
291 median_max_gap: float = pydantic.Field(20.0, ge=0.0)
292 side_band_margin: float = pydantic.Field(3.5, ge=0.0)
293 left_side_id: str = "B"
294 right_side_id: str = "A"
295 side_window_length: float = pydantic.Field(2000.0, gt=0.0)
296 side_window_stride: float = pydantic.Field(1000.0, gt=0.0)
297 side_max_median_offset: float = pydantic.Field(30.0, ge=0.0)
298 median_max_slope: float = pydantic.Field(0.04, ge=0.0)
299 median_max_extrapolation: float = pydantic.Field(3000.0, ge=0.0)
300
301
302class ClusterStepperLaneSegmentEndsConfig(config_loader.ConfigModel):
303 """Paint-marking end detection along a lane-segment axis."""
304
305 n_bins: int = pydantic.Field(40, ge=1)
306 max_distance: float = pydantic.Field(1.0, ge=0.0)
307 start_from_spline: float = pydantic.Field(0.3, ge=0.0)
308 start_from_line: float = pydantic.Field(0.5, ge=0.0)
309 show_histograms: bool = False
310 histogram_dir: str | None = None
311 axis_sample_spacing: float = pydantic.Field(0.5, gt=0.0)
312 lateral_gate: float = pydantic.Field(0.5, ge=0.0)
313 min_amplitude: float = pydantic.Field(3.0, ge=0.0)
314 min_points: int = pydantic.Field(30, ge=1)
315
316
317class ClusterStepperLaneSegmentWidthConfig(config_loader.ConfigModel):
318 """Paint-marking width measurement along a lane-segment axis."""
319
320 bin_width: float = pydantic.Field(0.01, gt=0.0)
321 lateral_max: float = pydantic.Field(0.5, ge=0.0)
322 axis_sample_spacing: float = pydantic.Field(0.5, gt=0.0)
323 end_margin: float = pydantic.Field(0.15, ge=0.0)
324 min_points: int = pydantic.Field(50, ge=1)
325 min_amplitude: float = pydantic.Field(3.0, ge=0.0)
326 width_min: float = pydantic.Field(0.04, ge=0.0)
327 width_max: float = pydantic.Field(0.6, ge=0.0)
328 flag_width_min: float = pydantic.Field(0.08, ge=0.0)
329 flag_width_max: float = pydantic.Field(0.35, ge=0.0)
330 integration_bin_length: float = pydantic.Field(0.25, gt=0.0)
331 min_bin_points: int = pydantic.Field(3, ge=1)
332 show_histograms: bool = False
333 histogram_dir: str | None = None
334
335
336class ClusterStepperLanePointsExportConfig(config_loader.ConfigModel):
337 """Optional NPZ dumps of lane points by marking type."""
338
339 save_all: bool = True
340 save_dashed: bool = True
341 save_solid: bool = True
342 output_dir: str = "lane_points_exports"
343 filename_all: str = "lane_points_all.npz"
344 filename_dashed: str = "lane_points_dashed.npz"
345 filename_solid: str = "lane_points_solid.npz"
346
347
348class ClusterStepperFileNamingConfig(config_loader.ConfigModel):
349 """Step 6/3 input discovery and Step 7 output stems."""
350
351 cluster_prefix: str = "run6_cluster_"
352 geoshift_filename: str = "run3_geoshift.json"
353 step7_prefix: str = "run7_"
354 lanes_stem: str = "lanes"
355
356
357class ClusterStepperConfig(config_loader.ConfigModel):
358 """Root cluster-stepper config mirroring ``cluster_stepper.default.json``."""
359
360 initial_outlier_removal: ClusterStepperInitialOutlierRemovalConfig = (
361 ClusterStepperInitialOutlierRemovalConfig()
362 )
363 clustering: ClusterStepperClusteringConfig = ClusterStepperClusteringConfig()
364 cluster_density_filtering: ClusterStepperClusterDensityFilteringConfig = (
365 ClusterStepperClusterDensityFilteringConfig()
366 )
367 cluster_intensity_filtering: ClusterStepperClusterIntensityFilteringConfig = (
368 ClusterStepperClusterIntensityFilteringConfig()
369 )
370 edge_lines: ClusterStepperEdgeLinesConfig = ClusterStepperEdgeLinesConfig()
371 line_connection: ClusterStepperLineConnectionConfig = (
372 ClusterStepperLineConnectionConfig()
373 )
374 cross_section: ClusterStepperCrossSectionConfig = ClusterStepperCrossSectionConfig()
375 xml_metadata: ClusterStepperXmlMetadataConfig = ClusterStepperXmlMetadataConfig()
376 start_section: ClusterStepperStartSectionConfig = ClusterStepperStartSectionConfig()
377 cluster_stepper: ClusterStepperSectionConfig = ClusterStepperSectionConfig()
378 lane_connection_2d: ClusterStepperLaneConnection2dConfig = (
379 ClusterStepperLaneConnection2dConfig()
380 )
381 lane_roles: ClusterStepperLaneRolesConfig = ClusterStepperLaneRolesConfig()
382 road_axis: ClusterStepperRoadAxisConfig = ClusterStepperRoadAxisConfig()
383 control_plot: ClusterStepperControlPlotConfig = ClusterStepperControlPlotConfig()
384 lane_state: ClusterStepperLaneStateConfig = ClusterStepperLaneStateConfig()
385 two_stage: ClusterStepperTwoStageConfig = ClusterStepperTwoStageConfig()
386 visualization_labels: ClusterStepperVisualizationLabelsConfig = (
387 ClusterStepperVisualizationLabelsConfig()
388 )
389 road_sides: ClusterStepperRoadSidesConfig = ClusterStepperRoadSidesConfig()
390 lane_segment_ends: ClusterStepperLaneSegmentEndsConfig = (
391 ClusterStepperLaneSegmentEndsConfig()
392 )
393 lane_segment_width: ClusterStepperLaneSegmentWidthConfig = (
394 ClusterStepperLaneSegmentWidthConfig()
395 )
396 lane_points_export: ClusterStepperLanePointsExportConfig = (
397 ClusterStepperLanePointsExportConfig()
398 )
399 file_naming: ClusterStepperFileNamingConfig = ClusterStepperFileNamingConfig()
0
Importance #15: tests/test_config.py @@ -19,17 +19,18 @@
19def test_model_defaults_match_packaged_json():19def test_model_defaults_match_packaged_json():
20 assert _config.ClusterStepperConfig().model_dump() == _packaged_defaults()20 assert _config.ClusterStepperConfig().model_dump() == _packaged_defaults()
2121
2222
23def test_load_defaults_matches_packaged_json():23def test_load_cluster_stepper_config_returns_packaged_defaults():
24 assert _config.load_cluster_stepper_config() == _packaged_defaults()24 assert _config.load_cluster_stepper_config() == _packaged_defaults()
2525
2626
27def test_error_class_is_a_config_error():27def test_error_class_is_config_error():
28 assert issubclass(_config.ClusterStepperConfigError, config_loader.ConfigError)28 assert issubclass(_config.ClusterStepperConfigError, config_loader.ConfigError)
29 assert issubclass(_config.ClusterStepperConfigError, ValueError)
2930
3031
31def test_unknown_root_key_is_rejected():32def test_unknown_top_level_key_is_rejected():
32 with pytest.raises(_config.ClusterStepperConfigError, match="bogus"):33 with pytest.raises(_config.ClusterStepperConfigError, match="bogus"):
33 _config.normalize_cluster_stepper_config({"bogus": 1})34 _config.normalize_cluster_stepper_config({"bogus": 1})
3435
3536
Importance #16: tests/test_config.py @@ -37,9 +38,9 @@
37 with pytest.raises(_config.ClusterStepperConfigError, match="nope"):38 with pytest.raises(_config.ClusterStepperConfigError, match="nope"):
38 _config.normalize_cluster_stepper_config({"clustering": {"nope": 1}})39 _config.normalize_cluster_stepper_config({"clustering": {"nope": 1}})
3940
4041
41def test_overrides_are_deep_merged_and_coerced():42def test_overrides_deep_merge_onto_defaults():
42 config = _config.build_cluster_stepper_config(43 config = _config.build_cluster_stepper_config(
43 overrides={"clustering": {"dbscan_eps": "2.5"}}44 overrides={"clustering": {"dbscan_eps": "2.5"}}
44 )45 )
45 defaults = _packaged_defaults()46 defaults = _packaged_defaults()
Importance #17: tests/test_config.py @@ -47,8 +48,28 @@
47 assert config["clustering"]["use_cache"] == defaults["clustering"]["use_cache"]48 assert config["clustering"]["use_cache"] == defaults["clustering"]["use_cache"]
48 assert config["road_axis"] == defaults["road_axis"]49 assert config["road_axis"] == defaults["road_axis"]
4950
5051
52def test_set_override_coercion_and_rejection():
53 overrides = config_loader.parse_set_overrides(
54 ["clustering.min_cluster_size=1e3", "clustering.use_cache=on"],
55 error_cls=_config.ClusterStepperConfigError,
56 nested=True,
57 )
58 config = _config.build_cluster_stepper_config(overrides=overrides)
59 assert config["clustering"]["min_cluster_size"] == 1000
60 assert config["clustering"]["use_cache"] is True
61 with pytest.raises(_config.ClusterStepperConfigError, match="use_cache"):
62 _config.build_cluster_stepper_config(
63 overrides={"clustering": {"use_cache": "flase"}}
64 )
65
66
67def test_out_of_range_value_is_rejected():
68 with pytest.raises(_config.ClusterStepperConfigError, match="dbscan_eps"):
69 _config.normalize_cluster_stepper_config({"clustering": {"dbscan_eps": -1.0}})
70
71
51def test_bool_is_not_accepted_for_an_int_field():72def test_bool_is_not_accepted_for_an_int_field():
52 with pytest.raises(_config.ClusterStepperConfigError, match="dbscan_min_points"):73 with pytest.raises(_config.ClusterStepperConfigError, match="dbscan_min_points"):
53 _config.normalize_cluster_stepper_config(74 _config.normalize_cluster_stepper_config(
54 {"clustering": {"dbscan_min_points": True}}75 {"clustering": {"dbscan_min_points": True}}