Step 7 modellinglines 03b2aa6: AI3D-379 Pydantic config models via iolabs-common ConfigModel

Miroslav Simko <ms@iolabs.ch> 2026-09-02T08:37:01+02:00

Commit #41 · 11 snippets

 AGENTS.md                                         |   4 +-
 README.md                                         |   2 +-
 pyproject.toml                                    |   5 +-
 src/iolabs_point_cloud_modelling_lines/_config.py | 424 ++++++++++++++++++++--
 tests/test_config.py                              |  91 +++++
 5 files changed, 496 insertions(+), 30 deletions(-)

Package config module rewritten on ConfigModel: nested section models mirror the packaged *.default.json key for key; whitelist sets and hand-rolled coercion deleted; loader built on config_loader.load_config. Public entry-point names and return types unchanged so lanefinder wrappers keep working.
Adds pydantic>=2.7 dependency.

Importance #1: src/iolabs_point_cloud_modelling_lines/_config.py @@ -1,51 +1,425 @@

		1	"""Cluster-stepper config: packaged JSON defaults validated by a ConfigModel tree."""
		2
1	from __future__ import annotations	3	from __future__ import annotations
2		4
3	import json	5	from collections.abc import Mapping
4	from pathlib import Path	6	from pathlib import Path
5	from typing import Any	7	from typing import Any
6		8
7	from iolabs.common import config_loader	9	from iolabs.common import config_loader
8		10
9	_PACKAGE_NAME = "iolabs_point_cloud_modelling_lines"	11	_PACKAGE_NAME = "iolabs_point_cloud_modelling_lines"
10	_DEFAULT_CONFIG_NAME = "cluster_stepper.default.json"	12	_DEFAULT_CONFIG_NAME = "cluster_stepper.default.json"
		13	_CONTEXT = "cluster-stepper config"
11		14
12		15
13	class ClusterStepperConfigError(config_loader.ConfigError):	16	class ClusterStepperConfigError(config_loader.ConfigError):
14	"""Raised when cluster-stepper config contains unsupported keys."""	17	"""Raised when cluster-stepper config contains unsupported keys or values."""
		18
		19
		20	class InitialOutlierRemovalConfig(config_loader.ConfigModel):
		21	"""Statistical outlier removal before clustering."""
		22
		23	nb_neighbors: int = 10
		24	std_ratio: float = 4.5
		25	max_points_per_segment: int = 800000
		26	n_times_max_points2kill_clustering: float = 2.0
		27
		28
		29	class ClusteringConfig(config_loader.ConfigModel):
		30	"""DBSCAN clustering and cache/visualization toggles."""
		31
		32	cluster_dir: str = "clusters"
		33	dashed_min_length: float = 0.7
		34	use_cache: bool = False
		35	save_cache: bool = True
		36	visualize: bool = False
		37	enable_density_filtering: bool = True
		38	enable_intensity_trimming: bool = False
		39	enable_intensity_peak_split: bool = False
		40	dbscan_eps: float = 1.0
		41	dbscan_min_points: int = 50
		42	min_cluster_size: int = 200
		43	dbscan_guard_threshold_points: int = 15000
		44
		45
		46	class ClusterDensityFilteringConfig(config_loader.ConfigModel):
		47	"""Density-cutoff filters applied to clusters."""
		48
		49	density_cutoffs: list[float] = [0.001, 0.002, 0.005, 0.012]
		50	cutoff_portions: list[float] = [0.85, 0.85, 0.15, 0.001]
		51
		52
		53	class ClusterIntensityFilteringConfig(config_loader.ConfigModel):
		54	"""Intensity-histogram filters applied to clusters."""
		55
		56	n_bins: int = 50
		57	intensity_range: list[int] = [0, 255]
		58	peak_distance: int = 5
		59	peak_prominence: int = 10
		60	sigma_estimate: float = 3.0
		61	sigma_estimate_peak_distance_fraction: float = 0.125
		62	n_sigma_intensity_cutoff: float = 5.0
		63	min_sigma: float = 0.5
		64
		65
		66	class SubclusterDbscanMemoryGuardConfig(config_loader.ConfigModel):
		67	"""Memory/time guard around subcluster DBSCAN."""
		68
		69	mem_limit_bytes: int = 16106127360
		70	mem_poll_interval_s: float = 0.5
		71	mem_hard_kill: bool = False
		72	timeout_s: float = 120.0
		73	progress_in_guarded: bool = False
		74	dbscan_guard_threshold_points: int = 15000
		75	dbscan_stats_enabled: bool = True
		76	dbscan_stats_output_path: str = (
		77	"data/00_external/250812_Color scans/inference/dbscan_stats.csv"
		78	)
		79	dbscan_stats_format: str = "csv"
		80
		81
		82	class SubclusterVisualizationConfig(config_loader.ConfigModel):
		83	"""Optional subcluster histogram visualization."""
		84
		85	visualize_every_cluster: bool = False
		86	save_histograms: bool = False
		87	histogram_dir: str = "histograms"
		88	n_bins: int = 30
		89
		90
		91	class EdgeLinesSplineFittingConfig(config_loader.ConfigModel):
		92	"""Spline fitting of edge-line clusters."""
		93
		94	max_sharp_bend_degrees: float = 7.5
		95	max_sharp_bend_rejection_rate: float = 0.2
		96	fit_line_length: float = 3.0
		97	spline_point_count: int = 1000
		98	target_distance: float = 0.3
		99	n_parts_start_line: int = 6
		100	min_width: float = 0.1
		101	subcluster_dbscan_eps: float = 0.2
		102	subcluster_dbscan_min_samples: int = 5
		103	subcluster_dbscan_n_jobs: int = -1
		104	subcluster_dbscan_memory_guard: SubclusterDbscanMemoryGuardConfig = (
		105	SubclusterDbscanMemoryGuardConfig()
		106	)
		107	subcluster_visualization: SubclusterVisualizationConfig = (
		108	SubclusterVisualizationConfig()
		109	)
		110
		111
		112	class EdgeLinesExtrapolationConfig(config_loader.ConfigModel):
		113	"""How far edge-line splines are extrapolated."""
		114
		115	extrapolate_to: float = 10.0
		116	extrapolation_points: int = 30
		117
		118
		119	class EdgeLinesSegmentPropertiesConfig(config_loader.ConfigModel):
		120	"""Curvature thresholds that split edge-line splines into segments."""
		121
		122	angle_change_rate: float = 5.0
		123	angle_change_threshold: float = 45.0
		124	angle_change_rate_last_segment: float = 5.0
		125
		126
		127	class EdgeLinesConfig(config_loader.ConfigModel):
		128	"""Edge-lane spline fitting, extrapolation, and segmentation."""
		129
		130	spline_fitting: EdgeLinesSplineFittingConfig = EdgeLinesSplineFittingConfig()
		131	extrapolation: EdgeLinesExtrapolationConfig = EdgeLinesExtrapolationConfig()
		132	segment_properties: EdgeLinesSegmentPropertiesConfig = (
		133	EdgeLinesSegmentPropertiesConfig()
		134	)
		135
		136
		137	class LineConnectionConfig(config_loader.ConfigModel):
		138	"""Legacy 3D line-connection gates."""
		139
		140	perpendicular_distance_threshold: float = 1.0
		141	longitudinal_distance_factor: float = 0.2
		142	negative_offset: float = 0.5
		143
		144
		145	class CrossSectionConfig(config_loader.ConfigModel):
		146	"""Cross-section sampling along the road axis."""
		147
		148	interval: float = 5.0
		149	visualize_crosssections: bool = False
		150
		151
		152	class XmlMetadataConfig(config_loader.ConfigModel):
		153	"""XML export metadata."""
		154
		155	software_version: str = "0.3.1"
		156
		157
		158	class StartSectionConfig(config_loader.ConfigModel):
		159	"""Where the walk starts relative to the first plane."""
		160
		161	distance_from_plane: float = 10.0
		162
15		163
		164	class ClusterStepperSectionConfig(config_loader.ConfigModel):
		165	"""Segment walk, visualization, and plane-association settings."""
16		166
17	def _load_raw_default_config() -> dict[str, Any]:	167	segment_dir_basename: str = "segment"
18	"""Return the packaged cluster-stepper defaults as a plain dict."""	168	planes_filename: str = "run3_planes.npz"
19	return config_loader.load_packaged_json(__package__ or _PACKAGE_NAME, _DEFAULT_CONFIG_NAME)	169	surface_filenames_base: str = "run4_road_surface_road_extension_*.ply"
		170	step_length: float = 10.0
		171	visualize_each_step: bool = False
		172	visualize_each_segment: bool = False
		173	visualize_road_state_plane_intersections: bool = False
		174	visualize_road_side_state_points: bool = False
		175	road_axis_color: list[float] = [0.0, 0.0, 1.0]
		176	plane_distance_from_lane_threshold: float = 50.0
		177	lane_distance_from_road_surface: float = 0.03
20		178
21		179
22	def normalize_cluster_stepper_config(raw_config: dict[str, Any]) -> dict[str, Any]:	180	class LaneConnection2dConfig(config_loader.ConfigModel):
23	defaults = _load_raw_default_config()	181	"""2D continuation matching used to assemble lanes."""
24	config_loader.validate_against_defaults(	182
25	raw_config,	183	lateral_threshold: float = 0.5
26	defaults,	184	lateral_slack_per_meter: float = 0.02
27	context="cluster-stepper config",	185	max_lateral: float = 1.5
		186	heading_threshold_degrees: float = 25.0
		187	heading_weight: float = 0.02
		188	gap_weight: float = 0.01
		189	max_along_gap: float = 40.0
		190	max_overlap: float = 1.0
		191	same_type_only: bool = True
		192	merge_max_along_gap: float = 80.0
		193	merge_lateral_threshold: float = 0.6
		194	merge_heading_threshold_degrees: float = 20.0
		195	duplicate_lateral_threshold: float = 0.7
		196	merge_max_duplicate_overlap: float = 40.0
		197	refinement_max_station_gap: float = 200.0
		198	line_chain_max_gap: float = 2000.0
		199	chain_max_endpoint_gap: float = 250.0
		200	chain_max_lateral_jump: float = 2.0
		201
		202
		203	class LaneRolesConfig(config_loader.ConfigModel):
		204	"""Line grouping and edge-role dedupe thresholds."""
		205
		206	line_group_lateral_threshold: float = 1.0
		207	min_line_length: float = 10.0
		208	min_profile_overlap: float = 10.0
		209	edge_dedupe_min_overlap: float = 30.0
		210	edge_dedupe_min_separation: float = 1.5
		211	edge_dedupe_max_covered_fraction: float = 0.7
		212
		213
		214	class RoadAxisConfig(config_loader.ConfigModel):
		215	"""Road-axis derivation from completed sides."""
		216
		217	max_angle_degrees: float = 5.0
		218	both_sides_min_coverage: float = 0.5
		219	both_sides_sample_spacing: float = 10.0
		220	both_sides_max_station_gap: float = 30.0
		221	station_merge_tolerance: float = 2.0
		222	median_width_min: float = 1.0
		223	median_width_max: float = 25.0
		224	max_lateral_deviation: float = 6.0
		225	max_extension_length: float = 200.0
		226	both_sides_merge_lateral_tol: float = 6.0
		227	max_blend_length: float = 300.0
		228
		229
		230	class ControlPlotConfig(config_loader.ConfigModel):
		231	"""Control-plot PDF output."""
		232
		233	enabled: bool = True
		234	filename: str = "run7_control_plot.pdf"
		235
		236
		237	class LaneStateConfig(config_loader.ConfigModel):
		238	"""Lane-state snapshot persistence."""
		239
		240	enabled: bool = True
		241	debug_subdir: str = "run7_lane_state"
		242	filename: str = "lane_state_snapshot.json"
		243	lazy_load: bool = True
		244	incremental_intersection_rebuild: bool = True
		245
		246
		247	class TwoStageConfig(config_loader.ConfigModel):
		248	"""Two-stage walk and boundary-reconnect settings."""
		249
		250	partial_snapshot_filename: str = "lane_state_partial.json"
		251	partial_glob: str = "*lane_state_partial.json"
		252	boundary_reconnect: bool = False
		253
		254
		255	class VisualizationLabelsConfig(config_loader.ConfigModel):
		256	"""On-geometry text labels."""
		257
		258	enabled: bool = True
		259	show_lane_labels: bool = True
		260	show_lane_segment_labels: bool = False
		261	show_road_surface_labels: bool = True
		262	text_depth: float = 0.02
		263	label_offset: float = 0.6
		264	z_lift: float = 0.08
		265	lane_text_scale: float = 0.18
		266	lane_segment_text_scale: float = 0.12
		267	road_surface_text_scale: float = 0.14
		268	lane_label_color: list[float] = [0.12, 0.12, 0.12]
		269	lane_segment_label_color: list[float] = [0.28, 0.36, 0.44]
		270
		271
		272	class RoadSidesConfig(config_loader.ConfigModel):
		273	"""Road-side assignment and median geometry."""
		274
		275	visualize_every_intersection: bool = False
		276	visualize_every_new_intersection: bool = False
		277	visualize_every_road_side_plane: bool = False
		278	middle_lane_color: list[float] = [0.0, 1.0, 1.0]
		279	edge_lane_color: list[float] = [1.0, 0.0, 1.0]
		280	extra_lane_color: list[float] = [0.0, 1.0, 0.0]
		281	intersection_extend_search_by_n_segments: int = 1
		282	lane_distance: float = 3.5
		283	lane_distance_limits: list[float] = [3.3, 3.9]
		284	side_split_lateral_gap: float = 3.0
		285	through_line_fraction: float = 0.3
		286	median_min_gap: float = 1.5
		287	median_max_gap: float = 20.0
		288	side_band_margin: float = 3.5
		289	left_side_id: str = "B"
		290	right_side_id: str = "A"
		291	side_window_length: float = 2000.0
		292	side_window_stride: float = 1000.0
		293	side_max_median_offset: float = 30.0
		294	median_max_slope: float = 0.04
		295	median_max_extrapolation: float = 3000.0
		296
		297
		298	class LaneSegmentEndsConfig(config_loader.ConfigModel):
		299	"""Paint-marking end detection along a lane-segment axis."""
		300
		301	n_bins: int = 40
		302	max_distance: float = 1.0
		303	start_from_spline: float = 0.3
		304	start_from_line: float = 0.5
		305	show_histograms: bool = False
		306	histogram_dir: str \| None = None
		307	axis_sample_spacing: float = 0.5
		308	lateral_gate: float = 0.5
		309	min_amplitude: float = 3.0
		310	min_points: int = 30
		311
		312
		313	class LaneSegmentWidthConfig(config_loader.ConfigModel):
		314	"""Paint-marking width measurement along a lane-segment axis."""
		315
		316	bin_width: float = 0.01
		317	lateral_max: float = 0.5
		318	axis_sample_spacing: float = 0.5
		319	end_margin: float = 0.15
		320	min_points: int = 50
		321	min_amplitude: float = 3.0
		322	width_min: float = 0.04
		323	width_max: float = 0.6
		324	flag_width_min: float = 0.08
		325	flag_width_max: float = 0.35
		326	integration_bin_length: float = 0.25
		327	min_bin_points: int = 3
		328	show_histograms: bool = False
		329	histogram_dir: str \| None = None
		330
		331
		332	class LanePointsExportConfig(config_loader.ConfigModel):
		333	"""Optional NPZ dumps of lane points by marking type."""
		334
		335	save_all: bool = True
		336	save_dashed: bool = True
		337	save_solid: bool = True
		338	output_dir: str = "lane_points_exports"
		339	filename_all: str = "lane_points_all.npz"
		340	filename_dashed: str = "lane_points_dashed.npz"
		341	filename_solid: str = "lane_points_solid.npz"
		342
		343
		344	class FileNamingConfig(config_loader.ConfigModel):
		345	"""Step 6/3 input discovery and Step 7 output stems."""
		346
		347	cluster_prefix: str = "run6_cluster_"
		348	geoshift_filename: str = "run3_geoshift.json"
		349	step7_prefix: str = "run7_"
		350	lanes_stem: str = "lanes"
		351
		352
		353	class ClusterStepperConfig(config_loader.ConfigModel):
		354	"""Root cluster-stepper config mirroring ``cluster_stepper.default.json``."""
		355
		356	initial_outlier_removal: InitialOutlierRemovalConfig = InitialOutlierRemovalConfig()
		357	clustering: ClusteringConfig = ClusteringConfig()
		358	cluster_density_filtering: ClusterDensityFilteringConfig = (
		359	ClusterDensityFilteringConfig()
		360	)
		361	cluster_intensity_filtering: ClusterIntensityFilteringConfig = (
		362	ClusterIntensityFilteringConfig()
		363	)
		364	edge_lines: EdgeLinesConfig = EdgeLinesConfig()
		365	line_connection: LineConnectionConfig = LineConnectionConfig()
		366	cross_section: CrossSectionConfig = CrossSectionConfig()
		367	xml_metadata: XmlMetadataConfig = XmlMetadataConfig()
		368	start_section: StartSectionConfig = StartSectionConfig()
		369	cluster_stepper: ClusterStepperSectionConfig = ClusterStepperSectionConfig()
		370	lane_connection_2d: LaneConnection2dConfig = LaneConnection2dConfig()
		371	lane_roles: LaneRolesConfig = LaneRolesConfig()
		372	road_axis: RoadAxisConfig = RoadAxisConfig()
		373	control_plot: ControlPlotConfig = ControlPlotConfig()
		374	lane_state: LaneStateConfig = LaneStateConfig()
		375	two_stage: TwoStageConfig = TwoStageConfig()
		376	visualization_labels: VisualizationLabelsConfig = VisualizationLabelsConfig()
		377	road_sides: RoadSidesConfig = RoadSidesConfig()
		378	lane_segment_ends: LaneSegmentEndsConfig = LaneSegmentEndsConfig()
		379	lane_segment_width: LaneSegmentWidthConfig = LaneSegmentWidthConfig()
		380	lane_points_export: LanePointsExportConfig = LanePointsExportConfig()
		381	file_naming: FileNamingConfig = FileNamingConfig()
		382
		383
		384	def _load_model(
		385	*,
		386	overrides: Mapping[str, Any] \| None = None,
		387	config_path: str \| Path \| None = None,
		388	) -> ClusterStepperConfig:
		389	"""Load packaged defaults, merge overrides, and validate the model."""
		390	return config_loader.load_config(
		391	ClusterStepperConfig,
		392	package=__package__ or _PACKAGE_NAME,
		393	filename=_DEFAULT_CONFIG_NAME,
		394	overrides=overrides,
		395	config_path=config_path,
		396	context=_CONTEXT,
28	error_cls=ClusterStepperConfigError,	397	error_cls=ClusterStepperConfigError,
29	)	398	)
30	return config_loader.deep_merge_dicts(defaults, dict(raw_config))
31		399
32		400
33	def load_cluster_stepper_config(config_path: str \| Path \| None = None) -> dict[str, Any]:	401	def normalize_cluster_stepper_config(raw_config: Mapping[str, Any]) -> dict[str, Any]:
34	if config_path is None:	402	"""Deep-merge raw_config onto the packaged defaults and validate it."""
35	raw_config = _load_raw_default_config()	403	return _load_model(overrides=raw_config).model_dump()
36	else:	404
37	resolved_path = Path(config_path)	405
38	with resolved_path.open("r", encoding="utf-8") as handle:	406	def load_cluster_stepper_config(
39	raw_config = json.load(handle)	407	config_path: str \| Path \| None = None,
40	return normalize_cluster_stepper_config(raw_config)	408	) -> dict[str, Any]:
		409	"""Return the validated config as a plain dict.
		410
		411	Args:
		412	config_path: JSON file read instead of the packaged defaults. Keys it
		413	omits fall back to the model defaults, which mirror
		414	``cluster_stepper.default.json``.
		415	"""
		416	return _load_model(config_path=config_path).model_dump()
41		417
42		418
43	def build_cluster_stepper_config(	419	def build_cluster_stepper_config(
44	*,	420	*,
45	overrides: dict[str, Any] \| None = None,	421	overrides: Mapping[str, Any] \| None = None,
46	config_path: str \| Path \| None = None,	422	config_path: str \| Path \| None = None,
47	) -> dict[str, Any]:	423	) -> dict[str, Any]:
48	config = load_cluster_stepper_config(config_path)	424	"""Load defaults (or config_path) and deep-merge overrides on top."""
49	if overrides:	425	return _load_model(overrides=overrides, config_path=config_path).model_dump()
50	config = config_loader.deep_merge_dicts(config, dict(overrides))
51	return normalize_cluster_stepper_config(config)

Importance #2: tests/test_config.py @@ -0,0 +1,91 @@

	1	"""Tests for the pydantic cluster-stepper config layer."""
	2
	3	from __future__ import annotations
	4
	5	import json
	6
	7	import pytest
	8	from iolabs.common import config_loader
	9
	10	from iolabs_point_cloud_modelling_lines import _config
	11
	12
	13	def _packaged_defaults() -> dict:
	14	return config_loader.load_packaged_json(
	15	"iolabs_point_cloud_modelling_lines", "cluster_stepper.default.json"
	16	)
	17
	18
	19	def test_model_defaults_match_packaged_json():
	20	assert _config.ClusterStepperConfig().model_dump() == _packaged_defaults()
	21
	22
	23	def test_load_defaults_matches_packaged_json():
	24	assert _config.load_cluster_stepper_config() == _packaged_defaults()
	25
	26
	27	def test_error_class_is_a_config_error():
	28	assert issubclass(_config.ClusterStepperConfigError, config_loader.ConfigError)
	29
	30
	31	def test_unknown_root_key_is_rejected():
	32	with pytest.raises(_config.ClusterStepperConfigError, match="bogus"):
	33	_config.normalize_cluster_stepper_config({"bogus": 1})
	34
	35
	36	def test_unknown_nested_key_is_rejected():
	37	with pytest.raises(_config.ClusterStepperConfigError, match="nope"):
	38	_config.normalize_cluster_stepper_config({"clustering": {"nope": 1}})
	39
	40
	41	def test_overrides_are_deep_merged_and_coerced():
	42	config = _config.build_cluster_stepper_config(
	43	overrides={"clustering": {"dbscan_eps": "2.5"}}
	44	)
	45	defaults = _packaged_defaults()
	46	assert config["clustering"]["dbscan_eps"] == 2.5
	47	assert config["clustering"]["use_cache"] == defaults["clustering"]["use_cache"]
	48	assert config["road_axis"] == defaults["road_axis"]
	49
	50
	51	def test_bool_is_not_accepted_for_an_int_field():
	52	with pytest.raises(_config.ClusterStepperConfigError, match="dbscan_min_points"):
	53	_config.normalize_cluster_stepper_config(
	54	{"clustering": {"dbscan_min_points": True}}
	55	)
	56
	57
	58	def test_partial_config_file_falls_back_to_defaults(tmp_path):
	59	path = tmp_path / "cluster_stepper.json"
	60	path.write_text(json.dumps({"clustering": {"dbscan_eps": 9.0}}), encoding="utf-8")
	61	config = _config.load_cluster_stepper_config(path)
	62	assert config["clustering"]["dbscan_eps"] == 9.0
	63	assert config["road_axis"] == _packaged_defaults()["road_axis"]
	64
	65
	66	def test_defaults_are_not_shared_between_loads():
	67	first = _config.load_cluster_stepper_config()
	68	first["road_sides"]["lane_distance_limits"].append(99.0)
	69	second = _config.load_cluster_stepper_config()
	70	expected = _packaged_defaults()["road_sides"]["lane_distance_limits"]
	71	assert second["road_sides"]["lane_distance_limits"] == expected
	72
	73
	74	def test_unknown_key_in_a_config_file_is_rejected(tmp_path):
	75	path = tmp_path / "cluster_stepper.json"
	76	path.write_text(json.dumps({"clustering": {"nope": 1}}), encoding="utf-8")
	77	with pytest.raises(_config.ClusterStepperConfigError, match="nope"):
	78	_config.load_cluster_stepper_config(path)
	79
	80
	81	def test_overrides_win_over_a_config_file(tmp_path):
	82	path = tmp_path / "cluster_stepper.json"
	83	path.write_text(
	84	json.dumps({"clustering": {"dbscan_eps": 9.0, "min_cluster_size": 7}}),
	85	encoding="utf-8",
	86	)
	87	config = _config.build_cluster_stepper_config(
	88	overrides={"clustering": {"dbscan_eps": 1.5}}, config_path=path
	89	)
	90	assert config["clustering"]["dbscan_eps"] == 1.5
	91	assert config["clustering"]["min_cluster_size"] == 7
0

Importance #3: pyproject.toml @@ -1,7 +1,7 @@

1	[project]	1	[project]
2	name = "iolabs-point-cloud-modelling-lines"	2	name = "iolabs-point-cloud-modelling-lines"
3	version = "0.12.1"	3	version = "0.12.2"
4	description = "Lane assembly from LIDAR clusters, road-side assignment, road-axis derivation, and XML export"	4	description = "Lane assembly from LIDAR clusters, road-side assignment, road-axis derivation, and XML export"
5	requires-python = ">=3.11,<3.13"	5	requires-python = ">=3.11,<3.13"
6	dependencies = [	6	dependencies = [
7	"numpy>=1.20.0",	7	"numpy>=1.20.0",

Importance #4: pyproject.toml @@ -10,9 +10,10 @@

     "scipy>=1.7.0",
     "matplotlib>=3.4.0",
     "methodtools>=0.4.0",
     "iolabs-logstash>=0.5.1",
-    "iolabs-common>=0.7.0",
+    "iolabs-common>=0.8.0",
+    "pydantic>=2.7",
     "iolabs-geometry-geometry>=0.11.0",
     "iolabs-geometry-visualization>=0.7.0",
     "iolabs-point-cloud-filtering-surface>=0.7.4",
     "iolabs-point-cloud-las-tools>=0.5.1",

Importance #5: AGENTS.md @@ -32,13 +32,13 @@

32		32
33	Main package: `src/iolabs_point_cloud_modelling_lines/`.	33	Main package: `src/iolabs_point_cloud_modelling_lines/`.
34		34
35	- `cluster_stepper.py` is the Step 7 driver. `ClusterStepper.process_lanes_in_dir()` walks `segment_*` folders, loads Step 3 planes/geoshift, assembles lanes from Step 6 clusters, assigns lanes to road sides, derives the road axis, and exports XML helpers.	35	- `cluster_stepper.py` is the Step 7 driver. `ClusterStepper.process_lanes_in_dir()` walks `segment_*` folders, loads Step 3 planes/geoshift, assembles lanes from Step 6 clusters, assigns lanes to road sides, derives the road axis, and exports XML helpers.
36	- `_config.py` strictly validates config keys against `cluster_stepper.default.json`; when adding config, update the default JSON and tests together.	36	- `_config.py` declares a pydantic `ConfigModel` tree that mirrors `cluster_stepper.default.json`. When adding config, add the field to the model and the JSON default; nothing else.
37	- `_log_props.py` sets the structured logging contract for Step 7: `pipeline_step=s7`, `step_logic=cluster_stepper`, `log_origin=pointcloud_internal`.	37	- `_log_props.py` sets the structured logging contract for Step 7: `pipeline_step=s7`, `step_logic=cluster_stepper`, `log_origin=pointcloud_internal`.
38		38
39	## Coding Guidelines	39	## Coding Guidelines
40		40
41	- Treat config as a public contract. Add new keys to `cluster_stepper.default.json`, validate them through `_config.py`, and cover default/backward-compatible behavior in tests.	41	- Treat config as a public contract. Add new keys to the pydantic model in `_config.py` and `cluster_stepper.default.json`; nothing else. Cover default/backward-compatible behavior in tests.
42	- Keep structured logging stable. New logs should use the existing `_log_props.py` contract and include enough segment/file context to debug batch pipeline runs.	42	- Keep structured logging stable. New logs should use the existing `_log_props.py` contract and include enough segment/file context to debug batch pipeline runs.
43	- Prefer deterministic outputs for XML, snapshots, debug files, and test fixtures. Avoid depending on filesystem iteration order, unordered mappings, or random sampling without a fixed seed.	43	- Prefer deterministic outputs for XML, snapshots, debug files, and test fixtures. Avoid depending on filesystem iteration order, unordered mappings, or random sampling without a fixed seed.
44	- Prefer module imports over importing classes or functions directly. For example, use `from iolabs_geometry_geometry import geometry_tools`, then refer to `geometry_tools.Plane` in code, instead of `from iolabs_geometry_geometry.geometry_tools import Plane`. `from pathlib import Path` and `from dataclasses import dataclass, field` are allowed exceptions because they are unambiguous and idiomatic. Other direct imports should have a clear readability or standard-library convention reason.	44	- Prefer module imports over importing classes or functions directly. For example, use `from iolabs_geometry_geometry import geometry_tools`, then refer to `geometry_tools.Plane` in code, instead of `from iolabs_geometry_geometry.geometry_tools import Plane`. `from pathlib import Path` and `from dataclasses import dataclass, field` are allowed exceptions because they are unambiguous and idiomatic. Other direct imports should have a clear readability or standard-library convention reason.

Importance #6: README.md @@ -16,9 +16,9 @@

16		16
17	## Requirements	17	## Requirements
18		18
19	- Python ≥3.11, <3.13	19	- Python ≥3.11, <3.13
20	- numpy, open3d, torch, scipy, matplotlib, methodtools, iolabs-common, iolabs-geometry-geometry, iolabs-geometry-visualization, iolabs-point-cloud-filtering-surface, iolabs-point-cloud-modelling-export, iolabs-point-cloud-las-tools	20	- numpy, open3d, torch, scipy, matplotlib, methodtools, pydantic, iolabs-common, iolabs-geometry-geometry, iolabs-geometry-visualization, iolabs-point-cloud-filtering-surface, iolabs-point-cloud-modelling-export, iolabs-point-cloud-las-tools
21		21
22	## Usage	22	## Usage
23		23
24	Builds lane models from Step 6 `run6_cluster_*.npz` outputs: spline-fit clusters, assemble lanes by 2D continuation matching, assign completed lanes to road sides, derive the road axis, and export XML. Consumes the GPU Step 6 cluster metadata API and preserves geoshift metadata in XML output.	24	Builds lane models from Step 6 `run6_cluster_*.npz` outputs: spline-fit clusters, assemble lanes by 2D continuation matching, assign completed lanes to road sides, derive the road axis, and export XML. Consumes the GPU Step 6 cluster metadata API and preserves geoshift metadata in XML output.

Importance #7: README.md @@ -16,9 +16,9 @@

16		16
17	## Requirements	17	## Requirements
18		18
19	- Python ≥3.11, <3.13	19	- Python ≥3.11, <3.13
20	- numpy, open3d, torch, scipy, matplotlib, methodtools, iolabs-common, iolabs-geometry-geometry, iolabs-geometry-visualization, iolabs-point-cloud-filtering-surface, iolabs-point-cloud-modelling-export, iolabs-point-cloud-las-tools	20	- numpy, open3d, torch, scipy, matplotlib, methodtools, pydantic, iolabs-common, iolabs-geometry-geometry, iolabs-geometry-visualization, iolabs-point-cloud-filtering-surface, iolabs-point-cloud-modelling-export, iolabs-point-cloud-las-tools
21		21
22	## Usage	22	## Usage
23		23
24	Builds lane models from Step 6 `run6_cluster_*.npz` outputs: spline-fit clusters, assemble lanes by 2D continuation matching, assign completed lanes to road sides, derive the road axis, and export XML. Consumes the GPU Step 6 cluster metadata API and preserves geoshift metadata in XML output.	24	Builds lane models from Step 6 `run6_cluster_*.npz` outputs: spline-fit clusters, assemble lanes by 2D continuation matching, assign completed lanes to road sides, derive the road axis, and export XML. Consumes the GPU Step 6 cluster metadata API and preserves geoshift metadata in XML output.

Importance #8: pyproject.toml @@ -1,7 +1,7 @@

1	[project]	1	[project]
2	name = "iolabs-point-cloud-modelling-lines"	2	name = "iolabs-point-cloud-modelling-lines"
3	version = "0.12.1"	3	version = "0.12.2"
4	description = "Lane assembly from LIDAR clusters, road-side assignment, road-axis derivation, and XML export"	4	description = "Lane assembly from LIDAR clusters, road-side assignment, road-axis derivation, and XML export"
5	requires-python = ">=3.11,<3.13"	5	requires-python = ">=3.11,<3.13"
6	dependencies = [	6	dependencies = [
7	"numpy>=1.20.0",	7	"numpy>=1.20.0",

Importance #9: pyproject.toml @@ -10,9 +10,10 @@

     "scipy>=1.7.0",
     "matplotlib>=3.4.0",
     "methodtools>=0.4.0",
     "iolabs-logstash>=0.5.1",
-    "iolabs-common>=0.7.0",
+    "iolabs-common>=0.8.0",
+    "pydantic>=2.7",
     "iolabs-geometry-geometry>=0.11.0",
     "iolabs-geometry-visualization>=0.7.0",
     "iolabs-point-cloud-filtering-surface>=0.7.4",
     "iolabs-point-cloud-las-tools>=0.5.1",

Importance #10: src/iolabs_point_cloud_modelling_lines/_config.py @@ -1,51 +1,425 @@

		1	"""Cluster-stepper config: packaged JSON defaults validated by a ConfigModel tree."""
		2
1	from __future__ import annotations	3	from __future__ import annotations
2		4
3	import json	5	from collections.abc import Mapping
4	from pathlib import Path	6	from pathlib import Path
5	from typing import Any	7	from typing import Any
6		8
7	from iolabs.common import config_loader	9	from iolabs.common import config_loader
8		10
9	_PACKAGE_NAME = "iolabs_point_cloud_modelling_lines"	11	_PACKAGE_NAME = "iolabs_point_cloud_modelling_lines"
10	_DEFAULT_CONFIG_NAME = "cluster_stepper.default.json"	12	_DEFAULT_CONFIG_NAME = "cluster_stepper.default.json"
		13	_CONTEXT = "cluster-stepper config"
11		14
12		15
13	class ClusterStepperConfigError(config_loader.ConfigError):	16	class ClusterStepperConfigError(config_loader.ConfigError):
14	"""Raised when cluster-stepper config contains unsupported keys."""	17	"""Raised when cluster-stepper config contains unsupported keys or values."""
		18
		19
		20	class InitialOutlierRemovalConfig(config_loader.ConfigModel):
		21	"""Statistical outlier removal before clustering."""
		22
		23	nb_neighbors: int = 10
		24	std_ratio: float = 4.5
		25	max_points_per_segment: int = 800000
		26	n_times_max_points2kill_clustering: float = 2.0
		27
		28
		29	class ClusteringConfig(config_loader.ConfigModel):
		30	"""DBSCAN clustering and cache/visualization toggles."""
		31
		32	cluster_dir: str = "clusters"
		33	dashed_min_length: float = 0.7
		34	use_cache: bool = False
		35	save_cache: bool = True
		36	visualize: bool = False
		37	enable_density_filtering: bool = True
		38	enable_intensity_trimming: bool = False
		39	enable_intensity_peak_split: bool = False
		40	dbscan_eps: float = 1.0
		41	dbscan_min_points: int = 50
		42	min_cluster_size: int = 200
		43	dbscan_guard_threshold_points: int = 15000
		44
		45
		46	class ClusterDensityFilteringConfig(config_loader.ConfigModel):
		47	"""Density-cutoff filters applied to clusters."""
		48
		49	density_cutoffs: list[float] = [0.001, 0.002, 0.005, 0.012]
		50	cutoff_portions: list[float] = [0.85, 0.85, 0.15, 0.001]
		51
		52
		53	class ClusterIntensityFilteringConfig(config_loader.ConfigModel):
		54	"""Intensity-histogram filters applied to clusters."""
		55
		56	n_bins: int = 50
		57	intensity_range: list[int] = [0, 255]
		58	peak_distance: int = 5
		59	peak_prominence: int = 10
		60	sigma_estimate: float = 3.0
		61	sigma_estimate_peak_distance_fraction: float = 0.125
		62	n_sigma_intensity_cutoff: float = 5.0
		63	min_sigma: float = 0.5
		64
		65
		66	class SubclusterDbscanMemoryGuardConfig(config_loader.ConfigModel):
		67	"""Memory/time guard around subcluster DBSCAN."""
		68
		69	mem_limit_bytes: int = 16106127360
		70	mem_poll_interval_s: float = 0.5
		71	mem_hard_kill: bool = False
		72	timeout_s: float = 120.0
		73	progress_in_guarded: bool = False
		74	dbscan_guard_threshold_points: int = 15000
		75	dbscan_stats_enabled: bool = True
		76	dbscan_stats_output_path: str = (
		77	"data/00_external/250812_Color scans/inference/dbscan_stats.csv"
		78	)
		79	dbscan_stats_format: str = "csv"
		80
		81
		82	class SubclusterVisualizationConfig(config_loader.ConfigModel):
		83	"""Optional subcluster histogram visualization."""
		84
		85	visualize_every_cluster: bool = False
		86	save_histograms: bool = False
		87	histogram_dir: str = "histograms"
		88	n_bins: int = 30
		89
		90
		91	class EdgeLinesSplineFittingConfig(config_loader.ConfigModel):
		92	"""Spline fitting of edge-line clusters."""
		93
		94	max_sharp_bend_degrees: float = 7.5
		95	max_sharp_bend_rejection_rate: float = 0.2
		96	fit_line_length: float = 3.0
		97	spline_point_count: int = 1000
		98	target_distance: float = 0.3
		99	n_parts_start_line: int = 6
		100	min_width: float = 0.1
		101	subcluster_dbscan_eps: float = 0.2
		102	subcluster_dbscan_min_samples: int = 5
		103	subcluster_dbscan_n_jobs: int = -1
		104	subcluster_dbscan_memory_guard: SubclusterDbscanMemoryGuardConfig = (
		105	SubclusterDbscanMemoryGuardConfig()
		106	)
		107	subcluster_visualization: SubclusterVisualizationConfig = (
		108	SubclusterVisualizationConfig()
		109	)
		110
		111
		112	class EdgeLinesExtrapolationConfig(config_loader.ConfigModel):
		113	"""How far edge-line splines are extrapolated."""
		114
		115	extrapolate_to: float = 10.0
		116	extrapolation_points: int = 30
		117
		118
		119	class EdgeLinesSegmentPropertiesConfig(config_loader.ConfigModel):
		120	"""Curvature thresholds that split edge-line splines into segments."""
		121
		122	angle_change_rate: float = 5.0
		123	angle_change_threshold: float = 45.0
		124	angle_change_rate_last_segment: float = 5.0
		125
		126
		127	class EdgeLinesConfig(config_loader.ConfigModel):
		128	"""Edge-lane spline fitting, extrapolation, and segmentation."""
		129
		130	spline_fitting: EdgeLinesSplineFittingConfig = EdgeLinesSplineFittingConfig()
		131	extrapolation: EdgeLinesExtrapolationConfig = EdgeLinesExtrapolationConfig()
		132	segment_properties: EdgeLinesSegmentPropertiesConfig = (
		133	EdgeLinesSegmentPropertiesConfig()
		134	)
		135
		136
		137	class LineConnectionConfig(config_loader.ConfigModel):
		138	"""Legacy 3D line-connection gates."""
		139
		140	perpendicular_distance_threshold: float = 1.0
		141	longitudinal_distance_factor: float = 0.2
		142	negative_offset: float = 0.5
		143
		144
		145	class CrossSectionConfig(config_loader.ConfigModel):
		146	"""Cross-section sampling along the road axis."""
		147
		148	interval: float = 5.0
		149	visualize_crosssections: bool = False
		150
		151
		152	class XmlMetadataConfig(config_loader.ConfigModel):
		153	"""XML export metadata."""
		154
		155	software_version: str = "0.3.1"
		156
		157
		158	class StartSectionConfig(config_loader.ConfigModel):
		159	"""Where the walk starts relative to the first plane."""
		160
		161	distance_from_plane: float = 10.0
		162
15		163
		164	class ClusterStepperSectionConfig(config_loader.ConfigModel):
		165	"""Segment walk, visualization, and plane-association settings."""
16		166
17	def _load_raw_default_config() -> dict[str, Any]:	167	segment_dir_basename: str = "segment"
18	"""Return the packaged cluster-stepper defaults as a plain dict."""	168	planes_filename: str = "run3_planes.npz"
19	return config_loader.load_packaged_json(__package__ or _PACKAGE_NAME, _DEFAULT_CONFIG_NAME)	169	surface_filenames_base: str = "run4_road_surface_road_extension_*.ply"
		170	step_length: float = 10.0
		171	visualize_each_step: bool = False
		172	visualize_each_segment: bool = False
		173	visualize_road_state_plane_intersections: bool = False
		174	visualize_road_side_state_points: bool = False
		175	road_axis_color: list[float] = [0.0, 0.0, 1.0]
		176	plane_distance_from_lane_threshold: float = 50.0
		177	lane_distance_from_road_surface: float = 0.03
20		178
21		179
22	def normalize_cluster_stepper_config(raw_config: dict[str, Any]) -> dict[str, Any]:	180	class LaneConnection2dConfig(config_loader.ConfigModel):
23	defaults = _load_raw_default_config()	181	"""2D continuation matching used to assemble lanes."""
24	config_loader.validate_against_defaults(	182
25	raw_config,	183	lateral_threshold: float = 0.5
26	defaults,	184	lateral_slack_per_meter: float = 0.02
27	context="cluster-stepper config",	185	max_lateral: float = 1.5
		186	heading_threshold_degrees: float = 25.0
		187	heading_weight: float = 0.02
		188	gap_weight: float = 0.01
		189	max_along_gap: float = 40.0
		190	max_overlap: float = 1.0
		191	same_type_only: bool = True
		192	merge_max_along_gap: float = 80.0
		193	merge_lateral_threshold: float = 0.6
		194	merge_heading_threshold_degrees: float = 20.0
		195	duplicate_lateral_threshold: float = 0.7
		196	merge_max_duplicate_overlap: float = 40.0
		197	refinement_max_station_gap: float = 200.0
		198	line_chain_max_gap: float = 2000.0
		199	chain_max_endpoint_gap: float = 250.0
		200	chain_max_lateral_jump: float = 2.0
		201
		202
		203	class LaneRolesConfig(config_loader.ConfigModel):
		204	"""Line grouping and edge-role dedupe thresholds."""
		205
		206	line_group_lateral_threshold: float = 1.0
		207	min_line_length: float = 10.0
		208	min_profile_overlap: float = 10.0
		209	edge_dedupe_min_overlap: float = 30.0
		210	edge_dedupe_min_separation: float = 1.5
		211	edge_dedupe_max_covered_fraction: float = 0.7
		212
		213
		214	class RoadAxisConfig(config_loader.ConfigModel):
		215	"""Road-axis derivation from completed sides."""
		216
		217	max_angle_degrees: float = 5.0
		218	both_sides_min_coverage: float = 0.5
		219	both_sides_sample_spacing: float = 10.0
		220	both_sides_max_station_gap: float = 30.0
		221	station_merge_tolerance: float = 2.0
		222	median_width_min: float = 1.0
		223	median_width_max: float = 25.0
		224	max_lateral_deviation: float = 6.0
		225	max_extension_length: float = 200.0
		226	both_sides_merge_lateral_tol: float = 6.0
		227	max_blend_length: float = 300.0
		228
		229
		230	class ControlPlotConfig(config_loader.ConfigModel):
		231	"""Control-plot PDF output."""
		232
		233	enabled: bool = True
		234	filename: str = "run7_control_plot.pdf"
		235
		236
		237	class LaneStateConfig(config_loader.ConfigModel):
		238	"""Lane-state snapshot persistence."""
		239
		240	enabled: bool = True
		241	debug_subdir: str = "run7_lane_state"
		242	filename: str = "lane_state_snapshot.json"
		243	lazy_load: bool = True
		244	incremental_intersection_rebuild: bool = True
		245
		246
		247	class TwoStageConfig(config_loader.ConfigModel):
		248	"""Two-stage walk and boundary-reconnect settings."""
		249
		250	partial_snapshot_filename: str = "lane_state_partial.json"
		251	partial_glob: str = "*lane_state_partial.json"
		252	boundary_reconnect: bool = False
		253
		254
		255	class VisualizationLabelsConfig(config_loader.ConfigModel):
		256	"""On-geometry text labels."""
		257
		258	enabled: bool = True
		259	show_lane_labels: bool = True
		260	show_lane_segment_labels: bool = False
		261	show_road_surface_labels: bool = True
		262	text_depth: float = 0.02
		263	label_offset: float = 0.6
		264	z_lift: float = 0.08
		265	lane_text_scale: float = 0.18
		266	lane_segment_text_scale: float = 0.12
		267	road_surface_text_scale: float = 0.14
		268	lane_label_color: list[float] = [0.12, 0.12, 0.12]
		269	lane_segment_label_color: list[float] = [0.28, 0.36, 0.44]
		270
		271
		272	class RoadSidesConfig(config_loader.ConfigModel):
		273	"""Road-side assignment and median geometry."""
		274
		275	visualize_every_intersection: bool = False
		276	visualize_every_new_intersection: bool = False
		277	visualize_every_road_side_plane: bool = False
		278	middle_lane_color: list[float] = [0.0, 1.0, 1.0]
		279	edge_lane_color: list[float] = [1.0, 0.0, 1.0]
		280	extra_lane_color: list[float] = [0.0, 1.0, 0.0]
		281	intersection_extend_search_by_n_segments: int = 1
		282	lane_distance: float = 3.5
		283	lane_distance_limits: list[float] = [3.3, 3.9]
		284	side_split_lateral_gap: float = 3.0
		285	through_line_fraction: float = 0.3
		286	median_min_gap: float = 1.5
		287	median_max_gap: float = 20.0
		288	side_band_margin: float = 3.5
		289	left_side_id: str = "B"
		290	right_side_id: str = "A"
		291	side_window_length: float = 2000.0
		292	side_window_stride: float = 1000.0
		293	side_max_median_offset: float = 30.0
		294	median_max_slope: float = 0.04
		295	median_max_extrapolation: float = 3000.0
		296
		297
		298	class LaneSegmentEndsConfig(config_loader.ConfigModel):
		299	"""Paint-marking end detection along a lane-segment axis."""
		300
		301	n_bins: int = 40
		302	max_distance: float = 1.0
		303	start_from_spline: float = 0.3
		304	start_from_line: float = 0.5
		305	show_histograms: bool = False
		306	histogram_dir: str \| None = None
		307	axis_sample_spacing: float = 0.5
		308	lateral_gate: float = 0.5
		309	min_amplitude: float = 3.0
		310	min_points: int = 30
		311
		312
		313	class LaneSegmentWidthConfig(config_loader.ConfigModel):
		314	"""Paint-marking width measurement along a lane-segment axis."""
		315
		316	bin_width: float = 0.01
		317	lateral_max: float = 0.5
		318	axis_sample_spacing: float = 0.5
		319	end_margin: float = 0.15
		320	min_points: int = 50
		321	min_amplitude: float = 3.0
		322	width_min: float = 0.04
		323	width_max: float = 0.6
		324	flag_width_min: float = 0.08
		325	flag_width_max: float = 0.35
		326	integration_bin_length: float = 0.25
		327	min_bin_points: int = 3
		328	show_histograms: bool = False
		329	histogram_dir: str \| None = None
		330
		331
		332	class LanePointsExportConfig(config_loader.ConfigModel):
		333	"""Optional NPZ dumps of lane points by marking type."""
		334
		335	save_all: bool = True
		336	save_dashed: bool = True
		337	save_solid: bool = True
		338	output_dir: str = "lane_points_exports"
		339	filename_all: str = "lane_points_all.npz"
		340	filename_dashed: str = "lane_points_dashed.npz"
		341	filename_solid: str = "lane_points_solid.npz"
		342
		343
		344	class FileNamingConfig(config_loader.ConfigModel):
		345	"""Step 6/3 input discovery and Step 7 output stems."""
		346
		347	cluster_prefix: str = "run6_cluster_"
		348	geoshift_filename: str = "run3_geoshift.json"
		349	step7_prefix: str = "run7_"
		350	lanes_stem: str = "lanes"
		351
		352
		353	class ClusterStepperConfig(config_loader.ConfigModel):
		354	"""Root cluster-stepper config mirroring ``cluster_stepper.default.json``."""
		355
		356	initial_outlier_removal: InitialOutlierRemovalConfig = InitialOutlierRemovalConfig()
		357	clustering: ClusteringConfig = ClusteringConfig()
		358	cluster_density_filtering: ClusterDensityFilteringConfig = (
		359	ClusterDensityFilteringConfig()
		360	)
		361	cluster_intensity_filtering: ClusterIntensityFilteringConfig = (
		362	ClusterIntensityFilteringConfig()
		363	)
		364	edge_lines: EdgeLinesConfig = EdgeLinesConfig()
		365	line_connection: LineConnectionConfig = LineConnectionConfig()
		366	cross_section: CrossSectionConfig = CrossSectionConfig()
		367	xml_metadata: XmlMetadataConfig = XmlMetadataConfig()
		368	start_section: StartSectionConfig = StartSectionConfig()
		369	cluster_stepper: ClusterStepperSectionConfig = ClusterStepperSectionConfig()
		370	lane_connection_2d: LaneConnection2dConfig = LaneConnection2dConfig()
		371	lane_roles: LaneRolesConfig = LaneRolesConfig()
		372	road_axis: RoadAxisConfig = RoadAxisConfig()
		373	control_plot: ControlPlotConfig = ControlPlotConfig()
		374	lane_state: LaneStateConfig = LaneStateConfig()
		375	two_stage: TwoStageConfig = TwoStageConfig()
		376	visualization_labels: VisualizationLabelsConfig = VisualizationLabelsConfig()
		377	road_sides: RoadSidesConfig = RoadSidesConfig()
		378	lane_segment_ends: LaneSegmentEndsConfig = LaneSegmentEndsConfig()
		379	lane_segment_width: LaneSegmentWidthConfig = LaneSegmentWidthConfig()
		380	lane_points_export: LanePointsExportConfig = LanePointsExportConfig()
		381	file_naming: FileNamingConfig = FileNamingConfig()
		382
		383
		384	def _load_model(
		385	*,
		386	overrides: Mapping[str, Any] \| None = None,
		387	config_path: str \| Path \| None = None,
		388	) -> ClusterStepperConfig:
		389	"""Load packaged defaults, merge overrides, and validate the model."""
		390	return config_loader.load_config(
		391	ClusterStepperConfig,
		392	package=__package__ or _PACKAGE_NAME,
		393	filename=_DEFAULT_CONFIG_NAME,
		394	overrides=overrides,
		395	config_path=config_path,
		396	context=_CONTEXT,
28	error_cls=ClusterStepperConfigError,	397	error_cls=ClusterStepperConfigError,
29	)	398	)
30	return config_loader.deep_merge_dicts(defaults, dict(raw_config))
31		399
32		400
33	def load_cluster_stepper_config(config_path: str \| Path \| None = None) -> dict[str, Any]:	401	def normalize_cluster_stepper_config(raw_config: Mapping[str, Any]) -> dict[str, Any]:
34	if config_path is None:	402	"""Deep-merge raw_config onto the packaged defaults and validate it."""
35	raw_config = _load_raw_default_config()	403	return _load_model(overrides=raw_config).model_dump()
36	else:	404
37	resolved_path = Path(config_path)	405
38	with resolved_path.open("r", encoding="utf-8") as handle:	406	def load_cluster_stepper_config(
39	raw_config = json.load(handle)	407	config_path: str \| Path \| None = None,
40	return normalize_cluster_stepper_config(raw_config)	408	) -> dict[str, Any]:
		409	"""Return the validated config as a plain dict.
		410
		411	Args:
		412	config_path: JSON file read instead of the packaged defaults. Keys it
		413	omits fall back to the model defaults, which mirror
		414	``cluster_stepper.default.json``.
		415	"""
		416	return _load_model(config_path=config_path).model_dump()
41		417
42		418
43	def build_cluster_stepper_config(	419	def build_cluster_stepper_config(
44	*,	420	*,
45	overrides: dict[str, Any] \| None = None,	421	overrides: Mapping[str, Any] \| None = None,
46	config_path: str \| Path \| None = None,	422	config_path: str \| Path \| None = None,
47	) -> dict[str, Any]:	423	) -> dict[str, Any]:
48	config = load_cluster_stepper_config(config_path)	424	"""Load defaults (or config_path) and deep-merge overrides on top."""
49	if overrides:	425	return _load_model(overrides=overrides, config_path=config_path).model_dump()
50	config = config_loader.deep_merge_dicts(config, dict(overrides))
51	return normalize_cluster_stepper_config(config)

Importance #11: tests/test_config.py @@ -0,0 +1,91 @@

	1	"""Tests for the pydantic cluster-stepper config layer."""
	2
	3	from __future__ import annotations
	4
	5	import json
	6
	7	import pytest
	8	from iolabs.common import config_loader
	9
	10	from iolabs_point_cloud_modelling_lines import _config
	11
	12
	13	def _packaged_defaults() -> dict:
	14	return config_loader.load_packaged_json(
	15	"iolabs_point_cloud_modelling_lines", "cluster_stepper.default.json"
	16	)
	17
	18
	19	def test_model_defaults_match_packaged_json():
	20	assert _config.ClusterStepperConfig().model_dump() == _packaged_defaults()
	21
	22
	23	def test_load_defaults_matches_packaged_json():
	24	assert _config.load_cluster_stepper_config() == _packaged_defaults()
	25
	26
	27	def test_error_class_is_a_config_error():
	28	assert issubclass(_config.ClusterStepperConfigError, config_loader.ConfigError)
	29
	30
	31	def test_unknown_root_key_is_rejected():
	32	with pytest.raises(_config.ClusterStepperConfigError, match="bogus"):
	33	_config.normalize_cluster_stepper_config({"bogus": 1})
	34
	35
	36	def test_unknown_nested_key_is_rejected():
	37	with pytest.raises(_config.ClusterStepperConfigError, match="nope"):
	38	_config.normalize_cluster_stepper_config({"clustering": {"nope": 1}})
	39
	40
	41	def test_overrides_are_deep_merged_and_coerced():
	42	config = _config.build_cluster_stepper_config(
	43	overrides={"clustering": {"dbscan_eps": "2.5"}}
	44	)
	45	defaults = _packaged_defaults()
	46	assert config["clustering"]["dbscan_eps"] == 2.5
	47	assert config["clustering"]["use_cache"] == defaults["clustering"]["use_cache"]
	48	assert config["road_axis"] == defaults["road_axis"]
	49
	50
	51	def test_bool_is_not_accepted_for_an_int_field():
	52	with pytest.raises(_config.ClusterStepperConfigError, match="dbscan_min_points"):
	53	_config.normalize_cluster_stepper_config(
	54	{"clustering": {"dbscan_min_points": True}}
	55	)
	56
	57
	58	def test_partial_config_file_falls_back_to_defaults(tmp_path):
	59	path = tmp_path / "cluster_stepper.json"
	60	path.write_text(json.dumps({"clustering": {"dbscan_eps": 9.0}}), encoding="utf-8")
	61	config = _config.load_cluster_stepper_config(path)
	62	assert config["clustering"]["dbscan_eps"] == 9.0
	63	assert config["road_axis"] == _packaged_defaults()["road_axis"]
	64
	65
	66	def test_defaults_are_not_shared_between_loads():
	67	first = _config.load_cluster_stepper_config()
	68	first["road_sides"]["lane_distance_limits"].append(99.0)
	69	second = _config.load_cluster_stepper_config()
	70	expected = _packaged_defaults()["road_sides"]["lane_distance_limits"]
	71	assert second["road_sides"]["lane_distance_limits"] == expected
	72
	73
	74	def test_unknown_key_in_a_config_file_is_rejected(tmp_path):
	75	path = tmp_path / "cluster_stepper.json"
	76	path.write_text(json.dumps({"clustering": {"nope": 1}}), encoding="utf-8")
	77	with pytest.raises(_config.ClusterStepperConfigError, match="nope"):
	78	_config.load_cluster_stepper_config(path)
	79
	80
	81	def test_overrides_win_over_a_config_file(tmp_path):
	82	path = tmp_path / "cluster_stepper.json"
	83	path.write_text(
	84	json.dumps({"clustering": {"dbscan_eps": 9.0, "min_cluster_size": 7}}),
	85	encoding="utf-8",
	86	)
	87	config = _config.build_cluster_stepper_config(
	88	overrides={"clustering": {"dbscan_eps": 1.5}}, config_path=path
	89	)
	90	assert config["clustering"]["dbscan_eps"] == 1.5
	91	assert config["clustering"]["min_cluster_size"] == 7
0