Back to report index

Step 5 filteringintensity ac8b5d1: AI3D-379 Pydantic config models via iolabs-common ConfigModel

Miroslav Simko <ms@iolabs.ch> 2026-09-02T08:34:11+02:00

Commit #25 · 21 snippets

 README.md                                          |   6 +-
 pyproject.toml                                     |   5 +-
 .../_config.py                                     | 331 +++++++--------------
 .../bright_points_filter.py                        |  54 +---
 tests/test_bright_points_config.py                 |  44 ++-
 5 files changed, 171 insertions(+), 269 deletions(-)
Importance #1: src/iolabs_point_cloud_filtering_intensity/_config.py @@ -1,230 +1,127 @@
1import json1"""Bright-points config: packaged JSON defaults, overrides, pydantic validation."""
2from importlib import resources
3from pathlib import Path
4from typing import Any
5
6ALLOWED_BRIGHT_POINTS_CONFIG_KEYS = frozenset(
7 {
8 "filter_mode",
9 "allow_missing_rgb",
10 "save_bright_points_pcd",
11 "save_all_delta_ply",
12 "device",
13 "intensity_min_cut",
14 "intensity_max_cut",
15 "saturation_min_cut",
16 "saturation_max_cut",
17 "hue_min_cut",
18 "hue_max_cut",
19 "laser_intensity_min_cut",
20 "laser_intensity_max_cut",
21 "laser_intensity_in_range",
22 "intensity_in_range",
23 "saturation_in_range",
24 "hue_in_range",
25 "intensity_cutoff",
26 "default_sigma",
27 "laser_intensity_fitting",
28 "color_intensity_fitting",
29 "file_naming",
30 }
31)
32
33ALLOWED_FILE_NAMING_KEYS = frozenset(
34 {
35 "road_surface_suffix",
36 "bright_filtered_suffix",
37 }
38)
39
40ALLOWED_FITTING_KEYS = frozenset(
41 {
42 "bins",
43 "prominence",
44 "fit_width",
45 "n_sigma",
46 "cutoff_n_sigma",
47 "max_sigma",
48 "default_sigma",
49 "min_mu",
50 "max_mu",
51 "min_sigma",
52 "angle_min",
53 "angle_max",
54 "angle_step",
55 }
56)
57
58DEFAULT_LASER_INTENSITY_FITTING: dict[str, Any] = {
59 "bins": 100,
60 "prominence": 30.0,
61 "fit_width": 10000.0,
62 "n_sigma": 4.0,
63 "cutoff_n_sigma": 7.0,
64 "max_sigma": 5000.0,
65 "min_mu": 0.0,
66 "max_mu": 68000.0,
67 "min_sigma": 200.0,
68 "angle_min": -80.0,
69 "angle_max": 80.0,
70 "angle_step": 10.0,
71}
72
73DEFAULT_COLOR_INTENSITY_FITTING: dict[str, Any] = {
74 "bins": 100,
75 "prominence": 30.0,
76 "fit_width": 15.26,
77 "n_sigma": 4.0,
78 "cutoff_n_sigma": 7.0,
79 "max_sigma": 7.63,
80 "default_sigma": 7.63,
81 "min_mu": 0.0,
82 "max_mu": 100.0,
83 "min_sigma": 0.31,
84 "angle_min": -80.0,
85 "angle_max": 80.0,
86 "angle_step": 10.0,
87}
88
89
90class BrightPointsConfigError(ValueError):
91 """Raised when bright-points config contains unsupported keys."""
92
93
94def _default_config_path() -> Path:
95 if __package__ in {None, ""}:
96 return Path(__file__).resolve().with_name("bright_points.default.json")
97 return Path(str(resources.files(__package__).joinpath("bright_points.default.json")))
98
99
100def _deep_merge_dicts(
101 base: dict[str, Any],
102 overrides: dict[str, Any],
103) -> dict[str, Any]:
104 for key, value in overrides.items():
105 if isinstance(value, dict) and isinstance(base.get(key), dict):
106 base[key] = _deep_merge_dicts(dict(base[key]), value)
107 else:
108 base[key] = value
109 return base
110
1112
112def _validate_allowed_keys(3import logging
113 config: dict[str, Any],4from collections.abc import Mapping
114 allowed_keys: frozenset[str],5from pathlib import Path
115 *,6from typing import Any, Literal
116 context: str,7
117) -> None:8from iolabs.common import config_loader
118 unknown_keys = sorted(set(config) - allowed_keys)9
119 if not unknown_keys:10logger = logging.getLogger(__name__)
120 return11
12112_PACKAGE = "iolabs_point_cloud_filtering_intensity"
122 allowed = ", ".join(sorted(allowed_keys))13_DEFAULT_FILENAME = "bright_points.default.json"
123 raise BrightPointsConfigError(14_CONTEXT = "bright-points config"
124 f"Unknown {context} key(s): {', '.join(unknown_keys)}. Allowed keys: {allowed}"15
125 )16
12617class FileNamingConfig(config_loader.ConfigModel):
12718 """Output filename suffixes for road-surface input and bright-filtered output."""
128def _normalize_file_naming(raw_file_naming: Any) -> dict[str, Any]:19
129 if raw_file_naming is None:20 road_surface_suffix: str = "_run4_road_surface"
130 file_naming: dict[str, Any] = {}21 bright_filtered_suffix: str = "_run5_bright_filtered"
131 elif isinstance(raw_file_naming, dict):22
132 file_naming = dict(raw_file_naming)23
133 else:24class LaserIntensityFittingConfig(config_loader.ConfigModel):
134 raise BrightPointsConfigError(25 """Per-scan-angle Gaussian fitting for laser intensity in [0, 65535]."""
135 "bright-points config field 'file_naming' must be a mapping"26
136 )27 bins: int = 100
13728 prominence: float = 30.0
138 _validate_allowed_keys(29 fit_width: float = 10000.0
139 file_naming,30 n_sigma: float = 4.0
140 ALLOWED_FILE_NAMING_KEYS,31 cutoff_n_sigma: float = 7.0
141 context="bright-points file_naming",32 max_sigma: float = 5000.0
142 )33 # None means "fall back to the top-level default_sigma".
143 file_naming.setdefault("road_surface_suffix", "_run4_road_surface")34 default_sigma: float | None = None
144 file_naming.setdefault("bright_filtered_suffix", "_run5_bright_filtered")35 min_mu: float = 0.0
145 return file_naming36 max_mu: float = 68000.0
14637 min_sigma: float = 200.0
14738 angle_min: float = -80.0
148def _normalize_fitting_config(39 angle_max: float = 80.0
149 raw_fitting_config: Any,40 angle_step: float = 10.0
150 *,41
151 defaults: dict[str, Any],42
152 context: str,43class ColorIntensityFittingConfig(config_loader.ConfigModel):
153) -> dict[str, Any]:44 """Per-scan-angle Gaussian fitting for HSI intensity in [0, 100] percent."""
154 if raw_fitting_config is None:45
155 fitting_config: dict[str, Any] = {}46 bins: int = 100
156 elif isinstance(raw_fitting_config, dict):47 prominence: float = 30.0
157 fitting_config = dict(raw_fitting_config)48 fit_width: float = 15.26
158 else:49 n_sigma: float = 4.0
159 raise BrightPointsConfigError(f"{context} must be a mapping")50 cutoff_n_sigma: float = 7.0
16051 max_sigma: float = 7.63
161 _validate_allowed_keys(52 default_sigma: float = 7.63
162 fitting_config,53 min_mu: float = 0.0
163 ALLOWED_FITTING_KEYS,54 max_mu: float = 100.0
164 context=context,55 min_sigma: float = 0.31
165 )56 angle_min: float = -80.0
16657 angle_max: float = 80.0
167 normalized = dict(defaults)58 angle_step: float = 10.0
168 normalized.update(fitting_config)59
169 return normalized60
61class BrightPointsConfig(config_loader.ConfigModel):
62 """Top-level bright-points filter config; keys match ``bright_points.default.json``."""
63
64 filter_mode: Literal[
65 "laser_intensity",
66 "color_intensity",
67 "color_cuts",
68 "simple_intensity_cutoff",
69 ] = "color_cuts"
70 allow_missing_rgb: bool = False
71 save_bright_points_pcd: bool = False
72 save_all_delta_ply: bool = True
73 device: str = "cpu"
74 intensity_min_cut: float = 20.0
75 intensity_max_cut: float = 100.0
76 saturation_min_cut: float = 0.0
77 saturation_max_cut: float = 30.0
78 hue_min_cut: float = 0.0
79 hue_max_cut: float = 360.0
80 laser_intensity_min_cut: float = 43000.0
81 laser_intensity_max_cut: float = 65535.0
82 laser_intensity_in_range: bool = True
83 intensity_in_range: bool = True
84 saturation_in_range: bool = True
85 hue_in_range: bool = True
86 intensity_cutoff: float = 43000.0
87 default_sigma: float = 5000.0
88 laser_intensity_fitting: LaserIntensityFittingConfig = LaserIntensityFittingConfig()
89 color_intensity_fitting: ColorIntensityFittingConfig = ColorIntensityFittingConfig()
90 file_naming: FileNamingConfig = FileNamingConfig()
91
92
93class BrightPointsConfigError(config_loader.ConfigError):
94 """Raised when bright-points config contains unsupported keys or values."""
17095
17196
172def normalize_bright_points_config(raw_config: dict[str, Any]) -> dict[str, Any]:97def normalize_bright_points_config(raw_config: dict[str, Any]) -> dict[str, Any]:
173 config = dict(raw_config)98 """Validate *raw_config* and return a complete dict with model defaults filled in."""
174 _validate_allowed_keys(99 return config_loader.validate_config(
175 config,100 BrightPointsConfig,
176 ALLOWED_BRIGHT_POINTS_CONFIG_KEYS,101 raw_config,
177 context="bright-points config",102 context=_CONTEXT,
178 )103 error_cls=BrightPointsConfigError,
179104 ).model_dump()
180 config.setdefault("filter_mode", "color_cuts")
181 config.setdefault("allow_missing_rgb", False)
182 config.setdefault("save_bright_points_pcd", False)
183 config.setdefault("save_all_delta_ply", True)
184 config.setdefault("device", "cpu")
185
186 config.setdefault("intensity_min_cut", 20.0)
187 config.setdefault("intensity_max_cut", 100.0)
188 config.setdefault("saturation_min_cut", 0.0)
189 config.setdefault("saturation_max_cut", 30.0)
190 config.setdefault("hue_min_cut", 0.0)
191 config.setdefault("hue_max_cut", 360.0)
192 config.setdefault("laser_intensity_min_cut", 43000.0)
193 config.setdefault("laser_intensity_max_cut", 65535.0)
194 config.setdefault("laser_intensity_in_range", True)
195 config.setdefault("intensity_in_range", True)
196 config.setdefault("saturation_in_range", True)
197 config.setdefault("hue_in_range", True)
198 config.setdefault("intensity_cutoff", 43000.0)
199 config.setdefault("default_sigma", 5000.0)
200
201 config["laser_intensity_fitting"] = _normalize_fitting_config(
202 config.get("laser_intensity_fitting"),
203 defaults=DEFAULT_LASER_INTENSITY_FITTING,
204 context="bright-points laser_intensity_fitting",
205 )
206 config["color_intensity_fitting"] = _normalize_fitting_config(
207 config.get("color_intensity_fitting"),
208 defaults=DEFAULT_COLOR_INTENSITY_FITTING,
209 context="bright-points color_intensity_fitting",
210 )
211 config["file_naming"] = _normalize_file_naming(config.get("file_naming"))
212 return config
213105
214106
215def load_bright_points_config(config_path: str | Path | None = None) -> dict[str, Any]:107def load_bright_points_config(config_path: str | Path | None = None) -> dict[str, Any]:
216 resolved_path = Path(config_path) if config_path is not None else _default_config_path()108 """Load packaged (or *config_path*) defaults and return a validated config dict."""
217 with resolved_path.open("r", encoding="utf-8") as handle:109 return build_bright_points_config(config_path=config_path)
218 raw_config: dict[str, Any] = json.load(handle)
219 return normalize_bright_points_config(raw_config)
220110
221111
222def build_bright_points_config(112def build_bright_points_config(
223 *,113 *,
224 overrides: dict[str, Any] | None = None,114 overrides: Mapping[str, Any] | None = None,
225 config_path: str | Path | None = None,115 config_path: str | Path | None = None,
226) -> dict[str, Any]:116) -> dict[str, Any]:
227 config = load_bright_points_config(config_path)117 """Load defaults, deep-merge *overrides*, and return a validated config dict."""
228 if overrides:118 logger.debug("Loading bright-points config (config_path=%s)", config_path)
229 config = _deep_merge_dicts(config, dict(overrides))119 return config_loader.load_config(
230 return normalize_bright_points_config(config)120 BrightPointsConfig,
121 package=_PACKAGE,
122 filename=_DEFAULT_FILENAME,
123 overrides=overrides,
124 config_path=config_path,
125 context=_CONTEXT,
126 error_cls=BrightPointsConfigError,
127 ).model_dump()
Importance #2: src/iolabs_point_cloud_filtering_intensity/bright_points_filter.py @@ -20,43 +20,8 @@
20from ._config import normalize_bright_points_config20from ._config import normalize_bright_points_config
21from ._log_props import LOG_PROPS21from ._log_props import LOG_PROPS
22from . import intensity_tools22from . import intensity_tools
2323
24LASER_INTENSITY_FITTING_DEFAULT: dict[str, Any] = {
25 # Mirrors the historical hardcoded defaults used for laser intensity in [0, 65535]
26 "bins": 100,
27 "prominence": 30.0,
28 "fit_width": 10000.0,
29 "n_sigma": 4.0,
30 "cutoff_n_sigma": 7.0,
31 "max_sigma": 5000.0,
32 "min_mu": 0.0,
33 "max_mu": 68000.0,
34 "min_sigma": 200.0,
35 "angle_min": -80.0,
36 "angle_max": 80.0,
37 "angle_step": 10.0,
38}
39
40# Color intensity (HSI-I) is in [0, 100] percent. Scale the intensity-unit parameters
41# (fit_width, max_sigma) from the laser defaults by 100/65535.
42COLOR_INTENSITY_FITTING_DEFAULT: dict[str, Any] = {
43 "bins": 100,
44 "prominence": 30.0,
45 "fit_width": 15.26,
46 "n_sigma": 4.0,
47 "cutoff_n_sigma": 7.0,
48 "max_sigma": 7.63,
49 "default_sigma": 7.63,
50 "min_mu": 0.0,
51 "max_mu": 100.0,
52 # Scale laser min_sigma=200 by 100/65535 -> ~0.31
53 "min_sigma": 0.31,
54 "angle_min": -80.0,
55 "angle_max": 80.0,
56 "angle_step": 10.0,
57}
58
5924
60def _save_version_json_atomic(25def _save_version_json_atomic(
61 output_path: Path,26 output_path: Path,
62 step_name: str,27 step_name: str,
Importance #3: src/iolabs_point_cloud_filtering_intensity/bright_points_filter.py @@ -331,12 +296,9 @@
331296
332 if filter_mode == "laser_intensity":297 if filter_mode == "laser_intensity":
333 logger.info("Using laser intensity scan-angle fitting for filtering")298 logger.info("Using laser intensity scan-angle fitting for filtering")
334 fit_values = np.asarray(color_data.intensity)299 fit_values = np.asarray(color_data.intensity)
335 fit_cfg = {300 fit_cfg = dict(self.config["laser_intensity_fitting"])
336 **LASER_INTENSITY_FITTING_DEFAULT,
337 **dict(self.config.get("laser_intensity_fitting", {})),
338 }
339 self._process_with_angle_slice_fitting(301 self._process_with_angle_slice_fitting(
340 points=points,302 points=points,
341 color_data=color_data,303 color_data=color_data,
342 output_pdf=output_pdf,304 output_pdf=output_pdf,
Importance #4: src/iolabs_point_cloud_filtering_intensity/bright_points_filter.py @@ -354,12 +316,9 @@
354316
355 if filter_mode == "color_intensity":317 if filter_mode == "color_intensity":
356 logger.info("Using color-intensity (HSI-I) scan-angle fitting for filtering")318 logger.info("Using color-intensity (HSI-I) scan-angle fitting for filtering")
357 fit_values = self._compute_color_intensity_percent(color_data)319 fit_values = self._compute_color_intensity_percent(color_data)
358 fit_cfg = {320 fit_cfg = dict(self.config["color_intensity_fitting"])
359 **COLOR_INTENSITY_FITTING_DEFAULT,
360 **dict(self.config.get("color_intensity_fitting", {})),
361 }
362 self._process_with_angle_slice_fitting(321 self._process_with_angle_slice_fitting(
363 points=points,322 points=points,
364 color_data=color_data,323 color_data=color_data,
365 output_pdf=output_pdf,324 output_pdf=output_pdf,
Importance #5: src/iolabs_point_cloud_filtering_intensity/bright_points_filter.py @@ -556,9 +515,11 @@
556 peak_sigmas_array = np.array(peak_sigmas)515 peak_sigmas_array = np.array(peak_sigmas)
557 peak_sigma_per_point_all = peak_sigmas_array[interval_indices_all]516 peak_sigma_per_point_all = peak_sigmas_array[interval_indices_all]
558 517
559 # Use default sigma if provided in config, otherwise use max_sigma fallback518 # Use default sigma if provided in config, otherwise use max_sigma fallback
560 default_sigma = fit_cfg.get("default_sigma", self.config.get("default_sigma", None))519 default_sigma = fit_cfg.get("default_sigma")
520 if default_sigma is None:
521 default_sigma = self.config.get("default_sigma")
561 max_sigma_fallback = float(fit_cfg.get("max_sigma", 5000.0))522 max_sigma_fallback = float(fit_cfg.get("max_sigma", 5000.0))
562 if default_sigma is not None:523 if default_sigma is not None:
563 # Replace any invalid (zero or negative) sigmas with default524 # Replace any invalid (zero or negative) sigmas with default
564 peak_sigma_per_point_all = np.where(525 peak_sigma_per_point_all = np.where(
Importance #6: src/iolabs_point_cloud_filtering_intensity/bright_points_filter.py @@ -691,12 +652,9 @@
691 output_npz: Path,652 output_npz: Path,
692 points_on_road_file: Path,653 points_on_road_file: Path,
693 ) -> None:654 ) -> None:
694 """Backward-compatible wrapper for laser intensity fitting."""655 """Backward-compatible wrapper for laser intensity fitting."""
695 fit_cfg = {656 fit_cfg = dict(self.config["laser_intensity_fitting"])
696 **LASER_INTENSITY_FITTING_DEFAULT,
697 **dict(self.config.get("laser_intensity_fitting", {})),
698 }
699 self._process_with_angle_slice_fitting(657 self._process_with_angle_slice_fitting(
700 points=points,658 points=points,
701 color_data=color_data,659 color_data=color_data,
702 output_pdf=output_pdf,660 output_pdf=output_pdf,
Importance #7: tests/test_bright_points_config.py @@ -1,21 +1,30 @@
1import pytest1import pytest
22
3from iolabs.common import config_loader
4from iolabs_point_cloud_filtering_intensity import _config
3from iolabs_point_cloud_filtering_intensity import (5from iolabs_point_cloud_filtering_intensity import (
4 BrightPointsConfigError,6 BrightPointsConfigError,
7 build_bright_points_config,
8 load_bright_points_config,
5 normalize_bright_points_config,9 normalize_bright_points_config,
6)10)
711
812
13def test_bright_points_config_error_is_config_error():
14 assert issubclass(BrightPointsConfigError, config_loader.ConfigError)
15 assert issubclass(BrightPointsConfigError, ValueError)
16
17
9def test_normalize_bright_points_config_rejects_unknown_top_level_keys():18def test_normalize_bright_points_config_rejects_unknown_top_level_keys():
10 with pytest.raises(BrightPointsConfigError, match="Unknown bright-points config key"):19 with pytest.raises(BrightPointsConfigError, match="Unknown bright-points config key"):
11 normalize_bright_points_config({"random_seed": 42})20 normalize_bright_points_config({"random_seed": 42})
1221
1322
14def test_normalize_bright_points_config_rejects_unknown_nested_keys():23def test_normalize_bright_points_config_rejects_unknown_nested_keys():
15 with pytest.raises(24 with pytest.raises(
16 BrightPointsConfigError,25 BrightPointsConfigError,
17 match="Unknown bright-points laser_intensity_fitting key",26 match="Unknown bright-points config.laser_intensity_fitting key",
18 ):27 ):
19 normalize_bright_points_config(28 normalize_bright_points_config(
20 {29 {
21 "laser_intensity_fitting": {30 "laser_intensity_fitting": {
Importance #8: tests/test_bright_points_config.py @@ -45,4 +54,37 @@
45def test_normalize_bright_points_config_accepts_allow_missing_rgb():54def test_normalize_bright_points_config_accepts_allow_missing_rgb():
46 config = normalize_bright_points_config({"allow_missing_rgb": True})55 config = normalize_bright_points_config({"allow_missing_rgb": True})
4756
48 assert config["allow_missing_rgb"] is True57 assert config["allow_missing_rgb"] is True
58
59
60def test_load_bright_points_config_validates_packaged_defaults():
61 config = load_bright_points_config()
62
63 assert type(config) is dict
64 assert config["filter_mode"] == "color_cuts"
65 assert "laser_intensity_fitting" in config
66 assert "color_intensity_fitting" in config
67 assert "file_naming" in config
68
69
70def test_build_bright_points_config_deep_merges_overrides():
71 config = build_bright_points_config(
72 overrides={"laser_intensity_fitting": {"bins": 50}}
73 )
74
75 assert config["laser_intensity_fitting"]["bins"] == 50
76 assert config["laser_intensity_fitting"]["prominence"] == 30.0
77 assert config["filter_mode"] == "color_cuts"
78
79
80def test_packaged_defaults_match_model_defaults():
81 assert load_bright_points_config() == _config.BrightPointsConfig().model_dump()
82
83
84def test_laser_intensity_fitting_accepts_default_sigma():
85 config = build_bright_points_config(
86 overrides={"laser_intensity_fitting": {"default_sigma": 1234.0}}
87 )
88
89 assert config["laser_intensity_fitting"]["default_sigma"] == 1234.0
90 assert load_bright_points_config()["laser_intensity_fitting"]["default_sigma"] is None
Importance #9: pyproject.toml @@ -1,7 +1,7 @@
1[project]1[project]
2name = "iolabs-point-cloud-filtering-intensity"2name = "iolabs-point-cloud-filtering-intensity"
3version = "0.6.2"3version = "0.6.3"
4description = "Intensity-based bright lane marking detection"4description = "Intensity-based bright lane marking detection"
5requires-python = ">=3.11,<3.13"5requires-python = ">=3.11,<3.13"
6dependencies = [6dependencies = [
7 "numpy>=1.20.0",7 "numpy>=1.20.0",
Importance #10: pyproject.toml @@ -12,9 +12,10 @@
12 "laspy>=2.0.0",12 "laspy>=2.0.0",
13 "pypdf>=4.3.1",13 "pypdf>=4.3.1",
14 "tqdm>=4.0.0",14 "tqdm>=4.0.0",
15 "iolabs-logstash>=0.5.1",15 "iolabs-logstash>=0.5.1",
16 "iolabs-common",16 "pydantic>=2.7",
17 "iolabs-common>=0.8.0",
17 "iolabs-geometry-geometry",18 "iolabs-geometry-geometry",
18 "iolabs-geometry-visualization",19 "iolabs-geometry-visualization",
19 "iolabs-point-cloud-filtering-surface>=0.5.0",20 "iolabs-point-cloud-filtering-surface>=0.5.0",
20 "iolabs-point-cloud-las-tools>=0.5.1",21 "iolabs-point-cloud-las-tools>=0.5.1",
Importance #11: README.md @@ -16,14 +16,18 @@
1616
17## Requirements17## Requirements
1818
19- Python ≥3.11, <3.1319- Python ≥3.11, <3.13
20- numpy, open3d, torch, scipy, matplotlib, laspy, pypdf, tqdm, iolabs-common, iolabs-geometry-geometry, iolabs-geometry-visualization, iolabs-point-cloud-filtering-surface, iolabs-point-cloud-las-tools20- numpy, open3d, torch, scipy, matplotlib, laspy, pypdf, tqdm, pydantic, iolabs-common (≥0.8.0), iolabs-geometry-geometry, iolabs-geometry-visualization, iolabs-point-cloud-filtering-surface, iolabs-point-cloud-las-tools
2121
22## Usage22## Usage
2323
24Identifies bright lane markings using intensity filtering on top of surface detection. Feeds into cluster-based middle lane detection and trajectory segmentation.24Identifies bright lane markings using intensity filtering on top of surface detection. Feeds into cluster-based middle lane detection and trajectory segmentation.
2525
26## Configuration
27
28Defaults live in `src/iolabs_point_cloud_filtering_intensity/bright_points.default.json`. The schema is the pydantic model tree in `_config.py` (`BrightPointsConfig` and nested section models). To add a config key, add the field to the model **and** the JSON default; unknown keys are rejected.
29
26## Develop locally (Nexus)30## Develop locally (Nexus)
2731
28Internal `iolabs-*` dependencies resolve through the private Nexus index declared in `pyproject.toml`. Export Nexus credentials before any `uv` command that touches private deps — e.g. by sourcing `../3dai.lanefinder/scripts/nexus_credentials.sh` from your shell rc — then:32Internal `iolabs-*` dependencies resolve through the private Nexus index declared in `pyproject.toml`. Export Nexus credentials before any `uv` command that touches private deps — e.g. by sourcing `../3dai.lanefinder/scripts/nexus_credentials.sh` from your shell rc — then:
2933
Importance #12: pyproject.toml @@ -1,7 +1,7 @@
1[project]1[project]
2name = "iolabs-point-cloud-filtering-intensity"2name = "iolabs-point-cloud-filtering-intensity"
3version = "0.6.2"3version = "0.6.3"
4description = "Intensity-based bright lane marking detection"4description = "Intensity-based bright lane marking detection"
5requires-python = ">=3.11,<3.13"5requires-python = ">=3.11,<3.13"
6dependencies = [6dependencies = [
7 "numpy>=1.20.0",7 "numpy>=1.20.0",
Importance #13: pyproject.toml @@ -12,9 +12,10 @@
12 "laspy>=2.0.0",12 "laspy>=2.0.0",
13 "pypdf>=4.3.1",13 "pypdf>=4.3.1",
14 "tqdm>=4.0.0",14 "tqdm>=4.0.0",
15 "iolabs-logstash>=0.5.1",15 "iolabs-logstash>=0.5.1",
16 "iolabs-common",16 "pydantic>=2.7",
17 "iolabs-common>=0.8.0",
17 "iolabs-geometry-geometry",18 "iolabs-geometry-geometry",
18 "iolabs-geometry-visualization",19 "iolabs-geometry-visualization",
19 "iolabs-point-cloud-filtering-surface>=0.5.0",20 "iolabs-point-cloud-filtering-surface>=0.5.0",
20 "iolabs-point-cloud-las-tools>=0.5.1",21 "iolabs-point-cloud-las-tools>=0.5.1",
Importance #14: src/iolabs_point_cloud_filtering_intensity/_config.py @@ -1,230 +1,127 @@
1import json1"""Bright-points config: packaged JSON defaults, overrides, pydantic validation."""
2from importlib import resources
3from pathlib import Path
4from typing import Any
5
6ALLOWED_BRIGHT_POINTS_CONFIG_KEYS = frozenset(
7 {
8 "filter_mode",
9 "allow_missing_rgb",
10 "save_bright_points_pcd",
11 "save_all_delta_ply",
12 "device",
13 "intensity_min_cut",
14 "intensity_max_cut",
15 "saturation_min_cut",
16 "saturation_max_cut",
17 "hue_min_cut",
18 "hue_max_cut",
19 "laser_intensity_min_cut",
20 "laser_intensity_max_cut",
21 "laser_intensity_in_range",
22 "intensity_in_range",
23 "saturation_in_range",
24 "hue_in_range",
25 "intensity_cutoff",
26 "default_sigma",
27 "laser_intensity_fitting",
28 "color_intensity_fitting",
29 "file_naming",
30 }
31)
32
33ALLOWED_FILE_NAMING_KEYS = frozenset(
34 {
35 "road_surface_suffix",
36 "bright_filtered_suffix",
37 }
38)
39
40ALLOWED_FITTING_KEYS = frozenset(
41 {
42 "bins",
43 "prominence",
44 "fit_width",
45 "n_sigma",
46 "cutoff_n_sigma",
47 "max_sigma",
48 "default_sigma",
49 "min_mu",
50 "max_mu",
51 "min_sigma",
52 "angle_min",
53 "angle_max",
54 "angle_step",
55 }
56)
57
58DEFAULT_LASER_INTENSITY_FITTING: dict[str, Any] = {
59 "bins": 100,
60 "prominence": 30.0,
61 "fit_width": 10000.0,
62 "n_sigma": 4.0,
63 "cutoff_n_sigma": 7.0,
64 "max_sigma": 5000.0,
65 "min_mu": 0.0,
66 "max_mu": 68000.0,
67 "min_sigma": 200.0,
68 "angle_min": -80.0,
69 "angle_max": 80.0,
70 "angle_step": 10.0,
71}
72
73DEFAULT_COLOR_INTENSITY_FITTING: dict[str, Any] = {
74 "bins": 100,
75 "prominence": 30.0,
76 "fit_width": 15.26,
77 "n_sigma": 4.0,
78 "cutoff_n_sigma": 7.0,
79 "max_sigma": 7.63,
80 "default_sigma": 7.63,
81 "min_mu": 0.0,
82 "max_mu": 100.0,
83 "min_sigma": 0.31,
84 "angle_min": -80.0,
85 "angle_max": 80.0,
86 "angle_step": 10.0,
87}
88
89
90class BrightPointsConfigError(ValueError):
91 """Raised when bright-points config contains unsupported keys."""
92
93
94def _default_config_path() -> Path:
95 if __package__ in {None, ""}:
96 return Path(__file__).resolve().with_name("bright_points.default.json")
97 return Path(str(resources.files(__package__).joinpath("bright_points.default.json")))
98
99
100def _deep_merge_dicts(
101 base: dict[str, Any],
102 overrides: dict[str, Any],
103) -> dict[str, Any]:
104 for key, value in overrides.items():
105 if isinstance(value, dict) and isinstance(base.get(key), dict):
106 base[key] = _deep_merge_dicts(dict(base[key]), value)
107 else:
108 base[key] = value
109 return base
110
1112
112def _validate_allowed_keys(3import logging
113 config: dict[str, Any],4from collections.abc import Mapping
114 allowed_keys: frozenset[str],5from pathlib import Path
115 *,6from typing import Any, Literal
116 context: str,7
117) -> None:8from iolabs.common import config_loader
118 unknown_keys = sorted(set(config) - allowed_keys)9
119 if not unknown_keys:10logger = logging.getLogger(__name__)
120 return11
12112_PACKAGE = "iolabs_point_cloud_filtering_intensity"
122 allowed = ", ".join(sorted(allowed_keys))13_DEFAULT_FILENAME = "bright_points.default.json"
123 raise BrightPointsConfigError(14_CONTEXT = "bright-points config"
124 f"Unknown {context} key(s): {', '.join(unknown_keys)}. Allowed keys: {allowed}"15
125 )16
12617class FileNamingConfig(config_loader.ConfigModel):
12718 """Output filename suffixes for road-surface input and bright-filtered output."""
128def _normalize_file_naming(raw_file_naming: Any) -> dict[str, Any]:19
129 if raw_file_naming is None:20 road_surface_suffix: str = "_run4_road_surface"
130 file_naming: dict[str, Any] = {}21 bright_filtered_suffix: str = "_run5_bright_filtered"
131 elif isinstance(raw_file_naming, dict):22
132 file_naming = dict(raw_file_naming)23
133 else:24class LaserIntensityFittingConfig(config_loader.ConfigModel):
134 raise BrightPointsConfigError(25 """Per-scan-angle Gaussian fitting for laser intensity in [0, 65535]."""
135 "bright-points config field 'file_naming' must be a mapping"26
136 )27 bins: int = 100
13728 prominence: float = 30.0
138 _validate_allowed_keys(29 fit_width: float = 10000.0
139 file_naming,30 n_sigma: float = 4.0
140 ALLOWED_FILE_NAMING_KEYS,31 cutoff_n_sigma: float = 7.0
141 context="bright-points file_naming",32 max_sigma: float = 5000.0
142 )33 # None means "fall back to the top-level default_sigma".
143 file_naming.setdefault("road_surface_suffix", "_run4_road_surface")34 default_sigma: float | None = None
144 file_naming.setdefault("bright_filtered_suffix", "_run5_bright_filtered")35 min_mu: float = 0.0
145 return file_naming36 max_mu: float = 68000.0
14637 min_sigma: float = 200.0
14738 angle_min: float = -80.0
148def _normalize_fitting_config(39 angle_max: float = 80.0
149 raw_fitting_config: Any,40 angle_step: float = 10.0
150 *,41
151 defaults: dict[str, Any],42
152 context: str,43class ColorIntensityFittingConfig(config_loader.ConfigModel):
153) -> dict[str, Any]:44 """Per-scan-angle Gaussian fitting for HSI intensity in [0, 100] percent."""
154 if raw_fitting_config is None:45
155 fitting_config: dict[str, Any] = {}46 bins: int = 100
156 elif isinstance(raw_fitting_config, dict):47 prominence: float = 30.0
157 fitting_config = dict(raw_fitting_config)48 fit_width: float = 15.26
158 else:49 n_sigma: float = 4.0
159 raise BrightPointsConfigError(f"{context} must be a mapping")50 cutoff_n_sigma: float = 7.0
16051 max_sigma: float = 7.63
161 _validate_allowed_keys(52 default_sigma: float = 7.63
162 fitting_config,53 min_mu: float = 0.0
163 ALLOWED_FITTING_KEYS,54 max_mu: float = 100.0
164 context=context,55 min_sigma: float = 0.31
165 )56 angle_min: float = -80.0
16657 angle_max: float = 80.0
167 normalized = dict(defaults)58 angle_step: float = 10.0
168 normalized.update(fitting_config)59
169 return normalized60
61class BrightPointsConfig(config_loader.ConfigModel):
62 """Top-level bright-points filter config; keys match ``bright_points.default.json``."""
63
64 filter_mode: Literal[
65 "laser_intensity",
66 "color_intensity",
67 "color_cuts",
68 "simple_intensity_cutoff",
69 ] = "color_cuts"
70 allow_missing_rgb: bool = False
71 save_bright_points_pcd: bool = False
72 save_all_delta_ply: bool = True
73 device: str = "cpu"
74 intensity_min_cut: float = 20.0
75 intensity_max_cut: float = 100.0
76 saturation_min_cut: float = 0.0
77 saturation_max_cut: float = 30.0
78 hue_min_cut: float = 0.0
79 hue_max_cut: float = 360.0
80 laser_intensity_min_cut: float = 43000.0
81 laser_intensity_max_cut: float = 65535.0
82 laser_intensity_in_range: bool = True
83 intensity_in_range: bool = True
84 saturation_in_range: bool = True
85 hue_in_range: bool = True
86 intensity_cutoff: float = 43000.0
87 default_sigma: float = 5000.0
88 laser_intensity_fitting: LaserIntensityFittingConfig = LaserIntensityFittingConfig()
89 color_intensity_fitting: ColorIntensityFittingConfig = ColorIntensityFittingConfig()
90 file_naming: FileNamingConfig = FileNamingConfig()
91
92
93class BrightPointsConfigError(config_loader.ConfigError):
94 """Raised when bright-points config contains unsupported keys or values."""
17095
17196
172def normalize_bright_points_config(raw_config: dict[str, Any]) -> dict[str, Any]:97def normalize_bright_points_config(raw_config: dict[str, Any]) -> dict[str, Any]:
173 config = dict(raw_config)98 """Validate *raw_config* and return a complete dict with model defaults filled in."""
174 _validate_allowed_keys(99 return config_loader.validate_config(
175 config,100 BrightPointsConfig,
176 ALLOWED_BRIGHT_POINTS_CONFIG_KEYS,101 raw_config,
177 context="bright-points config",102 context=_CONTEXT,
178 )103 error_cls=BrightPointsConfigError,
179104 ).model_dump()
180 config.setdefault("filter_mode", "color_cuts")
181 config.setdefault("allow_missing_rgb", False)
182 config.setdefault("save_bright_points_pcd", False)
183 config.setdefault("save_all_delta_ply", True)
184 config.setdefault("device", "cpu")
185
186 config.setdefault("intensity_min_cut", 20.0)
187 config.setdefault("intensity_max_cut", 100.0)
188 config.setdefault("saturation_min_cut", 0.0)
189 config.setdefault("saturation_max_cut", 30.0)
190 config.setdefault("hue_min_cut", 0.0)
191 config.setdefault("hue_max_cut", 360.0)
192 config.setdefault("laser_intensity_min_cut", 43000.0)
193 config.setdefault("laser_intensity_max_cut", 65535.0)
194 config.setdefault("laser_intensity_in_range", True)
195 config.setdefault("intensity_in_range", True)
196 config.setdefault("saturation_in_range", True)
197 config.setdefault("hue_in_range", True)
198 config.setdefault("intensity_cutoff", 43000.0)
199 config.setdefault("default_sigma", 5000.0)
200
201 config["laser_intensity_fitting"] = _normalize_fitting_config(
202 config.get("laser_intensity_fitting"),
203 defaults=DEFAULT_LASER_INTENSITY_FITTING,
204 context="bright-points laser_intensity_fitting",
205 )
206 config["color_intensity_fitting"] = _normalize_fitting_config(
207 config.get("color_intensity_fitting"),
208 defaults=DEFAULT_COLOR_INTENSITY_FITTING,
209 context="bright-points color_intensity_fitting",
210 )
211 config["file_naming"] = _normalize_file_naming(config.get("file_naming"))
212 return config
213105
214106
215def load_bright_points_config(config_path: str | Path | None = None) -> dict[str, Any]:107def load_bright_points_config(config_path: str | Path | None = None) -> dict[str, Any]:
216 resolved_path = Path(config_path) if config_path is not None else _default_config_path()108 """Load packaged (or *config_path*) defaults and return a validated config dict."""
217 with resolved_path.open("r", encoding="utf-8") as handle:109 return build_bright_points_config(config_path=config_path)
218 raw_config: dict[str, Any] = json.load(handle)
219 return normalize_bright_points_config(raw_config)
220110
221111
222def build_bright_points_config(112def build_bright_points_config(
223 *,113 *,
224 overrides: dict[str, Any] | None = None,114 overrides: Mapping[str, Any] | None = None,
225 config_path: str | Path | None = None,115 config_path: str | Path | None = None,
226) -> dict[str, Any]:116) -> dict[str, Any]:
227 config = load_bright_points_config(config_path)117 """Load defaults, deep-merge *overrides*, and return a validated config dict."""
228 if overrides:118 logger.debug("Loading bright-points config (config_path=%s)", config_path)
229 config = _deep_merge_dicts(config, dict(overrides))119 return config_loader.load_config(
230 return normalize_bright_points_config(config)120 BrightPointsConfig,
121 package=_PACKAGE,
122 filename=_DEFAULT_FILENAME,
123 overrides=overrides,
124 config_path=config_path,
125 context=_CONTEXT,
126 error_cls=BrightPointsConfigError,
127 ).model_dump()
Importance #15: src/iolabs_point_cloud_filtering_intensity/bright_points_filter.py @@ -20,43 +20,8 @@
20from ._config import normalize_bright_points_config20from ._config import normalize_bright_points_config
21from ._log_props import LOG_PROPS21from ._log_props import LOG_PROPS
22from . import intensity_tools22from . import intensity_tools
2323
24LASER_INTENSITY_FITTING_DEFAULT: dict[str, Any] = {
25 # Mirrors the historical hardcoded defaults used for laser intensity in [0, 65535]
26 "bins": 100,
27 "prominence": 30.0,
28 "fit_width": 10000.0,
29 "n_sigma": 4.0,
30 "cutoff_n_sigma": 7.0,
31 "max_sigma": 5000.0,
32 "min_mu": 0.0,
33 "max_mu": 68000.0,
34 "min_sigma": 200.0,
35 "angle_min": -80.0,
36 "angle_max": 80.0,
37 "angle_step": 10.0,
38}
39
40# Color intensity (HSI-I) is in [0, 100] percent. Scale the intensity-unit parameters
41# (fit_width, max_sigma) from the laser defaults by 100/65535.
42COLOR_INTENSITY_FITTING_DEFAULT: dict[str, Any] = {
43 "bins": 100,
44 "prominence": 30.0,
45 "fit_width": 15.26,
46 "n_sigma": 4.0,
47 "cutoff_n_sigma": 7.0,
48 "max_sigma": 7.63,
49 "default_sigma": 7.63,
50 "min_mu": 0.0,
51 "max_mu": 100.0,
52 # Scale laser min_sigma=200 by 100/65535 -> ~0.31
53 "min_sigma": 0.31,
54 "angle_min": -80.0,
55 "angle_max": 80.0,
56 "angle_step": 10.0,
57}
58
5924
60def _save_version_json_atomic(25def _save_version_json_atomic(
61 output_path: Path,26 output_path: Path,
62 step_name: str,27 step_name: str,
Importance #16: src/iolabs_point_cloud_filtering_intensity/bright_points_filter.py @@ -331,12 +296,9 @@
331296
332 if filter_mode == "laser_intensity":297 if filter_mode == "laser_intensity":
333 logger.info("Using laser intensity scan-angle fitting for filtering")298 logger.info("Using laser intensity scan-angle fitting for filtering")
334 fit_values = np.asarray(color_data.intensity)299 fit_values = np.asarray(color_data.intensity)
335 fit_cfg = {300 fit_cfg = dict(self.config["laser_intensity_fitting"])
336 **LASER_INTENSITY_FITTING_DEFAULT,
337 **dict(self.config.get("laser_intensity_fitting", {})),
338 }
339 self._process_with_angle_slice_fitting(301 self._process_with_angle_slice_fitting(
340 points=points,302 points=points,
341 color_data=color_data,303 color_data=color_data,
342 output_pdf=output_pdf,304 output_pdf=output_pdf,
Importance #17: src/iolabs_point_cloud_filtering_intensity/bright_points_filter.py @@ -354,12 +316,9 @@
354316
355 if filter_mode == "color_intensity":317 if filter_mode == "color_intensity":
356 logger.info("Using color-intensity (HSI-I) scan-angle fitting for filtering")318 logger.info("Using color-intensity (HSI-I) scan-angle fitting for filtering")
357 fit_values = self._compute_color_intensity_percent(color_data)319 fit_values = self._compute_color_intensity_percent(color_data)
358 fit_cfg = {320 fit_cfg = dict(self.config["color_intensity_fitting"])
359 **COLOR_INTENSITY_FITTING_DEFAULT,
360 **dict(self.config.get("color_intensity_fitting", {})),
361 }
362 self._process_with_angle_slice_fitting(321 self._process_with_angle_slice_fitting(
363 points=points,322 points=points,
364 color_data=color_data,323 color_data=color_data,
365 output_pdf=output_pdf,324 output_pdf=output_pdf,
Importance #18: src/iolabs_point_cloud_filtering_intensity/bright_points_filter.py @@ -556,9 +515,11 @@
556 peak_sigmas_array = np.array(peak_sigmas)515 peak_sigmas_array = np.array(peak_sigmas)
557 peak_sigma_per_point_all = peak_sigmas_array[interval_indices_all]516 peak_sigma_per_point_all = peak_sigmas_array[interval_indices_all]
558 517
559 # Use default sigma if provided in config, otherwise use max_sigma fallback518 # Use default sigma if provided in config, otherwise use max_sigma fallback
560 default_sigma = fit_cfg.get("default_sigma", self.config.get("default_sigma", None))519 default_sigma = fit_cfg.get("default_sigma")
520 if default_sigma is None:
521 default_sigma = self.config.get("default_sigma")
561 max_sigma_fallback = float(fit_cfg.get("max_sigma", 5000.0))522 max_sigma_fallback = float(fit_cfg.get("max_sigma", 5000.0))
562 if default_sigma is not None:523 if default_sigma is not None:
563 # Replace any invalid (zero or negative) sigmas with default524 # Replace any invalid (zero or negative) sigmas with default
564 peak_sigma_per_point_all = np.where(525 peak_sigma_per_point_all = np.where(
Importance #19: src/iolabs_point_cloud_filtering_intensity/bright_points_filter.py @@ -691,12 +652,9 @@
691 output_npz: Path,652 output_npz: Path,
692 points_on_road_file: Path,653 points_on_road_file: Path,
693 ) -> None:654 ) -> None:
694 """Backward-compatible wrapper for laser intensity fitting."""655 """Backward-compatible wrapper for laser intensity fitting."""
695 fit_cfg = {656 fit_cfg = dict(self.config["laser_intensity_fitting"])
696 **LASER_INTENSITY_FITTING_DEFAULT,
697 **dict(self.config.get("laser_intensity_fitting", {})),
698 }
699 self._process_with_angle_slice_fitting(657 self._process_with_angle_slice_fitting(
700 points=points,658 points=points,
701 color_data=color_data,659 color_data=color_data,
702 output_pdf=output_pdf,660 output_pdf=output_pdf,
Importance #20: tests/test_bright_points_config.py @@ -1,21 +1,30 @@
1import pytest1import pytest
22
3from iolabs.common import config_loader
4from iolabs_point_cloud_filtering_intensity import _config
3from iolabs_point_cloud_filtering_intensity import (5from iolabs_point_cloud_filtering_intensity import (
4 BrightPointsConfigError,6 BrightPointsConfigError,
7 build_bright_points_config,
8 load_bright_points_config,
5 normalize_bright_points_config,9 normalize_bright_points_config,
6)10)
711
812
13def test_bright_points_config_error_is_config_error():
14 assert issubclass(BrightPointsConfigError, config_loader.ConfigError)
15 assert issubclass(BrightPointsConfigError, ValueError)
16
17
9def test_normalize_bright_points_config_rejects_unknown_top_level_keys():18def test_normalize_bright_points_config_rejects_unknown_top_level_keys():
10 with pytest.raises(BrightPointsConfigError, match="Unknown bright-points config key"):19 with pytest.raises(BrightPointsConfigError, match="Unknown bright-points config key"):
11 normalize_bright_points_config({"random_seed": 42})20 normalize_bright_points_config({"random_seed": 42})
1221
1322
14def test_normalize_bright_points_config_rejects_unknown_nested_keys():23def test_normalize_bright_points_config_rejects_unknown_nested_keys():
15 with pytest.raises(24 with pytest.raises(
16 BrightPointsConfigError,25 BrightPointsConfigError,
17 match="Unknown bright-points laser_intensity_fitting key",26 match="Unknown bright-points config.laser_intensity_fitting key",
18 ):27 ):
19 normalize_bright_points_config(28 normalize_bright_points_config(
20 {29 {
21 "laser_intensity_fitting": {30 "laser_intensity_fitting": {
Importance #21: tests/test_bright_points_config.py @@ -45,4 +54,37 @@
45def test_normalize_bright_points_config_accepts_allow_missing_rgb():54def test_normalize_bright_points_config_accepts_allow_missing_rgb():
46 config = normalize_bright_points_config({"allow_missing_rgb": True})55 config = normalize_bright_points_config({"allow_missing_rgb": True})
4756
48 assert config["allow_missing_rgb"] is True57 assert config["allow_missing_rgb"] is True
58
59
60def test_load_bright_points_config_validates_packaged_defaults():
61 config = load_bright_points_config()
62
63 assert type(config) is dict
64 assert config["filter_mode"] == "color_cuts"
65 assert "laser_intensity_fitting" in config
66 assert "color_intensity_fitting" in config
67 assert "file_naming" in config
68
69
70def test_build_bright_points_config_deep_merges_overrides():
71 config = build_bright_points_config(
72 overrides={"laser_intensity_fitting": {"bins": 50}}
73 )
74
75 assert config["laser_intensity_fitting"]["bins"] == 50
76 assert config["laser_intensity_fitting"]["prominence"] == 30.0
77 assert config["filter_mode"] == "color_cuts"
78
79
80def test_packaged_defaults_match_model_defaults():
81 assert load_bright_points_config() == _config.BrightPointsConfig().model_dump()
82
83
84def test_laser_intensity_fitting_accepts_default_sigma():
85 config = build_bright_points_config(
86 overrides={"laser_intensity_fitting": {"default_sigma": 1234.0}}
87 )
88
89 assert config["laser_intensity_fitting"]["default_sigma"] == 1234.0
90 assert load_bright_points_config()["laser_intensity_fitting"]["default_sigma"] is None