Back to report index

Step 6 filteringclusters db67bdd: AI3D-379 Align config module with fleet pattern

Miroslav Simko <ms@iolabs.ch> 2026-09-02T09:38:40+02:00

Commit #35 · 33 snippets

 README.md                                          |   6 +-
 .../__init__.py                                    |   2 +-
 .../_config.py                                     | 137 +++++++++++++--------
 .../cluster_finder.py                              |   6 +-
 .../clustering_gpu_engine.py                       |   7 +-
 .../clustering_gpu_pipeline_recursive.py           |   3 +-
 .../clustering_gpu_pipeline_stages.py              |   3 +-
 .../clustering_gpu_split.py                        |   7 +-
 .../clustering_gpu_types.py                        |   2 +
 tests/test_clustering_gpu_config.py                |  81 ------------
 tests/test_config.py                               | 126 +++++++++++++++++++
 11 files changed, 230 insertions(+), 150 deletions(-)
Importance #1: src/iolabs_point_cloud_filtering_clusters/_config.py @@ -1,40 +1,50 @@
1"""Packaged GPU Step 6 clustering config: pydantic model tree plus load helpers."""1"""Packaged GPU clustering config: pydantic model tree plus load helpers.
2
3The schema is `ClusterFinderGPUConfig` (a `config_loader.ConfigModel`), mirroring
4`clustering_gpu.default.json` key for key.
5
6Adding a config key means adding the field to the model and the same key to
7`clustering_gpu.default.json` — nothing else. Unknown keys are rejected.
8
9The entry points return plain JSON-typed dicts (`dict[str, Any]`), because callers
10pass `--set`-style overrides around as dicts and embed the normalized mapping in
11run manifests.
12"""
213
3from __future__ import annotations14from __future__ import annotations
415
5import logging16import logging
17from collections.abc import Mapping
6from pathlib import Path18from pathlib import Path
7from typing import Any19from typing import Any
820
9import pydantic21import pydantic
10from iolabs.common import config_loader22from iolabs.common import config_loader
1123
12logger = logging.getLogger(__name__)24logger = logging.getLogger(__name__)
1325
14GPU_CLUSTERING_ENGINE_NAME = "cuml_dbscan"26_PACKAGE_NAME = "iolabs_point_cloud_filtering_clusters"
15
16_PACKAGE = "iolabs_point_cloud_filtering_clusters"
17_DEFAULT_FILENAME = "clustering_gpu.default.json"27_DEFAULT_FILENAME = "clustering_gpu.default.json"
18_CONTEXT = "step6 gpu config"28_CONTEXT = "clustering GPU config"
1929
2030
21class ClusterFinderGPUConfigError(config_loader.ConfigError):31class ClusterFinderGPUConfigError(config_loader.ConfigError):
22 """Raised when Step 6 GPU config contains unsupported keys or values."""32 """Raised when clustering GPU config contains unsupported keys or values."""
2333
2434
25class ClusterFinderGPUInitialOutlierRemovalConfig(config_loader.ConfigModel):35class ClusterFinderGPUInitialOutlierRemovalConfig(config_loader.ConfigModel):
26 """Statistical outlier-removal section of the GPU clustering config."""36 """Statistical outlier-removal section of the GPU clustering config."""
2737
28 nb_neighbors: int = 2038 nb_neighbors: int = pydantic.Field(default=20, ge=1)
29 std_ratio: float = 2.7539 std_ratio: float = pydantic.Field(default=2.75, gt=0.0)
3040
3141
32class ClusterFinderGPUVoxelizationConfig(config_loader.ConfigModel):42class ClusterFinderGPUVoxelizationConfig(config_loader.ConfigModel):
33 """Voxel-downsample section of the GPU clustering config."""43 """Voxel-downsample section of the GPU clustering config."""
3444
35 enabled: bool = False45 enabled: bool = False
36 voxel_size: float = 0.0346 voxel_size: float = pydantic.Field(default=0.03, gt=0.0)
3747
3848
39class ClusterFinderGPUFeaturesConfig(config_loader.ConfigModel):49class ClusterFinderGPUFeaturesConfig(config_loader.ConfigModel):
40 """Feature-vector section of the GPU clustering config."""50 """Feature-vector section of the GPU clustering config."""
Importance #2: src/iolabs_point_cloud_filtering_clusters/_config.py @@ -77,33 +87,50 @@
7787
78class ClusterFinderGPUDensityFilteringConfig(config_loader.ConfigModel):88class ClusterFinderGPUDensityFilteringConfig(config_loader.ConfigModel):
79 """Per-cluster density cutoff section of the GPU clustering config."""89 """Per-cluster density cutoff section of the GPU clustering config."""
8090
81 density_cutoffs: list[float] = [0.0008, 0.0015, 0.003, 0.006]91 density_cutoffs: tuple[float, ...] = (0.0008, 0.0015, 0.003, 0.006)
82 cutoff_portions: list[float] = [0.8, 0.8, 0.2, 0.02]92 cutoff_portions: tuple[float, ...] = (0.8, 0.8, 0.2, 0.02)
93
94 @pydantic.model_validator(mode="after")
95 def _check_cutoffs(self) -> "ClusterFinderGPUDensityFilteringConfig":
96 """Reject mismatched cutoff lists or portions outside ``(0, 1]``."""
97 if len(self.density_cutoffs) != len(self.cutoff_portions):
98 raise ValueError("density_cutoffs and cutoff_portions must have the same length")
99 if any(portion <= 0.0 or portion > 1.0 for portion in self.cutoff_portions):
100 raise ValueError("every cutoff_portions entry must satisfy 0 < portion <= 1")
101 return self
83102
84103
85class ClusterFinderGPUIntensityFilteringConfig(config_loader.ConfigModel):104class ClusterFinderGPUIntensityFilteringConfig(config_loader.ConfigModel):
86 """Per-cluster intensity histogram section of the GPU clustering config."""105 """Per-cluster intensity histogram section of the GPU clustering config."""
87106
88 n_bins: int = 50107 n_bins: int = pydantic.Field(default=50, ge=1)
89 intensity_range: list[int] = [0, 255]108 intensity_range: tuple[int, int] = (0, 255)
90 peak_distance: int = 5109 peak_distance: int = pydantic.Field(default=5, ge=1)
91 peak_prominence: int = 10110 peak_prominence: int = pydantic.Field(default=10, ge=0)
92 sigma_estimate: float = 3.0111 sigma_estimate: float = pydantic.Field(default=3.0, gt=0.0)
93 sigma_estimate_peak_distance_fraction: float = 0.125112 sigma_estimate_peak_distance_fraction: float = pydantic.Field(default=0.125, gt=0.0, le=1.0)
94 n_sigma_intensity_cutoff: float = 5.0113 n_sigma_intensity_cutoff: float = pydantic.Field(default=5.0, gt=0.0)
95 min_sigma: float = 0.5114 min_sigma: float = pydantic.Field(default=0.5, gt=0.0)
115
116 @pydantic.model_validator(mode="after")
117 def _check_intensity_range(self) -> "ClusterFinderGPUIntensityFilteringConfig":
118 """Reject a histogram range whose lower bound is not below its upper bound."""
119 low, high = self.intensity_range
120 if low >= high:
121 raise ValueError("intensity_range must be [low, high] with low < high")
122 return self
96123
97124
98class ClusterFinderGPUPostprocessConfig(config_loader.ConfigModel):125class ClusterFinderGPUPostprocessConfig(config_loader.ConfigModel):
99 """Geometry-filter section of the GPU clustering config."""126 """Geometry-filter section of the GPU clustering config."""
100127
101 enable_geometry_filtering: bool = False128 enable_geometry_filtering: bool = False
102 min_points: int = 100129 min_points: int = pydantic.Field(default=100, ge=0)
103 min_length: float = 1.0130 min_length: float = pydantic.Field(default=1.0, ge=0.0)
104 max_width: float = 1.6131 max_width: float = pydantic.Field(default=1.6, gt=0.0)
105 max_height: float = 0.2132 max_height: float = pydantic.Field(default=0.2, gt=0.0)
106133
107134
108class ClusterFinderGPUFileNamingConfig(config_loader.ConfigModel):135class ClusterFinderGPUFileNamingConfig(config_loader.ConfigModel):
109 """Input/output file-name section of the GPU clustering config."""136 """Input/output file-name section of the GPU clustering config."""
Importance #3: src/iolabs_point_cloud_filtering_clusters/_config.py @@ -112,9 +139,9 @@
112 cluster_prefix: str = "run6_cluster_"139 cluster_prefix: str = "run6_cluster_"
113140
114141
115class ClusterFinderGPUConfig(config_loader.ConfigModel):142class ClusterFinderGPUConfig(config_loader.ConfigModel):
116 """Root GPU Step 6 clustering config; field names match the packaged JSON."""143 """Root GPU clustering config; field names match the packaged JSON."""
117144
118 device: str = "CUDA:0"145 device: str = "CUDA:0"
119 voxelization: ClusterFinderGPUVoxelizationConfig = ClusterFinderGPUVoxelizationConfig()146 voxelization: ClusterFinderGPUVoxelizationConfig = ClusterFinderGPUVoxelizationConfig()
120 features: ClusterFinderGPUFeaturesConfig = ClusterFinderGPUFeaturesConfig()147 features: ClusterFinderGPUFeaturesConfig = ClusterFinderGPUFeaturesConfig()
Importance #4: src/iolabs_point_cloud_filtering_clusters/_config.py @@ -152,41 +179,47 @@
152 logger.debug("Using defaults for null %s section(s): %s", _CONTEXT, null_sections)179 logger.debug("Using defaults for null %s section(s): %s", _CONTEXT, null_sections)
153 return data180 return data
154181
155182
156def normalize_cluster_finder_gpu_config(raw_config: dict[str, Any]) -> dict[str, Any]:183def _load_model(
184 *,
185 overrides: Mapping[str, Any] | None = None,
186 config_path: str | Path | None = None,
187) -> ClusterFinderGPUConfig:
188 """Load the packaged (or *config_path*) defaults with *overrides* deep-merged on top."""
189 if overrides:
190 logger.info("Config overrides applied: %s", ", ".join(sorted(overrides)))
191 if config_path is not None:
192 logger.info("Config file applied: %s", config_path)
193 return config_loader.load_config(
194 ClusterFinderGPUConfig,
195 package=_PACKAGE_NAME,
196 filename=_DEFAULT_FILENAME,
197 overrides=dict(overrides) if overrides is not None else None,
198 config_path=config_path,
199 context=_CONTEXT,
200 error_cls=ClusterFinderGPUConfigError,
201 )
202
203
204def normalize_cluster_finder_gpu_config(raw_config: Mapping[str, Any]) -> dict[str, Any]:
157 """Validate *raw_config* against the model tree and return a plain dict."""205 """Validate *raw_config* against the model tree and return a plain dict."""
158 return config_loader.validate_config(206 return config_loader.validate_config(
159 ClusterFinderGPUConfig,207 ClusterFinderGPUConfig,
160 raw_config,208 raw_config,
161 context=_CONTEXT,209 context=_CONTEXT,
162 error_cls=ClusterFinderGPUConfigError,210 error_cls=ClusterFinderGPUConfigError,
163 ).model_dump()211 ).model_dump(mode="json")
164212
165213
166def load_cluster_finder_gpu_config(config_path: str | Path | None = None) -> dict[str, Any]:214def load_cluster_finder_gpu_config(config_path: str | Path | None = None) -> dict[str, Any]:
167 """Load packaged (or *config_path*) defaults, validate, and return a plain dict."""215 """Load packaged (or *config_path*) defaults, validate, and return a plain dict."""
168 return config_loader.load_config(216 return _load_model(config_path=config_path).model_dump(mode="json")
169 ClusterFinderGPUConfig,
170 package=_PACKAGE,
171 filename=_DEFAULT_FILENAME,
172 config_path=config_path,
173 context=_CONTEXT,
174 error_cls=ClusterFinderGPUConfigError,
175 ).model_dump()
176217
177218
178def build_cluster_finder_gpu_config(219def build_cluster_finder_gpu_config(
179 *,220 *,
180 overrides: dict[str, Any] | None = None,221 overrides: Mapping[str, Any] | None = None,
181 config_path: str | Path | None = None,222 config_path: str | Path | None = None,
182) -> dict[str, Any]:223) -> dict[str, Any]:
183 """Load defaults, deep-merge *overrides*, validate, and return a plain dict."""224 """Load defaults, deep-merge *overrides*, validate, and return a plain dict."""
184 return config_loader.load_config(225 return _load_model(overrides=overrides, config_path=config_path).model_dump(mode="json")
185 ClusterFinderGPUConfig,
186 package=_PACKAGE,
187 filename=_DEFAULT_FILENAME,
188 overrides=overrides,
189 config_path=config_path,
190 context=_CONTEXT,
191 error_cls=ClusterFinderGPUConfigError,
192 ).model_dump()
Importance #5: src/iolabs_point_cloud_filtering_clusters/_config.py @@ -49,19 +59,19 @@
4959
50 enable_density_filtering: bool = True60 enable_density_filtering: bool = True
51 enable_intensity_trimming: bool = False61 enable_intensity_trimming: bool = False
52 enable_intensity_peak_split: bool = False62 enable_intensity_peak_split: bool = False
53 dbscan_eps: float = 1.263 dbscan_eps: float = pydantic.Field(default=1.2, gt=0.0)
54 dbscan_guard_threshold_points: int = 1500064 dbscan_guard_threshold_points: int = pydantic.Field(default=15000, ge=0)
55 min_cluster_size: int = 20065 min_cluster_size: int = pydantic.Field(default=200, ge=1)
56 dbscan_min_points: int = 5066 dbscan_min_points: int = pydantic.Field(default=50, ge=1)
57 include_noise_in_second_pass: bool = False67 include_noise_in_second_pass: bool = False
58 skip_gpu_second_pass: bool = False68 skip_gpu_second_pass: bool = False
59 max_mbytes_per_batch: int = 25669 max_mbytes_per_batch: int = pydantic.Field(default=256, ge=1)
60 calc_core_sample_indices: bool = False70 calc_core_sample_indices: bool = False
61 safe_raw_points: int = 100000071 safe_raw_points: int = pydantic.Field(default=1000000, ge=1)
62 safe_post_outlier_points: int = 90000072 safe_post_outlier_points: int = pydantic.Field(default=900000, ge=1)
63 safe_engine_rows: int = 40000073 safe_engine_rows: int = pydantic.Field(default=400000, ge=1)
6474
65 @pydantic.model_validator(mode="before")75 @pydantic.model_validator(mode="before")
66 @classmethod76 @classmethod
67 def _map_min_samples_alias(cls, value: Any) -> Any:77 def _map_min_samples_alias(cls, value: Any) -> Any:
Importance #6: src/iolabs_point_cloud_filtering_clusters/__init__.py @@ -3,13 +3,13 @@
3from .cluster_finder import ClusterFinder3from .cluster_finder import ClusterFinder
4from .cluster_metadata import ClusterInfo, get_cluster_info, load_npz_clusters_from_dir, planes_between4from .cluster_metadata import ClusterInfo, get_cluster_info, load_npz_clusters_from_dir, planes_between
5from ._config import (5from ._config import (
6 ClusterFinderGPUConfigError,6 ClusterFinderGPUConfigError,
7 GPU_CLUSTERING_ENGINE_NAME,
8 build_cluster_finder_gpu_config,7 build_cluster_finder_gpu_config,
9 load_cluster_finder_gpu_config,8 load_cluster_finder_gpu_config,
10 normalize_cluster_finder_gpu_config,9 normalize_cluster_finder_gpu_config,
11)10)
11from .clustering_gpu_types import GPU_CLUSTERING_ENGINE_NAME
12from .clustering_gpu_pipeline_entrypoints import find_clusters_gpu12from .clustering_gpu_pipeline_entrypoints import find_clusters_gpu
13from . import (13from . import (
14 cluster_finder,14 cluster_finder,
15 cluster_metadata,15 cluster_metadata,
Importance #7: src/iolabs_point_cloud_filtering_clusters/cluster_finder.py @@ -11,12 +11,10 @@
11from iolabs.logstash import get_props_logger11from iolabs.logstash import get_props_logger
12from iolabs_geometry_geometry import geometry_tools12from iolabs_geometry_geometry import geometry_tools
1313
14from ._log_props import LOG_PROPS14from ._log_props import LOG_PROPS
15from ._config import (15from ._config import normalize_cluster_finder_gpu_config
16 GPU_CLUSTERING_ENGINE_NAME,16from .clustering_gpu_types import GPU_CLUSTERING_ENGINE_NAME
17 normalize_cluster_finder_gpu_config,
18)
19from .clustering_gpu_pipeline_entrypoints import find_clusters_gpu_for_segment17from .clustering_gpu_pipeline_entrypoints import find_clusters_gpu_for_segment
2018
2119
22class ClusterFinder:20class ClusterFinder:
Importance #8: src/iolabs_point_cloud_filtering_clusters/clustering_gpu_engine.py @@ -5,11 +5,14 @@
5import numpy as np5import numpy as np
6from iolabs.logstash import get_props_logger6from iolabs.logstash import get_props_logger
77
8from ._log_props import LOG_PROPS8from ._log_props import LOG_PROPS
9from ._config import GPU_CLUSTERING_ENGINE_NAME
10from .clustering_gpu_runtime import _release_gpu_memory9from .clustering_gpu_runtime import _release_gpu_memory
11from .clustering_gpu_types import EngineLabelResult, PreparedSegmentReplay10from .clustering_gpu_types import (
11 GPU_CLUSTERING_ENGINE_NAME,
12 EngineLabelResult,
13 PreparedSegmentReplay,
14)
1215
1316
14LOGGER = get_props_logger(__name__, LOG_PROPS)17LOGGER = get_props_logger(__name__, LOG_PROPS)
1518
Importance #9: src/iolabs_point_cloud_filtering_clusters/clustering_gpu_pipeline_recursive.py @@ -6,9 +6,8 @@
6import numpy as np6import numpy as np
7from iolabs.logstash import get_props_logger7from iolabs.logstash import get_props_logger
88
9from ._log_props import LOG_PROPS9from ._log_props import LOG_PROPS
10from ._config import GPU_CLUSTERING_ENGINE_NAME
11from .clustering_gpu_io import GPUClusterArtifact10from .clustering_gpu_io import GPUClusterArtifact
12from .clustering_gpu_metrics import GPUClusterRunMetrics11from .clustering_gpu_metrics import GPUClusterRunMetrics
13from .clustering_gpu_pipeline_stages import _run_prepared_segment_once12from .clustering_gpu_pipeline_stages import _run_prepared_segment_once
14from .clustering_gpu_runtime import _log_gpu_stage_snapshot, _release_gpu_memory13from .clustering_gpu_runtime import _log_gpu_stage_snapshot, _release_gpu_memory
Importance #10: src/iolabs_point_cloud_filtering_clusters/clustering_gpu_pipeline_recursive.py @@ -17,9 +16,9 @@
17 _is_gpu_memory_split_error,16 _is_gpu_memory_split_error,
18 _planned_split_plane_count,17 _planned_split_plane_count,
19 _subdivide_prepared_segment,18 _subdivide_prepared_segment,
20)19)
21from .clustering_gpu_types import PreparedSegmentReplay20from .clustering_gpu_types import GPU_CLUSTERING_ENGINE_NAME, PreparedSegmentReplay
2221
2322
24LOGGER = get_props_logger(__name__, LOG_PROPS)23LOGGER = get_props_logger(__name__, LOG_PROPS)
2524
Importance #11: src/iolabs_point_cloud_filtering_clusters/clustering_gpu_pipeline_stages.py @@ -8,9 +8,8 @@
88
9from iolabs.logstash import get_props_logger9from iolabs.logstash import get_props_logger
1010
11from ._log_props import LOG_PROPS11from ._log_props import LOG_PROPS
12from ._config import GPU_CLUSTERING_ENGINE_NAME
13from .clustering_gpu_engine import _label_points_with_engine12from .clustering_gpu_engine import _label_points_with_engine
14from .clustering_gpu_filters import (13from .clustering_gpu_filters import (
15 _apply_legacy_density_filter,14 _apply_legacy_density_filter,
16 _apply_legacy_intensity_peak_split,15 _apply_legacy_intensity_peak_split,
Importance #12: src/iolabs_point_cloud_filtering_clusters/clustering_gpu_pipeline_stages.py @@ -28,9 +27,9 @@
28 _count_cloud_points,27 _count_cloud_points,
29 _log_gpu_stage_snapshot,28 _log_gpu_stage_snapshot,
30 _safe_gpu_split_targets,29 _safe_gpu_split_targets,
31)30)
32from .clustering_gpu_types import PreparedSegmentReplay31from .clustering_gpu_types import GPU_CLUSTERING_ENGINE_NAME, PreparedSegmentReplay
3332
3433
35LOGGER = get_props_logger(__name__, LOG_PROPS)34LOGGER = get_props_logger(__name__, LOG_PROPS)
3635
Importance #13: src/iolabs_point_cloud_filtering_clusters/clustering_gpu_split.py @@ -5,15 +5,18 @@
55
6import numpy as np6import numpy as np
7from iolabs_geometry_geometry import geometry_tools7from iolabs_geometry_geometry import geometry_tools
88
9from ._config import GPU_CLUSTERING_ENGINE_NAME
10from .clustering_gpu_features import segment_frame_from_planes9from .clustering_gpu_features import segment_frame_from_planes
11from .clustering_gpu_io import MergedBrightPoints, load_bright_points_npz10from .clustering_gpu_io import MergedBrightPoints, load_bright_points_npz
12from .clustering_gpu_metrics import GPUClusterRunMetrics11from .clustering_gpu_metrics import GPUClusterRunMetrics
13from .clustering_gpu_prepare import _prepare_replay_from_bright_points12from .clustering_gpu_prepare import _prepare_replay_from_bright_points
14from .clustering_gpu_runtime import _safe_gpu_split_targets13from .clustering_gpu_runtime import _safe_gpu_split_targets
15from .clustering_gpu_types import PreparedSegmentReplay, RawSegmentChunk14from .clustering_gpu_types import (
15 GPU_CLUSTERING_ENGINE_NAME,
16 PreparedSegmentReplay,
17 RawSegmentChunk,
18)
1619
17def _project_points_along_axis(20def _project_points_along_axis(
18 points: np.ndarray,21 points: np.ndarray,
19 *,22 *,
Importance #14: src/iolabs_point_cloud_filtering_clusters/clustering_gpu_types.py @@ -5,8 +5,10 @@
5import numpy as np5import numpy as np
66
7from .clustering_gpu_io import MergedBrightPoints7from .clustering_gpu_io import MergedBrightPoints
88
9GPU_CLUSTERING_ENGINE_NAME = "cuml_dbscan"
10
9@dataclass11@dataclass
10class PreparedSegmentReplay:12class PreparedSegmentReplay:
11 input_points_before_outlier_removal: int13 input_points_before_outlier_removal: int
12 bright_points: MergedBrightPoints14 bright_points: MergedBrightPoints
Importance #15: tests/test_clustering_gpu_config.py @@ -1,81 +0,0 @@
1import json
2
3import pytest
4from iolabs.common import config_loader
5from iolabs_point_cloud_filtering_clusters import _config
6
7
8def test_load_cluster_finder_gpu_defaults() -> None:
9 config = _config.load_cluster_finder_gpu_config()
10
11 assert config["device"] == "CUDA:0"
12 assert config["voxelization"]["enabled"] is False
13 assert config["clustering"]["dbscan_min_points"] == 50
14 assert config["file_naming"]["cluster_prefix"] == "run6_cluster_"
15
16
17def test_unknown_step6_gpu_key_is_rejected() -> None:
18 with pytest.raises(_config.ClusterFinderGPUConfigError, match="random_seed"):
19 _config.build_cluster_finder_gpu_config(overrides={"random_seed": 1})
20
21
22def test_legacy_min_samples_alias_maps_to_dbscan_min_points() -> None:
23 config = _config.build_cluster_finder_gpu_config(
24 overrides={"clustering": {"min_samples": 12}}
25 )
26
27 assert config["clustering"]["dbscan_min_points"] == 12
28 assert "min_samples" not in config["clustering"]
29
30
31def test_cluster_finder_gpu_config_error_is_config_error() -> None:
32 assert issubclass(_config.ClusterFinderGPUConfigError, config_loader.ConfigError)
33 assert issubclass(_config.ClusterFinderGPUConfigError, ValueError)
34
35
36def test_defaults_round_trip_packaged_json() -> None:
37 packaged = json.loads(
38 config_loader.default_config_path(
39 "iolabs_point_cloud_filtering_clusters", "clustering_gpu.default.json"
40 ).read_text(encoding="utf-8")
41 )
42
43 assert _config.load_cluster_finder_gpu_config() == packaged
44
45
46def test_bad_value_type_is_rejected() -> None:
47 with pytest.raises(_config.ClusterFinderGPUConfigError, match="dbscan_eps"):
48 _config.build_cluster_finder_gpu_config(
49 overrides={"clustering": {"dbscan_eps": "not-a-number"}}
50 )
51
52
53def test_nested_unknown_key_names_its_section() -> None:
54 with pytest.raises(_config.ClusterFinderGPUConfigError, match="postprocess"):
55 _config.build_cluster_finder_gpu_config(overrides={"postprocess": {"bogus": 1}})
56
57
58def test_model_defaults_match_packaged_json() -> None:
59 """Model defaults must mirror the packaged JSON, so a partial config file agrees."""
60 packaged = json.loads(
61 config_loader.default_config_path(
62 "iolabs_point_cloud_filtering_clusters", "clustering_gpu.default.json"
63 ).read_text(encoding="utf-8")
64 )
65
66 assert _config.ClusterFinderGPUConfig().model_dump() == packaged
67
68
69def test_null_section_falls_back_to_section_defaults() -> None:
70 """An explicit ``null`` section means "use defaults", as before the pydantic move."""
71 config = _config.normalize_cluster_finder_gpu_config(
72 {"voxelization": None, "clustering": None}
73 )
74
75 assert config["voxelization"] == {"enabled": False, "voxel_size": 0.03}
76 assert config["clustering"]["dbscan_min_points"] == 50
77
78
79def test_non_mapping_section_is_still_rejected() -> None:
80 with pytest.raises(_config.ClusterFinderGPUConfigError, match="voxelization"):
81 _config.normalize_cluster_finder_gpu_config({"voxelization": 5})
0
Importance #16: tests/test_config.py @@ -0,0 +1,126 @@
1import json
2
3import pytest
4from iolabs.common import config_loader
5from iolabs_point_cloud_filtering_clusters import _config
6
7
8def _packaged_defaults() -> dict:
9 path = config_loader.default_config_path(
10 "iolabs_point_cloud_filtering_clusters", "clustering_gpu.default.json"
11 )
12 return json.loads(path.read_text(encoding="utf-8"))
13
14
15def test_model_defaults_match_packaged_json() -> None:
16 """Model defaults must mirror the packaged JSON, so a partial config file agrees."""
17 assert _config.ClusterFinderGPUConfig().model_dump(mode="json") == _packaged_defaults()
18
19
20def test_load_cluster_finder_gpu_config_returns_packaged_defaults() -> None:
21 config = _config.load_cluster_finder_gpu_config()
22
23 assert isinstance(config, dict)
24 assert config == _packaged_defaults()
25 assert config["device"] == "CUDA:0"
26 assert config["voxelization"]["enabled"] is False
27 assert config["clustering"]["dbscan_min_points"] == 50
28 assert config["file_naming"]["cluster_prefix"] == "run6_cluster_"
29
30
31def test_error_class_is_config_error() -> None:
32 assert issubclass(_config.ClusterFinderGPUConfigError, config_loader.ConfigError)
33 assert issubclass(_config.ClusterFinderGPUConfigError, ValueError)
34
35
36def test_unknown_top_level_key_is_rejected() -> None:
37 with pytest.raises(_config.ClusterFinderGPUConfigError, match="random_seed"):
38 _config.build_cluster_finder_gpu_config(overrides={"random_seed": 1})
39
40
41def test_unknown_nested_key_is_rejected() -> None:
42 with pytest.raises(_config.ClusterFinderGPUConfigError, match="postprocess"):
43 _config.build_cluster_finder_gpu_config(overrides={"postprocess": {"bogus": 1}})
44
45
46def test_overrides_deep_merge_onto_defaults() -> None:
47 config = _config.build_cluster_finder_gpu_config(
48 overrides={"clustering": {"dbscan_eps": 2.5}}
49 )
50
51 assert config["clustering"]["dbscan_eps"] == 2.5
52 assert config["clustering"]["dbscan_min_points"] == 50
53 assert config["file_naming"] == _packaged_defaults()["file_naming"]
54
55
56def test_set_override_coercion_and_rejection() -> None:
57 overrides = config_loader.parse_set_overrides(
58 ["clustering.safe_raw_points=1e3", "clustering.skip_gpu_second_pass=on"],
59 nested=True,
60 error_cls=_config.ClusterFinderGPUConfigError,
61 )
62 config = _config.build_cluster_finder_gpu_config(overrides=overrides)
63
64 assert config["clustering"]["safe_raw_points"] == 1000
65 assert config["clustering"]["skip_gpu_second_pass"] is True
66
67 bad = config_loader.parse_set_overrides(
68 ["clustering.skip_gpu_second_pass=flase"],
69 nested=True,
70 error_cls=_config.ClusterFinderGPUConfigError,
71 )
72 with pytest.raises(_config.ClusterFinderGPUConfigError, match="skip_gpu_second_pass"):
73 _config.build_cluster_finder_gpu_config(overrides=bad)
74
75
76def test_legacy_min_samples_alias_maps_to_dbscan_min_points() -> None:
77 config = _config.build_cluster_finder_gpu_config(
78 overrides={"clustering": {"min_samples": 12}}
79 )
80
81 assert config["clustering"]["dbscan_min_points"] == 12
82 assert "min_samples" not in config["clustering"]
83
84
85def test_bad_value_type_is_rejected() -> None:
86 with pytest.raises(_config.ClusterFinderGPUConfigError, match="dbscan_eps"):
87 _config.build_cluster_finder_gpu_config(
88 overrides={"clustering": {"dbscan_eps": "not-a-number"}}
89 )
90
91
92def test_out_of_range_value_is_rejected() -> None:
93 with pytest.raises(_config.ClusterFinderGPUConfigError, match="dbscan_eps"):
94 _config.build_cluster_finder_gpu_config(overrides={"clustering": {"dbscan_eps": 0.0}})
95
96
97def test_mismatched_density_cutoffs_are_rejected() -> None:
98 with pytest.raises(_config.ClusterFinderGPUConfigError, match="cutoff_portions"):
99 _config.build_cluster_finder_gpu_config(
100 overrides={"cluster_density_filtering": {"cutoff_portions": [0.5]}}
101 )
102
103
104def test_null_section_falls_back_to_section_defaults() -> None:
105 """An explicit ``null`` section means "use defaults", as before the pydantic move."""
106 config = _config.normalize_cluster_finder_gpu_config(
107 {"voxelization": None, "clustering": None}
108 )
109
110 assert config["voxelization"] == {"enabled": False, "voxel_size": 0.03}
111 assert config["clustering"]["dbscan_min_points"] == 50
112
113
114def test_non_mapping_section_is_still_rejected() -> None:
115 with pytest.raises(_config.ClusterFinderGPUConfigError, match="voxelization"):
116 _config.normalize_cluster_finder_gpu_config({"voxelization": 5})
117
118
119def test_config_path_replaces_packaged_defaults(tmp_path) -> None:
120 path = tmp_path / "custom.json"
121 path.write_text(json.dumps({"device": "CPU:0"}), encoding="utf-8")
122
123 config = _config.load_cluster_finder_gpu_config(path)
124
125 assert config["device"] == "CPU:0"
126 assert config["clustering"]["dbscan_min_points"] == 50
0
Importance #17: README.md @@ -31,13 +31,11 @@
31Runs the GPU Step 6 clustering pipeline over Step 5 bright-filtered points, emits per-cluster NPZ artifacts, and provides cluster metadata helpers used by the modelling-lines package.31Runs the GPU Step 6 clustering pipeline over Step 5 bright-filtered points, emits per-cluster NPZ artifacts, and provides cluster metadata helpers used by the modelling-lines package.
3232
33## Configuration33## Configuration
3434
35GPU Step 6 defaults live in `src/iolabs_point_cloud_filtering_clusters/clustering_gpu.default.json` and are mirrored by the pydantic model tree in `_config.py` (`ClusterFinderGPUConfig`, nested sections as nested models).35Defaults live in `src/iolabs_point_cloud_filtering_clusters/clustering_gpu.default.json`. The schema is `ClusterFinderGPUConfig` in `_config.py` (a `config_loader.ConfigModel`); nested JSON sections are nested models and unknown keys are rejected. **To add a config key: add the field (with its type, default and any `Field` range) to the model and the same key with the same default to the JSON — nothing else.** `load_cluster_finder_gpu_config`, `build_cluster_finder_gpu_config` and `normalize_cluster_finder_gpu_config` return a plain `dict`. Runtime overrides come from repeatable `--set KEY=VALUE`, never repo-local JSON.
3636
37To add a config key: add a field to the matching `config_loader.ConfigModel` and the same key to the packaged JSON default. Nothing else. Unknown keys are rejected; overrides deep-merge onto the packaged defaults.37To add a whole section: declare a new `ClusterFinderGPU<Section>Config` model, add it as a field on `ClusterFinderGPUConfig` with a default instance, and mirror the section in the packaged JSON. A section given as `null` falls back to that section's defaults; a section of any non-mapping type is rejected.
38
39To add a whole section: declare a new `config_loader.ConfigModel` subclass, add it as a field on `ClusterFinderGPUConfig` with a default instance, and mirror the section in the packaged JSON. A section given as `null` falls back to that section's defaults; a section of any non-mapping type is rejected.
4038
41## Develop locally (Nexus)39## Develop locally (Nexus)
4240
43Internal `iolabs-*` dependencies resolve through the private Nexus index declared in `pyproject.toml`. Export Nexus credentials before any `uv` command that touches private deps — e.g. by sourcing `../3dai.lanefinder/scripts/nexus_credentials.sh` from your shell rc — then:41Internal `iolabs-*` dependencies resolve through the private Nexus index declared in `pyproject.toml`. Export Nexus credentials before any `uv` command that touches private deps — e.g. by sourcing `../3dai.lanefinder/scripts/nexus_credentials.sh` from your shell rc — then:
Importance #18: src/iolabs_point_cloud_filtering_clusters/__init__.py @@ -3,13 +3,13 @@
3from .cluster_finder import ClusterFinder3from .cluster_finder import ClusterFinder
4from .cluster_metadata import ClusterInfo, get_cluster_info, load_npz_clusters_from_dir, planes_between4from .cluster_metadata import ClusterInfo, get_cluster_info, load_npz_clusters_from_dir, planes_between
5from ._config import (5from ._config import (
6 ClusterFinderGPUConfigError,6 ClusterFinderGPUConfigError,
7 GPU_CLUSTERING_ENGINE_NAME,
8 build_cluster_finder_gpu_config,7 build_cluster_finder_gpu_config,
9 load_cluster_finder_gpu_config,8 load_cluster_finder_gpu_config,
10 normalize_cluster_finder_gpu_config,9 normalize_cluster_finder_gpu_config,
11)10)
11from .clustering_gpu_types import GPU_CLUSTERING_ENGINE_NAME
12from .clustering_gpu_pipeline_entrypoints import find_clusters_gpu12from .clustering_gpu_pipeline_entrypoints import find_clusters_gpu
13from . import (13from . import (
14 cluster_finder,14 cluster_finder,
15 cluster_metadata,15 cluster_metadata,
Importance #19: src/iolabs_point_cloud_filtering_clusters/_config.py @@ -1,40 +1,50 @@
1"""Packaged GPU Step 6 clustering config: pydantic model tree plus load helpers."""1"""Packaged GPU clustering config: pydantic model tree plus load helpers.
2
3The schema is `ClusterFinderGPUConfig` (a `config_loader.ConfigModel`), mirroring
4`clustering_gpu.default.json` key for key.
5
6Adding a config key means adding the field to the model and the same key to
7`clustering_gpu.default.json` — nothing else. Unknown keys are rejected.
8
9The entry points return plain JSON-typed dicts (`dict[str, Any]`), because callers
10pass `--set`-style overrides around as dicts and embed the normalized mapping in
11run manifests.
12"""
213
3from __future__ import annotations14from __future__ import annotations
415
5import logging16import logging
17from collections.abc import Mapping
6from pathlib import Path18from pathlib import Path
7from typing import Any19from typing import Any
820
9import pydantic21import pydantic
10from iolabs.common import config_loader22from iolabs.common import config_loader
1123
12logger = logging.getLogger(__name__)24logger = logging.getLogger(__name__)
1325
14GPU_CLUSTERING_ENGINE_NAME = "cuml_dbscan"26_PACKAGE_NAME = "iolabs_point_cloud_filtering_clusters"
15
16_PACKAGE = "iolabs_point_cloud_filtering_clusters"
17_DEFAULT_FILENAME = "clustering_gpu.default.json"27_DEFAULT_FILENAME = "clustering_gpu.default.json"
18_CONTEXT = "step6 gpu config"28_CONTEXT = "clustering GPU config"
1929
2030
21class ClusterFinderGPUConfigError(config_loader.ConfigError):31class ClusterFinderGPUConfigError(config_loader.ConfigError):
22 """Raised when Step 6 GPU config contains unsupported keys or values."""32 """Raised when clustering GPU config contains unsupported keys or values."""
2333
2434
25class ClusterFinderGPUInitialOutlierRemovalConfig(config_loader.ConfigModel):35class ClusterFinderGPUInitialOutlierRemovalConfig(config_loader.ConfigModel):
26 """Statistical outlier-removal section of the GPU clustering config."""36 """Statistical outlier-removal section of the GPU clustering config."""
2737
28 nb_neighbors: int = 2038 nb_neighbors: int = pydantic.Field(default=20, ge=1)
29 std_ratio: float = 2.7539 std_ratio: float = pydantic.Field(default=2.75, gt=0.0)
3040
3141
32class ClusterFinderGPUVoxelizationConfig(config_loader.ConfigModel):42class ClusterFinderGPUVoxelizationConfig(config_loader.ConfigModel):
33 """Voxel-downsample section of the GPU clustering config."""43 """Voxel-downsample section of the GPU clustering config."""
3444
35 enabled: bool = False45 enabled: bool = False
36 voxel_size: float = 0.0346 voxel_size: float = pydantic.Field(default=0.03, gt=0.0)
3747
3848
39class ClusterFinderGPUFeaturesConfig(config_loader.ConfigModel):49class ClusterFinderGPUFeaturesConfig(config_loader.ConfigModel):
40 """Feature-vector section of the GPU clustering config."""50 """Feature-vector section of the GPU clustering config."""
Importance #20: src/iolabs_point_cloud_filtering_clusters/_config.py @@ -49,19 +59,19 @@
4959
50 enable_density_filtering: bool = True60 enable_density_filtering: bool = True
51 enable_intensity_trimming: bool = False61 enable_intensity_trimming: bool = False
52 enable_intensity_peak_split: bool = False62 enable_intensity_peak_split: bool = False
53 dbscan_eps: float = 1.263 dbscan_eps: float = pydantic.Field(default=1.2, gt=0.0)
54 dbscan_guard_threshold_points: int = 1500064 dbscan_guard_threshold_points: int = pydantic.Field(default=15000, ge=0)
55 min_cluster_size: int = 20065 min_cluster_size: int = pydantic.Field(default=200, ge=1)
56 dbscan_min_points: int = 5066 dbscan_min_points: int = pydantic.Field(default=50, ge=1)
57 include_noise_in_second_pass: bool = False67 include_noise_in_second_pass: bool = False
58 skip_gpu_second_pass: bool = False68 skip_gpu_second_pass: bool = False
59 max_mbytes_per_batch: int = 25669 max_mbytes_per_batch: int = pydantic.Field(default=256, ge=1)
60 calc_core_sample_indices: bool = False70 calc_core_sample_indices: bool = False
61 safe_raw_points: int = 100000071 safe_raw_points: int = pydantic.Field(default=1000000, ge=1)
62 safe_post_outlier_points: int = 90000072 safe_post_outlier_points: int = pydantic.Field(default=900000, ge=1)
63 safe_engine_rows: int = 40000073 safe_engine_rows: int = pydantic.Field(default=400000, ge=1)
6474
65 @pydantic.model_validator(mode="before")75 @pydantic.model_validator(mode="before")
66 @classmethod76 @classmethod
67 def _map_min_samples_alias(cls, value: Any) -> Any:77 def _map_min_samples_alias(cls, value: Any) -> Any:
Importance #21: src/iolabs_point_cloud_filtering_clusters/_config.py @@ -77,33 +87,50 @@
7787
78class ClusterFinderGPUDensityFilteringConfig(config_loader.ConfigModel):88class ClusterFinderGPUDensityFilteringConfig(config_loader.ConfigModel):
79 """Per-cluster density cutoff section of the GPU clustering config."""89 """Per-cluster density cutoff section of the GPU clustering config."""
8090
81 density_cutoffs: list[float] = [0.0008, 0.0015, 0.003, 0.006]91 density_cutoffs: tuple[float, ...] = (0.0008, 0.0015, 0.003, 0.006)
82 cutoff_portions: list[float] = [0.8, 0.8, 0.2, 0.02]92 cutoff_portions: tuple[float, ...] = (0.8, 0.8, 0.2, 0.02)
93
94 @pydantic.model_validator(mode="after")
95 def _check_cutoffs(self) -> "ClusterFinderGPUDensityFilteringConfig":
96 """Reject mismatched cutoff lists or portions outside ``(0, 1]``."""
97 if len(self.density_cutoffs) != len(self.cutoff_portions):
98 raise ValueError("density_cutoffs and cutoff_portions must have the same length")
99 if any(portion <= 0.0 or portion > 1.0 for portion in self.cutoff_portions):
100 raise ValueError("every cutoff_portions entry must satisfy 0 < portion <= 1")
101 return self
83102
84103
85class ClusterFinderGPUIntensityFilteringConfig(config_loader.ConfigModel):104class ClusterFinderGPUIntensityFilteringConfig(config_loader.ConfigModel):
86 """Per-cluster intensity histogram section of the GPU clustering config."""105 """Per-cluster intensity histogram section of the GPU clustering config."""
87106
88 n_bins: int = 50107 n_bins: int = pydantic.Field(default=50, ge=1)
89 intensity_range: list[int] = [0, 255]108 intensity_range: tuple[int, int] = (0, 255)
90 peak_distance: int = 5109 peak_distance: int = pydantic.Field(default=5, ge=1)
91 peak_prominence: int = 10110 peak_prominence: int = pydantic.Field(default=10, ge=0)
92 sigma_estimate: float = 3.0111 sigma_estimate: float = pydantic.Field(default=3.0, gt=0.0)
93 sigma_estimate_peak_distance_fraction: float = 0.125112 sigma_estimate_peak_distance_fraction: float = pydantic.Field(default=0.125, gt=0.0, le=1.0)
94 n_sigma_intensity_cutoff: float = 5.0113 n_sigma_intensity_cutoff: float = pydantic.Field(default=5.0, gt=0.0)
95 min_sigma: float = 0.5114 min_sigma: float = pydantic.Field(default=0.5, gt=0.0)
115
116 @pydantic.model_validator(mode="after")
117 def _check_intensity_range(self) -> "ClusterFinderGPUIntensityFilteringConfig":
118 """Reject a histogram range whose lower bound is not below its upper bound."""
119 low, high = self.intensity_range
120 if low >= high:
121 raise ValueError("intensity_range must be [low, high] with low < high")
122 return self
96123
97124
98class ClusterFinderGPUPostprocessConfig(config_loader.ConfigModel):125class ClusterFinderGPUPostprocessConfig(config_loader.ConfigModel):
99 """Geometry-filter section of the GPU clustering config."""126 """Geometry-filter section of the GPU clustering config."""
100127
101 enable_geometry_filtering: bool = False128 enable_geometry_filtering: bool = False
102 min_points: int = 100129 min_points: int = pydantic.Field(default=100, ge=0)
103 min_length: float = 1.0130 min_length: float = pydantic.Field(default=1.0, ge=0.0)
104 max_width: float = 1.6131 max_width: float = pydantic.Field(default=1.6, gt=0.0)
105 max_height: float = 0.2132 max_height: float = pydantic.Field(default=0.2, gt=0.0)
106133
107134
108class ClusterFinderGPUFileNamingConfig(config_loader.ConfigModel):135class ClusterFinderGPUFileNamingConfig(config_loader.ConfigModel):
109 """Input/output file-name section of the GPU clustering config."""136 """Input/output file-name section of the GPU clustering config."""
Importance #22: src/iolabs_point_cloud_filtering_clusters/_config.py @@ -112,9 +139,9 @@
112 cluster_prefix: str = "run6_cluster_"139 cluster_prefix: str = "run6_cluster_"
113140
114141
115class ClusterFinderGPUConfig(config_loader.ConfigModel):142class ClusterFinderGPUConfig(config_loader.ConfigModel):
116 """Root GPU Step 6 clustering config; field names match the packaged JSON."""143 """Root GPU clustering config; field names match the packaged JSON."""
117144
118 device: str = "CUDA:0"145 device: str = "CUDA:0"
119 voxelization: ClusterFinderGPUVoxelizationConfig = ClusterFinderGPUVoxelizationConfig()146 voxelization: ClusterFinderGPUVoxelizationConfig = ClusterFinderGPUVoxelizationConfig()
120 features: ClusterFinderGPUFeaturesConfig = ClusterFinderGPUFeaturesConfig()147 features: ClusterFinderGPUFeaturesConfig = ClusterFinderGPUFeaturesConfig()
Importance #23: src/iolabs_point_cloud_filtering_clusters/_config.py @@ -152,41 +179,47 @@
152 logger.debug("Using defaults for null %s section(s): %s", _CONTEXT, null_sections)179 logger.debug("Using defaults for null %s section(s): %s", _CONTEXT, null_sections)
153 return data180 return data
154181
155182
156def normalize_cluster_finder_gpu_config(raw_config: dict[str, Any]) -> dict[str, Any]:183def _load_model(
184 *,
185 overrides: Mapping[str, Any] | None = None,
186 config_path: str | Path | None = None,
187) -> ClusterFinderGPUConfig:
188 """Load the packaged (or *config_path*) defaults with *overrides* deep-merged on top."""
189 if overrides:
190 logger.info("Config overrides applied: %s", ", ".join(sorted(overrides)))
191 if config_path is not None:
192 logger.info("Config file applied: %s", config_path)
193 return config_loader.load_config(
194 ClusterFinderGPUConfig,
195 package=_PACKAGE_NAME,
196 filename=_DEFAULT_FILENAME,
197 overrides=dict(overrides) if overrides is not None else None,
198 config_path=config_path,
199 context=_CONTEXT,
200 error_cls=ClusterFinderGPUConfigError,
201 )
202
203
204def normalize_cluster_finder_gpu_config(raw_config: Mapping[str, Any]) -> dict[str, Any]:
157 """Validate *raw_config* against the model tree and return a plain dict."""205 """Validate *raw_config* against the model tree and return a plain dict."""
158 return config_loader.validate_config(206 return config_loader.validate_config(
159 ClusterFinderGPUConfig,207 ClusterFinderGPUConfig,
160 raw_config,208 raw_config,
161 context=_CONTEXT,209 context=_CONTEXT,
162 error_cls=ClusterFinderGPUConfigError,210 error_cls=ClusterFinderGPUConfigError,
163 ).model_dump()211 ).model_dump(mode="json")
164212
165213
166def load_cluster_finder_gpu_config(config_path: str | Path | None = None) -> dict[str, Any]:214def load_cluster_finder_gpu_config(config_path: str | Path | None = None) -> dict[str, Any]:
167 """Load packaged (or *config_path*) defaults, validate, and return a plain dict."""215 """Load packaged (or *config_path*) defaults, validate, and return a plain dict."""
168 return config_loader.load_config(216 return _load_model(config_path=config_path).model_dump(mode="json")
169 ClusterFinderGPUConfig,
170 package=_PACKAGE,
171 filename=_DEFAULT_FILENAME,
172 config_path=config_path,
173 context=_CONTEXT,
174 error_cls=ClusterFinderGPUConfigError,
175 ).model_dump()
176217
177218
178def build_cluster_finder_gpu_config(219def build_cluster_finder_gpu_config(
179 *,220 *,
180 overrides: dict[str, Any] | None = None,221 overrides: Mapping[str, Any] | None = None,
181 config_path: str | Path | None = None,222 config_path: str | Path | None = None,
182) -> dict[str, Any]:223) -> dict[str, Any]:
183 """Load defaults, deep-merge *overrides*, validate, and return a plain dict."""224 """Load defaults, deep-merge *overrides*, validate, and return a plain dict."""
184 return config_loader.load_config(225 return _load_model(overrides=overrides, config_path=config_path).model_dump(mode="json")
185 ClusterFinderGPUConfig,
186 package=_PACKAGE,
187 filename=_DEFAULT_FILENAME,
188 overrides=overrides,
189 config_path=config_path,
190 context=_CONTEXT,
191 error_cls=ClusterFinderGPUConfigError,
192 ).model_dump()
Importance #24: src/iolabs_point_cloud_filtering_clusters/cluster_finder.py @@ -11,12 +11,10 @@
11from iolabs.logstash import get_props_logger11from iolabs.logstash import get_props_logger
12from iolabs_geometry_geometry import geometry_tools12from iolabs_geometry_geometry import geometry_tools
1313
14from ._log_props import LOG_PROPS14from ._log_props import LOG_PROPS
15from ._config import (15from ._config import normalize_cluster_finder_gpu_config
16 GPU_CLUSTERING_ENGINE_NAME,16from .clustering_gpu_types import GPU_CLUSTERING_ENGINE_NAME
17 normalize_cluster_finder_gpu_config,
18)
19from .clustering_gpu_pipeline_entrypoints import find_clusters_gpu_for_segment17from .clustering_gpu_pipeline_entrypoints import find_clusters_gpu_for_segment
2018
2119
22class ClusterFinder:20class ClusterFinder:
Importance #25: src/iolabs_point_cloud_filtering_clusters/clustering_gpu_engine.py @@ -5,11 +5,14 @@
5import numpy as np5import numpy as np
6from iolabs.logstash import get_props_logger6from iolabs.logstash import get_props_logger
77
8from ._log_props import LOG_PROPS8from ._log_props import LOG_PROPS
9from ._config import GPU_CLUSTERING_ENGINE_NAME
10from .clustering_gpu_runtime import _release_gpu_memory9from .clustering_gpu_runtime import _release_gpu_memory
11from .clustering_gpu_types import EngineLabelResult, PreparedSegmentReplay10from .clustering_gpu_types import (
11 GPU_CLUSTERING_ENGINE_NAME,
12 EngineLabelResult,
13 PreparedSegmentReplay,
14)
1215
1316
14LOGGER = get_props_logger(__name__, LOG_PROPS)17LOGGER = get_props_logger(__name__, LOG_PROPS)
1518
Importance #26: src/iolabs_point_cloud_filtering_clusters/clustering_gpu_pipeline_recursive.py @@ -6,9 +6,8 @@
6import numpy as np6import numpy as np
7from iolabs.logstash import get_props_logger7from iolabs.logstash import get_props_logger
88
9from ._log_props import LOG_PROPS9from ._log_props import LOG_PROPS
10from ._config import GPU_CLUSTERING_ENGINE_NAME
11from .clustering_gpu_io import GPUClusterArtifact10from .clustering_gpu_io import GPUClusterArtifact
12from .clustering_gpu_metrics import GPUClusterRunMetrics11from .clustering_gpu_metrics import GPUClusterRunMetrics
13from .clustering_gpu_pipeline_stages import _run_prepared_segment_once12from .clustering_gpu_pipeline_stages import _run_prepared_segment_once
14from .clustering_gpu_runtime import _log_gpu_stage_snapshot, _release_gpu_memory13from .clustering_gpu_runtime import _log_gpu_stage_snapshot, _release_gpu_memory
Importance #27: src/iolabs_point_cloud_filtering_clusters/clustering_gpu_pipeline_recursive.py @@ -17,9 +16,9 @@
17 _is_gpu_memory_split_error,16 _is_gpu_memory_split_error,
18 _planned_split_plane_count,17 _planned_split_plane_count,
19 _subdivide_prepared_segment,18 _subdivide_prepared_segment,
20)19)
21from .clustering_gpu_types import PreparedSegmentReplay20from .clustering_gpu_types import GPU_CLUSTERING_ENGINE_NAME, PreparedSegmentReplay
2221
2322
24LOGGER = get_props_logger(__name__, LOG_PROPS)23LOGGER = get_props_logger(__name__, LOG_PROPS)
2524
Importance #28: src/iolabs_point_cloud_filtering_clusters/clustering_gpu_pipeline_stages.py @@ -8,9 +8,8 @@
88
9from iolabs.logstash import get_props_logger9from iolabs.logstash import get_props_logger
1010
11from ._log_props import LOG_PROPS11from ._log_props import LOG_PROPS
12from ._config import GPU_CLUSTERING_ENGINE_NAME
13from .clustering_gpu_engine import _label_points_with_engine12from .clustering_gpu_engine import _label_points_with_engine
14from .clustering_gpu_filters import (13from .clustering_gpu_filters import (
15 _apply_legacy_density_filter,14 _apply_legacy_density_filter,
16 _apply_legacy_intensity_peak_split,15 _apply_legacy_intensity_peak_split,
Importance #29: src/iolabs_point_cloud_filtering_clusters/clustering_gpu_pipeline_stages.py @@ -28,9 +27,9 @@
28 _count_cloud_points,27 _count_cloud_points,
29 _log_gpu_stage_snapshot,28 _log_gpu_stage_snapshot,
30 _safe_gpu_split_targets,29 _safe_gpu_split_targets,
31)30)
32from .clustering_gpu_types import PreparedSegmentReplay31from .clustering_gpu_types import GPU_CLUSTERING_ENGINE_NAME, PreparedSegmentReplay
3332
3433
35LOGGER = get_props_logger(__name__, LOG_PROPS)34LOGGER = get_props_logger(__name__, LOG_PROPS)
3635
Importance #30: src/iolabs_point_cloud_filtering_clusters/clustering_gpu_split.py @@ -5,15 +5,18 @@
55
6import numpy as np6import numpy as np
7from iolabs_geometry_geometry import geometry_tools7from iolabs_geometry_geometry import geometry_tools
88
9from ._config import GPU_CLUSTERING_ENGINE_NAME
10from .clustering_gpu_features import segment_frame_from_planes9from .clustering_gpu_features import segment_frame_from_planes
11from .clustering_gpu_io import MergedBrightPoints, load_bright_points_npz10from .clustering_gpu_io import MergedBrightPoints, load_bright_points_npz
12from .clustering_gpu_metrics import GPUClusterRunMetrics11from .clustering_gpu_metrics import GPUClusterRunMetrics
13from .clustering_gpu_prepare import _prepare_replay_from_bright_points12from .clustering_gpu_prepare import _prepare_replay_from_bright_points
14from .clustering_gpu_runtime import _safe_gpu_split_targets13from .clustering_gpu_runtime import _safe_gpu_split_targets
15from .clustering_gpu_types import PreparedSegmentReplay, RawSegmentChunk14from .clustering_gpu_types import (
15 GPU_CLUSTERING_ENGINE_NAME,
16 PreparedSegmentReplay,
17 RawSegmentChunk,
18)
1619
17def _project_points_along_axis(20def _project_points_along_axis(
18 points: np.ndarray,21 points: np.ndarray,
19 *,22 *,
Importance #31: src/iolabs_point_cloud_filtering_clusters/clustering_gpu_types.py @@ -5,8 +5,10 @@
5import numpy as np5import numpy as np
66
7from .clustering_gpu_io import MergedBrightPoints7from .clustering_gpu_io import MergedBrightPoints
88
9GPU_CLUSTERING_ENGINE_NAME = "cuml_dbscan"
10
9@dataclass11@dataclass
10class PreparedSegmentReplay:12class PreparedSegmentReplay:
11 input_points_before_outlier_removal: int13 input_points_before_outlier_removal: int
12 bright_points: MergedBrightPoints14 bright_points: MergedBrightPoints
Importance #32: tests/test_clustering_gpu_config.py @@ -1,81 +0,0 @@
1import json
2
3import pytest
4from iolabs.common import config_loader
5from iolabs_point_cloud_filtering_clusters import _config
6
7
8def test_load_cluster_finder_gpu_defaults() -> None:
9 config = _config.load_cluster_finder_gpu_config()
10
11 assert config["device"] == "CUDA:0"
12 assert config["voxelization"]["enabled"] is False
13 assert config["clustering"]["dbscan_min_points"] == 50
14 assert config["file_naming"]["cluster_prefix"] == "run6_cluster_"
15
16
17def test_unknown_step6_gpu_key_is_rejected() -> None:
18 with pytest.raises(_config.ClusterFinderGPUConfigError, match="random_seed"):
19 _config.build_cluster_finder_gpu_config(overrides={"random_seed": 1})
20
21
22def test_legacy_min_samples_alias_maps_to_dbscan_min_points() -> None:
23 config = _config.build_cluster_finder_gpu_config(
24 overrides={"clustering": {"min_samples": 12}}
25 )
26
27 assert config["clustering"]["dbscan_min_points"] == 12
28 assert "min_samples" not in config["clustering"]
29
30
31def test_cluster_finder_gpu_config_error_is_config_error() -> None:
32 assert issubclass(_config.ClusterFinderGPUConfigError, config_loader.ConfigError)
33 assert issubclass(_config.ClusterFinderGPUConfigError, ValueError)
34
35
36def test_defaults_round_trip_packaged_json() -> None:
37 packaged = json.loads(
38 config_loader.default_config_path(
39 "iolabs_point_cloud_filtering_clusters", "clustering_gpu.default.json"
40 ).read_text(encoding="utf-8")
41 )
42
43 assert _config.load_cluster_finder_gpu_config() == packaged
44
45
46def test_bad_value_type_is_rejected() -> None:
47 with pytest.raises(_config.ClusterFinderGPUConfigError, match="dbscan_eps"):
48 _config.build_cluster_finder_gpu_config(
49 overrides={"clustering": {"dbscan_eps": "not-a-number"}}
50 )
51
52
53def test_nested_unknown_key_names_its_section() -> None:
54 with pytest.raises(_config.ClusterFinderGPUConfigError, match="postprocess"):
55 _config.build_cluster_finder_gpu_config(overrides={"postprocess": {"bogus": 1}})
56
57
58def test_model_defaults_match_packaged_json() -> None:
59 """Model defaults must mirror the packaged JSON, so a partial config file agrees."""
60 packaged = json.loads(
61 config_loader.default_config_path(
62 "iolabs_point_cloud_filtering_clusters", "clustering_gpu.default.json"
63 ).read_text(encoding="utf-8")
64 )
65
66 assert _config.ClusterFinderGPUConfig().model_dump() == packaged
67
68
69def test_null_section_falls_back_to_section_defaults() -> None:
70 """An explicit ``null`` section means "use defaults", as before the pydantic move."""
71 config = _config.normalize_cluster_finder_gpu_config(
72 {"voxelization": None, "clustering": None}
73 )
74
75 assert config["voxelization"] == {"enabled": False, "voxel_size": 0.03}
76 assert config["clustering"]["dbscan_min_points"] == 50
77
78
79def test_non_mapping_section_is_still_rejected() -> None:
80 with pytest.raises(_config.ClusterFinderGPUConfigError, match="voxelization"):
81 _config.normalize_cluster_finder_gpu_config({"voxelization": 5})
0
Importance #33: tests/test_config.py @@ -0,0 +1,126 @@
1import json
2
3import pytest
4from iolabs.common import config_loader
5from iolabs_point_cloud_filtering_clusters import _config
6
7
8def _packaged_defaults() -> dict:
9 path = config_loader.default_config_path(
10 "iolabs_point_cloud_filtering_clusters", "clustering_gpu.default.json"
11 )
12 return json.loads(path.read_text(encoding="utf-8"))
13
14
15def test_model_defaults_match_packaged_json() -> None:
16 """Model defaults must mirror the packaged JSON, so a partial config file agrees."""
17 assert _config.ClusterFinderGPUConfig().model_dump(mode="json") == _packaged_defaults()
18
19
20def test_load_cluster_finder_gpu_config_returns_packaged_defaults() -> None:
21 config = _config.load_cluster_finder_gpu_config()
22
23 assert isinstance(config, dict)
24 assert config == _packaged_defaults()
25 assert config["device"] == "CUDA:0"
26 assert config["voxelization"]["enabled"] is False
27 assert config["clustering"]["dbscan_min_points"] == 50
28 assert config["file_naming"]["cluster_prefix"] == "run6_cluster_"
29
30
31def test_error_class_is_config_error() -> None:
32 assert issubclass(_config.ClusterFinderGPUConfigError, config_loader.ConfigError)
33 assert issubclass(_config.ClusterFinderGPUConfigError, ValueError)
34
35
36def test_unknown_top_level_key_is_rejected() -> None:
37 with pytest.raises(_config.ClusterFinderGPUConfigError, match="random_seed"):
38 _config.build_cluster_finder_gpu_config(overrides={"random_seed": 1})
39
40
41def test_unknown_nested_key_is_rejected() -> None:
42 with pytest.raises(_config.ClusterFinderGPUConfigError, match="postprocess"):
43 _config.build_cluster_finder_gpu_config(overrides={"postprocess": {"bogus": 1}})
44
45
46def test_overrides_deep_merge_onto_defaults() -> None:
47 config = _config.build_cluster_finder_gpu_config(
48 overrides={"clustering": {"dbscan_eps": 2.5}}
49 )
50
51 assert config["clustering"]["dbscan_eps"] == 2.5
52 assert config["clustering"]["dbscan_min_points"] == 50
53 assert config["file_naming"] == _packaged_defaults()["file_naming"]
54
55
56def test_set_override_coercion_and_rejection() -> None:
57 overrides = config_loader.parse_set_overrides(
58 ["clustering.safe_raw_points=1e3", "clustering.skip_gpu_second_pass=on"],
59 nested=True,
60 error_cls=_config.ClusterFinderGPUConfigError,
61 )
62 config = _config.build_cluster_finder_gpu_config(overrides=overrides)
63
64 assert config["clustering"]["safe_raw_points"] == 1000
65 assert config["clustering"]["skip_gpu_second_pass"] is True
66
67 bad = config_loader.parse_set_overrides(
68 ["clustering.skip_gpu_second_pass=flase"],
69 nested=True,
70 error_cls=_config.ClusterFinderGPUConfigError,
71 )
72 with pytest.raises(_config.ClusterFinderGPUConfigError, match="skip_gpu_second_pass"):
73 _config.build_cluster_finder_gpu_config(overrides=bad)
74
75
76def test_legacy_min_samples_alias_maps_to_dbscan_min_points() -> None:
77 config = _config.build_cluster_finder_gpu_config(
78 overrides={"clustering": {"min_samples": 12}}
79 )
80
81 assert config["clustering"]["dbscan_min_points"] == 12
82 assert "min_samples" not in config["clustering"]
83
84
85def test_bad_value_type_is_rejected() -> None:
86 with pytest.raises(_config.ClusterFinderGPUConfigError, match="dbscan_eps"):
87 _config.build_cluster_finder_gpu_config(
88 overrides={"clustering": {"dbscan_eps": "not-a-number"}}
89 )
90
91
92def test_out_of_range_value_is_rejected() -> None:
93 with pytest.raises(_config.ClusterFinderGPUConfigError, match="dbscan_eps"):
94 _config.build_cluster_finder_gpu_config(overrides={"clustering": {"dbscan_eps": 0.0}})
95
96
97def test_mismatched_density_cutoffs_are_rejected() -> None:
98 with pytest.raises(_config.ClusterFinderGPUConfigError, match="cutoff_portions"):
99 _config.build_cluster_finder_gpu_config(
100 overrides={"cluster_density_filtering": {"cutoff_portions": [0.5]}}
101 )
102
103
104def test_null_section_falls_back_to_section_defaults() -> None:
105 """An explicit ``null`` section means "use defaults", as before the pydantic move."""
106 config = _config.normalize_cluster_finder_gpu_config(
107 {"voxelization": None, "clustering": None}
108 )
109
110 assert config["voxelization"] == {"enabled": False, "voxel_size": 0.03}
111 assert config["clustering"]["dbscan_min_points"] == 50
112
113
114def test_non_mapping_section_is_still_rejected() -> None:
115 with pytest.raises(_config.ClusterFinderGPUConfigError, match="voxelization"):
116 _config.normalize_cluster_finder_gpu_config({"voxelization": 5})
117
118
119def test_config_path_replaces_packaged_defaults(tmp_path) -> None:
120 path = tmp_path / "custom.json"
121 path.write_text(json.dumps({"device": "CPU:0"}), encoding="utf-8")
122
123 config = _config.load_cluster_finder_gpu_config(path)
124
125 assert config["device"] == "CPU:0"
126 assert config["clustering"]["dbscan_min_points"] == 50
0