Miroslav Simko <ms@iolabs.ch> 2026-09-02T08:34:34+02:00
Commit #33 · 11 snippets
README.md | 8 +- pyproject.toml | 5 +- .../_config.py | 365 ++++++++------------- tests/test_clustering_gpu_config.py | 66 ++-- 4 files changed, 197 insertions(+), 247 deletions(-)
| 1 | """Packaged GPU Step 6 clustering config: pydantic model tree plus load helpers.""" | ||
| 2 | |||
| 1 | from __future__ import annotations | 3 | from __future__ import annotations |
| 2 | 4 | ||
| 3 | import json | 5 | import logging |
| 4 | from importlib import resources | ||
| 5 | from pathlib import Path | 6 | from pathlib import Path |
| 6 | from typing import Any | 7 | from typing import Any |
| 7 | 8 | ||
| 9 | import pydantic | ||
| 10 | from iolabs.common import config_loader | ||
| 11 | |||
| 12 | logger = logging.getLogger(__name__) | ||
| 13 | |||
| 8 | GPU_CLUSTERING_ENGINE_NAME = "cuml_dbscan" | 14 | GPU_CLUSTERING_ENGINE_NAME = "cuml_dbscan" |
| 9 | 15 | ||
| 10 | ALLOWED_CLUSTER_FINDER_GPU_CONFIG_KEYS = frozenset( | 16 | _PACKAGE = "iolabs_point_cloud_filtering_clusters" |
| 11 | { | 17 | _DEFAULT_FILENAME = "clustering_gpu.default.json" |
| 12 | "device", | 18 | _CONTEXT = "step6 gpu config" |
| 13 | "initial_outlier_removal", | ||
| 14 | "voxelization", | ||
| 15 | "features", | ||
| 16 | "clustering", | ||
| 17 | "cluster_density_filtering", | ||
| 18 | "cluster_intensity_filtering", | ||
| 19 | "postprocess", | ||
| 20 | "file_naming", | ||
| 21 | } | ||
| 22 | ) | ||
| 23 | |||
| 24 | ALLOWED_INITIAL_OUTLIER_REMOVAL_KEYS = frozenset({"nb_neighbors", "std_ratio"}) | ||
| 25 | ALLOWED_VOXELIZATION_KEYS = frozenset({"enabled", "voxel_size"}) | ||
| 26 | ALLOWED_FEATURE_KEYS = frozenset( | ||
| 27 | {"include_z_residual", "include_intensity", "normalize_intensity"} | ||
| 28 | ) | ||
| 29 | ALLOWED_CLUSTERING_KEYS = frozenset( | ||
| 30 | { | ||
| 31 | "enable_density_filtering", | ||
| 32 | "enable_intensity_trimming", | ||
| 33 | "enable_intensity_peak_split", | ||
| 34 | "dbscan_eps", | ||
| 35 | "dbscan_guard_threshold_points", | ||
| 36 | "min_cluster_size", | ||
| 37 | "max_mbytes_per_batch", | ||
| 38 | "calc_core_sample_indices", | ||
| 39 | "safe_raw_points", | ||
| 40 | "safe_post_outlier_points", | ||
| 41 | "safe_engine_rows", | ||
| 42 | "dbscan_min_points", | ||
| 43 | "min_samples", | ||
| 44 | "include_noise_in_second_pass", | ||
| 45 | "skip_gpu_second_pass", | ||
| 46 | } | ||
| 47 | ) | ||
| 48 | ALLOWED_CLUSTER_DENSITY_FILTERING_KEYS = frozenset( | ||
| 49 | {"density_cutoffs", "cutoff_portions"} | ||
| 50 | ) | ||
| 51 | ALLOWED_CLUSTER_INTENSITY_FILTERING_KEYS = frozenset( | ||
| 52 | { | ||
| 53 | "n_bins", | ||
| 54 | "intensity_range", | ||
| 55 | "peak_distance", | ||
| 56 | "peak_prominence", | ||
| 57 | "sigma_estimate", | ||
| 58 | "sigma_estimate_peak_distance_fraction", | ||
| 59 | "n_sigma_intensity_cutoff", | ||
| 60 | "min_sigma", | ||
| 61 | } | ||
| 62 | ) | ||
| 63 | ALLOWED_POSTPROCESS_KEYS = frozenset( | ||
| 64 | {"enable_geometry_filtering", "min_points", "min_length", "max_width", "max_height"} | ||
| 65 | ) | ||
| 66 | ALLOWED_FILE_NAMING_KEYS = frozenset({"bright_filtered_suffix", "cluster_prefix"}) | ||
| 67 | |||
| 68 | |||
| 69 | class ClusterFinderGPUConfigError(ValueError): | ||
| 70 | """Raised when Step 6 GPU config contains unsupported keys.""" | ||
| 71 | |||
| 72 | |||
| 73 | def _default_gpu_config_path() -> Path: | ||
| 74 | if __package__ in {None, ""}: | ||
| 75 | return Path(__file__).resolve().with_name("clustering_gpu.default.json") | ||
| 76 | return Path(str(resources.files(__package__).joinpath("clustering_gpu.default.json"))) | ||
| 77 | |||
| 78 | |||
| 79 | def _deep_merge_dicts( | ||
| 80 | base: dict[str, Any], | ||
| 81 | overrides: dict[str, Any], | ||
| 82 | ) -> dict[str, Any]: | ||
| 83 | for key, value in overrides.items(): | ||
| 84 | if isinstance(value, dict) and isinstance(base.get(key), dict): | ||
| 85 | base[key] = _deep_merge_dicts(dict(base[key]), value) | ||
| 86 | else: | ||
| 87 | base[key] = value | ||
| 88 | return base | ||
| 89 | 19 | ||
| 90 | 20 | ||
| 91 | def _validate_allowed_keys( | 21 | class ClusterFinderGPUConfigError(config_loader.ConfigError): |
| 92 | config: dict[str, Any], | 22 | """Raised when Step 6 GPU config contains unsupported keys or values.""" |
| 93 | allowed_keys: frozenset[str], | ||
| 94 | *, | ||
| 95 | context: str, | ||
| 96 | ) -> None: | ||
| 97 | unknown_keys = sorted(set(config) - allowed_keys) | ||
| 98 | if not unknown_keys: | ||
| 99 | return | ||
| 100 | |||
| 101 | allowed = ", ".join(sorted(allowed_keys)) | ||
| 102 | raise ClusterFinderGPUConfigError( | ||
| 103 | f"Unknown {context} key(s): {', '.join(unknown_keys)}. Allowed keys: {allowed}" | ||
| 104 | ) | ||
| 105 | 23 | ||
| 106 | 24 | ||
| 107 | def _normalize_section( | 25 | class ClusterFinderGPUInitialOutlierRemovalConfig(config_loader.ConfigModel): |
| 108 | raw_section: Any, | 26 | """Statistical outlier-removal section of the GPU clustering config.""" |
| 109 | *, | ||
| 110 | allowed_keys: frozenset[str], | ||
| 111 | context: str, | ||
| 112 | ) -> dict[str, Any]: | ||
| 113 | if raw_section is None: | ||
| 114 | section: dict[str, Any] = {} | ||
| 115 | elif isinstance(raw_section, dict): | ||
| 116 | section = dict(raw_section) | ||
| 117 | else: | ||
| 118 | raise ClusterFinderGPUConfigError(f"{context} must be a mapping") | ||
| 119 | 27 | ||
| 120 | _validate_allowed_keys(section, allowed_keys, context=context) | 28 | nb_neighbors: int = 20 |
| 121 | return section | 29 | std_ratio: float = 2.75 |
| 122 | 30 | ||
| 123 | 31 | ||
| 124 | def normalize_cluster_finder_gpu_config(raw_config: dict[str, Any]) -> dict[str, Any]: | 32 | class ClusterFinderGPUVoxelizationConfig(config_loader.ConfigModel): |
| 125 | config = dict(raw_config) | 33 | """Voxel-downsample section of the GPU clustering config.""" |
| 126 | if "engine" in config: | ||
| 127 | raise ClusterFinderGPUConfigError( | ||
| 128 | "Unknown step6 gpu config key(s): engine. Allowed keys: " | ||
| 129 | + ", ".join(sorted(ALLOWED_CLUSTER_FINDER_GPU_CONFIG_KEYS)) | ||
| 130 | ) | ||
| 131 | _validate_allowed_keys( | ||
| 132 | config, | ||
| 133 | ALLOWED_CLUSTER_FINDER_GPU_CONFIG_KEYS, | ||
| 134 | context="step6 gpu config", | ||
| 135 | ) | ||
| 136 | 34 | ||
| 137 | config.setdefault("device", "CUDA:0") | 35 | enabled: bool = False |
| 36 | voxel_size: float = 0.03 | ||
| 138 | 37 | ||
| 139 | initial_outlier_removal = _normalize_section( | 38 | |
| 140 | config.get("initial_outlier_removal"), | 39 | class ClusterFinderGPUFeaturesConfig(config_loader.ConfigModel): |
| 141 | allowed_keys=ALLOWED_INITIAL_OUTLIER_REMOVAL_KEYS, | 40 | """Feature-vector section of the GPU clustering config.""" |
| 142 | context="step6 gpu initial_outlier_removal", | 41 | |
| 143 | ) | 42 | include_z_residual: bool = True |
| 144 | initial_outlier_removal.setdefault("nb_neighbors", 20) | 43 | include_intensity: bool = False |
| 145 | initial_outlier_removal.setdefault("std_ratio", 2.75) | 44 | normalize_intensity: bool = True |
| 146 | config["initial_outlier_removal"] = initial_outlier_removal | 45 | |
| 147 | 46 | ||
| 148 | voxelization = _normalize_section( | 47 | class ClusterFinderGPUClusteringConfig(config_loader.ConfigModel): |
| 149 | config.get("voxelization"), | 48 | """DBSCAN and pass-control section of the GPU clustering config.""" |
| 150 | allowed_keys=ALLOWED_VOXELIZATION_KEYS, | 49 | |
| 151 | context="step6 gpu voxelization", | 50 | enable_density_filtering: bool = True |
| 152 | ) | 51 | enable_intensity_trimming: bool = False |
| 153 | voxelization.setdefault("enabled", True) | 52 | enable_intensity_peak_split: bool = False |
| 154 | voxelization.setdefault("voxel_size", 0.03) | 53 | dbscan_eps: float = 1.2 |
| 155 | config["voxelization"] = voxelization | 54 | dbscan_guard_threshold_points: int = 15000 |
| 156 | 55 | min_cluster_size: int = 200 | |
| 157 | features = _normalize_section( | 56 | dbscan_min_points: int = 50 |
| 158 | config.get("features"), | 57 | include_noise_in_second_pass: bool = False |
| 159 | allowed_keys=ALLOWED_FEATURE_KEYS, | 58 | skip_gpu_second_pass: bool = False |
| 160 | context="step6 gpu features", | 59 | max_mbytes_per_batch: int = 256 |
| 161 | ) | 60 | calc_core_sample_indices: bool = False |
| 162 | features.setdefault("include_z_residual", True) | 61 | safe_raw_points: int = 1000000 |
| 163 | features.setdefault("include_intensity", False) | 62 | safe_post_outlier_points: int = 900000 |
| 164 | features.setdefault("normalize_intensity", True) | 63 | safe_engine_rows: int = 400000 |
| 165 | config["features"] = features | 64 | |
| 166 | 65 | @pydantic.model_validator(mode="before") | |
| 167 | clustering = _normalize_section( | 66 | @classmethod |
| 168 | config.get("clustering"), | 67 | def _map_min_samples_alias(cls, value: Any) -> Any: |
| 169 | allowed_keys=ALLOWED_CLUSTERING_KEYS, | 68 | """Map the legacy ``min_samples`` clustering key onto ``dbscan_min_points``.""" |
| 170 | context="step6 gpu clustering", | 69 | if not isinstance(value, dict) or "min_samples" not in value: |
| 171 | ) | 70 | return value |
| 172 | clustering.setdefault("enable_density_filtering", True) | 71 | data = dict(value) |
| 173 | clustering.setdefault("enable_intensity_trimming", False) | 72 | min_samples = data.pop("min_samples") |
| 174 | clustering.setdefault("enable_intensity_peak_split", False) | 73 | if min_samples is not None: |
| 175 | clustering.setdefault("dbscan_eps", 1.2) | 74 | data["dbscan_min_points"] = min_samples |
| 176 | clustering.setdefault("dbscan_guard_threshold_points", 15000) | 75 | return data |
| 177 | clustering.setdefault("min_cluster_size", 200) | 76 | |
| 178 | clustering.setdefault("max_mbytes_per_batch", 256) | 77 | |
| 179 | clustering.setdefault("calc_core_sample_indices", False) | 78 | class ClusterFinderGPUDensityFilteringConfig(config_loader.ConfigModel): |
| 180 | clustering.setdefault("safe_raw_points", 1000000) | 79 | """Per-cluster density cutoff section of the GPU clustering config.""" |
| 181 | clustering.setdefault("safe_post_outlier_points", 900000) | 80 | |
| 182 | clustering.setdefault("safe_engine_rows", 400000) | 81 | density_cutoffs: list[float] = [0.0008, 0.0015, 0.003, 0.006] |
| 183 | min_samples = clustering.pop("min_samples", None) | 82 | cutoff_portions: list[float] = [0.8, 0.8, 0.2, 0.02] |
| 184 | if min_samples is not None: | 83 | |
| 185 | clustering["dbscan_min_points"] = int(min_samples) | 84 | |
| 186 | clustering.setdefault("dbscan_min_points", 50) | 85 | class ClusterFinderGPUIntensityFilteringConfig(config_loader.ConfigModel): |
| 187 | clustering.setdefault("include_noise_in_second_pass", False) | 86 | """Per-cluster intensity histogram section of the GPU clustering config.""" |
| 188 | clustering.setdefault("skip_gpu_second_pass", False) | 87 | |
| 189 | config["clustering"] = clustering | 88 | n_bins: int = 50 |
| 190 | 89 | intensity_range: list[int] = [0, 255] | |
| 191 | cluster_density_filtering = _normalize_section( | 90 | peak_distance: int = 5 |
| 192 | config.get("cluster_density_filtering"), | 91 | peak_prominence: int = 10 |
| 193 | allowed_keys=ALLOWED_CLUSTER_DENSITY_FILTERING_KEYS, | 92 | sigma_estimate: float = 3.0 |
| 194 | context="step6 gpu cluster_density_filtering", | 93 | sigma_estimate_peak_distance_fraction: float = 0.125 |
| 195 | ) | 94 | n_sigma_intensity_cutoff: float = 5.0 |
| 196 | cluster_density_filtering.setdefault("density_cutoffs", [0.0008, 0.0015, 0.003, 0.006]) | 95 | min_sigma: float = 0.5 |
| 197 | cluster_density_filtering.setdefault("cutoff_portions", [0.8, 0.8, 0.2, 0.02]) | 96 | |
| 198 | config["cluster_density_filtering"] = cluster_density_filtering | 97 | |
| 199 | 98 | class ClusterFinderGPUPostprocessConfig(config_loader.ConfigModel): | |
| 200 | cluster_intensity_filtering = _normalize_section( | 99 | """Geometry-filter section of the GPU clustering config.""" |
| 201 | config.get("cluster_intensity_filtering"), | 100 | |
| 202 | allowed_keys=ALLOWED_CLUSTER_INTENSITY_FILTERING_KEYS, | 101 | enable_geometry_filtering: bool = False |
| 203 | context="step6 gpu cluster_intensity_filtering", | 102 | min_points: int = 100 |
| 103 | min_length: float = 1.0 | ||
| 104 | max_width: float = 1.6 | ||
| 105 | max_height: float = 0.2 | ||
| 106 | |||
| 107 | |||
| 108 | class ClusterFinderGPUFileNamingConfig(config_loader.ConfigModel): | ||
| 109 | """Input/output file-name section of the GPU clustering config.""" | ||
| 110 | |||
| 111 | bright_filtered_suffix: str = "_run5_bright_filtered" | ||
| 112 | cluster_prefix: str = "run6_cluster_" | ||
| 113 | |||
| 114 | |||
| 115 | class ClusterFinderGPUConfig(config_loader.ConfigModel): | ||
| 116 | """Root GPU Step 6 clustering config; field names match the packaged JSON.""" | ||
| 117 | |||
| 118 | device: str = "CUDA:0" | ||
| 119 | voxelization: ClusterFinderGPUVoxelizationConfig = ClusterFinderGPUVoxelizationConfig() | ||
| 120 | features: ClusterFinderGPUFeaturesConfig = ClusterFinderGPUFeaturesConfig() | ||
| 121 | initial_outlier_removal: ClusterFinderGPUInitialOutlierRemovalConfig = ( | ||
| 122 | ClusterFinderGPUInitialOutlierRemovalConfig() | ||
| 204 | ) | 123 | ) |
| 205 | cluster_intensity_filtering.setdefault("n_bins", 50) | 124 | clustering: ClusterFinderGPUClusteringConfig = ClusterFinderGPUClusteringConfig() |
| 206 | cluster_intensity_filtering.setdefault("intensity_range", [0, 255]) | 125 | cluster_density_filtering: ClusterFinderGPUDensityFilteringConfig = ( |
| 207 | cluster_intensity_filtering.setdefault("peak_distance", 5) | 126 | ClusterFinderGPUDensityFilteringConfig() |
| 208 | cluster_intensity_filtering.setdefault("peak_prominence", 10) | ||
| 209 | cluster_intensity_filtering.setdefault("sigma_estimate", 3.0) | ||
| 210 | cluster_intensity_filtering.setdefault("sigma_estimate_peak_distance_fraction", 0.125) | ||
| 211 | cluster_intensity_filtering.setdefault("n_sigma_intensity_cutoff", 5.0) | ||
| 212 | cluster_intensity_filtering.setdefault("min_sigma", 0.5) | ||
| 213 | config["cluster_intensity_filtering"] = cluster_intensity_filtering | ||
| 214 | |||
| 215 | postprocess = _normalize_section( | ||
| 216 | config.get("postprocess"), | ||
| 217 | allowed_keys=ALLOWED_POSTPROCESS_KEYS, | ||
| 218 | context="step6 gpu postprocess", | ||
| 219 | ) | 127 | ) |
| 220 | postprocess.setdefault("enable_geometry_filtering", False) | 128 | cluster_intensity_filtering: ClusterFinderGPUIntensityFilteringConfig = ( |
| 221 | postprocess.setdefault("min_points", 100) | 129 | ClusterFinderGPUIntensityFilteringConfig() |
| 222 | postprocess.setdefault("min_length", 1.0) | ||
| 223 | postprocess.setdefault("max_width", 1.6) | ||
| 224 | postprocess.setdefault("max_height", 0.2) | ||
| 225 | config["postprocess"] = postprocess | ||
| 226 | |||
| 227 | file_naming = _normalize_section( | ||
| 228 | config.get("file_naming"), | ||
| 229 | allowed_keys=ALLOWED_FILE_NAMING_KEYS, | ||
| 230 | context="step6 gpu file_naming", | ||
| 231 | ) | 130 | ) |
| 232 | file_naming.setdefault("bright_filtered_suffix", "_run5_bright_filtered") | 131 | postprocess: ClusterFinderGPUPostprocessConfig = ClusterFinderGPUPostprocessConfig() |
| 233 | file_naming.setdefault("cluster_prefix", "run6_cluster_") | 132 | file_naming: ClusterFinderGPUFileNamingConfig = ClusterFinderGPUFileNamingConfig() |
| 234 | config["file_naming"] = file_naming | ||
| 235 | 133 | ||
| 236 | return config | 134 | |
| 135 | def normalize_cluster_finder_gpu_config(raw_config: dict[str, Any]) -> dict[str, Any]: | ||
| 136 | """Validate *raw_config* against the model tree and return a plain dict.""" | ||
| 137 | return config_loader.validate_config( | ||
| 138 | ClusterFinderGPUConfig, | ||
| 139 | raw_config, | ||
| 140 | context=_CONTEXT, | ||
| 141 | error_cls=ClusterFinderGPUConfigError, | ||
| 142 | ).model_dump() | ||
| 237 | 143 | ||
| 238 | 144 | ||
| 239 | def load_cluster_finder_gpu_config(config_path: str | Path | None = None) -> dict[str, Any]: | 145 | def load_cluster_finder_gpu_config(config_path: str | Path | None = None) -> dict[str, Any]: |
| 240 | resolved_path = Path(config_path) if config_path is not None else _default_gpu_config_path() | 146 | """Load packaged (or *config_path*) defaults, validate, and return a plain dict.""" |
| 241 | with resolved_path.open("r", encoding="utf-8") as handle: | 147 | return config_loader.load_config( |
| 242 | raw_config: dict[str, Any] = json.load(handle) | 148 | ClusterFinderGPUConfig, |
| 243 | return normalize_cluster_finder_gpu_config(raw_config) | 149 | package=_PACKAGE, |
| 150 | filename=_DEFAULT_FILENAME, | ||
| 151 | config_path=config_path, | ||
| 152 | context=_CONTEXT, | ||
| 153 | error_cls=ClusterFinderGPUConfigError, | ||
| 154 | ).model_dump() | ||
| 244 | 155 | ||
| 245 | 156 | ||
| 246 | def build_cluster_finder_gpu_config( | 157 | def build_cluster_finder_gpu_config( |
| 247 | *, | 158 | *, |
| 248 | overrides: dict[str, Any] | None = None, | 159 | overrides: dict[str, Any] | None = None, |
| 249 | config_path: str | Path | None = None, | 160 | config_path: str | Path | None = None, |
| 250 | ) -> dict[str, Any]: | 161 | ) -> dict[str, Any]: |
| 251 | config = load_cluster_finder_gpu_config(config_path) | 162 | """Load defaults, deep-merge *overrides*, validate, and return a plain dict.""" |
| 252 | if overrides: | 163 | return config_loader.load_config( |
| 253 | config = _deep_merge_dicts(config, dict(overrides)) | 164 | ClusterFinderGPUConfig, |
| 254 | return normalize_cluster_finder_gpu_config(config) | 165 | package=_PACKAGE, |
| 166 | filename=_DEFAULT_FILENAME, | ||
| 167 | overrides=overrides, | ||
| 168 | config_path=config_path, | ||
| 169 | context=_CONTEXT, | ||
| 170 | error_cls=ClusterFinderGPUConfigError, | ||
| 171 | ).model_dump() |
| 1 | import importlib.util | 1 | import json |
| 2 | from pathlib import Path | ||
| 3 | 2 | ||
| 4 | import pytest | 3 | import pytest |
| 5 | 4 | from iolabs.common import config_loader | |
| 6 | 5 | from iolabs_point_cloud_filtering_clusters import _config | |
| 7 | MODULE_PATH = ( | ||
| 8 | Path(__file__) | ||
| 9 | .resolve() | ||
| 10 | .parents[1] | ||
| 11 | / "src" | ||
| 12 | / "iolabs_point_cloud_filtering_clusters" | ||
| 13 | / "_config.py" | ||
| 14 | ) | ||
| 15 | |||
| 16 | SPEC = importlib.util.spec_from_file_location("step6_config", MODULE_PATH) | ||
| 17 | assert SPEC is not None and SPEC.loader is not None | ||
| 18 | MODULE = importlib.util.module_from_spec(SPEC) | ||
| 19 | SPEC.loader.exec_module(MODULE) | ||
| 20 | 6 | ||
| 21 | 7 | ||
| 22 | def test_load_cluster_finder_gpu_defaults() -> None: | 8 | def test_load_cluster_finder_gpu_defaults() -> None: |
| 23 | config = MODULE.load_cluster_finder_gpu_config() | 9 | config = _config.load_cluster_finder_gpu_config() |
| 24 | 10 | ||
| 25 | assert config["device"] == "CUDA:0" | 11 | assert config["device"] == "CUDA:0" |
| 26 | assert config["voxelization"]["enabled"] is False | 12 | assert config["voxelization"]["enabled"] is False |
| 27 | assert config["clustering"]["dbscan_min_points"] == 50 | 13 | assert config["clustering"]["dbscan_min_points"] == 50 |
| 28 | assert config["file_naming"]["cluster_prefix"] == "run6_cluster_" | 14 | assert config["file_naming"]["cluster_prefix"] == "run6_cluster_" |
| 29 | 15 | ||
| 30 | 16 | ||
| 31 | def test_unknown_step6_gpu_key_is_rejected() -> None: | 17 | def test_unknown_step6_gpu_key_is_rejected() -> None: |
| 32 | with pytest.raises(MODULE.ClusterFinderGPUConfigError, match="random_seed"): | 18 | with pytest.raises(_config.ClusterFinderGPUConfigError, match="random_seed"): |
| 33 | MODULE.build_cluster_finder_gpu_config(overrides={"random_seed": 1}) | 19 | _config.build_cluster_finder_gpu_config(overrides={"random_seed": 1}) |
| 34 | 20 | ||
| 35 | 21 | ||
| 36 | def test_legacy_min_samples_alias_maps_to_dbscan_min_points() -> None: | 22 | def test_legacy_min_samples_alias_maps_to_dbscan_min_points() -> None: |
| 37 | config = MODULE.build_cluster_finder_gpu_config( | 23 | config = _config.build_cluster_finder_gpu_config( |
| 38 | overrides={"clustering": {"min_samples": 12}} | 24 | overrides={"clustering": {"min_samples": 12}} |
| 39 | ) | 25 | ) |
| 40 | 26 | ||
| 41 | assert config["clustering"]["dbscan_min_points"] == 12 | 27 | assert config["clustering"]["dbscan_min_points"] == 12 |
| 42 | assert "min_samples" not in config["clustering"] | 28 | assert "min_samples" not in config["clustering"] |
| 29 | |||
| 30 | |||
| 31 | def test_cluster_finder_gpu_config_error_is_config_error() -> None: | ||
| 32 | assert issubclass(_config.ClusterFinderGPUConfigError, config_loader.ConfigError) | ||
| 33 | assert issubclass(_config.ClusterFinderGPUConfigError, ValueError) | ||
| 34 | |||
| 35 | |||
| 36 | def test_defaults_round_trip_packaged_json() -> None: | ||
| 37 | packaged = json.loads( | ||
| 38 | config_loader.default_config_path( | ||
| 39 | "iolabs_point_cloud_filtering_clusters", "clustering_gpu.default.json" | ||
| 40 | ).read_text(encoding="utf-8") | ||
| 41 | ) | ||
| 42 | |||
| 43 | assert _config.load_cluster_finder_gpu_config() == packaged | ||
| 44 | |||
| 45 | |||
| 46 | def test_bad_value_type_is_rejected() -> None: | ||
| 47 | with pytest.raises(_config.ClusterFinderGPUConfigError, match="dbscan_eps"): | ||
| 48 | _config.build_cluster_finder_gpu_config( | ||
| 49 | overrides={"clustering": {"dbscan_eps": "not-a-number"}} | ||
| 50 | ) | ||
| 51 | |||
| 52 | |||
| 53 | def test_nested_unknown_key_names_its_section() -> None: | ||
| 54 | with pytest.raises(_config.ClusterFinderGPUConfigError, match="postprocess"): | ||
| 55 | _config.build_cluster_finder_gpu_config(overrides={"postprocess": {"bogus": 1}}) | ||
| 56 | |||
| 57 | |||
| 58 | def test_model_defaults_match_packaged_json() -> None: | ||
| 59 | """Model defaults must mirror the packaged JSON, so a partial config file agrees.""" | ||
| 60 | packaged = json.loads( | ||
| 61 | config_loader.default_config_path( | ||
| 62 | "iolabs_point_cloud_filtering_clusters", "clustering_gpu.default.json" | ||
| 63 | ).read_text(encoding="utf-8") | ||
| 64 | ) | ||
| 65 | |||
| 66 | assert _config.ClusterFinderGPUConfig().model_dump() == packaged |
| 1 | [project] | 1 | [project] |
| 2 | name = "iolabs-point-cloud-filtering-clusters" | 2 | name = "iolabs-point-cloud-filtering-clusters" |
| 3 | version = "0.6.3" | 3 | version = "0.6.4" |
| 4 | description = "GPU Step 6 clustering plus cluster metadata helpers for lane modelling" | 4 | description = "GPU Step 6 clustering plus cluster metadata helpers for lane modelling" |
| 5 | requires-python = ">=3.11,<3.13" | 5 | requires-python = ">=3.11,<3.13" |
| 6 | dependencies = [ | 6 | dependencies = [ |
| 7 | "numpy>=1.20.0", | 7 | "numpy>=1.20.0", |
| 9 | "torch>=2.2.0", | 9 | "torch>=2.2.0", |
| 10 | "scipy>=1.7.0", | 10 | "scipy>=1.7.0", |
| 11 | "matplotlib>=3.4.0", | 11 | "matplotlib>=3.4.0", |
| 12 | "scikit-learn>=1.0.0", | 12 | "scikit-learn>=1.0.0", |
| 13 | "pydantic>=2.7", | ||
| 13 | "iolabs-logstash>=0.4.0", | 14 | "iolabs-logstash>=0.4.0", |
| 15 | "iolabs-common>=0.8.0", | ||
| 14 | "iolabs-geometry-geometry", | 16 | "iolabs-geometry-geometry", |
| 15 | "iolabs-geometry-visualization", | 17 | "iolabs-geometry-visualization", |
| 16 | "iolabs-point-cloud-filtering-intensity>=0.5.1", | 18 | "iolabs-point-cloud-filtering-intensity>=0.5.1", |
| 17 | ] | 19 | ] |
| 35 | authenticate = "always" | 37 | authenticate = "always" |
| 36 | 38 | ||
| 37 | [tool.uv.sources] | 39 | [tool.uv.sources] |
| 38 | iolabs-logstash = { index = "nexus" } | 40 | iolabs-logstash = { index = "nexus" } |
| 41 | iolabs-common = { index = "nexus" } | ||
| 39 | iolabs-geometry-geometry = { index = "nexus" } | 42 | iolabs-geometry-geometry = { index = "nexus" } |
| 40 | iolabs-geometry-visualization = { index = "nexus" } | 43 | iolabs-geometry-visualization = { index = "nexus" } |
| 41 | iolabs-point-cloud-filtering-intensity = { index = "nexus" } | 44 | iolabs-point-cloud-filtering-intensity = { index = "nexus" } |
| 42 | 45 |
| 22 | 22 | ||
| 23 | ## Requirements | 23 | ## Requirements |
| 24 | 24 | ||
| 25 | - Python ≥3.11, <3.13 | 25 | - Python ≥3.11, <3.13 |
| 26 | - numpy, open3d, torch, scipy, matplotlib, scikit-learn, iolabs-geometry-geometry, iolabs-geometry-visualization, iolabs-point-cloud-filtering-intensity | 26 | - numpy, open3d, torch, scipy, matplotlib, scikit-learn, pydantic, iolabs-common, iolabs-geometry-geometry, iolabs-geometry-visualization, iolabs-point-cloud-filtering-intensity |
| 27 | - optional GPU runtime: cupy-cuda12x, rmm-cu12, cuml-cu12 | 27 | - optional GPU runtime: cupy-cuda12x, rmm-cu12, cuml-cu12 |
| 28 | 28 | ||
| 29 | ## Usage | 29 | ## Usage |
| 30 | 30 | ||
| 31 | Runs the GPU Step 6 clustering pipeline over Step 5 bright-filtered points, emits per-cluster NPZ artifacts, and provides cluster metadata helpers used by the modelling-lines package. | 31 | Runs the GPU Step 6 clustering pipeline over Step 5 bright-filtered points, emits per-cluster NPZ artifacts, and provides cluster metadata helpers used by the modelling-lines package. |
| 32 | 32 | ||
| 33 | ## Configuration | ||
| 34 | |||
| 35 | GPU Step 6 defaults live in `src/iolabs_point_cloud_filtering_clusters/clustering_gpu.default.json` and are mirrored by the pydantic model tree in `_config.py` (`ClusterFinderGPUConfig`, nested sections as nested models). | ||
| 36 | |||
| 37 | To add a config key: add a field to the matching `config_loader.ConfigModel` and the same key to the packaged JSON default. Nothing else. Unknown keys are rejected; overrides deep-merge onto the packaged defaults. | ||
| 38 | |||
| 33 | ## Develop locally (Nexus) | 39 | ## Develop locally (Nexus) |
| 34 | 40 | ||
| 35 | Internal `iolabs-*` dependencies resolve through the private Nexus index declared in `pyproject.toml`. Export Nexus credentials before any `uv` command that touches private deps — e.g. by sourcing `../3dai.lanefinder/scripts/nexus_credentials.sh` from your shell rc — then: | 41 | Internal `iolabs-*` dependencies resolve through the private Nexus index declared in `pyproject.toml`. Export Nexus credentials before any `uv` command that touches private deps — e.g. by sourcing `../3dai.lanefinder/scripts/nexus_credentials.sh` from your shell rc — then: |
| 36 | 42 |
| 1 | [project] | 1 | [project] |
| 2 | name = "iolabs-point-cloud-filtering-clusters" | 2 | name = "iolabs-point-cloud-filtering-clusters" |
| 3 | version = "0.6.3" | 3 | version = "0.6.4" |
| 4 | description = "GPU Step 6 clustering plus cluster metadata helpers for lane modelling" | 4 | description = "GPU Step 6 clustering plus cluster metadata helpers for lane modelling" |
| 5 | requires-python = ">=3.11,<3.13" | 5 | requires-python = ">=3.11,<3.13" |
| 6 | dependencies = [ | 6 | dependencies = [ |
| 7 | "numpy>=1.20.0", | 7 | "numpy>=1.20.0", |
| 9 | "torch>=2.2.0", | 9 | "torch>=2.2.0", |
| 10 | "scipy>=1.7.0", | 10 | "scipy>=1.7.0", |
| 11 | "matplotlib>=3.4.0", | 11 | "matplotlib>=3.4.0", |
| 12 | "scikit-learn>=1.0.0", | 12 | "scikit-learn>=1.0.0", |
| 13 | "pydantic>=2.7", | ||
| 13 | "iolabs-logstash>=0.4.0", | 14 | "iolabs-logstash>=0.4.0", |
| 15 | "iolabs-common>=0.8.0", | ||
| 14 | "iolabs-geometry-geometry", | 16 | "iolabs-geometry-geometry", |
| 15 | "iolabs-geometry-visualization", | 17 | "iolabs-geometry-visualization", |
| 16 | "iolabs-point-cloud-filtering-intensity>=0.5.1", | 18 | "iolabs-point-cloud-filtering-intensity>=0.5.1", |
| 17 | ] | 19 | ] |
| 35 | authenticate = "always" | 37 | authenticate = "always" |
| 36 | 38 | ||
| 37 | [tool.uv.sources] | 39 | [tool.uv.sources] |
| 38 | iolabs-logstash = { index = "nexus" } | 40 | iolabs-logstash = { index = "nexus" } |
| 41 | iolabs-common = { index = "nexus" } | ||
| 39 | iolabs-geometry-geometry = { index = "nexus" } | 42 | iolabs-geometry-geometry = { index = "nexus" } |
| 40 | iolabs-geometry-visualization = { index = "nexus" } | 43 | iolabs-geometry-visualization = { index = "nexus" } |
| 41 | iolabs-point-cloud-filtering-intensity = { index = "nexus" } | 44 | iolabs-point-cloud-filtering-intensity = { index = "nexus" } |
| 42 | 45 |
| 1 | """Packaged GPU Step 6 clustering config: pydantic model tree plus load helpers.""" | ||
| 2 | |||
| 1 | from __future__ import annotations | 3 | from __future__ import annotations |
| 2 | 4 | ||
| 3 | import json | 5 | import logging |
| 4 | from importlib import resources | ||
| 5 | from pathlib import Path | 6 | from pathlib import Path |
| 6 | from typing import Any | 7 | from typing import Any |
| 7 | 8 | ||
| 9 | import pydantic | ||
| 10 | from iolabs.common import config_loader | ||
| 11 | |||
| 12 | logger = logging.getLogger(__name__) | ||
| 13 | |||
| 8 | GPU_CLUSTERING_ENGINE_NAME = "cuml_dbscan" | 14 | GPU_CLUSTERING_ENGINE_NAME = "cuml_dbscan" |
| 9 | 15 | ||
| 10 | ALLOWED_CLUSTER_FINDER_GPU_CONFIG_KEYS = frozenset( | 16 | _PACKAGE = "iolabs_point_cloud_filtering_clusters" |
| 11 | { | 17 | _DEFAULT_FILENAME = "clustering_gpu.default.json" |
| 12 | "device", | 18 | _CONTEXT = "step6 gpu config" |
| 13 | "initial_outlier_removal", | ||
| 14 | "voxelization", | ||
| 15 | "features", | ||
| 16 | "clustering", | ||
| 17 | "cluster_density_filtering", | ||
| 18 | "cluster_intensity_filtering", | ||
| 19 | "postprocess", | ||
| 20 | "file_naming", | ||
| 21 | } | ||
| 22 | ) | ||
| 23 | |||
| 24 | ALLOWED_INITIAL_OUTLIER_REMOVAL_KEYS = frozenset({"nb_neighbors", "std_ratio"}) | ||
| 25 | ALLOWED_VOXELIZATION_KEYS = frozenset({"enabled", "voxel_size"}) | ||
| 26 | ALLOWED_FEATURE_KEYS = frozenset( | ||
| 27 | {"include_z_residual", "include_intensity", "normalize_intensity"} | ||
| 28 | ) | ||
| 29 | ALLOWED_CLUSTERING_KEYS = frozenset( | ||
| 30 | { | ||
| 31 | "enable_density_filtering", | ||
| 32 | "enable_intensity_trimming", | ||
| 33 | "enable_intensity_peak_split", | ||
| 34 | "dbscan_eps", | ||
| 35 | "dbscan_guard_threshold_points", | ||
| 36 | "min_cluster_size", | ||
| 37 | "max_mbytes_per_batch", | ||
| 38 | "calc_core_sample_indices", | ||
| 39 | "safe_raw_points", | ||
| 40 | "safe_post_outlier_points", | ||
| 41 | "safe_engine_rows", | ||
| 42 | "dbscan_min_points", | ||
| 43 | "min_samples", | ||
| 44 | "include_noise_in_second_pass", | ||
| 45 | "skip_gpu_second_pass", | ||
| 46 | } | ||
| 47 | ) | ||
| 48 | ALLOWED_CLUSTER_DENSITY_FILTERING_KEYS = frozenset( | ||
| 49 | {"density_cutoffs", "cutoff_portions"} | ||
| 50 | ) | ||
| 51 | ALLOWED_CLUSTER_INTENSITY_FILTERING_KEYS = frozenset( | ||
| 52 | { | ||
| 53 | "n_bins", | ||
| 54 | "intensity_range", | ||
| 55 | "peak_distance", | ||
| 56 | "peak_prominence", | ||
| 57 | "sigma_estimate", | ||
| 58 | "sigma_estimate_peak_distance_fraction", | ||
| 59 | "n_sigma_intensity_cutoff", | ||
| 60 | "min_sigma", | ||
| 61 | } | ||
| 62 | ) | ||
| 63 | ALLOWED_POSTPROCESS_KEYS = frozenset( | ||
| 64 | {"enable_geometry_filtering", "min_points", "min_length", "max_width", "max_height"} | ||
| 65 | ) | ||
| 66 | ALLOWED_FILE_NAMING_KEYS = frozenset({"bright_filtered_suffix", "cluster_prefix"}) | ||
| 67 | |||
| 68 | |||
| 69 | class ClusterFinderGPUConfigError(ValueError): | ||
| 70 | """Raised when Step 6 GPU config contains unsupported keys.""" | ||
| 71 | |||
| 72 | |||
| 73 | def _default_gpu_config_path() -> Path: | ||
| 74 | if __package__ in {None, ""}: | ||
| 75 | return Path(__file__).resolve().with_name("clustering_gpu.default.json") | ||
| 76 | return Path(str(resources.files(__package__).joinpath("clustering_gpu.default.json"))) | ||
| 77 | |||
| 78 | |||
| 79 | def _deep_merge_dicts( | ||
| 80 | base: dict[str, Any], | ||
| 81 | overrides: dict[str, Any], | ||
| 82 | ) -> dict[str, Any]: | ||
| 83 | for key, value in overrides.items(): | ||
| 84 | if isinstance(value, dict) and isinstance(base.get(key), dict): | ||
| 85 | base[key] = _deep_merge_dicts(dict(base[key]), value) | ||
| 86 | else: | ||
| 87 | base[key] = value | ||
| 88 | return base | ||
| 89 | 19 | ||
| 90 | 20 | ||
| 91 | def _validate_allowed_keys( | 21 | class ClusterFinderGPUConfigError(config_loader.ConfigError): |
| 92 | config: dict[str, Any], | 22 | """Raised when Step 6 GPU config contains unsupported keys or values.""" |
| 93 | allowed_keys: frozenset[str], | ||
| 94 | *, | ||
| 95 | context: str, | ||
| 96 | ) -> None: | ||
| 97 | unknown_keys = sorted(set(config) - allowed_keys) | ||
| 98 | if not unknown_keys: | ||
| 99 | return | ||
| 100 | |||
| 101 | allowed = ", ".join(sorted(allowed_keys)) | ||
| 102 | raise ClusterFinderGPUConfigError( | ||
| 103 | f"Unknown {context} key(s): {', '.join(unknown_keys)}. Allowed keys: {allowed}" | ||
| 104 | ) | ||
| 105 | 23 | ||
| 106 | 24 | ||
| 107 | def _normalize_section( | 25 | class ClusterFinderGPUInitialOutlierRemovalConfig(config_loader.ConfigModel): |
| 108 | raw_section: Any, | 26 | """Statistical outlier-removal section of the GPU clustering config.""" |
| 109 | *, | ||
| 110 | allowed_keys: frozenset[str], | ||
| 111 | context: str, | ||
| 112 | ) -> dict[str, Any]: | ||
| 113 | if raw_section is None: | ||
| 114 | section: dict[str, Any] = {} | ||
| 115 | elif isinstance(raw_section, dict): | ||
| 116 | section = dict(raw_section) | ||
| 117 | else: | ||
| 118 | raise ClusterFinderGPUConfigError(f"{context} must be a mapping") | ||
| 119 | 27 | ||
| 120 | _validate_allowed_keys(section, allowed_keys, context=context) | 28 | nb_neighbors: int = 20 |
| 121 | return section | 29 | std_ratio: float = 2.75 |
| 122 | 30 | ||
| 123 | 31 | ||
| 124 | def normalize_cluster_finder_gpu_config(raw_config: dict[str, Any]) -> dict[str, Any]: | 32 | class ClusterFinderGPUVoxelizationConfig(config_loader.ConfigModel): |
| 125 | config = dict(raw_config) | 33 | """Voxel-downsample section of the GPU clustering config.""" |
| 126 | if "engine" in config: | ||
| 127 | raise ClusterFinderGPUConfigError( | ||
| 128 | "Unknown step6 gpu config key(s): engine. Allowed keys: " | ||
| 129 | + ", ".join(sorted(ALLOWED_CLUSTER_FINDER_GPU_CONFIG_KEYS)) | ||
| 130 | ) | ||
| 131 | _validate_allowed_keys( | ||
| 132 | config, | ||
| 133 | ALLOWED_CLUSTER_FINDER_GPU_CONFIG_KEYS, | ||
| 134 | context="step6 gpu config", | ||
| 135 | ) | ||
| 136 | 34 | ||
| 137 | config.setdefault("device", "CUDA:0") | 35 | enabled: bool = False |
| 36 | voxel_size: float = 0.03 | ||
| 138 | 37 | ||
| 139 | initial_outlier_removal = _normalize_section( | 38 | |
| 140 | config.get("initial_outlier_removal"), | 39 | class ClusterFinderGPUFeaturesConfig(config_loader.ConfigModel): |
| 141 | allowed_keys=ALLOWED_INITIAL_OUTLIER_REMOVAL_KEYS, | 40 | """Feature-vector section of the GPU clustering config.""" |
| 142 | context="step6 gpu initial_outlier_removal", | 41 | |
| 143 | ) | 42 | include_z_residual: bool = True |
| 144 | initial_outlier_removal.setdefault("nb_neighbors", 20) | 43 | include_intensity: bool = False |
| 145 | initial_outlier_removal.setdefault("std_ratio", 2.75) | 44 | normalize_intensity: bool = True |
| 146 | config["initial_outlier_removal"] = initial_outlier_removal | 45 | |
| 147 | 46 | ||
| 148 | voxelization = _normalize_section( | 47 | class ClusterFinderGPUClusteringConfig(config_loader.ConfigModel): |
| 149 | config.get("voxelization"), | 48 | """DBSCAN and pass-control section of the GPU clustering config.""" |
| 150 | allowed_keys=ALLOWED_VOXELIZATION_KEYS, | 49 | |
| 151 | context="step6 gpu voxelization", | 50 | enable_density_filtering: bool = True |
| 152 | ) | 51 | enable_intensity_trimming: bool = False |
| 153 | voxelization.setdefault("enabled", True) | 52 | enable_intensity_peak_split: bool = False |
| 154 | voxelization.setdefault("voxel_size", 0.03) | 53 | dbscan_eps: float = 1.2 |
| 155 | config["voxelization"] = voxelization | 54 | dbscan_guard_threshold_points: int = 15000 |
| 156 | 55 | min_cluster_size: int = 200 | |
| 157 | features = _normalize_section( | 56 | dbscan_min_points: int = 50 |
| 158 | config.get("features"), | 57 | include_noise_in_second_pass: bool = False |
| 159 | allowed_keys=ALLOWED_FEATURE_KEYS, | 58 | skip_gpu_second_pass: bool = False |
| 160 | context="step6 gpu features", | 59 | max_mbytes_per_batch: int = 256 |
| 161 | ) | 60 | calc_core_sample_indices: bool = False |
| 162 | features.setdefault("include_z_residual", True) | 61 | safe_raw_points: int = 1000000 |
| 163 | features.setdefault("include_intensity", False) | 62 | safe_post_outlier_points: int = 900000 |
| 164 | features.setdefault("normalize_intensity", True) | 63 | safe_engine_rows: int = 400000 |
| 165 | config["features"] = features | 64 | |
| 166 | 65 | @pydantic.model_validator(mode="before") | |
| 167 | clustering = _normalize_section( | 66 | @classmethod |
| 168 | config.get("clustering"), | 67 | def _map_min_samples_alias(cls, value: Any) -> Any: |
| 169 | allowed_keys=ALLOWED_CLUSTERING_KEYS, | 68 | """Map the legacy ``min_samples`` clustering key onto ``dbscan_min_points``.""" |
| 170 | context="step6 gpu clustering", | 69 | if not isinstance(value, dict) or "min_samples" not in value: |
| 171 | ) | 70 | return value |
| 172 | clustering.setdefault("enable_density_filtering", True) | 71 | data = dict(value) |
| 173 | clustering.setdefault("enable_intensity_trimming", False) | 72 | min_samples = data.pop("min_samples") |
| 174 | clustering.setdefault("enable_intensity_peak_split", False) | 73 | if min_samples is not None: |
| 175 | clustering.setdefault("dbscan_eps", 1.2) | 74 | data["dbscan_min_points"] = min_samples |
| 176 | clustering.setdefault("dbscan_guard_threshold_points", 15000) | 75 | return data |
| 177 | clustering.setdefault("min_cluster_size", 200) | 76 | |
| 178 | clustering.setdefault("max_mbytes_per_batch", 256) | 77 | |
| 179 | clustering.setdefault("calc_core_sample_indices", False) | 78 | class ClusterFinderGPUDensityFilteringConfig(config_loader.ConfigModel): |
| 180 | clustering.setdefault("safe_raw_points", 1000000) | 79 | """Per-cluster density cutoff section of the GPU clustering config.""" |
| 181 | clustering.setdefault("safe_post_outlier_points", 900000) | 80 | |
| 182 | clustering.setdefault("safe_engine_rows", 400000) | 81 | density_cutoffs: list[float] = [0.0008, 0.0015, 0.003, 0.006] |
| 183 | min_samples = clustering.pop("min_samples", None) | 82 | cutoff_portions: list[float] = [0.8, 0.8, 0.2, 0.02] |
| 184 | if min_samples is not None: | 83 | |
| 185 | clustering["dbscan_min_points"] = int(min_samples) | 84 | |
| 186 | clustering.setdefault("dbscan_min_points", 50) | 85 | class ClusterFinderGPUIntensityFilteringConfig(config_loader.ConfigModel): |
| 187 | clustering.setdefault("include_noise_in_second_pass", False) | 86 | """Per-cluster intensity histogram section of the GPU clustering config.""" |
| 188 | clustering.setdefault("skip_gpu_second_pass", False) | 87 | |
| 189 | config["clustering"] = clustering | 88 | n_bins: int = 50 |
| 190 | 89 | intensity_range: list[int] = [0, 255] | |
| 191 | cluster_density_filtering = _normalize_section( | 90 | peak_distance: int = 5 |
| 192 | config.get("cluster_density_filtering"), | 91 | peak_prominence: int = 10 |
| 193 | allowed_keys=ALLOWED_CLUSTER_DENSITY_FILTERING_KEYS, | 92 | sigma_estimate: float = 3.0 |
| 194 | context="step6 gpu cluster_density_filtering", | 93 | sigma_estimate_peak_distance_fraction: float = 0.125 |
| 195 | ) | 94 | n_sigma_intensity_cutoff: float = 5.0 |
| 196 | cluster_density_filtering.setdefault("density_cutoffs", [0.0008, 0.0015, 0.003, 0.006]) | 95 | min_sigma: float = 0.5 |
| 197 | cluster_density_filtering.setdefault("cutoff_portions", [0.8, 0.8, 0.2, 0.02]) | 96 | |
| 198 | config["cluster_density_filtering"] = cluster_density_filtering | 97 | |
| 199 | 98 | class ClusterFinderGPUPostprocessConfig(config_loader.ConfigModel): | |
| 200 | cluster_intensity_filtering = _normalize_section( | 99 | """Geometry-filter section of the GPU clustering config.""" |
| 201 | config.get("cluster_intensity_filtering"), | 100 | |
| 202 | allowed_keys=ALLOWED_CLUSTER_INTENSITY_FILTERING_KEYS, | 101 | enable_geometry_filtering: bool = False |
| 203 | context="step6 gpu cluster_intensity_filtering", | 102 | min_points: int = 100 |
| 103 | min_length: float = 1.0 | ||
| 104 | max_width: float = 1.6 | ||
| 105 | max_height: float = 0.2 | ||
| 106 | |||
| 107 | |||
| 108 | class ClusterFinderGPUFileNamingConfig(config_loader.ConfigModel): | ||
| 109 | """Input/output file-name section of the GPU clustering config.""" | ||
| 110 | |||
| 111 | bright_filtered_suffix: str = "_run5_bright_filtered" | ||
| 112 | cluster_prefix: str = "run6_cluster_" | ||
| 113 | |||
| 114 | |||
| 115 | class ClusterFinderGPUConfig(config_loader.ConfigModel): | ||
| 116 | """Root GPU Step 6 clustering config; field names match the packaged JSON.""" | ||
| 117 | |||
| 118 | device: str = "CUDA:0" | ||
| 119 | voxelization: ClusterFinderGPUVoxelizationConfig = ClusterFinderGPUVoxelizationConfig() | ||
| 120 | features: ClusterFinderGPUFeaturesConfig = ClusterFinderGPUFeaturesConfig() | ||
| 121 | initial_outlier_removal: ClusterFinderGPUInitialOutlierRemovalConfig = ( | ||
| 122 | ClusterFinderGPUInitialOutlierRemovalConfig() | ||
| 204 | ) | 123 | ) |
| 205 | cluster_intensity_filtering.setdefault("n_bins", 50) | 124 | clustering: ClusterFinderGPUClusteringConfig = ClusterFinderGPUClusteringConfig() |
| 206 | cluster_intensity_filtering.setdefault("intensity_range", [0, 255]) | 125 | cluster_density_filtering: ClusterFinderGPUDensityFilteringConfig = ( |
| 207 | cluster_intensity_filtering.setdefault("peak_distance", 5) | 126 | ClusterFinderGPUDensityFilteringConfig() |
| 208 | cluster_intensity_filtering.setdefault("peak_prominence", 10) | ||
| 209 | cluster_intensity_filtering.setdefault("sigma_estimate", 3.0) | ||
| 210 | cluster_intensity_filtering.setdefault("sigma_estimate_peak_distance_fraction", 0.125) | ||
| 211 | cluster_intensity_filtering.setdefault("n_sigma_intensity_cutoff", 5.0) | ||
| 212 | cluster_intensity_filtering.setdefault("min_sigma", 0.5) | ||
| 213 | config["cluster_intensity_filtering"] = cluster_intensity_filtering | ||
| 214 | |||
| 215 | postprocess = _normalize_section( | ||
| 216 | config.get("postprocess"), | ||
| 217 | allowed_keys=ALLOWED_POSTPROCESS_KEYS, | ||
| 218 | context="step6 gpu postprocess", | ||
| 219 | ) | 127 | ) |
| 220 | postprocess.setdefault("enable_geometry_filtering", False) | 128 | cluster_intensity_filtering: ClusterFinderGPUIntensityFilteringConfig = ( |
| 221 | postprocess.setdefault("min_points", 100) | 129 | ClusterFinderGPUIntensityFilteringConfig() |
| 222 | postprocess.setdefault("min_length", 1.0) | ||
| 223 | postprocess.setdefault("max_width", 1.6) | ||
| 224 | postprocess.setdefault("max_height", 0.2) | ||
| 225 | config["postprocess"] = postprocess | ||
| 226 | |||
| 227 | file_naming = _normalize_section( | ||
| 228 | config.get("file_naming"), | ||
| 229 | allowed_keys=ALLOWED_FILE_NAMING_KEYS, | ||
| 230 | context="step6 gpu file_naming", | ||
| 231 | ) | 130 | ) |
| 232 | file_naming.setdefault("bright_filtered_suffix", "_run5_bright_filtered") | 131 | postprocess: ClusterFinderGPUPostprocessConfig = ClusterFinderGPUPostprocessConfig() |
| 233 | file_naming.setdefault("cluster_prefix", "run6_cluster_") | 132 | file_naming: ClusterFinderGPUFileNamingConfig = ClusterFinderGPUFileNamingConfig() |
| 234 | config["file_naming"] = file_naming | ||
| 235 | 133 | ||
| 236 | return config | 134 | |
| 135 | def normalize_cluster_finder_gpu_config(raw_config: dict[str, Any]) -> dict[str, Any]: | ||
| 136 | """Validate *raw_config* against the model tree and return a plain dict.""" | ||
| 137 | return config_loader.validate_config( | ||
| 138 | ClusterFinderGPUConfig, | ||
| 139 | raw_config, | ||
| 140 | context=_CONTEXT, | ||
| 141 | error_cls=ClusterFinderGPUConfigError, | ||
| 142 | ).model_dump() | ||
| 237 | 143 | ||
| 238 | 144 | ||
| 239 | def load_cluster_finder_gpu_config(config_path: str | Path | None = None) -> dict[str, Any]: | 145 | def load_cluster_finder_gpu_config(config_path: str | Path | None = None) -> dict[str, Any]: |
| 240 | resolved_path = Path(config_path) if config_path is not None else _default_gpu_config_path() | 146 | """Load packaged (or *config_path*) defaults, validate, and return a plain dict.""" |
| 241 | with resolved_path.open("r", encoding="utf-8") as handle: | 147 | return config_loader.load_config( |
| 242 | raw_config: dict[str, Any] = json.load(handle) | 148 | ClusterFinderGPUConfig, |
| 243 | return normalize_cluster_finder_gpu_config(raw_config) | 149 | package=_PACKAGE, |
| 150 | filename=_DEFAULT_FILENAME, | ||
| 151 | config_path=config_path, | ||
| 152 | context=_CONTEXT, | ||
| 153 | error_cls=ClusterFinderGPUConfigError, | ||
| 154 | ).model_dump() | ||
| 244 | 155 | ||
| 245 | 156 | ||
| 246 | def build_cluster_finder_gpu_config( | 157 | def build_cluster_finder_gpu_config( |
| 247 | *, | 158 | *, |
| 248 | overrides: dict[str, Any] | None = None, | 159 | overrides: dict[str, Any] | None = None, |
| 249 | config_path: str | Path | None = None, | 160 | config_path: str | Path | None = None, |
| 250 | ) -> dict[str, Any]: | 161 | ) -> dict[str, Any]: |
| 251 | config = load_cluster_finder_gpu_config(config_path) | 162 | """Load defaults, deep-merge *overrides*, validate, and return a plain dict.""" |
| 252 | if overrides: | 163 | return config_loader.load_config( |
| 253 | config = _deep_merge_dicts(config, dict(overrides)) | 164 | ClusterFinderGPUConfig, |
| 254 | return normalize_cluster_finder_gpu_config(config) | 165 | package=_PACKAGE, |
| 166 | filename=_DEFAULT_FILENAME, | ||
| 167 | overrides=overrides, | ||
| 168 | config_path=config_path, | ||
| 169 | context=_CONTEXT, | ||
| 170 | error_cls=ClusterFinderGPUConfigError, | ||
| 171 | ).model_dump() |
| 1 | import importlib.util | 1 | import json |
| 2 | from pathlib import Path | ||
| 3 | 2 | ||
| 4 | import pytest | 3 | import pytest |
| 5 | 4 | from iolabs.common import config_loader | |
| 6 | 5 | from iolabs_point_cloud_filtering_clusters import _config | |
| 7 | MODULE_PATH = ( | ||
| 8 | Path(__file__) | ||
| 9 | .resolve() | ||
| 10 | .parents[1] | ||
| 11 | / "src" | ||
| 12 | / "iolabs_point_cloud_filtering_clusters" | ||
| 13 | / "_config.py" | ||
| 14 | ) | ||
| 15 | |||
| 16 | SPEC = importlib.util.spec_from_file_location("step6_config", MODULE_PATH) | ||
| 17 | assert SPEC is not None and SPEC.loader is not None | ||
| 18 | MODULE = importlib.util.module_from_spec(SPEC) | ||
| 19 | SPEC.loader.exec_module(MODULE) | ||
| 20 | 6 | ||
| 21 | 7 | ||
| 22 | def test_load_cluster_finder_gpu_defaults() -> None: | 8 | def test_load_cluster_finder_gpu_defaults() -> None: |
| 23 | config = MODULE.load_cluster_finder_gpu_config() | 9 | config = _config.load_cluster_finder_gpu_config() |
| 24 | 10 | ||
| 25 | assert config["device"] == "CUDA:0" | 11 | assert config["device"] == "CUDA:0" |
| 26 | assert config["voxelization"]["enabled"] is False | 12 | assert config["voxelization"]["enabled"] is False |
| 27 | assert config["clustering"]["dbscan_min_points"] == 50 | 13 | assert config["clustering"]["dbscan_min_points"] == 50 |
| 28 | assert config["file_naming"]["cluster_prefix"] == "run6_cluster_" | 14 | assert config["file_naming"]["cluster_prefix"] == "run6_cluster_" |
| 29 | 15 | ||
| 30 | 16 | ||
| 31 | def test_unknown_step6_gpu_key_is_rejected() -> None: | 17 | def test_unknown_step6_gpu_key_is_rejected() -> None: |
| 32 | with pytest.raises(MODULE.ClusterFinderGPUConfigError, match="random_seed"): | 18 | with pytest.raises(_config.ClusterFinderGPUConfigError, match="random_seed"): |
| 33 | MODULE.build_cluster_finder_gpu_config(overrides={"random_seed": 1}) | 19 | _config.build_cluster_finder_gpu_config(overrides={"random_seed": 1}) |
| 34 | 20 | ||
| 35 | 21 | ||
| 36 | def test_legacy_min_samples_alias_maps_to_dbscan_min_points() -> None: | 22 | def test_legacy_min_samples_alias_maps_to_dbscan_min_points() -> None: |
| 37 | config = MODULE.build_cluster_finder_gpu_config( | 23 | config = _config.build_cluster_finder_gpu_config( |
| 38 | overrides={"clustering": {"min_samples": 12}} | 24 | overrides={"clustering": {"min_samples": 12}} |
| 39 | ) | 25 | ) |
| 40 | 26 | ||
| 41 | assert config["clustering"]["dbscan_min_points"] == 12 | 27 | assert config["clustering"]["dbscan_min_points"] == 12 |
| 42 | assert "min_samples" not in config["clustering"] | 28 | assert "min_samples" not in config["clustering"] |
| 29 | |||
| 30 | |||
| 31 | def test_cluster_finder_gpu_config_error_is_config_error() -> None: | ||
| 32 | assert issubclass(_config.ClusterFinderGPUConfigError, config_loader.ConfigError) | ||
| 33 | assert issubclass(_config.ClusterFinderGPUConfigError, ValueError) | ||
| 34 | |||
| 35 | |||
| 36 | def test_defaults_round_trip_packaged_json() -> None: | ||
| 37 | packaged = json.loads( | ||
| 38 | config_loader.default_config_path( | ||
| 39 | "iolabs_point_cloud_filtering_clusters", "clustering_gpu.default.json" | ||
| 40 | ).read_text(encoding="utf-8") | ||
| 41 | ) | ||
| 42 | |||
| 43 | assert _config.load_cluster_finder_gpu_config() == packaged | ||
| 44 | |||
| 45 | |||
| 46 | def test_bad_value_type_is_rejected() -> None: | ||
| 47 | with pytest.raises(_config.ClusterFinderGPUConfigError, match="dbscan_eps"): | ||
| 48 | _config.build_cluster_finder_gpu_config( | ||
| 49 | overrides={"clustering": {"dbscan_eps": "not-a-number"}} | ||
| 50 | ) | ||
| 51 | |||
| 52 | |||
| 53 | def test_nested_unknown_key_names_its_section() -> None: | ||
| 54 | with pytest.raises(_config.ClusterFinderGPUConfigError, match="postprocess"): | ||
| 55 | _config.build_cluster_finder_gpu_config(overrides={"postprocess": {"bogus": 1}}) | ||
| 56 | |||
| 57 | |||
| 58 | def test_model_defaults_match_packaged_json() -> None: | ||
| 59 | """Model defaults must mirror the packaged JSON, so a partial config file agrees.""" | ||
| 60 | packaged = json.loads( | ||
| 61 | config_loader.default_config_path( | ||
| 62 | "iolabs_point_cloud_filtering_clusters", "clustering_gpu.default.json" | ||
| 63 | ).read_text(encoding="utf-8") | ||
| 64 | ) | ||
| 65 | |||
| 66 | assert _config.ClusterFinderGPUConfig().model_dump() == packaged |
ConfigModel: nested section models mirror the packaged*.default.jsonkey for key; whitelist sets and hand-rolled coercion deleted; loader built onconfig_loader.load_config. Public entry-point names and return types unchanged so lanefinder wrappers keep working.pydantic>=2.7dependency.