Back to report index

Step 3 segmentationtrajectory 88af931: AI3D-379 Review fixes: accept write_only_segments null, enforce las_points_per_chunk>0, doc/test gaps

Miroslav Simko <ms@iolabs.ch> 2026-09-02T09:07:34+02:00

Commit #18 · 15 snippets

 AGENTS.md                                          |  2 +-
 CLAUDE.md                                          |  2 +-
 docs/configuration.md                              |  5 ++++-
 knowledge.md                                       |  2 +-
 .../_config.py                                     | 10 ++++++++-
 tests/test_config.py                               | 24 ++++++++++++++++++++++
 6 files changed, 40 insertions(+), 5 deletions(-)
Importance #1: src/iolabs_point_cloud_segmentation_trajectory/_config.py @@ -53,9 +53,9 @@
53 device: str = "CPU:0"53 device: str = "CPU:0"
54 visualize: bool = False54 visualize: bool = False
55 visualize_las_segment_coloring: bool = False55 visualize_las_segment_coloring: bool = False
56 save_points_between_planes: bool = True56 save_points_between_planes: bool = True
57 las_points_per_chunk: int = 50000057 las_points_per_chunk: int = pydantic.Field(default=500000, gt=0)
58 max_parallel_las_files: int = 158 max_parallel_las_files: int = 1
59 angle_limit: int | None = 8059 angle_limit: int | None = 80
60 n_extra_planes: int = 460 n_extra_planes: int = 4
61 segments_base_dir_name: str = "lane_points"61 segments_base_dir_name: str = "lane_points"
Importance #2: src/iolabs_point_cloud_segmentation_trajectory/_config.py @@ -78,8 +78,16 @@
78 if value is None:78 if value is None:
79 return {}79 return {}
80 return value80 return value
8181
82 @pydantic.field_validator("write_only_segments", mode="before")
83 @classmethod
84 def _none_write_only_segments_is_empty(cls, value: Any) -> Any:
85 """Treat a JSON ``null`` as 'no restriction', as the pre-pydantic code did."""
86 if value is None:
87 return []
88 return value
89
8290
83def _load_segment_mapper_model(91def _load_segment_mapper_model(
84 *,92 *,
85 overrides: dict[str, Any] | None = None,93 overrides: dict[str, Any] | None = None,
Importance #3: tests/test_config.py @@ -164,8 +164,32 @@
164 config = normalize_segment_mapper_config({"file_naming": None})164 config = normalize_segment_mapper_config({"file_naming": None})
165 assert config["file_naming"] == EXPECTED_FILE_NAMING_DEFAULTS165 assert config["file_naming"] == EXPECTED_FILE_NAMING_DEFAULTS
166166
167167
168def test_visualization_colors_none_yields_defaults() -> None:
169 config = normalize_segment_mapper_config({"visualization_colors": None})
170 assert config["visualization_colors"] == (
171 EXPECTED_TOP_LEVEL_DEFAULTS["visualization_colors"]
172 )
173
174
175def test_write_only_segments_none_means_no_restriction() -> None:
176 """A JSON ``null`` keeps the pre-pydantic 'write every segment' behaviour."""
177 config = normalize_segment_mapper_config({"write_only_segments": None})
178 assert config["write_only_segments"] == []
179
180
181def test_write_only_segments_accepts_int_list() -> None:
182 config = normalize_segment_mapper_config({"write_only_segments": [3, 7]})
183 assert config["write_only_segments"] == [3, 7]
184
185
186def test_las_points_per_chunk_must_be_positive() -> None:
187 for bad in (0, -1):
188 with pytest.raises(SegmentMapperConfigError):
189 normalize_segment_mapper_config({"las_points_per_chunk": bad})
190
191
168def test_longitudinal_limit_distance_must_be_positive() -> None:192def test_longitudinal_limit_distance_must_be_positive() -> None:
169 with pytest.raises(SegmentMapperConfigError):193 with pytest.raises(SegmentMapperConfigError):
170 normalize_segment_mapper_config({"longitudinal_limit_distance_m": 0})194 normalize_segment_mapper_config({"longitudinal_limit_distance_m": 0})
171195
Importance #4: AGENTS.md @@ -35,9 +35,9 @@
35 6. If `save_points_between_planes`, call `divide_las_file_by_planes` for each LAS — single-threaded on CUDA, otherwise a `ThreadPoolExecutor(max_workers=max_parallel_las_files)`. Per-segment `.npz` files land under `<segments_base_dir>/segment_NNN/` (3-digit zero-padded index, matching the `point{i:03d}`/`normal{i:03d}` keys in `run3_planes.npz`) and `save_version_json` drops a `run3_versions.json` next to them.35 6. If `save_points_between_planes`, call `divide_las_file_by_planes` for each LAS — single-threaded on CUDA, otherwise a `ThreadPoolExecutor(max_workers=max_parallel_las_files)`. Per-segment `.npz` files land under `<segments_base_dir>/segment_NNN/` (3-digit zero-padded index, matching the `point{i:03d}`/`normal{i:03d}` keys in `run3_planes.npz`) and `save_version_json` drops a `run3_versions.json` next to them.
3636
37- **`divide_las_file_by_planes`** streams the LAS via `laspy.open(...).chunk_iterator(las_points_per_chunk)`, never loading the whole file. Per chunk: optional `|scan_angle| < angle_limit` filter, then a sign-count mask against all selected planes assigns each point to a segment bucket. Scan-angle field is auto-detected (`scan_angle_rank` legacy vs `scan_angle` newer); RGB and intensity are required and the code raises if missing. Each per-segment `.npz` also carries `number_of_returns` (uint8, AI3D-382); LAS files without that field degrade to zeros, which the run3 NPZ contract reads as "unknown" (0 is not a legal LAS return count). Output points are written **geoshift-relative**.37- **`divide_las_file_by_planes`** streams the LAS via `laspy.open(...).chunk_iterator(las_points_per_chunk)`, never loading the whole file. Per chunk: optional `|scan_angle| < angle_limit` filter, then a sign-count mask against all selected planes assigns each point to a segment bucket. Scan-angle field is auto-detected (`scan_angle_rank` legacy vs `scan_angle` newer); RGB and intensity are required and the code raises if missing. Each per-segment `.npz` also carries `number_of_returns` (uint8, AI3D-382); LAS files without that field degrade to zeros, which the run3 NPZ contract reads as "unknown" (0 is not a legal LAS return count). Output points are written **geoshift-relative**.
3838
39- **`_config.py`** — pydantic `config_loader.ConfigModel` tree (`SegmentMapperConfig` plus nested section models) validated by `iolabs.common.config_loader.load_config`. Unknown keys raise `SegmentMapperConfigError` (a `config_loader.ConfigError`). `normalize_segment_mapper_config` / `load_segment_mapper_config` / `build_segment_mapper_config` return plain dicts (`model.model_dump()`); overrides deep-merge onto the packaged JSON. **When adding a new config key, add a field to the model and a matching default in `segment_mapper.default.json`. Nothing else.**39- **`_config.py`** — pydantic `config_loader.ConfigModel` tree (`SegmentMapperConfig` plus nested section models) validated by `iolabs.common.config_loader.load_config`. Unknown keys raise `SegmentMapperConfigError` (a `config_loader.ConfigError`). `normalize_segment_mapper_config` / `load_segment_mapper_config` / `build_segment_mapper_config` return plain dicts (`model.model_dump()`); overrides deep-merge onto the packaged JSON, while an explicit `config_path` replaces it (omitted keys fall back to the model defaults, which the packaged JSON mirrors). **When adding a new config key, add a field to the model and a matching default in `segment_mapper.default.json`. Nothing else.**
4040
41- **`segment_mapper.default.json`** — bundled defaults. It is force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`; if you rename or move it, update that mapping or the installed package will be missing the file at runtime.41- **`segment_mapper.default.json`** — bundled defaults. It is force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`; if you rename or move it, update that mapping or the installed package will be missing the file at runtime.
4242
43- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS-file work is wrapped in `las_file_scope(las_file)` so log records carry the LAS filename context — keep new per-file code paths inside that scope.43- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS-file work is wrapped in `las_file_scope(las_file)` so log records carry the LAS filename context — keep new per-file code paths inside that scope.
Importance #5: CLAUDE.md @@ -43,9 +43,9 @@
43 4. an entry in `required_chunk_fields` inside `divide_las_file_by_planes` **if the field is mandatory** — that tuple is what turns a missing field into an upfront `ValueError` instead of a later `AttributeError`.43 4. an entry in `required_chunk_fields` inside `divide_las_file_by_planes` **if the field is mandatory** — that tuple is what turns a missing field into an upfront `ValueError` instead of a later `AttributeError`.
4444
45 `SEGMENT_NPZ_FIELD_NAMES` / `NUMBER_OF_RETURNS_KEY` / `NUMBER_OF_RETURNS_DTYPE` duplicate the schema owned by `iolabs.common.segment_points_io` (mirrored, not imported, so this module stays importable against older `iolabs-common`). `tests/test_number_of_returns.py::test_npz_schema_matches_common_segment_points_io` guards the duplication; it `importorskip`s the consumer module, so it is inert until the `iolabs-common` floor is raised to a release that ships it, and then activates on its own. For the same cross-version reason, `SegmentMapper.load_color_intensity_data` builds its `ColorIntensityData` kwargs filtered by `dataclasses.fields(...)`: passing a kwarg the installed dataclass does not declare is a `TypeError`, so a field the installed `iolabs-common` predates is dropped rather than forced.45 `SEGMENT_NPZ_FIELD_NAMES` / `NUMBER_OF_RETURNS_KEY` / `NUMBER_OF_RETURNS_DTYPE` duplicate the schema owned by `iolabs.common.segment_points_io` (mirrored, not imported, so this module stays importable against older `iolabs-common`). `tests/test_number_of_returns.py::test_npz_schema_matches_common_segment_points_io` guards the duplication; it `importorskip`s the consumer module, so it is inert until the `iolabs-common` floor is raised to a release that ships it, and then activates on its own. For the same cross-version reason, `SegmentMapper.load_color_intensity_data` builds its `ColorIntensityData` kwargs filtered by `dataclasses.fields(...)`: passing a kwarg the installed dataclass does not declare is a `TypeError`, so a field the installed `iolabs-common` predates is dropped rather than forced.
4646
47- **`_config.py`** — pydantic `config_loader.ConfigModel` tree (`SegmentMapperConfig` plus nested section models) validated by `iolabs.common.config_loader.load_config`. Unknown keys raise `SegmentMapperConfigError` (a `config_loader.ConfigError`). `normalize_segment_mapper_config` / `load_segment_mapper_config` / `build_segment_mapper_config` return plain dicts (`model.model_dump()`); overrides deep-merge onto the packaged JSON. **When adding a new config key, add a field to the model and a matching default in `segment_mapper.default.json`. Nothing else.**47- **`_config.py`** — pydantic `config_loader.ConfigModel` tree (`SegmentMapperConfig` plus nested section models) validated by `iolabs.common.config_loader.load_config`. Unknown keys raise `SegmentMapperConfigError` (a `config_loader.ConfigError`). `normalize_segment_mapper_config` / `load_segment_mapper_config` / `build_segment_mapper_config` return plain dicts (`model.model_dump()`); overrides deep-merge onto the packaged JSON, while an explicit `config_path` replaces it (omitted keys fall back to the model defaults, which the packaged JSON mirrors). **When adding a new config key, add a field to the model and a matching default in `segment_mapper.default.json`. Nothing else.**
4848
49- **`segment_mapper.default.json`** — bundled defaults. It is force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`; if you rename or move it, update that mapping or the installed package will be missing the file at runtime.49- **`segment_mapper.default.json`** — bundled defaults. It is force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`; if you rename or move it, update that mapping or the installed package will be missing the file at runtime.
5050
51- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS-file work is wrapped in `las_file_scope(las_file)` so log records carry the LAS filename context — keep new per-file code paths inside that scope.51- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS-file work is wrapped in `las_file_scope(las_file)` so log records carry the LAS filename context — keep new per-file code paths inside that scope.
Importance #6: docs/configuration.md @@ -5,9 +5,11 @@
5keys raise `SegmentMapperConfigError`.5keys raise `SegmentMapperConfigError`.
66
7Defaults are loaded from7Defaults are loaded from
8`src/iolabs_point_cloud_segmentation_trajectory/segment_mapper.default.json`.8`src/iolabs_point_cloud_segmentation_trajectory/segment_mapper.default.json`.
9Partial nested overrides are deep-merged with the bundled defaults.9Partial nested overrides are deep-merged with the bundled defaults. A config
10file passed explicitly by path replaces the bundled JSON rather than extending
11it; keys it omits fall back to the same values the bundled JSON carries.
1012
11## Example13## Example
1214
13```json15```json
Importance #7: docs/configuration.md @@ -51,8 +53,9 @@
51| `reuse_existing_planes` | `false` | Loads planes from `<output_dir>/lane_points/<planes_filename>` instead of regenerating them. Requires a compatible saved `.npz` archive. |53| `reuse_existing_planes` | `false` | Loads planes from `<output_dir>/lane_points/<planes_filename>` instead of regenerating them. Requires a compatible saved `.npz` archive. |
52| `reuse_existing_geoshift` | `false` | Loads geoshift from `<output_dir>/lane_points/<geoshift_filename>` instead of recomputing it from connected spline points. |54| `reuse_existing_geoshift` | `false` | Loads geoshift from `<output_dir>/lane_points/<geoshift_filename>` instead of recomputing it from connected spline points. |
53| `enable_longitudinal_limit_planes` | `true` | Builds side limit planes for each segment and drops saved points outside the left/right corridor. This affects LAS splitting and the LAS coloring visualization. |55| `enable_longitudinal_limit_planes` | `true` | Builds side limit planes for each segment and drops saved points outside the left/right corridor. This affects LAS splitting and the LAS coloring visualization. |
54| `longitudinal_limit_distance_m` | `100.0` | Left/right offset distance, in metres, used to build longitudinal limit planes. Must be greater than zero. |56| `longitudinal_limit_distance_m` | `100.0` | Left/right offset distance, in metres, used to build longitudinal limit planes. Must be greater than zero. |
57| `write_only_segments` | `[]` | Restricts LAS splitting to the listed segment indices; an empty list (or `null`) writes every segment. Useful for re-running a few failed segments without redoing the whole split. |
55| `save_longitudinal_limit_planes` | `true` | Writes longitudinal limit planes to `<output_dir>/lane_points/<longitudinal_limit_planes_filename>` when longitudinal limits are enabled. |58| `save_longitudinal_limit_planes` | `true` | Writes longitudinal limit planes to `<output_dir>/lane_points/<longitudinal_limit_planes_filename>` when longitudinal limits are enabled. |
56| `visualization_colors` | See below | RGB colors used by visualization-only code. Values are lists of three numbers. |59| `visualization_colors` | See below | RGB colors used by visualization-only code. Values are lists of three numbers. |
57| `file_naming` | See below | Output filename overrides. Partial overrides keep unspecified defaults. |60| `file_naming` | See below | Output filename overrides. Partial overrides keep unspecified defaults. |
5861
Importance #8: knowledge.md @@ -36,9 +36,9 @@
36 7. If `save_points_between_planes`: call `divide_las_file_by_planes` per LAS. **Single-threaded on CUDA**; otherwise `max_workers = max(1, min(max_parallel_las_files, os.cpu_count()))`, and a serial path is taken when `max_workers == 1` or only one LAS file is queued — else a `ThreadPoolExecutor` runs the splits in parallel. Per-segment `.npz` files land under `<segments_base_dir>/segment_NNN/` (3-digit zero-padded, matching the `point{i:03d}`/`normal{i:03d}` keys in `run3_planes.npz`). `save_version_json` then drops `run3_versions.json` in each segment dir.36 7. If `save_points_between_planes`: call `divide_las_file_by_planes` per LAS. **Single-threaded on CUDA**; otherwise `max_workers = max(1, min(max_parallel_las_files, os.cpu_count()))`, and a serial path is taken when `max_workers == 1` or only one LAS file is queued — else a `ThreadPoolExecutor` runs the splits in parallel. Per-segment `.npz` files land under `<segments_base_dir>/segment_NNN/` (3-digit zero-padded, matching the `point{i:03d}`/`normal{i:03d}` keys in `run3_planes.npz`). `save_version_json` then drops `run3_versions.json` in each segment dir.
3737
38- **`divide_las_file_by_planes`** streams the LAS via `laspy.open(...).chunk_iterator(las_points_per_chunk)` — never loads the whole file. Per chunk: optional `|scan_angle| < angle_limit` filter, then a sign-count mask against all selected planes assigns each point to a segment bucket. Scan-angle field is auto-detected (`scan_angle_rank` legacy vs `scan_angle` newer). RGB and intensity are **required** — missing fields raise. Each per-segment `.npz` also carries `number_of_returns` (uint8, AI3D-382); LAS files without that field degrade to zeros, which the run3 NPZ contract reads as "unknown" (0 is not a legal LAS return count). Output points are written **geoshift-relative**.38- **`divide_las_file_by_planes`** streams the LAS via `laspy.open(...).chunk_iterator(las_points_per_chunk)` — never loads the whole file. Per chunk: optional `|scan_angle| < angle_limit` filter, then a sign-count mask against all selected planes assigns each point to a segment bucket. Scan-angle field is auto-detected (`scan_angle_rank` legacy vs `scan_angle` newer). RGB and intensity are **required** — missing fields raise. Each per-segment `.npz` also carries `number_of_returns` (uint8, AI3D-382); LAS files without that field degrade to zeros, which the run3 NPZ contract reads as "unknown" (0 is not a legal LAS return count). Output points are written **geoshift-relative**.
3939
40- **`_config.py`** — pydantic `config_loader.ConfigModel` tree (`SegmentMapperConfig` plus nested section models) validated by `iolabs.common.config_loader.load_config`. Unknown keys raise `SegmentMapperConfigError` (a `config_loader.ConfigError`). `normalize_segment_mapper_config` / `load_segment_mapper_config` / `build_segment_mapper_config` return plain dicts (`model.model_dump()`); overrides deep-merge onto the packaged JSON.40- **`_config.py`** — pydantic `config_loader.ConfigModel` tree (`SegmentMapperConfig` plus nested section models) validated by `iolabs.common.config_loader.load_config`. Unknown keys raise `SegmentMapperConfigError` (a `config_loader.ConfigError`). `normalize_segment_mapper_config` / `load_segment_mapper_config` / `build_segment_mapper_config` return plain dicts (`model.model_dump()`); overrides deep-merge onto the packaged JSON, while an explicit `config_path` replaces it (omitted keys fall back to the model defaults, which the packaged JSON mirrors).
4141
42- **`segment_mapper.default.json`** — bundled defaults. Force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`. If you rename/move it, update that mapping or the installed package will be missing it at runtime.42- **`segment_mapper.default.json`** — bundled defaults. Force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`. If you rename/move it, update that mapping or the installed package will be missing it at runtime.
4343
44- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS work is wrapped in `las_file_scope(las_file)` so log records carry LAS filename context.44- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS work is wrapped in `las_file_scope(las_file)` so log records carry LAS filename context.
Importance #9: CLAUDE.md @@ -43,9 +43,9 @@
43 4. an entry in `required_chunk_fields` inside `divide_las_file_by_planes` **if the field is mandatory** — that tuple is what turns a missing field into an upfront `ValueError` instead of a later `AttributeError`.43 4. an entry in `required_chunk_fields` inside `divide_las_file_by_planes` **if the field is mandatory** — that tuple is what turns a missing field into an upfront `ValueError` instead of a later `AttributeError`.
4444
45 `SEGMENT_NPZ_FIELD_NAMES` / `NUMBER_OF_RETURNS_KEY` / `NUMBER_OF_RETURNS_DTYPE` duplicate the schema owned by `iolabs.common.segment_points_io` (mirrored, not imported, so this module stays importable against older `iolabs-common`). `tests/test_number_of_returns.py::test_npz_schema_matches_common_segment_points_io` guards the duplication; it `importorskip`s the consumer module, so it is inert until the `iolabs-common` floor is raised to a release that ships it, and then activates on its own. For the same cross-version reason, `SegmentMapper.load_color_intensity_data` builds its `ColorIntensityData` kwargs filtered by `dataclasses.fields(...)`: passing a kwarg the installed dataclass does not declare is a `TypeError`, so a field the installed `iolabs-common` predates is dropped rather than forced.45 `SEGMENT_NPZ_FIELD_NAMES` / `NUMBER_OF_RETURNS_KEY` / `NUMBER_OF_RETURNS_DTYPE` duplicate the schema owned by `iolabs.common.segment_points_io` (mirrored, not imported, so this module stays importable against older `iolabs-common`). `tests/test_number_of_returns.py::test_npz_schema_matches_common_segment_points_io` guards the duplication; it `importorskip`s the consumer module, so it is inert until the `iolabs-common` floor is raised to a release that ships it, and then activates on its own. For the same cross-version reason, `SegmentMapper.load_color_intensity_data` builds its `ColorIntensityData` kwargs filtered by `dataclasses.fields(...)`: passing a kwarg the installed dataclass does not declare is a `TypeError`, so a field the installed `iolabs-common` predates is dropped rather than forced.
4646
47- **`_config.py`** — pydantic `config_loader.ConfigModel` tree (`SegmentMapperConfig` plus nested section models) validated by `iolabs.common.config_loader.load_config`. Unknown keys raise `SegmentMapperConfigError` (a `config_loader.ConfigError`). `normalize_segment_mapper_config` / `load_segment_mapper_config` / `build_segment_mapper_config` return plain dicts (`model.model_dump()`); overrides deep-merge onto the packaged JSON. **When adding a new config key, add a field to the model and a matching default in `segment_mapper.default.json`. Nothing else.**47- **`_config.py`** — pydantic `config_loader.ConfigModel` tree (`SegmentMapperConfig` plus nested section models) validated by `iolabs.common.config_loader.load_config`. Unknown keys raise `SegmentMapperConfigError` (a `config_loader.ConfigError`). `normalize_segment_mapper_config` / `load_segment_mapper_config` / `build_segment_mapper_config` return plain dicts (`model.model_dump()`); overrides deep-merge onto the packaged JSON, while an explicit `config_path` replaces it (omitted keys fall back to the model defaults, which the packaged JSON mirrors). **When adding a new config key, add a field to the model and a matching default in `segment_mapper.default.json`. Nothing else.**
4848
49- **`segment_mapper.default.json`** — bundled defaults. It is force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`; if you rename or move it, update that mapping or the installed package will be missing the file at runtime.49- **`segment_mapper.default.json`** — bundled defaults. It is force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`; if you rename or move it, update that mapping or the installed package will be missing the file at runtime.
5050
51- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS-file work is wrapped in `las_file_scope(las_file)` so log records carry the LAS filename context — keep new per-file code paths inside that scope.51- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS-file work is wrapped in `las_file_scope(las_file)` so log records carry the LAS filename context — keep new per-file code paths inside that scope.
Importance #10: docs/configuration.md @@ -5,9 +5,11 @@
5keys raise `SegmentMapperConfigError`.5keys raise `SegmentMapperConfigError`.
66
7Defaults are loaded from7Defaults are loaded from
8`src/iolabs_point_cloud_segmentation_trajectory/segment_mapper.default.json`.8`src/iolabs_point_cloud_segmentation_trajectory/segment_mapper.default.json`.
9Partial nested overrides are deep-merged with the bundled defaults.9Partial nested overrides are deep-merged with the bundled defaults. A config
10file passed explicitly by path replaces the bundled JSON rather than extending
11it; keys it omits fall back to the same values the bundled JSON carries.
1012
11## Example13## Example
1214
13```json15```json
Importance #11: docs/configuration.md @@ -51,8 +53,9 @@
51| `reuse_existing_planes` | `false` | Loads planes from `<output_dir>/lane_points/<planes_filename>` instead of regenerating them. Requires a compatible saved `.npz` archive. |53| `reuse_existing_planes` | `false` | Loads planes from `<output_dir>/lane_points/<planes_filename>` instead of regenerating them. Requires a compatible saved `.npz` archive. |
52| `reuse_existing_geoshift` | `false` | Loads geoshift from `<output_dir>/lane_points/<geoshift_filename>` instead of recomputing it from connected spline points. |54| `reuse_existing_geoshift` | `false` | Loads geoshift from `<output_dir>/lane_points/<geoshift_filename>` instead of recomputing it from connected spline points. |
53| `enable_longitudinal_limit_planes` | `true` | Builds side limit planes for each segment and drops saved points outside the left/right corridor. This affects LAS splitting and the LAS coloring visualization. |55| `enable_longitudinal_limit_planes` | `true` | Builds side limit planes for each segment and drops saved points outside the left/right corridor. This affects LAS splitting and the LAS coloring visualization. |
54| `longitudinal_limit_distance_m` | `100.0` | Left/right offset distance, in metres, used to build longitudinal limit planes. Must be greater than zero. |56| `longitudinal_limit_distance_m` | `100.0` | Left/right offset distance, in metres, used to build longitudinal limit planes. Must be greater than zero. |
57| `write_only_segments` | `[]` | Restricts LAS splitting to the listed segment indices; an empty list (or `null`) writes every segment. Useful for re-running a few failed segments without redoing the whole split. |
55| `save_longitudinal_limit_planes` | `true` | Writes longitudinal limit planes to `<output_dir>/lane_points/<longitudinal_limit_planes_filename>` when longitudinal limits are enabled. |58| `save_longitudinal_limit_planes` | `true` | Writes longitudinal limit planes to `<output_dir>/lane_points/<longitudinal_limit_planes_filename>` when longitudinal limits are enabled. |
56| `visualization_colors` | See below | RGB colors used by visualization-only code. Values are lists of three numbers. |59| `visualization_colors` | See below | RGB colors used by visualization-only code. Values are lists of three numbers. |
57| `file_naming` | See below | Output filename overrides. Partial overrides keep unspecified defaults. |60| `file_naming` | See below | Output filename overrides. Partial overrides keep unspecified defaults. |
5861
Importance #12: knowledge.md @@ -36,9 +36,9 @@
36 7. If `save_points_between_planes`: call `divide_las_file_by_planes` per LAS. **Single-threaded on CUDA**; otherwise `max_workers = max(1, min(max_parallel_las_files, os.cpu_count()))`, and a serial path is taken when `max_workers == 1` or only one LAS file is queued — else a `ThreadPoolExecutor` runs the splits in parallel. Per-segment `.npz` files land under `<segments_base_dir>/segment_NNN/` (3-digit zero-padded, matching the `point{i:03d}`/`normal{i:03d}` keys in `run3_planes.npz`). `save_version_json` then drops `run3_versions.json` in each segment dir.36 7. If `save_points_between_planes`: call `divide_las_file_by_planes` per LAS. **Single-threaded on CUDA**; otherwise `max_workers = max(1, min(max_parallel_las_files, os.cpu_count()))`, and a serial path is taken when `max_workers == 1` or only one LAS file is queued — else a `ThreadPoolExecutor` runs the splits in parallel. Per-segment `.npz` files land under `<segments_base_dir>/segment_NNN/` (3-digit zero-padded, matching the `point{i:03d}`/`normal{i:03d}` keys in `run3_planes.npz`). `save_version_json` then drops `run3_versions.json` in each segment dir.
3737
38- **`divide_las_file_by_planes`** streams the LAS via `laspy.open(...).chunk_iterator(las_points_per_chunk)` — never loads the whole file. Per chunk: optional `|scan_angle| < angle_limit` filter, then a sign-count mask against all selected planes assigns each point to a segment bucket. Scan-angle field is auto-detected (`scan_angle_rank` legacy vs `scan_angle` newer). RGB and intensity are **required** — missing fields raise. Each per-segment `.npz` also carries `number_of_returns` (uint8, AI3D-382); LAS files without that field degrade to zeros, which the run3 NPZ contract reads as "unknown" (0 is not a legal LAS return count). Output points are written **geoshift-relative**.38- **`divide_las_file_by_planes`** streams the LAS via `laspy.open(...).chunk_iterator(las_points_per_chunk)` — never loads the whole file. Per chunk: optional `|scan_angle| < angle_limit` filter, then a sign-count mask against all selected planes assigns each point to a segment bucket. Scan-angle field is auto-detected (`scan_angle_rank` legacy vs `scan_angle` newer). RGB and intensity are **required** — missing fields raise. Each per-segment `.npz` also carries `number_of_returns` (uint8, AI3D-382); LAS files without that field degrade to zeros, which the run3 NPZ contract reads as "unknown" (0 is not a legal LAS return count). Output points are written **geoshift-relative**.
3939
40- **`_config.py`** — pydantic `config_loader.ConfigModel` tree (`SegmentMapperConfig` plus nested section models) validated by `iolabs.common.config_loader.load_config`. Unknown keys raise `SegmentMapperConfigError` (a `config_loader.ConfigError`). `normalize_segment_mapper_config` / `load_segment_mapper_config` / `build_segment_mapper_config` return plain dicts (`model.model_dump()`); overrides deep-merge onto the packaged JSON.40- **`_config.py`** — pydantic `config_loader.ConfigModel` tree (`SegmentMapperConfig` plus nested section models) validated by `iolabs.common.config_loader.load_config`. Unknown keys raise `SegmentMapperConfigError` (a `config_loader.ConfigError`). `normalize_segment_mapper_config` / `load_segment_mapper_config` / `build_segment_mapper_config` return plain dicts (`model.model_dump()`); overrides deep-merge onto the packaged JSON, while an explicit `config_path` replaces it (omitted keys fall back to the model defaults, which the packaged JSON mirrors).
4141
42- **`segment_mapper.default.json`** — bundled defaults. Force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`. If you rename/move it, update that mapping or the installed package will be missing it at runtime.42- **`segment_mapper.default.json`** — bundled defaults. Force-included into the wheel via `[tool.hatch.build.targets.wheel.force-include]` in `pyproject.toml`. If you rename/move it, update that mapping or the installed package will be missing it at runtime.
4343
44- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS work is wrapped in `las_file_scope(las_file)` so log records carry LAS filename context.44- **`_log_props.py`** — `LOG_PROPS = {"pipeline_step": "s3", ...}` is attached to every logger via `iolabs.logstash.get_props_logger`. Per-LAS work is wrapped in `las_file_scope(las_file)` so log records carry LAS filename context.
Importance #13: src/iolabs_point_cloud_segmentation_trajectory/_config.py @@ -53,9 +53,9 @@
53 device: str = "CPU:0"53 device: str = "CPU:0"
54 visualize: bool = False54 visualize: bool = False
55 visualize_las_segment_coloring: bool = False55 visualize_las_segment_coloring: bool = False
56 save_points_between_planes: bool = True56 save_points_between_planes: bool = True
57 las_points_per_chunk: int = 50000057 las_points_per_chunk: int = pydantic.Field(default=500000, gt=0)
58 max_parallel_las_files: int = 158 max_parallel_las_files: int = 1
59 angle_limit: int | None = 8059 angle_limit: int | None = 80
60 n_extra_planes: int = 460 n_extra_planes: int = 4
61 segments_base_dir_name: str = "lane_points"61 segments_base_dir_name: str = "lane_points"
Importance #14: src/iolabs_point_cloud_segmentation_trajectory/_config.py @@ -78,8 +78,16 @@
78 if value is None:78 if value is None:
79 return {}79 return {}
80 return value80 return value
8181
82 @pydantic.field_validator("write_only_segments", mode="before")
83 @classmethod
84 def _none_write_only_segments_is_empty(cls, value: Any) -> Any:
85 """Treat a JSON ``null`` as 'no restriction', as the pre-pydantic code did."""
86 if value is None:
87 return []
88 return value
89
8290
83def _load_segment_mapper_model(91def _load_segment_mapper_model(
84 *,92 *,
85 overrides: dict[str, Any] | None = None,93 overrides: dict[str, Any] | None = None,
Importance #15: tests/test_config.py @@ -164,8 +164,32 @@
164 config = normalize_segment_mapper_config({"file_naming": None})164 config = normalize_segment_mapper_config({"file_naming": None})
165 assert config["file_naming"] == EXPECTED_FILE_NAMING_DEFAULTS165 assert config["file_naming"] == EXPECTED_FILE_NAMING_DEFAULTS
166166
167167
168def test_visualization_colors_none_yields_defaults() -> None:
169 config = normalize_segment_mapper_config({"visualization_colors": None})
170 assert config["visualization_colors"] == (
171 EXPECTED_TOP_LEVEL_DEFAULTS["visualization_colors"]
172 )
173
174
175def test_write_only_segments_none_means_no_restriction() -> None:
176 """A JSON ``null`` keeps the pre-pydantic 'write every segment' behaviour."""
177 config = normalize_segment_mapper_config({"write_only_segments": None})
178 assert config["write_only_segments"] == []
179
180
181def test_write_only_segments_accepts_int_list() -> None:
182 config = normalize_segment_mapper_config({"write_only_segments": [3, 7]})
183 assert config["write_only_segments"] == [3, 7]
184
185
186def test_las_points_per_chunk_must_be_positive() -> None:
187 for bad in (0, -1):
188 with pytest.raises(SegmentMapperConfigError):
189 normalize_segment_mapper_config({"las_points_per_chunk": bad})
190
191
168def test_longitudinal_limit_distance_must_be_positive() -> None:192def test_longitudinal_limit_distance_must_be_positive() -> None:
169 with pytest.raises(SegmentMapperConfigError):193 with pytest.raises(SegmentMapperConfigError):
170 normalize_segment_mapper_config({"longitudinal_limit_distance_m": 0})194 normalize_segment_mapper_config({"longitudinal_limit_distance_m": 0})
171195