Back to report index

Step 3 3dsegmentation de39aa0: AI3D-382 Carry unknown run3 record keys through SegmentCloud

Miroslav Simko <ms@iolabs.ch> 2026-09-01T14:51:34+02:00

Commit #106 ยท 7 snippets

 src/iolabs_point_cloud_segmentation_3d/io_npz.py | 52 +++++++++++++++++++++++-
 tests/test_fusion.py                             | 41 +++++++++++++++++++
 2 files changed, 91 insertions(+), 2 deletions(-)
Importance #1: src/iolabs_point_cloud_segmentation_3d/io_npz.py @@ -8,9 +8,9 @@
8(the road-surface key set, its recall metric) and the `SegmentCloud` shape the8(the road-surface key set, its recall metric) and the `SegmentCloud` shape the
9fusion pipeline consumes.9fusion pipeline consumes.
10"""10"""
1111
12from dataclasses import dataclass12from dataclasses import dataclass, field
13from pathlib import Path13from pathlib import Path
1414
15import numpy as np15import numpy as np
16from iolabs.common.point_hash import (16from iolabs.common.point_hash import (
Importance #2: src/iolabs_point_cloud_segmentation_3d/io_npz.py @@ -30,18 +30,35 @@
3030
31# Storage dtypes every run3 record is coerced to on load. These are the31# Storage dtypes every run3 record is coerced to on load. These are the
32# historical seg3d coercions and they are load-bearing: `points` must stay32# historical seg3d coercions and they are load-bearing: `points` must stay
33# float64 all the way into the hash rounding, and the uint16/int8 channels are33# float64 all the way into the hash rounding, and the uint16/int8 channels are
34# what the npz/LAS writers expect.34# what the npz/LAS writers expect. Keys the loaded records do not carry are
35# ignored by the concatenator, so listing a key the installed
36# `iolabs.common.segment_points_io` contract does not (yet) produce is a no-op
37# rather than an error -- that is what keeps `number_of_returns` (AI3D-382,
38# uint8) safe to declare here ahead of the contract bump: once records carry
39# it, mixed uint8/int-width inputs are cast instead of rejected.
35RUN3_TARGET_DTYPES: dict[str, np.dtype] = {40RUN3_TARGET_DTYPES: dict[str, np.dtype] = {
36 "points": np.dtype(np.float64),41 "points": np.dtype(np.float64),
37 "intensity": np.dtype(np.uint16),42 "intensity": np.dtype(np.uint16),
38 "red": np.dtype(np.uint16),43 "red": np.dtype(np.uint16),
39 "green": np.dtype(np.uint16),44 "green": np.dtype(np.uint16),
40 "blue": np.dtype(np.uint16),45 "blue": np.dtype(np.uint16),
41 "scan_angle": np.dtype(np.int8),46 "scan_angle": np.dtype(np.int8),
47 "number_of_returns": np.dtype(np.uint8),
42}48}
4349
50# Record keys `SegmentCloud` exposes as named attributes. Every other key the
51# run3 contract grows lands in `SegmentCloud.extra` without a code change here.
52NAMED_RECORD_KEYS: tuple[str, ...] = (
53 "points",
54 "intensity",
55 "red",
56 "green",
57 "blue",
58 "scan_angle",
59)
60
44#: Filename suffix of the run4 road-surface records joined against run3.61#: Filename suffix of the run4 road-surface records joined against run3.
45RUN4_SURFACE_SUFFIX = "_run4_road_surface.npz"62RUN4_SURFACE_SUFFIX = "_run4_road_surface.npz"
4663
47#: One input record's identity and row range within the concatenated cloud.64#: One input record's identity and row range within the concatenated cloud.
Importance #3: src/iolabs_point_cloud_segmentation_3d/io_npz.py @@ -83,8 +100,13 @@
83 road-surface hash join.100 road-surface hash join.
84 records: Per-record identity/offset metadata, in concatenation101 records: Per-record identity/offset metadata, in concatenation
85 order.102 order.
86 surface_match_rate: Fraction of run4 keys found in run3.103 surface_match_rate: Fraction of run4 keys found in run3.
104 extra: Every further `(N,)` channel the loaded run3 records carried
105 (i.e. record keys outside `NAMED_RECORD_KEYS`), in record order.
106 Empty when the installed `iolabs.common` point-record contract
107 has no such keys, so consumers must treat a channel as optional
108 and never assume presence.
87 """109 """
88110
89 points: np.ndarray111 points: np.ndarray
90 intensity: np.ndarray112 intensity: np.ndarray
Importance #4: src/iolabs_point_cloud_segmentation_3d/io_npz.py @@ -94,14 +116,33 @@
94 scan_angle: np.ndarray116 scan_angle: np.ndarray
95 is_surface: np.ndarray117 is_surface: np.ndarray
96 records: list[Record]118 records: list[Record]
97 surface_match_rate: float119 surface_match_rate: float
120 extra: dict[str, np.ndarray] = field(default_factory=dict)
98121
99 @property122 @property
100 def n(self) -> int:123 def n(self) -> int:
101 """Number of points in the concatenated cloud."""124 """Number of points in the concatenated cloud."""
102 return self.points.shape[0]125 return self.points.shape[0]
103126
127 def channel(self, name: str) -> np.ndarray | None:
128 """Returns a point channel by record-key name, or `None` if absent.
129
130 Reads named attributes and `extra` through one lookup, so callers
131 that want an optional channel (e.g. `number_of_returns`) do not have
132 to know which of the two carries it.
133
134 Args:
135 name: Run3 record key, e.g. `"intensity"` or `"number_of_returns"`.
136
137 Returns:
138 The `(N,)` (or `(N, 3)` for `points`) array, or `None` when this
139 cloud has no such channel.
140 """
141 if name in NAMED_RECORD_KEYS:
142 return getattr(self, name)
143 return self.extra.get(name)
144
104 def record_names(self) -> list[str]:145 def record_names(self) -> list[str]:
105 """Returns the contributing record basenames, in concatenation order."""146 """Returns the contributing record basenames, in concatenation order."""
106 return [r.name for r in self.records]147 return [r.name for r in self.records]
107148
Importance #5: src/iolabs_point_cloud_segmentation_3d/io_npz.py @@ -211,5 +252,12 @@
211 scan_angle=record["scan_angle"],252 scan_angle=record["scan_angle"],
212 is_surface=is_surface,253 is_surface=is_surface,
213 records=records,254 records=records,
214 surface_match_rate=surface_match_rate,255 surface_match_rate=surface_match_rate,
256 # Carried generically: a record key added upstream reaches consumers
257 # without another edit here.
258 extra={
259 key: value
260 for key, value in record.items()
261 if key not in NAMED_RECORD_KEYS
262 },
215 )263 )
Importance #6: tests/test_fusion.py @@ -134,8 +134,49 @@
134 cloud = io_npz.load_segment_cloud(seg)134 cloud = io_npz.load_segment_cloud(seg)
135 assert not cloud.is_surface.any()135 assert not cloud.is_surface.any()
136136
137137
138def test_segment_cloud_channel_lookup(tmp_path):
139 """`channel` reads named and optional channels through one lookup."""
140 seg = tmp_path / "segment_003"
141 seg.mkdir()
142 _write_run3(seg / "a_run3_points.npz", np.zeros((3, 3)))
143 cloud = io_npz.load_segment_cloud(seg)
144
145 np.testing.assert_array_equal(cloud.channel("intensity"), cloud.intensity)
146 np.testing.assert_array_equal(cloud.channel("points"), cloud.points)
147 assert cloud.channel("not_a_channel") is None
148 # Named channels never leak into the generic bucket.
149 assert not set(cloud.extra) & set(io_npz.NAMED_RECORD_KEYS)
150 # Optional contract channels are present only when the installed
151 # iolabs.common point-record contract produces them.
152 returns = cloud.channel("number_of_returns")
153 assert returns is None or returns.shape == (cloud.n,)
154
155
156def test_segment_cloud_carries_unknown_record_keys(tmp_path, monkeypatch):
157 """A record key the loader grows reaches consumers with no edit here."""
158 seg = tmp_path / "segment_004"
159 seg.mkdir()
160 _write_run3(seg / "a_run3_points.npz", np.zeros((3, 3)))
161 real_load = io_npz.load_run3_segment
162
163 def _with_future_key(seg_dir, **kwargs):
164 record, records = real_load(seg_dir, **kwargs)
165 record = dict(record)
166 record["future_channel"] = np.arange(
167 record["points"].shape[0], dtype=np.uint8
168 )
169 return record, records
170
171 monkeypatch.setattr(io_npz, "load_run3_segment", _with_future_key)
172 cloud = io_npz.load_segment_cloud(seg)
173
174 expected = np.arange(3, dtype=np.uint8)
175 np.testing.assert_array_equal(cloud.extra["future_channel"], expected)
176 np.testing.assert_array_equal(cloud.channel("future_channel"), expected)
177
178
138# --------------------------------------------------------------------------- #179# --------------------------------------------------------------------------- #
139# pavement / polygon classify180# pavement / polygon classify
140# --------------------------------------------------------------------------- #181# --------------------------------------------------------------------------- #
141def test_polygon_classify():182def test_polygon_classify():
Importance #7: tests/test_fusion.py @@ -134,8 +134,49 @@
134 cloud = io_npz.load_segment_cloud(seg)134 cloud = io_npz.load_segment_cloud(seg)
135 assert not cloud.is_surface.any()135 assert not cloud.is_surface.any()
136136
137137
138def test_segment_cloud_channel_lookup(tmp_path):
139 """`channel` reads named and optional channels through one lookup."""
140 seg = tmp_path / "segment_003"
141 seg.mkdir()
142 _write_run3(seg / "a_run3_points.npz", np.zeros((3, 3)))
143 cloud = io_npz.load_segment_cloud(seg)
144
145 np.testing.assert_array_equal(cloud.channel("intensity"), cloud.intensity)
146 np.testing.assert_array_equal(cloud.channel("points"), cloud.points)
147 assert cloud.channel("not_a_channel") is None
148 # Named channels never leak into the generic bucket.
149 assert not set(cloud.extra) & set(io_npz.NAMED_RECORD_KEYS)
150 # Optional contract channels are present only when the installed
151 # iolabs.common point-record contract produces them.
152 returns = cloud.channel("number_of_returns")
153 assert returns is None or returns.shape == (cloud.n,)
154
155
156def test_segment_cloud_carries_unknown_record_keys(tmp_path, monkeypatch):
157 """A record key the loader grows reaches consumers with no edit here."""
158 seg = tmp_path / "segment_004"
159 seg.mkdir()
160 _write_run3(seg / "a_run3_points.npz", np.zeros((3, 3)))
161 real_load = io_npz.load_run3_segment
162
163 def _with_future_key(seg_dir, **kwargs):
164 record, records = real_load(seg_dir, **kwargs)
165 record = dict(record)
166 record["future_channel"] = np.arange(
167 record["points"].shape[0], dtype=np.uint8
168 )
169 return record, records
170
171 monkeypatch.setattr(io_npz, "load_run3_segment", _with_future_key)
172 cloud = io_npz.load_segment_cloud(seg)
173
174 expected = np.arange(3, dtype=np.uint8)
175 np.testing.assert_array_equal(cloud.extra["future_channel"], expected)
176 np.testing.assert_array_equal(cloud.channel("future_channel"), expected)
177
178
138# --------------------------------------------------------------------------- #179# --------------------------------------------------------------------------- #
139# pavement / polygon classify180# pavement / polygon classify
140# --------------------------------------------------------------------------- #181# --------------------------------------------------------------------------- #
141def test_polygon_classify():182def test_polygon_classify():