Back to report index

iolabs-common 3c981c5: AI3D-382 Add number_of_returns to the run3 point-NPZ contract

Miroslav Simko <ms@iolabs.ch> 2026-09-01T10:43:42+02:00

Commit #1 ยท 9 snippets

 src/iolabs/common/segment_points_io.py | 105 +++++++++++++++++++++++---
 tests/test_segment_points_io.py        | 130 +++++++++++++++++++++++++++++++++
 2 files changed, 224 insertions(+), 11 deletions(-)

Origin of the contract. segment_points_io gains NUMBER_OF_RETURNS_KEY/NUMBER_OF_RETURNS_DTYPE (uint8); save_points_npz always writes it, load_points_npz zero-fills it when absent (0 = unknown). Every other commit in this report exists to feed or honour this.

Importance #1: src/iolabs/common/segment_points_io.py @@ -24,69 +34,137 @@

The contract itself: key constants, save always writes uint8, load zero-fills when absent. Read this hunk first.

24import numpy as np34import numpy as np
2535
26logger = logging.getLogger(__name__)36logger = logging.getLogger(__name__)
2737
28POINT_RECORD_KEYS: tuple[str, ...] = (38#: Per-point LAS return count; see the module docstring for the 0 = unknown rule.
39NUMBER_OF_RETURNS_KEY = "number_of_returns"
40
41#: Storage dtype of :data:`NUMBER_OF_RETURNS_KEY` (LAS carries 3 bits, values 1-7).
42NUMBER_OF_RETURNS_DTYPE = np.uint8
43
44#: Keys every record must already carry; a file missing one of these is corrupt.
45REQUIRED_POINT_RECORD_KEYS: tuple[str, ...] = (
29 "points",46 "points",
30 "red",47 "red",
31 "green",48 "green",
32 "blue",49 "blue",
33 "intensity",50 "intensity",
34 "scan_angle",51 "scan_angle",
35)52)
3653
54#: Keys added after datasets were already on disk: zero-filled when absent on load.
55OPTIONAL_POINT_RECORD_KEYS: tuple[str, ...] = (NUMBER_OF_RETURNS_KEY,)
56
57#: Every key a record carries once loaded, and every key that is written.
58POINT_RECORD_KEYS: tuple[str, ...] = REQUIRED_POINT_RECORD_KEYS + OPTIONAL_POINT_RECORD_KEYS
59
37PointRecord = dict[str, np.ndarray]60PointRecord = dict[str, np.ndarray]
3861
39GEOSHIFT_NAME = "run3_geoshift.json"62GEOSHIFT_NAME = "run3_geoshift.json"
40LANE_POINTS_DIR_NAME = "lane_points"63LANE_POINTS_DIR_NAME = "lane_points"
41RUN3_POINTS_SUFFIX = "_run3_points.npz"64RUN3_POINTS_SUFFIX = "_run3_points.npz"
42RUN3_POINTS_GLOB = f"*{RUN3_POINTS_SUFFIX}"65RUN3_POINTS_GLOB = f"*{RUN3_POINTS_SUFFIX}"
4366
4467
68def _coerce_number_of_returns(values: np.ndarray, *, source: str) -> np.ndarray:
69 """Cast a return-count array to the contract's uint8 storage dtype.
70
71 Args:
72 values: Array-like of per-point return counts.
73 source: Label used in error messages (typically the target path).
74
75 Returns:
76 The values as uint8, unchanged when they already are.
77
78 Raises:
79 ValueError: The array is not of integer dtype, or holds a value
80 outside the uint8 range (which would wrap silently on cast).
81 """
82 array = np.asarray(values)
83 if array.dtype == NUMBER_OF_RETURNS_DTYPE:
84 return array
85 if array.dtype.kind not in "uib":
86 raise ValueError(
87 f"{source}: '{NUMBER_OF_RETURNS_KEY}' must be an integer array, "
88 f"got dtype {array.dtype}"
89 )
90 if array.size and (int(array.min()) < 0 or int(array.max()) > 255):
91 raise ValueError(
92 f"{source}: '{NUMBER_OF_RETURNS_KEY}' values must fit in uint8, got range "
93 f"[{int(array.min())}, {int(array.max())}]"
94 )
95 return array.astype(NUMBER_OF_RETURNS_DTYPE)
96
97
45def load_points_npz(path: str | Path) -> PointRecord:98def load_points_npz(path: str | Path) -> PointRecord:
46 """Load one ``*_run3_points.npz``-style file and validate the schema.99 """Load one ``*_run3_points.npz``-style file and validate the schema.
47100
48 Requires all :data:`POINT_RECORD_KEYS`. ``points`` must be shape ``(N, 3)``;101 Requires all :data:`REQUIRED_POINT_RECORD_KEYS`. ``points`` must be shape
49 every ancillary array (``red``, ``green``, ``blue``, ``intensity``,102 ``(N, 3)``; every ancillary array (``red``, ``green``, ``blue``,
50 ``scan_angle``) must be 1-D with shape ``(N,)``.103 ``intensity``, ``scan_angle``, ``number_of_returns``) must be 1-D with
104 shape ``(N,)``.
105
106 :data:`OPTIONAL_POINT_RECORD_KEYS` -- today just ``number_of_returns`` --
107 may be absent: records written before AI3D-382 predate the key, and are
108 filled with a zeros uint8 array (0 = unknown, see the module docstring)
109 rather than rejected. The returned mapping always holds every key in
110 :data:`POINT_RECORD_KEYS`.
51 """111 """
52 npz_path = Path(path)112 npz_path = Path(path)
53 with np.load(npz_path) as data:113 with np.load(npz_path) as data:
54 missing = [key for key in POINT_RECORD_KEYS if key not in data.files]114 missing = [key for key in REQUIRED_POINT_RECORD_KEYS if key not in data.files]
55 if missing:115 if missing:
56 raise ValueError(116 raise ValueError(
57 f"{npz_path}: missing required key(s) {missing}; "117 f"{npz_path}: missing required key(s) {missing}; "
58 f"expected {list(POINT_RECORD_KEYS)}"118 f"expected {list(REQUIRED_POINT_RECORD_KEYS)}"
59 )119 )
60 record = {key: np.asarray(data[key]) for key in POINT_RECORD_KEYS}120 record = {key: np.asarray(data[key]) for key in REQUIRED_POINT_RECORD_KEYS}
121 present_optional = [key for key in OPTIONAL_POINT_RECORD_KEYS if key in data.files]
122 record.update({key: np.asarray(data[key]) for key in present_optional})
61123
62 points = record["points"]124 points = record["points"]
63 if points.ndim != 2 or points.shape[1] != 3:125 if points.ndim != 2 or points.shape[1] != 3:
64 raise ValueError(126 raise ValueError(
65 f"{npz_path}: 'points' must have shape (N, 3), got {points.shape}"127 f"{npz_path}: 'points' must have shape (N, 3), got {points.shape}"
66 )128 )
67 n_points = int(points.shape[0])129 n_points = int(points.shape[0])
68 for key in POINT_RECORD_KEYS:130 for key, arr in record.items():
69 if key == "points":131 if key == "points":
70 continue132 continue
71 arr = record[key]
72 if arr.shape != (n_points,):133 if arr.shape != (n_points,):
73 raise ValueError(134 raise ValueError(
74 f"{npz_path}: '{key}' must have shape (N,), got {arr.shape}"135 f"{npz_path}: '{key}' must have shape (N,), got {arr.shape}"
75 )136 )
76 return record137 if NUMBER_OF_RETURNS_KEY not in record:
138 logger.debug(
139 "%s: no '%s' member (pre-AI3D-382 record); filling %d zeros (unknown)",
140 npz_path.name,
141 NUMBER_OF_RETURNS_KEY,
142 n_points,
143 )
144 record[NUMBER_OF_RETURNS_KEY] = np.zeros(n_points, dtype=NUMBER_OF_RETURNS_DTYPE)
145 return {key: record[key] for key in POINT_RECORD_KEYS}
77146
78147
79def save_points_npz(path: str | Path, record: Mapping[str, np.ndarray]) -> Path:148def save_points_npz(path: str | Path, record: Mapping[str, np.ndarray]) -> Path:
80 """Write a point record as a compressed NPZ (inverse of :func:`load_points_npz`)."""149 """Write a point record as a compressed NPZ (inverse of :func:`load_points_npz`).
150
151 Every key in :data:`POINT_RECORD_KEYS` must be present, ``number_of_returns``
152 included: it is optional on load only, to read datasets that predate it.
153 A producer that omits it here is not "unknown", it is out of date, so this
154 raises instead of zero-filling. ``number_of_returns`` is stored as uint8.
155 """
81 npz_path = Path(path)156 npz_path = Path(path)
82 missing = [key for key in POINT_RECORD_KEYS if key not in record]157 missing = [key for key in POINT_RECORD_KEYS if key not in record]
83 if missing:158 if missing:
84 raise ValueError(159 raise ValueError(
85 f"Cannot save {npz_path}: missing required key(s) {missing}; "160 f"Cannot save {npz_path}: missing required key(s) {missing}; "
86 f"expected {list(POINT_RECORD_KEYS)}"161 f"expected {list(POINT_RECORD_KEYS)}"
87 )162 )
88 payload = {key: np.asarray(record[key]) for key in POINT_RECORD_KEYS}163 payload = {key: np.asarray(record[key]) for key in POINT_RECORD_KEYS}
164 payload[NUMBER_OF_RETURNS_KEY] = _coerce_number_of_returns(
165 payload[NUMBER_OF_RETURNS_KEY], source=f"Cannot save {npz_path}"
166 )
89 points = payload["points"]167 points = payload["points"]
90 if points.ndim != 2 or points.shape[1] != 3:168 if points.ndim != 2 or points.shape[1] != 3:
91 raise ValueError(169 raise ValueError(
92 f"Cannot save {npz_path}: 'points' must have shape (N, 3), got {points.shape}"170 f"Cannot save {npz_path}: 'points' must have shape (N, 3), got {points.shape}"
Importance #2: src/iolabs/common/segment_points_io.py @@ -5,8 +5,18 @@
5segment's run3 records, chunked streaming of the ``points`` member,5segment's run3 records, chunked streaming of the ``points`` member,
6``run3_geoshift.json`` lookup/loading, and per-segment fnmatch blacklist6``run3_geoshift.json`` lookup/loading, and per-segment fnmatch blacklist
7filtering. Pure NumPy only.7filtering. Pure NumPy only.
88
9``number_of_returns`` (AI3D-382) is the LAS per-point return count, stored as
10uint8. It is **always written** by :func:`save_points_npz` -- a record that
11does not carry it is rejected rather than zero-filled, so a producer that
12still has the data cannot silently downgrade it. On the read side the key is
13**optional**: datasets written before AI3D-382 have no such member, so
14:func:`load_points_npz` fills a zeros uint8 array of matching point count.
15Zero is not a legal LAS return count (valid values are 1-7 for point formats
160-5), which makes it an unambiguous "unknown" sentinel. Never default to 1 --
17that fabricates a plausible-looking measurement.
18
9Geoshift sign convention: every function here returns the *shift itself*,19Geoshift sign convention: every function here returns the *shift itself*,
10exactly as the trajectory step recorded it. Upstream writes run3 points as20exactly as the trajectory step recorded it. Upstream writes run3 points as
11``source - geoshift``, so consumers that want world coordinates add it back,21``source - geoshift``, so consumers that want world coordinates add it back,
12and consumers working in the local frame ignore it. This module never22and consumers working in the local frame ignore it. This module never
Importance #3: src/iolabs/common/segment_points_io.py @@ -552,8 +630,13 @@
552 *target_dtypes*; every record is then cast to them before concatenation630 *target_dtypes*; every record is then cast to them before concatenation
553 and no mixed-dtype rejection applies to the listed keys. No decimation or631 and no mixed-dtype rejection applies to the listed keys. No decimation or
554 filtering is applied; that stays a caller policy.632 filtering is applied; that stays a caller policy.
555633
634 Records predating ``number_of_returns`` contribute the zero-filled uint8
635 array :func:`load_points_npz` synthesises for them, so a mixed-vintage
636 segment concatenates without a dtype clash and the missing rows stay
637 marked unknown.
638
556 Args:639 Args:
557 files: Record paths in the intended concatenation order.640 files: Record paths in the intended concatenation order.
558 target_dtypes: Optional per-key dtypes to cast every record to641 target_dtypes: Optional per-key dtypes to cast every record to
559 (e.g. ``{"points": np.float64, "intensity": np.uint16}``). Keys642 (e.g. ``{"points": np.float64, "intensity": np.uint16}``). Keys
Importance #4: tests/test_segment_points_io.py @@ -8,9 +8,11 @@
8import pytest8import pytest
99
10from iolabs.common.segment_points_io import (10from iolabs.common.segment_points_io import (
11 GEOSHIFT_NAME,11 GEOSHIFT_NAME,
12 NUMBER_OF_RETURNS_KEY,
12 POINT_RECORD_KEYS,13 POINT_RECORD_KEYS,
14 REQUIRED_POINT_RECORD_KEYS,
13 RecordSpan,15 RecordSpan,
14 concat_points_npz,16 concat_points_npz,
15 discover_run3_files,17 discover_run3_files,
16 filter_segment_files,18 filter_segment_files,
Importance #5: tests/test_segment_points_io.py @@ -38,11 +40,19 @@
38 "green": rng.integers(0, 65535, size=n, dtype=np.uint16),40 "green": rng.integers(0, 65535, size=n, dtype=np.uint16),
39 "blue": rng.integers(0, 65535, size=n, dtype=np.uint16),41 "blue": rng.integers(0, 65535, size=n, dtype=np.uint16),
40 "intensity": rng.integers(0, 65535, size=n, dtype=np.uint16),42 "intensity": rng.integers(0, 65535, size=n, dtype=np.uint16),
41 "scan_angle": rng.integers(-90, 90, size=n, dtype=np.int8),43 "scan_angle": rng.integers(-90, 90, size=n, dtype=np.int8),
44 "number_of_returns": rng.integers(1, 8, size=n, dtype=np.uint8),
42 }45 }
4346
4447
48def _make_legacy_record(n: int, *, seed: int = 0) -> dict[str, np.ndarray]:
49 """Build a pre-AI3D-382 record: every key except ``number_of_returns``."""
50 record = _make_record(n, seed=seed)
51 del record[NUMBER_OF_RETURNS_KEY]
52 return record
53
54
45def test_save_load_points_npz_round_trip(tmp_path: Path) -> None:55def test_save_load_points_npz_round_trip(tmp_path: Path) -> None:
46 record = _make_record(7, seed=1)56 record = _make_record(7, seed=1)
47 path = tmp_path / "scan_a_run3_points.npz"57 path = tmp_path / "scan_a_run3_points.npz"
48 save_points_npz(path, record)58 save_points_npz(path, record)
Importance #6: tests/test_segment_points_io.py @@ -118,8 +128,128 @@
118 with pytest.raises(ValueError, match="missing required key"):128 with pytest.raises(ValueError, match="missing required key"):
119 save_points_npz(tmp_path / "out.npz", {"points": np.zeros((1, 3))})129 save_points_npz(tmp_path / "out.npz", {"points": np.zeros((1, 3))})
120130
121131
132def test_number_of_returns_is_part_of_the_written_contract() -> None:
133 assert NUMBER_OF_RETURNS_KEY in POINT_RECORD_KEYS
134 assert NUMBER_OF_RETURNS_KEY not in REQUIRED_POINT_RECORD_KEYS
135 assert set(REQUIRED_POINT_RECORD_KEYS) < set(POINT_RECORD_KEYS)
136
137
138def test_save_points_npz_always_writes_number_of_returns(tmp_path: Path) -> None:
139 record = _make_record(6, seed=60)
140 path = save_points_npz(tmp_path / "with_returns_run3_points.npz", record)
141
142 with np.load(path) as data:
143 assert NUMBER_OF_RETURNS_KEY in data.files
144 stored = np.asarray(data[NUMBER_OF_RETURNS_KEY])
145 assert stored.dtype == np.uint8
146 np.testing.assert_array_equal(stored, record[NUMBER_OF_RETURNS_KEY])
147
148
149def test_save_points_npz_casts_number_of_returns_to_uint8(tmp_path: Path) -> None:
150 record = _make_record(4, seed=61)
151 record[NUMBER_OF_RETURNS_KEY] = record[NUMBER_OF_RETURNS_KEY].astype(np.int64)
152 path = save_points_npz(tmp_path / "cast_run3_points.npz", record)
153
154 loaded = load_points_npz(path)
155 assert loaded[NUMBER_OF_RETURNS_KEY].dtype == np.uint8
156
157
158def test_save_points_npz_rejects_out_of_range_number_of_returns(tmp_path: Path) -> None:
159 record = _make_record(3, seed=62)
160 record[NUMBER_OF_RETURNS_KEY] = np.array([1, -1, 3], dtype=np.int16)
161 with pytest.raises(ValueError, match="number_of_returns"):
162 save_points_npz(tmp_path / "out.npz", record)
163
164
165def test_save_points_npz_rejects_missing_number_of_returns(tmp_path: Path) -> None:
166 with pytest.raises(ValueError, match="number_of_returns"):
167 save_points_npz(tmp_path / "out.npz", _make_legacy_record(3, seed=63))
168
169
170def test_load_points_npz_fills_zeros_for_legacy_records(tmp_path: Path) -> None:
171 """Datasets written before AI3D-382 lack the key; 0 means unknown, never 1."""
172 legacy = _make_legacy_record(5, seed=64)
173 path = tmp_path / "legacy_run3_points.npz"
174 np.savez_compressed(path, **legacy)
175
176 loaded = load_points_npz(path)
177
178 assert list(loaded.keys()) == list(POINT_RECORD_KEYS)
179 returns = loaded[NUMBER_OF_RETURNS_KEY]
180 assert returns.shape == (5,)
181 assert returns.dtype == np.uint8
182 np.testing.assert_array_equal(returns, np.zeros(5, dtype=np.uint8))
183
184
185def test_load_points_npz_rejects_bad_number_of_returns_shape(tmp_path: Path) -> None:
186 record = _make_record(4, seed=65)
187 record[NUMBER_OF_RETURNS_KEY] = record[NUMBER_OF_RETURNS_KEY][:2]
188 path = tmp_path / "bad_returns.npz"
189 np.savez_compressed(path, **record)
190 with pytest.raises(
191 ValueError, match=r"'number_of_returns' must have shape \(N,\), got \(2,\)"
192 ):
193 load_points_npz(path)
194
195
196def test_load_segment_points_merges_legacy_and_new_records(tmp_path: Path) -> None:
197 legacy = _make_legacy_record(3, seed=66)
198 modern = _make_record(4, seed=67)
199 legacy_path = tmp_path / "alpha_run3_points.npz"
200 np.savez_compressed(legacy_path, **legacy)
201 modern_path = tmp_path / "beta_run3_points.npz"
202 save_points_npz(modern_path, modern)
203
204 merged, _, _ = load_segment_points([legacy_path, modern_path])
205
206 np.testing.assert_array_equal(
207 merged[NUMBER_OF_RETURNS_KEY],
208 np.concatenate(
209 [np.zeros(3, dtype=np.uint8), modern[NUMBER_OF_RETURNS_KEY]], axis=0
210 ),
211 )
212
213
214def test_concat_points_npz_carries_number_of_returns(tmp_path: Path) -> None:
215 first = _make_record(3, seed=68)
216 second = _make_legacy_record(2, seed=69)
217 save_points_npz(tmp_path / "a_run3_points.npz", first)
218 np.savez_compressed(tmp_path / "b_run3_points.npz", **second)
219
220 merged, spans = concat_points_npz(
221 [tmp_path / "a_run3_points.npz", tmp_path / "b_run3_points.npz"]
222 )
223
224 assert [span.count for span in spans] == [3, 2]
225 assert merged[NUMBER_OF_RETURNS_KEY].dtype == np.uint8
226 np.testing.assert_array_equal(
227 merged[NUMBER_OF_RETURNS_KEY],
228 np.concatenate(
229 [first[NUMBER_OF_RETURNS_KEY], np.zeros(2, dtype=np.uint8)], axis=0
230 ),
231 )
232
233
234def test_load_run3_segment_carries_number_of_returns(tmp_path: Path) -> None:
235 seg_dir = tmp_path / "segment_011"
236 seg_dir.mkdir()
237 first = _make_record(2, seed=70)
238 second = _make_record(3, seed=71)
239 save_points_npz(seg_dir / "a_run3_points.npz", first)
240 save_points_npz(seg_dir / "b_run3_points.npz", second)
241
242 merged, _ = load_run3_segment(seg_dir)
243
244 np.testing.assert_array_equal(
245 merged[NUMBER_OF_RETURNS_KEY],
246 np.concatenate(
247 [first[NUMBER_OF_RETURNS_KEY], second[NUMBER_OF_RETURNS_KEY]], axis=0
248 ),
249 )
250
251
122def test_load_segment_points_merges_and_tracks_file_ids(tmp_path: Path) -> None:252def test_load_segment_points_merges_and_tracks_file_ids(tmp_path: Path) -> None:
123 records = [_make_record(3, seed=10), _make_record(5, seed=11)]253 records = [_make_record(3, seed=10), _make_record(5, seed=11)]
124 paths = [254 paths = [
125 tmp_path / "alpha_run3_points.npz",255 tmp_path / "alpha_run3_points.npz",
Importance #7: tests/test_segment_points_io.py @@ -8,9 +8,11 @@
8import pytest8import pytest
99
10from iolabs.common.segment_points_io import (10from iolabs.common.segment_points_io import (
11 GEOSHIFT_NAME,11 GEOSHIFT_NAME,
12 NUMBER_OF_RETURNS_KEY,
12 POINT_RECORD_KEYS,13 POINT_RECORD_KEYS,
14 REQUIRED_POINT_RECORD_KEYS,
13 RecordSpan,15 RecordSpan,
14 concat_points_npz,16 concat_points_npz,
15 discover_run3_files,17 discover_run3_files,
16 filter_segment_files,18 filter_segment_files,
Importance #8: tests/test_segment_points_io.py @@ -38,11 +40,19 @@
38 "green": rng.integers(0, 65535, size=n, dtype=np.uint16),40 "green": rng.integers(0, 65535, size=n, dtype=np.uint16),
39 "blue": rng.integers(0, 65535, size=n, dtype=np.uint16),41 "blue": rng.integers(0, 65535, size=n, dtype=np.uint16),
40 "intensity": rng.integers(0, 65535, size=n, dtype=np.uint16),42 "intensity": rng.integers(0, 65535, size=n, dtype=np.uint16),
41 "scan_angle": rng.integers(-90, 90, size=n, dtype=np.int8),43 "scan_angle": rng.integers(-90, 90, size=n, dtype=np.int8),
44 "number_of_returns": rng.integers(1, 8, size=n, dtype=np.uint8),
42 }45 }
4346
4447
48def _make_legacy_record(n: int, *, seed: int = 0) -> dict[str, np.ndarray]:
49 """Build a pre-AI3D-382 record: every key except ``number_of_returns``."""
50 record = _make_record(n, seed=seed)
51 del record[NUMBER_OF_RETURNS_KEY]
52 return record
53
54
45def test_save_load_points_npz_round_trip(tmp_path: Path) -> None:55def test_save_load_points_npz_round_trip(tmp_path: Path) -> None:
46 record = _make_record(7, seed=1)56 record = _make_record(7, seed=1)
47 path = tmp_path / "scan_a_run3_points.npz"57 path = tmp_path / "scan_a_run3_points.npz"
48 save_points_npz(path, record)58 save_points_npz(path, record)
Importance #9: tests/test_segment_points_io.py @@ -118,8 +128,128 @@
118 with pytest.raises(ValueError, match="missing required key"):128 with pytest.raises(ValueError, match="missing required key"):
119 save_points_npz(tmp_path / "out.npz", {"points": np.zeros((1, 3))})129 save_points_npz(tmp_path / "out.npz", {"points": np.zeros((1, 3))})
120130
121131
132def test_number_of_returns_is_part_of_the_written_contract() -> None:
133 assert NUMBER_OF_RETURNS_KEY in POINT_RECORD_KEYS
134 assert NUMBER_OF_RETURNS_KEY not in REQUIRED_POINT_RECORD_KEYS
135 assert set(REQUIRED_POINT_RECORD_KEYS) < set(POINT_RECORD_KEYS)
136
137
138def test_save_points_npz_always_writes_number_of_returns(tmp_path: Path) -> None:
139 record = _make_record(6, seed=60)
140 path = save_points_npz(tmp_path / "with_returns_run3_points.npz", record)
141
142 with np.load(path) as data:
143 assert NUMBER_OF_RETURNS_KEY in data.files
144 stored = np.asarray(data[NUMBER_OF_RETURNS_KEY])
145 assert stored.dtype == np.uint8
146 np.testing.assert_array_equal(stored, record[NUMBER_OF_RETURNS_KEY])
147
148
149def test_save_points_npz_casts_number_of_returns_to_uint8(tmp_path: Path) -> None:
150 record = _make_record(4, seed=61)
151 record[NUMBER_OF_RETURNS_KEY] = record[NUMBER_OF_RETURNS_KEY].astype(np.int64)
152 path = save_points_npz(tmp_path / "cast_run3_points.npz", record)
153
154 loaded = load_points_npz(path)
155 assert loaded[NUMBER_OF_RETURNS_KEY].dtype == np.uint8
156
157
158def test_save_points_npz_rejects_out_of_range_number_of_returns(tmp_path: Path) -> None:
159 record = _make_record(3, seed=62)
160 record[NUMBER_OF_RETURNS_KEY] = np.array([1, -1, 3], dtype=np.int16)
161 with pytest.raises(ValueError, match="number_of_returns"):
162 save_points_npz(tmp_path / "out.npz", record)
163
164
165def test_save_points_npz_rejects_missing_number_of_returns(tmp_path: Path) -> None:
166 with pytest.raises(ValueError, match="number_of_returns"):
167 save_points_npz(tmp_path / "out.npz", _make_legacy_record(3, seed=63))
168
169
170def test_load_points_npz_fills_zeros_for_legacy_records(tmp_path: Path) -> None:
171 """Datasets written before AI3D-382 lack the key; 0 means unknown, never 1."""
172 legacy = _make_legacy_record(5, seed=64)
173 path = tmp_path / "legacy_run3_points.npz"
174 np.savez_compressed(path, **legacy)
175
176 loaded = load_points_npz(path)
177
178 assert list(loaded.keys()) == list(POINT_RECORD_KEYS)
179 returns = loaded[NUMBER_OF_RETURNS_KEY]
180 assert returns.shape == (5,)
181 assert returns.dtype == np.uint8
182 np.testing.assert_array_equal(returns, np.zeros(5, dtype=np.uint8))
183
184
185def test_load_points_npz_rejects_bad_number_of_returns_shape(tmp_path: Path) -> None:
186 record = _make_record(4, seed=65)
187 record[NUMBER_OF_RETURNS_KEY] = record[NUMBER_OF_RETURNS_KEY][:2]
188 path = tmp_path / "bad_returns.npz"
189 np.savez_compressed(path, **record)
190 with pytest.raises(
191 ValueError, match=r"'number_of_returns' must have shape \(N,\), got \(2,\)"
192 ):
193 load_points_npz(path)
194
195
196def test_load_segment_points_merges_legacy_and_new_records(tmp_path: Path) -> None:
197 legacy = _make_legacy_record(3, seed=66)
198 modern = _make_record(4, seed=67)
199 legacy_path = tmp_path / "alpha_run3_points.npz"
200 np.savez_compressed(legacy_path, **legacy)
201 modern_path = tmp_path / "beta_run3_points.npz"
202 save_points_npz(modern_path, modern)
203
204 merged, _, _ = load_segment_points([legacy_path, modern_path])
205
206 np.testing.assert_array_equal(
207 merged[NUMBER_OF_RETURNS_KEY],
208 np.concatenate(
209 [np.zeros(3, dtype=np.uint8), modern[NUMBER_OF_RETURNS_KEY]], axis=0
210 ),
211 )
212
213
214def test_concat_points_npz_carries_number_of_returns(tmp_path: Path) -> None:
215 first = _make_record(3, seed=68)
216 second = _make_legacy_record(2, seed=69)
217 save_points_npz(tmp_path / "a_run3_points.npz", first)
218 np.savez_compressed(tmp_path / "b_run3_points.npz", **second)
219
220 merged, spans = concat_points_npz(
221 [tmp_path / "a_run3_points.npz", tmp_path / "b_run3_points.npz"]
222 )
223
224 assert [span.count for span in spans] == [3, 2]
225 assert merged[NUMBER_OF_RETURNS_KEY].dtype == np.uint8
226 np.testing.assert_array_equal(
227 merged[NUMBER_OF_RETURNS_KEY],
228 np.concatenate(
229 [first[NUMBER_OF_RETURNS_KEY], np.zeros(2, dtype=np.uint8)], axis=0
230 ),
231 )
232
233
234def test_load_run3_segment_carries_number_of_returns(tmp_path: Path) -> None:
235 seg_dir = tmp_path / "segment_011"
236 seg_dir.mkdir()
237 first = _make_record(2, seed=70)
238 second = _make_record(3, seed=71)
239 save_points_npz(seg_dir / "a_run3_points.npz", first)
240 save_points_npz(seg_dir / "b_run3_points.npz", second)
241
242 merged, _ = load_run3_segment(seg_dir)
243
244 np.testing.assert_array_equal(
245 merged[NUMBER_OF_RETURNS_KEY],
246 np.concatenate(
247 [first[NUMBER_OF_RETURNS_KEY], second[NUMBER_OF_RETURNS_KEY]], axis=0
248 ),
249 )
250
251
122def test_load_segment_points_merges_and_tracks_file_ids(tmp_path: Path) -> None:252def test_load_segment_points_merges_and_tracks_file_ids(tmp_path: Path) -> None:
123 records = [_make_record(3, seed=10), _make_record(5, seed=11)]253 records = [_make_record(3, seed=10), _make_record(5, seed=11)]
124 paths = [254 paths = [
125 tmp_path / "alpha_run3_points.npz",255 tmp_path / "alpha_run3_points.npz",