Back to report index

iolabs-common (shared config layer) 14f041e: AI3D-382 Make ColorIntensityData.number_of_returns keyword-only and uint8-enforced

Miroslav Simko <ms@iolabs.ch> 2026-09-01T23:39:27+02:00

Commit #91 ยท 15 snippets

 src/iolabs/common/_dtype_coercion.py      |  64 +++++++++++++
 src/iolabs/common/color_intensity_data.py |  53 ++++++++--
 src/iolabs/common/segment_points_io.py    |  40 +++-----
 tests/test_color_intensity_data.py        | 154 ++++++++++++++++++++++++++++++
 4 files changed, 274 insertions(+), 37 deletions(-)
Importance #1: src/iolabs/common/_dtype_coercion.py @@ -0,0 +1,64 @@
1"""Shared storage-dtype coercion for per-point arrays (AI3D-382).
2
3One rule set, used by both ends of the ``number_of_returns`` contract:
4:mod:`iolabs.common.segment_points_io` applies it to NPZ members on save and
5load, and :class:`iolabs.common.color_intensity_data.ColorIntensityData`
6applies it to the array a caller hands the constructor. Keeping it here rather
7than in either module avoids a duplicate rule drifting out of step, and avoids
8making the in-memory data class depend on the file-format module.
9
10Deliberately spec-free and schema-free: it takes the target dtype and the words
11for the error message, nothing else.
12"""
13
14import numpy as np
15
16
17def coerce_storage_dtype(
18 values: np.ndarray, *, dtype: np.dtype | type, key: str, noun: str, source: str
19) -> np.ndarray:
20 """Cast an array to its declared storage dtype, rejecting lossy inputs.
21
22 For an integer target dtype the input must itself be integral -- a bool
23 array is a mask, and a float array is not a count -- and must fit the target
24 range, because ``astype`` would otherwise wrap silently.
25
26 Args:
27 values: Array-like as the producer supplied it or the file stored it.
28 dtype: Target storage dtype.
29 key: Field or member name, used in error messages.
30 noun: Singular name of one stored value, used in error messages
31 ("a boolean is a mask, not a return count").
32 source: Label used in error messages (typically the target path or the
33 owning class name).
34
35 Returns:
36 The values as *dtype*. When they already are it the array is returned
37 as-is, so an already-conforming array is never copied.
38
39 Raises:
40 ValueError: The target dtype is integral and the values are boolean,
41 non-integral, or outside that dtype's range.
42 """
43 array = np.asarray(values)
44 target = np.dtype(dtype)
45 if array.dtype == target:
46 return array
47 if target.kind in "ui":
48 if array.dtype.kind == "b":
49 raise ValueError(
50 f"{source}: '{key}' must be an integer array, got a bool array; a boolean "
51 f"is a mask, not a {noun} (casting it would fabricate {noun}s of "
52 "0 and 1)"
53 )
54 if array.dtype.kind not in "ui":
55 raise ValueError(
56 f"{source}: '{key}' must be an integer array, got dtype {array.dtype}"
57 )
58 info = np.iinfo(target)
59 if array.size and (int(array.min()) < info.min or int(array.max()) > info.max):
60 raise ValueError(
61 f"{source}: '{key}' values must fit in {target.name}, got range "
62 f"[{int(array.min())}, {int(array.max())}]"
63 )
64 return array.astype(target)
0
Importance #2: src/iolabs/common/color_intensity_data.py @@ -8,16 +8,23 @@
8constructor could not accept it.8constructor could not accept it.
9"""9"""
1010
11from collections.abc import Callable11from collections.abc import Callable
12from dataclasses import dataclass, fields12from dataclasses import dataclass, field, fields
13from typing import TypeVar13from typing import TypeVar
1414
15import numpy as np15import numpy as np
1616
17#: Storage dtype of :attr:`ColorIntensityData.number_of_returns` (LAS values 1-7).17from . import _dtype_coercion
18
19#: Storage dtype of :attr:`ColorIntensityData.number_of_returns`. LAS point
20#: formats 0-5 carry a 3-bit count (1-7) and formats 6-10 a 4-bit one (1-15),
21#: so uint8 holds every legal value; no narrower value domain is enforced.
18NUMBER_OF_RETURNS_DTYPE = np.uint822NUMBER_OF_RETURNS_DTYPE = np.uint8
1923
24#: Name of the return-count field, used in the constructor's dtype errors.
25NUMBER_OF_RETURNS_FIELD = "number_of_returns"
26
20#: Bound to the concrete type the transforms are called on: both rebuild27#: Bound to the concrete type the transforms are called on: both rebuild
21#: ``type(self)``, so a subclass stays its own type through them.28#: ``type(self)``, so a subclass stays its own type through them.
22ColorIntensityDataT = TypeVar("ColorIntensityDataT", bound="ColorIntensityData")29ColorIntensityDataT = TypeVar("ColorIntensityDataT", bound="ColorIntensityData")
2330
Importance #3: src/iolabs/common/color_intensity_data.py @@ -32,37 +39,63 @@
32 blue: Per-point blue channel.39 blue: Per-point blue channel.
33 intensity: Per-point LAS intensity.40 intensity: Per-point LAS intensity.
34 scan_angle_rank: Per-point LAS scan angle.41 scan_angle_rank: Per-point LAS scan angle.
35 number_of_returns: Per-point LAS return count as uint8, added by42 number_of_returns: Per-point LAS return count as uint8, added by
36 AI3D-382. Callers that have no such data omit it and get zeros:43 AI3D-382. Keyword-only, so a subclass may still declare a required
37 0 is not a legal LAS return count, so it reads as "unknown".44 (non-defaulted) field of its own without hitting "non-default
38 Never substitute 1 -- that fabricates a measurement. Non-``None``45 argument follows default argument". Callers that have no such data
39 after construction.46 omit it and get zeros: 0 is not a legal LAS return count, so it
47 reads as "unknown". Never substitute 1 -- that fabricates a
48 measurement. A supplied array is coerced to uint8 and must be an
49 integer array in range; ``None`` after construction is impossible.
40 """50 """
4151
42 red: np.ndarray52 red: np.ndarray
43 green: np.ndarray53 green: np.ndarray
44 blue: np.ndarray54 blue: np.ndarray
45 intensity: np.ndarray55 intensity: np.ndarray
46 scan_angle_rank: np.ndarray56 scan_angle_rank: np.ndarray
47 number_of_returns: np.ndarray | None = None57 number_of_returns: np.ndarray | None = field(default=None, kw_only=True)
4858
49 def __post_init__(self) -> None:59 def __post_init__(self) -> None:
50 """Fill an omitted ``number_of_returns`` with zeros (unknown) per point."""60 """Zero-fill an omitted ``number_of_returns``, or coerce a supplied one to uint8.
61
62 The coercion is the same rule :mod:`iolabs.common.segment_points_io`
63 applies when the array reaches an NPZ, so a producer cannot hold a
64 wider (or boolean) return count in memory and only discover it at save
65 time. An array that is already uint8 is kept as-is, not copied.
66
67 Raises:
68 ValueError: ``number_of_returns`` is a bool array (a mask is not a
69 count), a non-integer array, or holds values outside uint8.
70 """
51 if self.number_of_returns is None:71 if self.number_of_returns is None:
52 self.number_of_returns = np.zeros(len(self.red), dtype=NUMBER_OF_RETURNS_DTYPE)72 self.number_of_returns = np.zeros(len(self.red), dtype=NUMBER_OF_RETURNS_DTYPE)
73 return
74 self.number_of_returns = _dtype_coercion.coerce_storage_dtype(
75 self.number_of_returns,
76 dtype=NUMBER_OF_RETURNS_DTYPE,
77 key=NUMBER_OF_RETURNS_FIELD,
78 noun="return count",
79 source=type(self).__name__,
80 )
5381
54 def _map_fields(82 def _map_fields(
55 self: ColorIntensityDataT, transform: Callable[[str], np.ndarray]83 self: ColorIntensityDataT, transform: Callable[[str], np.ndarray]
56 ) -> ColorIntensityDataT:84 ) -> ColorIntensityDataT:
57 """Rebuild this instance's type by applying *transform* to every field name.85 """Rebuild this instance's type by applying *transform* to every field name.
5886
59 Only ``init=True`` fields are passed to the constructor: a subclass may87 Only ``init=True`` fields are passed to the constructor: a subclass may
60 declare a derived ``field(init=False)`` attribute, which the constructor88 declare a derived ``field(init=False)`` attribute, which the constructor
61 would reject as an unexpected keyword.89 would reject as an unexpected keyword. Everything is passed by keyword,
90 so a keyword-only field (``number_of_returns``) rides along unchanged.
62 """91 """
63 return type(self)(92 return type(self)(
64 **{field.name: transform(field.name) for field in fields(self) if field.init}93 **{
94 data_field.name: transform(data_field.name)
95 for data_field in fields(self)
96 if data_field.init
97 }
65 )98 )
6699
67 def select_by_mask(self: ColorIntensityDataT, mask: np.ndarray) -> ColorIntensityDataT:100 def select_by_mask(self: ColorIntensityDataT, mask: np.ndarray) -> ColorIntensityDataT:
68 """Return a new instance of this type holding only points where mask is True."""101 """Return a new instance of this type holding only points where mask is True."""
Importance #4: src/iolabs/common/segment_points_io.py @@ -46,8 +46,10 @@
46from types import MappingProxyType46from types import MappingProxyType
4747
48import numpy as np48import numpy as np
4949
50from . import _dtype_coercion
51
50logger = logging.getLogger(__name__)52logger = logging.getLogger(__name__)
5153
52#: Builds a member for a record that predates its key, given the point count.54#: Builds a member for a record that predates its key, given the point count.
53FillFactory = Callable[[int], np.ndarray]55FillFactory = Callable[[int], np.ndarray]
Importance #5: src/iolabs/common/segment_points_io.py @@ -153,14 +155,15 @@
153155
154def _coerce_storage_dtype(156def _coerce_storage_dtype(
155 values: np.ndarray, *, key: str, spec: PointFieldSpec, source: str157 values: np.ndarray, *, key: str, spec: PointFieldSpec, source: str
156) -> np.ndarray:158) -> np.ndarray:
157 """Cast one member to its declared storage dtype, rejecting lossy inputs.159 """Cast one member to its spec's storage dtype, rejecting lossy inputs.
158160
159 Keys whose spec declares no storage dtype are passed through untouched. For161 Keys whose spec declares no storage dtype are passed through untouched;
160 an integer storage dtype the input must itself be integral -- a bool array162 the rest are handed to :func:`iolabs.common._dtype_coercion.coerce_storage_dtype`,
161 is a mask, and a float array is not a count -- and must fit the target163 which is the one implementation of the rule (shared with
162 range, because ``astype`` would otherwise wrap silently.164 :class:`iolabs.common.color_intensity_data.ColorIntensityData`, so the array
165 a producer holds in memory and the bytes it writes obey the same contract).
163166
164 Args:167 Args:
165 values: Array-like member as the producer supplied or the file stored it.168 values: Array-like member as the producer supplied or the file stored it.
166 key: Member name, used in error messages.169 key: Member name, used in error messages.
Importance #6: src/iolabs/common/segment_points_io.py @@ -174,30 +177,13 @@
174 Raises:177 Raises:
175 ValueError: The declared dtype is integral and the values are boolean,178 ValueError: The declared dtype is integral and the values are boolean,
176 non-integral, or outside the target dtype's range.179 non-integral, or outside the target dtype's range.
177 """180 """
178 array = np.asarray(values)181 if spec.storage_dtype is None:
179 dtype = spec.storage_dtype182 return np.asarray(values)
180 if dtype is None or array.dtype == dtype:183 return _dtype_coercion.coerce_storage_dtype(
181 return array184 values, dtype=spec.storage_dtype, key=key, noun=spec.noun, source=source
182 if dtype.kind in "ui":185 )
183 if array.dtype.kind == "b":
184 raise ValueError(
185 f"{source}: '{key}' must be an integer array, got a bool array; a boolean "
186 f"is a mask, not a {spec.noun} (casting it would fabricate {spec.noun}s of "
187 "0 and 1)"
188 )
189 if array.dtype.kind not in "ui":
190 raise ValueError(
191 f"{source}: '{key}' must be an integer array, got dtype {array.dtype}"
192 )
193 info = np.iinfo(dtype)
194 if array.size and (int(array.min()) < info.min or int(array.max()) > info.max):
195 raise ValueError(
196 f"{source}: '{key}' values must fit in {dtype.name}, got range "
197 f"[{int(array.min())}, {int(array.max())}]"
198 )
199 return array.astype(dtype)
200186
201187
202def _shape_text(spec: PointFieldSpec) -> str:188def _shape_text(spec: PointFieldSpec) -> str:
203 """Render a spec's expected shape for an error message."""189 """Render a spec's expected shape for an error message."""
Importance #7: tests/test_color_intensity_data.py @@ -1,8 +1,9 @@
1"""Tests for ColorIntensityData selection and concatenation operations."""1"""Tests for ColorIntensityData selection and concatenation operations."""
2from dataclasses import dataclass, field2from dataclasses import dataclass, field
33
4import numpy as np4import numpy as np
5import pytest
56
6from iolabs.common.color_intensity_data import ColorIntensityData7from iolabs.common.color_intensity_data import ColorIntensityData
78
89
Importance #8: tests/test_color_intensity_data.py @@ -186,4 +187,157 @@
186 assert isinstance(masked, WithPointCount)187 assert isinstance(masked, WithPointCount)
187 assert masked.point_count == 2188 assert masked.point_count == 2
188 assert merged.point_count == 4189 assert merged.point_count == 4
189 np.testing.assert_array_equal(masked.red, first.red[np.array([True, False, True, False])])190 np.testing.assert_array_equal(masked.red, first.red[np.array([True, False, True, False])])
191
192
193class TestKeywordOnlyReturnCount:
194 """number_of_returns is kw_only so it does not poison subclass field order."""
195
196 def test_subclass_may_declare_a_required_field(self):
197 """A trailing defaulted field would make a required subclass field a TypeError."""
198
199 @dataclass
200 class WithRequiredClassification(ColorIntensityData):
201 classification: np.ndarray
202
203 data = WithRequiredClassification(
204 red=np.zeros(3, dtype=np.uint8),
205 green=np.zeros(3, dtype=np.uint8),
206 blue=np.zeros(3, dtype=np.uint8),
207 intensity=np.zeros(3, dtype=np.float64),
208 scan_angle_rank=np.zeros(3, dtype=np.float64),
209 classification=np.arange(3, dtype=np.uint8),
210 )
211
212 assert data.number_of_returns.dtype == np.uint8
213 np.testing.assert_array_equal(data.number_of_returns, np.zeros(3, dtype=np.uint8))
214
215 merged = data.select_by_mask(np.array([True, False, True])).append(data)
216 assert isinstance(merged, WithRequiredClassification)
217 assert len(merged.classification) == 5
218
219 def test_pre_ai3d_382_fields_still_take_positional_args(self):
220 """The five original fields keep their positional order for old call sites."""
221 data = ColorIntensityData(
222 np.zeros(2, dtype=np.uint8),
223 np.zeros(2, dtype=np.uint8),
224 np.zeros(2, dtype=np.uint8),
225 np.zeros(2, dtype=np.float64),
226 np.zeros(2, dtype=np.float64),
227 )
228
229 assert len(data.number_of_returns) == 2
230
231 def test_number_of_returns_is_not_positional(self):
232 """Passing it as a sixth positional arg is a TypeError, not a silent mismatch."""
233 with pytest.raises(TypeError):
234 ColorIntensityData(
235 np.zeros(2, dtype=np.uint8),
236 np.zeros(2, dtype=np.uint8),
237 np.zeros(2, dtype=np.uint8),
238 np.zeros(2, dtype=np.float64),
239 np.zeros(2, dtype=np.float64),
240 np.ones(2, dtype=np.uint8),
241 )
242
243
244class TestNumberOfReturnsDtypeContract:
245 """The constructor enforces the same uint8 contract as segment_points_io."""
246
247 @staticmethod
248 def _make(number_of_returns: np.ndarray) -> ColorIntensityData:
249 n = len(number_of_returns)
250 return ColorIntensityData(
251 red=np.zeros(n, dtype=np.uint8),
252 green=np.zeros(n, dtype=np.uint8),
253 blue=np.zeros(n, dtype=np.uint8),
254 intensity=np.zeros(n, dtype=np.float64),
255 scan_angle_rank=np.zeros(n, dtype=np.float64),
256 number_of_returns=number_of_returns,
257 )
258
259 def test_bool_array_is_rejected(self):
260 """A mask is not a count: casting it would fabricate counts of 0 and 1."""
261 with pytest.raises(ValueError, match="mask, not a return count"):
262 self._make(np.array([True, False, True]))
263
264 def test_float_array_is_rejected(self):
265 """A float array is not a count either."""
266 with pytest.raises(ValueError, match="must be an integer array"):
267 self._make(np.array([1.0, 2.0, 3.0]))
268
269 def test_in_range_int64_is_coerced_to_uint8(self):
270 """A wider integer array inside uint8 range is narrowed, values intact."""
271 data = self._make(np.array([1, 7, 15, 255], dtype=np.int64))
272
273 assert data.number_of_returns.dtype == np.uint8
274 np.testing.assert_array_equal(
275 data.number_of_returns, np.array([1, 7, 15, 255], dtype=np.uint8)
276 )
277
278 def test_values_above_255_raise_instead_of_wrapping(self):
279 """astype would silently wrap 256 to 0 -- the "unknown" sentinel."""
280 with pytest.raises(ValueError, match="must fit in uint8"):
281 self._make(np.array([1, 256], dtype=np.int64))
282
283 def test_negative_values_raise(self):
284 """A negative count cannot be stored unsigned."""
285 with pytest.raises(ValueError, match="must fit in uint8"):
286 self._make(np.array([-1, 2], dtype=np.int64))
287
288 def test_uint8_input_is_not_copied(self):
289 """An already-conforming array is stored as-is, no per-construction copy."""
290 counts = np.array([1, 2, 3], dtype=np.uint8)
291 data = self._make(counts)
292
293 assert data.number_of_returns is counts
294
295 def test_empty_array_is_accepted(self):
296 """An empty integer array has no values to range-check."""
297 data = self._make(np.array([], dtype=np.int64))
298
299 assert data.number_of_returns.dtype == np.uint8
300 assert len(data.number_of_returns) == 0
301
302 def test_mask_and_append_do_not_recopy_or_re_raise(self):
303 """The transforms rebuild instances, so coercion runs again -- on uint8 input."""
304 data = self._make(np.array([1, 2, 3, 4], dtype=np.uint8))
305
306 masked = data.select_by_mask(np.array([True, False, True, False]))
307 merged = masked.append(masked)
308
309 assert masked.number_of_returns.dtype == np.uint8
310 assert merged.number_of_returns.dtype == np.uint8
311 np.testing.assert_array_equal(
312 merged.number_of_returns, np.array([1, 3, 1, 3], dtype=np.uint8)
313 )
314
315 @pytest.mark.parametrize(
316 "counts",
317 [
318 pytest.param(np.array([True, False]), id="bool"),
319 pytest.param(np.array([1.0, 2.0]), id="float"),
320 pytest.param(np.array([1, 256], dtype=np.int64), id="out-of-range"),
321 ],
322 )
323 def test_rejects_exactly_what_the_npz_writer_rejects(self, tmp_path, counts):
324 """One rule, two doors: the constructor and save_points_npz agree."""
325 from iolabs.common import segment_points_io
326
327 record = {
328 "points": np.zeros((len(counts), 3), dtype=np.float64),
329 "red": np.zeros(len(counts), dtype=np.uint8),
330 "green": np.zeros(len(counts), dtype=np.uint8),
331 "blue": np.zeros(len(counts), dtype=np.uint8),
332 "intensity": np.zeros(len(counts), dtype=np.float64),
333 "scan_angle": np.zeros(len(counts), dtype=np.float64),
334 "number_of_returns": counts,
335 }
336 with pytest.raises(ValueError) as from_writer:
337 segment_points_io.save_points_npz(tmp_path / "p.npz", record)
338 with pytest.raises(ValueError) as from_constructor:
339 self._make(counts)
340
341 assert str(from_constructor.value).split(": ", 1)[1] == str(
342 from_writer.value
343 ).split(": ", 1)[1]
Importance #9: src/iolabs/common/color_intensity_data.py @@ -8,16 +8,23 @@
8constructor could not accept it.8constructor could not accept it.
9"""9"""
1010
11from collections.abc import Callable11from collections.abc import Callable
12from dataclasses import dataclass, fields12from dataclasses import dataclass, field, fields
13from typing import TypeVar13from typing import TypeVar
1414
15import numpy as np15import numpy as np
1616
17#: Storage dtype of :attr:`ColorIntensityData.number_of_returns` (LAS values 1-7).17from . import _dtype_coercion
18
19#: Storage dtype of :attr:`ColorIntensityData.number_of_returns`. LAS point
20#: formats 0-5 carry a 3-bit count (1-7) and formats 6-10 a 4-bit one (1-15),
21#: so uint8 holds every legal value; no narrower value domain is enforced.
18NUMBER_OF_RETURNS_DTYPE = np.uint822NUMBER_OF_RETURNS_DTYPE = np.uint8
1923
24#: Name of the return-count field, used in the constructor's dtype errors.
25NUMBER_OF_RETURNS_FIELD = "number_of_returns"
26
20#: Bound to the concrete type the transforms are called on: both rebuild27#: Bound to the concrete type the transforms are called on: both rebuild
21#: ``type(self)``, so a subclass stays its own type through them.28#: ``type(self)``, so a subclass stays its own type through them.
22ColorIntensityDataT = TypeVar("ColorIntensityDataT", bound="ColorIntensityData")29ColorIntensityDataT = TypeVar("ColorIntensityDataT", bound="ColorIntensityData")
2330
Importance #10: src/iolabs/common/color_intensity_data.py @@ -32,37 +39,63 @@
32 blue: Per-point blue channel.39 blue: Per-point blue channel.
33 intensity: Per-point LAS intensity.40 intensity: Per-point LAS intensity.
34 scan_angle_rank: Per-point LAS scan angle.41 scan_angle_rank: Per-point LAS scan angle.
35 number_of_returns: Per-point LAS return count as uint8, added by42 number_of_returns: Per-point LAS return count as uint8, added by
36 AI3D-382. Callers that have no such data omit it and get zeros:43 AI3D-382. Keyword-only, so a subclass may still declare a required
37 0 is not a legal LAS return count, so it reads as "unknown".44 (non-defaulted) field of its own without hitting "non-default
38 Never substitute 1 -- that fabricates a measurement. Non-``None``45 argument follows default argument". Callers that have no such data
39 after construction.46 omit it and get zeros: 0 is not a legal LAS return count, so it
47 reads as "unknown". Never substitute 1 -- that fabricates a
48 measurement. A supplied array is coerced to uint8 and must be an
49 integer array in range; ``None`` after construction is impossible.
40 """50 """
4151
42 red: np.ndarray52 red: np.ndarray
43 green: np.ndarray53 green: np.ndarray
44 blue: np.ndarray54 blue: np.ndarray
45 intensity: np.ndarray55 intensity: np.ndarray
46 scan_angle_rank: np.ndarray56 scan_angle_rank: np.ndarray
47 number_of_returns: np.ndarray | None = None57 number_of_returns: np.ndarray | None = field(default=None, kw_only=True)
4858
49 def __post_init__(self) -> None:59 def __post_init__(self) -> None:
50 """Fill an omitted ``number_of_returns`` with zeros (unknown) per point."""60 """Zero-fill an omitted ``number_of_returns``, or coerce a supplied one to uint8.
61
62 The coercion is the same rule :mod:`iolabs.common.segment_points_io`
63 applies when the array reaches an NPZ, so a producer cannot hold a
64 wider (or boolean) return count in memory and only discover it at save
65 time. An array that is already uint8 is kept as-is, not copied.
66
67 Raises:
68 ValueError: ``number_of_returns`` is a bool array (a mask is not a
69 count), a non-integer array, or holds values outside uint8.
70 """
51 if self.number_of_returns is None:71 if self.number_of_returns is None:
52 self.number_of_returns = np.zeros(len(self.red), dtype=NUMBER_OF_RETURNS_DTYPE)72 self.number_of_returns = np.zeros(len(self.red), dtype=NUMBER_OF_RETURNS_DTYPE)
73 return
74 self.number_of_returns = _dtype_coercion.coerce_storage_dtype(
75 self.number_of_returns,
76 dtype=NUMBER_OF_RETURNS_DTYPE,
77 key=NUMBER_OF_RETURNS_FIELD,
78 noun="return count",
79 source=type(self).__name__,
80 )
5381
54 def _map_fields(82 def _map_fields(
55 self: ColorIntensityDataT, transform: Callable[[str], np.ndarray]83 self: ColorIntensityDataT, transform: Callable[[str], np.ndarray]
56 ) -> ColorIntensityDataT:84 ) -> ColorIntensityDataT:
57 """Rebuild this instance's type by applying *transform* to every field name.85 """Rebuild this instance's type by applying *transform* to every field name.
5886
59 Only ``init=True`` fields are passed to the constructor: a subclass may87 Only ``init=True`` fields are passed to the constructor: a subclass may
60 declare a derived ``field(init=False)`` attribute, which the constructor88 declare a derived ``field(init=False)`` attribute, which the constructor
61 would reject as an unexpected keyword.89 would reject as an unexpected keyword. Everything is passed by keyword,
90 so a keyword-only field (``number_of_returns``) rides along unchanged.
62 """91 """
63 return type(self)(92 return type(self)(
64 **{field.name: transform(field.name) for field in fields(self) if field.init}93 **{
94 data_field.name: transform(data_field.name)
95 for data_field in fields(self)
96 if data_field.init
97 }
65 )98 )
6699
67 def select_by_mask(self: ColorIntensityDataT, mask: np.ndarray) -> ColorIntensityDataT:100 def select_by_mask(self: ColorIntensityDataT, mask: np.ndarray) -> ColorIntensityDataT:
68 """Return a new instance of this type holding only points where mask is True."""101 """Return a new instance of this type holding only points where mask is True."""
Importance #11: src/iolabs/common/segment_points_io.py @@ -46,8 +46,10 @@
46from types import MappingProxyType46from types import MappingProxyType
4747
48import numpy as np48import numpy as np
4949
50from . import _dtype_coercion
51
50logger = logging.getLogger(__name__)52logger = logging.getLogger(__name__)
5153
52#: Builds a member for a record that predates its key, given the point count.54#: Builds a member for a record that predates its key, given the point count.
53FillFactory = Callable[[int], np.ndarray]55FillFactory = Callable[[int], np.ndarray]
Importance #12: src/iolabs/common/segment_points_io.py @@ -153,14 +155,15 @@
153155
154def _coerce_storage_dtype(156def _coerce_storage_dtype(
155 values: np.ndarray, *, key: str, spec: PointFieldSpec, source: str157 values: np.ndarray, *, key: str, spec: PointFieldSpec, source: str
156) -> np.ndarray:158) -> np.ndarray:
157 """Cast one member to its declared storage dtype, rejecting lossy inputs.159 """Cast one member to its spec's storage dtype, rejecting lossy inputs.
158160
159 Keys whose spec declares no storage dtype are passed through untouched. For161 Keys whose spec declares no storage dtype are passed through untouched;
160 an integer storage dtype the input must itself be integral -- a bool array162 the rest are handed to :func:`iolabs.common._dtype_coercion.coerce_storage_dtype`,
161 is a mask, and a float array is not a count -- and must fit the target163 which is the one implementation of the rule (shared with
162 range, because ``astype`` would otherwise wrap silently.164 :class:`iolabs.common.color_intensity_data.ColorIntensityData`, so the array
165 a producer holds in memory and the bytes it writes obey the same contract).
163166
164 Args:167 Args:
165 values: Array-like member as the producer supplied or the file stored it.168 values: Array-like member as the producer supplied or the file stored it.
166 key: Member name, used in error messages.169 key: Member name, used in error messages.
Importance #13: src/iolabs/common/segment_points_io.py @@ -174,30 +177,13 @@
174 Raises:177 Raises:
175 ValueError: The declared dtype is integral and the values are boolean,178 ValueError: The declared dtype is integral and the values are boolean,
176 non-integral, or outside the target dtype's range.179 non-integral, or outside the target dtype's range.
177 """180 """
178 array = np.asarray(values)181 if spec.storage_dtype is None:
179 dtype = spec.storage_dtype182 return np.asarray(values)
180 if dtype is None or array.dtype == dtype:183 return _dtype_coercion.coerce_storage_dtype(
181 return array184 values, dtype=spec.storage_dtype, key=key, noun=spec.noun, source=source
182 if dtype.kind in "ui":185 )
183 if array.dtype.kind == "b":
184 raise ValueError(
185 f"{source}: '{key}' must be an integer array, got a bool array; a boolean "
186 f"is a mask, not a {spec.noun} (casting it would fabricate {spec.noun}s of "
187 "0 and 1)"
188 )
189 if array.dtype.kind not in "ui":
190 raise ValueError(
191 f"{source}: '{key}' must be an integer array, got dtype {array.dtype}"
192 )
193 info = np.iinfo(dtype)
194 if array.size and (int(array.min()) < info.min or int(array.max()) > info.max):
195 raise ValueError(
196 f"{source}: '{key}' values must fit in {dtype.name}, got range "
197 f"[{int(array.min())}, {int(array.max())}]"
198 )
199 return array.astype(dtype)
200186
201187
202def _shape_text(spec: PointFieldSpec) -> str:188def _shape_text(spec: PointFieldSpec) -> str:
203 """Render a spec's expected shape for an error message."""189 """Render a spec's expected shape for an error message."""
Importance #14: tests/test_color_intensity_data.py @@ -1,8 +1,9 @@
1"""Tests for ColorIntensityData selection and concatenation operations."""1"""Tests for ColorIntensityData selection and concatenation operations."""
2from dataclasses import dataclass, field2from dataclasses import dataclass, field
33
4import numpy as np4import numpy as np
5import pytest
56
6from iolabs.common.color_intensity_data import ColorIntensityData7from iolabs.common.color_intensity_data import ColorIntensityData
78
89
Importance #15: tests/test_color_intensity_data.py @@ -186,4 +187,157 @@
186 assert isinstance(masked, WithPointCount)187 assert isinstance(masked, WithPointCount)
187 assert masked.point_count == 2188 assert masked.point_count == 2
188 assert merged.point_count == 4189 assert merged.point_count == 4
189 np.testing.assert_array_equal(masked.red, first.red[np.array([True, False, True, False])])190 np.testing.assert_array_equal(masked.red, first.red[np.array([True, False, True, False])])
191
192
193class TestKeywordOnlyReturnCount:
194 """number_of_returns is kw_only so it does not poison subclass field order."""
195
196 def test_subclass_may_declare_a_required_field(self):
197 """A trailing defaulted field would make a required subclass field a TypeError."""
198
199 @dataclass
200 class WithRequiredClassification(ColorIntensityData):
201 classification: np.ndarray
202
203 data = WithRequiredClassification(
204 red=np.zeros(3, dtype=np.uint8),
205 green=np.zeros(3, dtype=np.uint8),
206 blue=np.zeros(3, dtype=np.uint8),
207 intensity=np.zeros(3, dtype=np.float64),
208 scan_angle_rank=np.zeros(3, dtype=np.float64),
209 classification=np.arange(3, dtype=np.uint8),
210 )
211
212 assert data.number_of_returns.dtype == np.uint8
213 np.testing.assert_array_equal(data.number_of_returns, np.zeros(3, dtype=np.uint8))
214
215 merged = data.select_by_mask(np.array([True, False, True])).append(data)
216 assert isinstance(merged, WithRequiredClassification)
217 assert len(merged.classification) == 5
218
219 def test_pre_ai3d_382_fields_still_take_positional_args(self):
220 """The five original fields keep their positional order for old call sites."""
221 data = ColorIntensityData(
222 np.zeros(2, dtype=np.uint8),
223 np.zeros(2, dtype=np.uint8),
224 np.zeros(2, dtype=np.uint8),
225 np.zeros(2, dtype=np.float64),
226 np.zeros(2, dtype=np.float64),
227 )
228
229 assert len(data.number_of_returns) == 2
230
231 def test_number_of_returns_is_not_positional(self):
232 """Passing it as a sixth positional arg is a TypeError, not a silent mismatch."""
233 with pytest.raises(TypeError):
234 ColorIntensityData(
235 np.zeros(2, dtype=np.uint8),
236 np.zeros(2, dtype=np.uint8),
237 np.zeros(2, dtype=np.uint8),
238 np.zeros(2, dtype=np.float64),
239 np.zeros(2, dtype=np.float64),
240 np.ones(2, dtype=np.uint8),
241 )
242
243
244class TestNumberOfReturnsDtypeContract:
245 """The constructor enforces the same uint8 contract as segment_points_io."""
246
247 @staticmethod
248 def _make(number_of_returns: np.ndarray) -> ColorIntensityData:
249 n = len(number_of_returns)
250 return ColorIntensityData(
251 red=np.zeros(n, dtype=np.uint8),
252 green=np.zeros(n, dtype=np.uint8),
253 blue=np.zeros(n, dtype=np.uint8),
254 intensity=np.zeros(n, dtype=np.float64),
255 scan_angle_rank=np.zeros(n, dtype=np.float64),
256 number_of_returns=number_of_returns,
257 )
258
259 def test_bool_array_is_rejected(self):
260 """A mask is not a count: casting it would fabricate counts of 0 and 1."""
261 with pytest.raises(ValueError, match="mask, not a return count"):
262 self._make(np.array([True, False, True]))
263
264 def test_float_array_is_rejected(self):
265 """A float array is not a count either."""
266 with pytest.raises(ValueError, match="must be an integer array"):
267 self._make(np.array([1.0, 2.0, 3.0]))
268
269 def test_in_range_int64_is_coerced_to_uint8(self):
270 """A wider integer array inside uint8 range is narrowed, values intact."""
271 data = self._make(np.array([1, 7, 15, 255], dtype=np.int64))
272
273 assert data.number_of_returns.dtype == np.uint8
274 np.testing.assert_array_equal(
275 data.number_of_returns, np.array([1, 7, 15, 255], dtype=np.uint8)
276 )
277
278 def test_values_above_255_raise_instead_of_wrapping(self):
279 """astype would silently wrap 256 to 0 -- the "unknown" sentinel."""
280 with pytest.raises(ValueError, match="must fit in uint8"):
281 self._make(np.array([1, 256], dtype=np.int64))
282
283 def test_negative_values_raise(self):
284 """A negative count cannot be stored unsigned."""
285 with pytest.raises(ValueError, match="must fit in uint8"):
286 self._make(np.array([-1, 2], dtype=np.int64))
287
288 def test_uint8_input_is_not_copied(self):
289 """An already-conforming array is stored as-is, no per-construction copy."""
290 counts = np.array([1, 2, 3], dtype=np.uint8)
291 data = self._make(counts)
292
293 assert data.number_of_returns is counts
294
295 def test_empty_array_is_accepted(self):
296 """An empty integer array has no values to range-check."""
297 data = self._make(np.array([], dtype=np.int64))
298
299 assert data.number_of_returns.dtype == np.uint8
300 assert len(data.number_of_returns) == 0
301
302 def test_mask_and_append_do_not_recopy_or_re_raise(self):
303 """The transforms rebuild instances, so coercion runs again -- on uint8 input."""
304 data = self._make(np.array([1, 2, 3, 4], dtype=np.uint8))
305
306 masked = data.select_by_mask(np.array([True, False, True, False]))
307 merged = masked.append(masked)
308
309 assert masked.number_of_returns.dtype == np.uint8
310 assert merged.number_of_returns.dtype == np.uint8
311 np.testing.assert_array_equal(
312 merged.number_of_returns, np.array([1, 3, 1, 3], dtype=np.uint8)
313 )
314
315 @pytest.mark.parametrize(
316 "counts",
317 [
318 pytest.param(np.array([True, False]), id="bool"),
319 pytest.param(np.array([1.0, 2.0]), id="float"),
320 pytest.param(np.array([1, 256], dtype=np.int64), id="out-of-range"),
321 ],
322 )
323 def test_rejects_exactly_what_the_npz_writer_rejects(self, tmp_path, counts):
324 """One rule, two doors: the constructor and save_points_npz agree."""
325 from iolabs.common import segment_points_io
326
327 record = {
328 "points": np.zeros((len(counts), 3), dtype=np.float64),
329 "red": np.zeros(len(counts), dtype=np.uint8),
330 "green": np.zeros(len(counts), dtype=np.uint8),
331 "blue": np.zeros(len(counts), dtype=np.uint8),
332 "intensity": np.zeros(len(counts), dtype=np.float64),
333 "scan_angle": np.zeros(len(counts), dtype=np.float64),
334 "number_of_returns": counts,
335 }
336 with pytest.raises(ValueError) as from_writer:
337 segment_points_io.save_points_npz(tmp_path / "p.npz", record)
338 with pytest.raises(ValueError) as from_constructor:
339 self._make(counts)
340
341 assert str(from_constructor.value).split(": ", 1)[1] == str(
342 from_writer.value
343 ).split(": ", 1)[1]