Back to report index

iolabs-common (shared config layer) cbbb210: AI3D-382 Harden generic record helpers and field mapping

Miroslav Simko <ms@iolabs.ch> 2026-09-01T15:08:19+02:00

Commit #89 ยท 12 snippets

 src/iolabs/common/color_intensity_data.py | 33 ++++++++++++++++++++++---------
 src/iolabs/common/segment_points_io.py    | 19 +++++++++++++++---
 tests/test_color_intensity_data.py        | 33 ++++++++++++++++++++++++++++++-
 tests/test_segment_points_io.py           | 14 +++++++++++++
 4 files changed, 86 insertions(+), 13 deletions(-)
Importance #1: src/iolabs/common/color_intensity_data.py @@ -1,19 +1,27 @@
1"""Color, intensity, scan angle, and return-count data aligned with point clouds.1"""Color, intensity, scan angle, and return-count data aligned with point clouds.
22
3Both transforms below (:meth:`ColorIntensityData.select_by_mask` and3Both transforms below (:meth:`ColorIntensityData.select_by_mask` and
4:meth:`ColorIntensityData.append`) walk :func:`dataclasses.fields`, so a new4:meth:`ColorIntensityData.append`) walk :func:`dataclasses.fields`, so a new
5per-point attribute is carried through them by declaring the field alone.5per-point attribute is carried through them by declaring the field alone. Only
6constructor (``init=True``) fields are mapped; a subclass's ``init=False``
7field is left to that subclass's ``__post_init__`` to derive, as the
8constructor could not accept it.
6"""9"""
710
8from collections.abc import Callable11from collections.abc import Callable
9from dataclasses import dataclass, fields12from dataclasses import dataclass, fields
13from typing import TypeVar
1014
11import numpy as np15import numpy as np
1216
13#: Storage dtype of :attr:`ColorIntensityData.number_of_returns` (LAS values 1-7).17#: Storage dtype of :attr:`ColorIntensityData.number_of_returns` (LAS values 1-7).
14NUMBER_OF_RETURNS_DTYPE = np.uint818NUMBER_OF_RETURNS_DTYPE = np.uint8
1519
20#: Bound to the concrete type the transforms are called on: both rebuild
21#: ``type(self)``, so a subclass stays its own type through them.
22ColorIntensityDataT = TypeVar("ColorIntensityDataT", bound="ColorIntensityData")
23
1624
17@dataclass25@dataclass
18class ColorIntensityData:26class ColorIntensityData:
19 """Stores per-point RGB, intensity, scan angle, and return-count arrays.27 """Stores per-point RGB, intensity, scan angle, and return-count arrays.
Importance #2: src/iolabs/common/color_intensity_data.py @@ -43,18 +51,25 @@
43 if self.number_of_returns is None:51 if self.number_of_returns is None:
44 self.number_of_returns = np.zeros(len(self.red), dtype=NUMBER_OF_RETURNS_DTYPE)52 self.number_of_returns = np.zeros(len(self.red), dtype=NUMBER_OF_RETURNS_DTYPE)
4553
46 def _map_fields(54 def _map_fields(
47 self, transform: Callable[[str], np.ndarray]55 self: ColorIntensityDataT, transform: Callable[[str], np.ndarray]
48 ) -> "ColorIntensityData":56 ) -> ColorIntensityDataT:
49 """Rebuild this instance's type by applying *transform* to every field name."""57 """Rebuild this instance's type by applying *transform* to every field name.
50 return type(self)(**{field.name: transform(field.name) for field in fields(self)})58
59 Only ``init=True`` fields are passed to the constructor: a subclass may
60 declare a derived ``field(init=False)`` attribute, which the constructor
61 would reject as an unexpected keyword.
62 """
63 return type(self)(
64 **{field.name: transform(field.name) for field in fields(self) if field.init}
65 )
5166
52 def select_by_mask(self, mask: np.ndarray) -> "ColorIntensityData":67 def select_by_mask(self: ColorIntensityDataT, mask: np.ndarray) -> ColorIntensityDataT:
53 """Return a new ColorIntensityData containing only points where mask is True."""68 """Return a new instance of this type holding only points where mask is True."""
54 return self._map_fields(lambda name: getattr(self, name)[mask])69 return self._map_fields(lambda name: getattr(self, name)[mask])
5570
56 def append(self, other: "ColorIntensityData") -> "ColorIntensityData":71 def append(self: ColorIntensityDataT, other: "ColorIntensityData") -> ColorIntensityDataT:
57 """Concatenate this instance with another, returning a new ColorIntensityData."""72 """Concatenate this instance with another, returning a new instance of this type."""
58 return self._map_fields(73 return self._map_fields(
59 lambda name: np.concatenate([getattr(self, name), getattr(other, name)])74 lambda name: np.concatenate([getattr(self, name), getattr(other, name)])
60 )75 )
Importance #3: src/iolabs/common/segment_points_io.py @@ -346,14 +346,24 @@
346 Returns:346 Returns:
347 A new record with the same keys, each member indexed by *mask*.347 A new record with the same keys, each member indexed by *mask*.
348348
349 Raises:349 Raises:
350 ValueError: The record is empty or its members disagree on point count.350 ValueError: The record is empty, carries a member that is not a 1-D or
351 2-D array (a 0-D member has no rows to select and would fail with a
352 bare ``IndexError``), or its members disagree on point count.
351 """353 """
352 if not record:354 if not record:
353 raise ValueError("Cannot mask an empty point record")355 raise ValueError("Cannot mask an empty point record")
354 arrays = {key: np.asarray(value) for key, value in record.items()}356 arrays = {key: np.asarray(value) for key, value in record.items()}
355 counts = {key: array.shape[0] if array.ndim else -1 for key, array in arrays.items()}357 bad_shapes = {
358 key: array.shape for key, array in arrays.items() if array.ndim not in (1, 2)
359 }
360 if bad_shapes:
361 raise ValueError(
362 "Cannot mask a point record whose members are not 1-D or 2-D arrays "
363 f"indexable along axis 0: {bad_shapes}"
364 )
365 counts = {key: array.shape[0] for key, array in arrays.items()}
356 if len(set(counts.values())) > 1:366 if len(set(counts.values())) > 1:
357 raise ValueError(367 raise ValueError(
358 f"Cannot mask a point record whose members disagree on point count: {counts}"368 f"Cannot mask a point record whose members disagree on point count: {counts}"
359 )369 )
Importance #4: src/iolabs/common/segment_points_io.py @@ -375,13 +385,16 @@
375 One record holding the concatenated members, keyed in the first385 One record holding the concatenated members, keyed in the first
376 record's order.386 record's order.
377387
378 Raises:388 Raises:
379 ValueError: *records* is empty or the records' key sets differ.389 ValueError: *records* is empty, the records carry no members at all, or
390 the records' key sets differ.
380 """391 """
381 if not records:392 if not records:
382 raise ValueError("Cannot concatenate an empty sequence of point records")393 raise ValueError("Cannot concatenate an empty sequence of point records")
383 keys = list(records[0])394 keys = list(records[0])
395 if not keys:
396 raise ValueError("Cannot concatenate empty point records: record 0 has no members")
384 for index, other in enumerate(records[1:], start=1):397 for index, other in enumerate(records[1:], start=1):
385 if set(other) != set(keys):398 if set(other) != set(keys):
386 raise ValueError(399 raise ValueError(
387 f"Cannot concatenate point records with different keys: record 0 has "400 f"Cannot concatenate point records with different keys: record 0 has "
Importance #5: tests/test_color_intensity_data.py @@ -1,6 +1,6 @@
1"""Tests for ColorIntensityData selection and concatenation operations."""1"""Tests for ColorIntensityData selection and concatenation operations."""
2from dataclasses import dataclass2from dataclasses import dataclass, field
33
4import numpy as np4import numpy as np
55
6from iolabs.common.color_intensity_data import ColorIntensityData6from iolabs.common.color_intensity_data import ColorIntensityData
Importance #6: tests/test_color_intensity_data.py @@ -155,4 +155,35 @@
155 np.testing.assert_array_equal(155 np.testing.assert_array_equal(
156 merged.number_of_returns,156 merged.number_of_returns,
157 np.concatenate([first.number_of_returns[mask], second.number_of_returns]),157 np.concatenate([first.number_of_returns[mask], second.number_of_returns]),
158 )158 )
159
160 def test_non_init_subclass_field_is_not_passed_to_the_constructor(self):
161 """A derived ``init=False`` field must not break the generic transforms."""
162
163 @dataclass
164 class WithPointCount(ColorIntensityData):
165 point_count: int = field(init=False, default=0)
166
167 def __post_init__(self) -> None:
168 super().__post_init__()
169 self.point_count = len(self.red)
170
171 def _make(n: int, offset: int) -> WithPointCount:
172 base = _make_sample(n, offset)
173 return WithPointCount(
174 red=base.red,
175 green=base.green,
176 blue=base.blue,
177 intensity=base.intensity,
178 scan_angle_rank=base.scan_angle_rank,
179 number_of_returns=base.number_of_returns,
180 )
181
182 first = _make(4, 0)
183 masked = first.select_by_mask(np.array([True, False, True, False]))
184 merged = masked.append(_make(2, 50))
185
186 assert isinstance(masked, WithPointCount)
187 assert masked.point_count == 2
188 assert merged.point_count == 4
189 np.testing.assert_array_equal(masked.red, first.red[np.array([True, False, True, False])])
Importance #7: tests/test_segment_points_io.py @@ -815,8 +815,22 @@
815 with pytest.raises(ValueError, match="empty sequence"):815 with pytest.raises(ValueError, match="empty sequence"):
816 concat_records([])816 concat_records([])
817817
818818
819def test_concat_records_rejects_records_without_members() -> None:
820 """Memberless records must raise, not silently concatenate to ``{}``."""
821 with pytest.raises(ValueError, match="no members"):
822 concat_records([{}, {}])
823
824
825def test_mask_record_rejects_zero_dimensional_members() -> None:
826 """A 0-D member is not row-indexable: raise ValueError, not a raw IndexError."""
827 record = {key: np.asarray(1, dtype=np.uint8) for key in POINT_RECORD_KEYS}
828
829 with pytest.raises(ValueError, match="not 1-D or 2-D"):
830 mask_record(record, np.array([True]))
831
832
819# --- Adding a field to the contract must be a registry entry and nothing else ---833# --- Adding a field to the contract must be a registry entry and nothing else ---
820834
821EXTRA_KEY = "point_source_id"835EXTRA_KEY = "point_source_id"
822836
Importance #8: src/iolabs/common/segment_points_io.py @@ -346,14 +346,24 @@
346 Returns:346 Returns:
347 A new record with the same keys, each member indexed by *mask*.347 A new record with the same keys, each member indexed by *mask*.
348348
349 Raises:349 Raises:
350 ValueError: The record is empty or its members disagree on point count.350 ValueError: The record is empty, carries a member that is not a 1-D or
351 2-D array (a 0-D member has no rows to select and would fail with a
352 bare ``IndexError``), or its members disagree on point count.
351 """353 """
352 if not record:354 if not record:
353 raise ValueError("Cannot mask an empty point record")355 raise ValueError("Cannot mask an empty point record")
354 arrays = {key: np.asarray(value) for key, value in record.items()}356 arrays = {key: np.asarray(value) for key, value in record.items()}
355 counts = {key: array.shape[0] if array.ndim else -1 for key, array in arrays.items()}357 bad_shapes = {
358 key: array.shape for key, array in arrays.items() if array.ndim not in (1, 2)
359 }
360 if bad_shapes:
361 raise ValueError(
362 "Cannot mask a point record whose members are not 1-D or 2-D arrays "
363 f"indexable along axis 0: {bad_shapes}"
364 )
365 counts = {key: array.shape[0] for key, array in arrays.items()}
356 if len(set(counts.values())) > 1:366 if len(set(counts.values())) > 1:
357 raise ValueError(367 raise ValueError(
358 f"Cannot mask a point record whose members disagree on point count: {counts}"368 f"Cannot mask a point record whose members disagree on point count: {counts}"
359 )369 )
Importance #9: src/iolabs/common/segment_points_io.py @@ -375,13 +385,16 @@
375 One record holding the concatenated members, keyed in the first385 One record holding the concatenated members, keyed in the first
376 record's order.386 record's order.
377387
378 Raises:388 Raises:
379 ValueError: *records* is empty or the records' key sets differ.389 ValueError: *records* is empty, the records carry no members at all, or
390 the records' key sets differ.
380 """391 """
381 if not records:392 if not records:
382 raise ValueError("Cannot concatenate an empty sequence of point records")393 raise ValueError("Cannot concatenate an empty sequence of point records")
383 keys = list(records[0])394 keys = list(records[0])
395 if not keys:
396 raise ValueError("Cannot concatenate empty point records: record 0 has no members")
384 for index, other in enumerate(records[1:], start=1):397 for index, other in enumerate(records[1:], start=1):
385 if set(other) != set(keys):398 if set(other) != set(keys):
386 raise ValueError(399 raise ValueError(
387 f"Cannot concatenate point records with different keys: record 0 has "400 f"Cannot concatenate point records with different keys: record 0 has "
Importance #10: tests/test_color_intensity_data.py @@ -1,6 +1,6 @@
1"""Tests for ColorIntensityData selection and concatenation operations."""1"""Tests for ColorIntensityData selection and concatenation operations."""
2from dataclasses import dataclass2from dataclasses import dataclass, field
33
4import numpy as np4import numpy as np
55
6from iolabs.common.color_intensity_data import ColorIntensityData6from iolabs.common.color_intensity_data import ColorIntensityData
Importance #11: tests/test_color_intensity_data.py @@ -155,4 +155,35 @@
155 np.testing.assert_array_equal(155 np.testing.assert_array_equal(
156 merged.number_of_returns,156 merged.number_of_returns,
157 np.concatenate([first.number_of_returns[mask], second.number_of_returns]),157 np.concatenate([first.number_of_returns[mask], second.number_of_returns]),
158 )158 )
159
160 def test_non_init_subclass_field_is_not_passed_to_the_constructor(self):
161 """A derived ``init=False`` field must not break the generic transforms."""
162
163 @dataclass
164 class WithPointCount(ColorIntensityData):
165 point_count: int = field(init=False, default=0)
166
167 def __post_init__(self) -> None:
168 super().__post_init__()
169 self.point_count = len(self.red)
170
171 def _make(n: int, offset: int) -> WithPointCount:
172 base = _make_sample(n, offset)
173 return WithPointCount(
174 red=base.red,
175 green=base.green,
176 blue=base.blue,
177 intensity=base.intensity,
178 scan_angle_rank=base.scan_angle_rank,
179 number_of_returns=base.number_of_returns,
180 )
181
182 first = _make(4, 0)
183 masked = first.select_by_mask(np.array([True, False, True, False]))
184 merged = masked.append(_make(2, 50))
185
186 assert isinstance(masked, WithPointCount)
187 assert masked.point_count == 2
188 assert merged.point_count == 4
189 np.testing.assert_array_equal(masked.red, first.red[np.array([True, False, True, False])])
Importance #12: tests/test_segment_points_io.py @@ -815,8 +815,22 @@
815 with pytest.raises(ValueError, match="empty sequence"):815 with pytest.raises(ValueError, match="empty sequence"):
816 concat_records([])816 concat_records([])
817817
818818
819def test_concat_records_rejects_records_without_members() -> None:
820 """Memberless records must raise, not silently concatenate to ``{}``."""
821 with pytest.raises(ValueError, match="no members"):
822 concat_records([{}, {}])
823
824
825def test_mask_record_rejects_zero_dimensional_members() -> None:
826 """A 0-D member is not row-indexable: raise ValueError, not a raw IndexError."""
827 record = {key: np.asarray(1, dtype=np.uint8) for key in POINT_RECORD_KEYS}
828
829 with pytest.raises(ValueError, match="not 1-D or 2-D"):
830 mask_record(record, np.array([True]))
831
832
819# --- Adding a field to the contract must be a registry entry and nothing else ---833# --- Adding a field to the contract must be a registry entry and nothing else ---
820834
821EXTRA_KEY = "point_source_id"835EXTRA_KEY = "point_source_id"
822836