Back to report index

Step 6 maskclustering 2a79902: AI3D-382 Forward extra ColorIntensityData channels into cluster NPZs

Miroslav Simko <ms@iolabs.ch> 2026-09-01T23:36:12+02:00

Commit #114 ยท 9 snippets

 .../cluster_io.py                                  |  68 ++++++++++++-
 tests/test_cluster_io.py                           | 112 +++++++++++++++++++++
 2 files changed, 178 insertions(+), 2 deletions(-)
Importance #1: src/iolabs_point_cloud_mask_clustering/cluster_io.py @@ -1,25 +1,86 @@
1"""Write Step 6-compatible cluster NPZ artifacts."""1"""Write Step 6-compatible cluster NPZ artifacts.
22
3The five colour/intensity channels Step 6 has always required are written
4explicitly, with their fixed output names and storage dtypes. Every *other*
5``ColorIntensityData`` field is forwarded generically (see
6:func:`_extra_channel_arrays`), so a per-point channel added to that dataclass
7-- ``number_of_returns`` in AI3D-382, for instance -- reaches the cluster NPZ
8without an edit here. Against an ``iolabs-common`` whose ``ColorIntensityData``
9declares only the five explicit fields, the generic set is empty and the
10archive is byte-identical to the pre-AI3D-382 output.
11"""
12
13import dataclasses
3from pathlib import Path14from pathlib import Path
415
5import numpy as np16import numpy as np
6from iolabs.common.atomic_io import atomic_savez17from iolabs.common.atomic_io import atomic_savez
18from iolabs.common.color_intensity_data import ColorIntensityData
7from iolabs.logstash import get_props_logger19from iolabs.logstash import get_props_logger
820
9from ._log_props import LOG_PROPS21from ._log_props import LOG_PROPS
10from .types import PointChannels, SegmentType22from .types import PointChannels, SegmentType
1123
12logger = get_props_logger(__name__, LOG_PROPS)24logger = get_props_logger(__name__, LOG_PROPS)
1325
26#: ``ColorIntensityData`` fields :func:`write_cluster_npz` writes itself, with
27#: their own dtype handling; they must not be forwarded a second time.
28EXPLICIT_CHANNEL_FIELDS: frozenset[str] = frozenset(
29 {"red", "green", "blue", "intensity", "scan_angle_rank"}
30)
31
32#: NPZ member names :func:`write_cluster_npz` always writes. A forwarded field
33#: whose name lands here would silently overwrite a fixed member, so it is
34#: dropped with a warning instead.
35FIXED_MEMBER_NAMES: frozenset[str] = frozenset(
36 {"points", "segment_type", "scan_angle", "intensity", "red", "green", "blue"}
37)
38
1439
15def _round_clip(values: np.ndarray, dtype: np.dtype) -> np.ndarray:40def _round_clip(values: np.ndarray, dtype: np.dtype) -> np.ndarray:
16 info = np.iinfo(dtype)41 info = np.iinfo(dtype)
17 return np.clip(42 return np.clip(
18 np.rint(np.asarray(values, dtype=np.float64)), info.min, info.max43 np.rint(np.asarray(values, dtype=np.float64)), info.min, info.max
19 ).astype(dtype)44 ).astype(dtype)
2045
2146
47def _extra_channel_arrays(data: ColorIntensityData) -> dict[str, np.ndarray]:
48 """Return the channels to forward verbatim, keyed by NPZ member name.
49
50 The installed dataclass is inspected with :func:`dataclasses.fields` rather
51 than matched against a version, so this works on any ``iolabs-common``:
52 fields the installed release does not declare simply do not appear. Only
53 constructor (``init=True``) fields are considered -- a derived
54 ``field(init=False)`` attribute is not point data. Each array is passed on
55 with its own dtype; the explicit members own the round-and-clip casts.
56
57 Args:
58 data: Aligned per-point channels from the loaded Step 3 record.
59
60 Returns:
61 Member name -> array, for every ``init=True`` field that is neither
62 written explicitly nor ``None``. Names colliding with a fixed member
63 are dropped and logged.
64 """
65 extras: dict[str, np.ndarray] = {}
66 for field in dataclasses.fields(type(data)):
67 if not field.init or field.name in EXPLICIT_CHANNEL_FIELDS:
68 continue
69 value = getattr(data, field.name, None)
70 if value is None:
71 continue
72 if field.name in FIXED_MEMBER_NAMES:
73 logger.warning(
74 "Skipping ColorIntensityData field %r: the name collides with a "
75 "fixed cluster NPZ member",
76 field.name,
77 )
78 continue
79 extras[field.name] = np.asarray(value)
80 return extras
81
82
22def write_cluster_npz(83def write_cluster_npz(
23 output_path: Path,84 output_path: Path,
24 channels: PointChannels,85 channels: PointChannels,
25 segment_type: SegmentType,86 segment_type: SegmentType,
Importance #2: src/iolabs_point_cloud_mask_clustering/cluster_io.py @@ -31,9 +92,11 @@
31 not switch this to ``compress=True``.92 not switch this to ``compress=True``.
3293
33 Args:94 Args:
34 output_path: Target ``.npz`` path; parent directories are created.95 output_path: Target ``.npz`` path; parent directories are created.
35 channels: Step 3 XYZ plus the aligned colour/intensity channels.96 channels: Step 3 XYZ plus the aligned colour/intensity channels. Any
97 channel beyond the five explicit ones is forwarded under its field
98 name, with its own dtype.
36 segment_type: Road-marking class recorded alongside the points.99 segment_type: Road-marking class recorded alongside the points.
37 """100 """
38 atomic_savez(101 atomic_savez(
39 output_path,102 output_path,
Importance #3: src/iolabs_point_cloud_mask_clustering/cluster_io.py @@ -44,5 +107,6 @@
44 red=_round_clip(channels.data.red, np.dtype(np.uint16)),107 red=_round_clip(channels.data.red, np.dtype(np.uint16)),
45 green=_round_clip(channels.data.green, np.dtype(np.uint16)),108 green=_round_clip(channels.data.green, np.dtype(np.uint16)),
46 blue=_round_clip(channels.data.blue, np.dtype(np.uint16)),109 blue=_round_clip(channels.data.blue, np.dtype(np.uint16)),
47 segment_type=np.asarray(segment_type.value),110 segment_type=np.asarray(segment_type.value),
111 **_extra_channel_arrays(channels.data),
48 )112 )
Importance #4: tests/test_cluster_io.py @@ -1,4 +1,6 @@
1import dataclasses
2import logging
1import zipfile3import zipfile
2from pathlib import Path4from pathlib import Path
3from typing import BinaryIO5from typing import BinaryIO
46
Importance #5: tests/test_cluster_io.py @@ -7,11 +9,57 @@
7from iolabs.common import atomic_io9from iolabs.common import atomic_io
8from iolabs.common.color_intensity_data import ColorIntensityData10from iolabs.common.color_intensity_data import ColorIntensityData
9from iolabs_point_cloud_filtering_clusters.clustering_gpu_io import load_cluster_artifact_npz11from iolabs_point_cloud_filtering_clusters.clustering_gpu_io import load_cluster_artifact_npz
1012
13from iolabs_point_cloud_mask_clustering import cluster_io
11from iolabs_point_cloud_mask_clustering.cluster_io import write_cluster_npz14from iolabs_point_cloud_mask_clustering.cluster_io import write_cluster_npz
12from iolabs_point_cloud_mask_clustering.types import PointChannels, SegmentType15from iolabs_point_cloud_mask_clustering.types import PointChannels, SegmentType
1316
17FIXED_MEMBERS = {
18 "points",
19 "scan_angle",
20 "intensity",
21 "red",
22 "green",
23 "blue",
24 "segment_type",
25}
26
27
28@dataclasses.dataclass
29class _FiveChannels:
30 """Stand-in for a ColorIntensityData that declares only the explicit five."""
31
32 red: np.ndarray
33 green: np.ndarray
34 blue: np.ndarray
35 intensity: np.ndarray
36 scan_angle_rank: np.ndarray
37
38
39@dataclasses.dataclass
40class _CollidingChannels(_FiveChannels):
41 """Stand-in whose extra field name collides with a fixed NPZ member."""
42
43 scan_angle: np.ndarray = dataclasses.field(default_factory=lambda: np.array([7, 8]))
44
45
46def _installed_channel_fields() -> set[str]:
47 return {field.name for field in dataclasses.fields(ColorIntensityData)}
48
49
50def _stub_channels(stub: type) -> PointChannels:
51 return PointChannels(
52 points=np.array([[1.123456789, 2, 3], [4, 5, 6]], dtype=np.float64),
53 data=stub(
54 red=np.array([1.6, 2.4]),
55 green=np.array([3.5, 4.5]),
56 blue=np.array([5.49, 6.51]),
57 intensity=np.array([-1.0, 70000.0]),
58 scan_angle_rank=np.array([-200.1, 127.4]),
59 ),
60 )
61
1462
15def _sample() -> PointChannels:63def _sample() -> PointChannels:
16 return PointChannels(64 return PointChannels(
17 points=np.array([[1.123456789, 2, 3], [4, 5, 6]], dtype=np.float64),65 points=np.array([[1.123456789, 2, 3], [4, 5, 6]], dtype=np.float64),
Importance #6: tests/test_cluster_io.py @@ -53,8 +101,72 @@
53 assert methods101 assert methods
54 assert set(methods.values()) == {zipfile.ZIP_STORED}102 assert set(methods.values()) == {zipfile.ZIP_STORED}
55103
56104
105def test_five_field_channels_write_only_the_fixed_members(tmp_path: Path) -> None:
106 # A ColorIntensityData declaring only the explicit five (iolabs-common 0.7.0)
107 # must produce exactly the pre-AI3D-382 member set: no generic extras.
108 path = tmp_path / "run6_cluster_000.npz"
109 write_cluster_npz(path, _stub_channels(_FiveChannels), SegmentType.SOLID)
110 with np.load(path) as payload:
111 assert set(payload.files) == FIXED_MEMBERS
112 assert payload["scan_angle"].dtype == np.int8
113 np.testing.assert_array_equal(payload["scan_angle"], [-128, 127])
114
115
116def test_member_set_follows_the_installed_channel_schema(tmp_path: Path) -> None:
117 # Holds on any iolabs-common: the extras are exactly the installed
118 # dataclass's non-explicit init fields.
119 path = tmp_path / "run6_cluster_000.npz"
120 write_cluster_npz(path, _sample(), SegmentType.SOLID)
121 expected_extras = _installed_channel_fields() - cluster_io.EXPLICIT_CHANNEL_FIELDS
122 with np.load(path) as payload:
123 assert set(payload.files) == FIXED_MEMBERS | expected_extras
124
125
126@pytest.mark.skipif(
127 "number_of_returns" not in _installed_channel_fields(),
128 reason="installed ColorIntensityData predates number_of_returns",
129)
130def test_number_of_returns_is_forwarded_aligned_with_points(tmp_path: Path) -> None:
131 source = _sample()
132 source.data.number_of_returns = np.array([1, 3], dtype=np.uint8)
133 path = tmp_path / "run6_cluster_000.npz"
134 write_cluster_npz(path, source, SegmentType.DASHED)
135 with np.load(path) as payload:
136 returns = payload["number_of_returns"]
137 assert returns.dtype == np.uint8
138 assert len(returns) == len(source.points)
139 np.testing.assert_array_equal(returns, [1, 3])
140
141
142@pytest.mark.skipif(
143 "number_of_returns" not in _installed_channel_fields(),
144 reason="installed ColorIntensityData predates number_of_returns",
145)
146def test_omitted_number_of_returns_is_forwarded_as_unknown_zeros(tmp_path: Path) -> None:
147 path = tmp_path / "run6_cluster_000.npz"
148 write_cluster_npz(path, _sample(), SegmentType.DASHED)
149 with np.load(path) as payload:
150 returns = payload["number_of_returns"]
151 assert returns.dtype == np.uint8
152 np.testing.assert_array_equal(returns, [0, 0])
153
154
155def test_field_colliding_with_a_fixed_member_is_skipped_with_a_warning(
156 tmp_path: Path, caplog: pytest.LogCaptureFixture
157) -> None:
158 path = tmp_path / "run6_cluster_000.npz"
159 with caplog.at_level(logging.WARNING, logger=cluster_io.__name__):
160 write_cluster_npz(path, _stub_channels(_CollidingChannels), SegmentType.SOLID)
161 assert any("scan_angle" in record.getMessage() for record in caplog.records)
162 with np.load(path) as payload:
163 assert set(payload.files) == FIXED_MEMBERS
164 # The fixed member survived: it is the clipped scan_angle_rank, not [7, 8].
165 assert payload["scan_angle"].dtype == np.int8
166 np.testing.assert_array_equal(payload["scan_angle"], [-128, 127])
167
168
57def test_atomic_failure_leaves_no_partial_output(169def test_atomic_failure_leaves_no_partial_output(
58 tmp_path: Path, monkeypatch: pytest.MonkeyPatch170 tmp_path: Path, monkeypatch: pytest.MonkeyPatch
59) -> None:171) -> None:
60 path = tmp_path / "run6_cluster_000.npz"172 path = tmp_path / "run6_cluster_000.npz"
Importance #7: tests/test_cluster_io.py @@ -1,4 +1,6 @@
1import dataclasses
2import logging
1import zipfile3import zipfile
2from pathlib import Path4from pathlib import Path
3from typing import BinaryIO5from typing import BinaryIO
46
Importance #8: tests/test_cluster_io.py @@ -7,11 +9,57 @@
7from iolabs.common import atomic_io9from iolabs.common import atomic_io
8from iolabs.common.color_intensity_data import ColorIntensityData10from iolabs.common.color_intensity_data import ColorIntensityData
9from iolabs_point_cloud_filtering_clusters.clustering_gpu_io import load_cluster_artifact_npz11from iolabs_point_cloud_filtering_clusters.clustering_gpu_io import load_cluster_artifact_npz
1012
13from iolabs_point_cloud_mask_clustering import cluster_io
11from iolabs_point_cloud_mask_clustering.cluster_io import write_cluster_npz14from iolabs_point_cloud_mask_clustering.cluster_io import write_cluster_npz
12from iolabs_point_cloud_mask_clustering.types import PointChannels, SegmentType15from iolabs_point_cloud_mask_clustering.types import PointChannels, SegmentType
1316
17FIXED_MEMBERS = {
18 "points",
19 "scan_angle",
20 "intensity",
21 "red",
22 "green",
23 "blue",
24 "segment_type",
25}
26
27
28@dataclasses.dataclass
29class _FiveChannels:
30 """Stand-in for a ColorIntensityData that declares only the explicit five."""
31
32 red: np.ndarray
33 green: np.ndarray
34 blue: np.ndarray
35 intensity: np.ndarray
36 scan_angle_rank: np.ndarray
37
38
39@dataclasses.dataclass
40class _CollidingChannels(_FiveChannels):
41 """Stand-in whose extra field name collides with a fixed NPZ member."""
42
43 scan_angle: np.ndarray = dataclasses.field(default_factory=lambda: np.array([7, 8]))
44
45
46def _installed_channel_fields() -> set[str]:
47 return {field.name for field in dataclasses.fields(ColorIntensityData)}
48
49
50def _stub_channels(stub: type) -> PointChannels:
51 return PointChannels(
52 points=np.array([[1.123456789, 2, 3], [4, 5, 6]], dtype=np.float64),
53 data=stub(
54 red=np.array([1.6, 2.4]),
55 green=np.array([3.5, 4.5]),
56 blue=np.array([5.49, 6.51]),
57 intensity=np.array([-1.0, 70000.0]),
58 scan_angle_rank=np.array([-200.1, 127.4]),
59 ),
60 )
61
1462
15def _sample() -> PointChannels:63def _sample() -> PointChannels:
16 return PointChannels(64 return PointChannels(
17 points=np.array([[1.123456789, 2, 3], [4, 5, 6]], dtype=np.float64),65 points=np.array([[1.123456789, 2, 3], [4, 5, 6]], dtype=np.float64),
Importance #9: tests/test_cluster_io.py @@ -53,8 +101,72 @@
53 assert methods101 assert methods
54 assert set(methods.values()) == {zipfile.ZIP_STORED}102 assert set(methods.values()) == {zipfile.ZIP_STORED}
55103
56104
105def test_five_field_channels_write_only_the_fixed_members(tmp_path: Path) -> None:
106 # A ColorIntensityData declaring only the explicit five (iolabs-common 0.7.0)
107 # must produce exactly the pre-AI3D-382 member set: no generic extras.
108 path = tmp_path / "run6_cluster_000.npz"
109 write_cluster_npz(path, _stub_channels(_FiveChannels), SegmentType.SOLID)
110 with np.load(path) as payload:
111 assert set(payload.files) == FIXED_MEMBERS
112 assert payload["scan_angle"].dtype == np.int8
113 np.testing.assert_array_equal(payload["scan_angle"], [-128, 127])
114
115
116def test_member_set_follows_the_installed_channel_schema(tmp_path: Path) -> None:
117 # Holds on any iolabs-common: the extras are exactly the installed
118 # dataclass's non-explicit init fields.
119 path = tmp_path / "run6_cluster_000.npz"
120 write_cluster_npz(path, _sample(), SegmentType.SOLID)
121 expected_extras = _installed_channel_fields() - cluster_io.EXPLICIT_CHANNEL_FIELDS
122 with np.load(path) as payload:
123 assert set(payload.files) == FIXED_MEMBERS | expected_extras
124
125
126@pytest.mark.skipif(
127 "number_of_returns" not in _installed_channel_fields(),
128 reason="installed ColorIntensityData predates number_of_returns",
129)
130def test_number_of_returns_is_forwarded_aligned_with_points(tmp_path: Path) -> None:
131 source = _sample()
132 source.data.number_of_returns = np.array([1, 3], dtype=np.uint8)
133 path = tmp_path / "run6_cluster_000.npz"
134 write_cluster_npz(path, source, SegmentType.DASHED)
135 with np.load(path) as payload:
136 returns = payload["number_of_returns"]
137 assert returns.dtype == np.uint8
138 assert len(returns) == len(source.points)
139 np.testing.assert_array_equal(returns, [1, 3])
140
141
142@pytest.mark.skipif(
143 "number_of_returns" not in _installed_channel_fields(),
144 reason="installed ColorIntensityData predates number_of_returns",
145)
146def test_omitted_number_of_returns_is_forwarded_as_unknown_zeros(tmp_path: Path) -> None:
147 path = tmp_path / "run6_cluster_000.npz"
148 write_cluster_npz(path, _sample(), SegmentType.DASHED)
149 with np.load(path) as payload:
150 returns = payload["number_of_returns"]
151 assert returns.dtype == np.uint8
152 np.testing.assert_array_equal(returns, [0, 0])
153
154
155def test_field_colliding_with_a_fixed_member_is_skipped_with_a_warning(
156 tmp_path: Path, caplog: pytest.LogCaptureFixture
157) -> None:
158 path = tmp_path / "run6_cluster_000.npz"
159 with caplog.at_level(logging.WARNING, logger=cluster_io.__name__):
160 write_cluster_npz(path, _stub_channels(_CollidingChannels), SegmentType.SOLID)
161 assert any("scan_angle" in record.getMessage() for record in caplog.records)
162 with np.load(path) as payload:
163 assert set(payload.files) == FIXED_MEMBERS
164 # The fixed member survived: it is the clipped scan_angle_rank, not [7, 8].
165 assert payload["scan_angle"].dtype == np.int8
166 np.testing.assert_array_equal(payload["scan_angle"], [-128, 127])
167
168
57def test_atomic_failure_leaves_no_partial_output(169def test_atomic_failure_leaves_no_partial_output(
58 tmp_path: Path, monkeypatch: pytest.MonkeyPatch170 tmp_path: Path, monkeypatch: pytest.MonkeyPatch
59) -> None:171) -> None:
60 path = tmp_path / "run6_cluster_000.npz"172 path = tmp_path / "run6_cluster_000.npz"