Miroslav Simko <ms@iolabs.ch> 2026-09-01T10:58:14+02:00
Commit #4 ยท 8 snippets
scripts/s3_segment_mapper/s3_segment_mapper.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+)
_chunk_number_of_returns helper in the orchestrator's range-split writer; untested (no pytest infra there).
| 1083 | ) | 1083 | ) |
| 1084 | return mapper | 1084 | return mapper |
| 1085 | 1085 | ||
| 1086 | 1086 | ||
| 1087 | #: Storage dtype for the run3 NPZ ``number_of_returns`` member (AI3D-382); LAS | ||
| 1088 | #: carries the field in 3 bits (valid values 1-7), so uint8 stores it exactly. | ||
| 1089 | NUMBER_OF_RETURNS_DTYPE = np.uint8 | ||
| 1090 | |||
| 1091 | |||
| 1092 | def _chunk_number_of_returns(chunk: Any, point_count: int) -> np.ndarray: | ||
| 1093 | """Return per-point LAS return counts as uint8, zero-filled when absent. | ||
| 1094 | |||
| 1095 | Mirrors ``SegmentMapper._chunk_scan_angle``'s field-fallback pattern: a | ||
| 1096 | chunk without ``number_of_returns`` degrades to zeros (read downstream as | ||
| 1097 | "unknown") instead of raising. Never substitute 1 -- that fabricates data. | ||
| 1098 | """ | ||
| 1099 | if hasattr(chunk, "number_of_returns"): | ||
| 1100 | return np.asarray(chunk.number_of_returns, dtype=NUMBER_OF_RETURNS_DTYPE) | ||
| 1101 | return np.zeros(point_count, dtype=NUMBER_OF_RETURNS_DTYPE) | ||
| 1102 | |||
| 1103 | |||
| 1087 | class _RangeSplitAccumulator: | 1104 | class _RangeSplitAccumulator: |
| 1088 | def __init__( | 1105 | def __init__( |
| 1089 | self, | 1106 | self, |
| 1090 | *, | 1107 | *, |
| 1122 | "intensity": [], | 1140 | "intensity": [], |
| 1123 | "red": [], | 1141 | "red": [], |
| 1124 | "green": [], | 1142 | "green": [], |
| 1125 | "blue": [], | 1143 | "blue": [], |
| 1144 | "number_of_returns": [], | ||
| 1126 | }, | 1145 | }, |
| 1127 | ) | 1146 | ) |
| 1128 | buffers["points"].append(np.asarray(points - self.geoshift)) | 1147 | buffers["points"].append(np.asarray(points - self.geoshift)) |
| 1129 | buffers["scan_angle"].append(np.asarray(scan_angle)) | 1148 | buffers["scan_angle"].append(np.asarray(scan_angle)) |
| 1130 | buffers["intensity"].append(np.asarray(intensity)) | 1149 | buffers["intensity"].append(np.asarray(intensity)) |
| 1131 | buffers["red"].append(np.asarray(red)) | 1150 | buffers["red"].append(np.asarray(red)) |
| 1132 | buffers["green"].append(np.asarray(green)) | 1151 | buffers["green"].append(np.asarray(green)) |
| 1133 | buffers["blue"].append(np.asarray(blue)) | 1152 | buffers["blue"].append(np.asarray(blue)) |
| 1153 | buffers["number_of_returns"].append(np.asarray(number_of_returns)) | ||
| 1134 | 1154 | ||
| 1135 | def finalize(self) -> list[Path]: | 1155 | def finalize(self) -> list[Path]: |
| 1136 | written: list[Path] = [] | 1156 | written: list[Path] = [] |
| 1137 | for segment_idx, buffers in sorted(self._buffers.items()): | 1157 | for segment_idx, buffers in sorted(self._buffers.items()): |
| 1110 | intensity: np.ndarray, | 1127 | intensity: np.ndarray, |
| 1111 | red: np.ndarray, | 1128 | red: np.ndarray, |
| 1112 | green: np.ndarray, | 1129 | green: np.ndarray, |
| 1113 | blue: np.ndarray, | 1130 | blue: np.ndarray, |
| 1131 | number_of_returns: np.ndarray, | ||
| 1114 | ) -> None: | 1132 | ) -> None: |
| 1115 | if points.shape[0] == 0: | 1133 | if points.shape[0] == 0: |
| 1116 | return | 1134 | return |
| 1117 | buffers = self._buffers.setdefault( | 1135 | buffers = self._buffers.setdefault( |
| 1258 | ) | 1278 | ) |
| 1259 | scan_angle = mapper._chunk_scan_angle(chunk, las_file=las_file, logger=logger) | 1279 | scan_angle = mapper._chunk_scan_angle(chunk, las_file=las_file, logger=logger) |
| 1260 | intensity = np.asarray(chunk.intensity) | 1280 | intensity = np.asarray(chunk.intensity) |
| 1261 | red, green, blue = mapper._chunk_rgb(chunk=chunk, las_file=las_file) | 1281 | red, green, blue = mapper._chunk_rgb(chunk=chunk, las_file=las_file) |
| 1282 | number_of_returns = _chunk_number_of_returns(chunk, int(points.shape[0])) | ||
| 1262 | 1283 | ||
| 1263 | processed_points += int(points.shape[0]) | 1284 | processed_points += int(points.shape[0]) |
| 1264 | 1285 | ||
| 1265 | if angle_limit is not None: | 1286 | if angle_limit is not None: |
| 1271 | intensity = intensity[angle_mask] | 1292 | intensity = intensity[angle_mask] |
| 1272 | red = red[angle_mask] | 1293 | red = red[angle_mask] |
| 1273 | green = green[angle_mask] | 1294 | green = green[angle_mask] |
| 1274 | blue = blue[angle_mask] | 1295 | blue = blue[angle_mask] |
| 1296 | number_of_returns = number_of_returns[angle_mask] | ||
| 1275 | 1297 | ||
| 1276 | if points.shape[0] == 0: | 1298 | if points.shape[0] == 0: |
| 1277 | continue | 1299 | continue |
| 1278 | 1300 |
| 1320 | intensity_to_save = intensity[points_mask] | 1342 | intensity_to_save = intensity[points_mask] |
| 1321 | red_to_save = red[points_mask] | 1343 | red_to_save = red[points_mask] |
| 1322 | green_to_save = green[points_mask] | 1344 | green_to_save = green[points_mask] |
| 1323 | blue_to_save = blue[points_mask] | 1345 | blue_to_save = blue[points_mask] |
| 1346 | number_of_returns_to_save = number_of_returns[points_mask] | ||
| 1324 | longitudinal_planes = (longitudinal_limit_planes_by_segment or {}).get(segment_idx) | 1347 | longitudinal_planes = (longitudinal_limit_planes_by_segment or {}).get(segment_idx) |
| 1325 | if longitudinal_planes is not None: | 1348 | if longitudinal_planes is not None: |
| 1326 | left_plane, right_plane = longitudinal_planes | 1349 | left_plane, right_plane = longitudinal_planes |
| 1327 | longitudinal_mask = points_inside_longitudinal_limits( | 1350 | longitudinal_mask = points_inside_longitudinal_limits( |
| 1338 | intensity_to_save = intensity_to_save[longitudinal_mask] | 1361 | intensity_to_save = intensity_to_save[longitudinal_mask] |
| 1339 | red_to_save = red_to_save[longitudinal_mask] | 1362 | red_to_save = red_to_save[longitudinal_mask] |
| 1340 | green_to_save = green_to_save[longitudinal_mask] | 1363 | green_to_save = green_to_save[longitudinal_mask] |
| 1341 | blue_to_save = blue_to_save[longitudinal_mask] | 1364 | blue_to_save = blue_to_save[longitudinal_mask] |
| 1365 | number_of_returns_to_save = number_of_returns_to_save[longitudinal_mask] | ||
| 1342 | 1366 | ||
| 1343 | final_point_count = int(points_to_save.shape[0]) | 1367 | final_point_count = int(points_to_save.shape[0]) |
| 1344 | point_count_by_segment[segment_idx] = ( | 1368 | point_count_by_segment[segment_idx] = ( |
| 1345 | point_count_by_segment.get(segment_idx, 0) + final_point_count | 1369 | point_count_by_segment.get(segment_idx, 0) + final_point_count |
| 1352 | intensity=intensity_to_save, | 1376 | intensity=intensity_to_save, |
| 1353 | red=red_to_save, | 1377 | red=red_to_save, |
| 1354 | green=green_to_save, | 1378 | green=green_to_save, |
| 1355 | blue=blue_to_save, | 1379 | blue=blue_to_save, |
| 1380 | number_of_returns=number_of_returns_to_save, | ||
| 1356 | ) | 1381 | ) |
| 1357 | 1382 | ||
| 1358 | return LasRangeSplitResult( | 1383 | return LasRangeSplitResult( |
| 1359 | point_count_by_segment=point_count_by_segment, | 1384 | point_count_by_segment=point_count_by_segment, |
Orchestrator's own range-split run3 writer (the non-chunked path in
s3_segment_mapper.py) mirrored the producer change so both write paths emit the same 7-key record. Note the orchestrator conda env still pins the producer at 0.7.2, so this path emits legacy records until repinned. No pytest infrastructure there; untested, flagged as follow-up.