Back to report index

Lanefinder (job config + wrappers) 93a0953: AI3D-379 Align config module with fleet pattern

Miroslav Simko <ms@iolabs.ch> 2026-09-02T09:39:11+02:00

Commit #8 ยท 15 snippets

 README.md                  |  9 ++++++---
 src/pipeline/job_config.py | 41 ++++++++++++++++++++++++++++++-----------
 test/test_job_config.py    | 35 +++++++++++++++++++++++++++++++++++
 3 files changed, 71 insertions(+), 14 deletions(-)
Importance #1: src/pipeline/job_config.py @@ -52,13 +63,19 @@
52 return {} if value is None else value63 return {} if value is None else value
5364
54 @pydantic.field_validator("spine_las_files", mode="after")65 @pydantic.field_validator("spine_las_files", mode="after")
55 @classmethod66 @classmethod
56 def _empty_spine_list_as_none(cls, value: list[str] | None) -> list[str] | None:67 def _empty_spine_list_as_none(
57 """Preserve the legacy empty-list -> ``None`` behaviour."""68 cls, value: tuple[str, ...] | None
58 if not value:69 ) -> tuple[str, ...] | None:
59 return None70 """Map an empty list to ``None``.
60 return list(value)71
72 ``"spine_las_files": []`` is the checked-in way of saying "use every
73 spine", i.e. exactly what omitting the key means; collapsing it here
74 keeps :func:`merge_into_args` from passing an empty ``--spine-las-files``
75 string to Step 3.
76 """
77 return value or None
6178
6279
63def load_job_config(path: Path) -> JobConfig:80def load_job_config(path: Path) -> JobConfig:
64 """Parse ``path`` into a :class:`JobConfig`.81 """Parse ``path`` into a :class:`JobConfig`.
Importance #2: src/pipeline/job_config.py @@ -10,25 +10,36 @@
10``args.config_overrides`` **before** any CLI ``--set`` flags so that CLI10``args.config_overrides`` **before** any CLI ``--set`` flags so that CLI
11overrides win on key collision.11overrides win on key collision.
1212
13The schema is :class:`JobConfig` (a :class:`config_loader.ConfigModel`). Adding13The schema is :class:`JobConfig` (a :class:`config_loader.ConfigModel`). Adding
14a key means adding a field here and using it in ``configs/jobs/*.json``;14a config key means adding the field to the model and using the same key in
15unknown keys are rejected.15``configs/jobs/*.json`` -- nothing else. Unknown keys are rejected.
16
17Unlike the pipeline packages this module has no packaged ``*.default.json`` and
18no ``--set`` entry points: a job config is always a user-supplied file, so
19:func:`load_job_config` is the only loader. The ``set`` field keeps the name of
20its public JSON key and therefore shadows the ``set`` builtin inside the class
21body -- read it as ``cfg.set`` and never call ``set(...)`` in this module.
16"""22"""
1723
18from __future__ import annotations24from __future__ import annotations
1925
20import argparse26import argparse
21import json27import json
22import logging28import logging
23from pathlib import Path29from pathlib import Path
24from typing import Any30from typing import Annotated, Any
2531
26import pydantic32import pydantic
27from iolabs.common import config_loader33from iolabs.common import config_loader
2834
29logger = logging.getLogger(__name__)35logger = logging.getLogger(__name__)
3036
37_DEVICE_PATTERN = r"^(CPU|CUDA):\d+$"
38
39Device = Annotated[str, pydantic.StringConstraints(pattern=_DEVICE_PATTERN)]
40"""Open3D device string as accepted by the pipeline wrappers (``CUDA:0``/``CPU:0``)."""
41
3142
32class JobConfigError(config_loader.ConfigError):43class JobConfigError(config_loader.ConfigError):
33 """Raised when a job config JSON is missing keys, has unknown keys, or is invalid."""44 """Raised when a job config JSON is missing keys, has unknown keys, or is invalid."""
3445
Importance #3: src/pipeline/job_config.py @@ -39,11 +50,11 @@
39 job_id: str50 job_id: str
40 data_dir: Path51 data_dir: Path
41 from_segment: int | None = None52 from_segment: int | None = None
42 to_segment: int | None = None53 to_segment: int | None = None
43 device: str | None = None54 device: Device | None = None
44 random_seed: int | None = None55 random_seed: int | None = None
45 spine_las_files: list[str] | None = None56 spine_las_files: tuple[str, ...] | None = None
46 set: dict[str, Any] = pydantic.Field(default_factory=dict)57 set: dict[str, Any] = pydantic.Field(default_factory=dict)
4758
48 @pydantic.field_validator("set", mode="before")59 @pydantic.field_validator("set", mode="before")
49 @classmethod60 @classmethod
Importance #4: src/pipeline/job_config.py @@ -109,9 +126,11 @@
109 and getattr(args, "spine_las_files", None) is None126 and getattr(args, "spine_las_files", None) is None
110 ):127 ):
111 args.spine_las_files = ",".join(cfg.spine_las_files)128 args.spine_las_files = ",".join(cfg.spine_las_files)
112129
113 cfg_overrides = [f"{path}={_encode_override_value(value)}" for path, value in cfg.set.items()]130 cfg_overrides = [
131 f"{key}={_encode_override_value(value)}" for key, value in cfg.set.items()
132 ]
114 existing_overrides = list(getattr(args, "config_overrides", None) or [])133 existing_overrides = list(getattr(args, "config_overrides", None) or [])
115 args.config_overrides = cfg_overrides + existing_overrides134 args.config_overrides = cfg_overrides + existing_overrides
116135
117136
Importance #5: test/test_job_config.py @@ -3,8 +3,9 @@
3from pathlib import Path3from pathlib import Path
44
5import pydantic5import pydantic
6import pytest6import pytest
7from iolabs.common import config_loader
78
8from src.pipeline import job_config9from src.pipeline import job_config
910
10_JOBS_DIR = Path(__file__).resolve().parents[1] / "configs" / "jobs"11_JOBS_DIR = Path(__file__).resolve().parents[1] / "configs" / "jobs"
Importance #6: test/test_job_config.py @@ -13,8 +14,13 @@
13def _write_json(path: Path, payload: dict) -> None:14def _write_json(path: Path, payload: dict) -> None:
14 path.write_text(json.dumps(payload))15 path.write_text(json.dumps(payload))
1516
1617
18def test_error_class_is_config_error() -> None:
19 assert issubclass(job_config.JobConfigError, config_loader.ConfigError)
20 assert issubclass(job_config.JobConfigError, ValueError)
21
22
17def test_load_job_config_parses_full_schema(tmp_path: Path) -> None:23def test_load_job_config_parses_full_schema(tmp_path: Path) -> None:
18 cfg_path = tmp_path / "job.json"24 cfg_path = tmp_path / "job.json"
19 _write_json(25 _write_json(
20 cfg_path,26 cfg_path,
Importance #7: test/test_job_config.py @@ -267,8 +273,37 @@
267 with pytest.raises(job_config.JobConfigError, match="not valid JSON"):273 with pytest.raises(job_config.JobConfigError, match="not valid JSON"):
268 job_config.load_job_config(cfg_path)274 job_config.load_job_config(cfg_path)
269275
270276
277def test_load_job_config_rejects_unknown_device(tmp_path: Path) -> None:
278 cfg_path = tmp_path / "job.json"
279 _write_json(cfg_path, {"job_id": "x", "data_dir": "d", "device": "gpu"})
280
281 with pytest.raises(job_config.JobConfigError, match="device"):
282 job_config.load_job_config(cfg_path)
283
284
285def test_spine_las_files_is_a_tuple(tmp_path: Path) -> None:
286 cfg_path = tmp_path / "job.json"
287 _write_json(
288 cfg_path,
289 {"job_id": "x", "data_dir": "d", "spine_las_files": ["a", "b"]},
290 )
291
292 cfg = job_config.load_job_config(cfg_path)
293
294 assert cfg.spine_las_files == ("a", "b")
295
296
297def test_empty_spine_las_files_means_all_spines(tmp_path: Path) -> None:
298 cfg_path = tmp_path / "job.json"
299 _write_json(cfg_path, {"job_id": "x", "data_dir": "d", "spine_las_files": []})
300
301 cfg = job_config.load_job_config(cfg_path)
302
303 assert cfg.spine_las_files is None
304
305
271def test_job_config_is_frozen() -> None:306def test_job_config_is_frozen() -> None:
272 cfg = job_config.JobConfig(job_id="j", data_dir=Path("/x"))307 cfg = job_config.JobConfig(job_id="j", data_dir=Path("/x"))
273308
274 with pytest.raises(pydantic.ValidationError):309 with pytest.raises(pydantic.ValidationError):
Importance #8: README.md @@ -114,11 +114,14 @@
114Step 5 `line_bitmap_inference.*`, Step 6 `mask_clustering.*`, Step 7114Step 5 `line_bitmap_inference.*`, Step 6 `mask_clustering.*`, Step 7
115`iolabs-point-cloud-modelling-lines`.115`iolabs-point-cloud-modelling-lines`.
116116
117Job-config keys (`configs/jobs/*.json`) are declared on `JobConfig` in117Job-config keys (`configs/jobs/*.json`) are declared on `JobConfig` in
118`src/pipeline/job_config.py`. Adding a key means adding a field to that118`src/pipeline/job_config.py` (a `config_loader.ConfigModel`); unknown keys are
119`config_loader.ConfigModel` and using it in the job JSON; there is no separate119rejected. **To add a job-config key: add the field (with its type, default and
120allow-list.120any `Field` range) to `JobConfig` and use the same key in the job JSON --
121nothing else.** `load_job_config()` returns the frozen `JobConfig`. Unlike the
122pipeline packages there is no packaged `job.default.json` and no `--set` entry
123point; `device` must match `CUDA:<n>`/`CPU:<n>`.
121124
122Use a dataset shortcut to keep commands short:125Use a dataset shortcut to keep commands short:
123126
124```bash127```bash
Importance #9: src/pipeline/job_config.py @@ -10,25 +10,36 @@
10``args.config_overrides`` **before** any CLI ``--set`` flags so that CLI10``args.config_overrides`` **before** any CLI ``--set`` flags so that CLI
11overrides win on key collision.11overrides win on key collision.
1212
13The schema is :class:`JobConfig` (a :class:`config_loader.ConfigModel`). Adding13The schema is :class:`JobConfig` (a :class:`config_loader.ConfigModel`). Adding
14a key means adding a field here and using it in ``configs/jobs/*.json``;14a config key means adding the field to the model and using the same key in
15unknown keys are rejected.15``configs/jobs/*.json`` -- nothing else. Unknown keys are rejected.
16
17Unlike the pipeline packages this module has no packaged ``*.default.json`` and
18no ``--set`` entry points: a job config is always a user-supplied file, so
19:func:`load_job_config` is the only loader. The ``set`` field keeps the name of
20its public JSON key and therefore shadows the ``set`` builtin inside the class
21body -- read it as ``cfg.set`` and never call ``set(...)`` in this module.
16"""22"""
1723
18from __future__ import annotations24from __future__ import annotations
1925
20import argparse26import argparse
21import json27import json
22import logging28import logging
23from pathlib import Path29from pathlib import Path
24from typing import Any30from typing import Annotated, Any
2531
26import pydantic32import pydantic
27from iolabs.common import config_loader33from iolabs.common import config_loader
2834
29logger = logging.getLogger(__name__)35logger = logging.getLogger(__name__)
3036
37_DEVICE_PATTERN = r"^(CPU|CUDA):\d+$"
38
39Device = Annotated[str, pydantic.StringConstraints(pattern=_DEVICE_PATTERN)]
40"""Open3D device string as accepted by the pipeline wrappers (``CUDA:0``/``CPU:0``)."""
41
3142
32class JobConfigError(config_loader.ConfigError):43class JobConfigError(config_loader.ConfigError):
33 """Raised when a job config JSON is missing keys, has unknown keys, or is invalid."""44 """Raised when a job config JSON is missing keys, has unknown keys, or is invalid."""
3445
Importance #10: src/pipeline/job_config.py @@ -39,11 +50,11 @@
39 job_id: str50 job_id: str
40 data_dir: Path51 data_dir: Path
41 from_segment: int | None = None52 from_segment: int | None = None
42 to_segment: int | None = None53 to_segment: int | None = None
43 device: str | None = None54 device: Device | None = None
44 random_seed: int | None = None55 random_seed: int | None = None
45 spine_las_files: list[str] | None = None56 spine_las_files: tuple[str, ...] | None = None
46 set: dict[str, Any] = pydantic.Field(default_factory=dict)57 set: dict[str, Any] = pydantic.Field(default_factory=dict)
4758
48 @pydantic.field_validator("set", mode="before")59 @pydantic.field_validator("set", mode="before")
49 @classmethod60 @classmethod
Importance #11: src/pipeline/job_config.py @@ -52,13 +63,19 @@
52 return {} if value is None else value63 return {} if value is None else value
5364
54 @pydantic.field_validator("spine_las_files", mode="after")65 @pydantic.field_validator("spine_las_files", mode="after")
55 @classmethod66 @classmethod
56 def _empty_spine_list_as_none(cls, value: list[str] | None) -> list[str] | None:67 def _empty_spine_list_as_none(
57 """Preserve the legacy empty-list -> ``None`` behaviour."""68 cls, value: tuple[str, ...] | None
58 if not value:69 ) -> tuple[str, ...] | None:
59 return None70 """Map an empty list to ``None``.
60 return list(value)71
72 ``"spine_las_files": []`` is the checked-in way of saying "use every
73 spine", i.e. exactly what omitting the key means; collapsing it here
74 keeps :func:`merge_into_args` from passing an empty ``--spine-las-files``
75 string to Step 3.
76 """
77 return value or None
6178
6279
63def load_job_config(path: Path) -> JobConfig:80def load_job_config(path: Path) -> JobConfig:
64 """Parse ``path`` into a :class:`JobConfig`.81 """Parse ``path`` into a :class:`JobConfig`.
Importance #12: src/pipeline/job_config.py @@ -109,9 +126,11 @@
109 and getattr(args, "spine_las_files", None) is None126 and getattr(args, "spine_las_files", None) is None
110 ):127 ):
111 args.spine_las_files = ",".join(cfg.spine_las_files)128 args.spine_las_files = ",".join(cfg.spine_las_files)
112129
113 cfg_overrides = [f"{path}={_encode_override_value(value)}" for path, value in cfg.set.items()]130 cfg_overrides = [
131 f"{key}={_encode_override_value(value)}" for key, value in cfg.set.items()
132 ]
114 existing_overrides = list(getattr(args, "config_overrides", None) or [])133 existing_overrides = list(getattr(args, "config_overrides", None) or [])
115 args.config_overrides = cfg_overrides + existing_overrides134 args.config_overrides = cfg_overrides + existing_overrides
116135
117136
Importance #13: test/test_job_config.py @@ -3,8 +3,9 @@
3from pathlib import Path3from pathlib import Path
44
5import pydantic5import pydantic
6import pytest6import pytest
7from iolabs.common import config_loader
78
8from src.pipeline import job_config9from src.pipeline import job_config
910
10_JOBS_DIR = Path(__file__).resolve().parents[1] / "configs" / "jobs"11_JOBS_DIR = Path(__file__).resolve().parents[1] / "configs" / "jobs"
Importance #14: test/test_job_config.py @@ -13,8 +14,13 @@
13def _write_json(path: Path, payload: dict) -> None:14def _write_json(path: Path, payload: dict) -> None:
14 path.write_text(json.dumps(payload))15 path.write_text(json.dumps(payload))
1516
1617
18def test_error_class_is_config_error() -> None:
19 assert issubclass(job_config.JobConfigError, config_loader.ConfigError)
20 assert issubclass(job_config.JobConfigError, ValueError)
21
22
17def test_load_job_config_parses_full_schema(tmp_path: Path) -> None:23def test_load_job_config_parses_full_schema(tmp_path: Path) -> None:
18 cfg_path = tmp_path / "job.json"24 cfg_path = tmp_path / "job.json"
19 _write_json(25 _write_json(
20 cfg_path,26 cfg_path,
Importance #15: test/test_job_config.py @@ -267,8 +273,37 @@
267 with pytest.raises(job_config.JobConfigError, match="not valid JSON"):273 with pytest.raises(job_config.JobConfigError, match="not valid JSON"):
268 job_config.load_job_config(cfg_path)274 job_config.load_job_config(cfg_path)
269275
270276
277def test_load_job_config_rejects_unknown_device(tmp_path: Path) -> None:
278 cfg_path = tmp_path / "job.json"
279 _write_json(cfg_path, {"job_id": "x", "data_dir": "d", "device": "gpu"})
280
281 with pytest.raises(job_config.JobConfigError, match="device"):
282 job_config.load_job_config(cfg_path)
283
284
285def test_spine_las_files_is_a_tuple(tmp_path: Path) -> None:
286 cfg_path = tmp_path / "job.json"
287 _write_json(
288 cfg_path,
289 {"job_id": "x", "data_dir": "d", "spine_las_files": ["a", "b"]},
290 )
291
292 cfg = job_config.load_job_config(cfg_path)
293
294 assert cfg.spine_las_files == ("a", "b")
295
296
297def test_empty_spine_las_files_means_all_spines(tmp_path: Path) -> None:
298 cfg_path = tmp_path / "job.json"
299 _write_json(cfg_path, {"job_id": "x", "data_dir": "d", "spine_las_files": []})
300
301 cfg = job_config.load_job_config(cfg_path)
302
303 assert cfg.spine_las_files is None
304
305
271def test_job_config_is_frozen() -> None:306def test_job_config_is_frozen() -> None:
272 cfg = job_config.JobConfig(job_id="j", data_dir=Path("/x"))307 cfg = job_config.JobConfig(job_id="j", data_dir=Path("/x"))
273308
274 with pytest.raises(pydantic.ValidationError):309 with pytest.raises(pydantic.ValidationError):