Back to report index

Lanefinder (job config + wrappers) 89c38e6: AI3D-379 Pydantic config models via iolabs-common ConfigModel

Miroslav Simko <ms@iolabs.ch> 2026-09-02T08:35:55+02:00

Commit #6 · 23 snippets

 AGENTS.md                               |  1 +
 README.md                               |  5 ++
 configs/jobs/README.md                  |  5 ++
 pyproject.toml                          |  5 +-
 scripts/pipeline/run_8_asphalt_edges.py | 12 ++---
 src/pipeline/job_config.py              | 95 +++++++++++++++------------------
 test/test_job_config.py                 | 15 ++++++
 7 files changed, 76 insertions(+), 62 deletions(-)
Importance #1: src/pipeline/job_config.py @@ -8,33 +8,33 @@
8CLI flags always win over config values: if the user passes ``--from-segment``,8CLI flags always win over config values: if the user passes ``--from-segment``,
9it overrides the config's ``from_segment``. Entries in ``set`` are merged into9it overrides the config's ``from_segment``. Entries in ``set`` are merged into
10``args.config_overrides`` **before** any CLI ``--set`` flags so that CLI10``args.config_overrides`` **before** any CLI ``--set`` flags so that CLI
11overrides win on key collision.11overrides win on key collision.
12
13The schema is :class:`JobConfig` (a :class:`config_loader.ConfigModel`). Adding
14a key means adding a field here and using it in ``configs/jobs/*.json``;
15unknown keys are rejected.
12"""16"""
1317
14from __future__ import annotations18from __future__ import annotations
1519
16import argparse20import argparse
17import json21import json
18from dataclasses import dataclass, field22import logging
19from pathlib import Path23from pathlib import Path
20from typing import Any24from typing import Any
2125
22_REQUIRED_KEYS = ("job_id", "data_dir")26import pydantic
23_KNOWN_KEYS = {27from iolabs.common import config_loader
24 "job_id",28
25 "data_dir",29logger = logging.getLogger(__name__)
26 "from_segment",30
27 "to_segment",31
28 "device",32class JobConfigError(config_loader.ConfigError):
29 "random_seed",33 """Raised when a job config JSON is missing keys, has unknown keys, or is invalid."""
30 "spine_las_files",34
31 "set",35
32}36class JobConfig(config_loader.ConfigModel):
33
34
35@dataclass
36class JobConfig:
37 """Parsed contents of a ``configs/jobs/<name>.json`` file."""37 """Parsed contents of a ``configs/jobs/<name>.json`` file."""
3838
39 job_id: str39 job_id: str
40 data_dir: Path40 data_dir: Path
Importance #2: src/pipeline/job_config.py @@ -42,59 +42,52 @@
42 to_segment: int | None = None42 to_segment: int | None = None
43 device: str | None = None43 device: str | None = None
44 random_seed: int | None = None44 random_seed: int | None = None
45 spine_las_files: list[str] | None = None45 spine_las_files: list[str] | None = None
46 set: dict[str, Any] = field(default_factory=dict)46 set: dict[str, Any] = {}
47
48 @pydantic.field_validator("set", mode="before")
49 @classmethod
50 def _empty_set_mapping(cls, value: Any) -> Any:
51 """Treat a missing/null ``set`` object as an empty mapping."""
52 return {} if value is None else value
53
54 @pydantic.field_validator("spine_las_files", mode="after")
55 @classmethod
56 def _empty_spine_list_as_none(cls, value: list[str] | None) -> list[str] | None:
57 """Preserve the legacy empty-list -> ``None`` behaviour."""
58 if not value:
59 return None
60 return list(value)
4761
4862
49def load_job_config(path: Path) -> JobConfig:63def load_job_config(path: Path) -> JobConfig:
50 """Parse ``path`` into a :class:`JobConfig`.64 """Parse ``path`` into a :class:`JobConfig`.
5165
52 Relative ``data_dir`` entries are resolved against the config file's66 Relative ``data_dir`` entries are resolved against the config file's
53 directory. Missing required keys or unknown top-level keys raise67 directory. Missing required keys or unknown top-level keys raise
54 :class:`ValueError`.68 :class:`JobConfigError` (a :class:`ValueError`).
55 """69 """
56 path = Path(path)70 path = Path(path)
57 raw = json.loads(path.read_text())71 raw = json.loads(path.read_text(encoding="utf-8"))
5872
59 if not isinstance(raw, dict):73 if not isinstance(raw, dict):
60 raise ValueError(f"Job config {path} must be a JSON object")74 raise JobConfigError(f"Job config {path} must be a JSON object")
61
62 for key in _REQUIRED_KEYS:
63 if key not in raw:
64 raise ValueError(f"Job config {path} missing required key: {key}")
65
66 unknown = set(raw) - _KNOWN_KEYS
67 if unknown:
68 raise ValueError(
69 f"Job config {path} has unknown keys: {sorted(unknown)}"
70 )
7175
72 data_dir = Path(raw["data_dir"])76 cfg = config_loader.validate_config(
77 JobConfig,
78 raw,
79 context=f"job config {path}",
80 error_cls=JobConfigError,
81 )
82 data_dir = Path(cfg.data_dir)
73 if not data_dir.is_absolute():83 if not data_dir.is_absolute():
74 data_dir = (path.parent / data_dir).resolve()84 data_dir = (path.parent / data_dir).resolve()
75 else:85 else:
76 data_dir = data_dir.resolve()86 data_dir = data_dir.resolve()
7787 resolved = cfg.model_copy(update={"data_dir": data_dir})
78 spine_las_files = raw.get("spine_las_files")88 logger.debug("Loaded job config %s (job_id=%s)", path, resolved.job_id)
79 if spine_las_files is not None:89 return resolved
80 if not isinstance(spine_las_files, list) or not all(
81 isinstance(s, str) for s in spine_las_files
82 ):
83 raise ValueError(
84 f"Job config {path} 'spine_las_files' must be a list of strings"
85 )
86
87 return JobConfig(
88 job_id=str(raw["job_id"]),
89 data_dir=data_dir,
90 from_segment=raw.get("from_segment"),
91 to_segment=raw.get("to_segment"),
92 device=raw.get("device"),
93 random_seed=raw.get("random_seed"),
94 spine_las_files=list(spine_las_files) if spine_las_files else None,
95 set=dict(raw.get("set") or {}),
96 )
9790
9891
99def merge_into_args(cfg: JobConfig, args: argparse.Namespace) -> None:92def merge_into_args(cfg: JobConfig, args: argparse.Namespace) -> None:
100 """Populate unset CLI fields from ``cfg``. CLI values always win."""93 """Populate unset CLI fields from ``cfg``. CLI values always win."""
Importance #3: scripts/pipeline/run_8_asphalt_edges.py @@ -1,7 +1,7 @@
1"""Pipeline step 8: Detect asphalt/pavement edges per segment."""1"""Pipeline step 8: Detect asphalt/pavement edges per segment."""
22
3import json3import dataclasses
4import logging4import logging
5from pathlib import Path5from pathlib import Path
66
7import numpy as np7import numpy as np
Importance #4: scripts/pipeline/run_8_asphalt_edges.py @@ -293,8 +293,9 @@
293 FLAG_INTERPOLATED,293 FLAG_INTERPOLATED,
294 FLAG_MEASURED,294 FLAG_MEASURED,
295 config_from_dict,295 config_from_dict,
296 detect_edges,296 detect_edges,
297 load_asphalt_edge_config,
297 )298 )
298 from iolabs_point_cloud_detection_asphaltedge.io import (299 from iolabs_point_cloud_detection_asphaltedge.io import (
299 assert_aligned_frames,300 assert_aligned_frames,
300 build_segment_axis,301 build_segment_axis,
Importance #5: scripts/pipeline/run_8_asphalt_edges.py @@ -310,16 +311,9 @@
310 "iolabs-point-cloud-detection-asphaltedge (unpublished -- "311 "iolabs-point-cloud-detection-asphaltedge (unpublished -- "
311 "pip install -e /path/to/3dai.iolabs.pointcloud.asphaltedge)"312 "pip install -e /path/to/3dai.iolabs.pointcloud.asphaltedge)"
312 ) from exc313 ) from exc
313314
314 from iolabs.common import config_loader315 base_config = dataclasses.asdict(load_asphalt_edge_config())
315
316 default_path = config_loader.default_config_path(
317 "iolabs_point_cloud_detection_asphaltedge",
318 "asphalt_edge.default.json",
319 )
320 with open(default_path, encoding="utf-8") as handle:
321 base_config = json.load(handle)
322 prepared = cli.prepare_package_config(316 prepared = cli.prepare_package_config(
323 args,317 args,
324 base_config,318 base_config,
325 override_sections=("asphalt_edge",),319 override_sections=("asphalt_edge",),
Importance #6: test/test_job_config.py @@ -5,8 +5,10 @@
5import pytest5import pytest
66
7from src.pipeline import job_config7from src.pipeline import job_config
88
9_JOBS_DIR = Path(__file__).resolve().parents[1] / "configs" / "jobs"
10
911
10def _write_json(path: Path, payload: dict) -> None:12def _write_json(path: Path, payload: dict) -> None:
11 path.write_text(json.dumps(payload))13 path.write_text(json.dumps(payload))
1214
Importance #7: test/test_job_config.py @@ -241,4 +243,17 @@
241243
242 assert "job_from_segment" not in props244 assert "job_from_segment" not in props
243 assert "job_to_segment" not in props245 assert "job_to_segment" not in props
244 assert props["job_id"] == "j"246 assert props["job_id"] == "j"
247
248
249@pytest.mark.parametrize(
250 "job_path",
251 sorted(_JOBS_DIR.glob("*.json")),
252 ids=lambda path: path.name,
253)
254def test_checked_in_job_configs_load(job_path: Path) -> None:
255 cfg = job_config.load_job_config(job_path)
256
257 assert isinstance(cfg, job_config.JobConfig)
258 assert cfg.job_id
259 assert cfg.data_dir.is_absolute()
Importance #8: pyproject.toml @@ -1,7 +1,7 @@
1[project]1[project]
2name = "3dai-lanefinder"2name = "3dai-lanefinder"
3version = "0.9.1"3version = "0.9.2"
4description = "3D AI Lane Finder - LIDAR point cloud processing pipeline"4description = "3D AI Lane Finder - LIDAR point cloud processing pipeline"
5requires-python = ">=3.11,<3.13"5requires-python = ">=3.11,<3.13"
6dependencies = [6dependencies = [
7 "numpy>=1.20.0",7 "numpy>=1.20.0",
Importance #9: pyproject.toml @@ -11,9 +11,10 @@
11 "pypdf>=4.3.1",11 "pypdf>=4.3.1",
12 "matplotlib>=3.7",12 "matplotlib>=3.7",
13 "Pillow>=10.0",13 "Pillow>=10.0",
14 "scikit-image>=0.22",14 "scikit-image>=0.22",
15 "iolabs-common",15 "pydantic>=2.7",
16 "iolabs-common>=0.8.0",
16 "iolabs-logstash>=0.5.1",17 "iolabs-logstash>=0.5.1",
17 "python-dotenv>=1.0.0",18 "python-dotenv>=1.0.0",
18 "iolabs-geometry-geometry",19 "iolabs-geometry-geometry",
19 "iolabs-geometry-visualization",20 "iolabs-geometry-visualization",
Importance #10: AGENTS.md @@ -31,8 +31,9 @@
31## Config ownership31## Config ownership
3232
33- Wrappers (`scripts/pipeline/`) own runtime context: `--data-dir`, `--from-segment`/`--to-segment`, `--device`, `--seed`, output paths33- Wrappers (`scripts/pipeline/`) own runtime context: `--data-dir`, `--from-segment`/`--to-segment`, `--device`, `--seed`, output paths
34- Packages own algorithm defaults; override with repeatable `--set PATH=VALUE` — never repo-local JSON34- Packages own algorithm defaults; override with repeatable `--set PATH=VALUE` — never repo-local JSON
35- Job config (`src/pipeline/job_config.py`): add a field to `JobConfig` (`config_loader.ConfigModel`) and the same key to the `configs/jobs/*.json` instance — nothing else. Unknown keys are rejected.
35- Key namespaces: Step 4 `topdown_rasterizer.*` (wrapper-owned defaults), Step 5 `line_bitmap_inference.*`, Step 6 `mask_clustering.*`; Steps 1–3 and 7 use their package's keys (see README)36- Key namespaces: Step 4 `topdown_rasterizer.*` (wrapper-owned defaults), Step 5 `line_bitmap_inference.*`, Step 6 `mask_clustering.*`; Steps 1–3 and 7 use their package's keys (see README)
36- Step 1 special case: `--seed` is runtime-only (never inject into package config); Step 1 package config is strict — unknown keys must fail fast37- Step 1 special case: `--seed` is runtime-only (never inject into package config); Step 1 package config is strict — unknown keys must fail fast
3738
38## Conventions39## Conventions
Importance #11: README.md @@ -113,8 +113,13 @@
113`topdown_rasterizer.*` (defaults in `scripts/pipeline/run_4_rasterize_topdown.py`),113`topdown_rasterizer.*` (defaults in `scripts/pipeline/run_4_rasterize_topdown.py`),
114Step 5 `line_bitmap_inference.*`, Step 6 `mask_clustering.*`, Step 7114Step 5 `line_bitmap_inference.*`, Step 6 `mask_clustering.*`, Step 7
115`iolabs-point-cloud-modelling-lines`.115`iolabs-point-cloud-modelling-lines`.
116116
117Job-config keys (`configs/jobs/*.json`) are declared on `JobConfig` in
118`src/pipeline/job_config.py`. Adding a key means adding a field to that
119`config_loader.ConfigModel` and using it in the job JSON; there is no separate
120allow-list.
121
117Use a dataset shortcut to keep commands short:122Use a dataset shortcut to keep commands short:
118123
119```bash124```bash
120export DATASET=data/00_external/<dataset>125export DATASET=data/00_external/<dataset>
Importance #12: configs/jobs/README.md @@ -28,8 +28,13 @@
28 }28 }
29}29}
30```30```
3131
32The schema is `JobConfig` in `src/pipeline/job_config.py` (a
33`config_loader.ConfigModel`). Adding a key means adding a field there and using
34it in the JSON; unknown keys are rejected. There is no packaged default — these
35files are per-run instances.
36
32Required: `job_id`, `data_dir`. Everything else is optional.37Required: `job_id`, `data_dir`. Everything else is optional.
3338
34`data_dir` resolves relative to the config file's directory if it isn't absolute.39`data_dir` resolves relative to the config file's directory if it isn't absolute.
3540
Importance #13: README.md @@ -113,8 +113,13 @@
113`topdown_rasterizer.*` (defaults in `scripts/pipeline/run_4_rasterize_topdown.py`),113`topdown_rasterizer.*` (defaults in `scripts/pipeline/run_4_rasterize_topdown.py`),
114Step 5 `line_bitmap_inference.*`, Step 6 `mask_clustering.*`, Step 7114Step 5 `line_bitmap_inference.*`, Step 6 `mask_clustering.*`, Step 7
115`iolabs-point-cloud-modelling-lines`.115`iolabs-point-cloud-modelling-lines`.
116116
117Job-config keys (`configs/jobs/*.json`) are declared on `JobConfig` in
118`src/pipeline/job_config.py`. Adding a key means adding a field to that
119`config_loader.ConfigModel` and using it in the job JSON; there is no separate
120allow-list.
121
117Use a dataset shortcut to keep commands short:122Use a dataset shortcut to keep commands short:
118123
119```bash124```bash
120export DATASET=data/00_external/<dataset>125export DATASET=data/00_external/<dataset>
Importance #14: configs/jobs/README.md @@ -28,8 +28,13 @@
28 }28 }
29}29}
30```30```
3131
32The schema is `JobConfig` in `src/pipeline/job_config.py` (a
33`config_loader.ConfigModel`). Adding a key means adding a field there and using
34it in the JSON; unknown keys are rejected. There is no packaged default — these
35files are per-run instances.
36
32Required: `job_id`, `data_dir`. Everything else is optional.37Required: `job_id`, `data_dir`. Everything else is optional.
3338
34`data_dir` resolves relative to the config file's directory if it isn't absolute.39`data_dir` resolves relative to the config file's directory if it isn't absolute.
3540
Importance #15: pyproject.toml @@ -1,7 +1,7 @@
1[project]1[project]
2name = "3dai-lanefinder"2name = "3dai-lanefinder"
3version = "0.9.1"3version = "0.9.2"
4description = "3D AI Lane Finder - LIDAR point cloud processing pipeline"4description = "3D AI Lane Finder - LIDAR point cloud processing pipeline"
5requires-python = ">=3.11,<3.13"5requires-python = ">=3.11,<3.13"
6dependencies = [6dependencies = [
7 "numpy>=1.20.0",7 "numpy>=1.20.0",
Importance #16: pyproject.toml @@ -11,9 +11,10 @@
11 "pypdf>=4.3.1",11 "pypdf>=4.3.1",
12 "matplotlib>=3.7",12 "matplotlib>=3.7",
13 "Pillow>=10.0",13 "Pillow>=10.0",
14 "scikit-image>=0.22",14 "scikit-image>=0.22",
15 "iolabs-common",15 "pydantic>=2.7",
16 "iolabs-common>=0.8.0",
16 "iolabs-logstash>=0.5.1",17 "iolabs-logstash>=0.5.1",
17 "python-dotenv>=1.0.0",18 "python-dotenv>=1.0.0",
18 "iolabs-geometry-geometry",19 "iolabs-geometry-geometry",
19 "iolabs-geometry-visualization",20 "iolabs-geometry-visualization",
Importance #17: scripts/pipeline/run_8_asphalt_edges.py @@ -1,7 +1,7 @@
1"""Pipeline step 8: Detect asphalt/pavement edges per segment."""1"""Pipeline step 8: Detect asphalt/pavement edges per segment."""
22
3import json3import dataclasses
4import logging4import logging
5from pathlib import Path5from pathlib import Path
66
7import numpy as np7import numpy as np
Importance #18: scripts/pipeline/run_8_asphalt_edges.py @@ -293,8 +293,9 @@
293 FLAG_INTERPOLATED,293 FLAG_INTERPOLATED,
294 FLAG_MEASURED,294 FLAG_MEASURED,
295 config_from_dict,295 config_from_dict,
296 detect_edges,296 detect_edges,
297 load_asphalt_edge_config,
297 )298 )
298 from iolabs_point_cloud_detection_asphaltedge.io import (299 from iolabs_point_cloud_detection_asphaltedge.io import (
299 assert_aligned_frames,300 assert_aligned_frames,
300 build_segment_axis,301 build_segment_axis,
Importance #19: scripts/pipeline/run_8_asphalt_edges.py @@ -310,16 +311,9 @@
310 "iolabs-point-cloud-detection-asphaltedge (unpublished -- "311 "iolabs-point-cloud-detection-asphaltedge (unpublished -- "
311 "pip install -e /path/to/3dai.iolabs.pointcloud.asphaltedge)"312 "pip install -e /path/to/3dai.iolabs.pointcloud.asphaltedge)"
312 ) from exc313 ) from exc
313314
314 from iolabs.common import config_loader315 base_config = dataclasses.asdict(load_asphalt_edge_config())
315
316 default_path = config_loader.default_config_path(
317 "iolabs_point_cloud_detection_asphaltedge",
318 "asphalt_edge.default.json",
319 )
320 with open(default_path, encoding="utf-8") as handle:
321 base_config = json.load(handle)
322 prepared = cli.prepare_package_config(316 prepared = cli.prepare_package_config(
323 args,317 args,
324 base_config,318 base_config,
325 override_sections=("asphalt_edge",),319 override_sections=("asphalt_edge",),
Importance #20: src/pipeline/job_config.py @@ -8,33 +8,33 @@
8CLI flags always win over config values: if the user passes ``--from-segment``,8CLI flags always win over config values: if the user passes ``--from-segment``,
9it overrides the config's ``from_segment``. Entries in ``set`` are merged into9it overrides the config's ``from_segment``. Entries in ``set`` are merged into
10``args.config_overrides`` **before** any CLI ``--set`` flags so that CLI10``args.config_overrides`` **before** any CLI ``--set`` flags so that CLI
11overrides win on key collision.11overrides win on key collision.
12
13The schema is :class:`JobConfig` (a :class:`config_loader.ConfigModel`). Adding
14a key means adding a field here and using it in ``configs/jobs/*.json``;
15unknown keys are rejected.
12"""16"""
1317
14from __future__ import annotations18from __future__ import annotations
1519
16import argparse20import argparse
17import json21import json
18from dataclasses import dataclass, field22import logging
19from pathlib import Path23from pathlib import Path
20from typing import Any24from typing import Any
2125
22_REQUIRED_KEYS = ("job_id", "data_dir")26import pydantic
23_KNOWN_KEYS = {27from iolabs.common import config_loader
24 "job_id",28
25 "data_dir",29logger = logging.getLogger(__name__)
26 "from_segment",30
27 "to_segment",31
28 "device",32class JobConfigError(config_loader.ConfigError):
29 "random_seed",33 """Raised when a job config JSON is missing keys, has unknown keys, or is invalid."""
30 "spine_las_files",34
31 "set",35
32}36class JobConfig(config_loader.ConfigModel):
33
34
35@dataclass
36class JobConfig:
37 """Parsed contents of a ``configs/jobs/<name>.json`` file."""37 """Parsed contents of a ``configs/jobs/<name>.json`` file."""
3838
39 job_id: str39 job_id: str
40 data_dir: Path40 data_dir: Path
Importance #21: src/pipeline/job_config.py @@ -42,59 +42,52 @@
42 to_segment: int | None = None42 to_segment: int | None = None
43 device: str | None = None43 device: str | None = None
44 random_seed: int | None = None44 random_seed: int | None = None
45 spine_las_files: list[str] | None = None45 spine_las_files: list[str] | None = None
46 set: dict[str, Any] = field(default_factory=dict)46 set: dict[str, Any] = {}
47
48 @pydantic.field_validator("set", mode="before")
49 @classmethod
50 def _empty_set_mapping(cls, value: Any) -> Any:
51 """Treat a missing/null ``set`` object as an empty mapping."""
52 return {} if value is None else value
53
54 @pydantic.field_validator("spine_las_files", mode="after")
55 @classmethod
56 def _empty_spine_list_as_none(cls, value: list[str] | None) -> list[str] | None:
57 """Preserve the legacy empty-list -> ``None`` behaviour."""
58 if not value:
59 return None
60 return list(value)
4761
4862
49def load_job_config(path: Path) -> JobConfig:63def load_job_config(path: Path) -> JobConfig:
50 """Parse ``path`` into a :class:`JobConfig`.64 """Parse ``path`` into a :class:`JobConfig`.
5165
52 Relative ``data_dir`` entries are resolved against the config file's66 Relative ``data_dir`` entries are resolved against the config file's
53 directory. Missing required keys or unknown top-level keys raise67 directory. Missing required keys or unknown top-level keys raise
54 :class:`ValueError`.68 :class:`JobConfigError` (a :class:`ValueError`).
55 """69 """
56 path = Path(path)70 path = Path(path)
57 raw = json.loads(path.read_text())71 raw = json.loads(path.read_text(encoding="utf-8"))
5872
59 if not isinstance(raw, dict):73 if not isinstance(raw, dict):
60 raise ValueError(f"Job config {path} must be a JSON object")74 raise JobConfigError(f"Job config {path} must be a JSON object")
61
62 for key in _REQUIRED_KEYS:
63 if key not in raw:
64 raise ValueError(f"Job config {path} missing required key: {key}")
65
66 unknown = set(raw) - _KNOWN_KEYS
67 if unknown:
68 raise ValueError(
69 f"Job config {path} has unknown keys: {sorted(unknown)}"
70 )
7175
72 data_dir = Path(raw["data_dir"])76 cfg = config_loader.validate_config(
77 JobConfig,
78 raw,
79 context=f"job config {path}",
80 error_cls=JobConfigError,
81 )
82 data_dir = Path(cfg.data_dir)
73 if not data_dir.is_absolute():83 if not data_dir.is_absolute():
74 data_dir = (path.parent / data_dir).resolve()84 data_dir = (path.parent / data_dir).resolve()
75 else:85 else:
76 data_dir = data_dir.resolve()86 data_dir = data_dir.resolve()
7787 resolved = cfg.model_copy(update={"data_dir": data_dir})
78 spine_las_files = raw.get("spine_las_files")88 logger.debug("Loaded job config %s (job_id=%s)", path, resolved.job_id)
79 if spine_las_files is not None:89 return resolved
80 if not isinstance(spine_las_files, list) or not all(
81 isinstance(s, str) for s in spine_las_files
82 ):
83 raise ValueError(
84 f"Job config {path} 'spine_las_files' must be a list of strings"
85 )
86
87 return JobConfig(
88 job_id=str(raw["job_id"]),
89 data_dir=data_dir,
90 from_segment=raw.get("from_segment"),
91 to_segment=raw.get("to_segment"),
92 device=raw.get("device"),
93 random_seed=raw.get("random_seed"),
94 spine_las_files=list(spine_las_files) if spine_las_files else None,
95 set=dict(raw.get("set") or {}),
96 )
9790
9891
99def merge_into_args(cfg: JobConfig, args: argparse.Namespace) -> None:92def merge_into_args(cfg: JobConfig, args: argparse.Namespace) -> None:
100 """Populate unset CLI fields from ``cfg``. CLI values always win."""93 """Populate unset CLI fields from ``cfg``. CLI values always win."""
Importance #22: test/test_job_config.py @@ -5,8 +5,10 @@
5import pytest5import pytest
66
7from src.pipeline import job_config7from src.pipeline import job_config
88
9_JOBS_DIR = Path(__file__).resolve().parents[1] / "configs" / "jobs"
10
911
10def _write_json(path: Path, payload: dict) -> None:12def _write_json(path: Path, payload: dict) -> None:
11 path.write_text(json.dumps(payload))13 path.write_text(json.dumps(payload))
1214
Importance #23: test/test_job_config.py @@ -241,4 +243,17 @@
241243
242 assert "job_from_segment" not in props244 assert "job_from_segment" not in props
243 assert "job_to_segment" not in props245 assert "job_to_segment" not in props
244 assert props["job_id"] == "j"246 assert props["job_id"] == "j"
247
248
249@pytest.mark.parametrize(
250 "job_path",
251 sorted(_JOBS_DIR.glob("*.json")),
252 ids=lambda path: path.name,
253)
254def test_checked_in_job_configs_load(job_path: Path) -> None:
255 cfg = job_config.load_job_config(job_path)
256
257 assert isinstance(cfg, job_config.JobConfig)
258 assert cfg.job_id
259 assert cfg.data_dir.is_absolute()