Back to report index

iolabs-common (shared config layer) 943e710: AI3D-379 Align config module with fleet pattern

Miroslav Simko <ms@iolabs.ch> 2026-09-02T09:37:11+02:00

Commit #5 · 20 snippets

 README.md                          | 50 ++++++++++++++++++-----
 src/iolabs/common/config_loader.py | 40 +++++++++++++++----
 src/iolabs/common/config_model.py  | 81 ++++++++++++++++++++++++++++++++------
 tests/test_config_model.py         | 43 ++++++++++++++++++--
 4 files changed, 183 insertions(+), 31 deletions(-)
Importance #1: src/iolabs/common/config_model.py @@ -9,14 +9,21 @@
9pydantic itself.9pydantic itself.
1010
11Canonical package pattern::11Canonical package pattern::
1212
13 import logging
13 from collections.abc import Mapping14 from collections.abc import Mapping
14 from pathlib import Path15 from pathlib import Path
15 from typing import Any, Literal16 from typing import Any, Literal
1617
17 from iolabs.common import config_loader18 from iolabs.common import config_loader
1819
20 logger = logging.getLogger(__name__)
21
22 _PACKAGE_NAME = "iolabs_foo"
23 _DEFAULT_FILENAME = "foo.default.json"
24 _CONTEXT = "foo config"
25
1926
20 class FooGroundConfig(config_loader.ConfigModel):27 class FooGroundConfig(config_loader.ConfigModel):
21 cell_m: float = 0.528 cell_m: float = 0.5
22 enabled: bool = True29 enabled: bool = True
Importance #2: src/iolabs/common/config_model.py @@ -27,38 +34,66 @@
27 ground: FooGroundConfig = FooGroundConfig()34 ground: FooGroundConfig = FooGroundConfig()
2835
2936
30 class FooConfigError(config_loader.ConfigError):37 class FooConfigError(config_loader.ConfigError):
31 \"\"\"Raised for an invalid foo config.\"\"\"38 \"\"\"Raised when foo config contains unsupported keys or values.\"\"\"
3239
3340
34 def build_foo_config(41 def build_foo_config(
42 *,
35 overrides: Mapping[str, Any] | None = None,43 overrides: Mapping[str, Any] | None = None,
36 config_path: str | Path | None = None,44 config_path: str | Path | None = None,
37 ) -> dict[str, Any]:45 ) -> dict[str, Any]:
38 return config_loader.load_config(46 return config_loader.load_config(
39 FooConfig,47 FooConfig,
40 package="iolabs_foo",48 package=_PACKAGE_NAME,
41 filename="default_config.json",49 filename=_DEFAULT_FILENAME,
42 overrides=overrides,50 overrides=overrides,
43 config_path=config_path,51 config_path=config_path,
44 context="foo config",52 context=_CONTEXT,
45 error_cls=FooConfigError,53 error_cls=FooConfigError,
46 ).model_dump()54 ).model_dump()
4755
48Aliased fields must be dumped with ``model_dump(by_alias=True)``, so the result56Naming is part of the pattern. The root model is ``<Name>Config`` and every
49stays keyed by the config-file keys rather than the Python field names.57nested section is ``<Name><Section>Config`` (``FooGroundConfig``, never a bare
58``GroundConfig``): same-named models collide in the process-global registry that
59resolves the ``Allowed keys: ...`` hints, and a lanefinder run imports many
60packages into one process. Each package declares exactly one error class,
61``<Name>ConfigError``, and passes it as ``error_cls=`` to every call in this
62module. The three module constants above replace inline ``package=`` /
63``filename=`` / ``context=`` literals, and ``_CONTEXT`` names the package
64(``"mask clustering config"``, not ``"config"``). Entry points take their
65``overrides`` and ``config_path`` arguments keyword-only.
66
67Do not use field aliases: the config-file key is the Python field name. The
68``alias`` / `pydantic.AliasChoices` handling in this module -- and the
69``model_dump(by_alias=True)`` that an aliased model would need in order to stay
70keyed by config-file keys -- is a documented escape hatch, not fleet practice.
5071
51Nest sections as `ConfigModel` subclasses, not stdlib dataclasses: a dataclass72Nest sections as `ConfigModel` subclasses, not stdlib dataclasses: a dataclass
52section is coerced, but its errors do not carry the section path. Writers that73section is coerced, but its errors do not carry the section path. Writers that
53serialise a config (run stats, provenance) should use ``model_dump(mode="json")``74serialise a config (run stats, provenance) should use ``model_dump(mode="json")``
54so tuples, paths and enums become JSON-native values.75so tuples, paths and enums become JSON-native values.
5576
77A user-supplied JSON override file is read with
78:func:`iolabs.common.config_loader.load_json_overrides`, which raises the
79package error class -- never with a hand-rolled reader.
80
56Unknown keys are rejected (``extra="forbid"``) with the same message shape as81Unknown keys are rejected (``extra="forbid"``) with the same message shape as
57:func:`iolabs.common.config_loader.validate_allowed_keys`; instances are frozen,82:func:`iolabs.common.config_loader.validate_allowed_keys`; instances are frozen,
58so no field can be rebound after construction. Freezing is shallow, as in83so no field can be rebound after construction. Freezing is shallow, as in
59pydantic itself: a ``list``-valued field is still a mutable list, so prefer84pydantic itself: a ``list``-valued field is still a mutable list, so sequence
60``tuple`` for sequence fields that must not change.85fields are declared ``tuple[...]``. That is a hard rule for a package whose
86entry points return the model; ``list`` is acceptable only where the entry point
87returns a plain dict that the caller is meant to mutate.
88
89Package tests live in ``tests/test_config.py`` under fixed names:
90``test_model_defaults_match_packaged_json`` (whole-dict parity between
91``<X>Config().model_dump()`` and the packaged JSON),
92``test_load_<x>_config_returns_packaged_defaults``,
93``test_error_class_is_config_error``, ``test_unknown_top_level_key_is_rejected``,
94``test_unknown_nested_key_is_rejected``, ``test_overrides_deep_merge_onto_defaults``
95and ``test_set_override_coercion_and_rejection``.
61"""96"""
6297
63from __future__ import annotations98from __future__ import annotations
6499
Importance #3: src/iolabs/common/config_model.py @@ -418,10 +453,34 @@

Sanctioned reader for user-supplied JSON override files; raises the package error class.

418 )453 )
419 return loaded454 return loaded
420455
421456
422def _load_json_file(path: Path, *, error_cls: type[ValueError]) -> dict[str, Any]:457def load_json_overrides(
423 """Load a JSON config file, wrapping decode errors in *error_cls*."""458 path: str | Path,
459 *,
460 error_cls: type[ValueError] = config_loader.ConfigError,
461) -> dict[str, Any]:
462 """Read a user-supplied JSON config/override file as a plain mapping.
463
464 This is the one sanctioned way for a package to read an override file the
465 user passed on the command line; the result is normally handed to
466 :func:`load_config` as ``overrides=`` (merge semantics) or used directly.
467 Packages must not hand-roll their own reader.
468
469 Args:
470 path: Path to the JSON file.
471 error_cls: Exception class raised for malformed JSON or a non-object
472 top level. Defaults to `iolabs.common.config_loader.ConfigError`;
473 packages pass their own ``<Name>ConfigError``.
474
475 Returns:
476 The decoded JSON object as a dict.
477
478 Raises:
479 error_cls: The file is not valid JSON, or does not hold a JSON object.
480 OSError: The file could not be read.
481 """
482 path = Path(path)
424 try:483 try:
425 with path.open("r", encoding="utf-8") as handle:484 with path.open("r", encoding="utf-8") as handle:
426 loaded = json.load(handle)485 loaded = json.load(handle)
427 except json.JSONDecodeError as exc:486 except json.JSONDecodeError as exc:
Importance #4: src/iolabs/common/config_model.py @@ -202,9 +237,9 @@
202 error_cls: The JSON is malformed, or the merged config is invalid.237 error_cls: The JSON is malformed, or the merged config is invalid.
203 OSError: The config file could not be read.238 OSError: The config file could not be read.
204 """239 """
205 if config_path is not None:240 if config_path is not None:
206 raw = _load_json_file(Path(config_path), error_cls=error_cls)241 raw = load_json_overrides(config_path, error_cls=error_cls)
207 logger.debug("Loaded %s from %s", context, config_path)242 logger.debug("Loaded %s from %s", context, config_path)
208 else:243 else:
209 raw = _load_packaged(package, filename, error_cls=error_cls)244 raw = _load_packaged(package, filename, error_cls=error_cls)
210 logger.debug("Loaded %s defaults from %s:%s", context, package, filename)245 logger.debug("Loaded %s defaults from %s:%s", context, package, filename)
Importance #5: src/iolabs/common/config_loader.py @@ -6,15 +6,24 @@
66
7The current way to declare a config is a pydantic model derived from7The current way to declare a config is a pydantic model derived from
8`ConfigModel`, validated by :func:`load_config`; both are defined in8`ConfigModel`, validated by :func:`load_config`; both are defined in
9:mod:`iolabs.common.config_model` and re-exported here, together with9:mod:`iolabs.common.config_model` and re-exported here, together with
10:func:`validate_config` and :func:`format_validation_error`. The canonical10:func:`validate_config`, :func:`format_validation_error` and
11package pattern is::11:func:`load_json_overrides`. The canonical package pattern is::
1212
13 import logging
14 from collections.abc import Mapping
15 from pathlib import Path
13 from typing import Any, Literal16 from typing import Any, Literal
1417
15 from iolabs.common import config_loader18 from iolabs.common import config_loader
1619
20 logger = logging.getLogger(__name__)
21
22 _PACKAGE_NAME = "iolabs_foo"
23 _DEFAULT_FILENAME = "foo.default.json"
24 _CONTEXT = "foo config"
25
1726
18 class FooGroundConfig(config_loader.ConfigModel):27 class FooGroundConfig(config_loader.ConfigModel):
19 cell_m: float = 0.528 cell_m: float = 0.5
2029
Importance #6: src/iolabs/common/config_loader.py @@ -24,22 +33,31 @@
24 ground: FooGroundConfig = FooGroundConfig()33 ground: FooGroundConfig = FooGroundConfig()
2534
2635
27 class FooConfigError(config_loader.ConfigError):36 class FooConfigError(config_loader.ConfigError):
28 \"\"\"Raised for an invalid foo config.\"\"\"37 \"\"\"Raised when foo config contains unsupported keys or values.\"\"\"
2938
3039
31 def build_foo_config(overrides=None, config_path=None) -> dict[str, Any]:40 def build_foo_config(
41 *,
42 overrides: Mapping[str, Any] | None = None,
43 config_path: str | Path | None = None,
44 ) -> dict[str, Any]:
32 return config_loader.load_config(45 return config_loader.load_config(
33 FooConfig,46 FooConfig,
34 package="iolabs_foo",47 package=_PACKAGE_NAME,
35 filename="default_config.json",48 filename=_DEFAULT_FILENAME,
36 overrides=overrides,49 overrides=overrides,
37 config_path=config_path,50 config_path=config_path,
38 context="foo config",51 context=_CONTEXT,
39 error_cls=FooConfigError,52 error_cls=FooConfigError,
40 ).model_dump()53 ).model_dump()
4154
55The naming rules (``<Name>Config`` root, ``<Name><Section>Config`` sections, one
56``<Name>ConfigError`` per package), the keyword-only entry-point signature, the
57"no field aliases" rule and the canonical package test names are spelled out in
58:mod:`iolabs.common.config_model`.
59
42`ConfigError`, `default_config_path`, `load_packaged_json`, `deep_merge_dicts`60`ConfigError`, `default_config_path`, `load_packaged_json`, `deep_merge_dicts`
43and `parse_set_overrides` stay first-class. The hand-rolled dataclass helpers61and `parse_set_overrides` stay first-class. The hand-rolled dataclass helpers
44(`validate_allowed_keys`, `coerce_to_field_type`, `dataclass_from_mapping`,62(`validate_allowed_keys`, `coerce_to_field_type`, `dataclass_from_mapping`,
45`validate_against_defaults`) still work for published leaf wheels but emit a63`validate_against_defaults`) still work for published leaf wheels but emit a
Importance #7: src/iolabs/common/config_loader.py @@ -68,9 +86,15 @@
68 get_type_hints,86 get_type_hints,
69)87)
7088
71_PYDANTIC_EXPORTS = frozenset(89_PYDANTIC_EXPORTS = frozenset(
72 {"ConfigModel", "load_config", "validate_config", "format_validation_error"}90 {
91 "ConfigModel",
92 "load_config",
93 "validate_config",
94 "format_validation_error",
95 "load_json_overrides",
96 }
73)97)
7498
7599
76class ConfigError(ValueError):100class ConfigError(ValueError):
Importance #8: tests/test_config_model.py @@ -570,11 +570,15 @@
570570
571def test_lazy_reexports_are_visible_to_dir() -> None:571def test_lazy_reexports_are_visible_to_dir() -> None:
572 """The lazily re-exported pydantic names show up in dir(config_loader)."""572 """The lazily re-exported pydantic names show up in dir(config_loader)."""
573 names = dir(config_loader)573 names = dir(config_loader)
574 assert {"ConfigModel", "load_config", "validate_config", "format_validation_error"} <= set(574 assert {
575 names575 "ConfigModel",
576 )576 "load_config",
577 "validate_config",
578 "format_validation_error",
579 "load_json_overrides",
580 } <= set(names)
577 assert names == sorted(names)581 assert names == sorted(names)
578582
579583
580def test_coerce_config_value_does_not_warn() -> None:584def test_coerce_config_value_does_not_warn() -> None:
Importance #9: tests/test_config_model.py @@ -699,4 +703,37 @@
699703
700 with pytest.raises(config_loader.ConfigError) as excinfo:704 with pytest.raises(config_loader.ConfigError) as excinfo:
701 config_model.validate_config(ChoicesConfig, {"bogus": 1}, context="cfg")705 config_model.validate_config(ChoicesConfig, {"bogus": 1}, context="cfg")
702 assert "Allowed keys: count, n-points" in str(excinfo.value)706 assert "Allowed keys: count, n-points" in str(excinfo.value)
707
708
709def test_load_json_overrides_reads_a_json_object(tmp_path: Path) -> None:
710 """load_json_overrides returns the decoded object of a user config file."""
711 path = tmp_path / "overrides.json"
712 path.write_text(json.dumps({"mode": "exact", "ground": {"cell_m": 0.25}}), encoding="utf-8")
713 assert config_loader.load_json_overrides(path) == {
714 "mode": "exact",
715 "ground": {"cell_m": 0.25},
716 }
717
718
719def test_load_json_overrides_accepts_a_string_path(tmp_path: Path) -> None:
720 """A ``str`` path is accepted, like every other config_path argument."""
721 path = tmp_path / "overrides.json"
722 path.write_text('{"mode": "fast"}', encoding="utf-8")
723 assert config_loader.load_json_overrides(str(path)) == {"mode": "fast"}
724
725
726def test_load_json_overrides_rejects_malformed_json(tmp_path: Path) -> None:
727 """Malformed JSON is reported through the caller's error class."""
728 path = tmp_path / "overrides.json"
729 path.write_text("{not json", encoding="utf-8")
730 with pytest.raises(SampleConfigError, match="Invalid JSON in config file"):
731 config_loader.load_json_overrides(path, error_cls=SampleConfigError)
732
733
734def test_load_json_overrides_rejects_non_object_top_level(tmp_path: Path) -> None:
735 """A JSON array is not a config mapping."""
736 path = tmp_path / "overrides.json"
737 path.write_text("[1, 2]", encoding="utf-8")
738 with pytest.raises(SampleConfigError, match="must hold a JSON object"):
739 config_loader.load_json_overrides(path, error_cls=SampleConfigError)
Importance #10: README.md @@ -20,14 +20,21 @@
20fleet coercion matrix for `bool`/`int`/`float`/`str`) and are built with20fleet coercion matrix for `bool`/`int`/`float`/`str`) and are built with
21`config_loader.load_config(...)`:21`config_loader.load_config(...)`:
2222
23```python23```python
24import logging
24from collections.abc import Mapping25from collections.abc import Mapping
25from pathlib import Path26from pathlib import Path
26from typing import Any, Literal27from typing import Any, Literal
2728
28from iolabs.common import config_loader29from iolabs.common import config_loader
2930
31logger = logging.getLogger(__name__)
32
33_PACKAGE_NAME = "iolabs_foo"
34_DEFAULT_FILENAME = "foo.default.json"
35_CONTEXT = "foo config"
36
3037
31class FooGroundConfig(config_loader.ConfigModel):38class FooGroundConfig(config_loader.ConfigModel):
32 cell_m: float = 0.539 cell_m: float = 0.5
3340
Importance #11: README.md @@ -37,32 +44,57 @@
37 ground: FooGroundConfig = FooGroundConfig()44 ground: FooGroundConfig = FooGroundConfig()
3845
3946
40class FooConfigError(config_loader.ConfigError):47class FooConfigError(config_loader.ConfigError):
41 """Raised for an invalid foo config."""48 """Raised when foo config contains unsupported keys or values."""
4249
4350
44def build_foo_config(51def build_foo_config(
52 *,
45 overrides: Mapping[str, Any] | None = None,53 overrides: Mapping[str, Any] | None = None,
46 config_path: str | Path | None = None,54 config_path: str | Path | None = None,
47) -> dict[str, Any]:55) -> dict[str, Any]:
48 return config_loader.load_config(56 return config_loader.load_config(
49 FooConfig,57 FooConfig,
50 package="iolabs_foo",58 package=_PACKAGE_NAME,
51 filename="default_config.json",59 filename=_DEFAULT_FILENAME,
52 overrides=overrides,60 overrides=overrides,
53 config_path=config_path,61 config_path=config_path,
54 context="foo config",62 context=_CONTEXT,
55 error_cls=FooConfigError,63 error_cls=FooConfigError,
56 ).model_dump()64 ).model_dump()
57```65```
5866
59A model whose fields carry aliases must dump with ``model_dump(by_alias=True)``,67Naming is part of the pattern: the root model is `<Name>Config`, nested sections are
60otherwise the returned mapping is keyed by the Python field names and no longer68`<Name><Section>Config` (`FooGroundConfig`, never a bare `GroundConfig` — same-named
61round-trips through the config JSON.69models collide in the registry that resolves the `Allowed keys: ...` hints), and each
70package declares exactly one `<Name>ConfigError` that is passed as `error_cls=` to every
71shared-layer call. The three module constants replace inline `package=` / `filename=` /
72`context=` literals; entry points take `overrides` and `config_path` keyword-only.
73
74Do not use field aliases: the config-file key is the Python field name. The alias
75handling (and the `model_dump(by_alias=True)` an aliased model would need to stay keyed
76by config-file keys) is a documented escape hatch, not fleet practice.
77
78A user-supplied JSON override file is read with
79`config_loader.load_json_overrides(path, error_cls=FooConfigError)` — packages must not
80hand-roll their own reader.
81
82**To add a config key: add the field (with its type, default and any `Field` range) to
83the model and the same key with the same default to the packaged JSON — nothing else.**
84Unknown keys are rejected. Runtime overrides come from repeatable `--set KEY=VALUE`,
85never repo-local JSON.
6286
63The hand-rolled dataclass helpers (`validate_allowed_keys`, `coerce_to_field_type`,87The hand-rolled dataclass helpers (`validate_allowed_keys`, `coerce_to_field_type`,
64`dataclass_from_mapping`, `validate_against_defaults`) still work for published leaf88`dataclass_from_mapping`, `validate_against_defaults`) still work for published leaf
65wheels but emit a `DeprecationWarning` (0.9.0).89wheels but emit a `DeprecationWarning` (0.9.0).
6690
67Models are frozen, but only shallowly (as in pydantic itself): use `tuple` rather91Models are frozen, but only shallowly (as in pydantic itself): sequence fields are
68than `list` for sequence fields that must not be mutated after construction.92declared `tuple[...]`, not `list[...]`. That is a hard rule for a package whose entry
93points return the model; `list` is acceptable only where the entry point returns a plain
94dict the caller is meant to mutate.
95
96Package tests live in `tests/test_config.py` under fixed names:
97`test_model_defaults_match_packaged_json` (whole-dict parity with the packaged JSON),
98`test_load_<x>_config_returns_packaged_defaults`, `test_error_class_is_config_error`,
99`test_unknown_top_level_key_is_rejected`, `test_unknown_nested_key_is_rejected`,
100`test_overrides_deep_merge_onto_defaults`, `test_set_override_coercion_and_rejection`.
Importance #12: src/iolabs/common/config_loader.py @@ -6,15 +6,24 @@
66
7The current way to declare a config is a pydantic model derived from7The current way to declare a config is a pydantic model derived from
8`ConfigModel`, validated by :func:`load_config`; both are defined in8`ConfigModel`, validated by :func:`load_config`; both are defined in
9:mod:`iolabs.common.config_model` and re-exported here, together with9:mod:`iolabs.common.config_model` and re-exported here, together with
10:func:`validate_config` and :func:`format_validation_error`. The canonical10:func:`validate_config`, :func:`format_validation_error` and
11package pattern is::11:func:`load_json_overrides`. The canonical package pattern is::
1212
13 import logging
14 from collections.abc import Mapping
15 from pathlib import Path
13 from typing import Any, Literal16 from typing import Any, Literal
1417
15 from iolabs.common import config_loader18 from iolabs.common import config_loader
1619
20 logger = logging.getLogger(__name__)
21
22 _PACKAGE_NAME = "iolabs_foo"
23 _DEFAULT_FILENAME = "foo.default.json"
24 _CONTEXT = "foo config"
25
1726
18 class FooGroundConfig(config_loader.ConfigModel):27 class FooGroundConfig(config_loader.ConfigModel):
19 cell_m: float = 0.528 cell_m: float = 0.5
2029
Importance #13: src/iolabs/common/config_loader.py @@ -24,22 +33,31 @@
24 ground: FooGroundConfig = FooGroundConfig()33 ground: FooGroundConfig = FooGroundConfig()
2534
2635
27 class FooConfigError(config_loader.ConfigError):36 class FooConfigError(config_loader.ConfigError):
28 \"\"\"Raised for an invalid foo config.\"\"\"37 \"\"\"Raised when foo config contains unsupported keys or values.\"\"\"
2938
3039
31 def build_foo_config(overrides=None, config_path=None) -> dict[str, Any]:40 def build_foo_config(
41 *,
42 overrides: Mapping[str, Any] | None = None,
43 config_path: str | Path | None = None,
44 ) -> dict[str, Any]:
32 return config_loader.load_config(45 return config_loader.load_config(
33 FooConfig,46 FooConfig,
34 package="iolabs_foo",47 package=_PACKAGE_NAME,
35 filename="default_config.json",48 filename=_DEFAULT_FILENAME,
36 overrides=overrides,49 overrides=overrides,
37 config_path=config_path,50 config_path=config_path,
38 context="foo config",51 context=_CONTEXT,
39 error_cls=FooConfigError,52 error_cls=FooConfigError,
40 ).model_dump()53 ).model_dump()
4154
55The naming rules (``<Name>Config`` root, ``<Name><Section>Config`` sections, one
56``<Name>ConfigError`` per package), the keyword-only entry-point signature, the
57"no field aliases" rule and the canonical package test names are spelled out in
58:mod:`iolabs.common.config_model`.
59
42`ConfigError`, `default_config_path`, `load_packaged_json`, `deep_merge_dicts`60`ConfigError`, `default_config_path`, `load_packaged_json`, `deep_merge_dicts`
43and `parse_set_overrides` stay first-class. The hand-rolled dataclass helpers61and `parse_set_overrides` stay first-class. The hand-rolled dataclass helpers
44(`validate_allowed_keys`, `coerce_to_field_type`, `dataclass_from_mapping`,62(`validate_allowed_keys`, `coerce_to_field_type`, `dataclass_from_mapping`,
45`validate_against_defaults`) still work for published leaf wheels but emit a63`validate_against_defaults`) still work for published leaf wheels but emit a
Importance #14: src/iolabs/common/config_loader.py @@ -68,9 +86,15 @@
68 get_type_hints,86 get_type_hints,
69)87)
7088
71_PYDANTIC_EXPORTS = frozenset(89_PYDANTIC_EXPORTS = frozenset(
72 {"ConfigModel", "load_config", "validate_config", "format_validation_error"}90 {
91 "ConfigModel",
92 "load_config",
93 "validate_config",
94 "format_validation_error",
95 "load_json_overrides",
96 }
73)97)
7498
7599
76class ConfigError(ValueError):100class ConfigError(ValueError):
Importance #15: src/iolabs/common/config_model.py @@ -9,14 +9,21 @@
9pydantic itself.9pydantic itself.
1010
11Canonical package pattern::11Canonical package pattern::
1212
13 import logging
13 from collections.abc import Mapping14 from collections.abc import Mapping
14 from pathlib import Path15 from pathlib import Path
15 from typing import Any, Literal16 from typing import Any, Literal
1617
17 from iolabs.common import config_loader18 from iolabs.common import config_loader
1819
20 logger = logging.getLogger(__name__)
21
22 _PACKAGE_NAME = "iolabs_foo"
23 _DEFAULT_FILENAME = "foo.default.json"
24 _CONTEXT = "foo config"
25
1926
20 class FooGroundConfig(config_loader.ConfigModel):27 class FooGroundConfig(config_loader.ConfigModel):
21 cell_m: float = 0.528 cell_m: float = 0.5
22 enabled: bool = True29 enabled: bool = True
Importance #16: src/iolabs/common/config_model.py @@ -27,38 +34,66 @@
27 ground: FooGroundConfig = FooGroundConfig()34 ground: FooGroundConfig = FooGroundConfig()
2835
2936
30 class FooConfigError(config_loader.ConfigError):37 class FooConfigError(config_loader.ConfigError):
31 \"\"\"Raised for an invalid foo config.\"\"\"38 \"\"\"Raised when foo config contains unsupported keys or values.\"\"\"
3239
3340
34 def build_foo_config(41 def build_foo_config(
42 *,
35 overrides: Mapping[str, Any] | None = None,43 overrides: Mapping[str, Any] | None = None,
36 config_path: str | Path | None = None,44 config_path: str | Path | None = None,
37 ) -> dict[str, Any]:45 ) -> dict[str, Any]:
38 return config_loader.load_config(46 return config_loader.load_config(
39 FooConfig,47 FooConfig,
40 package="iolabs_foo",48 package=_PACKAGE_NAME,
41 filename="default_config.json",49 filename=_DEFAULT_FILENAME,
42 overrides=overrides,50 overrides=overrides,
43 config_path=config_path,51 config_path=config_path,
44 context="foo config",52 context=_CONTEXT,
45 error_cls=FooConfigError,53 error_cls=FooConfigError,
46 ).model_dump()54 ).model_dump()
4755
48Aliased fields must be dumped with ``model_dump(by_alias=True)``, so the result56Naming is part of the pattern. The root model is ``<Name>Config`` and every
49stays keyed by the config-file keys rather than the Python field names.57nested section is ``<Name><Section>Config`` (``FooGroundConfig``, never a bare
58``GroundConfig``): same-named models collide in the process-global registry that
59resolves the ``Allowed keys: ...`` hints, and a lanefinder run imports many
60packages into one process. Each package declares exactly one error class,
61``<Name>ConfigError``, and passes it as ``error_cls=`` to every call in this
62module. The three module constants above replace inline ``package=`` /
63``filename=`` / ``context=`` literals, and ``_CONTEXT`` names the package
64(``"mask clustering config"``, not ``"config"``). Entry points take their
65``overrides`` and ``config_path`` arguments keyword-only.
66
67Do not use field aliases: the config-file key is the Python field name. The
68``alias`` / `pydantic.AliasChoices` handling in this module -- and the
69``model_dump(by_alias=True)`` that an aliased model would need in order to stay
70keyed by config-file keys -- is a documented escape hatch, not fleet practice.
5071
51Nest sections as `ConfigModel` subclasses, not stdlib dataclasses: a dataclass72Nest sections as `ConfigModel` subclasses, not stdlib dataclasses: a dataclass
52section is coerced, but its errors do not carry the section path. Writers that73section is coerced, but its errors do not carry the section path. Writers that
53serialise a config (run stats, provenance) should use ``model_dump(mode="json")``74serialise a config (run stats, provenance) should use ``model_dump(mode="json")``
54so tuples, paths and enums become JSON-native values.75so tuples, paths and enums become JSON-native values.
5576
77A user-supplied JSON override file is read with
78:func:`iolabs.common.config_loader.load_json_overrides`, which raises the
79package error class -- never with a hand-rolled reader.
80
56Unknown keys are rejected (``extra="forbid"``) with the same message shape as81Unknown keys are rejected (``extra="forbid"``) with the same message shape as
57:func:`iolabs.common.config_loader.validate_allowed_keys`; instances are frozen,82:func:`iolabs.common.config_loader.validate_allowed_keys`; instances are frozen,
58so no field can be rebound after construction. Freezing is shallow, as in83so no field can be rebound after construction. Freezing is shallow, as in
59pydantic itself: a ``list``-valued field is still a mutable list, so prefer84pydantic itself: a ``list``-valued field is still a mutable list, so sequence
60``tuple`` for sequence fields that must not change.85fields are declared ``tuple[...]``. That is a hard rule for a package whose
86entry points return the model; ``list`` is acceptable only where the entry point
87returns a plain dict that the caller is meant to mutate.
88
89Package tests live in ``tests/test_config.py`` under fixed names:
90``test_model_defaults_match_packaged_json`` (whole-dict parity between
91``<X>Config().model_dump()`` and the packaged JSON),
92``test_load_<x>_config_returns_packaged_defaults``,
93``test_error_class_is_config_error``, ``test_unknown_top_level_key_is_rejected``,
94``test_unknown_nested_key_is_rejected``, ``test_overrides_deep_merge_onto_defaults``
95and ``test_set_override_coercion_and_rejection``.
61"""96"""
6297
63from __future__ import annotations98from __future__ import annotations
6499
Importance #17: src/iolabs/common/config_model.py @@ -202,9 +237,9 @@
202 error_cls: The JSON is malformed, or the merged config is invalid.237 error_cls: The JSON is malformed, or the merged config is invalid.
203 OSError: The config file could not be read.238 OSError: The config file could not be read.
204 """239 """
205 if config_path is not None:240 if config_path is not None:
206 raw = _load_json_file(Path(config_path), error_cls=error_cls)241 raw = load_json_overrides(config_path, error_cls=error_cls)
207 logger.debug("Loaded %s from %s", context, config_path)242 logger.debug("Loaded %s from %s", context, config_path)
208 else:243 else:
209 raw = _load_packaged(package, filename, error_cls=error_cls)244 raw = _load_packaged(package, filename, error_cls=error_cls)
210 logger.debug("Loaded %s defaults from %s:%s", context, package, filename)245 logger.debug("Loaded %s defaults from %s:%s", context, package, filename)
Importance #18: src/iolabs/common/config_model.py @@ -418,10 +453,34 @@

Sanctioned reader for user-supplied JSON override files; raises the package error class.

418 )453 )
419 return loaded454 return loaded
420455
421456
422def _load_json_file(path: Path, *, error_cls: type[ValueError]) -> dict[str, Any]:457def load_json_overrides(
423 """Load a JSON config file, wrapping decode errors in *error_cls*."""458 path: str | Path,
459 *,
460 error_cls: type[ValueError] = config_loader.ConfigError,
461) -> dict[str, Any]:
462 """Read a user-supplied JSON config/override file as a plain mapping.
463
464 This is the one sanctioned way for a package to read an override file the
465 user passed on the command line; the result is normally handed to
466 :func:`load_config` as ``overrides=`` (merge semantics) or used directly.
467 Packages must not hand-roll their own reader.
468
469 Args:
470 path: Path to the JSON file.
471 error_cls: Exception class raised for malformed JSON or a non-object
472 top level. Defaults to `iolabs.common.config_loader.ConfigError`;
473 packages pass their own ``<Name>ConfigError``.
474
475 Returns:
476 The decoded JSON object as a dict.
477
478 Raises:
479 error_cls: The file is not valid JSON, or does not hold a JSON object.
480 OSError: The file could not be read.
481 """
482 path = Path(path)
424 try:483 try:
425 with path.open("r", encoding="utf-8") as handle:484 with path.open("r", encoding="utf-8") as handle:
426 loaded = json.load(handle)485 loaded = json.load(handle)
427 except json.JSONDecodeError as exc:486 except json.JSONDecodeError as exc:
Importance #19: tests/test_config_model.py @@ -570,11 +570,15 @@
570570
571def test_lazy_reexports_are_visible_to_dir() -> None:571def test_lazy_reexports_are_visible_to_dir() -> None:
572 """The lazily re-exported pydantic names show up in dir(config_loader)."""572 """The lazily re-exported pydantic names show up in dir(config_loader)."""
573 names = dir(config_loader)573 names = dir(config_loader)
574 assert {"ConfigModel", "load_config", "validate_config", "format_validation_error"} <= set(574 assert {
575 names575 "ConfigModel",
576 )576 "load_config",
577 "validate_config",
578 "format_validation_error",
579 "load_json_overrides",
580 } <= set(names)
577 assert names == sorted(names)581 assert names == sorted(names)
578582
579583
580def test_coerce_config_value_does_not_warn() -> None:584def test_coerce_config_value_does_not_warn() -> None:
Importance #20: tests/test_config_model.py @@ -699,4 +703,37 @@
699703
700 with pytest.raises(config_loader.ConfigError) as excinfo:704 with pytest.raises(config_loader.ConfigError) as excinfo:
701 config_model.validate_config(ChoicesConfig, {"bogus": 1}, context="cfg")705 config_model.validate_config(ChoicesConfig, {"bogus": 1}, context="cfg")
702 assert "Allowed keys: count, n-points" in str(excinfo.value)706 assert "Allowed keys: count, n-points" in str(excinfo.value)
707
708
709def test_load_json_overrides_reads_a_json_object(tmp_path: Path) -> None:
710 """load_json_overrides returns the decoded object of a user config file."""
711 path = tmp_path / "overrides.json"
712 path.write_text(json.dumps({"mode": "exact", "ground": {"cell_m": 0.25}}), encoding="utf-8")
713 assert config_loader.load_json_overrides(path) == {
714 "mode": "exact",
715 "ground": {"cell_m": 0.25},
716 }
717
718
719def test_load_json_overrides_accepts_a_string_path(tmp_path: Path) -> None:
720 """A ``str`` path is accepted, like every other config_path argument."""
721 path = tmp_path / "overrides.json"
722 path.write_text('{"mode": "fast"}', encoding="utf-8")
723 assert config_loader.load_json_overrides(str(path)) == {"mode": "fast"}
724
725
726def test_load_json_overrides_rejects_malformed_json(tmp_path: Path) -> None:
727 """Malformed JSON is reported through the caller's error class."""
728 path = tmp_path / "overrides.json"
729 path.write_text("{not json", encoding="utf-8")
730 with pytest.raises(SampleConfigError, match="Invalid JSON in config file"):
731 config_loader.load_json_overrides(path, error_cls=SampleConfigError)
732
733
734def test_load_json_overrides_rejects_non_object_top_level(tmp_path: Path) -> None:
735 """A JSON array is not a config mapping."""
736 path = tmp_path / "overrides.json"
737 path.write_text("[1, 2]", encoding="utf-8")
738 with pytest.raises(SampleConfigError, match="must hold a JSON object"):
739 config_loader.load_json_overrides(path, error_cls=SampleConfigError)