mlsegmentation e60e7ec: AI3D-379 Pydantic config models via iolabs-common ConfigModel

Importance #1: src/train/config.py @@ -491,41 +175,8 @@

491	document: str	175	document: str
492	config: HarnessConfig	176	config: HarnessConfig
493		177
494		178
495	_ROOT_KEYS = {
496	"schema_version",
497	"experiment",
498	"study",
499	"seed",
500	"task",
501	"data",
502	"adapter",
503	"model",
504	"loss",
505	"train",
506	"evaluation",
507	"runtime",
508	"provenance",
509	"visualization",
510	}
511	_REQUIRED_ROOT_KEYS = _ROOT_KEYS - {"visualization"}
512	_RAW_ATTRIBUTE = "_raw_document_mapping"
513	_FIELD_TYPE_CACHE: dict[type[Any], Mapping[str, Any]] = {}
514	_GATE_PARAM_CLASSES: Mapping[str, type[GateParams]] = MappingProxyType(
515	{
516	"spt_partition_oracle": SptPartitionOracleParams,
517	"implementation_ticket": ImplementationTicketParams,
518	"artifact_exists": ArtifactExistsParams,
519	"data_available": DataAvailableParams,
520	"license_approval": LicenseApprovalParams,
521	"operational_smoke": OperationalSmokeParams,
522	"checkpoint_policy": CheckpointPolicyParams,
523	"human_workflow": HumanWorkflowParams,
524	}
525	)
526
527
528	def load_config(path: str \| Path) -> HarnessConfig:	179	def load_config(path: str \| Path) -> HarnessConfig:
529	"""Load and strictly validate one E1--E14 experiment YAML.	180	"""Load and strictly validate one E1--E14 experiment YAML.
530		181
531	Args:	182	Args:

Importance #2: src/train/config.py @@ -610,34 +261,20 @@

     return tuple(cells)
 def _build_config(raw: Any, config_path: Path, sha256: str) -> HarnessConfig:
-    root = _mapping(raw, str(config_path))
+    """Validate one raw document into a configuration carrying its digest."""
-    _keys(root, _REQUIRED_ROOT_KEYS, _ROOT_KEYS, str(config_path))
+    root = config_values.mapping(raw, str(config_path))
-    config = HarnessConfig(
+    payload = {
-        schema_version=_integer(root["schema_version"], "schema_version"),
+        **root,
-        experiment=_parse_experiment(root["experiment"]),
+        config_sections.SOURCE_PATH_ALIAS: config_path,
-        study=_parse_study(root["study"]),
+        config_sections.SHA256_ALIAS: sha256,
-        seed=_integer(root["seed"], "seed"),
+    }
-        task=_parse_task(root["task"]),
+    config = config_loader.validate_config(
-        data=_parse_data(root["data"]),
+        HarnessConfig, payload, context=_CONTEXT, error_cls=ConfigError
-        adapter=_parse_adapter(root["adapter"]),
-        model=_parse_model(root["model"]),
-        loss=_parse_loss(root["loss"]),
-        train=_parse_train(root["train"], _mapping(root["model"], "model")),
-        evaluation=_parse_evaluation(root["evaluation"]),
-        runtime=_parse_runtime(root["runtime"]),
-        provenance=_parse_provenance(root["provenance"]),
-        visualization=(
-            None
-            if "visualization" not in root
-            else _parse_visualization(root["visualization"])
-        ),
-        source_path=config_path,
-        sha256=sha256,
     )
-    _validate_config(config, root)
+    config_rules.validate_config(config, root)
-    object.__setattr__(config, _RAW_ATTRIBUTE, copy.deepcopy(dict(root)))
+    config._raw_document = copy.deepcopy(dict(root))
     return config
 def _raw_document(config: HarnessConfig) -> Mapping[str, Any]:

Importance #3: src/train/config_schema.py @@ -0,0 +1,332 @@

	1	"""Pydantic models for the experiment identity, gate, and study blocks.
	2
	3	Every model derives from :class:`StrictConfigModel`, the repository's
	4	fail-closed flavour of :class:`iolabs.common.config_loader.ConfigModel`:
	5	unknown keys are rejected, instances are frozen, and leaf values must already
	6	carry their declared YAML type (see :mod:`src.train.config_values`).
	7	"""
	8
	9	from __future__ import annotations
	10
	11	import logging
	12	from collections.abc import Mapping
	13	from pathlib import Path
	14	from typing import Annotated, Any, Literal, TypeAlias
	15
	16	import pydantic
	17	from iolabs.common import config_loader
	18
	19	from src.train import config_values
	20
	21	logger = logging.getLogger(__name__)
	22
	23	ExperimentStatus: TypeAlias = Literal["implement-now", "template-only", "gated-later"]
	24	StudyKind: TypeAlias = Literal["single", "variants", "matrix", "sweep"]
	25
	26
	27	class StrictConfigModel(config_loader.ConfigModel):
	28	"""Fleet config model whose leaves keep this repository's strict typing."""
	29
	30	@pydantic.field_validator("*", mode="before")
	31	@classmethod
	32	def _coerce_fleet_scalars(cls, value: Any, info: pydantic.ValidationInfo) -> Any:
	33	"""Parse a raw YAML value strictly as the field's declared type."""
	34	name = info.field_name or ""
	35	field = cls.model_fields.get(name)
	36	if field is None or field.annotation is None:
	37	return value
	38	return config_values.typed_value(field.annotation, value, name)
	39
	40
	41	class SptPartitionOracleParams(StrictConfigModel):
	42	"""Required SPT partition-purity report and per-class thresholds."""
	43
	44	report: Path
	45	minimum_purity_by_class: Mapping[str, float]
	46
	47
	48	class ImplementationTicketParams(StrictConfigModel):
	49	"""Implementation ticket whose external state must reach a required value."""
	50
	51	ticket: str
	52	required_status: str
	53
	54
	55	class ArtifactExistsParams(StrictConfigModel):
	56	"""Required local artifact and optional expected JSON status."""
	57
	58	path: Path
	59	expected_status: str \| None = None
	60
	61
	62	class DataAvailableParams(StrictConfigModel):
	63	"""Required data manifest and declared dataset contract."""
	64
	65	manifest: Path
	66	dataset: str
	67
	68
	69	class LicenseApprovalParams(StrictConfigModel):
	70	"""Required license-review decision artifact."""
	71
	72	decision: Path
	73	required_decision: str
	74
	75
	76	class OperationalSmokeParams(StrictConfigModel):
	77	"""Required operational-smoke report and accepted result."""
	78
	79	report: Path
	80	required_status: str
	81
	82
	83	class CheckpointPolicyParams(StrictConfigModel):
	84	"""Required checkpoint-policy file and checkpoint metadata artifact."""
	85
	86	policy: Path
	87	metadata: Path
	88
	89
	90	class HumanWorkflowParams(StrictConfigModel):
	91	"""Required human-workflow protocol and acceptance artifact."""
	92
	93	protocol: Path
	94	acceptance: Path
	95
	96
	97	GateParams: TypeAlias = (
	98	SptPartitionOracleParams
	99	\| ImplementationTicketParams
	100	\| ArtifactExistsParams
	101	\| DataAvailableParams
	102	\| LicenseApprovalParams
	103	\| OperationalSmokeParams
	104	\| CheckpointPolicyParams
	105	\| HumanWorkflowParams
	106	)
	107
	108
	109	class GateBase(StrictConfigModel):
	110	"""Shared identity of one typed, fail-closed experiment gate."""
	111
	112	name: str
	113
	114
	115	class SptPartitionOracleGate(GateBase):
	116	"""Gate on an SPT partition-purity report."""
	117
	118	type: Literal["spt_partition_oracle"]
	119	required: bool
	120	params: SptPartitionOracleParams
	121
	122
	123	class ImplementationTicketGate(GateBase):
	124	"""Gate on an external implementation ticket."""
	125
	126	type: Literal["implementation_ticket"]
	127	required: bool
	128	params: ImplementationTicketParams
	129
	130
	131	class ArtifactExistsGate(GateBase):
	132	"""Gate on a local artifact and its optional status."""
	133
	134	type: Literal["artifact_exists"]
	135	required: bool
	136	params: ArtifactExistsParams
	137
	138
	139	class DataAvailableGate(GateBase):
	140	"""Gate on a declared dataset manifest."""
	141
	142	type: Literal["data_available"]
	143	required: bool
	144	params: DataAvailableParams
	145
	146
	147	class LicenseApprovalGate(GateBase):
	148	"""Gate on a license-review decision."""
	149
	150	type: Literal["license_approval"]
	151	required: bool
	152	params: LicenseApprovalParams
	153
	154
	155	class OperationalSmokeGate(GateBase):
	156	"""Gate on an operational-smoke report."""
	157
	158	type: Literal["operational_smoke"]
	159	required: bool
	160	params: OperationalSmokeParams
	161
	162
	163	class CheckpointPolicyGate(GateBase):
	164	"""Gate on the checkpoint policy and its metadata artifact."""
	165
	166	type: Literal["checkpoint_policy"]
	167	required: bool
	168	params: CheckpointPolicyParams
	169
	170
	171	class HumanWorkflowGate(GateBase):
	172	"""Gate on a human-workflow protocol and its acceptance artifact."""
	173
	174	type: Literal["human_workflow"]
	175	required: bool
	176	params: HumanWorkflowParams
	177
	178
	179	def _gate_tag(value: Any) -> str \| None:
	180	"""Return the declared gate type a raw or parsed gate is tagged with."""
	181	if isinstance(value, Mapping):
	182	tag = value.get("type")
	183	return tag if isinstance(tag, str) else None
	184	return getattr(value, "type", None)
	185
	186
	187	GateConfig: TypeAlias = Annotated[
	188	Annotated[SptPartitionOracleGate, pydantic.Tag("spt_partition_oracle")]
	189	\| Annotated[ImplementationTicketGate, pydantic.Tag("implementation_ticket")]
	190	\| Annotated[ArtifactExistsGate, pydantic.Tag("artifact_exists")]
	191	\| Annotated[DataAvailableGate, pydantic.Tag("data_available")]
	192	\| Annotated[LicenseApprovalGate, pydantic.Tag("license_approval")]
	193	\| Annotated[OperationalSmokeGate, pydantic.Tag("operational_smoke")]
	194	\| Annotated[CheckpointPolicyGate, pydantic.Tag("checkpoint_policy")]
	195	\| Annotated[HumanWorkflowGate, pydantic.Tag("human_workflow")],
	196	pydantic.Discriminator(_gate_tag),
	197	]
	198
	199
	200	class ExperimentConfig(StrictConfigModel):
	201	"""Experiment identity, readiness, hypothesis, metrics, and gates."""
	202
	203	id: str
	204	name: str
	205	phase: int
	206	status: ExperimentStatus
	207	hypothesis: str
	208	deciding_metrics: tuple[str, ...]
	209	gates: tuple[GateConfig, ...]
	210
	211
	212	class VariantConfig(StrictConfigModel):
	213	"""One named study variant expressed as typed dotted-path overrides."""
	214
	215	id: str
	216	overrides: Mapping[str, config_values.OverrideValue]
	217	tags: tuple[str, ...] = ()
	218
	219
	220	class MatrixConfig(StrictConfigModel):
	221	"""Typed Cartesian matrix definition with optional cells."""
	222
	223	axes: Mapping[str, tuple[config_values.OverrideValue, ...]]
	224	include: tuple[Mapping[str, config_values.OverrideValue], ...] = ()
	225	exclude: tuple[Mapping[str, config_values.OverrideValue], ...] = ()
	226
	227	@pydantic.field_validator("axes")
	228	@classmethod
	229	def _axes_are_populated(
	230	cls, value: Mapping[str, tuple[config_values.OverrideValue, ...]]
	231	) -> Mapping[str, tuple[config_values.OverrideValue, ...]]:
	232	"""Reject an axis that declares no value."""
	233	for path, values in value.items():
	234	if not values:
	235	raise config_values.ConfigError(
	236	f"study.matrix.axes.{path} cannot be empty"
	237	)
	238	return value
	239
	240
	241	class SweepParameterConfig(StrictConfigModel):
	242	"""One finite or bounded sweep parameter."""
	243
	244	values: tuple[config_values.OverrideValue, ...] \| None = None
	245	minimum: float \| None = None
	246	maximum: float \| None = None
	247	distribution: str \| None = None
	248
	249	@pydantic.model_validator(mode="after")
	250	def _bounds_are_complete(self) -> SweepParameterConfig:
	251	"""Reject a parameter that is neither finite nor fully bounded."""
	252	if self.values is not None and not self.values:
	253	raise config_values.ConfigError("values cannot be empty")
	254	if self.values is None and (
	255	self.minimum is None or self.maximum is None or self.distribution is None
	256	):
	257	raise config_values.ConfigError(
	258	"requires values or minimum/maximum/distribution"
	259	)
	260	if (
	261	self.minimum is not None
	262	and self.maximum is not None
	263	and self.minimum >= self.maximum
	264	):
	265	raise config_values.ConfigError("minimum must be smaller than maximum")
	266	return self
	267
	268
	269	class SweepConfig(StrictConfigModel):
	270	"""Typed bounded sweep contract."""
	271
	272	method: str
	273	parameters: Mapping[str, SweepParameterConfig]
	274	budget: int = pydantic.Field(ge=1)
	275	objective: str
	276
	277	@pydantic.field_validator("parameters")
	278	@classmethod
	279	def _parameters_are_declared(
	280	cls, value: Mapping[str, SweepParameterConfig]
	281	) -> Mapping[str, SweepParameterConfig]:
	282	"""Reject a sweep that declares no parameter."""
	283	if not value:
	284	raise config_values.ConfigError("study.sweep.parameters cannot be empty")
	285	return value
	286
	287
	288	class ContrastConfig(StrictConfigModel):
	289	"""Predeclared comparison between two study cells."""
	290
	291	name: str
	292	left: str
	293	right: str
	294	metric: str
	295
	296
	297	class StudyConfig(StrictConfigModel):
	298	"""Discriminated single, variants, matrix, or sweep study definition."""
	299
	300	kind: StudyKind
	301	variants: tuple[VariantConfig, ...]
	302	matrix: MatrixConfig \| None
	303	sweep: SweepConfig \| None
	304	contrasts: tuple[ContrastConfig, ...]
	305
	306	@pydantic.model_validator(mode="after")
	307	def _kind_matches_its_payload(self) -> StudyConfig:
	308	"""Reject a study whose kind and payload disagree."""
	309	if self.kind == "single" and (
	310	self.variants or self.matrix is not None or self.sweep is not None
	311	):
	312	raise config_values.ConfigError(
	313	"study.kind single cannot carry variants, matrix, or sweep"
	314	)
	315	if self.kind == "variants" and (
	316	not self.variants or self.matrix is not None or self.sweep is not None
	317	):
	318	raise config_values.ConfigError(
	319	"study.kind variants requires only a non-empty variants payload"
	320	)
	321	if self.kind == "matrix" and (
	322	self.variants or self.matrix is None or self.sweep is not None
	323	):
	324	raise config_values.ConfigError("study.kind matrix requires only matrix")
	325	if self.kind == "sweep" and (
	326	self.variants or self.matrix is not None or self.sweep is None
	327	):
	328	raise config_values.ConfigError("study.kind sweep requires only sweep")
	329	identities = [item.id for item in self.variants]
	330	if len(set(identities)) != len(identities):
	331	raise config_values.ConfigError("study.variants contains duplicate IDs")
	332	return self
0

Importance #4: src/train/config.py @@ -1,475 +1,159 @@

1	"""Strict experiment configuration schema for corridor segmentation studies."""	1	"""Strict experiment configuration schema for corridor segmentation studies.
		2
		3	The schema itself is a pydantic model tree built on
		4	:class:`iolabs.common.config_loader.ConfigModel`: :mod:`src.train.config_schema`
		5	holds the experiment/gate/study models, :mod:`src.train.config_sections` the
		6	data, model, training, and evaluation blocks. This module is the entry point
		7	every script imports: it loads one YAML document, validates it, applies the
		8	rules of :mod:`src.train.config_rules`, and expands a study into re-validated
		9	cells with :mod:`src.train.config_study`.
		10
		11	Adding a configuration key means adding a field to its model (and to the YAML
		12	documents under ``configs/``); nothing else has to be touched.
		13	"""
2		14
3	from __future__ import annotations	15	from __future__ import annotations
4		16
5	import copy	17	import copy
6	import hashlib	18	import hashlib
7	import itertools	19	import logging
8	import math	20	from collections.abc import Mapping
9	import random	21	from dataclasses import dataclass
10	import re
11	from collections.abc import Mapping, Sequence
12	from dataclasses import MISSING, asdict, dataclass, field, fields, is_dataclass
13	from pathlib import Path	22	from pathlib import Path
14	from types import MappingProxyType, UnionType	23	from types import MappingProxyType
15	from typing import (	24	from typing import Any
16	Any,
17	Literal,
18	TypeAlias,
19	Union,
20	get_args,
21	get_origin,
22	get_type_hints,
23	)
24		25
25	import yaml	26	import yaml
		27	from iolabs.common import config_loader
26		28
27	from src.contracts.ontology import (	29	from src.train import (
28	Ontology,	30	config_rules,
29	OntologyError,	31	config_sections,
30	load_ontology,	32	config_study,
31	macro_interest_all_suffix,	33	config_values,
32	)	34	)
33		35	from src.train.config_rules import is_canonical_metric, resolve_ontology_path
34	try:	36	from src.train.config_schema import (
35	from iolabs_ml_harness.config import TrainerConfig as _TrainerConfig	37	ArtifactExistsGate,
36	except ModuleNotFoundError:	38	ArtifactExistsParams,
37		39	CheckpointPolicyGate,
38	@dataclass	40	CheckpointPolicyParams,
39	class _TrainerConfig: # type: ignore[no-redef]	41	ContrastConfig,
40	"""Core-compatible fallback used when the optional ML extra is absent."""	42	DataAvailableGate,
41		43	DataAvailableParams,
42	max_epochs: int = -1	44	ExperimentConfig,
43	lr: float = 3.0e-4	45	ExperimentStatus,
44	weight_decay: float = 1.0e-4	46	GateBase,
45	precision: str = "auto"	47	GateConfig,
46	accumulate_grad_batches: int = 1	48	GateParams,
47	accelerator: str = "auto"	49	HumanWorkflowGate,
48	devices: int \| str = 1	50	HumanWorkflowParams,
49	viz_every_n_epochs: int = 2	51	ImplementationTicketGate,
50	viz_samples: int = 4	52	ImplementationTicketParams,
51	monitor: str = "val/f1_mean_fg"	53	LicenseApprovalGate,
52	monitor_mode: str = "max"	54	LicenseApprovalParams,
53	early_stop_monitor: str = "val/loss"	55	MatrixConfig,
54	early_stop_mode: str = "min"	56	OperationalSmokeGate,
55	early_stop_patience: int = 4	57	OperationalSmokeParams,
56	log_dir: str = "runs"	58	SptPartitionOracleGate,
57	log_every_n_steps: int = 10	59	SptPartitionOracleParams,
58		60	StrictConfigModel,
59		61	StudyConfig,
60	Scalar: TypeAlias = str \| int \| float \| bool \| None	62	StudyKind,
61	OverrideValue: TypeAlias = Scalar \| list[Scalar]	63	SweepConfig,
62	ExperimentStatus: TypeAlias = Literal["implement-now", "template-only", "gated-later"]	64	SweepParameterConfig,
63	StudyKind: TypeAlias = Literal["single", "variants", "matrix", "sweep"]	65	VariantConfig,
64
65
66	class ConfigError(ValueError):
67	"""Raised when an experiment configuration violates its strict schema."""
68
69
70	@dataclass(frozen=True)
71	class VariantConfig:
72	"""One named study variant expressed as typed dotted-path overrides."""
73
74	id: str
75	overrides: Mapping[str, OverrideValue]
76	tags: tuple[str, ...] = ()
77
78
79	@dataclass(frozen=True)
80	class MatrixConfig:
81	"""Typed Cartesian matrix definition with optional cells."""
82
83	axes: Mapping[str, tuple[OverrideValue, ...]]
84	include: tuple[Mapping[str, OverrideValue], ...] = ()
85	exclude: tuple[Mapping[str, OverrideValue], ...] = ()
86
87
88	@dataclass(frozen=True)
89	class SweepParameterConfig:
90	"""One finite or bounded sweep parameter."""
91
92	values: tuple[OverrideValue, ...] \| None = None
93	minimum: float \| None = None
94	maximum: float \| None = None
95	distribution: str \| None = None
96
97
98	@dataclass(frozen=True)
99	class SweepConfig:
100	"""Typed bounded sweep contract."""
101
102	method: str
103	parameters: Mapping[str, SweepParameterConfig]
104	budget: int
105	objective: str
106
107
108	@dataclass(frozen=True)
109	class ContrastConfig:
110	"""Predeclared comparison between two study cells."""
111
112	name: str
113	left: str
114	right: str
115	metric: str
116
117
118	@dataclass(frozen=True)
119	class StudyConfig:
120	"""Discriminated single, variants, matrix, or sweep study definition."""
121
122	kind: StudyKind
123	variants: tuple[VariantConfig, ...]
124	matrix: MatrixConfig \| None
125	sweep: SweepConfig \| None
126	contrasts: tuple[ContrastConfig, ...]
127
128
129	@dataclass(frozen=True)
130	class SptPartitionOracleParams:
131	"""Required SPT partition-purity report and per-class thresholds."""
132
133	report: Path
134	minimum_purity_by_class: Mapping[str, float]
135
136
137	@dataclass(frozen=True)
138	class ImplementationTicketParams:
139	"""Implementation ticket whose external state must reach a required value."""
140
141	ticket: str
142	required_status: str
143
144
145	@dataclass(frozen=True)
146	class ArtifactExistsParams:
147	"""Required local artifact and optional expected JSON status."""
148
149	path: Path
150	expected_status: str \| None = None
151
152
153	@dataclass(frozen=True)
154	class DataAvailableParams:
155	"""Required data manifest and declared dataset contract."""
156
157	manifest: Path
158	dataset: str
159
160
161	@dataclass(frozen=True)
162	class LicenseApprovalParams:
163	"""Required license-review decision artifact."""
164
165	decision: Path
166	required_decision: str
167
168
169	@dataclass(frozen=True)
170	class OperationalSmokeParams:
171	"""Required operational-smoke report and accepted result."""
172
173	report: Path
174	required_status: str
175
176
177	@dataclass(frozen=True)
178	class CheckpointPolicyParams:
179	"""Required checkpoint-policy file and checkpoint metadata artifact."""
180
181	policy: Path
182	metadata: Path
183
184
185	@dataclass(frozen=True)
186	class HumanWorkflowParams:
187	"""Required human-workflow protocol and acceptance artifact."""
188
189	protocol: Path
190	acceptance: Path
191
192
193	GateParams: TypeAlias = (
194	SptPartitionOracleParams
195	\| ImplementationTicketParams
196	\| ArtifactExistsParams
197	\| DataAvailableParams
198	\| LicenseApprovalParams
199	\| OperationalSmokeParams
200	\| CheckpointPolicyParams
201	\| HumanWorkflowParams
202	)	66	)
203		67	from src.train.config_sections import (
204		68	AdapterConfig,
205	@dataclass(frozen=True)	69	BootstrapConfig,
206	class GateConfig:	70	ContinuityConfig,
207	"""One typed, fail-closed experiment gate."""	71	CorridorSelectionConfig,
208		72	DataConfig,
209	name: str	73	EvaluationConfig,
210	type: str	74	FeatureConfig,
211	required: bool	75	HarnessConfig,
212	params: GateParams	76	LabelSourceConfig,
213		77	LossConfig,
214		78	ModelConfig,
215	@dataclass(frozen=True)	79	ObjectMatchingConfig,
216	class ExperimentConfig:	80	PointceptAdapterConfig,
217	"""Experiment identity, readiness, hypothesis, metrics, and gates."""	81	PromotionConfig,
218		82	ProvenanceConfig,
219	id: str	83	RuntimeConfig,
220	name: str	84	SptAdapterConfig,
221	phase: int	85	TaskConfig,
222	status: ExperimentStatus	86	TilingConfig,
223	hypothesis: str	87	TrainConfig,
224	deciding_metrics: tuple[str, ...]	88	VisualizationConfig,
225	gates: tuple[GateConfig, ...]	89	)
226		90	from src.train.config_values import ConfigError, OverrideValue, Scalar
227		91
228	@dataclass(frozen=True)	92	logger = logging.getLogger(__name__)
229	class TaskConfig:	93
230	"""Frozen semantic task contract."""	94	_CONTEXT = "experiment config"
231		95
232	ontology: Path	96	__all__ = [
233	num_classes: int	97	"AdapterConfig",
234	ignore_index: int	98	"ArtifactExistsGate",
235	classes_of_interest: tuple[int, ...]	99	"ArtifactExistsParams",
236	linear_classes: tuple[str, ...]	100	"BootstrapConfig",
237		101	"CheckpointPolicyGate",
238		102	"CheckpointPolicyParams",
239	@dataclass(frozen=True)	103	"ConfigError",
240	class LabelSourceConfig:	104	"ContinuityConfig",
241	"""Label provenance and fail-closed join policy."""	105	"ContrastConfig",
242		106	"CorridorSelectionConfig",
243	mode: Literal["artifact", "regenerate_full_resolution"]	107	"DataAvailableGate",
244	source_geometry_glob: str	108	"DataAvailableParams",
245	fuse_config: Path \| None	109	"DataConfig",
246	prefer: str	110	"EvaluationConfig",
247	classical_glob: str \| None	111	"ExperimentConfig",
248	recap_glob: str \| None	112	"ExperimentStatus",
249	stats_glob: str	113	"FeatureConfig",
250	unmatched_policy: Literal["void"]	114	"GateBase",
251	max_unmatched_fraction: float	115	"GateConfig",
252		116	"GateParams",
253		117	"HarnessConfig",
254	@dataclass(frozen=True)	118	"HumanWorkflowGate",
255	class CorridorSelectionConfig:	119	"HumanWorkflowParams",
256	"""Config-declared corridor allow-list."""	120	"ImplementationTicketGate",
257		121	"ImplementationTicketParams",
258	include: tuple[str, ...]	122	"LabelSourceConfig",
259		123	"LicenseApprovalGate",
260		124	"LicenseApprovalParams",
261	@dataclass(frozen=True)	125	"LossConfig",
262	class FeatureConfig:	126	"MatrixConfig",
263	"""Ordered features and train-only normalization contract."""	127	"ModelConfig",
264		128	"ObjectMatchingConfig",
265	names: tuple[str, ...]	129	"OperationalSmokeGate",
266	normalization_manifest: Path	130	"OperationalSmokeParams",
267	fit_on: Literal["train_corridors_only"]	131	"OverrideValue",
268	scanner_conditioning: bool	132	"PointceptAdapterConfig",
269		133	"PromotionConfig",
270		134	"ProvenanceConfig",
271	@dataclass(frozen=True)	135	"RuntimeConfig",
272	class TilingConfig:	136	"Scalar",
273	"""Deterministic corridor tiling and overlap blending contract."""	137	"SptAdapterConfig",
274		138	"SptPartitionOracleGate",
275	mode: Literal["corridor_axis"]	139	"SptPartitionOracleParams",
276	length_m: float	140	"StrictConfigModel",
277	overlap_m: float	141	"StudyCell",
278	origin: Literal["dataset_manifest"]	142	"StudyConfig",
279	min_points: int	143	"StudyKind",
280	blend: Literal["linear_edge_weight"]	144	"SweepConfig",
281		145	"SweepParameterConfig",
282		146	"TaskConfig",
283	@dataclass(frozen=True)	147	"TilingConfig",
284	class DataConfig:	148	"TrainConfig",
285	"""Input roots, splits, labels, features, and tiling."""	149	"VariantConfig",
286		150	"VisualizationConfig",
287	root: Path	151	"expand_study",
288	canonical_root: Path	152	"is_canonical_metric",
289	processed_root: Path	153	"load_config",
290	split_manifest: Path	154	"resolve_ontology_path",
291	label_source: LabelSourceConfig	155	]
292	corridors: CorridorSelectionConfig
293	features: FeatureConfig
294	tiling: TilingConfig
295
296
297	@dataclass(frozen=True)
298	class SptAdapterConfig:
299	"""SPT raw-dataset emission contract."""
300
301	raw_root: Path
302	pc_tiling: int
303	voxel_m: float
304	base_family: str
305	raw_row_sidecar_keys: tuple[str, ...]
306	audit_only_data_keys: tuple[str, ...]
307
308
309	@dataclass(frozen=True)
310	class PointceptAdapterConfig:
311	"""Pointcept default-dataset emission contract."""
312
313	root: Path
314	grid_size_m: float
315	preserve_keys: tuple[str, ...]
316
317
318	@dataclass(frozen=True)
319	class AdapterConfig:
320	"""Framework-neutral and external-format adapter contract."""
321
322	emit: tuple[str, ...]
323	canonical_version: int
324	identity: Literal["source_file_and_row"]
325	spt: SptAdapterConfig
326	pointcept: PointceptAdapterConfig
327
328
329	@dataclass(frozen=True)
330	class ModelConfig:
331	"""Local or external model/runner selection."""
332
333	framework: Literal["cpu", "spt", "pointcept"]
334	runner: Path
335	name: str
336	base_config: Path \| None
337	checkout_env: str \| None
338	commit_env: str \| None
339	checkpoint: Path \| None
340	args: Mapping[str, Any]
341
342
343	@dataclass(frozen=True)
344	class LossConfig:
345	"""Shared-registry or external-native loss selection."""
346
347	name: str
348	args: Mapping[str, Any]
349
350
351	@dataclass
352	class TrainConfig(_TrainerConfig):
353	"""Harness-visible training settings for local and external runners."""
354
355	batch_size: int = 1
356	num_workers: int = 4
357	optimizer: str = "adamw"
358	scheduler: str = "cosine"
359	distributed: bool = False
360
361
362	@dataclass(frozen=True)
363	class ContinuityConfig:
364	"""Linear-continuity binning contract."""
365
366	chainage_bin_m: float
367	gap_threshold_m: float
368
369
370	@dataclass(frozen=True)
371	class ObjectMatchingConfig:
372	"""Object-clustering profile source and fallback tolerances."""
373
374	cluster_profiles: Path
375	minimum_iou: float
376	centroid_tolerance_m: float
377
378
379	@dataclass(frozen=True)
380	class BootstrapConfig:
381	"""Corridor/spatial bootstrap contract."""
382
383	unit: Literal["corridor"]
384	spatial_block_m: float
385	samples: int
386	confidence: float
387	seed: int
388
389
390	@dataclass(frozen=True)
391	class PromotionConfig:
392	"""Locked-test promotion margins and superiority conditions."""
393
394	enabled: bool
395	delta_quality: float \| None
396	delta_fp_per_km: float \| None
397	superiority_conditions: tuple[str, ...]
398
399
400	@dataclass(frozen=True)
401	class EvaluationConfig:
402	"""Held-out evaluation and promotion protocol."""
403
404	split: Literal["validation", "promotion_test"]
405	metrics: tuple[str, ...]
406	precision_floors: Mapping[str, float]
407	continuity: ContinuityConfig
408	object_matching: ObjectMatchingConfig
409	bootstrap: BootstrapConfig
410	promotion: PromotionConfig
411	worst_k_tiles: int = 4
412
413
414	@dataclass(frozen=True)
415	class RuntimeConfig:
416	"""Operational hardware and kernel constraints."""
417
418	target: str
419	cuda: str
420	spconv: str
421	flash_attention: bool
422	system_ram_gb: int
423	gpu_memory_gb: int
424
425
426	@dataclass(frozen=True)
427	class ProvenanceConfig:
428	"""Mandatory run-evidence policy."""
429
430	manifest: Literal["required"]
431	data_hash_source: Literal["dvc"]
432	record_environment: bool
433	record_commands: bool
434	checkpoint_policy: Path
435
436
437	@dataclass(frozen=True)
438	class VisualizationConfig:
439	"""Opt-in training-time TensorBoard class-mask visualization."""
440
441	masks_every_n_epochs: int
442	masks_tiles: int \| tuple[str, ...] = 2
443
444
445	@dataclass(frozen=True)
446	class HarnessConfig:
447	"""Fully parsed, cross-field-validated experiment configuration."""
448
449	schema_version: int
450	experiment: ExperimentConfig
451	study: StudyConfig
452	seed: int
453	task: TaskConfig
454	data: DataConfig
455	adapter: AdapterConfig
456	model: ModelConfig
457	loss: LossConfig
458	train: TrainConfig
459	evaluation: EvaluationConfig
460	runtime: RuntimeConfig
461	provenance: ProvenanceConfig
462	source_path: Path = field(compare=False)
463	sha256: str = field(compare=False)
464	visualization: VisualizationConfig \| None = None
465
466	def as_dict(self) -> dict[str, Any]:
467	"""Return the resolved dataclass tree as JSON-safe primitives."""
468	payload = _json_safe(self)
469	if not isinstance(payload, dict): # pragma: no cover - defensive
470	raise ConfigError("Resolved configuration is not a mapping")
471	return payload
472		156
473		157
474	@dataclass(frozen=True)	158	@dataclass(frozen=True)
475	class StudyCell:	159	class StudyCell:

Importance #5: src/train/config.py @@ -569,9 +220,9 @@

569	ConfigError: If the study payload, an override, a cell identity, or a	220	ConfigError: If the study payload, an override, a cell identity, or a
570	resolved cell configuration violates the strict schema.	221	resolved cell configuration violates the strict schema.
571	"""	222	"""
572	raw = _raw_document(config)	223	raw = _raw_document(config)
573	definitions = _cell_definitions(config)	224	definitions = config_study.cell_definitions(config)
574	if not definitions:	225	if not definitions:
575	raise ConfigError(	226	raise ConfigError(
576	f"{config.experiment.id} study kind {config.study.kind} expanded to "	227	f"{config.experiment.id} study kind {config.study.kind} expanded to "
577	"no cells"	228	"no cells"

Importance #6: src/train/config.py @@ -584,9 +235,9 @@

584	)	235	)
585	cells: list[StudyCell] = []	236	cells: list[StudyCell] = []
586	for index, (cell_id, overrides) in enumerate(definitions):	237	for index, (cell_id, overrides) in enumerate(definitions):
587	where = f"{config.experiment.id} study cell {cell_id!r}"	238	where = f"{config.experiment.id} study cell {cell_id!r}"
588	cell_raw = _apply_overrides(raw, overrides, where)	239	cell_raw = config_study.apply_overrides(raw, overrides, where)
589	document = yaml.safe_dump(cell_raw, sort_keys=True, default_flow_style=False)	240	document = yaml.safe_dump(cell_raw, sort_keys=True, default_flow_style=False)
590	digest = hashlib.sha256(document.encode("utf-8")).hexdigest()	241	digest = hashlib.sha256(document.encode("utf-8")).hexdigest()
591	try:	242	try:
592	cell_config = _build_config(cell_raw, config.source_path, digest)	243	cell_config = _build_config(cell_raw, config.source_path, digest)

Importance #7: src/train/config.py @@ -652,9 +289,9 @@

652		289
653	Raises:	290	Raises:
654	ConfigError: If the source file must be re-read and cannot be parsed.	291	ConfigError: If the source file must be re-read and cannot be parsed.
655	"""	292	"""
656	stashed = getattr(config, _RAW_ATTRIBUTE, None)	293	stashed = config._raw_document
657	if stashed is not None:	294	if stashed is not None:
658	return stashed	295	return stashed
659	try:	296	try:
660	payload = config.source_path.read_bytes()	297	payload = config.source_path.read_bytes()

Importance #8: src/train/config.py @@ -662,1437 +299,5 @@

662	except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc:	299	except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc:
663	raise ConfigError(	300	raise ConfigError(
664	f"Cannot re-read config {config.source_path}: {exc}"	301	f"Cannot re-read config {config.source_path}: {exc}"
665	) from exc	302	) from exc
666	return _mapping(raw, str(config.source_path))	303	return config_values.mapping(raw, str(config.source_path))
667
668
669	def is_canonical_metric(name: str, *, ontology: Ontology) -> bool:
670	"""Return whether a metric belongs to an ontology's val/eval namespace.
671
672	Args:
673	name: Metric tag to validate.
674	ontology: Ontology whose predicted class names and interest count fix
675	the per-class tags and the structural macro suffix.
676
677	Returns:
678	True for a canonical tag, otherwise False.
679	"""
680	if name == "val/loss":
681	return True
682	match = re.fullmatch(r"(val\|eval)/(.+)", name)
683	if match is None:
684	return False
685	metric = match.group(2)
686	suffix = macro_interest_all_suffix(ontology)
687	fixed = {
688	"iou_macro_interest",
689	"f1_macro_interest",
690	"precision_macro_interest",
691	"recall_macro_interest",
692	f"iou_macro_interest_{suffix}",
693	f"f1_macro_interest_{suffix}",
694	f"precision_macro_interest_{suffix}",
695	f"recall_macro_interest_{suffix}",
696	"miou_all_classes",
697	}
698	if metric in fixed:
699	return True
700	class_name = "\|".join(re.escape(item) for item in ontology.class_names)
701	patterns = (
702	rf"(?:iou\|precision\|recall\|f1\|support)_(?:{class_name})",
703	rf"(?:fp_per_km\|detections_per_km\|matched_recall)_(?:{class_name})",
704	rf"continuity_(?:covered_fraction\|total_missing_length_m\|gaps\|"
705	rf"gap_median_m\|gap_p95_m\|gap_max_m)_(?:{class_name})",
706	rf"seam_[a-zA-Z0-9_.-]+_(?:{class_name})",
707	)
708	return any(re.fullmatch(pattern, metric) is not None for pattern in patterns)
709
710
711	def _parse_experiment(value: Any) -> ExperimentConfig:
712	raw = _section(
713	value,
714	{"id", "name", "phase", "status", "hypothesis", "deciding_metrics", "gates"},
715	"experiment",
716	)
717	gates = tuple(
718	_parse_gate(item, index)
719	for index, item in enumerate(
720	_sequence(raw["gates"], "experiment.gates")
721	)
722	)
723	return ExperimentConfig(
724	id=_string(raw["id"], "experiment.id"),
725	name=_string(raw["name"], "experiment.name"),
726	phase=_integer(raw["phase"], "experiment.phase"),
727	status=_choice(
728	raw["status"],
729	{"implement-now", "template-only", "gated-later"},
730	"experiment.status",
731	),
732	hypothesis=_string(raw["hypothesis"], "experiment.hypothesis"),
733	deciding_metrics=_strings(
734	raw["deciding_metrics"], "experiment.deciding_metrics"
735	),
736	gates=gates,
737	)
738
739
740	def _parse_gate(value: Any, index: int) -> GateConfig:
741	where = f"experiment.gates[{index}]"
742	raw = _section(value, {"name", "type", "required", "params"}, where)
743	gate_type = _string(raw["type"], f"{where}.type")
744	try:
745	params_class = _GATE_PARAM_CLASSES[gate_type]
746	except KeyError as exc:
747	raise ConfigError(
748	f"{where}.type has unsupported gate type {gate_type!r}"
749	) from exc
750	params = _dataclass_section(params_class, raw["params"], f"{where}.params")
751	return GateConfig(
752	name=_string(raw["name"], f"{where}.name"),
753	type=gate_type,
754	required=_boolean(raw["required"], f"{where}.required"),
755	params=params,
756	)
757
758
759	def _parse_study(value: Any) -> StudyConfig:
760	raw = _section(
761	value,
762	{"kind", "variants", "matrix", "sweep", "contrasts"},
763	"study",
764	)
765	kind = _choice(
766	raw["kind"], {"single", "variants", "matrix", "sweep"}, "study.kind"
767	)
768	variants = tuple(
769	_parse_variant(item, index)
770	for index, item in enumerate(
771	_sequence(raw["variants"], "study.variants")
772	)
773	)
774	matrix = None if raw["matrix"] is None else _parse_matrix(raw["matrix"])
775	sweep = None if raw["sweep"] is None else _parse_sweep(raw["sweep"])
776	contrasts = tuple(
777	_dataclass_section(
778	ContrastConfig, item, f"study.contrasts[{index}]"
779	)
780	for index, item in enumerate(
781	_sequence(raw["contrasts"], "study.contrasts")
782	)
783	)
784	if kind == "single" and (variants or matrix is not None or sweep is not None):
785	raise ConfigError("study.kind single cannot carry variants, matrix, or sweep")
786	if kind == "variants" and (
787	not variants or matrix is not None or sweep is not None
788	):
789	raise ConfigError(
790	"study.kind variants requires only a non-empty variants payload"
791	)
792	if kind == "matrix" and (variants or matrix is None or sweep is not None):
793	raise ConfigError("study.kind matrix requires only matrix")
794	if kind == "sweep" and (variants or matrix is not None or sweep is None):
795	raise ConfigError("study.kind sweep requires only sweep")
796	variant_ids = [item.id for item in variants]
797	if len(set(variant_ids)) != len(variant_ids):
798	raise ConfigError("study.variants contains duplicate IDs")
799	return StudyConfig(kind, variants, matrix, sweep, contrasts)
800
801
802	def _parse_variant(value: Any, index: int) -> VariantConfig:
803	where = f"study.variants[{index}]"
804	raw = _mapping(value, where)
805	allowed = {"id", "overrides", "tags"}
806	required = {"id", "overrides"}
807	_keys(raw, required, allowed, where)
808	return VariantConfig(
809	id=_string(raw["id"], f"{where}.id"),
810	overrides=_overrides(raw["overrides"], f"{where}.overrides"),
811	tags=_strings(raw.get("tags", []), f"{where}.tags"),
812	)
813
814
815	def _parse_matrix(value: Any) -> MatrixConfig:
816	raw = _mapping(value, "study.matrix")
817	_keys(raw, {"axes"}, {"axes", "include", "exclude"}, "study.matrix")
818	axes_raw = _mapping(raw["axes"], "study.matrix.axes")
819	axes: dict[str, tuple[OverrideValue, ...]] = {}
820	for path, values in axes_raw.items():
821	sequence = tuple(
822	_override_value(item, f"study.matrix.axes.{path}")
823	for item in _sequence(values, f"study.matrix.axes.{path}")
824	)
825	if not sequence:
826	raise ConfigError(f"study.matrix.axes.{path} cannot be empty")
827	axes[_string(path, "study.matrix axis")] = sequence
828	return MatrixConfig(
829	axes=MappingProxyType(axes),
830	include=tuple(
831	_overrides(item, f"study.matrix.include[{index}]")
832	for index, item in enumerate(
833	_sequence(raw.get("include", []), "study.matrix.include")
834	)
835	),
836	exclude=tuple(
837	_overrides(item, f"study.matrix.exclude[{index}]")
838	for index, item in enumerate(
839	_sequence(raw.get("exclude", []), "study.matrix.exclude")
840	)
841	),
842	)
843
844
845	def _parse_sweep(value: Any) -> SweepConfig:
846	raw = _section(
847	value,
848	{"method", "parameters", "budget", "objective"},
849	"study.sweep",
850	)
851	parameters_raw = _mapping(raw["parameters"], "study.sweep.parameters")
852	parameters = {
853	path: _parse_sweep_parameter(
854	item, f"study.sweep.parameters.{path}"
855	)
856	for path, item in parameters_raw.items()
857	}
858	if not parameters:
859	raise ConfigError("study.sweep.parameters cannot be empty")
860	return SweepConfig(
861	method=_string(raw["method"], "study.sweep.method"),
862	parameters=MappingProxyType(parameters),
863	budget=_integer(raw["budget"], "study.sweep.budget"),
864	objective=_string(raw["objective"], "study.sweep.objective"),
865	)
866
867
868	def _parse_sweep_parameter(value: Any, where: str) -> SweepParameterConfig:
869	raw = _mapping(value, where)
870	_keys(raw, set(), {"values", "minimum", "maximum", "distribution"}, where)
871	values = None
872	if "values" in raw:
873	values = tuple(
874	_override_value(item, f"{where}.values")
875	for item in _sequence(raw["values"], f"{where}.values")
876	)
877	if not values:
878	raise ConfigError(f"{where}.values cannot be empty")
879	minimum = _optional_float(raw.get("minimum"), f"{where}.minimum")
880	maximum = _optional_float(raw.get("maximum"), f"{where}.maximum")
881	distribution = (
882	None
883	if raw.get("distribution") is None
884	else _string(raw["distribution"], f"{where}.distribution")
885	)
886	if values is None and (
887	minimum is None or maximum is None or distribution is None
888	):
889	raise ConfigError(f"{where} requires values or minimum/maximum/distribution")
890	if minimum is not None and maximum is not None and minimum >= maximum:
891	raise ConfigError(f"{where}.minimum must be smaller than maximum")
892	return SweepParameterConfig(values, minimum, maximum, distribution)
893
894
895	def _parse_task(value: Any) -> TaskConfig:
896	raw = _section(
897	value,
898	{
899	"ontology",
900	"num_classes",
901	"ignore_index",
902	"classes_of_interest",
903	"linear_classes",
904	},
905	"task",
906	)
907	return TaskConfig(
908	_path(raw["ontology"], "task.ontology"),
909	_integer(raw["num_classes"], "task.num_classes"),
910	_integer(raw["ignore_index"], "task.ignore_index"),
911	_integers(raw["classes_of_interest"], "task.classes_of_interest"),
912	_strings(raw["linear_classes"], "task.linear_classes"),
913	)
914
915
916	def _parse_data(value: Any) -> DataConfig:
917	raw = _section(
918	value,
919	{
920	"root",
921	"canonical_root",
922	"processed_root",
923	"split_manifest",
924	"label_source",
925	"corridors",
926	"features",
927	"tiling",
928	},
929	"data",
930	)
931	return DataConfig(
932	root=_path(raw["root"], "data.root"),
933	canonical_root=_path(raw["canonical_root"], "data.canonical_root"),
934	processed_root=_path(raw["processed_root"], "data.processed_root"),
935	split_manifest=_path(raw["split_manifest"], "data.split_manifest"),
936	label_source=_dataclass_section(
937	LabelSourceConfig, raw["label_source"], "data.label_source"
938	),
939	corridors=_dataclass_section(
940	CorridorSelectionConfig, raw["corridors"], "data.corridors"
941	),
942	features=_dataclass_section(
943	FeatureConfig, raw["features"], "data.features"
944	),
945	tiling=_dataclass_section(TilingConfig, raw["tiling"], "data.tiling"),
946	)
947
948
949	def _parse_adapter(value: Any) -> AdapterConfig:
950	raw = _section(
951	value,
952	{"emit", "canonical_version", "identity", "spt", "pointcept"},
953	"adapter",
954	)
955	return AdapterConfig(
956	emit=_strings(raw["emit"], "adapter.emit"),
957	canonical_version=_integer(
958	raw["canonical_version"], "adapter.canonical_version"
959	),
960	identity=_choice(
961	raw["identity"], {"source_file_and_row"}, "adapter.identity"
962	),
963	spt=_dataclass_section(SptAdapterConfig, raw["spt"], "adapter.spt"),
964	pointcept=_dataclass_section(
965	PointceptAdapterConfig, raw["pointcept"], "adapter.pointcept"
966	),
967	)
968
969
970	def _parse_model(value: Any) -> ModelConfig:
971	raw = _section(
972	value,
973	{
974	"framework", "runner", "name", "base_config", "checkout_env",
975	"commit_env", "checkpoint", "args",
976	},
977	"model",
978	)
979	args = dict(_mapping(raw["args"], "model.args"))
980	allowed_args = {
981	"aggregation", "annotation_mode", "balanced_crops",
982	"confidence_only_forbidden", "enable_flash", "geometry_context",
983	"head", "label_fraction", "max_num_edges", "max_num_nodes",
984	"num_classes", "ontology_priority", "own_unlabeled_only",
985	"partition", "partition_stage", "patch_size", "published_weights",
986	"require_multiview_agreement", "round", "scanner_holdout", "selector",
987	"semantic_stage", "timing_instrumentation", "training_population",
988	"uncertainty_tier", "unlicensed_scribblekitti_code", "voxel_sizes_m",
989	}
990	_keys(args, set(), allowed_args, "model.args")
991	if "partition" in args:
992	partition = _mapping(args["partition"], "model.args.partition")
993	_exact_keys(
994	partition,
995	{
996	"regularization", "spatial_weight", "cutoff", "graph_k_max",
997	"graph_gap_m",
998	},
999	"model.args.partition",
1000	)
1001	return ModelConfig(
1002	framework=_choice(
1003	raw["framework"], {"cpu", "spt", "pointcept"}, "model.framework"
1004	),
1005	runner=_path(raw["runner"], "model.runner"),
1006	name=_string(raw["name"], "model.name"),
1007	base_config=_optional_path(raw["base_config"], "model.base_config"),
1008	checkout_env=_optional_string(raw["checkout_env"], "model.checkout_env"),
1009	commit_env=_optional_string(raw["commit_env"], "model.commit_env"),
1010	checkpoint=_optional_path(raw["checkpoint"], "model.checkpoint"),
1011	args=MappingProxyType(args),
1012	)
1013
1014
1015	def _parse_loss(value: Any) -> LossConfig:
1016	raw = _section(value, {"name", "args"}, "loss")
1017	args = dict(_mapping(raw["args"], "loss.args"))
1018	_keys(
1019	args,
1020	set(),
1021	{"alpha", "beta", "class_weighting", "gamma", "ignore_index", "reason"},
1022	"loss.args",
1023	)
1024	return LossConfig(_string(raw["name"], "loss.name"), MappingProxyType(args))
1025
1026
1027	def _parse_train(value: Any, model_raw: Mapping[str, Any]) -> TrainConfig:
1028	raw = _mapping(value, "train")
1029	inherited = {
1030	"max_epochs", "lr", "weight_decay", "precision",
1031	"accumulate_grad_batches", "accelerator", "devices",
1032	"viz_every_n_epochs", "viz_samples", "monitor", "monitor_mode",
1033	"early_stop_monitor", "early_stop_mode", "early_stop_patience",
1034	"log_dir", "log_every_n_steps",
1035	}
1036	allowed = inherited \| {
1037	"batch_size", "num_workers", "optimizer", "scheduler", "distributed"
1038	}
1039	_keys(raw, set(), allowed, "train")
1040	if model_raw.get("framework") in {"spt", "pointcept"} and (
1041	{"viz_every_n_epochs", "viz_samples"} & set(raw)
1042	):
1043	raise ConfigError(
1044	"train.viz_every_n_epochs and train.viz_samples are forbidden for "
1045	"external frameworks"
1046	)
1047	hints = _field_types(TrainConfig)
1048	converted = {
1049	name: _typed_value(hints.get(name), item, f"train.{name}")
1050	for name, item in raw.items()
1051	}
1052	try:
1053	return TrainConfig(**converted)
1054	except TypeError as exc:
1055	raise ConfigError(f"Invalid train section: {exc}") from exc
1056
1057
1058	def _parse_evaluation(value: Any) -> EvaluationConfig:
1059	raw = _mapping(value, "evaluation")
1060	required = {
1061	"split", "metrics", "precision_floors", "continuity",
1062	"object_matching", "bootstrap", "promotion",
1063	}
1064	_keys(raw, required, required \| {"worst_k_tiles"}, "evaluation")
1065	floors_raw = _mapping(raw["precision_floors"], "evaluation.precision_floors")
1066	floors = {
1067	_string(name, "precision floor class"): _float(
1068	value, f"evaluation.precision_floors.{name}"
1069	)
1070	for name, value in floors_raw.items()
1071	}
1072	return EvaluationConfig(
1073	split=_choice(
1074	raw["split"], {"validation", "promotion_test"}, "evaluation.split"
1075	),
1076	metrics=_strings(raw["metrics"], "evaluation.metrics"),
1077	precision_floors=MappingProxyType(floors),
1078	continuity=_dataclass_section(
1079	ContinuityConfig, raw["continuity"], "evaluation.continuity"
1080	),
1081	object_matching=_dataclass_section(
1082	ObjectMatchingConfig,
1083	raw["object_matching"],
1084	"evaluation.object_matching",
1085	),
1086	bootstrap=_dataclass_section(
1087	BootstrapConfig, raw["bootstrap"], "evaluation.bootstrap"
1088	),
1089	promotion=_dataclass_section(
1090	PromotionConfig, raw["promotion"], "evaluation.promotion"
1091	),
1092	worst_k_tiles=_integer(
1093	raw.get("worst_k_tiles", 4), "evaluation.worst_k_tiles"
1094	),
1095	)
1096
1097
1098	def _parse_runtime(value: Any) -> RuntimeConfig:
1099	return _dataclass_section(RuntimeConfig, value, "runtime")
1100
1101
1102	def _parse_visualization(value: Any) -> VisualizationConfig:
1103	parsed = _dataclass_section(VisualizationConfig, value, "visualization")
1104	if parsed.masks_every_n_epochs < 1:
1105	raise ConfigError("visualization.masks_every_n_epochs must be >= 1")
1106	tiles = parsed.masks_tiles
1107	if isinstance(tiles, int):
1108	if tiles < 1:
1109	raise ConfigError("visualization.masks_tiles must be >= 1")
1110	elif not tiles:
1111	raise ConfigError("visualization.masks_tiles cannot be empty")
1112	return parsed
1113
1114
1115	def _parse_provenance(value: Any) -> ProvenanceConfig:
1116	return _dataclass_section(ProvenanceConfig, value, "provenance")
1117
1118
1119	def _validate_config(config: HarnessConfig, raw: Mapping[str, Any]) -> None:
1120	if config.schema_version != 1:
1121	raise ConfigError(f"Unsupported schema_version {config.schema_version}")
1122	if not re.fullmatch(r"E(?:[1-9]\|1[0-4])", config.experiment.id):
1123	raise ConfigError(f"Invalid experiment.id {config.experiment.id!r}")
1124	ontology = _load_task_ontology(config)
1125	_validate_task_against_ontology(config, ontology)
1126	if config.seed < 0:
1127	raise ConfigError("seed must be non-negative")
1128	_validate_typed_sections(config, ontology)
1129	if (
1130	config.data.label_source.mode == "regenerate_full_resolution"
1131	and config.data.label_source.fuse_config is None
1132	):
1133	raise ConfigError(
1134	"regenerate_full_resolution requires data.label_source.fuse_config"
1135	)
1136	if (
1137	config.data.label_source.mode == "artifact"
1138	and config.data.label_source.fuse_config is not None
1139	):
1140	raise ConfigError(
1141	"artifact label mode requires data.label_source.fuse_config: null"
1142	)
1143	if not 0.0 <= config.data.label_source.max_unmatched_fraction <= 1.0:
1144	raise ConfigError("data.label_source.max_unmatched_fraction must be in [0, 1]")
1145	if config.data.tiling.length_m <= 0.0 or not (
1146	0.0 <= config.data.tiling.overlap_m < config.data.tiling.length_m
1147	):
1148	raise ConfigError("tiling requires 0 <= overlap_m < length_m")
1149	if config.data.tiling.min_points < 1:
1150	raise ConfigError("data.tiling.min_points must be positive")
1151	if set(config.adapter.emit) - {"canonical", "spt", "pointcept"}:
1152	raise ConfigError("adapter.emit contains an unsupported output format")
1153	if "canonical" not in config.adapter.emit:
1154	raise ConfigError("adapter.emit must include canonical")
1155	if config.adapter.canonical_version != 1:
1156	raise ConfigError("adapter.canonical_version must be 1")
1157	if config.model.framework == "cpu":
1158	if config.model.checkout_env is not None or config.model.commit_env is not None:
1159	raise ConfigError(
1160	"CPU experiments cannot declare external checkout variables"
1161	)
1162	elif (
1163	config.model.base_config is None
1164	or not config.model.checkout_env
1165	or not config.model.commit_env
1166	):
1167	raise ConfigError(
1168	"External models require base_config, checkout_env, and commit_env"
1169	)
1170	if config.model.framework == "pointcept" and config.runtime.flash_attention:
1171	raise ConfigError(
1172	"Pointcept PTv3/LitePT configurations must keep FlashAttention disabled"
1173	)
1174	if config.visualization is not None and config.model.framework != "pointcept":
1175	raise ConfigError(
1176	"visualization is only supported for model.framework pointcept; "
1177	f"{config.model.framework} configs must omit the block"
1178	)
1179	if config.runtime.spconv not in {
1180	"disabled",
1181	"spconv-cu124>=2.3.0,<2.4.0",
1182	"spconv-cu126>=2.3.0,<2.4.0",
1183	}:
1184	raise ConfigError("runtime.spconv is outside the permitted package range")
1185	if config.train.monitor_mode not in {"min", "max"} or (
1186	config.train.early_stop_mode not in {"min", "max"}
1187	):
1188	raise ConfigError("train monitor modes must be min or max")
1189	for metric in config.experiment.deciding_metrics:
1190	if not is_canonical_metric(metric, ontology=ontology):
1191	raise ConfigError(
1192	"experiment.deciding_metrics contains non-canonical metric "
1193	f"{metric!r} for ontology {ontology.name}"
1194	)
1195	for metric in (config.train.monitor, config.train.early_stop_monitor):
1196	if not is_canonical_metric(metric, ontology=ontology):
1197	raise ConfigError(
1198	f"train monitor {metric!r} is outside the canonical namespace "
1199	f"of ontology {ontology.name}"
1200	)
1201	unknown_floor_classes = sorted(
1202	set(config.evaluation.precision_floors) - set(ontology.class_names)
1203	)
1204	if unknown_floor_classes:
1205	raise ConfigError(
1206	"evaluation.precision_floors has unknown classes "
1207	f"{unknown_floor_classes} for ontology {ontology.name}"
1208	)
1209	if config.evaluation.worst_k_tiles < 1:
1210	raise ConfigError("evaluation.worst_k_tiles must be positive")
1211	promotion = config.evaluation.promotion
1212	if promotion.enabled and (
1213	promotion.delta_quality is None
1214	or promotion.delta_fp_per_km is None
1215	or not promotion.superiority_conditions
1216	):
1217	raise ConfigError(
1218	"enabled promotion requires non-null margins and superiority conditions"
1219	)
1220	if config.evaluation.split == "promotion_test" and not promotion.enabled:
1221	raise ConfigError(
1222	"promotion_test evaluation requires evaluation.promotion.enabled"
1223	)
1224	gate_types = {gate.type for gate in config.experiment.gates if gate.required}
1225	if (
1226	config.experiment.status == "template-only"
1227	and "implementation_ticket" not in gate_types
1228	):
1229	raise ConfigError(
1230	"template-only experiments require an implementation_ticket gate"
1231	)
1232	if config.experiment.status == "gated-later" and not gate_types:
1233	raise ConfigError("gated-later experiments require at least one required gate")
1234	if (
1235	config.experiment.status == "implement-now"
1236	and config.experiment.id not in {"E1", "E2"}
1237	):
1238	raise ConfigError("Only E1 and E2 are implement-now in schema version 1")
1239	_validate_study_overrides(config, raw)
1240
1241
1242	def resolve_ontology_path(config: HarnessConfig) -> Path:
1243	"""Resolve the repository-relative ``task.ontology`` path to a real file.
1244
1245	The declared path is relative to the repository that owns the config, so
1246	the ancestors of the config file are searched first, nearest ancestor
1247	first, and the working directory is only consulted last. A run launched
1248	from another checkout therefore reads the ontology of the repository its
1249	config lives in instead of a same-named file that happens to sit under the
1250	working directory. This is the single ontology resolver: every script,
1251	runner, and provenance writer calls it so a run can never validate against
1252	one ontology file and train against another.
1253
1254	Args:
1255	config: Parsed configuration naming the ontology.
1256
1257	Returns:
1258	An absolute, existing ontology path.
1259
1260	Raises:
1261	ConfigError: If no candidate path exists.
1262	"""
1263	declared = config.task.ontology
1264	if declared.is_absolute():
1265	if not declared.is_file():
1266	raise ConfigError(
1267	f"{config.source_path}: task.ontology {declared.as_posix()} "
1268	f"does not exist"
1269	)
1270	return declared
1271	candidates = [
1272	ancestor / declared for ancestor in config.source_path.resolve().parents
1273	]
1274	candidates.append(Path.cwd().resolve() / declared)
1275	for candidate in candidates:
1276	if candidate.is_file():
1277	return candidate.resolve()
1278	searched = ", ".join(
1279	sorted({candidate.parent.as_posix() for candidate in candidates})
1280	)
1281	raise ConfigError(
1282	f"{config.source_path}: task.ontology {declared.as_posix()} was not "
1283	f"found relative to any parent of the config or to the working "
1284	f"directory {Path.cwd().as_posix()}; searched {searched}"
1285	)
1286
1287
1288	def _load_task_ontology(config: HarnessConfig) -> Ontology:
1289	"""Load the ontology the config declares, failing closed as a ConfigError.
1290
1291	Args:
1292	config: Parsed configuration whose ``task.ontology`` path is resolved
1293	relative to the repository root.
1294
1295	Returns:
1296	The validated ontology every other contract is checked against.
1297
1298	Raises:
1299	ConfigError: If the ontology cannot be located, loaded, or is invalid.
1300	"""
1301	resolved = resolve_ontology_path(config)
1302	try:
1303	return load_ontology(resolved)
1304	except OntologyError as exc:
1305	raise ConfigError(
1306	f"task.ontology {config.task.ontology.as_posix()} is not a valid "
1307	f"ontology: {exc}"
1308	) from exc
1309
1310
1311	def _validate_task_against_ontology(
1312	config: HarnessConfig, ontology: Ontology
1313	) -> None:
1314	"""Check that the task block restates the loaded ontology exactly.
1315
1316	Args:
1317	config: Parsed configuration.
1318	ontology: Ontology loaded from ``task.ontology``.
1319
1320	Raises:
1321	ConfigError: If any task, evaluation, model, or loss class contract
1322	disagrees with the loaded ontology.
1323	"""
1324	task = config.task
1325	if task.num_classes != ontology.num_predicted_classes:
1326	raise ConfigError(
1327	f"task.num_classes {task.num_classes} must equal ontology "
1328	f"{ontology.name} num_predicted_classes "
1329	f"{ontology.num_predicted_classes}"
1330	)
1331	if task.ignore_index != ontology.void_id:
1332	raise ConfigError(
1333	f"task.ignore_index {task.ignore_index} must equal ontology "
1334	f"{ontology.name} void ID {ontology.void_id}"
1335	)
1336	if task.classes_of_interest != ontology.interest_ids:
1337	raise ConfigError(
1338	f"task.classes_of_interest {list(task.classes_of_interest)} must "
1339	f"equal ontology {ontology.name} interest IDs "
1340	f"{list(ontology.interest_ids)}"
1341	)
1342	linear_names = tuple(
1343	ontology.class_for_id(train_id).name for train_id in ontology.linear_class_ids
1344	)
1345	if len(set(task.linear_classes)) != len(task.linear_classes) or set(
1346	task.linear_classes
1347	) != set(linear_names):
1348	raise ConfigError(
1349	f"task.linear_classes {list(task.linear_classes)} must be exactly "
1350	f"the linear classes {list(linear_names)} of ontology "
1351	f"{ontology.name}"
1352	)
1353	profiles = config.evaluation.object_matching.cluster_profiles
1354	if profiles != task.ontology:
1355	raise ConfigError(
1356	"evaluation.object_matching.cluster_profiles "
1357	f"{profiles.as_posix()} must be the task ontology "
1358	f"{task.ontology.as_posix()}"
1359	)
1360	if "num_classes" in config.model.args:
1361	declared = config.model.args["num_classes"]
1362	if declared != task.num_classes:
1363	raise ConfigError(
1364	f"model.args.num_classes {declared!r} must equal "
1365	f"task.num_classes {task.num_classes}"
1366	)
1367	if "ignore_index" in config.loss.args:
1368	declared = config.loss.args["ignore_index"]
1369	if declared != task.ignore_index:
1370	raise ConfigError(
1371	f"loss.args.ignore_index {declared!r} must equal "
1372	f"task.ignore_index {task.ignore_index}"
1373	)
1374
1375
1376	def _validate_typed_sections(config: HarnessConfig, ontology: Ontology) -> None:
1377	"""Validate runtime types not enforced by dataclass constructors.
1378
1379	Args:
1380	config: Parsed configuration.
1381	ontology: Ontology loaded from ``task.ontology``.
1382
1383	Raises:
1384	ConfigError: If a typed section or a class-naming gate is invalid.
1385	"""
1386	_require_int(config.experiment.phase, "experiment.phase")
1387	for gate in config.experiment.gates:
1388	params = gate.params
1389	if isinstance(params, SptPartitionOracleParams):
1390	unknown = sorted(
1391	set(params.minimum_purity_by_class) - set(ontology.class_names)
1392	)
1393	if unknown:
1394	raise ConfigError(
1395	f"gate {gate.name}.minimum_purity_by_class contains unknown "
1396	f"ontology classes {unknown} for ontology {ontology.name}"
1397	)
1398	if isinstance(params, ImplementationTicketParams):
1399	_require_string(params.ticket, f"gate {gate.name}.ticket")
1400	_require_string(params.required_status, f"gate {gate.name}.required_status")
1401	elif isinstance(params, ArtifactExistsParams):
1402	if params.expected_status is not None:
1403	_require_string(
1404	params.expected_status, f"gate {gate.name}.expected_status"
1405	)
1406	elif isinstance(params, DataAvailableParams):
1407	_require_string(params.dataset, f"gate {gate.name}.dataset")
1408	elif isinstance(params, LicenseApprovalParams):
1409	_require_string(
1410	params.required_decision, f"gate {gate.name}.required_decision"
1411	)
1412	elif isinstance(params, OperationalSmokeParams):
1413	_require_string(
1414	params.required_status, f"gate {gate.name}.required_status"
1415	)
1416	for contrast in config.study.contrasts:
1417	for name, value in asdict(contrast).items():
1418	_require_string(value, f"study.contrasts.{name}")
1419	if not is_canonical_metric(contrast.metric, ontology=ontology):
1420	raise ConfigError(
1421	f"study contrast metric {contrast.metric!r} is not canonical "
1422	f"for ontology {ontology.name}"
1423	)
1424	if config.study.sweep is not None:
1425	_require_string(config.study.sweep.method, "study.sweep.method")
1426	_require_int(config.study.sweep.budget, "study.sweep.budget")
1427	if config.study.sweep.budget < 1:
1428	raise ConfigError("study.sweep.budget must be positive")
1429	if not is_canonical_metric(config.study.sweep.objective, ontology=ontology):
1430	raise ConfigError(
1431	f"study.sweep.objective must be canonical for ontology "
1432	f"{ontology.name}"
1433	)
1434	source = config.data.label_source
1435	if source.mode not in {"artifact", "regenerate_full_resolution"}:
1436	raise ConfigError("data.label_source.mode is unsupported")
1437	for name, value in (
1438	("source_geometry_glob", source.source_geometry_glob),
1439	("prefer", source.prefer),
1440	("stats_glob", source.stats_glob),
1441	):
1442	_require_string(value, f"data.label_source.{name}")
1443	if source.unmatched_policy != "void":
1444	raise ConfigError("data.label_source.unmatched_policy must be void")
1445	_require_number(
1446	source.max_unmatched_fraction,
1447	"data.label_source.max_unmatched_fraction",
1448	)
1449	features = config.data.features
1450	if features.fit_on != "train_corridors_only":
1451	raise ConfigError("data.features.fit_on must be train_corridors_only")
1452	_require_bool(features.scanner_conditioning, "data.features.scanner_conditioning")
1453	tiling = config.data.tiling
1454	if tiling.mode != "corridor_axis" or tiling.origin != "dataset_manifest":
1455	raise ConfigError("data.tiling requires corridor_axis and dataset_manifest")
1456	if tiling.blend != "linear_edge_weight":
1457	raise ConfigError("data.tiling.blend must be linear_edge_weight")
1458	_require_number(tiling.length_m, "data.tiling.length_m")
1459	_require_number(tiling.overlap_m, "data.tiling.overlap_m")
1460	_require_int(tiling.min_points, "data.tiling.min_points")
1461	_require_int(config.adapter.canonical_version, "adapter.canonical_version")
1462	_require_int(config.adapter.spt.pc_tiling, "adapter.spt.pc_tiling")
1463	_require_number(config.adapter.spt.voxel_m, "adapter.spt.voxel_m")
1464	_require_number(
1465	config.adapter.pointcept.grid_size_m, "adapter.pointcept.grid_size_m"
1466	)
1467	if config.adapter.spt.voxel_m <= 0.0 or config.adapter.pointcept.grid_size_m <= 0.0:
1468	raise ConfigError("adapter voxel/grid sizes must be positive")
1469	if config.loss.name not in {
1470	"framework_native",
1471	"cross_entropy",
1472	"focal_cross_entropy",
1473	"masked_focal_tversky",
1474	}:
1475	raise ConfigError(f"Unsupported loss.name {config.loss.name!r}")
1476	train = config.train
1477	for name in (
1478	"max_epochs", "accumulate_grad_batches", "batch_size", "num_workers",
1479	"early_stop_patience", "log_every_n_steps",
1480	):
1481	_require_int(getattr(train, name), f"train.{name}")
1482	for name in ("lr", "weight_decay"):
1483	_require_number(getattr(train, name), f"train.{name}")
1484	_require_bool(train.distributed, "train.distributed")
1485	for name in (
1486	"precision", "accelerator", "optimizer", "scheduler", "monitor",
1487	"monitor_mode", "early_stop_monitor", "early_stop_mode", "log_dir",
1488	):
1489	_require_string(getattr(train, name), f"train.{name}")
1490	evaluation = config.evaluation
1491	for value in evaluation.metrics:
1492	_require_string(value, "evaluation.metrics")
1493	for name, value in evaluation.precision_floors.items():
1494	_require_number(value, f"evaluation.precision_floors.{name}")
1495	if not 0.0 <= value <= 1.0:
1496	raise ConfigError(
1497	f"evaluation precision floor for {name} must be in [0, 1]"
1498	)
1499	_require_number(
1500	evaluation.continuity.chainage_bin_m,
1501	"evaluation.continuity.chainage_bin_m",
1502	)
1503	_require_number(
1504	evaluation.continuity.gap_threshold_m,
1505	"evaluation.continuity.gap_threshold_m",
1506	)
1507	bootstrap = evaluation.bootstrap
1508	if bootstrap.unit != "corridor":
1509	raise ConfigError("evaluation.bootstrap.unit must be corridor")
1510	_require_number(bootstrap.spatial_block_m, "evaluation.bootstrap.spatial_block_m")
1511	_require_int(bootstrap.samples, "evaluation.bootstrap.samples")
1512	_require_number(bootstrap.confidence, "evaluation.bootstrap.confidence")
1513	_require_int(bootstrap.seed, "evaluation.bootstrap.seed")
1514	_require_bool(evaluation.promotion.enabled, "evaluation.promotion.enabled")
1515	runtime = config.runtime
1516	_require_bool(runtime.flash_attention, "runtime.flash_attention")
1517	_require_int(runtime.system_ram_gb, "runtime.system_ram_gb")
1518	_require_int(runtime.gpu_memory_gb, "runtime.gpu_memory_gb")
1519	for name in ("target", "cuda", "spconv"):
1520	_require_string(getattr(runtime, name), f"runtime.{name}")
1521	provenance = config.provenance
1522	if provenance.manifest != "required" or provenance.data_hash_source != "dvc":
1523	raise ConfigError(
1524	"provenance requires manifest=required and data_hash_source=dvc"
1525	)
1526	_require_bool(provenance.record_environment, "provenance.record_environment")
1527	_require_bool(provenance.record_commands, "provenance.record_commands")
1528
1529
1530	def _require_string(value: Any, where: str) -> None:
1531	if not isinstance(value, str) or not value:
1532	raise ConfigError(f"{where} must be a non-empty string")
1533
1534
1535	def _require_int(value: Any, where: str) -> None:
1536	if isinstance(value, bool) or not isinstance(value, int):
1537	raise ConfigError(f"{where} must be an integer")
1538
1539
1540	def _require_number(value: Any, where: str) -> None:
1541	if isinstance(value, bool) or not isinstance(value, (int, float)):
1542	raise ConfigError(f"{where} must be numeric")
1543
1544
1545	def _require_bool(value: Any, where: str) -> None:
1546	if not isinstance(value, bool):
1547	raise ConfigError(f"{where} must be boolean")
1548
1549
1550	def _validate_study_overrides(config: HarnessConfig, raw: Mapping[str, Any]) -> None:
1551	leaves = _leaf_values(raw)
1552	override_groups: list[Mapping[str, OverrideValue]] = [
1553	item.overrides for item in config.study.variants
1554	]
1555	if config.study.matrix is not None:
1556	override_groups.extend(
1557	{path: value}
1558	for path, values in config.study.matrix.axes.items()
1559	for value in values
1560	)
1561	override_groups.extend(config.study.matrix.include)
1562	override_groups.extend(config.study.matrix.exclude)
1563	if config.study.sweep is not None:
1564	for path, parameter in config.study.sweep.parameters.items():
1565	if parameter.values is not None:
1566	override_groups.extend({path: value} for value in parameter.values)
1567	else:
1568	override_groups.extend(
1569	({path: parameter.minimum}, {path: parameter.maximum})
1570	)
1571	for overrides in override_groups:
1572	for path, value in overrides.items():
1573	if path.startswith("study.") or path not in leaves:
1574	raise ConfigError(
1575	f"Study override path {path!r} is not a declared scalar/list leaf"
1576	)
1577	expected = leaves[path]
1578	if not _same_leaf_type(expected, value):
1579	raise ConfigError(
1580	f"Study override {path!r} has incompatible value {value!r}; "
1581	f"expected type of {expected!r}"
1582	)
1583
1584
1585	def _cell_definitions(
1586	config: HarnessConfig,
1587	) -> tuple[tuple[str, Mapping[str, OverrideValue]], ...]:
1588	"""Return the ordered (identity, overrides) pairs of one study."""
1589	study = config.study
1590	if study.kind == "single":
1591	return ((config.experiment.id, MappingProxyType({})),)
1592	if study.kind == "variants":
1593	if not study.variants:
1594	raise ConfigError(
1595	f"{config.experiment.id} study.kind variants has no variants payload"
1596	)
1597	return tuple((item.id, item.overrides) for item in study.variants)
1598	if study.kind == "matrix":
1599	if study.matrix is None:
1600	raise ConfigError(
1601	f"{config.experiment.id} study.kind matrix has no matrix payload"
1602	)
1603	return _matrix_cells(study.matrix)
1604	if study.sweep is None:
1605	raise ConfigError(
1606	f"{config.experiment.id} study.kind sweep has no sweep payload"
1607	)
1608	return _sweep_cells(study.sweep, config.seed)
1609
1610
1611	def _matrix_cells(
1612	matrix: MatrixConfig,
1613	) -> tuple[tuple[str, Mapping[str, OverrideValue]], ...]:
1614	"""Expand typed matrix axes into deterministic cells."""
1615	axis_paths = tuple(matrix.axes)
1616	unknown = sorted(
1617	{path for item in matrix.exclude for path in item} - set(axis_paths)
1618	)
1619	if unknown:
1620	raise ConfigError(f"study.matrix.exclude references non-axis paths {unknown}")
1621	definitions: list[tuple[str, Mapping[str, OverrideValue]]] = []
1622	excluded = [0] * len(matrix.exclude)
1623	for combination in itertools.product(
1624	*(matrix.axes[path] for path in axis_paths)
1625	):
1626	overrides = dict(zip(axis_paths, combination, strict=True))
1627	dropped = False
1628	for index, item in enumerate(matrix.exclude):
1629	if all(overrides[path] == value for path, value in item.items()):
1630	excluded[index] += 1
1631	dropped = True
1632	if not dropped:
1633	definitions.append((_cell_id(overrides), MappingProxyType(overrides)))
1634	for index, count in enumerate(excluded):
1635	if not count:
1636	raise ConfigError(
1637	f"study.matrix.exclude[{index}] matches no matrix cell"
1638	)
1639	for index, item in enumerate(matrix.include):
1640	if not item:
1641	raise ConfigError(f"study.matrix.include[{index}] cannot be empty")
1642	definitions.append((_cell_id(item), item))
1643	if not definitions:
1644	raise ConfigError("study.matrix excludes every cell")
1645	return tuple(definitions)
1646
1647
1648	def _sweep_cells(
1649	sweep: SweepConfig, seed: int
1650	) -> tuple[tuple[str, Mapping[str, OverrideValue]], ...]:
1651	"""Expand a bounded sweep deterministically under the study seed."""
1652	paths = tuple(sweep.parameters)
1653	if sweep.method == "grid":
1654	unbounded = sorted(
1655	path for path, item in sweep.parameters.items() if item.values is None
1656	)
1657	if unbounded:
1658	raise ConfigError(
1659	"study.sweep.method grid requires explicit values for "
1660	f"{unbounded}"
1661	)
1662	definitions: list[tuple[str, Mapping[str, OverrideValue]]] = []
1663	for combination in itertools.product(
1664	*(tuple(sweep.parameters[path].values or ()) for path in paths)
1665	):
1666	overrides = dict(zip(paths, combination, strict=True))
1667	definitions.append((_cell_id(overrides), MappingProxyType(overrides)))
1668	return tuple(definitions[: sweep.budget])
1669	if sweep.method == "random":
1670	generator = random.Random(seed)
1671	return tuple(
1672	(
1673	f"sample_{index:03d}",
1674	MappingProxyType(
1675	{
1676	path: _sweep_sample(
1677	sweep.parameters[path],
1678	generator,
1679	f"study.sweep.parameters.{path}",
1680	)
1681	for path in paths
1682	}
1683	),
1684	)
1685	for index in range(sweep.budget)
1686	)
1687	raise ConfigError(
1688	f"study.sweep.method {sweep.method!r} is not implemented; supported "
1689	"methods are grid and random"
1690	)
1691
1692
1693	def _sweep_sample(
1694	parameter: SweepParameterConfig, generator: random.Random, where: str
1695	) -> OverrideValue:
1696	"""Draw one deterministic value for a sweep parameter."""
1697	if parameter.values is not None:
1698	return parameter.values[generator.randrange(len(parameter.values))]
1699	if parameter.minimum is None or parameter.maximum is None:
1700	raise ConfigError(f"{where} requires minimum and maximum for sampling")
1701	if parameter.distribution == "uniform":
1702	drawn = generator.uniform(parameter.minimum, parameter.maximum)
1703	elif parameter.distribution == "log_uniform":
1704	if parameter.minimum <= 0.0:
1705	raise ConfigError(f"{where}.minimum must be positive for log_uniform")
1706	drawn = math.exp(
1707	generator.uniform(
1708	math.log(parameter.minimum), math.log(parameter.maximum)
1709	)
1710	)
1711	else:
1712	raise ConfigError(
1713	f"{where}.distribution {parameter.distribution!r} is not implemented; "
1714	"supported distributions are uniform and log_uniform"
1715	)
1716	return float(f"{drawn:.6g}")
1717
1718
1719	def _cell_id(overrides: Mapping[str, OverrideValue]) -> str:
1720	"""Derive a stable, filesystem-safe identity from a cell's overrides."""
1721	if not overrides:
1722	raise ConfigError("A study cell requires at least one override")
1723	names = [path.rsplit(".", 1)[-1] for path in overrides]
1724	if len(set(names)) != len(names):
1725	names = [path.replace(".", "_") for path in overrides]
1726	return "__".join(
1727	f"{name}-{_value_token(value)}"
1728	for name, value in zip(names, overrides.values(), strict=True)
1729	)
1730
1731
1732	def _value_token(value: OverrideValue) -> str:
1733	"""Render one override value as a filesystem-safe token."""
1734	if isinstance(value, bool):
1735	text = "true" if value else "false"
1736	elif value is None:
1737	text = "null"
1738	elif isinstance(value, float):
1739	text = repr(value)
1740	elif isinstance(value, list):
1741	text = "+".join(_value_token(item) for item in value)
1742	else:
1743	text = str(value)
1744	return re.sub(r"[^A-Za-z0-9._+-]", "_", text)
1745
1746
1747	def _apply_overrides(
1748	raw: Mapping[str, Any], overrides: Mapping[str, OverrideValue], where: str
1749	) -> dict[str, Any]:
1750	"""Apply dotted-path overrides to a raw configuration mapping.
1751
1752	Args:
1753	raw: Raw mapping of the base configuration.
1754	overrides: Dotted leaf paths mapped to their replacement values.
1755	where: Cell identification used in error messages.
1756
1757	Returns:
1758	A deep copy of ``raw`` carrying the overridden leaves.
1759
1760	Raises:
1761	ConfigError: If a path is not a declared leaf or the value type differs.
1762	"""
1763	result = copy.deepcopy(dict(raw))
1764	leaves = _leaf_values(raw)
1765	for path in sorted(overrides):
1766	value = overrides[path]
1767	if path.startswith("study.") or path not in leaves:
1768	raise ConfigError(
1769	f"{where} override path {path!r} is not a declared scalar/list leaf"
1770	)
1771	expected = leaves[path]
1772	if not _same_leaf_type(expected, value):
1773	raise ConfigError(
1774	f"{where} override {path!r} has incompatible value {value!r}; "
1775	f"expected type of {expected!r}"
1776	)
1777	_set_leaf(result, path, _coerce_leaf(expected, value), where)
1778	return result
1779
1780
1781	def _set_leaf(
1782	target: dict[str, Any], path: str, value: OverrideValue, where: str
1783	) -> None:
1784	segments = path.split(".")
1785	node: Any = target
1786	for segment in segments[:-1]:
1787	if not isinstance(node, dict) or segment not in node:
1788	raise ConfigError(f"{where} override path {path!r} is not addressable")
1789	node = node[segment]
1790	if not isinstance(node, dict) or segments[-1] not in node:
1791	raise ConfigError(f"{where} override path {path!r} is not addressable")
1792	node[segments[-1]] = value
1793
1794
1795	def _coerce_leaf(expected: OverrideValue, value: OverrideValue) -> OverrideValue:
1796	"""Parse an override value as the declared leaf's type."""
1797	if isinstance(expected, list):
1798	items = list(value) if isinstance(value, list) else [value]
1799	if expected and isinstance(expected[0], float):
1800	return [_coerce_leaf(expected[0], item) for item in items]
1801	return items
1802	if (
1803	isinstance(expected, float)
1804	and isinstance(value, int)
1805	and not isinstance(value, bool)
1806	):
1807	return float(value)
1808	return value
1809
1810
1811	def _leaf_values(value: Any, prefix: str = "") -> dict[str, OverrideValue]:
1812	result: dict[str, OverrideValue] = {}
1813	if isinstance(value, Mapping):
1814	for key, item in value.items():
1815	path = f"{prefix}.{key}" if prefix else str(key)
1816	if path == "study" or path.startswith("study."):
1817	continue
1818	result.update(_leaf_values(item, path))
1819	elif isinstance(value, list):
1820	if all(
1821	isinstance(item, (str, int, float, bool)) or item is None
1822	for item in value
1823	):
1824	result[prefix] = value
1825	elif isinstance(value, (str, int, float, bool)) or value is None:
1826	result[prefix] = value
1827	return result
1828
1829
1830	def _same_leaf_type(expected: OverrideValue, actual: OverrideValue) -> bool:
1831	if isinstance(expected, list):
1832	if not isinstance(actual, list):
1833	return False
1834	if not expected or not actual:
1835	return True
1836	return all(_same_leaf_type(expected[0], item) for item in actual)
1837	if isinstance(expected, bool):
1838	return isinstance(actual, bool)
1839	if isinstance(expected, int) and not isinstance(expected, bool):
1840	return isinstance(actual, int) and not isinstance(actual, bool)
1841	if isinstance(expected, float):
1842	return isinstance(actual, (int, float)) and not isinstance(actual, bool)
1843	return actual is None if expected is None else isinstance(actual, type(expected))
1844
1845
1846	def _dataclass_section(cls: type[Any], value: Any, where: str) -> Any:
1847	raw = _mapping(value, where)
1848	fields = cls.__dataclass_fields__
1849	required = {
1850	name
1851	for name, item in fields.items()
1852	if item.default is MISSING and item.default_factory is MISSING
1853	}
1854	_keys(raw, required, set(fields), where)
1855	hints = _field_types(cls)
1856	converted: dict[str, Any] = {
1857	name: _typed_value(hints.get(name), item, f"{where}.{name}")
1858	for name, item in raw.items()
1859	}
1860	try:
1861	return cls(**converted)
1862	except TypeError as exc:
1863	raise ConfigError(f"Invalid {where}: {exc}") from exc
1864
1865
1866	def _field_types(cls: type[Any]) -> Mapping[str, Any]:
1867	"""Return resolved field annotations, or an empty mapping if unresolvable.
1868
1869	Args:
1870	cls: Dataclass whose annotations declare the leaf types.
1871
1872	Returns:
1873	Mapping of field name to resolved annotation object.
1874	"""
1875	cached = _FIELD_TYPE_CACHE.get(cls)
1876	if cached is not None:
1877	return cached
1878	try:
1879	hints: Mapping[str, Any] = dict(get_type_hints(cls))
1880	except (NameError, TypeError): # pragma: no cover - defensive
1881	hints = {}
1882	_FIELD_TYPE_CACHE[cls] = hints
1883	return hints
1884
1885
1886	def _typed_value(annotation: Any, value: Any, where: str) -> Any:
1887	"""Parse one config leaf strictly as its declared annotation.
1888
1889	Args:
1890	annotation: Resolved field annotation, or None when unknown.
1891	value: Raw YAML value.
1892	where: Dotted key path used in error messages.
1893
1894	Returns:
1895	The value converted to the declared type.
1896
1897	Raises:
1898	ConfigError: If the value does not match the declared type.
1899	"""
1900	if annotation is None or annotation is Any:
1901	return value
1902	origin = get_origin(annotation)
1903	if origin is Literal:
1904	return _choice(value, {str(item) for item in get_args(annotation)}, where)
1905	if origin in (Union, UnionType):
1906	return _typed_union(annotation, value, where)
1907	if annotation is Path:
1908	return _path(value, where)
1909	if annotation is bool:
1910	return _boolean(value, where)
1911	if annotation is int:
1912	return _integer(value, where)
1913	if annotation is float:
1914	return _float(value, where)
1915	if annotation is str:
1916	return _string(value, where)
1917	if origin is tuple:
1918	args = get_args(annotation)
1919	item_type = args[0] if args else None
1920	return tuple(
1921	_typed_value(item_type, item, where) for item in _sequence(value, where)
1922	)
1923	if origin in (dict, Mapping) or (
1924	isinstance(origin, type) and issubclass(origin, Mapping)
1925	):
1926	args = get_args(annotation)
1927	item_type = args[1] if len(args) == 2 else None
1928	raw = _mapping(value, where)
1929	return MappingProxyType(
1930	{
1931	_string(key, where): _typed_value(item_type, item, f"{where}.{key}")
1932	for key, item in raw.items()
1933	}
1934	)
1935	return value
1936
1937
1938	def _typed_union(annotation: Any, value: Any, where: str) -> Any:
1939	"""Parse a value against a union annotation, rejecting every mismatch."""
1940	members = get_args(annotation)
1941	optional = type(None) in members
1942	if value is None:
1943	if optional:
1944	return None
1945	raise ConfigError(f"{where} must not be null")
1946	candidates = [item for item in members if item is not type(None)]
1947	if len(candidates) == 1:
1948	return _typed_value(candidates[0], value, where)
1949	for candidate in candidates:
1950	try:
1951	return _typed_value(candidate, value, where)
1952	except ConfigError:
1953	continue
1954	names = sorted(getattr(item, "__name__", str(item)) for item in candidates)
1955	raise ConfigError(
1956	f"{where} must be one of {names}, got {type(value).__name__} {value!r}"
1957	)
1958
1959
1960	def _json_safe(value: Any) -> Any:
1961	if is_dataclass(value) and not isinstance(value, type):
1962	return {
1963	item.name: _json_safe(getattr(value, item.name))
1964	for item in fields(value)
1965	}
1966	if isinstance(value, Path):
1967	return value.as_posix()
1968	if isinstance(value, Mapping):
1969	return {str(key): _json_safe(item) for key, item in value.items()}
1970	if isinstance(value, (tuple, list)):
1971	return [_json_safe(item) for item in value]
1972	return value
1973
1974
1975	def _overrides(value: Any, where: str) -> Mapping[str, OverrideValue]:
1976	raw = _mapping(value, where)
1977	return MappingProxyType(
1978	{
1979	_string(path, where): _override_value(item, f"{where}.{path}")
1980	for path, item in raw.items()
1981	}
1982	)
1983
1984
1985	def _override_value(value: Any, where: str) -> OverrideValue:
1986	if isinstance(value, list):
1987	return [_scalar(item, where) for item in value]
1988	return _scalar(value, where)
1989
1990
1991	def _section(value: Any, keys: set[str], where: str) -> Mapping[str, Any]:
1992	raw = _mapping(value, where)
1993	_exact_keys(raw, keys, where)
1994	return raw
1995
1996
1997	def _mapping(value: Any, where: str) -> Mapping[str, Any]:
1998	if not isinstance(value, Mapping):
1999	raise ConfigError(f"{where} must be a mapping")
2000	if any(not isinstance(key, str) for key in value):
2001	raise ConfigError(f"{where} keys must be strings")
2002	return value
2003
2004
2005	def _sequence(value: Any, where: str) -> Sequence[Any]:
2006	if isinstance(value, (str, bytes)) or not isinstance(value, Sequence):
2007	raise ConfigError(f"{where} must be a sequence")
2008	return value
2009
2010
2011	def _exact_keys(value: Mapping[str, Any], expected: set[str], where: str) -> None:
2012	_keys(value, expected, expected, where)
2013
2014
2015	def _keys(
2016	value: Mapping[str, Any],
2017	required: set[str],
2018	allowed: set[str],
2019	where: str,
2020	) -> None:
2021	missing = sorted(required - set(value))
2022	unknown = sorted(set(value) - allowed)
2023	if missing or unknown:
2024	details = []
2025	if missing:
2026	details.append(f"missing {missing}")
2027	if unknown:
2028	details.append(f"unknown {unknown}")
2029	raise ConfigError(f"{where} has " + " and ".join(details))
2030
2031
2032	def _string(value: Any, where: str) -> str:
2033	if not isinstance(value, str) or not value.strip():
2034	raise ConfigError(f"{where} must be a non-empty string")
2035	return value
2036
2037
2038	def _optional_string(value: Any, where: str) -> str \| None:
2039	return None if value is None else _string(value, where)
2040
2041
2042	def _integer(value: Any, where: str) -> int:
2043	if isinstance(value, bool) or not isinstance(value, int):
2044	raise ConfigError(f"{where} must be an integer")
2045	return value
2046
2047
2048	def _float(value: Any, where: str) -> float:
2049	if isinstance(value, bool) or not isinstance(value, (int, float)):
2050	raise ConfigError(f"{where} must be numeric")
2051	return float(value)
2052
2053
2054	def _optional_float(value: Any, where: str) -> float \| None:
2055	return None if value is None else _float(value, where)
2056
2057
2058	def _boolean(value: Any, where: str) -> bool:
2059	if not isinstance(value, bool):
2060	raise ConfigError(f"{where} must be boolean")
2061	return value
2062
2063
2064	def _scalar(value: Any, where: str) -> Scalar:
2065	if value is None or isinstance(value, (str, bool)):
2066	return value
2067	if isinstance(value, int) and not isinstance(value, bool):
2068	return value
2069	if isinstance(value, float):
2070	return value
2071	raise ConfigError(f"{where} must be a scalar or scalar list")
2072
2073
2074	def _path(value: Any, where: str) -> Path:
2075	text = _string(value, where)
2076	path = Path(text)
2077	if path.is_absolute():
2078	raise ConfigError(f"{where} must be repository-relative, got {path}")
2079	return path
2080
2081
2082	def _optional_path(value: Any, where: str) -> Path \| None:
2083	return None if value is None else _path(value, where)
2084
2085
2086	def _strings(value: Any, where: str) -> tuple[str, ...]:
2087	return tuple(_string(item, where) for item in _sequence(value, where))
2088
2089
2090	def _integers(value: Any, where: str) -> tuple[int, ...]:
2091	return tuple(_integer(item, where) for item in _sequence(value, where))
2092
2093
2094	def _choice(value: Any, choices: set[str], where: str) -> Any:
2095	text = _string(value, where)
2096	if text not in choices:
2097	raise ConfigError(f"{where} must be one of {sorted(choices)}, got {text!r}")
2098	return text

Importance #9: src/train/config_rules.py @@ -0,0 +1,364 @@

	1	"""Cross-section and ontology-dependent rules of an experiment configuration.
	2
	3	The pydantic models in :mod:`src.train.config_schema` and
	4	:mod:`src.train.config_sections` own every key, type, and single-section rule.
	5	This module owns what a model cannot see: the ontology file the task declares,
	6	the metric namespace it fixes, and the rules that span two sections.
	7	"""
	8
	9	from __future__ import annotations
	10
	11	import logging
	12	import re
	13	from collections.abc import Mapping
	14	from pathlib import Path
	15	from typing import Any
	16
	17	from src.contracts.ontology import (
	18	Ontology,
	19	OntologyError,
	20	load_ontology,
	21	macro_interest_all_suffix,
	22	)
	23	from src.train import config_schema, config_sections, config_values
	24
	25	logger = logging.getLogger(__name__)
	26
	27	_EXPERIMENT_ID_PATTERN = re.compile(r"E(?:[1-9]\|1[0-4])")
	28	_VIZ_TRAIN_KEYS = frozenset({"viz_every_n_epochs", "viz_samples"})
	29
	30
	31	def is_canonical_metric(name: str, *, ontology: Ontology) -> bool:
	32	"""Return whether a metric belongs to an ontology's val/eval namespace.
	33
	34	Args:
	35	name: Metric tag to validate.
	36	ontology: Ontology whose predicted class names and interest count fix
	37	the per-class tags and the structural macro suffix.
	38
	39	Returns:
	40	True for a canonical tag, otherwise False.
	41	"""
	42	if name == "val/loss":
	43	return True
	44	match = re.fullmatch(r"(val\|eval)/(.+)", name)
	45	if match is None:
	46	return False
	47	metric = match.group(2)
	48	suffix = macro_interest_all_suffix(ontology)
	49	fixed = {
	50	"iou_macro_interest",
	51	"f1_macro_interest",
	52	"precision_macro_interest",
	53	"recall_macro_interest",
	54	f"iou_macro_interest_{suffix}",
	55	f"f1_macro_interest_{suffix}",
	56	f"precision_macro_interest_{suffix}",
	57	f"recall_macro_interest_{suffix}",
	58	"miou_all_classes",
	59	}
	60	if metric in fixed:
	61	return True
	62	class_name = "\|".join(re.escape(item) for item in ontology.class_names)
	63	patterns = (
	64	rf"(?:iou\|precision\|recall\|f1\|support)_(?:{class_name})",
	65	rf"(?:fp_per_km\|detections_per_km\|matched_recall)_(?:{class_name})",
	66	rf"continuity_(?:covered_fraction\|total_missing_length_m\|gaps\|"
	67	rf"gap_median_m\|gap_p95_m\|gap_max_m)_(?:{class_name})",
	68	rf"seam_[a-zA-Z0-9_.-]+_(?:{class_name})",
	69	)
	70	return any(re.fullmatch(pattern, metric) is not None for pattern in patterns)
	71
	72
	73	def resolve_ontology_path(config: config_sections.HarnessConfig) -> Path:
	74	"""Resolve the repository-relative ``task.ontology`` path to a real file.
	75
	76	The declared path is relative to the repository that owns the config, so
	77	the ancestors of the config file are searched first, nearest ancestor
	78	first, and the working directory is only consulted last. A run launched
	79	from another checkout therefore reads the ontology of the repository its
	80	config lives in instead of a same-named file that happens to sit under the
	81	working directory. This is the single ontology resolver: every script,
	82	runner, and provenance writer calls it so a run can never validate against
	83	one ontology file and train against another.
	84
	85	Args:
	86	config: Parsed configuration naming the ontology.
	87
	88	Returns:
	89	An absolute, existing ontology path.
	90
	91	Raises:
	92	ConfigError: If no candidate path exists.
	93	"""
	94	declared = config.task.ontology
	95	if declared.is_absolute():
	96	if not declared.is_file():
	97	raise config_values.ConfigError(
	98	f"{config.source_path}: task.ontology {declared.as_posix()} "
	99	f"does not exist"
	100	)
	101	return declared
	102	candidates = [
	103	ancestor / declared for ancestor in config.source_path.resolve().parents
	104	]
	105	candidates.append(Path.cwd().resolve() / declared)
	106	for candidate in candidates:
	107	if candidate.is_file():
	108	return candidate.resolve()
	109	searched = ", ".join(
	110	sorted({candidate.parent.as_posix() for candidate in candidates})
	111	)
	112	raise config_values.ConfigError(
	113	f"{config.source_path}: task.ontology {declared.as_posix()} was not "
	114	f"found relative to any parent of the config or to the working "
	115	f"directory {Path.cwd().as_posix()}; searched {searched}"
	116	)
	117
	118
	119	def validate_config(
	120	config: config_sections.HarnessConfig, raw: Mapping[str, Any]
	121	) -> None:
	122	"""Apply every rule a single section's model cannot decide on its own.
	123
	124	Args:
	125	config: Configuration whose sections are already model-validated.
	126	raw: Raw mapping the configuration was parsed from, used to check the
	127	declared study overrides against the document's own leaves.
	128
	129	Raises:
	130	ConfigError: If a cross-section, identity, or ontology rule fails.
	131	"""
	132	if config.schema_version != 1:
	133	raise config_values.ConfigError(
	134	f"Unsupported schema_version {config.schema_version}"
	135	)
	136	if not _EXPERIMENT_ID_PATTERN.fullmatch(config.experiment.id):
	137	raise config_values.ConfigError(
	138	f"Invalid experiment.id {config.experiment.id!r}"
	139	)
	140	ontology = _load_task_ontology(config)
	141	_validate_task_against_ontology(config, ontology)
	142	_validate_frameworks(config)
	143	_validate_metrics(config, ontology)
	144	_validate_gates(config, ontology)
	145	_validate_study_overrides(config, raw)
	146
	147
	148	def _validate_frameworks(config: config_sections.HarnessConfig) -> None:
	149	"""Check the rules that tie a section to the declared model framework."""
	150	if config.model.framework in {"spt", "pointcept"} and (
	151	_VIZ_TRAIN_KEYS & config.train.model_fields_set
	152	):
	153	raise config_values.ConfigError(
	154	"train.viz_every_n_epochs and train.viz_samples are forbidden for "
	155	"external frameworks"
	156	)
	157	if config.model.framework == "pointcept" and config.runtime.flash_attention:
	158	raise config_values.ConfigError(
	159	"Pointcept PTv3/LitePT configurations must keep FlashAttention disabled"
	160	)
	161	if config.visualization is not None and config.model.framework != "pointcept":
	162	raise config_values.ConfigError(
	163	"visualization is only supported for model.framework pointcept; "
	164	f"{config.model.framework} configs must omit the block"
	165	)
	166
	167
	168	def _validate_metrics(
	169	config: config_sections.HarnessConfig, ontology: Ontology
	170	) -> None:
	171	"""Check every declared metric tag against the ontology's namespace."""
	172	for metric in config.experiment.deciding_metrics:
	173	if not is_canonical_metric(metric, ontology=ontology):
	174	raise config_values.ConfigError(
	175	"experiment.deciding_metrics contains non-canonical metric "
	176	f"{metric!r} for ontology {ontology.name}"
	177	)
	178	for metric in (config.train.monitor, config.train.early_stop_monitor):
	179	if not is_canonical_metric(metric, ontology=ontology):
	180	raise config_values.ConfigError(
	181	f"train monitor {metric!r} is outside the canonical namespace "
	182	f"of ontology {ontology.name}"
	183	)
	184	unknown_floor_classes = sorted(
	185	set(config.evaluation.precision_floors) - set(ontology.class_names)
	186	)
	187	if unknown_floor_classes:
	188	raise config_values.ConfigError(
	189	"evaluation.precision_floors has unknown classes "
	190	f"{unknown_floor_classes} for ontology {ontology.name}"
	191	)
	192	for contrast in config.study.contrasts:
	193	if not is_canonical_metric(contrast.metric, ontology=ontology):
	194	raise config_values.ConfigError(
	195	f"study contrast metric {contrast.metric!r} is not canonical "
	196	f"for ontology {ontology.name}"
	197	)
	198	if config.study.sweep is not None and not is_canonical_metric(
	199	config.study.sweep.objective, ontology=ontology
	200	):
	201	raise config_values.ConfigError(
	202	f"study.sweep.objective must be canonical for ontology {ontology.name}"
	203	)
	204
	205
	206	def _validate_gates(
	207	config: config_sections.HarnessConfig, ontology: Ontology
	208	) -> None:
	209	"""Check gate payloads and the readiness status they must back."""
	210	for gate in config.experiment.gates:
	211	if isinstance(gate.params, config_schema.SptPartitionOracleParams):
	212	unknown = sorted(
	213	set(gate.params.minimum_purity_by_class) - set(ontology.class_names)
	214	)
	215	if unknown:
	216	raise config_values.ConfigError(
	217	f"gate {gate.name}.minimum_purity_by_class contains unknown "
	218	f"ontology classes {unknown} for ontology {ontology.name}"
	219	)
	220	gate_types = {gate.type for gate in config.experiment.gates if gate.required}
	221	if (
	222	config.experiment.status == "template-only"
	223	and "implementation_ticket" not in gate_types
	224	):
	225	raise config_values.ConfigError(
	226	"template-only experiments require an implementation_ticket gate"
	227	)
	228	if config.experiment.status == "gated-later" and not gate_types:
	229	raise config_values.ConfigError(
	230	"gated-later experiments require at least one required gate"
	231	)
	232	if config.experiment.status == "implement-now" and config.experiment.id not in {
	233	"E1",
	234	"E2",
	235	}:
	236	raise config_values.ConfigError(
	237	"Only E1 and E2 are implement-now in schema version 1"
	238	)
	239
	240
	241	def _load_task_ontology(config: config_sections.HarnessConfig) -> Ontology:
	242	"""Load the ontology the config declares, failing closed as a ConfigError.
	243
	244	Args:
	245	config: Parsed configuration whose ``task.ontology`` path is resolved
	246	relative to the repository root.
	247
	248	Returns:
	249	The validated ontology every other contract is checked against.
	250
	251	Raises:
	252	ConfigError: If the ontology cannot be located, loaded, or is invalid.
	253	"""
	254	resolved = resolve_ontology_path(config)
	255	try:
	256	return load_ontology(resolved)
	257	except OntologyError as exc:
	258	raise config_values.ConfigError(
	259	f"task.ontology {config.task.ontology.as_posix()} is not a valid "
	260	f"ontology: {exc}"
	261	) from exc
	262
	263
	264	def _validate_task_against_ontology(
	265	config: config_sections.HarnessConfig, ontology: Ontology
	266	) -> None:
	267	"""Check that the task block restates the loaded ontology exactly.
	268
	269	Args:
	270	config: Parsed configuration.
	271	ontology: Ontology loaded from ``task.ontology``.
	272
	273	Raises:
	274	ConfigError: If any task, evaluation, model, or loss class contract
	275	disagrees with the loaded ontology.
	276	"""
	277	task = config.task
	278	if task.num_classes != ontology.num_predicted_classes:
	279	raise config_values.ConfigError(
	280	f"task.num_classes {task.num_classes} must equal ontology "
	281	f"{ontology.name} num_predicted_classes "
	282	f"{ontology.num_predicted_classes}"
	283	)
	284	if task.ignore_index != ontology.void_id:
	285	raise config_values.ConfigError(
	286	f"task.ignore_index {task.ignore_index} must equal ontology "
	287	f"{ontology.name} void ID {ontology.void_id}"
	288	)
	289	if task.classes_of_interest != ontology.interest_ids:
	290	raise config_values.ConfigError(
	291	f"task.classes_of_interest {list(task.classes_of_interest)} must "
	292	f"equal ontology {ontology.name} interest IDs "
	293	f"{list(ontology.interest_ids)}"
	294	)
	295	linear_names = tuple(
	296	ontology.class_for_id(train_id).name for train_id in ontology.linear_class_ids
	297	)
	298	if len(set(task.linear_classes)) != len(task.linear_classes) or set(
	299	task.linear_classes
	300	) != set(linear_names):
	301	raise config_values.ConfigError(
	302	f"task.linear_classes {list(task.linear_classes)} must be exactly "
	303	f"the linear classes {list(linear_names)} of ontology "
	304	f"{ontology.name}"
	305	)
	306	profiles = config.evaluation.object_matching.cluster_profiles
	307	if profiles != task.ontology:
	308	raise config_values.ConfigError(
	309	"evaluation.object_matching.cluster_profiles "
	310	f"{profiles.as_posix()} must be the task ontology "
	311	f"{task.ontology.as_posix()}"
	312	)
	313	if "num_classes" in config.model.args:
	314	declared = config.model.args["num_classes"]
	315	if declared != task.num_classes:
	316	raise config_values.ConfigError(
	317	f"model.args.num_classes {declared!r} must equal "
	318	f"task.num_classes {task.num_classes}"
	319	)
	320	if "ignore_index" in config.loss.args:
	321	declared = config.loss.args["ignore_index"]
	322	if declared != task.ignore_index:
	323	raise config_values.ConfigError(
	324	f"loss.args.ignore_index {declared!r} must equal "
	325	f"task.ignore_index {task.ignore_index}"
	326	)
	327
	328
	329	def _validate_study_overrides(
	330	config: config_sections.HarnessConfig, raw: Mapping[str, Any]
	331	) -> None:
	332	"""Check every declared override against the document's own leaves."""
	333	leaves = config_values.leaf_values(raw)
	334	override_groups: list[Mapping[str, config_values.OverrideValue]] = [
	335	item.overrides for item in config.study.variants
	336	]
	337	if config.study.matrix is not None:
	338	override_groups.extend(
	339	{path: value}
	340	for path, values in config.study.matrix.axes.items()
	341	for value in values
	342	)
	343	override_groups.extend(config.study.matrix.include)
	344	override_groups.extend(config.study.matrix.exclude)
	345	if config.study.sweep is not None:
	346	for path, parameter in config.study.sweep.parameters.items():
	347	if parameter.values is not None:
	348	override_groups.extend({path: value} for value in parameter.values)
	349	else:
	350	override_groups.extend(
	351	({path: parameter.minimum}, {path: parameter.maximum})
	352	)
	353	for overrides in override_groups:
	354	for path, value in overrides.items():
	355	if path.startswith("study.") or path not in leaves:
	356	raise config_values.ConfigError(
	357	f"Study override path {path!r} is not a declared scalar/list leaf"
	358	)
	359	expected = leaves[path]
	360	if not config_values.same_leaf_type(expected, value):
	361	raise config_values.ConfigError(
	362	f"Study override {path!r} has incompatible value {value!r}; "
	363	f"expected type of {expected!r}"
	364	)
0

Importance #10: src/train/config_sections.py @@ -0,0 +1,414 @@

	1	"""Pydantic models for the data, model, training, and evaluation blocks.
	2
	3	The models mirror the experiment YAML one-to-one: a nested block is a nested
	4	model, a field name is the YAML key, and every value keeps the strict typing of
	5	:mod:`src.train.config_values`. Cross-section rules that need the loaded
	6	ontology stay in :mod:`src.train.config_rules`.
	7	"""
	8
	9	from __future__ import annotations
	10
	11	import logging
	12	from collections.abc import Mapping
	13	from pathlib import Path
	14	from typing import Any, Literal
	15
	16	import pydantic
	17
	18	from src.train import config_schema, config_values
	19
	20	logger = logging.getLogger(__name__)
	21
	22	SOURCE_PATH_ALIAS = "__source_path__"
	23	SHA256_ALIAS = "__sha256__"
	24	_MODEL_ARG_KEYS = frozenset(
	25	{
	26	"aggregation", "annotation_mode", "balanced_crops",
	27	"confidence_only_forbidden", "enable_flash", "geometry_context",
	28	"head", "label_fraction", "max_num_edges", "max_num_nodes",
	29	"num_classes", "ontology_priority", "own_unlabeled_only",
	30	"partition", "partition_stage", "patch_size", "published_weights",
	31	"require_multiview_agreement", "round", "scanner_holdout", "selector",
	32	"semantic_stage", "timing_instrumentation", "training_population",
	33	"uncertainty_tier", "unlicensed_scribblekitti_code", "voxel_sizes_m",
	34	}
	35	)
	36	_PARTITION_KEYS = frozenset(
	37	{"regularization", "spatial_weight", "cutoff", "graph_k_max", "graph_gap_m"}
	38	)
	39	_LOSS_ARG_KEYS = frozenset(
	40	{"alpha", "beta", "class_weighting", "gamma", "ignore_index", "reason"}
	41	)
	42
	43
	44	class TaskConfig(config_schema.StrictConfigModel):
	45	"""Frozen semantic task contract."""
	46
	47	ontology: Path
	48	num_classes: int
	49	ignore_index: int
	50	classes_of_interest: tuple[int, ...]
	51	linear_classes: tuple[str, ...]
	52
	53
	54	class LabelSourceConfig(config_schema.StrictConfigModel):
	55	"""Label provenance and fail-closed join policy."""
	56
	57	mode: Literal["artifact", "regenerate_full_resolution"]
	58	source_geometry_glob: str
	59	fuse_config: Path \| None
	60	prefer: str
	61	classical_glob: str \| None
	62	recap_glob: str \| None
	63	stats_glob: str
	64	unmatched_policy: Literal["void"]
	65	max_unmatched_fraction: float = pydantic.Field(ge=0.0, le=1.0)
	66
	67	@pydantic.model_validator(mode="after")
	68	def _fuse_config_matches_mode(self) -> LabelSourceConfig:
	69	"""Tie the fusion config to the declared label mode."""
	70	if self.mode == "regenerate_full_resolution" and self.fuse_config is None:
	71	raise config_values.ConfigError(
	72	"regenerate_full_resolution requires data.label_source.fuse_config"
	73	)
	74	if self.mode == "artifact" and self.fuse_config is not None:
	75	raise config_values.ConfigError(
	76	"artifact label mode requires data.label_source.fuse_config: null"
	77	)
	78	return self
	79
	80
	81	class CorridorSelectionConfig(config_schema.StrictConfigModel):
	82	"""Config-declared corridor allow-list."""
	83
	84	include: tuple[str, ...]
	85
	86
	87	class FeatureConfig(config_schema.StrictConfigModel):
	88	"""Ordered features and train-only normalization contract."""
	89
	90	names: tuple[str, ...]
	91	normalization_manifest: Path
	92	fit_on: Literal["train_corridors_only"]
	93	scanner_conditioning: bool
	94
	95
	96	class TilingConfig(config_schema.StrictConfigModel):
	97	"""Deterministic corridor tiling and overlap blending contract."""
	98
	99	mode: Literal["corridor_axis"]
	100	length_m: float = pydantic.Field(gt=0.0)
	101	overlap_m: float = pydantic.Field(ge=0.0)
	102	origin: Literal["dataset_manifest"]
	103	min_points: int = pydantic.Field(ge=1)
	104	blend: Literal["linear_edge_weight"]
	105
	106	@pydantic.model_validator(mode="after")
	107	def _overlap_fits_in_a_tile(self) -> TilingConfig:
	108	"""Reject an overlap that is not shorter than the tile."""
	109	if self.overlap_m >= self.length_m:
	110	raise config_values.ConfigError("tiling requires 0 <= overlap_m < length_m")
	111	return self
	112
	113
	114	class DataConfig(config_schema.StrictConfigModel):
	115	"""Input roots, splits, labels, features, and tiling."""
	116
	117	root: Path
	118	canonical_root: Path
	119	processed_root: Path
	120	split_manifest: Path
	121	label_source: LabelSourceConfig
	122	corridors: CorridorSelectionConfig
	123	features: FeatureConfig
	124	tiling: TilingConfig
	125
	126
	127	class SptAdapterConfig(config_schema.StrictConfigModel):
	128	"""SPT raw-dataset emission contract."""
	129
	130	raw_root: Path
	131	pc_tiling: int
	132	voxel_m: float = pydantic.Field(gt=0.0)
	133	base_family: str
	134	raw_row_sidecar_keys: tuple[str, ...]
	135	audit_only_data_keys: tuple[str, ...]
	136
	137
	138	class PointceptAdapterConfig(config_schema.StrictConfigModel):
	139	"""Pointcept default-dataset emission contract."""
	140
	141	root: Path
	142	grid_size_m: float = pydantic.Field(gt=0.0)
	143	preserve_keys: tuple[str, ...]
	144
	145
	146	class AdapterConfig(config_schema.StrictConfigModel):
	147	"""Framework-neutral and external-format adapter contract."""
	148
	149	emit: tuple[str, ...]
	150	canonical_version: int
	151	identity: Literal["source_file_and_row"]
	152	spt: SptAdapterConfig
	153	pointcept: PointceptAdapterConfig
	154
	155	@pydantic.field_validator("emit")
	156	@classmethod
	157	def _emit_is_supported(cls, value: tuple[str, ...]) -> tuple[str, ...]:
	158	"""Reject unknown output formats and a missing canonical emission."""
	159	if set(value) - {"canonical", "spt", "pointcept"}:
	160	raise config_values.ConfigError(
	161	"adapter.emit contains an unsupported output format"
	162	)
	163	if "canonical" not in value:
	164	raise config_values.ConfigError("adapter.emit must include canonical")
	165	return value
	166
	167	@pydantic.field_validator("canonical_version")
	168	@classmethod
	169	def _canonical_version_is_one(cls, value: int) -> int:
	170	"""Freeze the canonical dataset version at 1."""
	171	if value != 1:
	172	raise config_values.ConfigError("adapter.canonical_version must be 1")
	173	return value
	174
	175
	176	class ModelConfig(config_schema.StrictConfigModel):
	177	"""Local or external model/runner selection."""
	178
	179	framework: Literal["cpu", "spt", "pointcept"]
	180	runner: Path
	181	name: str
	182	base_config: Path \| None
	183	checkout_env: str \| None
	184	commit_env: str \| None
	185	checkpoint: Path \| None
	186	args: Mapping[str, Any]
	187
	188	@pydantic.field_validator("args")
	189	@classmethod
	190	def _args_are_declared(cls, value: Mapping[str, Any]) -> Mapping[str, Any]:
	191	"""Reject undeclared model arguments and partition keys."""
	192	config_values.check_keys(value, set(), set(_MODEL_ARG_KEYS), "model.args")
	193	if "partition" in value:
	194	partition = config_values.mapping(
	195	value["partition"], "model.args.partition"
	196	)
	197	config_values.check_keys(
	198	partition,
	199	set(_PARTITION_KEYS),
	200	set(_PARTITION_KEYS),
	201	"model.args.partition",
	202	)
	203	return value
	204
	205	@pydantic.model_validator(mode="after")
	206	def _external_models_pin_their_checkout(self) -> ModelConfig:
	207	"""Tie the external checkout variables to the declared framework."""
	208	if self.framework == "cpu":
	209	if self.checkout_env is not None or self.commit_env is not None:
	210	raise config_values.ConfigError(
	211	"CPU experiments cannot declare external checkout variables"
	212	)
	213	elif not (self.base_config and self.checkout_env and self.commit_env):
	214	raise config_values.ConfigError(
	215	"External models require base_config, checkout_env, and commit_env"
	216	)
	217	return self
	218
	219
	220	class LossConfig(config_schema.StrictConfigModel):
	221	"""Shared-registry or external-native loss selection."""
	222
	223	name: Literal[
	224	"framework_native",
	225	"cross_entropy",
	226	"focal_cross_entropy",
	227	"masked_focal_tversky",
	228	]
	229	args: Mapping[str, Any]
	230
	231	@pydantic.field_validator("args")
	232	@classmethod
	233	def _args_are_declared(cls, value: Mapping[str, Any]) -> Mapping[str, Any]:
	234	"""Reject undeclared loss arguments."""
	235	config_values.check_keys(value, set(), set(_LOSS_ARG_KEYS), "loss.args")
	236	return value
	237
	238
	239	class TrainConfig(config_schema.StrictConfigModel):
	240	"""Harness-visible training settings for local and external runners."""
	241
	242	max_epochs: int = -1
	243	lr: float = 3.0e-4
	244	weight_decay: float = 1.0e-4
	245	precision: str = "auto"
	246	accumulate_grad_batches: int = 1
	247	accelerator: str = "auto"
	248	devices: int \| str = 1
	249	viz_every_n_epochs: int = 2
	250	viz_samples: int = 4
	251	monitor: str = "val/f1_mean_fg"
	252	monitor_mode: Literal["min", "max"] = "max"
	253	early_stop_monitor: str = "val/loss"
	254	early_stop_mode: Literal["min", "max"] = "min"
	255	early_stop_patience: int = 4
	256	log_dir: str = "runs"
	257	log_every_n_steps: int = 10
	258	batch_size: int = 1
	259	num_workers: int = 4
	260	optimizer: str = "adamw"
	261	scheduler: str = "cosine"
	262	distributed: bool = False
	263
	264
	265	class ContinuityConfig(config_schema.StrictConfigModel):
	266	"""Linear-continuity binning contract."""
	267
	268	chainage_bin_m: float
	269	gap_threshold_m: float
	270
	271
	272	class ObjectMatchingConfig(config_schema.StrictConfigModel):
	273	"""Object-clustering profile source and fallback tolerances."""
	274
	275	cluster_profiles: Path
	276	minimum_iou: float
	277	centroid_tolerance_m: float
	278
	279
	280	class BootstrapConfig(config_schema.StrictConfigModel):
	281	"""Corridor/spatial bootstrap contract."""
	282
	283	unit: Literal["corridor"]
	284	spatial_block_m: float
	285	samples: int
	286	confidence: float
	287	seed: int
	288
	289
	290	class PromotionConfig(config_schema.StrictConfigModel):
	291	"""Locked-test promotion margins and superiority conditions."""
	292
	293	enabled: bool
	294	delta_quality: float \| None
	295	delta_fp_per_km: float \| None
	296	superiority_conditions: tuple[str, ...]
	297
	298	@pydantic.model_validator(mode="after")
	299	def _enabled_promotion_is_complete(self) -> PromotionConfig:
	300	"""Reject an enabled promotion without margins or conditions."""
	301	if self.enabled and (
	302	self.delta_quality is None
	303	or self.delta_fp_per_km is None
	304	or not self.superiority_conditions
	305	):
	306	raise config_values.ConfigError(
	307	"enabled promotion requires non-null margins and superiority "
	308	"conditions"
	309	)
	310	return self
	311
	312
	313	class EvaluationConfig(config_schema.StrictConfigModel):
	314	"""Held-out evaluation and promotion protocol."""
	315
	316	split: Literal["validation", "promotion_test"]
	317	metrics: tuple[str, ...]
	318	precision_floors: Mapping[str, float]
	319	continuity: ContinuityConfig
	320	object_matching: ObjectMatchingConfig
	321	bootstrap: BootstrapConfig
	322	promotion: PromotionConfig
	323	worst_k_tiles: int = pydantic.Field(default=4, ge=1)
	324
	325	@pydantic.field_validator("precision_floors")
	326	@classmethod
	327	def _floors_are_fractions(cls, value: Mapping[str, float]) -> Mapping[str, float]:
	328	"""Reject a precision floor outside the unit interval."""
	329	for name, floor in value.items():
	330	if not 0.0 <= floor <= 1.0:
	331	raise config_values.ConfigError(
	332	f"evaluation precision floor for {name} must be in [0, 1]"
	333	)
	334	return value
	335
	336	@pydantic.model_validator(mode="after")
	337	def _promotion_test_is_promotable(self) -> EvaluationConfig:
	338	"""Reject a locked-test split whose promotion protocol is disabled."""
	339	if self.split == "promotion_test" and not self.promotion.enabled:
	340	raise config_values.ConfigError(
	341	"promotion_test evaluation requires evaluation.promotion.enabled"
	342	)
	343	return self
	344
	345
	346	class RuntimeConfig(config_schema.StrictConfigModel):
	347	"""Operational hardware and kernel constraints."""
	348
	349	target: str
	350	cuda: str
	351	spconv: Literal[
	352	"disabled",
	353	"spconv-cu124>=2.3.0,<2.4.0",
	354	"spconv-cu126>=2.3.0,<2.4.0",
	355	]
	356	flash_attention: bool
	357	system_ram_gb: int
	358	gpu_memory_gb: int
	359
	360
	361	class ProvenanceConfig(config_schema.StrictConfigModel):
	362	"""Mandatory run-evidence policy."""
	363
	364	manifest: Literal["required"]
	365	data_hash_source: Literal["dvc"]
	366	record_environment: bool
	367	record_commands: bool
	368	checkpoint_policy: Path
	369
	370
	371	class VisualizationConfig(config_schema.StrictConfigModel):
	372	"""Opt-in training-time TensorBoard class-mask visualization."""
	373
	374	masks_every_n_epochs: int = pydantic.Field(ge=1)
	375	masks_tiles: int \| tuple[str, ...] = 2
	376
	377	@pydantic.model_validator(mode="after")
	378	def _tiles_select_something(self) -> VisualizationConfig:
	379	"""Reject an empty or non-positive tile selection."""
	380	if isinstance(self.masks_tiles, int):
	381	if self.masks_tiles < 1:
	382	raise config_values.ConfigError(
	383	"visualization.masks_tiles must be >= 1"
	384	)
	385	elif not self.masks_tiles:
	386	raise config_values.ConfigError("visualization.masks_tiles cannot be empty")
	387	return self
	388
	389
	390	class HarnessConfig(config_schema.StrictConfigModel):
	391	"""Fully parsed, cross-field-validated experiment configuration."""
	392
	393	schema_version: int
	394	experiment: config_schema.ExperimentConfig
	395	study: config_schema.StudyConfig
	396	seed: int = pydantic.Field(ge=0)
	397	task: TaskConfig
	398	data: DataConfig
	399	adapter: AdapterConfig
	400	model: ModelConfig
	401	loss: LossConfig
	402	train: TrainConfig
	403	evaluation: EvaluationConfig
	404	runtime: RuntimeConfig
	405	provenance: ProvenanceConfig
	406	source_path: Path = pydantic.Field(validation_alias=SOURCE_PATH_ALIAS)
	407	sha256: str = pydantic.Field(validation_alias=SHA256_ALIAS)
	408	visualization: VisualizationConfig \| None = None
	409
	410	_raw_document: Mapping[str, Any] \| None = pydantic.PrivateAttr(default=None)
	411
	412	def as_dict(self) -> dict[str, Any]:
	413	"""Return the resolved model tree as JSON-safe primitives."""
	414	return self.model_dump(mode="json")
0

Importance #11: src/train/config_values.py @@ -0,0 +1,256 @@

	1	"""Strict YAML value typing shared by the experiment configuration models.
	2
	3	The experiment YAML is a fail-closed contract: a value must already carry its
	4	declared type, so ``"50"`` is not a float, ``1`` is not a boolean, and ``3.0``
	5	is not an integer. That is deliberately stricter than the fleet coercion
	6	matrix of :func:`iolabs.common.config_loader.coerce_config_value`, so the
	7	models in :mod:`src.train.config_schema` route every field through
	8	:func:`typed_value` instead of the inherited coercion.
	9	"""
	10
	11	from __future__ import annotations
	12
	13	import logging
	14	from collections.abc import Mapping, Sequence
	15	from pathlib import Path
	16	from types import MappingProxyType, UnionType
	17	from typing import Any, Literal, TypeAlias, Union, get_args, get_origin
	18
	19	from iolabs.common import config_loader
	20
	21	logger = logging.getLogger(__name__)
	22
	23	Scalar: TypeAlias = str \| int \| float \| bool \| None
	24	OverrideValue: TypeAlias = Scalar \| list[Scalar]
	25
	26
	27	class ConfigError(config_loader.ConfigError):
	28	"""Raised when an experiment configuration violates its strict schema."""
	29
	30
	31	def typed_value(annotation: Any, value: Any, where: str) -> Any:
	32	"""Parse one config leaf strictly as its declared annotation.
	33
	34	Nested models, ``Annotated`` aliases and unresolved annotations are
	35	returned unchanged so pydantic validates them itself.
	36
	37	Args:
	38	annotation: Resolved field annotation, or None when unknown.
	39	value: Raw YAML value.
	40	where: Dotted key path used in error messages.
	41
	42	Returns:
	43	The value converted to the declared type.
	44
	45	Raises:
	46	ConfigError: If the value does not match the declared type.
	47	"""
	48	if annotation is None or annotation is Any:
	49	return value
	50	origin = get_origin(annotation)
	51	if origin is Literal:
	52	return choice(value, {str(item) for item in get_args(annotation)}, where)
	53	if origin in (Union, UnionType):
	54	return _typed_union(annotation, value, where)
	55	if annotation is Path:
	56	return value if isinstance(value, Path) else path(value, where)
	57	if annotation is bool:
	58	return boolean(value, where)
	59	if annotation is int:
	60	return integer(value, where)
	61	if annotation is float:
	62	return number(value, where)
	63	if annotation is str:
	64	return string(value, where)
	65	if origin in (tuple, list):
	66	args = get_args(annotation)
	67	item_type = args[0] if args else None
	68	items = [typed_value(item_type, item, where) for item in sequence(value, where)]
	69	return tuple(items) if origin is tuple else items
	70	if origin in (dict, Mapping) or (
	71	isinstance(origin, type) and issubclass(origin, Mapping)
	72	):
	73	args = get_args(annotation)
	74	item_type = args[1] if len(args) == 2 else None
	75	raw = mapping(value, where)
	76	return MappingProxyType(
	77	{
	78	string(key, where): typed_value(item_type, item, f"{where}.{key}")
	79	for key, item in raw.items()
	80	}
	81	)
	82	return value
	83
	84
	85	def _typed_union(annotation: Any, value: Any, where: str) -> Any:
	86	"""Parse a value against a union annotation, rejecting every mismatch."""
	87	members = get_args(annotation)
	88	optional = type(None) in members
	89	if value is None:
	90	if optional:
	91	return None
	92	raise ConfigError(f"{where} must not be null")
	93	candidates = [item for item in members if item is not type(None)]
	94	if len(candidates) == 1:
	95	return typed_value(candidates[0], value, where)
	96	for candidate in candidates:
	97	try:
	98	return typed_value(candidate, value, where)
	99	except ConfigError:
	100	continue
	101	names = sorted(getattr(item, "__name__", str(item)) for item in candidates)
	102	raise ConfigError(
	103	f"{where} must be one of {names}, got {type(value).__name__} {value!r}"
	104	)
	105
	106
	107	def mapping(value: Any, where: str) -> Mapping[str, Any]:
	108	"""Return value as a string-keyed mapping or raise."""
	109	if not isinstance(value, Mapping):
	110	raise ConfigError(f"{where} must be a mapping")
	111	if any(not isinstance(key, str) for key in value):
	112	raise ConfigError(f"{where} keys must be strings")
	113	return value
	114
	115
	116	def sequence(value: Any, where: str) -> Sequence[Any]:
	117	"""Return value as a non-string sequence or raise."""
	118	if isinstance(value, (str, bytes)) or not isinstance(value, Sequence):
	119	raise ConfigError(f"{where} must be a sequence")
	120	return value
	121
	122
	123	def string(value: Any, where: str) -> str:
	124	"""Return value as a non-empty string or raise."""
	125	if not isinstance(value, str) or not value.strip():
	126	raise ConfigError(f"{where} must be a non-empty string")
	127	return value
	128
	129
	130	def integer(value: Any, where: str) -> int:
	131	"""Return value as an integer, rejecting booleans, or raise."""
	132	if isinstance(value, bool) or not isinstance(value, int):
	133	raise ConfigError(f"{where} must be an integer")
	134	return value
	135
	136
	137	def number(value: Any, where: str) -> float:
	138	"""Return value as a float, rejecting booleans, or raise."""
	139	if isinstance(value, bool) or not isinstance(value, (int, float)):
	140	raise ConfigError(f"{where} must be numeric")
	141	return float(value)
	142
	143
	144	def boolean(value: Any, where: str) -> bool:
	145	"""Return value as a boolean or raise."""
	146	if not isinstance(value, bool):
	147	raise ConfigError(f"{where} must be boolean")
	148	return value
	149
	150
	151	def path(value: Any, where: str) -> Path:
	152	"""Return value as a repository-relative path or raise."""
	153	text = string(value, where)
	154	parsed = Path(text)
	155	if parsed.is_absolute():
	156	raise ConfigError(f"{where} must be repository-relative, got {parsed}")
	157	return parsed
	158
	159
	160	def choice(value: Any, choices: set[str], where: str) -> str:
	161	"""Return value when it is one of the allowed string choices."""
	162	text = string(value, where)
	163	if text not in choices:
	164	raise ConfigError(f"{where} must be one of {sorted(choices)}, got {text!r}")
	165	return text
	166
	167
	168	def check_keys(
	169	value: Mapping[str, Any],
	170	required: set[str],
	171	allowed: set[str],
	172	where: str,
	173	) -> None:
	174	"""Raise when value misses a required key or carries an unknown one.
	175
	176	Args:
	177	value: Free-form mapping whose keys are not model fields.
	178	required: Keys that must be present.
	179	allowed: Keys that may be present.
	180	where: Dotted key path used in error messages.
	181
	182	Raises:
	183	ConfigError: If a key is missing or unknown.
	184	"""
	185	missing = sorted(required - set(value))
	186	unknown = sorted(set(value) - allowed)
	187	if missing or unknown:
	188	details = []
	189	if missing:
	190	details.append(f"missing {missing}")
	191	if unknown:
	192	details.append(f"unknown {unknown}")
	193	raise ConfigError(f"{where} has " + " and ".join(details))
	194
	195
	196	def leaf_values(value: Any, prefix: str = "") -> dict[str, OverrideValue]:
	197	"""Return every dotted scalar/list leaf of a raw configuration mapping.
	198
	199	The ``study`` block is skipped: a study never overrides itself.
	200
	201	Args:
	202	value: Raw mapping, sequence, or scalar.
	203	prefix: Dotted path of value inside the document.
	204
	205	Returns:
	206	Mapping of dotted leaf path to its declared value.
	207	"""
	208	result: dict[str, OverrideValue] = {}
	209	if isinstance(value, Mapping):
	210	for key, item in value.items():
	211	dotted = f"{prefix}.{key}" if prefix else str(key)
	212	if dotted == "study" or dotted.startswith("study."):
	213	continue
	214	result.update(leaf_values(item, dotted))
	215	elif isinstance(value, list):
	216	if all(
	217	isinstance(item, (str, int, float, bool)) or item is None
	218	for item in value
	219	):
	220	result[prefix] = value
	221	elif isinstance(value, (str, int, float, bool)) or value is None:
	222	result[prefix] = value
	223	return result
	224
	225
	226	def same_leaf_type(expected: OverrideValue, actual: OverrideValue) -> bool:
	227	"""Return whether actual may replace the declared leaf expected."""
	228	if isinstance(expected, list):
	229	if not isinstance(actual, list):
	230	return False
	231	if not expected or not actual:
	232	return True
	233	return all(same_leaf_type(expected[0], item) for item in actual)
	234	if isinstance(expected, bool):
	235	return isinstance(actual, bool)
	236	if isinstance(expected, int) and not isinstance(expected, bool):
	237	return isinstance(actual, int) and not isinstance(actual, bool)
	238	if isinstance(expected, float):
	239	return isinstance(actual, (int, float)) and not isinstance(actual, bool)
	240	return actual is None if expected is None else isinstance(actual, type(expected))
	241
	242
	243	def coerce_leaf(expected: OverrideValue, value: OverrideValue) -> OverrideValue:
	244	"""Parse an override value as the declared leaf's type."""
	245	if isinstance(expected, list):
	246	items = list(value) if isinstance(value, list) else [value]
	247	if expected and isinstance(expected[0], float):
	248	return [coerce_leaf(expected[0], item) for item in items]
	249	return items
	250	if (
	251	isinstance(expected, float)
	252	and isinstance(value, int)
	253	and not isinstance(value, bool)
	254	):
	255	return float(value)
	256	return value
0

Importance #12: dvc.yaml @@ -37,8 +37,13 @@

       - scripts/ingest_preannotations.py
       - src/contracts/splits.py
       - src/dataset/preannotations.py
       - src/train/config.py
+      - src/train/config_rules.py
+      - src/train/config_schema.py
+      - src/train/config_sections.py
+      - src/train/config_study.py
+      - src/train/config_values.py
     outs:
       - data/01_interim/kirioll_v1/labels
   prepare_canonical:

Importance #13: dvc.yaml @@ -62,8 +67,13 @@

       - src/dataset/las.py
       - src/dataset/preannotations.py
       - src/dataset/tiling.py
       - src/train/config.py
+      - src/train/config_rules.py
+      - src/train/config_schema.py
+      - src/train/config_sections.py
+      - src/train/config_study.py
+      - src/train/config_values.py
     outs:
       # The canonical lane holds the tiles, per-corridor <id>.adapter.json
       # manifests, the pinned axis/grid_origin files, and manifest.json.
       # The normalization manifest is keyed by the config's feature stack, so

Importance #14: dvc.yaml @@ -94,8 +104,13 @@

       - src/dataset/canonical.py
       - src/dataset/features.py
       - src/dataset/tiling.py
       - src/train/config.py
+      - src/train/config_rules.py
+      - src/train/config_schema.py
+      - src/train/config_sections.py
+      - src/train/config_study.py
+      - src/train/config_values.py
     outs:
       # Tier 2: versioned by recipe. Flip to cache: true on dataset freeze.
       - data/02_processed/kirioll_v1/spt/raw:
           cache: false

Importance #15: dvc.yaml @@ -119,8 +134,13 @@

       - src/dataset/canonical.py
       - src/dataset/features.py
       - src/dataset/tiling.py
       - src/train/config.py
+      - src/train/config_rules.py
+      - src/train/config_schema.py
+      - src/train/config_sections.py
+      - src/train/config_study.py
+      - src/train/config_values.py
     outs:
       # Tier 2: versioned by recipe. Flip to cache: true on dataset freeze.
       - data/02_processed/kirioll_v1/pointcept:
           cache: false

Importance #16: dvc.yaml @@ -150,8 +170,13 @@

       - scripts/ingest_preannotations.py
       - src/contracts/splits.py
       - src/dataset/preannotations.py
       - src/train/config.py
+      - src/train/config_rules.py
+      - src/train/config_schema.py
+      - src/train/config_sections.py
+      - src/train/config_study.py
+      - src/train/config_values.py
     outs:
       - data/01_interim/a1_recap_v1/labels
   prepare_canonical_a1_recap:

Importance #17: dvc.yaml @@ -175,8 +200,13 @@

       - src/dataset/las.py
       - src/dataset/preannotations.py
       - src/dataset/tiling.py
       - src/train/config.py
+      - src/train/config_rules.py
+      - src/train/config_schema.py
+      - src/train/config_sections.py
+      - src/train/config_study.py
+      - src/train/config_values.py
     outs:
       - data/01_interim/a1_recap_v1/canonical
       - data/01_interim/a1_recap_v1/normalization_xyz_intensity_rgb.json

Importance #18: dvc.yaml @@ -199,8 +229,13 @@

       - src/dataset/canonical.py
       - src/dataset/features.py
       - src/dataset/tiling.py
       - src/train/config.py
+      - src/train/config_rules.py
+      - src/train/config_schema.py
+      - src/train/config_sections.py
+      - src/train/config_study.py
+      - src/train/config_values.py
     outs:
       # Tier 2: versioned by recipe. Flip to cache: true on dataset freeze.
       - data/02_processed/a1_recap_v1/pointcept:
           cache: false

Importance #19: frameworks/run_provenance.py @@ -311,9 +310,9 @@

311	"schema": config.adapter.canonical_version,	310	"schema": config.adapter.canonical_version,
312	"feature_normalization": (	311	"feature_normalization": (
313	config.data.features.normalization_manifest.as_posix()	312	config.data.features.normalization_manifest.as_posix()
314	),	313	),
315	"tiling": _json_safe(asdict(config.data.tiling)),	314	"tiling": config.data.tiling.model_dump(mode="json"),
316	"voxel": {	315	"voxel": {
317	"spt": config.adapter.spt.voxel_m,	316	"spt": config.adapter.spt.voxel_m,
318	"pointcept": config.adapter.pointcept.grid_size_m,	317	"pointcept": config.adapter.pointcept.grid_size_m,
319	"grid_origin_xyz": pinned_grid_origin(	318	"grid_origin_xyz": pinned_grid_origin(

Importance #20: scripts/evaluate.py @@ -2191,9 +2191,9 @@

2191	"schema": config.adapter.canonical_version,	2191	"schema": config.adapter.canonical_version,
2192	"feature_normalization": (	2192	"feature_normalization": (
2193	config.data.features.normalization_manifest.as_posix()	2193	config.data.features.normalization_manifest.as_posix()
2194	),	2194	),
2195	"tiling": _json_safe(asdict(config.data.tiling)),	2195	"tiling": config.data.tiling.model_dump(mode="json"),
2196	"voxel": {	2196	"voxel": {
2197	"spt": config.adapter.spt.voxel_m,	2197	"spt": config.adapter.spt.voxel_m,
2198	"pointcept": config.adapter.pointcept.grid_size_m,	2198	"pointcept": config.adapter.pointcept.grid_size_m,
2199	"grid_origin_xyz": _pinned_grid_origin(config, records),	2199	"grid_origin_xyz": _pinned_grid_origin(config, records),

Importance #21: frameworks/run_provenance.py @@ -17,9 +17,8 @@

17	import json	17	import json
18	import os	18	import os
19	import shlex	19	import shlex
20	from collections.abc import Mapping, Sequence	20	from collections.abc import Mapping, Sequence
21	from dataclasses import asdict
22	from pathlib import Path	21	from pathlib import Path
23	from typing import Any	22	from typing import Any
24		23
25	from src.contracts.manifest import (	24	from src.contracts.manifest import (

Importance #22: frameworks/run_provenance.py @@ -323,10 +322,10 @@

323	"point_coverage": "training run; see evaluation remap_report.json",	322	"point_coverage": "training run; see evaluation remap_report.json",
324	"remap_report": "reports/remap_report.json",	323	"remap_report": "reports/remap_report.json",
325	},	324	},
326	metric_protocol={	325	metric_protocol={
327	"bootstrap": _json_safe(asdict(config.evaluation.bootstrap)),	326	"bootstrap": config.evaluation.bootstrap.model_dump(mode="json"),
328	"promotion_margins": _json_safe(asdict(config.evaluation.promotion)),	327	"promotion_margins": config.evaluation.promotion.model_dump(mode="json"),
329	},	328	},
330	locked_test_access=False,	329	locked_test_access=False,
331	checkpoint_path=checkpoint.get("path"),	330	checkpoint_path=checkpoint.get("path"),
332	extra={	331	extra={

Importance #23: frameworks/run_provenance.py @@ -477,24 +476,6 @@

477	)	476	)
478	return {key: os.environ.get(key) for key in keys}	477	return {key: os.environ.get(key) for key in keys}
479		478
480		479
481	def _json_safe(value: Any) -> Any:
482	"""Convert a dataclass tree into JSON-safe primitives.
483
484	Args:
485	value: Mapping, sequence, path, or scalar.
486
487	Returns:
488	JSON-serializable value.
489	"""
490	if isinstance(value, Path):
491	return value.as_posix()
492	if isinstance(value, Mapping):
493	return {str(key): _json_safe(item) for key, item in value.items()}
494	if isinstance(value, (list, tuple)):
495	return [_json_safe(item) for item in value]
496	return value
497
498
499	if __name__ == "__main__": # pragma: no cover - runner entry point	480	if __name__ == "__main__": # pragma: no cover - runner entry point
500	raise SystemExit(main())	481	raise SystemExit(main())

Importance #24: scripts/evaluate.py @@ -2201,12 +2201,10 @@

             "point_coverage": "see reports/remap_report.json",
             "remap_report": "reports/remap_report.json",
         },
         metric_protocol={
-            "bootstrap": _json_safe(asdict(config.evaluation.bootstrap)),
+            "bootstrap": config.evaluation.bootstrap.model_dump(mode="json"),
-            "promotion_margins": _json_safe(
+            "promotion_margins": config.evaluation.promotion.model_dump(mode="json"),
-                asdict(config.evaluation.promotion)
-            ),
         },
         locked_test_access=locked_test_access,
         checkpoint_path=selection["best_model_path"],
         extra={"status": status, "metrics": metrics, "command": sys.argv},

Importance #25: src/train/config_study.py @@ -0,0 +1,259 @@

	1	"""Deterministic expansion of a typed study definition into cell overrides.
	2
	3	The models own the study schema; this module owns the study algebra: the
	4	ordered cell definitions of a variants, matrix, or sweep study, the identity
	5	token of a cell, and the application of dotted-path overrides to the raw
	6	document a cell is re-validated from.
	7	"""
	8
	9	from __future__ import annotations
	10
	11	import copy
	12	import itertools
	13	import logging
	14	import math
	15	import random
	16	import re
	17	from collections.abc import Mapping
	18	from types import MappingProxyType
	19	from typing import Any
	20
	21	from src.train import config_schema, config_sections, config_values
	22
	23	logger = logging.getLogger(__name__)
	24
	25	CellDefinition = tuple[str, Mapping[str, config_values.OverrideValue]]
	26
	27
	28	def cell_definitions(
	29	config: config_sections.HarnessConfig,
	30	) -> tuple[CellDefinition, ...]:
	31	"""Return the ordered (identity, overrides) pairs of one study.
	32
	33	Args:
	34	config: Strictly parsed experiment configuration.
	35
	36	Returns:
	37	The deterministic cell definitions of the declared study kind.
	38
	39	Raises:
	40	ConfigError: If the study payload of the declared kind is absent or
	41	expands to nothing.
	42	"""
	43	study = config.study
	44	if study.kind == "single":
	45	return ((config.experiment.id, MappingProxyType({})),)
	46	if study.kind == "variants":
	47	if not study.variants:
	48	raise config_values.ConfigError(
	49	f"{config.experiment.id} study.kind variants has no variants payload"
	50	)
	51	return tuple((item.id, item.overrides) for item in study.variants)
	52	if study.kind == "matrix":
	53	if study.matrix is None:
	54	raise config_values.ConfigError(
	55	f"{config.experiment.id} study.kind matrix has no matrix payload"
	56	)
	57	return matrix_cells(study.matrix)
	58	if study.sweep is None:
	59	raise config_values.ConfigError(
	60	f"{config.experiment.id} study.kind sweep has no sweep payload"
	61	)
	62	return sweep_cells(study.sweep, config.seed)
	63
	64
	65	def matrix_cells(matrix: config_schema.MatrixConfig) -> tuple[CellDefinition, ...]:
	66	"""Expand typed matrix axes into deterministic cells."""
	67	axis_paths = tuple(matrix.axes)
	68	unknown = sorted(
	69	{path for item in matrix.exclude for path in item} - set(axis_paths)
	70	)
	71	if unknown:
	72	raise config_values.ConfigError(
	73	f"study.matrix.exclude references non-axis paths {unknown}"
	74	)
	75	definitions: list[CellDefinition] = []
	76	excluded = [0] * len(matrix.exclude)
	77	for combination in itertools.product(*(matrix.axes[path] for path in axis_paths)):
	78	overrides = dict(zip(axis_paths, combination, strict=True))
	79	dropped = False
	80	for index, item in enumerate(matrix.exclude):
	81	if all(overrides[path] == value for path, value in item.items()):
	82	excluded[index] += 1
	83	dropped = True
	84	if not dropped:
	85	definitions.append((cell_id(overrides), MappingProxyType(overrides)))
	86	for index, count in enumerate(excluded):
	87	if not count:
	88	raise config_values.ConfigError(
	89	f"study.matrix.exclude[{index}] matches no matrix cell"
	90	)
	91	for index, item in enumerate(matrix.include):
	92	if not item:
	93	raise config_values.ConfigError(
	94	f"study.matrix.include[{index}] cannot be empty"
	95	)
	96	definitions.append((cell_id(item), item))
	97	if not definitions:
	98	raise config_values.ConfigError("study.matrix excludes every cell")
	99	return tuple(definitions)
	100
	101
	102	def sweep_cells(
	103	sweep: config_schema.SweepConfig, seed: int
	104	) -> tuple[CellDefinition, ...]:
	105	"""Expand a bounded sweep deterministically under the study seed."""
	106	paths = tuple(sweep.parameters)
	107	if sweep.method == "grid":
	108	unbounded = sorted(
	109	path for path, item in sweep.parameters.items() if item.values is None
	110	)
	111	if unbounded:
	112	raise config_values.ConfigError(
	113	f"study.sweep.method grid requires explicit values for {unbounded}"
	114	)
	115	definitions: list[CellDefinition] = []
	116	for combination in itertools.product(
	117	*(tuple(sweep.parameters[path].values or ()) for path in paths)
	118	):
	119	overrides = dict(zip(paths, combination, strict=True))
	120	definitions.append((cell_id(overrides), MappingProxyType(overrides)))
	121	return tuple(definitions[: sweep.budget])
	122	if sweep.method == "random":
	123	generator = random.Random(seed)
	124	return tuple(
	125	(
	126	f"sample_{index:03d}",
	127	MappingProxyType(
	128	{
	129	path: _sweep_sample(
	130	sweep.parameters[path],
	131	generator,
	132	f"study.sweep.parameters.{path}",
	133	)
	134	for path in paths
	135	}
	136	),
	137	)
	138	for index in range(sweep.budget)
	139	)
	140	raise config_values.ConfigError(
	141	f"study.sweep.method {sweep.method!r} is not implemented; supported "
	142	"methods are grid and random"
	143	)
	144
	145
	146	def _sweep_sample(
	147	parameter: config_schema.SweepParameterConfig,
	148	generator: random.Random,
	149	where: str,
	150	) -> config_values.OverrideValue:
	151	"""Draw one deterministic value for a sweep parameter."""
	152	if parameter.values is not None:
	153	return parameter.values[generator.randrange(len(parameter.values))]
	154	if parameter.minimum is None or parameter.maximum is None:
	155	raise config_values.ConfigError(
	156	f"{where} requires minimum and maximum for sampling"
	157	)
	158	if parameter.distribution == "uniform":
	159	drawn = generator.uniform(parameter.minimum, parameter.maximum)
	160	elif parameter.distribution == "log_uniform":
	161	if parameter.minimum <= 0.0:
	162	raise config_values.ConfigError(
	163	f"{where}.minimum must be positive for log_uniform"
	164	)
	165	drawn = math.exp(
	166	generator.uniform(math.log(parameter.minimum), math.log(parameter.maximum))
	167	)
	168	else:
	169	raise config_values.ConfigError(
	170	f"{where}.distribution {parameter.distribution!r} is not implemented; "
	171	"supported distributions are uniform and log_uniform"
	172	)
	173	return float(f"{drawn:.6g}")
	174
	175
	176	def cell_id(overrides: Mapping[str, config_values.OverrideValue]) -> str:
	177	"""Derive a stable, filesystem-safe identity from a cell's overrides."""
	178	if not overrides:
	179	raise config_values.ConfigError("A study cell requires at least one override")
	180	names = [path.rsplit(".", 1)[-1] for path in overrides]
	181	if len(set(names)) != len(names):
	182	names = [path.replace(".", "_") for path in overrides]
	183	return "__".join(
	184	f"{name}-{_value_token(value)}"
	185	for name, value in zip(names, overrides.values(), strict=True)
	186	)
	187
	188
	189	def _value_token(value: config_values.OverrideValue) -> str:
	190	"""Render one override value as a filesystem-safe token."""
	191	if isinstance(value, bool):
	192	text = "true" if value else "false"
	193	elif value is None:
	194	text = "null"
	195	elif isinstance(value, float):
	196	text = repr(value)
	197	elif isinstance(value, list):
	198	text = "+".join(_value_token(item) for item in value)
	199	else:
	200	text = str(value)
	201	return re.sub(r"[^A-Za-z0-9._+-]", "_", text)
	202
	203
	204	def apply_overrides(
	205	raw: Mapping[str, Any],
	206	overrides: Mapping[str, config_values.OverrideValue],
	207	where: str,
	208	) -> dict[str, Any]:
	209	"""Apply dotted-path overrides to a raw configuration mapping.
	210
	211	Args:
	212	raw: Raw mapping of the base configuration.
	213	overrides: Dotted leaf paths mapped to their replacement values.
	214	where: Cell identification used in error messages.
	215
	216	Returns:
	217	A deep copy of ``raw`` carrying the overridden leaves.
	218
	219	Raises:
	220	ConfigError: If a path is not a declared leaf or the value type differs.
	221	"""
	222	result = copy.deepcopy(dict(raw))
	223	leaves = config_values.leaf_values(raw)
	224	for path in sorted(overrides):
	225	value = overrides[path]
	226	if path.startswith("study.") or path not in leaves:
	227	raise config_values.ConfigError(
	228	f"{where} override path {path!r} is not a declared scalar/list leaf"
	229	)
	230	expected = leaves[path]
	231	if not config_values.same_leaf_type(expected, value):
	232	raise config_values.ConfigError(
	233	f"{where} override {path!r} has incompatible value {value!r}; "
	234	f"expected type of {expected!r}"
	235	)
	236	_set_leaf(result, path, config_values.coerce_leaf(expected, value), where)
	237	return result
	238
	239
	240	def _set_leaf(
	241	target: dict[str, Any],
	242	path: str,
	243	value: config_values.OverrideValue,
	244	where: str,
	245	) -> None:
	246	"""Replace one addressable dotted leaf of a raw mapping in place."""
	247	segments = path.split(".")
	248	node: Any = target
	249	for segment in segments[:-1]:
	250	if not isinstance(node, dict) or segment not in node:
	251	raise config_values.ConfigError(
	252	f"{where} override path {path!r} is not addressable"
	253	)
	254	node = node[segment]
	255	if not isinstance(node, dict) or segments[-1] not in node:
	256	raise config_values.ConfigError(
	257	f"{where} override path {path!r} is not addressable"
	258	)
	259	node[segments[-1]] = value
0

Importance #26: tests/test_checkpoint_seam.py @@ -16,9 +16,8 @@

16	"""	16	"""
17		17
18	from __future__ import annotations	18	from __future__ import annotations
19		19
20	import dataclasses
21	import hashlib	20	import hashlib
22	import importlib.util	21	import importlib.util
23	import json	22	import json
24	import sys	23	import sys

Importance #27: tests/test_checkpoint_seam.py @@ -154,12 +153,10 @@

         (canonical_root / "canonical" / f"{corridor_id}.adapter.json").write_text(
             json.dumps({"tiling": {"grid_origin_xyz": GRID_ORIGIN}}),
             encoding="utf-8",
         )
-    return dataclasses.replace(
+    data = config.data.model_copy(update={"canonical_root": canonical_root})
-        config,
+    return config.model_copy(update={"data": data})
-        data=dataclasses.replace(config.data, canonical_root=canonical_root),
-    )
 def _training_run(
     tmp_path: Path,

Importance #28: tests/test_framework_run_provenance.py @@ -1,9 +1,8 @@

1	"""Training-run provenance for framework runners (spec section 9.7)."""	1	"""Training-run provenance for framework runners (spec section 9.7)."""
2		2
3	from __future__ import annotations	3	from __future__ import annotations
4		4
5	import dataclasses
6	import json	5	import json
7	import sys	6	import sys
8	from pathlib import Path	7	from pathlib import Path
9	from types import ModuleType	8	from types import ModuleType

Importance #29: tests/test_framework_run_provenance.py @@ -68,12 +67,10 @@

         (canonical_root / "canonical" / f"{corridor_id}.adapter.json").write_text(
             json.dumps({"tiling": {"grid_origin_xyz": GRID_ORIGIN}}),
             encoding="utf-8",
         )
-    return dataclasses.replace(
+    data = config.data.model_copy(update={"canonical_root": canonical_root})
-        config,
+    return config.model_copy(update={"data": data})
-        data=dataclasses.replace(config.data, canonical_root=canonical_root),
-    )
 def _records_and_manifest(config: HarnessConfig) -> tuple[Any, Any]:
     """Authorize the config's corridors for a training run."""

Importance #30: pyproject.toml @@ -3,17 +3,19 @@

 build-backend = "hatchling.build"
 [project]
 name = "iolabs-point-cloud-ml-segmentation"
-version = "0.1.0"
+version = "0.1.1"
 description = "Experiment harness for semantic segmentation of roadside mobile-laser-scanning corridors."
 readme = "README.md"
 requires-python = ">=3.11,<3.13"
 dependencies = [
     "dvc[azure]>=3.0",
+    "iolabs-common>=0.9.0",
     "iolabs-point-cloud-segmentation-3d>=0.1.0,<0.3.0",
     "laspy[lazrs]>=2.5,<3.0",
     "numpy>=2.0,<3.0",
+    "pydantic>=2.7",
     "pye57>=0.4.19,<0.5",
     "pyyaml>=6.0,<7.0",
     "scipy>=1.13,<2.0",
 ]

Importance #31: pyproject.toml @@ -39,8 +41,9 @@

39	url = "https://nexus.iolabs.ch/repository/pypi-private/simple/"	41	url = "https://nexus.iolabs.ch/repository/pypi-private/simple/"
40	authenticate = "always"	42	authenticate = "always"
41		43
42	[tool.uv.sources]	44	[tool.uv.sources]
		45	iolabs-common = { index = "nexus" }
43	iolabs-ml-harness = { index = "nexus" } # published 0.2.0 on 2026-08-13	46	iolabs-ml-harness = { index = "nexus" } # published 0.2.0 on 2026-08-13
44	iolabs-point-cloud-segmentation-3d = { index = "nexus" } # published 0.2.0 on 2026-08-13	47	iolabs-point-cloud-segmentation-3d = { index = "nexus" } # published 0.2.0 on 2026-08-13
45		48
46	[tool.pytest.ini_options]	49	[tool.pytest.ini_options]

Importance #32: CLAUDE.md @@ -18,8 +18,9 @@

 ## Data and experiments
 - Keep source and generated datasets under the DVC lanes in `data/`; never commit data products directly.
 - Every experiment is configured by one strict YAML file in `configs/`.
+- The configuration schema is a pydantic model tree on `iolabs.common.config_loader.ConfigModel`: `src/train/config_schema.py` (experiment, gates, study), `src/train/config_sections.py` (task, data, adapter, model, train, evaluation, runtime, provenance). Adding a key means adding a typed field to its model and to the YAML documents under `configs/`; nothing else. Cross-section and ontology-dependent rules live in `src/train/config_rules.py`, study expansion in `src/train/config_study.py`, and the strict YAML leaf typing in `src/train/config_values.py`. `src/train/config.py` stays the only import site (`load_config`, `expand_study`, `ConfigError`, the model classes).
 - Class counts come from the experiment's ontology (`task.ontology`): the model predicts train IDs `0..N-1` and ID `N` is void with no output channel (v1: N=9, roadside classes; v2: N=11, the ReCap annotation classes with junk merged into unclassified/void). Never hard-code a class count.
 - Boundary LAS codes belong to the code space the ontology, export map, and artifact all declare (`las_code_space`): `seg3d` is the legacy upstream space (v1), `asprs` is ASPRS LAS 1.4 and the ReCap lane (v2). Never read a LAS code without its space; resolve it through `registry_for(space)` in `src/contracts/las_codes.py`.
 - Never train on, evaluate on, or inspect `promotion_test` data without the command's explicit promotion authorization arguments.
 - E2 accepts only canonical data whose manifest proves full-resolution, in-memory `FuseResult.classification` provenance.

Importance #33: dvc.yaml @@ -37,8 +37,13 @@

       - scripts/ingest_preannotations.py
       - src/contracts/splits.py
       - src/dataset/preannotations.py
       - src/train/config.py
+      - src/train/config_rules.py
+      - src/train/config_schema.py
+      - src/train/config_sections.py
+      - src/train/config_study.py
+      - src/train/config_values.py
     outs:
       - data/01_interim/kirioll_v1/labels
   prepare_canonical:

Importance #34: dvc.yaml @@ -62,8 +67,13 @@

       - src/dataset/las.py
       - src/dataset/preannotations.py
       - src/dataset/tiling.py
       - src/train/config.py
+      - src/train/config_rules.py
+      - src/train/config_schema.py
+      - src/train/config_sections.py
+      - src/train/config_study.py
+      - src/train/config_values.py
     outs:
       # The canonical lane holds the tiles, per-corridor <id>.adapter.json
       # manifests, the pinned axis/grid_origin files, and manifest.json.
       # The normalization manifest is keyed by the config's feature stack, so

Importance #35: dvc.yaml @@ -94,8 +104,13 @@

       - src/dataset/canonical.py
       - src/dataset/features.py
       - src/dataset/tiling.py
       - src/train/config.py
+      - src/train/config_rules.py
+      - src/train/config_schema.py
+      - src/train/config_sections.py
+      - src/train/config_study.py
+      - src/train/config_values.py
     outs:
       # Tier 2: versioned by recipe. Flip to cache: true on dataset freeze.
       - data/02_processed/kirioll_v1/spt/raw:
           cache: false

Importance #36: dvc.yaml @@ -119,8 +134,13 @@

       - src/dataset/canonical.py
       - src/dataset/features.py
       - src/dataset/tiling.py
       - src/train/config.py
+      - src/train/config_rules.py
+      - src/train/config_schema.py
+      - src/train/config_sections.py
+      - src/train/config_study.py
+      - src/train/config_values.py
     outs:
       # Tier 2: versioned by recipe. Flip to cache: true on dataset freeze.
       - data/02_processed/kirioll_v1/pointcept:
           cache: false

Importance #37: dvc.yaml @@ -150,8 +170,13 @@

       - scripts/ingest_preannotations.py
       - src/contracts/splits.py
       - src/dataset/preannotations.py
       - src/train/config.py
+      - src/train/config_rules.py
+      - src/train/config_schema.py
+      - src/train/config_sections.py
+      - src/train/config_study.py
+      - src/train/config_values.py
     outs:
       - data/01_interim/a1_recap_v1/labels
   prepare_canonical_a1_recap:

Importance #38: dvc.yaml @@ -175,8 +200,13 @@

       - src/dataset/las.py
       - src/dataset/preannotations.py
       - src/dataset/tiling.py
       - src/train/config.py
+      - src/train/config_rules.py
+      - src/train/config_schema.py
+      - src/train/config_sections.py
+      - src/train/config_study.py
+      - src/train/config_values.py
     outs:
       - data/01_interim/a1_recap_v1/canonical
       - data/01_interim/a1_recap_v1/normalization_xyz_intensity_rgb.json

Importance #39: dvc.yaml @@ -199,8 +229,13 @@

       - src/dataset/canonical.py
       - src/dataset/features.py
       - src/dataset/tiling.py
       - src/train/config.py
+      - src/train/config_rules.py
+      - src/train/config_schema.py
+      - src/train/config_sections.py
+      - src/train/config_study.py
+      - src/train/config_values.py
     outs:
       # Tier 2: versioned by recipe. Flip to cache: true on dataset freeze.
       - data/02_processed/a1_recap_v1/pointcept:
           cache: false

Importance #40: frameworks/run_provenance.py @@ -17,9 +17,8 @@

17	import json	17	import json
18	import os	18	import os
19	import shlex	19	import shlex
20	from collections.abc import Mapping, Sequence	20	from collections.abc import Mapping, Sequence
21	from dataclasses import asdict
22	from pathlib import Path	21	from pathlib import Path
23	from typing import Any	22	from typing import Any
24		23
25	from src.contracts.manifest import (	24	from src.contracts.manifest import (

Importance #41: frameworks/run_provenance.py @@ -311,9 +310,9 @@

311	"schema": config.adapter.canonical_version,	310	"schema": config.adapter.canonical_version,
312	"feature_normalization": (	311	"feature_normalization": (
313	config.data.features.normalization_manifest.as_posix()	312	config.data.features.normalization_manifest.as_posix()
314	),	313	),
315	"tiling": _json_safe(asdict(config.data.tiling)),	314	"tiling": config.data.tiling.model_dump(mode="json"),
316	"voxel": {	315	"voxel": {
317	"spt": config.adapter.spt.voxel_m,	316	"spt": config.adapter.spt.voxel_m,
318	"pointcept": config.adapter.pointcept.grid_size_m,	317	"pointcept": config.adapter.pointcept.grid_size_m,
319	"grid_origin_xyz": pinned_grid_origin(	318	"grid_origin_xyz": pinned_grid_origin(

Importance #42: frameworks/run_provenance.py @@ -323,10 +322,10 @@

323	"point_coverage": "training run; see evaluation remap_report.json",	322	"point_coverage": "training run; see evaluation remap_report.json",
324	"remap_report": "reports/remap_report.json",	323	"remap_report": "reports/remap_report.json",
325	},	324	},
326	metric_protocol={	325	metric_protocol={
327	"bootstrap": _json_safe(asdict(config.evaluation.bootstrap)),	326	"bootstrap": config.evaluation.bootstrap.model_dump(mode="json"),
328	"promotion_margins": _json_safe(asdict(config.evaluation.promotion)),	327	"promotion_margins": config.evaluation.promotion.model_dump(mode="json"),
329	},	328	},
330	locked_test_access=False,	329	locked_test_access=False,
331	checkpoint_path=checkpoint.get("path"),	330	checkpoint_path=checkpoint.get("path"),
332	extra={	331	extra={

Importance #43: frameworks/run_provenance.py @@ -477,24 +476,6 @@

477	)	476	)
478	return {key: os.environ.get(key) for key in keys}	477	return {key: os.environ.get(key) for key in keys}
479		478
480		479
481	def _json_safe(value: Any) -> Any:
482	"""Convert a dataclass tree into JSON-safe primitives.
483
484	Args:
485	value: Mapping, sequence, path, or scalar.
486
487	Returns:
488	JSON-serializable value.
489	"""
490	if isinstance(value, Path):
491	return value.as_posix()
492	if isinstance(value, Mapping):
493	return {str(key): _json_safe(item) for key, item in value.items()}
494	if isinstance(value, (list, tuple)):
495	return [_json_safe(item) for item in value]
496	return value
497
498
499	if __name__ == "__main__": # pragma: no cover - runner entry point	480	if __name__ == "__main__": # pragma: no cover - runner entry point
500	raise SystemExit(main())	481	raise SystemExit(main())

Importance #44: pyproject.toml @@ -3,17 +3,19 @@

 build-backend = "hatchling.build"
 [project]
 name = "iolabs-point-cloud-ml-segmentation"
-version = "0.1.0"
+version = "0.1.1"
 description = "Experiment harness for semantic segmentation of roadside mobile-laser-scanning corridors."
 readme = "README.md"
 requires-python = ">=3.11,<3.13"
 dependencies = [
     "dvc[azure]>=3.0",
+    "iolabs-common>=0.9.0",
     "iolabs-point-cloud-segmentation-3d>=0.1.0,<0.3.0",
     "laspy[lazrs]>=2.5,<3.0",
     "numpy>=2.0,<3.0",
+    "pydantic>=2.7",
     "pye57>=0.4.19,<0.5",
     "pyyaml>=6.0,<7.0",
     "scipy>=1.13,<2.0",
 ]

Importance #45: pyproject.toml @@ -39,8 +41,9 @@

39	url = "https://nexus.iolabs.ch/repository/pypi-private/simple/"	41	url = "https://nexus.iolabs.ch/repository/pypi-private/simple/"
40	authenticate = "always"	42	authenticate = "always"
41		43
42	[tool.uv.sources]	44	[tool.uv.sources]
		45	iolabs-common = { index = "nexus" }
43	iolabs-ml-harness = { index = "nexus" } # published 0.2.0 on 2026-08-13	46	iolabs-ml-harness = { index = "nexus" } # published 0.2.0 on 2026-08-13
44	iolabs-point-cloud-segmentation-3d = { index = "nexus" } # published 0.2.0 on 2026-08-13	47	iolabs-point-cloud-segmentation-3d = { index = "nexus" } # published 0.2.0 on 2026-08-13
45		48
46	[tool.pytest.ini_options]	49	[tool.pytest.ini_options]

Importance #46: scripts/evaluate.py @@ -2191,9 +2191,9 @@

2191	"schema": config.adapter.canonical_version,	2191	"schema": config.adapter.canonical_version,
2192	"feature_normalization": (	2192	"feature_normalization": (
2193	config.data.features.normalization_manifest.as_posix()	2193	config.data.features.normalization_manifest.as_posix()
2194	),	2194	),
2195	"tiling": _json_safe(asdict(config.data.tiling)),	2195	"tiling": config.data.tiling.model_dump(mode="json"),
2196	"voxel": {	2196	"voxel": {
2197	"spt": config.adapter.spt.voxel_m,	2197	"spt": config.adapter.spt.voxel_m,
2198	"pointcept": config.adapter.pointcept.grid_size_m,	2198	"pointcept": config.adapter.pointcept.grid_size_m,
2199	"grid_origin_xyz": _pinned_grid_origin(config, records),	2199	"grid_origin_xyz": _pinned_grid_origin(config, records),

Importance #47: scripts/evaluate.py @@ -2201,12 +2201,10 @@

             "point_coverage": "see reports/remap_report.json",
             "remap_report": "reports/remap_report.json",
         },
         metric_protocol={
-            "bootstrap": _json_safe(asdict(config.evaluation.bootstrap)),
+            "bootstrap": config.evaluation.bootstrap.model_dump(mode="json"),
-            "promotion_margins": _json_safe(
+            "promotion_margins": config.evaluation.promotion.model_dump(mode="json"),
-                asdict(config.evaluation.promotion)
-            ),
         },
         locked_test_access=locked_test_access,
         checkpoint_path=selection["best_model_path"],
         extra={"status": status, "metrics": metrics, "command": sys.argv},

Importance #48: src/train/config.py @@ -1,475 +1,159 @@

1	"""Strict experiment configuration schema for corridor segmentation studies."""	1	"""Strict experiment configuration schema for corridor segmentation studies.
		2
		3	The schema itself is a pydantic model tree built on
		4	:class:`iolabs.common.config_loader.ConfigModel`: :mod:`src.train.config_schema`
		5	holds the experiment/gate/study models, :mod:`src.train.config_sections` the
		6	data, model, training, and evaluation blocks. This module is the entry point
		7	every script imports: it loads one YAML document, validates it, applies the
		8	rules of :mod:`src.train.config_rules`, and expands a study into re-validated
		9	cells with :mod:`src.train.config_study`.
		10
		11	Adding a configuration key means adding a field to its model (and to the YAML
		12	documents under ``configs/``); nothing else has to be touched.
		13	"""
2		14
3	from __future__ import annotations	15	from __future__ import annotations
4		16
5	import copy	17	import copy
6	import hashlib	18	import hashlib
7	import itertools	19	import logging
8	import math	20	from collections.abc import Mapping
9	import random	21	from dataclasses import dataclass
10	import re
11	from collections.abc import Mapping, Sequence
12	from dataclasses import MISSING, asdict, dataclass, field, fields, is_dataclass
13	from pathlib import Path	22	from pathlib import Path
14	from types import MappingProxyType, UnionType	23	from types import MappingProxyType
15	from typing import (	24	from typing import Any
16	Any,
17	Literal,
18	TypeAlias,
19	Union,
20	get_args,
21	get_origin,
22	get_type_hints,
23	)
24		25
25	import yaml	26	import yaml
		27	from iolabs.common import config_loader
26		28
27	from src.contracts.ontology import (	29	from src.train import (
28	Ontology,	30	config_rules,
29	OntologyError,	31	config_sections,
30	load_ontology,	32	config_study,
31	macro_interest_all_suffix,	33	config_values,
32	)	34	)
33		35	from src.train.config_rules import is_canonical_metric, resolve_ontology_path
34	try:	36	from src.train.config_schema import (
35	from iolabs_ml_harness.config import TrainerConfig as _TrainerConfig	37	ArtifactExistsGate,
36	except ModuleNotFoundError:	38	ArtifactExistsParams,
37		39	CheckpointPolicyGate,
38	@dataclass	40	CheckpointPolicyParams,
39	class _TrainerConfig: # type: ignore[no-redef]	41	ContrastConfig,
40	"""Core-compatible fallback used when the optional ML extra is absent."""	42	DataAvailableGate,
41		43	DataAvailableParams,
42	max_epochs: int = -1	44	ExperimentConfig,
43	lr: float = 3.0e-4	45	ExperimentStatus,
44	weight_decay: float = 1.0e-4	46	GateBase,
45	precision: str = "auto"	47	GateConfig,
46	accumulate_grad_batches: int = 1	48	GateParams,
47	accelerator: str = "auto"	49	HumanWorkflowGate,
48	devices: int \| str = 1	50	HumanWorkflowParams,
49	viz_every_n_epochs: int = 2	51	ImplementationTicketGate,
50	viz_samples: int = 4	52	ImplementationTicketParams,
51	monitor: str = "val/f1_mean_fg"	53	LicenseApprovalGate,
52	monitor_mode: str = "max"	54	LicenseApprovalParams,
53	early_stop_monitor: str = "val/loss"	55	MatrixConfig,
54	early_stop_mode: str = "min"	56	OperationalSmokeGate,
55	early_stop_patience: int = 4	57	OperationalSmokeParams,
56	log_dir: str = "runs"	58	SptPartitionOracleGate,
57	log_every_n_steps: int = 10	59	SptPartitionOracleParams,
58		60	StrictConfigModel,
59		61	StudyConfig,
60	Scalar: TypeAlias = str \| int \| float \| bool \| None	62	StudyKind,
61	OverrideValue: TypeAlias = Scalar \| list[Scalar]	63	SweepConfig,
62	ExperimentStatus: TypeAlias = Literal["implement-now", "template-only", "gated-later"]	64	SweepParameterConfig,
63	StudyKind: TypeAlias = Literal["single", "variants", "matrix", "sweep"]	65	VariantConfig,
64
65
66	class ConfigError(ValueError):
67	"""Raised when an experiment configuration violates its strict schema."""
68
69
70	@dataclass(frozen=True)
71	class VariantConfig:
72	"""One named study variant expressed as typed dotted-path overrides."""
73
74	id: str
75	overrides: Mapping[str, OverrideValue]
76	tags: tuple[str, ...] = ()
77
78
79	@dataclass(frozen=True)
80	class MatrixConfig:
81	"""Typed Cartesian matrix definition with optional cells."""
82
83	axes: Mapping[str, tuple[OverrideValue, ...]]
84	include: tuple[Mapping[str, OverrideValue], ...] = ()
85	exclude: tuple[Mapping[str, OverrideValue], ...] = ()
86
87
88	@dataclass(frozen=True)
89	class SweepParameterConfig:
90	"""One finite or bounded sweep parameter."""
91
92	values: tuple[OverrideValue, ...] \| None = None
93	minimum: float \| None = None
94	maximum: float \| None = None
95	distribution: str \| None = None
96
97
98	@dataclass(frozen=True)
99	class SweepConfig:
100	"""Typed bounded sweep contract."""
101
102	method: str
103	parameters: Mapping[str, SweepParameterConfig]
104	budget: int
105	objective: str
106
107
108	@dataclass(frozen=True)
109	class ContrastConfig:
110	"""Predeclared comparison between two study cells."""
111
112	name: str
113	left: str
114	right: str
115	metric: str
116
117
118	@dataclass(frozen=True)
119	class StudyConfig:
120	"""Discriminated single, variants, matrix, or sweep study definition."""
121
122	kind: StudyKind
123	variants: tuple[VariantConfig, ...]
124	matrix: MatrixConfig \| None
125	sweep: SweepConfig \| None
126	contrasts: tuple[ContrastConfig, ...]
127
128
129	@dataclass(frozen=True)
130	class SptPartitionOracleParams:
131	"""Required SPT partition-purity report and per-class thresholds."""
132
133	report: Path
134	minimum_purity_by_class: Mapping[str, float]
135
136
137	@dataclass(frozen=True)
138	class ImplementationTicketParams:
139	"""Implementation ticket whose external state must reach a required value."""
140
141	ticket: str
142	required_status: str
143
144
145	@dataclass(frozen=True)
146	class ArtifactExistsParams:
147	"""Required local artifact and optional expected JSON status."""
148
149	path: Path
150	expected_status: str \| None = None
151
152
153	@dataclass(frozen=True)
154	class DataAvailableParams:
155	"""Required data manifest and declared dataset contract."""
156
157	manifest: Path
158	dataset: str
159
160
161	@dataclass(frozen=True)
162	class LicenseApprovalParams:
163	"""Required license-review decision artifact."""
164
165	decision: Path
166	required_decision: str
167
168
169	@dataclass(frozen=True)
170	class OperationalSmokeParams:
171	"""Required operational-smoke report and accepted result."""
172
173	report: Path
174	required_status: str
175
176
177	@dataclass(frozen=True)
178	class CheckpointPolicyParams:
179	"""Required checkpoint-policy file and checkpoint metadata artifact."""
180
181	policy: Path
182	metadata: Path
183
184
185	@dataclass(frozen=True)
186	class HumanWorkflowParams:
187	"""Required human-workflow protocol and acceptance artifact."""
188
189	protocol: Path
190	acceptance: Path
191
192
193	GateParams: TypeAlias = (
194	SptPartitionOracleParams
195	\| ImplementationTicketParams
196	\| ArtifactExistsParams
197	\| DataAvailableParams
198	\| LicenseApprovalParams
199	\| OperationalSmokeParams
200	\| CheckpointPolicyParams
201	\| HumanWorkflowParams
202	)	66	)
203		67	from src.train.config_sections import (
204		68	AdapterConfig,
205	@dataclass(frozen=True)	69	BootstrapConfig,
206	class GateConfig:	70	ContinuityConfig,
207	"""One typed, fail-closed experiment gate."""	71	CorridorSelectionConfig,
208		72	DataConfig,
209	name: str	73	EvaluationConfig,
210	type: str	74	FeatureConfig,
211	required: bool	75	HarnessConfig,
212	params: GateParams	76	LabelSourceConfig,
213		77	LossConfig,
214		78	ModelConfig,
215	@dataclass(frozen=True)	79	ObjectMatchingConfig,
216	class ExperimentConfig:	80	PointceptAdapterConfig,
217	"""Experiment identity, readiness, hypothesis, metrics, and gates."""	81	PromotionConfig,
218		82	ProvenanceConfig,
219	id: str	83	RuntimeConfig,
220	name: str	84	SptAdapterConfig,
221	phase: int	85	TaskConfig,
222	status: ExperimentStatus	86	TilingConfig,
223	hypothesis: str	87	TrainConfig,
224	deciding_metrics: tuple[str, ...]	88	VisualizationConfig,
225	gates: tuple[GateConfig, ...]	89	)
226		90	from src.train.config_values import ConfigError, OverrideValue, Scalar
227		91
228	@dataclass(frozen=True)	92	logger = logging.getLogger(__name__)
229	class TaskConfig:	93
230	"""Frozen semantic task contract."""	94	_CONTEXT = "experiment config"
231		95
232	ontology: Path	96	__all__ = [
233	num_classes: int	97	"AdapterConfig",
234	ignore_index: int	98	"ArtifactExistsGate",
235	classes_of_interest: tuple[int, ...]	99	"ArtifactExistsParams",
236	linear_classes: tuple[str, ...]	100	"BootstrapConfig",
237		101	"CheckpointPolicyGate",
238		102	"CheckpointPolicyParams",
239	@dataclass(frozen=True)	103	"ConfigError",
240	class LabelSourceConfig:	104	"ContinuityConfig",
241	"""Label provenance and fail-closed join policy."""	105	"ContrastConfig",
242		106	"CorridorSelectionConfig",
243	mode: Literal["artifact", "regenerate_full_resolution"]	107	"DataAvailableGate",
244	source_geometry_glob: str	108	"DataAvailableParams",
245	fuse_config: Path \| None	109	"DataConfig",
246	prefer: str	110	"EvaluationConfig",
247	classical_glob: str \| None	111	"ExperimentConfig",
248	recap_glob: str \| None	112	"ExperimentStatus",
249	stats_glob: str	113	"FeatureConfig",
250	unmatched_policy: Literal["void"]	114	"GateBase",
251	max_unmatched_fraction: float	115	"GateConfig",
252		116	"GateParams",
253		117	"HarnessConfig",
254	@dataclass(frozen=True)	118	"HumanWorkflowGate",
255	class CorridorSelectionConfig:	119	"HumanWorkflowParams",
256	"""Config-declared corridor allow-list."""	120	"ImplementationTicketGate",
257		121	"ImplementationTicketParams",
258	include: tuple[str, ...]	122	"LabelSourceConfig",
259		123	"LicenseApprovalGate",
260		124	"LicenseApprovalParams",
261	@dataclass(frozen=True)	125	"LossConfig",
262	class FeatureConfig:	126	"MatrixConfig",
263	"""Ordered features and train-only normalization contract."""	127	"ModelConfig",
264		128	"ObjectMatchingConfig",
265	names: tuple[str, ...]	129	"OperationalSmokeGate",
266	normalization_manifest: Path	130	"OperationalSmokeParams",
267	fit_on: Literal["train_corridors_only"]	131	"OverrideValue",
268	scanner_conditioning: bool	132	"PointceptAdapterConfig",
269		133	"PromotionConfig",
270		134	"ProvenanceConfig",
271	@dataclass(frozen=True)	135	"RuntimeConfig",
272	class TilingConfig:	136	"Scalar",
273	"""Deterministic corridor tiling and overlap blending contract."""	137	"SptAdapterConfig",
274		138	"SptPartitionOracleGate",
275	mode: Literal["corridor_axis"]	139	"SptPartitionOracleParams",
276	length_m: float	140	"StrictConfigModel",
277	overlap_m: float	141	"StudyCell",
278	origin: Literal["dataset_manifest"]	142	"StudyConfig",
279	min_points: int	143	"StudyKind",
280	blend: Literal["linear_edge_weight"]	144	"SweepConfig",
281		145	"SweepParameterConfig",
282		146	"TaskConfig",
283	@dataclass(frozen=True)	147	"TilingConfig",
284	class DataConfig:	148	"TrainConfig",
285	"""Input roots, splits, labels, features, and tiling."""	149	"VariantConfig",
286		150	"VisualizationConfig",
287	root: Path	151	"expand_study",
288	canonical_root: Path	152	"is_canonical_metric",
289	processed_root: Path	153	"load_config",
290	split_manifest: Path	154	"resolve_ontology_path",
291	label_source: LabelSourceConfig	155	]
292	corridors: CorridorSelectionConfig
293	features: FeatureConfig
294	tiling: TilingConfig
295
296
297	@dataclass(frozen=True)
298	class SptAdapterConfig:
299	"""SPT raw-dataset emission contract."""
300
301	raw_root: Path
302	pc_tiling: int
303	voxel_m: float
304	base_family: str
305	raw_row_sidecar_keys: tuple[str, ...]
306	audit_only_data_keys: tuple[str, ...]
307
308
309	@dataclass(frozen=True)
310	class PointceptAdapterConfig:
311	"""Pointcept default-dataset emission contract."""
312
313	root: Path
314	grid_size_m: float
315	preserve_keys: tuple[str, ...]
316
317
318	@dataclass(frozen=True)
319	class AdapterConfig:
320	"""Framework-neutral and external-format adapter contract."""
321
322	emit: tuple[str, ...]
323	canonical_version: int
324	identity: Literal["source_file_and_row"]
325	spt: SptAdapterConfig
326	pointcept: PointceptAdapterConfig
327
328
329	@dataclass(frozen=True)
330	class ModelConfig:
331	"""Local or external model/runner selection."""
332
333	framework: Literal["cpu", "spt", "pointcept"]
334	runner: Path
335	name: str
336	base_config: Path \| None
337	checkout_env: str \| None
338	commit_env: str \| None
339	checkpoint: Path \| None
340	args: Mapping[str, Any]
341
342
343	@dataclass(frozen=True)
344	class LossConfig:
345	"""Shared-registry or external-native loss selection."""
346
347	name: str
348	args: Mapping[str, Any]
349
350
351	@dataclass
352	class TrainConfig(_TrainerConfig):
353	"""Harness-visible training settings for local and external runners."""
354
355	batch_size: int = 1
356	num_workers: int = 4
357	optimizer: str = "adamw"
358	scheduler: str = "cosine"
359	distributed: bool = False
360
361
362	@dataclass(frozen=True)
363	class ContinuityConfig:
364	"""Linear-continuity binning contract."""
365
366	chainage_bin_m: float
367	gap_threshold_m: float
368
369
370	@dataclass(frozen=True)
371	class ObjectMatchingConfig:
372	"""Object-clustering profile source and fallback tolerances."""
373
374	cluster_profiles: Path
375	minimum_iou: float
376	centroid_tolerance_m: float
377
378
379	@dataclass(frozen=True)
380	class BootstrapConfig:
381	"""Corridor/spatial bootstrap contract."""
382
383	unit: Literal["corridor"]
384	spatial_block_m: float
385	samples: int
386	confidence: float
387	seed: int
388
389
390	@dataclass(frozen=True)
391	class PromotionConfig:
392	"""Locked-test promotion margins and superiority conditions."""
393
394	enabled: bool
395	delta_quality: float \| None
396	delta_fp_per_km: float \| None
397	superiority_conditions: tuple[str, ...]
398
399
400	@dataclass(frozen=True)
401	class EvaluationConfig:
402	"""Held-out evaluation and promotion protocol."""
403
404	split: Literal["validation", "promotion_test"]
405	metrics: tuple[str, ...]
406	precision_floors: Mapping[str, float]
407	continuity: ContinuityConfig
408	object_matching: ObjectMatchingConfig
409	bootstrap: BootstrapConfig
410	promotion: PromotionConfig
411	worst_k_tiles: int = 4
412
413
414	@dataclass(frozen=True)
415	class RuntimeConfig:
416	"""Operational hardware and kernel constraints."""
417
418	target: str
419	cuda: str
420	spconv: str
421	flash_attention: bool
422	system_ram_gb: int
423	gpu_memory_gb: int
424
425
426	@dataclass(frozen=True)
427	class ProvenanceConfig:
428	"""Mandatory run-evidence policy."""
429
430	manifest: Literal["required"]
431	data_hash_source: Literal["dvc"]
432	record_environment: bool
433	record_commands: bool
434	checkpoint_policy: Path
435
436
437	@dataclass(frozen=True)
438	class VisualizationConfig:
439	"""Opt-in training-time TensorBoard class-mask visualization."""
440
441	masks_every_n_epochs: int
442	masks_tiles: int \| tuple[str, ...] = 2
443
444
445	@dataclass(frozen=True)
446	class HarnessConfig:
447	"""Fully parsed, cross-field-validated experiment configuration."""
448
449	schema_version: int
450	experiment: ExperimentConfig
451	study: StudyConfig
452	seed: int
453	task: TaskConfig
454	data: DataConfig
455	adapter: AdapterConfig
456	model: ModelConfig
457	loss: LossConfig
458	train: TrainConfig
459	evaluation: EvaluationConfig
460	runtime: RuntimeConfig
461	provenance: ProvenanceConfig
462	source_path: Path = field(compare=False)
463	sha256: str = field(compare=False)
464	visualization: VisualizationConfig \| None = None
465
466	def as_dict(self) -> dict[str, Any]:
467	"""Return the resolved dataclass tree as JSON-safe primitives."""
468	payload = _json_safe(self)
469	if not isinstance(payload, dict): # pragma: no cover - defensive
470	raise ConfigError("Resolved configuration is not a mapping")
471	return payload
472		156
473		157
474	@dataclass(frozen=True)	158	@dataclass(frozen=True)
475	class StudyCell:	159	class StudyCell:

Importance #49: src/train/config.py @@ -491,41 +175,8 @@

491	document: str	175	document: str
492	config: HarnessConfig	176	config: HarnessConfig
493		177
494		178
495	_ROOT_KEYS = {
496	"schema_version",
497	"experiment",
498	"study",
499	"seed",
500	"task",
501	"data",
502	"adapter",
503	"model",
504	"loss",
505	"train",
506	"evaluation",
507	"runtime",
508	"provenance",
509	"visualization",
510	}
511	_REQUIRED_ROOT_KEYS = _ROOT_KEYS - {"visualization"}
512	_RAW_ATTRIBUTE = "_raw_document_mapping"
513	_FIELD_TYPE_CACHE: dict[type[Any], Mapping[str, Any]] = {}
514	_GATE_PARAM_CLASSES: Mapping[str, type[GateParams]] = MappingProxyType(
515	{
516	"spt_partition_oracle": SptPartitionOracleParams,
517	"implementation_ticket": ImplementationTicketParams,
518	"artifact_exists": ArtifactExistsParams,
519	"data_available": DataAvailableParams,
520	"license_approval": LicenseApprovalParams,
521	"operational_smoke": OperationalSmokeParams,
522	"checkpoint_policy": CheckpointPolicyParams,
523	"human_workflow": HumanWorkflowParams,
524	}
525	)
526
527
528	def load_config(path: str \| Path) -> HarnessConfig:	179	def load_config(path: str \| Path) -> HarnessConfig:
529	"""Load and strictly validate one E1--E14 experiment YAML.	180	"""Load and strictly validate one E1--E14 experiment YAML.
530		181
531	Args:	182	Args:

Importance #50: src/train/config.py @@ -569,9 +220,9 @@

569	ConfigError: If the study payload, an override, a cell identity, or a	220	ConfigError: If the study payload, an override, a cell identity, or a
570	resolved cell configuration violates the strict schema.	221	resolved cell configuration violates the strict schema.
571	"""	222	"""
572	raw = _raw_document(config)	223	raw = _raw_document(config)
573	definitions = _cell_definitions(config)	224	definitions = config_study.cell_definitions(config)
574	if not definitions:	225	if not definitions:
575	raise ConfigError(	226	raise ConfigError(
576	f"{config.experiment.id} study kind {config.study.kind} expanded to "	227	f"{config.experiment.id} study kind {config.study.kind} expanded to "
577	"no cells"	228	"no cells"

Importance #51: src/train/config.py @@ -584,9 +235,9 @@

584	)	235	)
585	cells: list[StudyCell] = []	236	cells: list[StudyCell] = []
586	for index, (cell_id, overrides) in enumerate(definitions):	237	for index, (cell_id, overrides) in enumerate(definitions):
587	where = f"{config.experiment.id} study cell {cell_id!r}"	238	where = f"{config.experiment.id} study cell {cell_id!r}"
588	cell_raw = _apply_overrides(raw, overrides, where)	239	cell_raw = config_study.apply_overrides(raw, overrides, where)
589	document = yaml.safe_dump(cell_raw, sort_keys=True, default_flow_style=False)	240	document = yaml.safe_dump(cell_raw, sort_keys=True, default_flow_style=False)
590	digest = hashlib.sha256(document.encode("utf-8")).hexdigest()	241	digest = hashlib.sha256(document.encode("utf-8")).hexdigest()
591	try:	242	try:
592	cell_config = _build_config(cell_raw, config.source_path, digest)	243	cell_config = _build_config(cell_raw, config.source_path, digest)

Importance #52: src/train/config.py @@ -610,34 +261,20 @@

     return tuple(cells)
 def _build_config(raw: Any, config_path: Path, sha256: str) -> HarnessConfig:
-    root = _mapping(raw, str(config_path))
+    """Validate one raw document into a configuration carrying its digest."""
-    _keys(root, _REQUIRED_ROOT_KEYS, _ROOT_KEYS, str(config_path))
+    root = config_values.mapping(raw, str(config_path))
-    config = HarnessConfig(
+    payload = {
-        schema_version=_integer(root["schema_version"], "schema_version"),
+        **root,
-        experiment=_parse_experiment(root["experiment"]),
+        config_sections.SOURCE_PATH_ALIAS: config_path,
-        study=_parse_study(root["study"]),
+        config_sections.SHA256_ALIAS: sha256,
-        seed=_integer(root["seed"], "seed"),
+    }
-        task=_parse_task(root["task"]),
+    config = config_loader.validate_config(
-        data=_parse_data(root["data"]),
+        HarnessConfig, payload, context=_CONTEXT, error_cls=ConfigError
-        adapter=_parse_adapter(root["adapter"]),
-        model=_parse_model(root["model"]),
-        loss=_parse_loss(root["loss"]),
-        train=_parse_train(root["train"], _mapping(root["model"], "model")),
-        evaluation=_parse_evaluation(root["evaluation"]),
-        runtime=_parse_runtime(root["runtime"]),
-        provenance=_parse_provenance(root["provenance"]),
-        visualization=(
-            None
-            if "visualization" not in root
-            else _parse_visualization(root["visualization"])
-        ),
-        source_path=config_path,
-        sha256=sha256,
     )
-    _validate_config(config, root)
+    config_rules.validate_config(config, root)
-    object.__setattr__(config, _RAW_ATTRIBUTE, copy.deepcopy(dict(root)))
+    config._raw_document = copy.deepcopy(dict(root))
     return config
 def _raw_document(config: HarnessConfig) -> Mapping[str, Any]:

Importance #53: src/train/config.py @@ -652,9 +289,9 @@

652		289
653	Raises:	290	Raises:
654	ConfigError: If the source file must be re-read and cannot be parsed.	291	ConfigError: If the source file must be re-read and cannot be parsed.
655	"""	292	"""
656	stashed = getattr(config, _RAW_ATTRIBUTE, None)	293	stashed = config._raw_document
657	if stashed is not None:	294	if stashed is not None:
658	return stashed	295	return stashed
659	try:	296	try:
660	payload = config.source_path.read_bytes()	297	payload = config.source_path.read_bytes()

Importance #54: src/train/config.py @@ -662,1437 +299,5 @@

662	except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc:	299	except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc:
663	raise ConfigError(	300	raise ConfigError(
664	f"Cannot re-read config {config.source_path}: {exc}"	301	f"Cannot re-read config {config.source_path}: {exc}"
665	) from exc	302	) from exc
666	return _mapping(raw, str(config.source_path))	303	return config_values.mapping(raw, str(config.source_path))
667
668
669	def is_canonical_metric(name: str, *, ontology: Ontology) -> bool:
670	"""Return whether a metric belongs to an ontology's val/eval namespace.
671
672	Args:
673	name: Metric tag to validate.
674	ontology: Ontology whose predicted class names and interest count fix
675	the per-class tags and the structural macro suffix.
676
677	Returns:
678	True for a canonical tag, otherwise False.
679	"""
680	if name == "val/loss":
681	return True
682	match = re.fullmatch(r"(val\|eval)/(.+)", name)
683	if match is None:
684	return False
685	metric = match.group(2)
686	suffix = macro_interest_all_suffix(ontology)
687	fixed = {
688	"iou_macro_interest",
689	"f1_macro_interest",
690	"precision_macro_interest",
691	"recall_macro_interest",
692	f"iou_macro_interest_{suffix}",
693	f"f1_macro_interest_{suffix}",
694	f"precision_macro_interest_{suffix}",
695	f"recall_macro_interest_{suffix}",
696	"miou_all_classes",
697	}
698	if metric in fixed:
699	return True
700	class_name = "\|".join(re.escape(item) for item in ontology.class_names)
701	patterns = (
702	rf"(?:iou\|precision\|recall\|f1\|support)_(?:{class_name})",
703	rf"(?:fp_per_km\|detections_per_km\|matched_recall)_(?:{class_name})",
704	rf"continuity_(?:covered_fraction\|total_missing_length_m\|gaps\|"
705	rf"gap_median_m\|gap_p95_m\|gap_max_m)_(?:{class_name})",
706	rf"seam_[a-zA-Z0-9_.-]+_(?:{class_name})",
707	)
708	return any(re.fullmatch(pattern, metric) is not None for pattern in patterns)
709
710
711	def _parse_experiment(value: Any) -> ExperimentConfig:
712	raw = _section(
713	value,
714	{"id", "name", "phase", "status", "hypothesis", "deciding_metrics", "gates"},
715	"experiment",
716	)
717	gates = tuple(
718	_parse_gate(item, index)
719	for index, item in enumerate(
720	_sequence(raw["gates"], "experiment.gates")
721	)
722	)
723	return ExperimentConfig(
724	id=_string(raw["id"], "experiment.id"),
725	name=_string(raw["name"], "experiment.name"),
726	phase=_integer(raw["phase"], "experiment.phase"),
727	status=_choice(
728	raw["status"],
729	{"implement-now", "template-only", "gated-later"},
730	"experiment.status",
731	),
732	hypothesis=_string(raw["hypothesis"], "experiment.hypothesis"),
733	deciding_metrics=_strings(
734	raw["deciding_metrics"], "experiment.deciding_metrics"
735	),
736	gates=gates,
737	)
738
739
740	def _parse_gate(value: Any, index: int) -> GateConfig:
741	where = f"experiment.gates[{index}]"
742	raw = _section(value, {"name", "type", "required", "params"}, where)
743	gate_type = _string(raw["type"], f"{where}.type")
744	try:
745	params_class = _GATE_PARAM_CLASSES[gate_type]
746	except KeyError as exc:
747	raise ConfigError(
748	f"{where}.type has unsupported gate type {gate_type!r}"
749	) from exc
750	params = _dataclass_section(params_class, raw["params"], f"{where}.params")
751	return GateConfig(
752	name=_string(raw["name"], f"{where}.name"),
753	type=gate_type,
754	required=_boolean(raw["required"], f"{where}.required"),
755	params=params,
756	)
757
758
759	def _parse_study(value: Any) -> StudyConfig:
760	raw = _section(
761	value,
762	{"kind", "variants", "matrix", "sweep", "contrasts"},
763	"study",
764	)
765	kind = _choice(
766	raw["kind"], {"single", "variants", "matrix", "sweep"}, "study.kind"
767	)
768	variants = tuple(
769	_parse_variant(item, index)
770	for index, item in enumerate(
771	_sequence(raw["variants"], "study.variants")
772	)
773	)
774	matrix = None if raw["matrix"] is None else _parse_matrix(raw["matrix"])
775	sweep = None if raw["sweep"] is None else _parse_sweep(raw["sweep"])
776	contrasts = tuple(
777	_dataclass_section(
778	ContrastConfig, item, f"study.contrasts[{index}]"
779	)
780	for index, item in enumerate(
781	_sequence(raw["contrasts"], "study.contrasts")
782	)
783	)
784	if kind == "single" and (variants or matrix is not None or sweep is not None):
785	raise ConfigError("study.kind single cannot carry variants, matrix, or sweep")
786	if kind == "variants" and (
787	not variants or matrix is not None or sweep is not None
788	):
789	raise ConfigError(
790	"study.kind variants requires only a non-empty variants payload"
791	)
792	if kind == "matrix" and (variants or matrix is None or sweep is not None):
793	raise ConfigError("study.kind matrix requires only matrix")
794	if kind == "sweep" and (variants or matrix is not None or sweep is None):
795	raise ConfigError("study.kind sweep requires only sweep")
796	variant_ids = [item.id for item in variants]
797	if len(set(variant_ids)) != len(variant_ids):
798	raise ConfigError("study.variants contains duplicate IDs")
799	return StudyConfig(kind, variants, matrix, sweep, contrasts)
800
801
802	def _parse_variant(value: Any, index: int) -> VariantConfig:
803	where = f"study.variants[{index}]"
804	raw = _mapping(value, where)
805	allowed = {"id", "overrides", "tags"}
806	required = {"id", "overrides"}
807	_keys(raw, required, allowed, where)
808	return VariantConfig(
809	id=_string(raw["id"], f"{where}.id"),
810	overrides=_overrides(raw["overrides"], f"{where}.overrides"),
811	tags=_strings(raw.get("tags", []), f"{where}.tags"),
812	)
813
814
815	def _parse_matrix(value: Any) -> MatrixConfig:
816	raw = _mapping(value, "study.matrix")
817	_keys(raw, {"axes"}, {"axes", "include", "exclude"}, "study.matrix")
818	axes_raw = _mapping(raw["axes"], "study.matrix.axes")
819	axes: dict[str, tuple[OverrideValue, ...]] = {}
820	for path, values in axes_raw.items():
821	sequence = tuple(
822	_override_value(item, f"study.matrix.axes.{path}")
823	for item in _sequence(values, f"study.matrix.axes.{path}")
824	)
825	if not sequence:
826	raise ConfigError(f"study.matrix.axes.{path} cannot be empty")
827	axes[_string(path, "study.matrix axis")] = sequence
828	return MatrixConfig(
829	axes=MappingProxyType(axes),
830	include=tuple(
831	_overrides(item, f"study.matrix.include[{index}]")
832	for index, item in enumerate(
833	_sequence(raw.get("include", []), "study.matrix.include")
834	)
835	),
836	exclude=tuple(
837	_overrides(item, f"study.matrix.exclude[{index}]")
838	for index, item in enumerate(
839	_sequence(raw.get("exclude", []), "study.matrix.exclude")
840	)
841	),
842	)
843
844
845	def _parse_sweep(value: Any) -> SweepConfig:
846	raw = _section(
847	value,
848	{"method", "parameters", "budget", "objective"},
849	"study.sweep",
850	)
851	parameters_raw = _mapping(raw["parameters"], "study.sweep.parameters")
852	parameters = {
853	path: _parse_sweep_parameter(
854	item, f"study.sweep.parameters.{path}"
855	)
856	for path, item in parameters_raw.items()
857	}
858	if not parameters:
859	raise ConfigError("study.sweep.parameters cannot be empty")
860	return SweepConfig(
861	method=_string(raw["method"], "study.sweep.method"),
862	parameters=MappingProxyType(parameters),
863	budget=_integer(raw["budget"], "study.sweep.budget"),
864	objective=_string(raw["objective"], "study.sweep.objective"),
865	)
866
867
868	def _parse_sweep_parameter(value: Any, where: str) -> SweepParameterConfig:
869	raw = _mapping(value, where)
870	_keys(raw, set(), {"values", "minimum", "maximum", "distribution"}, where)
871	values = None
872	if "values" in raw:
873	values = tuple(
874	_override_value(item, f"{where}.values")
875	for item in _sequence(raw["values"], f"{where}.values")
876	)
877	if not values:
878	raise ConfigError(f"{where}.values cannot be empty")
879	minimum = _optional_float(raw.get("minimum"), f"{where}.minimum")
880	maximum = _optional_float(raw.get("maximum"), f"{where}.maximum")
881	distribution = (
882	None
883	if raw.get("distribution") is None
884	else _string(raw["distribution"], f"{where}.distribution")
885	)
886	if values is None and (
887	minimum is None or maximum is None or distribution is None
888	):
889	raise ConfigError(f"{where} requires values or minimum/maximum/distribution")
890	if minimum is not None and maximum is not None and minimum >= maximum:
891	raise ConfigError(f"{where}.minimum must be smaller than maximum")
892	return SweepParameterConfig(values, minimum, maximum, distribution)
893
894
895	def _parse_task(value: Any) -> TaskConfig:
896	raw = _section(
897	value,
898	{
899	"ontology",
900	"num_classes",
901	"ignore_index",
902	"classes_of_interest",
903	"linear_classes",
904	},
905	"task",
906	)
907	return TaskConfig(
908	_path(raw["ontology"], "task.ontology"),
909	_integer(raw["num_classes"], "task.num_classes"),
910	_integer(raw["ignore_index"], "task.ignore_index"),
911	_integers(raw["classes_of_interest"], "task.classes_of_interest"),
912	_strings(raw["linear_classes"], "task.linear_classes"),
913	)
914
915
916	def _parse_data(value: Any) -> DataConfig:
917	raw = _section(
918	value,
919	{
920	"root",
921	"canonical_root",
922	"processed_root",
923	"split_manifest",
924	"label_source",
925	"corridors",
926	"features",
927	"tiling",
928	},
929	"data",
930	)
931	return DataConfig(
932	root=_path(raw["root"], "data.root"),
933	canonical_root=_path(raw["canonical_root"], "data.canonical_root"),
934	processed_root=_path(raw["processed_root"], "data.processed_root"),
935	split_manifest=_path(raw["split_manifest"], "data.split_manifest"),
936	label_source=_dataclass_section(
937	LabelSourceConfig, raw["label_source"], "data.label_source"
938	),
939	corridors=_dataclass_section(
940	CorridorSelectionConfig, raw["corridors"], "data.corridors"
941	),
942	features=_dataclass_section(
943	FeatureConfig, raw["features"], "data.features"
944	),
945	tiling=_dataclass_section(TilingConfig, raw["tiling"], "data.tiling"),
946	)
947
948
949	def _parse_adapter(value: Any) -> AdapterConfig:
950	raw = _section(
951	value,
952	{"emit", "canonical_version", "identity", "spt", "pointcept"},
953	"adapter",
954	)
955	return AdapterConfig(
956	emit=_strings(raw["emit"], "adapter.emit"),
957	canonical_version=_integer(
958	raw["canonical_version"], "adapter.canonical_version"
959	),
960	identity=_choice(
961	raw["identity"], {"source_file_and_row"}, "adapter.identity"
962	),
963	spt=_dataclass_section(SptAdapterConfig, raw["spt"], "adapter.spt"),
964	pointcept=_dataclass_section(
965	PointceptAdapterConfig, raw["pointcept"], "adapter.pointcept"
966	),
967	)
968
969
970	def _parse_model(value: Any) -> ModelConfig:
971	raw = _section(
972	value,
973	{
974	"framework", "runner", "name", "base_config", "checkout_env",
975	"commit_env", "checkpoint", "args",
976	},
977	"model",
978	)
979	args = dict(_mapping(raw["args"], "model.args"))
980	allowed_args = {
981	"aggregation", "annotation_mode", "balanced_crops",
982	"confidence_only_forbidden", "enable_flash", "geometry_context",
983	"head", "label_fraction", "max_num_edges", "max_num_nodes",
984	"num_classes", "ontology_priority", "own_unlabeled_only",
985	"partition", "partition_stage", "patch_size", "published_weights",
986	"require_multiview_agreement", "round", "scanner_holdout", "selector",
987	"semantic_stage", "timing_instrumentation", "training_population",
988	"uncertainty_tier", "unlicensed_scribblekitti_code", "voxel_sizes_m",
989	}
990	_keys(args, set(), allowed_args, "model.args")
991	if "partition" in args:
992	partition = _mapping(args["partition"], "model.args.partition")
993	_exact_keys(
994	partition,
995	{
996	"regularization", "spatial_weight", "cutoff", "graph_k_max",
997	"graph_gap_m",
998	},
999	"model.args.partition",
1000	)
1001	return ModelConfig(
1002	framework=_choice(
1003	raw["framework"], {"cpu", "spt", "pointcept"}, "model.framework"
1004	),
1005	runner=_path(raw["runner"], "model.runner"),
1006	name=_string(raw["name"], "model.name"),
1007	base_config=_optional_path(raw["base_config"], "model.base_config"),
1008	checkout_env=_optional_string(raw["checkout_env"], "model.checkout_env"),
1009	commit_env=_optional_string(raw["commit_env"], "model.commit_env"),
1010	checkpoint=_optional_path(raw["checkpoint"], "model.checkpoint"),
1011	args=MappingProxyType(args),
1012	)
1013
1014
1015	def _parse_loss(value: Any) -> LossConfig:
1016	raw = _section(value, {"name", "args"}, "loss")
1017	args = dict(_mapping(raw["args"], "loss.args"))
1018	_keys(
1019	args,
1020	set(),
1021	{"alpha", "beta", "class_weighting", "gamma", "ignore_index", "reason"},
1022	"loss.args",
1023	)
1024	return LossConfig(_string(raw["name"], "loss.name"), MappingProxyType(args))
1025
1026
1027	def _parse_train(value: Any, model_raw: Mapping[str, Any]) -> TrainConfig:
1028	raw = _mapping(value, "train")
1029	inherited = {
1030	"max_epochs", "lr", "weight_decay", "precision",
1031	"accumulate_grad_batches", "accelerator", "devices",
1032	"viz_every_n_epochs", "viz_samples", "monitor", "monitor_mode",
1033	"early_stop_monitor", "early_stop_mode", "early_stop_patience",
1034	"log_dir", "log_every_n_steps",
1035	}
1036	allowed = inherited \| {
1037	"batch_size", "num_workers", "optimizer", "scheduler", "distributed"
1038	}
1039	_keys(raw, set(), allowed, "train")
1040	if model_raw.get("framework") in {"spt", "pointcept"} and (
1041	{"viz_every_n_epochs", "viz_samples"} & set(raw)
1042	):
1043	raise ConfigError(
1044	"train.viz_every_n_epochs and train.viz_samples are forbidden for "
1045	"external frameworks"
1046	)
1047	hints = _field_types(TrainConfig)
1048	converted = {
1049	name: _typed_value(hints.get(name), item, f"train.{name}")
1050	for name, item in raw.items()
1051	}
1052	try:
1053	return TrainConfig(**converted)
1054	except TypeError as exc:
1055	raise ConfigError(f"Invalid train section: {exc}") from exc
1056
1057
1058	def _parse_evaluation(value: Any) -> EvaluationConfig:
1059	raw = _mapping(value, "evaluation")
1060	required = {
1061	"split", "metrics", "precision_floors", "continuity",
1062	"object_matching", "bootstrap", "promotion",
1063	}
1064	_keys(raw, required, required \| {"worst_k_tiles"}, "evaluation")
1065	floors_raw = _mapping(raw["precision_floors"], "evaluation.precision_floors")
1066	floors = {
1067	_string(name, "precision floor class"): _float(
1068	value, f"evaluation.precision_floors.{name}"
1069	)
1070	for name, value in floors_raw.items()
1071	}
1072	return EvaluationConfig(
1073	split=_choice(
1074	raw["split"], {"validation", "promotion_test"}, "evaluation.split"
1075	),
1076	metrics=_strings(raw["metrics"], "evaluation.metrics"),
1077	precision_floors=MappingProxyType(floors),
1078	continuity=_dataclass_section(
1079	ContinuityConfig, raw["continuity"], "evaluation.continuity"
1080	),
1081	object_matching=_dataclass_section(
1082	ObjectMatchingConfig,
1083	raw["object_matching"],
1084	"evaluation.object_matching",
1085	),
1086	bootstrap=_dataclass_section(
1087	BootstrapConfig, raw["bootstrap"], "evaluation.bootstrap"
1088	),
1089	promotion=_dataclass_section(
1090	PromotionConfig, raw["promotion"], "evaluation.promotion"
1091	),
1092	worst_k_tiles=_integer(
1093	raw.get("worst_k_tiles", 4), "evaluation.worst_k_tiles"
1094	),
1095	)
1096
1097
1098	def _parse_runtime(value: Any) -> RuntimeConfig:
1099	return _dataclass_section(RuntimeConfig, value, "runtime")
1100
1101
1102	def _parse_visualization(value: Any) -> VisualizationConfig:
1103	parsed = _dataclass_section(VisualizationConfig, value, "visualization")
1104	if parsed.masks_every_n_epochs < 1:
1105	raise ConfigError("visualization.masks_every_n_epochs must be >= 1")
1106	tiles = parsed.masks_tiles
1107	if isinstance(tiles, int):
1108	if tiles < 1:
1109	raise ConfigError("visualization.masks_tiles must be >= 1")
1110	elif not tiles:
1111	raise ConfigError("visualization.masks_tiles cannot be empty")
1112	return parsed
1113
1114
1115	def _parse_provenance(value: Any) -> ProvenanceConfig:
1116	return _dataclass_section(ProvenanceConfig, value, "provenance")
1117
1118
1119	def _validate_config(config: HarnessConfig, raw: Mapping[str, Any]) -> None:
1120	if config.schema_version != 1:
1121	raise ConfigError(f"Unsupported schema_version {config.schema_version}")
1122	if not re.fullmatch(r"E(?:[1-9]\|1[0-4])", config.experiment.id):
1123	raise ConfigError(f"Invalid experiment.id {config.experiment.id!r}")
1124	ontology = _load_task_ontology(config)
1125	_validate_task_against_ontology(config, ontology)
1126	if config.seed < 0:
1127	raise ConfigError("seed must be non-negative")
1128	_validate_typed_sections(config, ontology)
1129	if (
1130	config.data.label_source.mode == "regenerate_full_resolution"
1131	and config.data.label_source.fuse_config is None
1132	):
1133	raise ConfigError(
1134	"regenerate_full_resolution requires data.label_source.fuse_config"
1135	)
1136	if (
1137	config.data.label_source.mode == "artifact"
1138	and config.data.label_source.fuse_config is not None
1139	):
1140	raise ConfigError(
1141	"artifact label mode requires data.label_source.fuse_config: null"
1142	)
1143	if not 0.0 <= config.data.label_source.max_unmatched_fraction <= 1.0:
1144	raise ConfigError("data.label_source.max_unmatched_fraction must be in [0, 1]")
1145	if config.data.tiling.length_m <= 0.0 or not (
1146	0.0 <= config.data.tiling.overlap_m < config.data.tiling.length_m
1147	):
1148	raise ConfigError("tiling requires 0 <= overlap_m < length_m")
1149	if config.data.tiling.min_points < 1:
1150	raise ConfigError("data.tiling.min_points must be positive")
1151	if set(config.adapter.emit) - {"canonical", "spt", "pointcept"}:
1152	raise ConfigError("adapter.emit contains an unsupported output format")
1153	if "canonical" not in config.adapter.emit:
1154	raise ConfigError("adapter.emit must include canonical")
1155	if config.adapter.canonical_version != 1:
1156	raise ConfigError("adapter.canonical_version must be 1")
1157	if config.model.framework == "cpu":
1158	if config.model.checkout_env is not None or config.model.commit_env is not None:
1159	raise ConfigError(
1160	"CPU experiments cannot declare external checkout variables"
1161	)
1162	elif (
1163	config.model.base_config is None
1164	or not config.model.checkout_env
1165	or not config.model.commit_env
1166	):
1167	raise ConfigError(
1168	"External models require base_config, checkout_env, and commit_env"
1169	)
1170	if config.model.framework == "pointcept" and config.runtime.flash_attention:
1171	raise ConfigError(
1172	"Pointcept PTv3/LitePT configurations must keep FlashAttention disabled"
1173	)
1174	if config.visualization is not None and config.model.framework != "pointcept":
1175	raise ConfigError(
1176	"visualization is only supported for model.framework pointcept; "
1177	f"{config.model.framework} configs must omit the block"
1178	)
1179	if config.runtime.spconv not in {
1180	"disabled",
1181	"spconv-cu124>=2.3.0,<2.4.0",
1182	"spconv-cu126>=2.3.0,<2.4.0",
1183	}:
1184	raise ConfigError("runtime.spconv is outside the permitted package range")
1185	if config.train.monitor_mode not in {"min", "max"} or (
1186	config.train.early_stop_mode not in {"min", "max"}
1187	):
1188	raise ConfigError("train monitor modes must be min or max")
1189	for metric in config.experiment.deciding_metrics:
1190	if not is_canonical_metric(metric, ontology=ontology):
1191	raise ConfigError(
1192	"experiment.deciding_metrics contains non-canonical metric "
1193	f"{metric!r} for ontology {ontology.name}"
1194	)
1195	for metric in (config.train.monitor, config.train.early_stop_monitor):
1196	if not is_canonical_metric(metric, ontology=ontology):
1197	raise ConfigError(
1198	f"train monitor {metric!r} is outside the canonical namespace "
1199	f"of ontology {ontology.name}"
1200	)
1201	unknown_floor_classes = sorted(
1202	set(config.evaluation.precision_floors) - set(ontology.class_names)
1203	)
1204	if unknown_floor_classes:
1205	raise ConfigError(
1206	"evaluation.precision_floors has unknown classes "
1207	f"{unknown_floor_classes} for ontology {ontology.name}"
1208	)
1209	if config.evaluation.worst_k_tiles < 1:
1210	raise ConfigError("evaluation.worst_k_tiles must be positive")
1211	promotion = config.evaluation.promotion
1212	if promotion.enabled and (
1213	promotion.delta_quality is None
1214	or promotion.delta_fp_per_km is None
1215	or not promotion.superiority_conditions
1216	):
1217	raise ConfigError(
1218	"enabled promotion requires non-null margins and superiority conditions"
1219	)
1220	if config.evaluation.split == "promotion_test" and not promotion.enabled:
1221	raise ConfigError(
1222	"promotion_test evaluation requires evaluation.promotion.enabled"
1223	)
1224	gate_types = {gate.type for gate in config.experiment.gates if gate.required}
1225	if (
1226	config.experiment.status == "template-only"
1227	and "implementation_ticket" not in gate_types
1228	):
1229	raise ConfigError(
1230	"template-only experiments require an implementation_ticket gate"
1231	)
1232	if config.experiment.status == "gated-later" and not gate_types:
1233	raise ConfigError("gated-later experiments require at least one required gate")
1234	if (
1235	config.experiment.status == "implement-now"
1236	and config.experiment.id not in {"E1", "E2"}
1237	):
1238	raise ConfigError("Only E1 and E2 are implement-now in schema version 1")
1239	_validate_study_overrides(config, raw)
1240
1241
1242	def resolve_ontology_path(config: HarnessConfig) -> Path:
1243	"""Resolve the repository-relative ``task.ontology`` path to a real file.
1244
1245	The declared path is relative to the repository that owns the config, so
1246	the ancestors of the config file are searched first, nearest ancestor
1247	first, and the working directory is only consulted last. A run launched
1248	from another checkout therefore reads the ontology of the repository its
1249	config lives in instead of a same-named file that happens to sit under the
1250	working directory. This is the single ontology resolver: every script,
1251	runner, and provenance writer calls it so a run can never validate against
1252	one ontology file and train against another.
1253
1254	Args:
1255	config: Parsed configuration naming the ontology.
1256
1257	Returns:
1258	An absolute, existing ontology path.
1259
1260	Raises:
1261	ConfigError: If no candidate path exists.
1262	"""
1263	declared = config.task.ontology
1264	if declared.is_absolute():
1265	if not declared.is_file():
1266	raise ConfigError(
1267	f"{config.source_path}: task.ontology {declared.as_posix()} "
1268	f"does not exist"
1269	)
1270	return declared
1271	candidates = [
1272	ancestor / declared for ancestor in config.source_path.resolve().parents
1273	]
1274	candidates.append(Path.cwd().resolve() / declared)
1275	for candidate in candidates:
1276	if candidate.is_file():
1277	return candidate.resolve()
1278	searched = ", ".join(
1279	sorted({candidate.parent.as_posix() for candidate in candidates})
1280	)
1281	raise ConfigError(
1282	f"{config.source_path}: task.ontology {declared.as_posix()} was not "
1283	f"found relative to any parent of the config or to the working "
1284	f"directory {Path.cwd().as_posix()}; searched {searched}"
1285	)
1286
1287
1288	def _load_task_ontology(config: HarnessConfig) -> Ontology:
1289	"""Load the ontology the config declares, failing closed as a ConfigError.
1290
1291	Args:
1292	config: Parsed configuration whose ``task.ontology`` path is resolved
1293	relative to the repository root.
1294
1295	Returns:
1296	The validated ontology every other contract is checked against.
1297
1298	Raises:
1299	ConfigError: If the ontology cannot be located, loaded, or is invalid.
1300	"""
1301	resolved = resolve_ontology_path(config)
1302	try:
1303	return load_ontology(resolved)
1304	except OntologyError as exc:
1305	raise ConfigError(
1306	f"task.ontology {config.task.ontology.as_posix()} is not a valid "
1307	f"ontology: {exc}"
1308	) from exc
1309
1310
1311	def _validate_task_against_ontology(
1312	config: HarnessConfig, ontology: Ontology
1313	) -> None:
1314	"""Check that the task block restates the loaded ontology exactly.
1315
1316	Args:
1317	config: Parsed configuration.
1318	ontology: Ontology loaded from ``task.ontology``.
1319
1320	Raises:
1321	ConfigError: If any task, evaluation, model, or loss class contract
1322	disagrees with the loaded ontology.
1323	"""
1324	task = config.task
1325	if task.num_classes != ontology.num_predicted_classes:
1326	raise ConfigError(
1327	f"task.num_classes {task.num_classes} must equal ontology "
1328	f"{ontology.name} num_predicted_classes "
1329	f"{ontology.num_predicted_classes}"
1330	)
1331	if task.ignore_index != ontology.void_id:
1332	raise ConfigError(
1333	f"task.ignore_index {task.ignore_index} must equal ontology "
1334	f"{ontology.name} void ID {ontology.void_id}"
1335	)
1336	if task.classes_of_interest != ontology.interest_ids:
1337	raise ConfigError(
1338	f"task.classes_of_interest {list(task.classes_of_interest)} must "
1339	f"equal ontology {ontology.name} interest IDs "
1340	f"{list(ontology.interest_ids)}"
1341	)
1342	linear_names = tuple(
1343	ontology.class_for_id(train_id).name for train_id in ontology.linear_class_ids
1344	)
1345	if len(set(task.linear_classes)) != len(task.linear_classes) or set(
1346	task.linear_classes
1347	) != set(linear_names):
1348	raise ConfigError(
1349	f"task.linear_classes {list(task.linear_classes)} must be exactly "
1350	f"the linear classes {list(linear_names)} of ontology "
1351	f"{ontology.name}"
1352	)
1353	profiles = config.evaluation.object_matching.cluster_profiles
1354	if profiles != task.ontology:
1355	raise ConfigError(
1356	"evaluation.object_matching.cluster_profiles "
1357	f"{profiles.as_posix()} must be the task ontology "
1358	f"{task.ontology.as_posix()}"
1359	)
1360	if "num_classes" in config.model.args:
1361	declared = config.model.args["num_classes"]
1362	if declared != task.num_classes:
1363	raise ConfigError(
1364	f"model.args.num_classes {declared!r} must equal "
1365	f"task.num_classes {task.num_classes}"
1366	)
1367	if "ignore_index" in config.loss.args:
1368	declared = config.loss.args["ignore_index"]
1369	if declared != task.ignore_index:
1370	raise ConfigError(
1371	f"loss.args.ignore_index {declared!r} must equal "
1372	f"task.ignore_index {task.ignore_index}"
1373	)
1374
1375
1376	def _validate_typed_sections(config: HarnessConfig, ontology: Ontology) -> None:
1377	"""Validate runtime types not enforced by dataclass constructors.
1378
1379	Args:
1380	config: Parsed configuration.
1381	ontology: Ontology loaded from ``task.ontology``.
1382
1383	Raises:
1384	ConfigError: If a typed section or a class-naming gate is invalid.
1385	"""
1386	_require_int(config.experiment.phase, "experiment.phase")
1387	for gate in config.experiment.gates:
1388	params = gate.params
1389	if isinstance(params, SptPartitionOracleParams):
1390	unknown = sorted(
1391	set(params.minimum_purity_by_class) - set(ontology.class_names)
1392	)
1393	if unknown:
1394	raise ConfigError(
1395	f"gate {gate.name}.minimum_purity_by_class contains unknown "
1396	f"ontology classes {unknown} for ontology {ontology.name}"
1397	)
1398	if isinstance(params, ImplementationTicketParams):
1399	_require_string(params.ticket, f"gate {gate.name}.ticket")
1400	_require_string(params.required_status, f"gate {gate.name}.required_status")
1401	elif isinstance(params, ArtifactExistsParams):
1402	if params.expected_status is not None:
1403	_require_string(
1404	params.expected_status, f"gate {gate.name}.expected_status"
1405	)
1406	elif isinstance(params, DataAvailableParams):
1407	_require_string(params.dataset, f"gate {gate.name}.dataset")
1408	elif isinstance(params, LicenseApprovalParams):
1409	_require_string(
1410	params.required_decision, f"gate {gate.name}.required_decision"
1411	)
1412	elif isinstance(params, OperationalSmokeParams):
1413	_require_string(
1414	params.required_status, f"gate {gate.name}.required_status"
1415	)
1416	for contrast in config.study.contrasts:
1417	for name, value in asdict(contrast).items():
1418	_require_string(value, f"study.contrasts.{name}")
1419	if not is_canonical_metric(contrast.metric, ontology=ontology):
1420	raise ConfigError(
1421	f"study contrast metric {contrast.metric!r} is not canonical "
1422	f"for ontology {ontology.name}"
1423	)
1424	if config.study.sweep is not None:
1425	_require_string(config.study.sweep.method, "study.sweep.method")
1426	_require_int(config.study.sweep.budget, "study.sweep.budget")
1427	if config.study.sweep.budget < 1:
1428	raise ConfigError("study.sweep.budget must be positive")
1429	if not is_canonical_metric(config.study.sweep.objective, ontology=ontology):
1430	raise ConfigError(
1431	f"study.sweep.objective must be canonical for ontology "
1432	f"{ontology.name}"
1433	)
1434	source = config.data.label_source
1435	if source.mode not in {"artifact", "regenerate_full_resolution"}:
1436	raise ConfigError("data.label_source.mode is unsupported")
1437	for name, value in (
1438	("source_geometry_glob", source.source_geometry_glob),
1439	("prefer", source.prefer),
1440	("stats_glob", source.stats_glob),
1441	):
1442	_require_string(value, f"data.label_source.{name}")
1443	if source.unmatched_policy != "void":
1444	raise ConfigError("data.label_source.unmatched_policy must be void")
1445	_require_number(
1446	source.max_unmatched_fraction,
1447	"data.label_source.max_unmatched_fraction",
1448	)
1449	features = config.data.features
1450	if features.fit_on != "train_corridors_only":
1451	raise ConfigError("data.features.fit_on must be train_corridors_only")
1452	_require_bool(features.scanner_conditioning, "data.features.scanner_conditioning")
1453	tiling = config.data.tiling
1454	if tiling.mode != "corridor_axis" or tiling.origin != "dataset_manifest":
1455	raise ConfigError("data.tiling requires corridor_axis and dataset_manifest")
1456	if tiling.blend != "linear_edge_weight":
1457	raise ConfigError("data.tiling.blend must be linear_edge_weight")
1458	_require_number(tiling.length_m, "data.tiling.length_m")
1459	_require_number(tiling.overlap_m, "data.tiling.overlap_m")
1460	_require_int(tiling.min_points, "data.tiling.min_points")
1461	_require_int(config.adapter.canonical_version, "adapter.canonical_version")
1462	_require_int(config.adapter.spt.pc_tiling, "adapter.spt.pc_tiling")
1463	_require_number(config.adapter.spt.voxel_m, "adapter.spt.voxel_m")
1464	_require_number(
1465	config.adapter.pointcept.grid_size_m, "adapter.pointcept.grid_size_m"
1466	)
1467	if config.adapter.spt.voxel_m <= 0.0 or config.adapter.pointcept.grid_size_m <= 0.0:
1468	raise ConfigError("adapter voxel/grid sizes must be positive")
1469	if config.loss.name not in {
1470	"framework_native",
1471	"cross_entropy",
1472	"focal_cross_entropy",
1473	"masked_focal_tversky",
1474	}:
1475	raise ConfigError(f"Unsupported loss.name {config.loss.name!r}")
1476	train = config.train
1477	for name in (
1478	"max_epochs", "accumulate_grad_batches", "batch_size", "num_workers",
1479	"early_stop_patience", "log_every_n_steps",
1480	):
1481	_require_int(getattr(train, name), f"train.{name}")
1482	for name in ("lr", "weight_decay"):
1483	_require_number(getattr(train, name), f"train.{name}")
1484	_require_bool(train.distributed, "train.distributed")
1485	for name in (
1486	"precision", "accelerator", "optimizer", "scheduler", "monitor",
1487	"monitor_mode", "early_stop_monitor", "early_stop_mode", "log_dir",
1488	):
1489	_require_string(getattr(train, name), f"train.{name}")
1490	evaluation = config.evaluation
1491	for value in evaluation.metrics:
1492	_require_string(value, "evaluation.metrics")
1493	for name, value in evaluation.precision_floors.items():
1494	_require_number(value, f"evaluation.precision_floors.{name}")
1495	if not 0.0 <= value <= 1.0:
1496	raise ConfigError(
1497	f"evaluation precision floor for {name} must be in [0, 1]"
1498	)
1499	_require_number(
1500	evaluation.continuity.chainage_bin_m,
1501	"evaluation.continuity.chainage_bin_m",
1502	)
1503	_require_number(
1504	evaluation.continuity.gap_threshold_m,
1505	"evaluation.continuity.gap_threshold_m",
1506	)
1507	bootstrap = evaluation.bootstrap
1508	if bootstrap.unit != "corridor":
1509	raise ConfigError("evaluation.bootstrap.unit must be corridor")
1510	_require_number(bootstrap.spatial_block_m, "evaluation.bootstrap.spatial_block_m")
1511	_require_int(bootstrap.samples, "evaluation.bootstrap.samples")
1512	_require_number(bootstrap.confidence, "evaluation.bootstrap.confidence")
1513	_require_int(bootstrap.seed, "evaluation.bootstrap.seed")
1514	_require_bool(evaluation.promotion.enabled, "evaluation.promotion.enabled")
1515	runtime = config.runtime
1516	_require_bool(runtime.flash_attention, "runtime.flash_attention")
1517	_require_int(runtime.system_ram_gb, "runtime.system_ram_gb")
1518	_require_int(runtime.gpu_memory_gb, "runtime.gpu_memory_gb")
1519	for name in ("target", "cuda", "spconv"):
1520	_require_string(getattr(runtime, name), f"runtime.{name}")
1521	provenance = config.provenance
1522	if provenance.manifest != "required" or provenance.data_hash_source != "dvc":
1523	raise ConfigError(
1524	"provenance requires manifest=required and data_hash_source=dvc"
1525	)
1526	_require_bool(provenance.record_environment, "provenance.record_environment")
1527	_require_bool(provenance.record_commands, "provenance.record_commands")
1528
1529
1530	def _require_string(value: Any, where: str) -> None:
1531	if not isinstance(value, str) or not value:
1532	raise ConfigError(f"{where} must be a non-empty string")
1533
1534
1535	def _require_int(value: Any, where: str) -> None:
1536	if isinstance(value, bool) or not isinstance(value, int):
1537	raise ConfigError(f"{where} must be an integer")
1538
1539
1540	def _require_number(value: Any, where: str) -> None:
1541	if isinstance(value, bool) or not isinstance(value, (int, float)):
1542	raise ConfigError(f"{where} must be numeric")
1543
1544
1545	def _require_bool(value: Any, where: str) -> None:
1546	if not isinstance(value, bool):
1547	raise ConfigError(f"{where} must be boolean")
1548
1549
1550	def _validate_study_overrides(config: HarnessConfig, raw: Mapping[str, Any]) -> None:
1551	leaves = _leaf_values(raw)
1552	override_groups: list[Mapping[str, OverrideValue]] = [
1553	item.overrides for item in config.study.variants
1554	]
1555	if config.study.matrix is not None:
1556	override_groups.extend(
1557	{path: value}
1558	for path, values in config.study.matrix.axes.items()
1559	for value in values
1560	)
1561	override_groups.extend(config.study.matrix.include)
1562	override_groups.extend(config.study.matrix.exclude)
1563	if config.study.sweep is not None:
1564	for path, parameter in config.study.sweep.parameters.items():
1565	if parameter.values is not None:
1566	override_groups.extend({path: value} for value in parameter.values)
1567	else:
1568	override_groups.extend(
1569	({path: parameter.minimum}, {path: parameter.maximum})
1570	)
1571	for overrides in override_groups:
1572	for path, value in overrides.items():
1573	if path.startswith("study.") or path not in leaves:
1574	raise ConfigError(
1575	f"Study override path {path!r} is not a declared scalar/list leaf"
1576	)
1577	expected = leaves[path]
1578	if not _same_leaf_type(expected, value):
1579	raise ConfigError(
1580	f"Study override {path!r} has incompatible value {value!r}; "
1581	f"expected type of {expected!r}"
1582	)
1583
1584
1585	def _cell_definitions(
1586	config: HarnessConfig,
1587	) -> tuple[tuple[str, Mapping[str, OverrideValue]], ...]:
1588	"""Return the ordered (identity, overrides) pairs of one study."""
1589	study = config.study
1590	if study.kind == "single":
1591	return ((config.experiment.id, MappingProxyType({})),)
1592	if study.kind == "variants":
1593	if not study.variants:
1594	raise ConfigError(
1595	f"{config.experiment.id} study.kind variants has no variants payload"
1596	)
1597	return tuple((item.id, item.overrides) for item in study.variants)
1598	if study.kind == "matrix":
1599	if study.matrix is None:
1600	raise ConfigError(
1601	f"{config.experiment.id} study.kind matrix has no matrix payload"
1602	)
1603	return _matrix_cells(study.matrix)
1604	if study.sweep is None:
1605	raise ConfigError(
1606	f"{config.experiment.id} study.kind sweep has no sweep payload"
1607	)
1608	return _sweep_cells(study.sweep, config.seed)
1609
1610
1611	def _matrix_cells(
1612	matrix: MatrixConfig,
1613	) -> tuple[tuple[str, Mapping[str, OverrideValue]], ...]:
1614	"""Expand typed matrix axes into deterministic cells."""
1615	axis_paths = tuple(matrix.axes)
1616	unknown = sorted(
1617	{path for item in matrix.exclude for path in item} - set(axis_paths)
1618	)
1619	if unknown:
1620	raise ConfigError(f"study.matrix.exclude references non-axis paths {unknown}")
1621	definitions: list[tuple[str, Mapping[str, OverrideValue]]] = []
1622	excluded = [0] * len(matrix.exclude)
1623	for combination in itertools.product(
1624	*(matrix.axes[path] for path in axis_paths)
1625	):
1626	overrides = dict(zip(axis_paths, combination, strict=True))
1627	dropped = False
1628	for index, item in enumerate(matrix.exclude):
1629	if all(overrides[path] == value for path, value in item.items()):
1630	excluded[index] += 1
1631	dropped = True
1632	if not dropped:
1633	definitions.append((_cell_id(overrides), MappingProxyType(overrides)))
1634	for index, count in enumerate(excluded):
1635	if not count:
1636	raise ConfigError(
1637	f"study.matrix.exclude[{index}] matches no matrix cell"
1638	)
1639	for index, item in enumerate(matrix.include):
1640	if not item:
1641	raise ConfigError(f"study.matrix.include[{index}] cannot be empty")
1642	definitions.append((_cell_id(item), item))
1643	if not definitions:
1644	raise ConfigError("study.matrix excludes every cell")
1645	return tuple(definitions)
1646
1647
1648	def _sweep_cells(
1649	sweep: SweepConfig, seed: int
1650	) -> tuple[tuple[str, Mapping[str, OverrideValue]], ...]:
1651	"""Expand a bounded sweep deterministically under the study seed."""
1652	paths = tuple(sweep.parameters)
1653	if sweep.method == "grid":
1654	unbounded = sorted(
1655	path for path, item in sweep.parameters.items() if item.values is None
1656	)
1657	if unbounded:
1658	raise ConfigError(
1659	"study.sweep.method grid requires explicit values for "
1660	f"{unbounded}"
1661	)
1662	definitions: list[tuple[str, Mapping[str, OverrideValue]]] = []
1663	for combination in itertools.product(
1664	*(tuple(sweep.parameters[path].values or ()) for path in paths)
1665	):
1666	overrides = dict(zip(paths, combination, strict=True))
1667	definitions.append((_cell_id(overrides), MappingProxyType(overrides)))
1668	return tuple(definitions[: sweep.budget])
1669	if sweep.method == "random":
1670	generator = random.Random(seed)
1671	return tuple(
1672	(
1673	f"sample_{index:03d}",
1674	MappingProxyType(
1675	{
1676	path: _sweep_sample(
1677	sweep.parameters[path],
1678	generator,
1679	f"study.sweep.parameters.{path}",
1680	)
1681	for path in paths
1682	}
1683	),
1684	)
1685	for index in range(sweep.budget)
1686	)
1687	raise ConfigError(
1688	f"study.sweep.method {sweep.method!r} is not implemented; supported "
1689	"methods are grid and random"
1690	)
1691
1692
1693	def _sweep_sample(
1694	parameter: SweepParameterConfig, generator: random.Random, where: str
1695	) -> OverrideValue:
1696	"""Draw one deterministic value for a sweep parameter."""
1697	if parameter.values is not None:
1698	return parameter.values[generator.randrange(len(parameter.values))]
1699	if parameter.minimum is None or parameter.maximum is None:
1700	raise ConfigError(f"{where} requires minimum and maximum for sampling")
1701	if parameter.distribution == "uniform":
1702	drawn = generator.uniform(parameter.minimum, parameter.maximum)
1703	elif parameter.distribution == "log_uniform":
1704	if parameter.minimum <= 0.0:
1705	raise ConfigError(f"{where}.minimum must be positive for log_uniform")
1706	drawn = math.exp(
1707	generator.uniform(
1708	math.log(parameter.minimum), math.log(parameter.maximum)
1709	)
1710	)
1711	else:
1712	raise ConfigError(
1713	f"{where}.distribution {parameter.distribution!r} is not implemented; "
1714	"supported distributions are uniform and log_uniform"
1715	)
1716	return float(f"{drawn:.6g}")
1717
1718
1719	def _cell_id(overrides: Mapping[str, OverrideValue]) -> str:
1720	"""Derive a stable, filesystem-safe identity from a cell's overrides."""
1721	if not overrides:
1722	raise ConfigError("A study cell requires at least one override")
1723	names = [path.rsplit(".", 1)[-1] for path in overrides]
1724	if len(set(names)) != len(names):
1725	names = [path.replace(".", "_") for path in overrides]
1726	return "__".join(
1727	f"{name}-{_value_token(value)}"
1728	for name, value in zip(names, overrides.values(), strict=True)
1729	)
1730
1731
1732	def _value_token(value: OverrideValue) -> str:
1733	"""Render one override value as a filesystem-safe token."""
1734	if isinstance(value, bool):
1735	text = "true" if value else "false"
1736	elif value is None:
1737	text = "null"
1738	elif isinstance(value, float):
1739	text = repr(value)
1740	elif isinstance(value, list):
1741	text = "+".join(_value_token(item) for item in value)
1742	else:
1743	text = str(value)
1744	return re.sub(r"[^A-Za-z0-9._+-]", "_", text)
1745
1746
1747	def _apply_overrides(
1748	raw: Mapping[str, Any], overrides: Mapping[str, OverrideValue], where: str
1749	) -> dict[str, Any]:
1750	"""Apply dotted-path overrides to a raw configuration mapping.
1751
1752	Args:
1753	raw: Raw mapping of the base configuration.
1754	overrides: Dotted leaf paths mapped to their replacement values.
1755	where: Cell identification used in error messages.
1756
1757	Returns:
1758	A deep copy of ``raw`` carrying the overridden leaves.
1759
1760	Raises:
1761	ConfigError: If a path is not a declared leaf or the value type differs.
1762	"""
1763	result = copy.deepcopy(dict(raw))
1764	leaves = _leaf_values(raw)
1765	for path in sorted(overrides):
1766	value = overrides[path]
1767	if path.startswith("study.") or path not in leaves:
1768	raise ConfigError(
1769	f"{where} override path {path!r} is not a declared scalar/list leaf"
1770	)
1771	expected = leaves[path]
1772	if not _same_leaf_type(expected, value):
1773	raise ConfigError(
1774	f"{where} override {path!r} has incompatible value {value!r}; "
1775	f"expected type of {expected!r}"
1776	)
1777	_set_leaf(result, path, _coerce_leaf(expected, value), where)
1778	return result
1779
1780
1781	def _set_leaf(
1782	target: dict[str, Any], path: str, value: OverrideValue, where: str
1783	) -> None:
1784	segments = path.split(".")
1785	node: Any = target
1786	for segment in segments[:-1]:
1787	if not isinstance(node, dict) or segment not in node:
1788	raise ConfigError(f"{where} override path {path!r} is not addressable")
1789	node = node[segment]
1790	if not isinstance(node, dict) or segments[-1] not in node:
1791	raise ConfigError(f"{where} override path {path!r} is not addressable")
1792	node[segments[-1]] = value
1793
1794
1795	def _coerce_leaf(expected: OverrideValue, value: OverrideValue) -> OverrideValue:
1796	"""Parse an override value as the declared leaf's type."""
1797	if isinstance(expected, list):
1798	items = list(value) if isinstance(value, list) else [value]
1799	if expected and isinstance(expected[0], float):
1800	return [_coerce_leaf(expected[0], item) for item in items]
1801	return items
1802	if (
1803	isinstance(expected, float)
1804	and isinstance(value, int)
1805	and not isinstance(value, bool)
1806	):
1807	return float(value)
1808	return value
1809
1810
1811	def _leaf_values(value: Any, prefix: str = "") -> dict[str, OverrideValue]:
1812	result: dict[str, OverrideValue] = {}
1813	if isinstance(value, Mapping):
1814	for key, item in value.items():
1815	path = f"{prefix}.{key}" if prefix else str(key)
1816	if path == "study" or path.startswith("study."):
1817	continue
1818	result.update(_leaf_values(item, path))
1819	elif isinstance(value, list):
1820	if all(
1821	isinstance(item, (str, int, float, bool)) or item is None
1822	for item in value
1823	):
1824	result[prefix] = value
1825	elif isinstance(value, (str, int, float, bool)) or value is None:
1826	result[prefix] = value
1827	return result
1828
1829
1830	def _same_leaf_type(expected: OverrideValue, actual: OverrideValue) -> bool:
1831	if isinstance(expected, list):
1832	if not isinstance(actual, list):
1833	return False
1834	if not expected or not actual:
1835	return True
1836	return all(_same_leaf_type(expected[0], item) for item in actual)
1837	if isinstance(expected, bool):
1838	return isinstance(actual, bool)
1839	if isinstance(expected, int) and not isinstance(expected, bool):
1840	return isinstance(actual, int) and not isinstance(actual, bool)
1841	if isinstance(expected, float):
1842	return isinstance(actual, (int, float)) and not isinstance(actual, bool)
1843	return actual is None if expected is None else isinstance(actual, type(expected))
1844
1845
1846	def _dataclass_section(cls: type[Any], value: Any, where: str) -> Any:
1847	raw = _mapping(value, where)
1848	fields = cls.__dataclass_fields__
1849	required = {
1850	name
1851	for name, item in fields.items()
1852	if item.default is MISSING and item.default_factory is MISSING
1853	}
1854	_keys(raw, required, set(fields), where)
1855	hints = _field_types(cls)
1856	converted: dict[str, Any] = {
1857	name: _typed_value(hints.get(name), item, f"{where}.{name}")
1858	for name, item in raw.items()
1859	}
1860	try:
1861	return cls(**converted)
1862	except TypeError as exc:
1863	raise ConfigError(f"Invalid {where}: {exc}") from exc
1864
1865
1866	def _field_types(cls: type[Any]) -> Mapping[str, Any]:
1867	"""Return resolved field annotations, or an empty mapping if unresolvable.
1868
1869	Args:
1870	cls: Dataclass whose annotations declare the leaf types.
1871
1872	Returns:
1873	Mapping of field name to resolved annotation object.
1874	"""
1875	cached = _FIELD_TYPE_CACHE.get(cls)
1876	if cached is not None:
1877	return cached
1878	try:
1879	hints: Mapping[str, Any] = dict(get_type_hints(cls))
1880	except (NameError, TypeError): # pragma: no cover - defensive
1881	hints = {}
1882	_FIELD_TYPE_CACHE[cls] = hints
1883	return hints
1884
1885
1886	def _typed_value(annotation: Any, value: Any, where: str) -> Any:
1887	"""Parse one config leaf strictly as its declared annotation.
1888
1889	Args:
1890	annotation: Resolved field annotation, or None when unknown.
1891	value: Raw YAML value.
1892	where: Dotted key path used in error messages.
1893
1894	Returns:
1895	The value converted to the declared type.
1896
1897	Raises:
1898	ConfigError: If the value does not match the declared type.
1899	"""
1900	if annotation is None or annotation is Any:
1901	return value
1902	origin = get_origin(annotation)
1903	if origin is Literal:
1904	return _choice(value, {str(item) for item in get_args(annotation)}, where)
1905	if origin in (Union, UnionType):
1906	return _typed_union(annotation, value, where)
1907	if annotation is Path:
1908	return _path(value, where)
1909	if annotation is bool:
1910	return _boolean(value, where)
1911	if annotation is int:
1912	return _integer(value, where)
1913	if annotation is float:
1914	return _float(value, where)
1915	if annotation is str:
1916	return _string(value, where)
1917	if origin is tuple:
1918	args = get_args(annotation)
1919	item_type = args[0] if args else None
1920	return tuple(
1921	_typed_value(item_type, item, where) for item in _sequence(value, where)
1922	)
1923	if origin in (dict, Mapping) or (
1924	isinstance(origin, type) and issubclass(origin, Mapping)
1925	):
1926	args = get_args(annotation)
1927	item_type = args[1] if len(args) == 2 else None
1928	raw = _mapping(value, where)
1929	return MappingProxyType(
1930	{
1931	_string(key, where): _typed_value(item_type, item, f"{where}.{key}")
1932	for key, item in raw.items()
1933	}
1934	)
1935	return value
1936
1937
1938	def _typed_union(annotation: Any, value: Any, where: str) -> Any:
1939	"""Parse a value against a union annotation, rejecting every mismatch."""
1940	members = get_args(annotation)
1941	optional = type(None) in members
1942	if value is None:
1943	if optional:
1944	return None
1945	raise ConfigError(f"{where} must not be null")
1946	candidates = [item for item in members if item is not type(None)]
1947	if len(candidates) == 1:
1948	return _typed_value(candidates[0], value, where)
1949	for candidate in candidates:
1950	try:
1951	return _typed_value(candidate, value, where)
1952	except ConfigError:
1953	continue
1954	names = sorted(getattr(item, "__name__", str(item)) for item in candidates)
1955	raise ConfigError(
1956	f"{where} must be one of {names}, got {type(value).__name__} {value!r}"
1957	)
1958
1959
1960	def _json_safe(value: Any) -> Any:
1961	if is_dataclass(value) and not isinstance(value, type):
1962	return {
1963	item.name: _json_safe(getattr(value, item.name))
1964	for item in fields(value)
1965	}
1966	if isinstance(value, Path):
1967	return value.as_posix()
1968	if isinstance(value, Mapping):
1969	return {str(key): _json_safe(item) for key, item in value.items()}
1970	if isinstance(value, (tuple, list)):
1971	return [_json_safe(item) for item in value]
1972	return value
1973
1974
1975	def _overrides(value: Any, where: str) -> Mapping[str, OverrideValue]:
1976	raw = _mapping(value, where)
1977	return MappingProxyType(
1978	{
1979	_string(path, where): _override_value(item, f"{where}.{path}")
1980	for path, item in raw.items()
1981	}
1982	)
1983
1984
1985	def _override_value(value: Any, where: str) -> OverrideValue:
1986	if isinstance(value, list):
1987	return [_scalar(item, where) for item in value]
1988	return _scalar(value, where)
1989
1990
1991	def _section(value: Any, keys: set[str], where: str) -> Mapping[str, Any]:
1992	raw = _mapping(value, where)
1993	_exact_keys(raw, keys, where)
1994	return raw
1995
1996
1997	def _mapping(value: Any, where: str) -> Mapping[str, Any]:
1998	if not isinstance(value, Mapping):
1999	raise ConfigError(f"{where} must be a mapping")
2000	if any(not isinstance(key, str) for key in value):
2001	raise ConfigError(f"{where} keys must be strings")
2002	return value
2003
2004
2005	def _sequence(value: Any, where: str) -> Sequence[Any]:
2006	if isinstance(value, (str, bytes)) or not isinstance(value, Sequence):
2007	raise ConfigError(f"{where} must be a sequence")
2008	return value
2009
2010
2011	def _exact_keys(value: Mapping[str, Any], expected: set[str], where: str) -> None:
2012	_keys(value, expected, expected, where)
2013
2014
2015	def _keys(
2016	value: Mapping[str, Any],
2017	required: set[str],
2018	allowed: set[str],
2019	where: str,
2020	) -> None:
2021	missing = sorted(required - set(value))
2022	unknown = sorted(set(value) - allowed)
2023	if missing or unknown:
2024	details = []
2025	if missing:
2026	details.append(f"missing {missing}")
2027	if unknown:
2028	details.append(f"unknown {unknown}")
2029	raise ConfigError(f"{where} has " + " and ".join(details))
2030
2031
2032	def _string(value: Any, where: str) -> str:
2033	if not isinstance(value, str) or not value.strip():
2034	raise ConfigError(f"{where} must be a non-empty string")
2035	return value
2036
2037
2038	def _optional_string(value: Any, where: str) -> str \| None:
2039	return None if value is None else _string(value, where)
2040
2041
2042	def _integer(value: Any, where: str) -> int:
2043	if isinstance(value, bool) or not isinstance(value, int):
2044	raise ConfigError(f"{where} must be an integer")
2045	return value
2046
2047
2048	def _float(value: Any, where: str) -> float:
2049	if isinstance(value, bool) or not isinstance(value, (int, float)):
2050	raise ConfigError(f"{where} must be numeric")
2051	return float(value)
2052
2053
2054	def _optional_float(value: Any, where: str) -> float \| None:
2055	return None if value is None else _float(value, where)
2056
2057
2058	def _boolean(value: Any, where: str) -> bool:
2059	if not isinstance(value, bool):
2060	raise ConfigError(f"{where} must be boolean")
2061	return value
2062
2063
2064	def _scalar(value: Any, where: str) -> Scalar:
2065	if value is None or isinstance(value, (str, bool)):
2066	return value
2067	if isinstance(value, int) and not isinstance(value, bool):
2068	return value
2069	if isinstance(value, float):
2070	return value
2071	raise ConfigError(f"{where} must be a scalar or scalar list")
2072
2073
2074	def _path(value: Any, where: str) -> Path:
2075	text = _string(value, where)
2076	path = Path(text)
2077	if path.is_absolute():
2078	raise ConfigError(f"{where} must be repository-relative, got {path}")
2079	return path
2080
2081
2082	def _optional_path(value: Any, where: str) -> Path \| None:
2083	return None if value is None else _path(value, where)
2084
2085
2086	def _strings(value: Any, where: str) -> tuple[str, ...]:
2087	return tuple(_string(item, where) for item in _sequence(value, where))
2088
2089
2090	def _integers(value: Any, where: str) -> tuple[int, ...]:
2091	return tuple(_integer(item, where) for item in _sequence(value, where))
2092
2093
2094	def _choice(value: Any, choices: set[str], where: str) -> Any:
2095	text = _string(value, where)
2096	if text not in choices:
2097	raise ConfigError(f"{where} must be one of {sorted(choices)}, got {text!r}")
2098	return text

Importance #55: src/train/config_rules.py @@ -0,0 +1,364 @@

	1	"""Cross-section and ontology-dependent rules of an experiment configuration.
	2
	3	The pydantic models in :mod:`src.train.config_schema` and
	4	:mod:`src.train.config_sections` own every key, type, and single-section rule.
	5	This module owns what a model cannot see: the ontology file the task declares,
	6	the metric namespace it fixes, and the rules that span two sections.
	7	"""
	8
	9	from __future__ import annotations
	10
	11	import logging
	12	import re
	13	from collections.abc import Mapping
	14	from pathlib import Path
	15	from typing import Any
	16
	17	from src.contracts.ontology import (
	18	Ontology,
	19	OntologyError,
	20	load_ontology,
	21	macro_interest_all_suffix,
	22	)
	23	from src.train import config_schema, config_sections, config_values
	24
	25	logger = logging.getLogger(__name__)
	26
	27	_EXPERIMENT_ID_PATTERN = re.compile(r"E(?:[1-9]\|1[0-4])")
	28	_VIZ_TRAIN_KEYS = frozenset({"viz_every_n_epochs", "viz_samples"})
	29
	30
	31	def is_canonical_metric(name: str, *, ontology: Ontology) -> bool:
	32	"""Return whether a metric belongs to an ontology's val/eval namespace.
	33
	34	Args:
	35	name: Metric tag to validate.
	36	ontology: Ontology whose predicted class names and interest count fix
	37	the per-class tags and the structural macro suffix.
	38
	39	Returns:
	40	True for a canonical tag, otherwise False.
	41	"""
	42	if name == "val/loss":
	43	return True
	44	match = re.fullmatch(r"(val\|eval)/(.+)", name)
	45	if match is None:
	46	return False
	47	metric = match.group(2)
	48	suffix = macro_interest_all_suffix(ontology)
	49	fixed = {
	50	"iou_macro_interest",
	51	"f1_macro_interest",
	52	"precision_macro_interest",
	53	"recall_macro_interest",
	54	f"iou_macro_interest_{suffix}",
	55	f"f1_macro_interest_{suffix}",
	56	f"precision_macro_interest_{suffix}",
	57	f"recall_macro_interest_{suffix}",
	58	"miou_all_classes",
	59	}
	60	if metric in fixed:
	61	return True
	62	class_name = "\|".join(re.escape(item) for item in ontology.class_names)
	63	patterns = (
	64	rf"(?:iou\|precision\|recall\|f1\|support)_(?:{class_name})",
	65	rf"(?:fp_per_km\|detections_per_km\|matched_recall)_(?:{class_name})",
	66	rf"continuity_(?:covered_fraction\|total_missing_length_m\|gaps\|"
	67	rf"gap_median_m\|gap_p95_m\|gap_max_m)_(?:{class_name})",
	68	rf"seam_[a-zA-Z0-9_.-]+_(?:{class_name})",
	69	)
	70	return any(re.fullmatch(pattern, metric) is not None for pattern in patterns)
	71
	72
	73	def resolve_ontology_path(config: config_sections.HarnessConfig) -> Path:
	74	"""Resolve the repository-relative ``task.ontology`` path to a real file.
	75
	76	The declared path is relative to the repository that owns the config, so
	77	the ancestors of the config file are searched first, nearest ancestor
	78	first, and the working directory is only consulted last. A run launched
	79	from another checkout therefore reads the ontology of the repository its
	80	config lives in instead of a same-named file that happens to sit under the
	81	working directory. This is the single ontology resolver: every script,
	82	runner, and provenance writer calls it so a run can never validate against
	83	one ontology file and train against another.
	84
	85	Args:
	86	config: Parsed configuration naming the ontology.
	87
	88	Returns:
	89	An absolute, existing ontology path.
	90
	91	Raises:
	92	ConfigError: If no candidate path exists.
	93	"""
	94	declared = config.task.ontology
	95	if declared.is_absolute():
	96	if not declared.is_file():
	97	raise config_values.ConfigError(
	98	f"{config.source_path}: task.ontology {declared.as_posix()} "
	99	f"does not exist"
	100	)
	101	return declared
	102	candidates = [
	103	ancestor / declared for ancestor in config.source_path.resolve().parents
	104	]
	105	candidates.append(Path.cwd().resolve() / declared)
	106	for candidate in candidates:
	107	if candidate.is_file():
	108	return candidate.resolve()
	109	searched = ", ".join(
	110	sorted({candidate.parent.as_posix() for candidate in candidates})
	111	)
	112	raise config_values.ConfigError(
	113	f"{config.source_path}: task.ontology {declared.as_posix()} was not "
	114	f"found relative to any parent of the config or to the working "
	115	f"directory {Path.cwd().as_posix()}; searched {searched}"
	116	)
	117
	118
	119	def validate_config(
	120	config: config_sections.HarnessConfig, raw: Mapping[str, Any]
	121	) -> None:
	122	"""Apply every rule a single section's model cannot decide on its own.
	123
	124	Args:
	125	config: Configuration whose sections are already model-validated.
	126	raw: Raw mapping the configuration was parsed from, used to check the
	127	declared study overrides against the document's own leaves.
	128
	129	Raises:
	130	ConfigError: If a cross-section, identity, or ontology rule fails.
	131	"""
	132	if config.schema_version != 1:
	133	raise config_values.ConfigError(
	134	f"Unsupported schema_version {config.schema_version}"
	135	)
	136	if not _EXPERIMENT_ID_PATTERN.fullmatch(config.experiment.id):
	137	raise config_values.ConfigError(
	138	f"Invalid experiment.id {config.experiment.id!r}"
	139	)
	140	ontology = _load_task_ontology(config)
	141	_validate_task_against_ontology(config, ontology)
	142	_validate_frameworks(config)
	143	_validate_metrics(config, ontology)
	144	_validate_gates(config, ontology)
	145	_validate_study_overrides(config, raw)
	146
	147
	148	def _validate_frameworks(config: config_sections.HarnessConfig) -> None:
	149	"""Check the rules that tie a section to the declared model framework."""
	150	if config.model.framework in {"spt", "pointcept"} and (
	151	_VIZ_TRAIN_KEYS & config.train.model_fields_set
	152	):
	153	raise config_values.ConfigError(
	154	"train.viz_every_n_epochs and train.viz_samples are forbidden for "
	155	"external frameworks"
	156	)
	157	if config.model.framework == "pointcept" and config.runtime.flash_attention:
	158	raise config_values.ConfigError(
	159	"Pointcept PTv3/LitePT configurations must keep FlashAttention disabled"
	160	)
	161	if config.visualization is not None and config.model.framework != "pointcept":
	162	raise config_values.ConfigError(
	163	"visualization is only supported for model.framework pointcept; "
	164	f"{config.model.framework} configs must omit the block"
	165	)
	166
	167
	168	def _validate_metrics(
	169	config: config_sections.HarnessConfig, ontology: Ontology
	170	) -> None:
	171	"""Check every declared metric tag against the ontology's namespace."""
	172	for metric in config.experiment.deciding_metrics:
	173	if not is_canonical_metric(metric, ontology=ontology):
	174	raise config_values.ConfigError(
	175	"experiment.deciding_metrics contains non-canonical metric "
	176	f"{metric!r} for ontology {ontology.name}"
	177	)
	178	for metric in (config.train.monitor, config.train.early_stop_monitor):
	179	if not is_canonical_metric(metric, ontology=ontology):
	180	raise config_values.ConfigError(
	181	f"train monitor {metric!r} is outside the canonical namespace "
	182	f"of ontology {ontology.name}"
	183	)
	184	unknown_floor_classes = sorted(
	185	set(config.evaluation.precision_floors) - set(ontology.class_names)
	186	)
	187	if unknown_floor_classes:
	188	raise config_values.ConfigError(
	189	"evaluation.precision_floors has unknown classes "
	190	f"{unknown_floor_classes} for ontology {ontology.name}"
	191	)
	192	for contrast in config.study.contrasts:
	193	if not is_canonical_metric(contrast.metric, ontology=ontology):
	194	raise config_values.ConfigError(
	195	f"study contrast metric {contrast.metric!r} is not canonical "
	196	f"for ontology {ontology.name}"
	197	)
	198	if config.study.sweep is not None and not is_canonical_metric(
	199	config.study.sweep.objective, ontology=ontology
	200	):
	201	raise config_values.ConfigError(
	202	f"study.sweep.objective must be canonical for ontology {ontology.name}"
	203	)
	204
	205
	206	def _validate_gates(
	207	config: config_sections.HarnessConfig, ontology: Ontology
	208	) -> None:
	209	"""Check gate payloads and the readiness status they must back."""
	210	for gate in config.experiment.gates:
	211	if isinstance(gate.params, config_schema.SptPartitionOracleParams):
	212	unknown = sorted(
	213	set(gate.params.minimum_purity_by_class) - set(ontology.class_names)
	214	)
	215	if unknown:
	216	raise config_values.ConfigError(
	217	f"gate {gate.name}.minimum_purity_by_class contains unknown "
	218	f"ontology classes {unknown} for ontology {ontology.name}"
	219	)
	220	gate_types = {gate.type for gate in config.experiment.gates if gate.required}
	221	if (
	222	config.experiment.status == "template-only"
	223	and "implementation_ticket" not in gate_types
	224	):
	225	raise config_values.ConfigError(
	226	"template-only experiments require an implementation_ticket gate"
	227	)
	228	if config.experiment.status == "gated-later" and not gate_types:
	229	raise config_values.ConfigError(
	230	"gated-later experiments require at least one required gate"
	231	)
	232	if config.experiment.status == "implement-now" and config.experiment.id not in {
	233	"E1",
	234	"E2",
	235	}:
	236	raise config_values.ConfigError(
	237	"Only E1 and E2 are implement-now in schema version 1"
	238	)
	239
	240
	241	def _load_task_ontology(config: config_sections.HarnessConfig) -> Ontology:
	242	"""Load the ontology the config declares, failing closed as a ConfigError.
	243
	244	Args:
	245	config: Parsed configuration whose ``task.ontology`` path is resolved
	246	relative to the repository root.
	247
	248	Returns:
	249	The validated ontology every other contract is checked against.
	250
	251	Raises:
	252	ConfigError: If the ontology cannot be located, loaded, or is invalid.
	253	"""
	254	resolved = resolve_ontology_path(config)
	255	try:
	256	return load_ontology(resolved)
	257	except OntologyError as exc:
	258	raise config_values.ConfigError(
	259	f"task.ontology {config.task.ontology.as_posix()} is not a valid "
	260	f"ontology: {exc}"
	261	) from exc
	262
	263
	264	def _validate_task_against_ontology(
	265	config: config_sections.HarnessConfig, ontology: Ontology
	266	) -> None:
	267	"""Check that the task block restates the loaded ontology exactly.
	268
	269	Args:
	270	config: Parsed configuration.
	271	ontology: Ontology loaded from ``task.ontology``.
	272
	273	Raises:
	274	ConfigError: If any task, evaluation, model, or loss class contract
	275	disagrees with the loaded ontology.
	276	"""
	277	task = config.task
	278	if task.num_classes != ontology.num_predicted_classes:
	279	raise config_values.ConfigError(
	280	f"task.num_classes {task.num_classes} must equal ontology "
	281	f"{ontology.name} num_predicted_classes "
	282	f"{ontology.num_predicted_classes}"
	283	)
	284	if task.ignore_index != ontology.void_id:
	285	raise config_values.ConfigError(
	286	f"task.ignore_index {task.ignore_index} must equal ontology "
	287	f"{ontology.name} void ID {ontology.void_id}"
	288	)
	289	if task.classes_of_interest != ontology.interest_ids:
	290	raise config_values.ConfigError(
	291	f"task.classes_of_interest {list(task.classes_of_interest)} must "
	292	f"equal ontology {ontology.name} interest IDs "
	293	f"{list(ontology.interest_ids)}"
	294	)
	295	linear_names = tuple(
	296	ontology.class_for_id(train_id).name for train_id in ontology.linear_class_ids
	297	)
	298	if len(set(task.linear_classes)) != len(task.linear_classes) or set(
	299	task.linear_classes
	300	) != set(linear_names):
	301	raise config_values.ConfigError(
	302	f"task.linear_classes {list(task.linear_classes)} must be exactly "
	303	f"the linear classes {list(linear_names)} of ontology "
	304	f"{ontology.name}"
	305	)
	306	profiles = config.evaluation.object_matching.cluster_profiles
	307	if profiles != task.ontology:
	308	raise config_values.ConfigError(
	309	"evaluation.object_matching.cluster_profiles "
	310	f"{profiles.as_posix()} must be the task ontology "
	311	f"{task.ontology.as_posix()}"
	312	)
	313	if "num_classes" in config.model.args:
	314	declared = config.model.args["num_classes"]
	315	if declared != task.num_classes:
	316	raise config_values.ConfigError(
	317	f"model.args.num_classes {declared!r} must equal "
	318	f"task.num_classes {task.num_classes}"
	319	)
	320	if "ignore_index" in config.loss.args:
	321	declared = config.loss.args["ignore_index"]
	322	if declared != task.ignore_index:
	323	raise config_values.ConfigError(
	324	f"loss.args.ignore_index {declared!r} must equal "
	325	f"task.ignore_index {task.ignore_index}"
	326	)
	327
	328
	329	def _validate_study_overrides(
	330	config: config_sections.HarnessConfig, raw: Mapping[str, Any]
	331	) -> None:
	332	"""Check every declared override against the document's own leaves."""
	333	leaves = config_values.leaf_values(raw)
	334	override_groups: list[Mapping[str, config_values.OverrideValue]] = [
	335	item.overrides for item in config.study.variants
	336	]
	337	if config.study.matrix is not None:
	338	override_groups.extend(
	339	{path: value}
	340	for path, values in config.study.matrix.axes.items()
	341	for value in values
	342	)
	343	override_groups.extend(config.study.matrix.include)
	344	override_groups.extend(config.study.matrix.exclude)
	345	if config.study.sweep is not None:
	346	for path, parameter in config.study.sweep.parameters.items():
	347	if parameter.values is not None:
	348	override_groups.extend({path: value} for value in parameter.values)
	349	else:
	350	override_groups.extend(
	351	({path: parameter.minimum}, {path: parameter.maximum})
	352	)
	353	for overrides in override_groups:
	354	for path, value in overrides.items():
	355	if path.startswith("study.") or path not in leaves:
	356	raise config_values.ConfigError(
	357	f"Study override path {path!r} is not a declared scalar/list leaf"
	358	)
	359	expected = leaves[path]
	360	if not config_values.same_leaf_type(expected, value):
	361	raise config_values.ConfigError(
	362	f"Study override {path!r} has incompatible value {value!r}; "
	363	f"expected type of {expected!r}"
	364	)
0

Importance #56: src/train/config_schema.py @@ -0,0 +1,332 @@

	1	"""Pydantic models for the experiment identity, gate, and study blocks.
	2
	3	Every model derives from :class:`StrictConfigModel`, the repository's
	4	fail-closed flavour of :class:`iolabs.common.config_loader.ConfigModel`:
	5	unknown keys are rejected, instances are frozen, and leaf values must already
	6	carry their declared YAML type (see :mod:`src.train.config_values`).
	7	"""
	8
	9	from __future__ import annotations
	10
	11	import logging
	12	from collections.abc import Mapping
	13	from pathlib import Path
	14	from typing import Annotated, Any, Literal, TypeAlias
	15
	16	import pydantic
	17	from iolabs.common import config_loader
	18
	19	from src.train import config_values
	20
	21	logger = logging.getLogger(__name__)
	22
	23	ExperimentStatus: TypeAlias = Literal["implement-now", "template-only", "gated-later"]
	24	StudyKind: TypeAlias = Literal["single", "variants", "matrix", "sweep"]
	25
	26
	27	class StrictConfigModel(config_loader.ConfigModel):
	28	"""Fleet config model whose leaves keep this repository's strict typing."""
	29
	30	@pydantic.field_validator("*", mode="before")
	31	@classmethod
	32	def _coerce_fleet_scalars(cls, value: Any, info: pydantic.ValidationInfo) -> Any:
	33	"""Parse a raw YAML value strictly as the field's declared type."""
	34	name = info.field_name or ""
	35	field = cls.model_fields.get(name)
	36	if field is None or field.annotation is None:
	37	return value
	38	return config_values.typed_value(field.annotation, value, name)
	39
	40
	41	class SptPartitionOracleParams(StrictConfigModel):
	42	"""Required SPT partition-purity report and per-class thresholds."""
	43
	44	report: Path
	45	minimum_purity_by_class: Mapping[str, float]
	46
	47
	48	class ImplementationTicketParams(StrictConfigModel):
	49	"""Implementation ticket whose external state must reach a required value."""
	50
	51	ticket: str
	52	required_status: str
	53
	54
	55	class ArtifactExistsParams(StrictConfigModel):
	56	"""Required local artifact and optional expected JSON status."""
	57
	58	path: Path
	59	expected_status: str \| None = None
	60
	61
	62	class DataAvailableParams(StrictConfigModel):
	63	"""Required data manifest and declared dataset contract."""
	64
	65	manifest: Path
	66	dataset: str
	67
	68
	69	class LicenseApprovalParams(StrictConfigModel):
	70	"""Required license-review decision artifact."""
	71
	72	decision: Path
	73	required_decision: str
	74
	75
	76	class OperationalSmokeParams(StrictConfigModel):
	77	"""Required operational-smoke report and accepted result."""
	78
	79	report: Path
	80	required_status: str
	81
	82
	83	class CheckpointPolicyParams(StrictConfigModel):
	84	"""Required checkpoint-policy file and checkpoint metadata artifact."""
	85
	86	policy: Path
	87	metadata: Path
	88
	89
	90	class HumanWorkflowParams(StrictConfigModel):
	91	"""Required human-workflow protocol and acceptance artifact."""
	92
	93	protocol: Path
	94	acceptance: Path
	95
	96
	97	GateParams: TypeAlias = (
	98	SptPartitionOracleParams
	99	\| ImplementationTicketParams
	100	\| ArtifactExistsParams
	101	\| DataAvailableParams
	102	\| LicenseApprovalParams
	103	\| OperationalSmokeParams
	104	\| CheckpointPolicyParams
	105	\| HumanWorkflowParams
	106	)
	107
	108
	109	class GateBase(StrictConfigModel):
	110	"""Shared identity of one typed, fail-closed experiment gate."""
	111
	112	name: str
	113
	114
	115	class SptPartitionOracleGate(GateBase):
	116	"""Gate on an SPT partition-purity report."""
	117
	118	type: Literal["spt_partition_oracle"]
	119	required: bool
	120	params: SptPartitionOracleParams
	121
	122
	123	class ImplementationTicketGate(GateBase):
	124	"""Gate on an external implementation ticket."""
	125
	126	type: Literal["implementation_ticket"]
	127	required: bool
	128	params: ImplementationTicketParams
	129
	130
	131	class ArtifactExistsGate(GateBase):
	132	"""Gate on a local artifact and its optional status."""
	133
	134	type: Literal["artifact_exists"]
	135	required: bool
	136	params: ArtifactExistsParams
	137
	138
	139	class DataAvailableGate(GateBase):
	140	"""Gate on a declared dataset manifest."""
	141
	142	type: Literal["data_available"]
	143	required: bool
	144	params: DataAvailableParams
	145
	146
	147	class LicenseApprovalGate(GateBase):
	148	"""Gate on a license-review decision."""
	149
	150	type: Literal["license_approval"]
	151	required: bool
	152	params: LicenseApprovalParams
	153
	154
	155	class OperationalSmokeGate(GateBase):
	156	"""Gate on an operational-smoke report."""
	157
	158	type: Literal["operational_smoke"]
	159	required: bool
	160	params: OperationalSmokeParams
	161
	162
	163	class CheckpointPolicyGate(GateBase):
	164	"""Gate on the checkpoint policy and its metadata artifact."""
	165
	166	type: Literal["checkpoint_policy"]
	167	required: bool
	168	params: CheckpointPolicyParams
	169
	170
	171	class HumanWorkflowGate(GateBase):
	172	"""Gate on a human-workflow protocol and its acceptance artifact."""
	173
	174	type: Literal["human_workflow"]
	175	required: bool
	176	params: HumanWorkflowParams
	177
	178
	179	def _gate_tag(value: Any) -> str \| None:
	180	"""Return the declared gate type a raw or parsed gate is tagged with."""
	181	if isinstance(value, Mapping):
	182	tag = value.get("type")
	183	return tag if isinstance(tag, str) else None
	184	return getattr(value, "type", None)
	185
	186
	187	GateConfig: TypeAlias = Annotated[
	188	Annotated[SptPartitionOracleGate, pydantic.Tag("spt_partition_oracle")]
	189	\| Annotated[ImplementationTicketGate, pydantic.Tag("implementation_ticket")]
	190	\| Annotated[ArtifactExistsGate, pydantic.Tag("artifact_exists")]
	191	\| Annotated[DataAvailableGate, pydantic.Tag("data_available")]
	192	\| Annotated[LicenseApprovalGate, pydantic.Tag("license_approval")]
	193	\| Annotated[OperationalSmokeGate, pydantic.Tag("operational_smoke")]
	194	\| Annotated[CheckpointPolicyGate, pydantic.Tag("checkpoint_policy")]
	195	\| Annotated[HumanWorkflowGate, pydantic.Tag("human_workflow")],
	196	pydantic.Discriminator(_gate_tag),
	197	]
	198
	199
	200	class ExperimentConfig(StrictConfigModel):
	201	"""Experiment identity, readiness, hypothesis, metrics, and gates."""
	202
	203	id: str
	204	name: str
	205	phase: int
	206	status: ExperimentStatus
	207	hypothesis: str
	208	deciding_metrics: tuple[str, ...]
	209	gates: tuple[GateConfig, ...]
	210
	211
	212	class VariantConfig(StrictConfigModel):
	213	"""One named study variant expressed as typed dotted-path overrides."""
	214
	215	id: str
	216	overrides: Mapping[str, config_values.OverrideValue]
	217	tags: tuple[str, ...] = ()
	218
	219
	220	class MatrixConfig(StrictConfigModel):
	221	"""Typed Cartesian matrix definition with optional cells."""
	222
	223	axes: Mapping[str, tuple[config_values.OverrideValue, ...]]
	224	include: tuple[Mapping[str, config_values.OverrideValue], ...] = ()
	225	exclude: tuple[Mapping[str, config_values.OverrideValue], ...] = ()
	226
	227	@pydantic.field_validator("axes")
	228	@classmethod
	229	def _axes_are_populated(
	230	cls, value: Mapping[str, tuple[config_values.OverrideValue, ...]]
	231	) -> Mapping[str, tuple[config_values.OverrideValue, ...]]:
	232	"""Reject an axis that declares no value."""
	233	for path, values in value.items():
	234	if not values:
	235	raise config_values.ConfigError(
	236	f"study.matrix.axes.{path} cannot be empty"
	237	)
	238	return value
	239
	240
	241	class SweepParameterConfig(StrictConfigModel):
	242	"""One finite or bounded sweep parameter."""
	243
	244	values: tuple[config_values.OverrideValue, ...] \| None = None
	245	minimum: float \| None = None
	246	maximum: float \| None = None
	247	distribution: str \| None = None
	248
	249	@pydantic.model_validator(mode="after")
	250	def _bounds_are_complete(self) -> SweepParameterConfig:
	251	"""Reject a parameter that is neither finite nor fully bounded."""
	252	if self.values is not None and not self.values:
	253	raise config_values.ConfigError("values cannot be empty")
	254	if self.values is None and (
	255	self.minimum is None or self.maximum is None or self.distribution is None
	256	):
	257	raise config_values.ConfigError(
	258	"requires values or minimum/maximum/distribution"
	259	)
	260	if (
	261	self.minimum is not None
	262	and self.maximum is not None
	263	and self.minimum >= self.maximum
	264	):
	265	raise config_values.ConfigError("minimum must be smaller than maximum")
	266	return self
	267
	268
	269	class SweepConfig(StrictConfigModel):
	270	"""Typed bounded sweep contract."""
	271
	272	method: str
	273	parameters: Mapping[str, SweepParameterConfig]
	274	budget: int = pydantic.Field(ge=1)
	275	objective: str
	276
	277	@pydantic.field_validator("parameters")
	278	@classmethod
	279	def _parameters_are_declared(
	280	cls, value: Mapping[str, SweepParameterConfig]
	281	) -> Mapping[str, SweepParameterConfig]:
	282	"""Reject a sweep that declares no parameter."""
	283	if not value:
	284	raise config_values.ConfigError("study.sweep.parameters cannot be empty")
	285	return value
	286
	287
	288	class ContrastConfig(StrictConfigModel):
	289	"""Predeclared comparison between two study cells."""
	290
	291	name: str
	292	left: str
	293	right: str
	294	metric: str
	295
	296
	297	class StudyConfig(StrictConfigModel):
	298	"""Discriminated single, variants, matrix, or sweep study definition."""
	299
	300	kind: StudyKind
	301	variants: tuple[VariantConfig, ...]
	302	matrix: MatrixConfig \| None
	303	sweep: SweepConfig \| None
	304	contrasts: tuple[ContrastConfig, ...]
	305
	306	@pydantic.model_validator(mode="after")
	307	def _kind_matches_its_payload(self) -> StudyConfig:
	308	"""Reject a study whose kind and payload disagree."""
	309	if self.kind == "single" and (
	310	self.variants or self.matrix is not None or self.sweep is not None
	311	):
	312	raise config_values.ConfigError(
	313	"study.kind single cannot carry variants, matrix, or sweep"
	314	)
	315	if self.kind == "variants" and (
	316	not self.variants or self.matrix is not None or self.sweep is not None
	317	):
	318	raise config_values.ConfigError(
	319	"study.kind variants requires only a non-empty variants payload"
	320	)
	321	if self.kind == "matrix" and (
	322	self.variants or self.matrix is None or self.sweep is not None
	323	):
	324	raise config_values.ConfigError("study.kind matrix requires only matrix")
	325	if self.kind == "sweep" and (
	326	self.variants or self.matrix is not None or self.sweep is None
	327	):
	328	raise config_values.ConfigError("study.kind sweep requires only sweep")
	329	identities = [item.id for item in self.variants]
	330	if len(set(identities)) != len(identities):
	331	raise config_values.ConfigError("study.variants contains duplicate IDs")
	332	return self
0

Importance #57: src/train/config_sections.py @@ -0,0 +1,414 @@

	1	"""Pydantic models for the data, model, training, and evaluation blocks.
	2
	3	The models mirror the experiment YAML one-to-one: a nested block is a nested
	4	model, a field name is the YAML key, and every value keeps the strict typing of
	5	:mod:`src.train.config_values`. Cross-section rules that need the loaded
	6	ontology stay in :mod:`src.train.config_rules`.
	7	"""
	8
	9	from __future__ import annotations
	10
	11	import logging
	12	from collections.abc import Mapping
	13	from pathlib import Path
	14	from typing import Any, Literal
	15
	16	import pydantic
	17
	18	from src.train import config_schema, config_values
	19
	20	logger = logging.getLogger(__name__)
	21
	22	SOURCE_PATH_ALIAS = "__source_path__"
	23	SHA256_ALIAS = "__sha256__"
	24	_MODEL_ARG_KEYS = frozenset(
	25	{
	26	"aggregation", "annotation_mode", "balanced_crops",
	27	"confidence_only_forbidden", "enable_flash", "geometry_context",
	28	"head", "label_fraction", "max_num_edges", "max_num_nodes",
	29	"num_classes", "ontology_priority", "own_unlabeled_only",
	30	"partition", "partition_stage", "patch_size", "published_weights",
	31	"require_multiview_agreement", "round", "scanner_holdout", "selector",
	32	"semantic_stage", "timing_instrumentation", "training_population",
	33	"uncertainty_tier", "unlicensed_scribblekitti_code", "voxel_sizes_m",
	34	}
	35	)
	36	_PARTITION_KEYS = frozenset(
	37	{"regularization", "spatial_weight", "cutoff", "graph_k_max", "graph_gap_m"}
	38	)
	39	_LOSS_ARG_KEYS = frozenset(
	40	{"alpha", "beta", "class_weighting", "gamma", "ignore_index", "reason"}
	41	)
	42
	43
	44	class TaskConfig(config_schema.StrictConfigModel):
	45	"""Frozen semantic task contract."""
	46
	47	ontology: Path
	48	num_classes: int
	49	ignore_index: int
	50	classes_of_interest: tuple[int, ...]
	51	linear_classes: tuple[str, ...]
	52
	53
	54	class LabelSourceConfig(config_schema.StrictConfigModel):
	55	"""Label provenance and fail-closed join policy."""
	56
	57	mode: Literal["artifact", "regenerate_full_resolution"]
	58	source_geometry_glob: str
	59	fuse_config: Path \| None
	60	prefer: str
	61	classical_glob: str \| None
	62	recap_glob: str \| None
	63	stats_glob: str
	64	unmatched_policy: Literal["void"]
	65	max_unmatched_fraction: float = pydantic.Field(ge=0.0, le=1.0)
	66
	67	@pydantic.model_validator(mode="after")
	68	def _fuse_config_matches_mode(self) -> LabelSourceConfig:
	69	"""Tie the fusion config to the declared label mode."""
	70	if self.mode == "regenerate_full_resolution" and self.fuse_config is None:
	71	raise config_values.ConfigError(
	72	"regenerate_full_resolution requires data.label_source.fuse_config"
	73	)
	74	if self.mode == "artifact" and self.fuse_config is not None:
	75	raise config_values.ConfigError(
	76	"artifact label mode requires data.label_source.fuse_config: null"
	77	)
	78	return self
	79
	80
	81	class CorridorSelectionConfig(config_schema.StrictConfigModel):
	82	"""Config-declared corridor allow-list."""
	83
	84	include: tuple[str, ...]
	85
	86
	87	class FeatureConfig(config_schema.StrictConfigModel):
	88	"""Ordered features and train-only normalization contract."""
	89
	90	names: tuple[str, ...]
	91	normalization_manifest: Path
	92	fit_on: Literal["train_corridors_only"]
	93	scanner_conditioning: bool
	94
	95
	96	class TilingConfig(config_schema.StrictConfigModel):
	97	"""Deterministic corridor tiling and overlap blending contract."""
	98
	99	mode: Literal["corridor_axis"]
	100	length_m: float = pydantic.Field(gt=0.0)
	101	overlap_m: float = pydantic.Field(ge=0.0)
	102	origin: Literal["dataset_manifest"]
	103	min_points: int = pydantic.Field(ge=1)
	104	blend: Literal["linear_edge_weight"]
	105
	106	@pydantic.model_validator(mode="after")
	107	def _overlap_fits_in_a_tile(self) -> TilingConfig:
	108	"""Reject an overlap that is not shorter than the tile."""
	109	if self.overlap_m >= self.length_m:
	110	raise config_values.ConfigError("tiling requires 0 <= overlap_m < length_m")
	111	return self
	112
	113
	114	class DataConfig(config_schema.StrictConfigModel):
	115	"""Input roots, splits, labels, features, and tiling."""
	116
	117	root: Path
	118	canonical_root: Path
	119	processed_root: Path
	120	split_manifest: Path
	121	label_source: LabelSourceConfig
	122	corridors: CorridorSelectionConfig
	123	features: FeatureConfig
	124	tiling: TilingConfig
	125
	126
	127	class SptAdapterConfig(config_schema.StrictConfigModel):
	128	"""SPT raw-dataset emission contract."""
	129
	130	raw_root: Path
	131	pc_tiling: int
	132	voxel_m: float = pydantic.Field(gt=0.0)
	133	base_family: str
	134	raw_row_sidecar_keys: tuple[str, ...]
	135	audit_only_data_keys: tuple[str, ...]
	136
	137
	138	class PointceptAdapterConfig(config_schema.StrictConfigModel):
	139	"""Pointcept default-dataset emission contract."""
	140
	141	root: Path
	142	grid_size_m: float = pydantic.Field(gt=0.0)
	143	preserve_keys: tuple[str, ...]
	144
	145
	146	class AdapterConfig(config_schema.StrictConfigModel):
	147	"""Framework-neutral and external-format adapter contract."""
	148
	149	emit: tuple[str, ...]
	150	canonical_version: int
	151	identity: Literal["source_file_and_row"]
	152	spt: SptAdapterConfig
	153	pointcept: PointceptAdapterConfig
	154
	155	@pydantic.field_validator("emit")
	156	@classmethod
	157	def _emit_is_supported(cls, value: tuple[str, ...]) -> tuple[str, ...]:
	158	"""Reject unknown output formats and a missing canonical emission."""
	159	if set(value) - {"canonical", "spt", "pointcept"}:
	160	raise config_values.ConfigError(
	161	"adapter.emit contains an unsupported output format"
	162	)
	163	if "canonical" not in value:
	164	raise config_values.ConfigError("adapter.emit must include canonical")
	165	return value
	166
	167	@pydantic.field_validator("canonical_version")
	168	@classmethod
	169	def _canonical_version_is_one(cls, value: int) -> int:
	170	"""Freeze the canonical dataset version at 1."""
	171	if value != 1:
	172	raise config_values.ConfigError("adapter.canonical_version must be 1")
	173	return value
	174
	175
	176	class ModelConfig(config_schema.StrictConfigModel):
	177	"""Local or external model/runner selection."""
	178
	179	framework: Literal["cpu", "spt", "pointcept"]
	180	runner: Path
	181	name: str
	182	base_config: Path \| None
	183	checkout_env: str \| None
	184	commit_env: str \| None
	185	checkpoint: Path \| None
	186	args: Mapping[str, Any]
	187
	188	@pydantic.field_validator("args")
	189	@classmethod
	190	def _args_are_declared(cls, value: Mapping[str, Any]) -> Mapping[str, Any]:
	191	"""Reject undeclared model arguments and partition keys."""
	192	config_values.check_keys(value, set(), set(_MODEL_ARG_KEYS), "model.args")
	193	if "partition" in value:
	194	partition = config_values.mapping(
	195	value["partition"], "model.args.partition"
	196	)
	197	config_values.check_keys(
	198	partition,
	199	set(_PARTITION_KEYS),
	200	set(_PARTITION_KEYS),
	201	"model.args.partition",
	202	)
	203	return value
	204
	205	@pydantic.model_validator(mode="after")
	206	def _external_models_pin_their_checkout(self) -> ModelConfig:
	207	"""Tie the external checkout variables to the declared framework."""
	208	if self.framework == "cpu":
	209	if self.checkout_env is not None or self.commit_env is not None:
	210	raise config_values.ConfigError(
	211	"CPU experiments cannot declare external checkout variables"
	212	)
	213	elif not (self.base_config and self.checkout_env and self.commit_env):
	214	raise config_values.ConfigError(
	215	"External models require base_config, checkout_env, and commit_env"
	216	)
	217	return self
	218
	219
	220	class LossConfig(config_schema.StrictConfigModel):
	221	"""Shared-registry or external-native loss selection."""
	222
	223	name: Literal[
	224	"framework_native",
	225	"cross_entropy",
	226	"focal_cross_entropy",
	227	"masked_focal_tversky",
	228	]
	229	args: Mapping[str, Any]
	230
	231	@pydantic.field_validator("args")
	232	@classmethod
	233	def _args_are_declared(cls, value: Mapping[str, Any]) -> Mapping[str, Any]:
	234	"""Reject undeclared loss arguments."""
	235	config_values.check_keys(value, set(), set(_LOSS_ARG_KEYS), "loss.args")
	236	return value
	237
	238
	239	class TrainConfig(config_schema.StrictConfigModel):
	240	"""Harness-visible training settings for local and external runners."""
	241
	242	max_epochs: int = -1
	243	lr: float = 3.0e-4
	244	weight_decay: float = 1.0e-4
	245	precision: str = "auto"
	246	accumulate_grad_batches: int = 1
	247	accelerator: str = "auto"
	248	devices: int \| str = 1
	249	viz_every_n_epochs: int = 2
	250	viz_samples: int = 4
	251	monitor: str = "val/f1_mean_fg"
	252	monitor_mode: Literal["min", "max"] = "max"
	253	early_stop_monitor: str = "val/loss"
	254	early_stop_mode: Literal["min", "max"] = "min"
	255	early_stop_patience: int = 4
	256	log_dir: str = "runs"
	257	log_every_n_steps: int = 10
	258	batch_size: int = 1
	259	num_workers: int = 4
	260	optimizer: str = "adamw"
	261	scheduler: str = "cosine"
	262	distributed: bool = False
	263
	264
	265	class ContinuityConfig(config_schema.StrictConfigModel):
	266	"""Linear-continuity binning contract."""
	267
	268	chainage_bin_m: float
	269	gap_threshold_m: float
	270
	271
	272	class ObjectMatchingConfig(config_schema.StrictConfigModel):
	273	"""Object-clustering profile source and fallback tolerances."""
	274
	275	cluster_profiles: Path
	276	minimum_iou: float
	277	centroid_tolerance_m: float
	278
	279
	280	class BootstrapConfig(config_schema.StrictConfigModel):
	281	"""Corridor/spatial bootstrap contract."""
	282
	283	unit: Literal["corridor"]
	284	spatial_block_m: float
	285	samples: int
	286	confidence: float
	287	seed: int
	288
	289
	290	class PromotionConfig(config_schema.StrictConfigModel):
	291	"""Locked-test promotion margins and superiority conditions."""
	292
	293	enabled: bool
	294	delta_quality: float \| None
	295	delta_fp_per_km: float \| None
	296	superiority_conditions: tuple[str, ...]
	297
	298	@pydantic.model_validator(mode="after")
	299	def _enabled_promotion_is_complete(self) -> PromotionConfig:
	300	"""Reject an enabled promotion without margins or conditions."""
	301	if self.enabled and (
	302	self.delta_quality is None
	303	or self.delta_fp_per_km is None
	304	or not self.superiority_conditions
	305	):
	306	raise config_values.ConfigError(
	307	"enabled promotion requires non-null margins and superiority "
	308	"conditions"
	309	)
	310	return self
	311
	312
	313	class EvaluationConfig(config_schema.StrictConfigModel):
	314	"""Held-out evaluation and promotion protocol."""
	315
	316	split: Literal["validation", "promotion_test"]
	317	metrics: tuple[str, ...]
	318	precision_floors: Mapping[str, float]
	319	continuity: ContinuityConfig
	320	object_matching: ObjectMatchingConfig
	321	bootstrap: BootstrapConfig
	322	promotion: PromotionConfig
	323	worst_k_tiles: int = pydantic.Field(default=4, ge=1)
	324
	325	@pydantic.field_validator("precision_floors")
	326	@classmethod
	327	def _floors_are_fractions(cls, value: Mapping[str, float]) -> Mapping[str, float]:
	328	"""Reject a precision floor outside the unit interval."""
	329	for name, floor in value.items():
	330	if not 0.0 <= floor <= 1.0:
	331	raise config_values.ConfigError(
	332	f"evaluation precision floor for {name} must be in [0, 1]"
	333	)
	334	return value
	335
	336	@pydantic.model_validator(mode="after")
	337	def _promotion_test_is_promotable(self) -> EvaluationConfig:
	338	"""Reject a locked-test split whose promotion protocol is disabled."""
	339	if self.split == "promotion_test" and not self.promotion.enabled:
	340	raise config_values.ConfigError(
	341	"promotion_test evaluation requires evaluation.promotion.enabled"
	342	)
	343	return self
	344
	345
	346	class RuntimeConfig(config_schema.StrictConfigModel):
	347	"""Operational hardware and kernel constraints."""
	348
	349	target: str
	350	cuda: str
	351	spconv: Literal[
	352	"disabled",
	353	"spconv-cu124>=2.3.0,<2.4.0",
	354	"spconv-cu126>=2.3.0,<2.4.0",
	355	]
	356	flash_attention: bool
	357	system_ram_gb: int
	358	gpu_memory_gb: int
	359
	360
	361	class ProvenanceConfig(config_schema.StrictConfigModel):
	362	"""Mandatory run-evidence policy."""
	363
	364	manifest: Literal["required"]
	365	data_hash_source: Literal["dvc"]
	366	record_environment: bool
	367	record_commands: bool
	368	checkpoint_policy: Path
	369
	370
	371	class VisualizationConfig(config_schema.StrictConfigModel):
	372	"""Opt-in training-time TensorBoard class-mask visualization."""
	373
	374	masks_every_n_epochs: int = pydantic.Field(ge=1)
	375	masks_tiles: int \| tuple[str, ...] = 2
	376
	377	@pydantic.model_validator(mode="after")
	378	def _tiles_select_something(self) -> VisualizationConfig:
	379	"""Reject an empty or non-positive tile selection."""
	380	if isinstance(self.masks_tiles, int):
	381	if self.masks_tiles < 1:
	382	raise config_values.ConfigError(
	383	"visualization.masks_tiles must be >= 1"
	384	)
	385	elif not self.masks_tiles:
	386	raise config_values.ConfigError("visualization.masks_tiles cannot be empty")
	387	return self
	388
	389
	390	class HarnessConfig(config_schema.StrictConfigModel):
	391	"""Fully parsed, cross-field-validated experiment configuration."""
	392
	393	schema_version: int
	394	experiment: config_schema.ExperimentConfig
	395	study: config_schema.StudyConfig
	396	seed: int = pydantic.Field(ge=0)
	397	task: TaskConfig
	398	data: DataConfig
	399	adapter: AdapterConfig
	400	model: ModelConfig
	401	loss: LossConfig
	402	train: TrainConfig
	403	evaluation: EvaluationConfig
	404	runtime: RuntimeConfig
	405	provenance: ProvenanceConfig
	406	source_path: Path = pydantic.Field(validation_alias=SOURCE_PATH_ALIAS)
	407	sha256: str = pydantic.Field(validation_alias=SHA256_ALIAS)
	408	visualization: VisualizationConfig \| None = None
	409
	410	_raw_document: Mapping[str, Any] \| None = pydantic.PrivateAttr(default=None)
	411
	412	def as_dict(self) -> dict[str, Any]:
	413	"""Return the resolved model tree as JSON-safe primitives."""
	414	return self.model_dump(mode="json")
0

Importance #58: src/train/config_study.py @@ -0,0 +1,259 @@

	1	"""Deterministic expansion of a typed study definition into cell overrides.
	2
	3	The models own the study schema; this module owns the study algebra: the
	4	ordered cell definitions of a variants, matrix, or sweep study, the identity
	5	token of a cell, and the application of dotted-path overrides to the raw
	6	document a cell is re-validated from.
	7	"""
	8
	9	from __future__ import annotations
	10
	11	import copy
	12	import itertools
	13	import logging
	14	import math
	15	import random
	16	import re
	17	from collections.abc import Mapping
	18	from types import MappingProxyType
	19	from typing import Any
	20
	21	from src.train import config_schema, config_sections, config_values
	22
	23	logger = logging.getLogger(__name__)
	24
	25	CellDefinition = tuple[str, Mapping[str, config_values.OverrideValue]]
	26
	27
	28	def cell_definitions(
	29	config: config_sections.HarnessConfig,
	30	) -> tuple[CellDefinition, ...]:
	31	"""Return the ordered (identity, overrides) pairs of one study.
	32
	33	Args:
	34	config: Strictly parsed experiment configuration.
	35
	36	Returns:
	37	The deterministic cell definitions of the declared study kind.
	38
	39	Raises:
	40	ConfigError: If the study payload of the declared kind is absent or
	41	expands to nothing.
	42	"""
	43	study = config.study
	44	if study.kind == "single":
	45	return ((config.experiment.id, MappingProxyType({})),)
	46	if study.kind == "variants":
	47	if not study.variants:
	48	raise config_values.ConfigError(
	49	f"{config.experiment.id} study.kind variants has no variants payload"
	50	)
	51	return tuple((item.id, item.overrides) for item in study.variants)
	52	if study.kind == "matrix":
	53	if study.matrix is None:
	54	raise config_values.ConfigError(
	55	f"{config.experiment.id} study.kind matrix has no matrix payload"
	56	)
	57	return matrix_cells(study.matrix)
	58	if study.sweep is None:
	59	raise config_values.ConfigError(
	60	f"{config.experiment.id} study.kind sweep has no sweep payload"
	61	)
	62	return sweep_cells(study.sweep, config.seed)
	63
	64
	65	def matrix_cells(matrix: config_schema.MatrixConfig) -> tuple[CellDefinition, ...]:
	66	"""Expand typed matrix axes into deterministic cells."""
	67	axis_paths = tuple(matrix.axes)
	68	unknown = sorted(
	69	{path for item in matrix.exclude for path in item} - set(axis_paths)
	70	)
	71	if unknown:
	72	raise config_values.ConfigError(
	73	f"study.matrix.exclude references non-axis paths {unknown}"
	74	)
	75	definitions: list[CellDefinition] = []
	76	excluded = [0] * len(matrix.exclude)
	77	for combination in itertools.product(*(matrix.axes[path] for path in axis_paths)):
	78	overrides = dict(zip(axis_paths, combination, strict=True))
	79	dropped = False
	80	for index, item in enumerate(matrix.exclude):
	81	if all(overrides[path] == value for path, value in item.items()):
	82	excluded[index] += 1
	83	dropped = True
	84	if not dropped:
	85	definitions.append((cell_id(overrides), MappingProxyType(overrides)))
	86	for index, count in enumerate(excluded):
	87	if not count:
	88	raise config_values.ConfigError(
	89	f"study.matrix.exclude[{index}] matches no matrix cell"
	90	)
	91	for index, item in enumerate(matrix.include):
	92	if not item:
	93	raise config_values.ConfigError(
	94	f"study.matrix.include[{index}] cannot be empty"
	95	)
	96	definitions.append((cell_id(item), item))
	97	if not definitions:
	98	raise config_values.ConfigError("study.matrix excludes every cell")
	99	return tuple(definitions)
	100
	101
	102	def sweep_cells(
	103	sweep: config_schema.SweepConfig, seed: int
	104	) -> tuple[CellDefinition, ...]:
	105	"""Expand a bounded sweep deterministically under the study seed."""
	106	paths = tuple(sweep.parameters)
	107	if sweep.method == "grid":
	108	unbounded = sorted(
	109	path for path, item in sweep.parameters.items() if item.values is None
	110	)
	111	if unbounded:
	112	raise config_values.ConfigError(
	113	f"study.sweep.method grid requires explicit values for {unbounded}"
	114	)
	115	definitions: list[CellDefinition] = []
	116	for combination in itertools.product(
	117	*(tuple(sweep.parameters[path].values or ()) for path in paths)
	118	):
	119	overrides = dict(zip(paths, combination, strict=True))
	120	definitions.append((cell_id(overrides), MappingProxyType(overrides)))
	121	return tuple(definitions[: sweep.budget])
	122	if sweep.method == "random":
	123	generator = random.Random(seed)
	124	return tuple(
	125	(
	126	f"sample_{index:03d}",
	127	MappingProxyType(
	128	{
	129	path: _sweep_sample(
	130	sweep.parameters[path],
	131	generator,
	132	f"study.sweep.parameters.{path}",
	133	)
	134	for path in paths
	135	}
	136	),
	137	)
	138	for index in range(sweep.budget)
	139	)
	140	raise config_values.ConfigError(
	141	f"study.sweep.method {sweep.method!r} is not implemented; supported "
	142	"methods are grid and random"
	143	)
	144
	145
	146	def _sweep_sample(
	147	parameter: config_schema.SweepParameterConfig,
	148	generator: random.Random,
	149	where: str,
	150	) -> config_values.OverrideValue:
	151	"""Draw one deterministic value for a sweep parameter."""
	152	if parameter.values is not None:
	153	return parameter.values[generator.randrange(len(parameter.values))]
	154	if parameter.minimum is None or parameter.maximum is None:
	155	raise config_values.ConfigError(
	156	f"{where} requires minimum and maximum for sampling"
	157	)
	158	if parameter.distribution == "uniform":
	159	drawn = generator.uniform(parameter.minimum, parameter.maximum)
	160	elif parameter.distribution == "log_uniform":
	161	if parameter.minimum <= 0.0:
	162	raise config_values.ConfigError(
	163	f"{where}.minimum must be positive for log_uniform"
	164	)
	165	drawn = math.exp(
	166	generator.uniform(math.log(parameter.minimum), math.log(parameter.maximum))
	167	)
	168	else:
	169	raise config_values.ConfigError(
	170	f"{where}.distribution {parameter.distribution!r} is not implemented; "
	171	"supported distributions are uniform and log_uniform"
	172	)
	173	return float(f"{drawn:.6g}")
	174
	175
	176	def cell_id(overrides: Mapping[str, config_values.OverrideValue]) -> str:
	177	"""Derive a stable, filesystem-safe identity from a cell's overrides."""
	178	if not overrides:
	179	raise config_values.ConfigError("A study cell requires at least one override")
	180	names = [path.rsplit(".", 1)[-1] for path in overrides]
	181	if len(set(names)) != len(names):
	182	names = [path.replace(".", "_") for path in overrides]
	183	return "__".join(
	184	f"{name}-{_value_token(value)}"
	185	for name, value in zip(names, overrides.values(), strict=True)
	186	)
	187
	188
	189	def _value_token(value: config_values.OverrideValue) -> str:
	190	"""Render one override value as a filesystem-safe token."""
	191	if isinstance(value, bool):
	192	text = "true" if value else "false"
	193	elif value is None:
	194	text = "null"
	195	elif isinstance(value, float):
	196	text = repr(value)
	197	elif isinstance(value, list):
	198	text = "+".join(_value_token(item) for item in value)
	199	else:
	200	text = str(value)
	201	return re.sub(r"[^A-Za-z0-9._+-]", "_", text)
	202
	203
	204	def apply_overrides(
	205	raw: Mapping[str, Any],
	206	overrides: Mapping[str, config_values.OverrideValue],
	207	where: str,
	208	) -> dict[str, Any]:
	209	"""Apply dotted-path overrides to a raw configuration mapping.
	210
	211	Args:
	212	raw: Raw mapping of the base configuration.
	213	overrides: Dotted leaf paths mapped to their replacement values.
	214	where: Cell identification used in error messages.
	215
	216	Returns:
	217	A deep copy of ``raw`` carrying the overridden leaves.
	218
	219	Raises:
	220	ConfigError: If a path is not a declared leaf or the value type differs.
	221	"""
	222	result = copy.deepcopy(dict(raw))
	223	leaves = config_values.leaf_values(raw)
	224	for path in sorted(overrides):
	225	value = overrides[path]
	226	if path.startswith("study.") or path not in leaves:
	227	raise config_values.ConfigError(
	228	f"{where} override path {path!r} is not a declared scalar/list leaf"
	229	)
	230	expected = leaves[path]
	231	if not config_values.same_leaf_type(expected, value):
	232	raise config_values.ConfigError(
	233	f"{where} override {path!r} has incompatible value {value!r}; "
	234	f"expected type of {expected!r}"
	235	)
	236	_set_leaf(result, path, config_values.coerce_leaf(expected, value), where)
	237	return result
	238
	239
	240	def _set_leaf(
	241	target: dict[str, Any],
	242	path: str,
	243	value: config_values.OverrideValue,
	244	where: str,
	245	) -> None:
	246	"""Replace one addressable dotted leaf of a raw mapping in place."""
	247	segments = path.split(".")
	248	node: Any = target
	249	for segment in segments[:-1]:
	250	if not isinstance(node, dict) or segment not in node:
	251	raise config_values.ConfigError(
	252	f"{where} override path {path!r} is not addressable"
	253	)
	254	node = node[segment]
	255	if not isinstance(node, dict) or segments[-1] not in node:
	256	raise config_values.ConfigError(
	257	f"{where} override path {path!r} is not addressable"
	258	)
	259	node[segments[-1]] = value
0

Importance #59: src/train/config_values.py @@ -0,0 +1,256 @@

	1	"""Strict YAML value typing shared by the experiment configuration models.
	2
	3	The experiment YAML is a fail-closed contract: a value must already carry its
	4	declared type, so ``"50"`` is not a float, ``1`` is not a boolean, and ``3.0``
	5	is not an integer. That is deliberately stricter than the fleet coercion
	6	matrix of :func:`iolabs.common.config_loader.coerce_config_value`, so the
	7	models in :mod:`src.train.config_schema` route every field through
	8	:func:`typed_value` instead of the inherited coercion.
	9	"""
	10
	11	from __future__ import annotations
	12
	13	import logging
	14	from collections.abc import Mapping, Sequence
	15	from pathlib import Path
	16	from types import MappingProxyType, UnionType
	17	from typing import Any, Literal, TypeAlias, Union, get_args, get_origin
	18
	19	from iolabs.common import config_loader
	20
	21	logger = logging.getLogger(__name__)
	22
	23	Scalar: TypeAlias = str \| int \| float \| bool \| None
	24	OverrideValue: TypeAlias = Scalar \| list[Scalar]
	25
	26
	27	class ConfigError(config_loader.ConfigError):
	28	"""Raised when an experiment configuration violates its strict schema."""
	29
	30
	31	def typed_value(annotation: Any, value: Any, where: str) -> Any:
	32	"""Parse one config leaf strictly as its declared annotation.
	33
	34	Nested models, ``Annotated`` aliases and unresolved annotations are
	35	returned unchanged so pydantic validates them itself.
	36
	37	Args:
	38	annotation: Resolved field annotation, or None when unknown.
	39	value: Raw YAML value.
	40	where: Dotted key path used in error messages.
	41
	42	Returns:
	43	The value converted to the declared type.
	44
	45	Raises:
	46	ConfigError: If the value does not match the declared type.
	47	"""
	48	if annotation is None or annotation is Any:
	49	return value
	50	origin = get_origin(annotation)
	51	if origin is Literal:
	52	return choice(value, {str(item) for item in get_args(annotation)}, where)
	53	if origin in (Union, UnionType):
	54	return _typed_union(annotation, value, where)
	55	if annotation is Path:
	56	return value if isinstance(value, Path) else path(value, where)
	57	if annotation is bool:
	58	return boolean(value, where)
	59	if annotation is int:
	60	return integer(value, where)
	61	if annotation is float:
	62	return number(value, where)
	63	if annotation is str:
	64	return string(value, where)
	65	if origin in (tuple, list):
	66	args = get_args(annotation)
	67	item_type = args[0] if args else None
	68	items = [typed_value(item_type, item, where) for item in sequence(value, where)]
	69	return tuple(items) if origin is tuple else items
	70	if origin in (dict, Mapping) or (
	71	isinstance(origin, type) and issubclass(origin, Mapping)
	72	):
	73	args = get_args(annotation)
	74	item_type = args[1] if len(args) == 2 else None
	75	raw = mapping(value, where)
	76	return MappingProxyType(
	77	{
	78	string(key, where): typed_value(item_type, item, f"{where}.{key}")
	79	for key, item in raw.items()
	80	}
	81	)
	82	return value
	83
	84
	85	def _typed_union(annotation: Any, value: Any, where: str) -> Any:
	86	"""Parse a value against a union annotation, rejecting every mismatch."""
	87	members = get_args(annotation)
	88	optional = type(None) in members
	89	if value is None:
	90	if optional:
	91	return None
	92	raise ConfigError(f"{where} must not be null")
	93	candidates = [item for item in members if item is not type(None)]
	94	if len(candidates) == 1:
	95	return typed_value(candidates[0], value, where)
	96	for candidate in candidates:
	97	try:
	98	return typed_value(candidate, value, where)
	99	except ConfigError:
	100	continue
	101	names = sorted(getattr(item, "__name__", str(item)) for item in candidates)
	102	raise ConfigError(
	103	f"{where} must be one of {names}, got {type(value).__name__} {value!r}"
	104	)
	105
	106
	107	def mapping(value: Any, where: str) -> Mapping[str, Any]:
	108	"""Return value as a string-keyed mapping or raise."""
	109	if not isinstance(value, Mapping):
	110	raise ConfigError(f"{where} must be a mapping")
	111	if any(not isinstance(key, str) for key in value):
	112	raise ConfigError(f"{where} keys must be strings")
	113	return value
	114
	115
	116	def sequence(value: Any, where: str) -> Sequence[Any]:
	117	"""Return value as a non-string sequence or raise."""
	118	if isinstance(value, (str, bytes)) or not isinstance(value, Sequence):
	119	raise ConfigError(f"{where} must be a sequence")
	120	return value
	121
	122
	123	def string(value: Any, where: str) -> str:
	124	"""Return value as a non-empty string or raise."""
	125	if not isinstance(value, str) or not value.strip():
	126	raise ConfigError(f"{where} must be a non-empty string")
	127	return value
	128
	129
	130	def integer(value: Any, where: str) -> int:
	131	"""Return value as an integer, rejecting booleans, or raise."""
	132	if isinstance(value, bool) or not isinstance(value, int):
	133	raise ConfigError(f"{where} must be an integer")
	134	return value
	135
	136
	137	def number(value: Any, where: str) -> float:
	138	"""Return value as a float, rejecting booleans, or raise."""
	139	if isinstance(value, bool) or not isinstance(value, (int, float)):
	140	raise ConfigError(f"{where} must be numeric")
	141	return float(value)
	142
	143
	144	def boolean(value: Any, where: str) -> bool:
	145	"""Return value as a boolean or raise."""
	146	if not isinstance(value, bool):
	147	raise ConfigError(f"{where} must be boolean")
	148	return value
	149
	150
	151	def path(value: Any, where: str) -> Path:
	152	"""Return value as a repository-relative path or raise."""
	153	text = string(value, where)
	154	parsed = Path(text)
	155	if parsed.is_absolute():
	156	raise ConfigError(f"{where} must be repository-relative, got {parsed}")
	157	return parsed
	158
	159
	160	def choice(value: Any, choices: set[str], where: str) -> str:
	161	"""Return value when it is one of the allowed string choices."""
	162	text = string(value, where)
	163	if text not in choices:
	164	raise ConfigError(f"{where} must be one of {sorted(choices)}, got {text!r}")
	165	return text
	166
	167
	168	def check_keys(
	169	value: Mapping[str, Any],
	170	required: set[str],
	171	allowed: set[str],
	172	where: str,
	173	) -> None:
	174	"""Raise when value misses a required key or carries an unknown one.
	175
	176	Args:
	177	value: Free-form mapping whose keys are not model fields.
	178	required: Keys that must be present.
	179	allowed: Keys that may be present.
	180	where: Dotted key path used in error messages.
	181
	182	Raises:
	183	ConfigError: If a key is missing or unknown.
	184	"""
	185	missing = sorted(required - set(value))
	186	unknown = sorted(set(value) - allowed)
	187	if missing or unknown:
	188	details = []
	189	if missing:
	190	details.append(f"missing {missing}")
	191	if unknown:
	192	details.append(f"unknown {unknown}")
	193	raise ConfigError(f"{where} has " + " and ".join(details))
	194
	195
	196	def leaf_values(value: Any, prefix: str = "") -> dict[str, OverrideValue]:
	197	"""Return every dotted scalar/list leaf of a raw configuration mapping.
	198
	199	The ``study`` block is skipped: a study never overrides itself.
	200
	201	Args:
	202	value: Raw mapping, sequence, or scalar.
	203	prefix: Dotted path of value inside the document.
	204
	205	Returns:
	206	Mapping of dotted leaf path to its declared value.
	207	"""
	208	result: dict[str, OverrideValue] = {}
	209	if isinstance(value, Mapping):
	210	for key, item in value.items():
	211	dotted = f"{prefix}.{key}" if prefix else str(key)
	212	if dotted == "study" or dotted.startswith("study."):
	213	continue
	214	result.update(leaf_values(item, dotted))
	215	elif isinstance(value, list):
	216	if all(
	217	isinstance(item, (str, int, float, bool)) or item is None
	218	for item in value
	219	):
	220	result[prefix] = value
	221	elif isinstance(value, (str, int, float, bool)) or value is None:
	222	result[prefix] = value
	223	return result
	224
	225
	226	def same_leaf_type(expected: OverrideValue, actual: OverrideValue) -> bool:
	227	"""Return whether actual may replace the declared leaf expected."""
	228	if isinstance(expected, list):
	229	if not isinstance(actual, list):
	230	return False
	231	if not expected or not actual:
	232	return True
	233	return all(same_leaf_type(expected[0], item) for item in actual)
	234	if isinstance(expected, bool):
	235	return isinstance(actual, bool)
	236	if isinstance(expected, int) and not isinstance(expected, bool):
	237	return isinstance(actual, int) and not isinstance(actual, bool)
	238	if isinstance(expected, float):
	239	return isinstance(actual, (int, float)) and not isinstance(actual, bool)
	240	return actual is None if expected is None else isinstance(actual, type(expected))
	241
	242
	243	def coerce_leaf(expected: OverrideValue, value: OverrideValue) -> OverrideValue:
	244	"""Parse an override value as the declared leaf's type."""
	245	if isinstance(expected, list):
	246	items = list(value) if isinstance(value, list) else [value]
	247	if expected and isinstance(expected[0], float):
	248	return [coerce_leaf(expected[0], item) for item in items]
	249	return items
	250	if (
	251	isinstance(expected, float)
	252	and isinstance(value, int)
	253	and not isinstance(value, bool)
	254	):
	255	return float(value)
	256	return value
0

Importance #60: tests/test_checkpoint_seam.py @@ -16,9 +16,8 @@

16	"""	16	"""
17		17
18	from __future__ import annotations	18	from __future__ import annotations
19		19
20	import dataclasses
21	import hashlib	20	import hashlib
22	import importlib.util	21	import importlib.util
23	import json	22	import json
24	import sys	23	import sys

Importance #61: tests/test_checkpoint_seam.py @@ -154,12 +153,10 @@

         (canonical_root / "canonical" / f"{corridor_id}.adapter.json").write_text(
             json.dumps({"tiling": {"grid_origin_xyz": GRID_ORIGIN}}),
             encoding="utf-8",
         )
-    return dataclasses.replace(
+    data = config.data.model_copy(update={"canonical_root": canonical_root})
-        config,
+    return config.model_copy(update={"data": data})
-        data=dataclasses.replace(config.data, canonical_root=canonical_root),
-    )
 def _training_run(
     tmp_path: Path,

Importance #62: tests/test_framework_run_provenance.py @@ -1,9 +1,8 @@

1	"""Training-run provenance for framework runners (spec section 9.7)."""	1	"""Training-run provenance for framework runners (spec section 9.7)."""
2		2
3	from __future__ import annotations	3	from __future__ import annotations
4		4
5	import dataclasses
6	import json	5	import json
7	import sys	6	import sys
8	from pathlib import Path	7	from pathlib import Path
9	from types import ModuleType	8	from types import ModuleType

Importance #63: tests/test_framework_run_provenance.py @@ -68,12 +67,10 @@

         (canonical_root / "canonical" / f"{corridor_id}.adapter.json").write_text(
             json.dumps({"tiling": {"grid_origin_xyz": GRID_ORIGIN}}),
             encoding="utf-8",
         )
-    return dataclasses.replace(
+    data = config.data.model_copy(update={"canonical_root": canonical_root})
-        config,
+    return config.model_copy(update={"data": data})
-        data=dataclasses.replace(config.data, canonical_root=canonical_root),
-    )
 def _records_and_manifest(config: HarnessConfig) -> tuple[Any, Any]:
     """Authorize the config's corridors for a training run."""