Miroslav Simko <ms@iolabs.ch> 2026-09-02T09:40:19+02:00
Commit #74 ยท 271 snippets
CLAUDE.md | 2 +- README.md | 21 +++++ frameworks/emission_check.py | 8 +- frameworks/run_provenance.py | 4 +- frameworks/runner_options.py | 11 +-- scripts/evaluate.py | 13 ++- scripts/ingest_preannotations.py | 4 +- scripts/prepare_dataset.py | 4 +- scripts/train.py | 6 +- scripts/verify_adapter_roundtrip.py | 4 +- scripts/voxel_oracle.py | 7 +- src/train/__init__.py | 5 +- src/train/config.py | 71 ++++++++------- src/train/config_rules.py | 62 ++++++------- src/train/config_schema.py | 30 ++++--- src/train/config_sections.py | 58 ++++++++++--- src/train/config_study.py | 41 +++++---- src/train/config_values.py | 48 ++++++----- src/train/dispatch.py | 4 +- tests/test_a1_recap_wiring.py | 4 +- tests/test_checkpoint_seam.py | 4 +- tests/test_config.py | 153 ++++++++++++++++++--------------- tests/test_dispatch_cells.py | 19 ++-- tests/test_dispatch_environment.py | 6 +- tests/test_framework_run_provenance.py | 4 +- tests/test_study_expansion.py | 38 ++++---- tests/test_voxel_oracle.py | 20 ++--- 27 files changed, 379 insertions(+), 272 deletions(-)
| 126 | raw: Raw mapping the configuration was parsed from, used to check the | 126 | raw: Raw mapping the configuration was parsed from, used to check the |
| 127 | declared study overrides against the document's own leaves. | 127 | declared study overrides against the document's own leaves. |
| 128 | 128 | ||
| 129 | Raises: | 129 | Raises: |
| 130 | ConfigError: If a cross-section, identity, or ontology rule fails. | 130 | ExperimentConfigError: If a cross-section, identity, or ontology rule fails. |
| 131 | """ | 131 | """ |
| 132 | if config.schema_version != 1: | 132 | if config.schema_version != 1: |
| 133 | raise config_values.ConfigError( | 133 | raise config_values.ExperimentConfigError( |
| 134 | f"Unsupported schema_version {config.schema_version}" | 134 | f"Unsupported schema_version {config.schema_version}" |
| 135 | ) | 135 | ) |
| 136 | if not _EXPERIMENT_ID_PATTERN.fullmatch(config.experiment.id): | 136 | if not _EXPERIMENT_ID_PATTERN.fullmatch(config.experiment.id): |
| 137 | raise config_values.ConfigError( | 137 | raise config_values.ExperimentConfigError( |
| 138 | f"Invalid experiment.id {config.experiment.id!r}" | 138 | f"Invalid experiment.id {config.experiment.id!r}" |
| 139 | ) | 139 | ) |
| 140 | ontology = _load_task_ontology(config) | 140 | ontology = _load_task_ontology(config) |
| 141 | _validate_task_against_ontology(config, ontology) | 141 | _validate_task_against_ontology(config, ontology) |
| 212 | unknown = sorted( | 212 | unknown = sorted( |
| 213 | set(gate.params.minimum_purity_by_class) - set(ontology.class_names) | 213 | set(gate.params.minimum_purity_by_class) - set(ontology.class_names) |
| 214 | ) | 214 | ) |
| 215 | if unknown: | 215 | if unknown: |
| 216 | raise config_values.ConfigError( | 216 | raise config_values.ExperimentConfigError( |
| 217 | f"gate {gate.name}.minimum_purity_by_class contains unknown " | 217 | f"gate {gate.name}.minimum_purity_by_class contains unknown " |
| 218 | f"ontology classes {unknown} for ontology {ontology.name}" | 218 | f"ontology classes {unknown} for ontology {ontology.name}" |
| 219 | ) | 219 | ) |
| 220 | gate_types = {gate.type for gate in config.experiment.gates if gate.required} | 220 | gate_types = {gate.type for gate in config.experiment.gates if gate.required} |
| 221 | if ( | 221 | if ( |
| 222 | config.experiment.status == "template-only" | 222 | config.experiment.status == "template-only" |
| 223 | and "implementation_ticket" not in gate_types | 223 | and "implementation_ticket" not in gate_types |
| 224 | ): | 224 | ): |
| 225 | raise config_values.ConfigError( | 225 | raise config_values.ExperimentConfigError( |
| 226 | "template-only experiments require an implementation_ticket gate" | 226 | "template-only experiments require an implementation_ticket gate" |
| 227 | ) | 227 | ) |
| 228 | if config.experiment.status == "gated-later" and not gate_types: | 228 | if config.experiment.status == "gated-later" and not gate_types: |
| 229 | raise config_values.ConfigError( | 229 | raise config_values.ExperimentConfigError( |
| 230 | "gated-later experiments require at least one required gate" | 230 | "gated-later experiments require at least one required gate" |
| 231 | ) | 231 | ) |
| 232 | if config.experiment.status == "implement-now" and config.experiment.id not in { | 232 | if config.experiment.status == "implement-now" and config.experiment.id not in { |
| 233 | "E1", | 233 | "E1", |
| 234 | "E2", | 234 | "E2", |
| 235 | }: | 235 | }: |
| 236 | raise config_values.ConfigError( | 236 | raise config_values.ExperimentConfigError( |
| 237 | "Only E1 and E2 are implement-now in schema version 1" | 237 | "Only E1 and E2 are implement-now in schema version 1" |
| 238 | ) | 238 | ) |
| 239 | 239 | ||
| 240 | 240 | ||
| 241 | def _load_task_ontology(config: config_sections.HarnessConfig) -> Ontology: | 241 | def _load_task_ontology(config: config_sections.HarnessConfig) -> Ontology: |
| 242 | """Load the ontology the config declares, failing closed as a ConfigError. | 242 | """Load the ontology the config declares, failing closed as a ExperimentConfigError. |
| 243 | 243 | ||
| 244 | Args: | 244 | Args: |
| 245 | config: Parsed configuration whose ``task.ontology`` path is resolved | 245 | config: Parsed configuration whose ``task.ontology`` path is resolved |
| 246 | relative to the repository root. | 246 | relative to the repository root. |
| 154 | @classmethod | 156 | @classmethod |
| 155 | def _emit_is_supported(cls, value: tuple[str, ...]) -> tuple[str, ...]: | 157 | def _emit_is_supported(cls, value: tuple[str, ...]) -> tuple[str, ...]: |
| 156 | """Reject unknown output formats and a missing canonical emission.""" | 158 | """Reject unknown output formats and a missing canonical emission.""" |
| 157 | if set(value) - {"canonical", "spt", "pointcept"}: | 159 | if set(value) - {"canonical", "spt", "pointcept"}: |
| 158 | raise config_values.ConfigError( | 160 | raise config_values.ExperimentConfigError( |
| 159 | "adapter.emit contains an unsupported output format" | 161 | "adapter.emit contains an unsupported output format" |
| 160 | ) | 162 | ) |
| 161 | if "canonical" not in value: | 163 | if "canonical" not in value: |
| 162 | raise config_values.ConfigError("adapter.emit must include canonical") | 164 | raise config_values.ExperimentConfigError( |
| 165 | "adapter.emit must include canonical" | ||
| 166 | ) | ||
| 163 | return value | 167 | return value |
| 164 | 168 | ||
| 165 | @pydantic.field_validator("canonical_version") | 169 | @pydantic.field_validator("canonical_version") |
| 166 | @classmethod | 170 | @classmethod |
| 167 | def _canonical_version_is_one(cls, value: int) -> int: | 171 | def _canonical_version_is_one(cls, value: int) -> int: |
| 168 | """Freeze the canonical dataset version at 1.""" | 172 | """Freeze the canonical dataset version at 1.""" |
| 169 | if value != 1: | 173 | if value != 1: |
| 170 | raise config_values.ConfigError("adapter.canonical_version must be 1") | 174 | raise config_values.ExperimentConfigError( |
| 175 | "adapter.canonical_version must be 1" | ||
| 176 | ) | ||
| 171 | return value | 177 | return value |
| 172 | 178 | ||
| 173 | 179 | ||
| 174 | class ModelConfig(config_schema.StrictConfigModel): | 180 | class ModelConfig(config_schema.StrictConfigModel): |
| 1 | """Strict experiment configuration schema for corridor segmentation studies. | 1 | """Strict experiment configuration schema for corridor segmentation studies. |
| 2 | 2 | ||
| 3 | The schema itself is a pydantic model tree built on | 3 | The schema is a pydantic model tree on |
| 4 | :class:`iolabs.common.config_loader.ConfigModel`: :mod:`src.train.config_schema` | 4 | :class:`iolabs.common.config_loader.ConfigModel`, split by section because it |
| 5 | holds the experiment/gate/study models, :mod:`src.train.config_sections` the | 5 | exceeds one module: :mod:`src.train.config_schema` holds the experiment, gate, |
| 6 | data, model, training, and evaluation blocks. This module is the entry point | 6 | and study models, :mod:`src.train.config_sections` the data, model, training, |
| 7 | every script imports: it loads one YAML document, validates it, applies the | 7 | and evaluation blocks, :mod:`src.train.config_values` the strict YAML leaf |
| 8 | rules of :mod:`src.train.config_rules`, and expands a study into re-validated | 8 | typing, :mod:`src.train.config_rules` the cross-section rules, and |
| 9 | cells with :mod:`src.train.config_study`. | 9 | :mod:`src.train.config_study` the study algebra. This module is the entry point |
| 10 | 10 | every script imports: :meth:`HarnessConfig.from_yaml` loads and validates one | |
| 11 | Adding a configuration key means adding a field to its model (and to the YAML | 11 | YAML document, :func:`expand_study` expands a study into re-validated cells, |
| 12 | documents under ``configs/``); nothing else has to be touched. | 12 | and both raise :class:`ExperimentConfigError`. |
| 13 | |||
| 14 | Adding a configuration key means adding the field to its model and the same key | ||
| 15 | to the YAML documents under ``configs/`` -- nothing else. Unknown keys are | ||
| 16 | rejected. | ||
| 13 | """ | 17 | """ |
| 14 | 18 | ||
| 15 | from __future__ import annotations | 19 | from __future__ import annotations |
| 16 | 20 |
| 85 | TilingConfig, | 89 | TilingConfig, |
| 86 | TrainConfig, | 90 | TrainConfig, |
| 87 | VisualizationConfig, | 91 | VisualizationConfig, |
| 88 | ) | 92 | ) |
| 89 | from src.train.config_values import ConfigError, OverrideValue, Scalar | 93 | from src.train.config_values import ExperimentConfigError, OverrideValue, Scalar |
| 90 | 94 | ||
| 91 | logger = logging.getLogger(__name__) | 95 | logger = logging.getLogger(__name__) |
| 92 | 96 | ||
| 93 | _CONTEXT = "experiment config" | 97 | _CONTEXT = "experiment config" |
| 98 | "ArtifactExistsParams", | 102 | "ArtifactExistsParams", |
| 99 | "BootstrapConfig", | 103 | "BootstrapConfig", |
| 100 | "CheckpointPolicyGate", | 104 | "CheckpointPolicyGate", |
| 101 | "CheckpointPolicyParams", | 105 | "CheckpointPolicyParams", |
| 102 | "ConfigError", | ||
| 103 | "ContinuityConfig", | 106 | "ContinuityConfig", |
| 104 | "ContrastConfig", | 107 | "ContrastConfig", |
| 105 | "CorridorSelectionConfig", | 108 | "CorridorSelectionConfig", |
| 106 | "DataAvailableGate", | 109 | "DataAvailableGate", |
| 107 | "DataAvailableParams", | 110 | "DataAvailableParams", |
| 108 | "DataConfig", | 111 | "DataConfig", |
| 109 | "EvaluationConfig", | 112 | "EvaluationConfig", |
| 110 | "ExperimentConfig", | 113 | "ExperimentConfig", |
| 114 | "ExperimentConfigError", | ||
| 111 | "ExperimentStatus", | 115 | "ExperimentStatus", |
| 112 | "FeatureConfig", | 116 | "FeatureConfig", |
| 113 | "GateBase", | 117 | "GateBase", |
| 114 | "GateConfig", | 118 | "GateConfig", |
| 148 | "VariantConfig", | 152 | "VariantConfig", |
| 149 | "VisualizationConfig", | 153 | "VisualizationConfig", |
| 150 | "expand_study", | 154 | "expand_study", |
| 151 | "is_canonical_metric", | 155 | "is_canonical_metric", |
| 152 | "load_config", | ||
| 153 | "resolve_ontology_path", | 156 | "resolve_ontology_path", |
| 154 | ] | 157 | ] |
| 155 | 158 | ||
| 156 | 159 |
| 174 | document: str | 177 | document: str |
| 175 | config: HarnessConfig | 178 | config: HarnessConfig |
| 176 | 179 | ||
| 177 | 180 | ||
| 178 | def load_config(path: str | Path) -> HarnessConfig: | 181 | def _load_yaml_document(path: str | Path) -> HarnessConfig: |
| 179 | """Load and strictly validate one E1--E14 experiment YAML. | 182 | """Read, strictly validate, and rule-check one experiment YAML. |
| 183 | |||
| 184 | Implementation of :meth:`HarnessConfig.from_yaml`, which is the public | ||
| 185 | entry point; it lives here because the loading pipeline needs | ||
| 186 | :mod:`src.train.config_rules` and :mod:`src.train.config_study`. | ||
| 180 | 187 | ||
| 181 | Args: | 188 | Args: |
| 182 | path: Repository-relative or absolute YAML path. | 189 | path: Repository-relative or absolute YAML path. |
| 183 | 190 | ||
| 184 | Returns: | 191 | Returns: |
| 185 | Fully typed immutable configuration. | 192 | Fully typed immutable configuration. |
| 186 | 193 | ||
| 187 | Raises: | 194 | Raises: |
| 188 | ConfigError: If loading, strict parsing, or validation fails. | 195 | ExperimentConfigError: If loading, strict parsing, or validation fails. |
| 189 | """ | 196 | """ |
| 190 | config_path = Path(path) | 197 | config_path = Path(path) |
| 191 | try: | 198 | try: |
| 192 | payload = config_path.read_bytes() | 199 | payload = config_path.read_bytes() |
| 193 | raw = yaml.safe_load(payload.decode("utf-8")) | 200 | raw = yaml.safe_load(payload.decode("utf-8")) |
| 194 | except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: | 201 | except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: |
| 195 | raise ConfigError(f"Cannot load config {config_path}: {exc}") from exc | 202 | raise ExperimentConfigError( |
| 203 | f"Cannot load config {config_path}: {exc}" | ||
| 204 | ) from exc | ||
| 196 | try: | 205 | try: |
| 197 | return _build_config(raw, config_path, hashlib.sha256(payload).hexdigest()) | 206 | return _build_config(raw, config_path, hashlib.sha256(payload).hexdigest()) |
| 198 | except ConfigError as exc: | 207 | except ExperimentConfigError as exc: |
| 199 | raise ConfigError(f"{config_path}: {exc}") from exc | 208 | raise ExperimentConfigError(f"{config_path}: {exc}") from exc |
| 200 | 209 | ||
| 201 | 210 | ||
| 202 | def expand_study(config: HarnessConfig) -> tuple[StudyCell, ...]: | 211 | def expand_study(config: HarnessConfig) -> tuple[StudyCell, ...]: |
| 203 | """Expand a typed study definition into ordered, re-validated cells. | 212 | """Expand a typed study definition into ordered, re-validated cells. |
| 206 | per declared variant, ``matrix`` the deterministic cross product of its | 215 | per declared variant, ``matrix`` the deterministic cross product of its |
| 207 | axes after ``include``/``exclude``, and ``sweep`` the deterministic | 216 | axes after ``include``/``exclude``, and ``sweep`` the deterministic |
| 208 | enumeration of its typed parameters under the study ``seed``. Every cell's | 217 | enumeration of its typed parameters under the study ``seed``. Every cell's |
| 209 | overrides are re-applied to the raw mapping and re-validated through | 218 | overrides are re-applied to the raw mapping and re-validated through |
| 210 | :func:`load_config`'s machinery, so each cell carries its own SHA-256. | 219 | :meth:`HarnessConfig.from_yaml`'s machinery, so each cell carries its own SHA-256. |
| 211 | 220 | ||
| 212 | Args: | 221 | Args: |
| 213 | config: Strictly parsed experiment configuration. | 222 | config: Strictly parsed experiment configuration. |
| 214 | 223 | ||
| 215 | Returns: | 224 | Returns: |
| 216 | Deterministically ordered study cells, never empty. | 225 | Deterministically ordered study cells, never empty. |
| 217 | 226 | ||
| 218 | Raises: | 227 | Raises: |
| 219 | ConfigError: If the study payload, an override, a cell identity, or a | 228 | ExperimentConfigError: If the study payload, an override, a cell identity, or a |
| 220 | resolved cell configuration violates the strict schema. | 229 | resolved cell configuration violates the strict schema. |
| 221 | """ | 230 | """ |
| 222 | raw = _raw_document(config) | 231 | raw = _raw_document(config) |
| 223 | definitions = config_study.cell_definitions(config) | 232 | definitions = config_study.cell_definitions(config) |
| 224 | if not definitions: | 233 | if not definitions: |
| 225 | raise ConfigError( | 234 | raise ExperimentConfigError( |
| 226 | f"{config.experiment.id} study kind {config.study.kind} expanded to " | 235 | f"{config.experiment.id} study kind {config.study.kind} expanded to " |
| 227 | "no cells" | 236 | "no cells" |
| 228 | ) | 237 | ) |
| 229 | identities = [cell_id for cell_id, _ in definitions] | 238 | identities = [cell_id for cell_id, _ in definitions] |
| 230 | duplicates = sorted({item for item in identities if identities.count(item) > 1}) | 239 | duplicates = sorted({item for item in identities if identities.count(item) > 1}) |
| 231 | if duplicates: | 240 | if duplicates: |
| 232 | raise ConfigError( | 241 | raise ExperimentConfigError( |
| 233 | f"{config.experiment.id} study has duplicate cell IDs {duplicates}" | 242 | f"{config.experiment.id} study has duplicate cell IDs {duplicates}" |
| 234 | ) | 243 | ) |
| 235 | cells: list[StudyCell] = [] | 244 | cells: list[StudyCell] = [] |
| 236 | for index, (cell_id, overrides) in enumerate(definitions): | 245 | for index, (cell_id, overrides) in enumerate(definitions): |
| 239 | document = yaml.safe_dump(cell_raw, sort_keys=True, default_flow_style=False) | 248 | document = yaml.safe_dump(cell_raw, sort_keys=True, default_flow_style=False) |
| 240 | digest = hashlib.sha256(document.encode("utf-8")).hexdigest() | 249 | digest = hashlib.sha256(document.encode("utf-8")).hexdigest() |
| 241 | try: | 250 | try: |
| 242 | cell_config = _build_config(cell_raw, config.source_path, digest) | 251 | cell_config = _build_config(cell_raw, config.source_path, digest) |
| 243 | except ConfigError as exc: | 252 | except ExperimentConfigError as exc: |
| 244 | raise ConfigError(f"{config.source_path} cell {cell_id!r}: {exc}") from exc | 253 | raise ExperimentConfigError( |
| 254 | f"{config.source_path} cell {cell_id!r}: {exc}" | ||
| 255 | ) from exc | ||
| 245 | run_name = ( | 256 | run_name = ( |
| 246 | config.experiment.id | 257 | config.experiment.id |
| 247 | if config.study.kind == "single" | 258 | if config.study.kind == "single" |
| 248 | else f"{config.experiment.id}_{cell_id}" | 259 | else f"{config.experiment.id}_{cell_id}" |
| 263 | def _build_config(raw: Any, config_path: Path, sha256: str) -> HarnessConfig: | 274 | def _build_config(raw: Any, config_path: Path, sha256: str) -> HarnessConfig: |
| 264 | """Validate one raw document into a configuration carrying its digest.""" | 275 | """Validate one raw document into a configuration carrying its digest.""" |
| 265 | root = config_values.mapping(raw, str(config_path)) | 276 | root = config_values.mapping(raw, str(config_path)) |
| 266 | config = config_loader.validate_config( | 277 | config = config_loader.validate_config( |
| 267 | HarnessConfig, dict(root), context=_CONTEXT, error_cls=ConfigError | 278 | HarnessConfig, dict(root), context=_CONTEXT, error_cls=ExperimentConfigError |
| 268 | ) | 279 | ) |
| 269 | config.attach_provenance(config_path, sha256, copy.deepcopy(dict(root))) | 280 | config.attach_provenance(config_path, sha256, copy.deepcopy(dict(root))) |
| 270 | config_rules.validate_config(config, root) | 281 | config_rules.validate_config(config, root) |
| 271 | return config | 282 | return config |
| 274 | def _raw_document(config: HarnessConfig) -> Mapping[str, Any]: | 285 | def _raw_document(config: HarnessConfig) -> Mapping[str, Any]: |
| 275 | """Return the raw mapping a configuration was validated from. | 286 | """Return the raw mapping a configuration was validated from. |
| 276 | 287 | ||
| 277 | Args: | 288 | Args: |
| 278 | config: Configuration produced by :func:`load_config` or | 289 | config: Configuration produced by :meth:`HarnessConfig.from_yaml` |
| 279 | :func:`expand_study`. | 290 | or :func:`expand_study`. |
| 280 | 291 | ||
| 281 | Returns: | 292 | Returns: |
| 282 | The stashed raw mapping, or a fresh parse of ``config.source_path``. | 293 | The stashed raw mapping, or a fresh parse of ``config.source_path``. |
| 283 | 294 | ||
| 284 | Raises: | 295 | Raises: |
| 285 | ConfigError: If the source file must be re-read and cannot be parsed. | 296 | ExperimentConfigError: If the source file must be re-read and cannot be parsed. |
| 286 | """ | 297 | """ |
| 287 | stashed = config._raw_document | 298 | stashed = config._raw_document |
| 288 | if stashed is not None: | 299 | if stashed is not None: |
| 289 | return stashed | 300 | return stashed |
| 290 | try: | 301 | try: |
| 291 | payload = config.source_path.read_bytes() | 302 | payload = config.source_path.read_bytes() |
| 292 | raw = yaml.safe_load(payload.decode("utf-8")) | 303 | raw = yaml.safe_load(payload.decode("utf-8")) |
| 293 | except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: | 304 | except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: |
| 294 | raise ConfigError( | 305 | raise ExperimentConfigError( |
| 295 | f"Cannot re-read config {config.source_path}: {exc}" | 306 | f"Cannot re-read config {config.source_path}: {exc}" |
| 296 | ) from exc | 307 | ) from exc |
| 297 | return config_values.mapping(raw, str(config.source_path)) | 308 | return config_values.mapping(raw, str(config.source_path)) |
| 88 | Returns: | 88 | Returns: |
| 89 | An absolute, existing ontology path. | 89 | An absolute, existing ontology path. |
| 90 | 90 | ||
| 91 | Raises: | 91 | Raises: |
| 92 | ConfigError: If no candidate path exists. | 92 | ExperimentConfigError: If no candidate path exists. |
| 93 | """ | 93 | """ |
| 94 | declared = config.task.ontology | 94 | declared = config.task.ontology |
| 95 | if declared.is_absolute(): | 95 | if declared.is_absolute(): |
| 96 | if not declared.is_file(): | 96 | if not declared.is_file(): |
| 97 | raise config_values.ConfigError( | 97 | raise config_values.ExperimentConfigError( |
| 98 | f"{config.source_path}: task.ontology {declared.as_posix()} " | 98 | f"{config.source_path}: task.ontology {declared.as_posix()} " |
| 99 | f"does not exist" | 99 | f"does not exist" |
| 100 | ) | 100 | ) |
| 101 | return declared | 101 | return declared |
| 108 | return candidate.resolve() | 108 | return candidate.resolve() |
| 109 | searched = ", ".join( | 109 | searched = ", ".join( |
| 110 | sorted({candidate.parent.as_posix() for candidate in candidates}) | 110 | sorted({candidate.parent.as_posix() for candidate in candidates}) |
| 111 | ) | 111 | ) |
| 112 | raise config_values.ConfigError( | 112 | raise config_values.ExperimentConfigError( |
| 113 | f"{config.source_path}: task.ontology {declared.as_posix()} was not " | 113 | f"{config.source_path}: task.ontology {declared.as_posix()} was not " |
| 114 | f"found relative to any parent of the config or to the working " | 114 | f"found relative to any parent of the config or to the working " |
| 115 | f"directory {Path.cwd().as_posix()}; searched {searched}" | 115 | f"directory {Path.cwd().as_posix()}; searched {searched}" |
| 116 | ) | 116 | ) |
| 149 | """Check the rules that tie a section to the declared model framework.""" | 149 | """Check the rules that tie a section to the declared model framework.""" |
| 150 | if config.model.framework in {"spt", "pointcept"} and ( | 150 | if config.model.framework in {"spt", "pointcept"} and ( |
| 151 | _VIZ_TRAIN_KEYS & config.train.model_fields_set | 151 | _VIZ_TRAIN_KEYS & config.train.model_fields_set |
| 152 | ): | 152 | ): |
| 153 | raise config_values.ConfigError( | 153 | raise config_values.ExperimentConfigError( |
| 154 | "train.viz_every_n_epochs and train.viz_samples are forbidden for " | 154 | "train.viz_every_n_epochs and train.viz_samples are forbidden for " |
| 155 | "external frameworks" | 155 | "external frameworks" |
| 156 | ) | 156 | ) |
| 157 | if config.model.framework == "pointcept" and config.runtime.flash_attention: | 157 | if config.model.framework == "pointcept" and config.runtime.flash_attention: |
| 158 | raise config_values.ConfigError( | 158 | raise config_values.ExperimentConfigError( |
| 159 | "Pointcept PTv3/LitePT configurations must keep FlashAttention disabled" | 159 | "Pointcept PTv3/LitePT configurations must keep FlashAttention disabled" |
| 160 | ) | 160 | ) |
| 161 | if config.visualization is not None and config.model.framework != "pointcept": | 161 | if config.visualization is not None and config.model.framework != "pointcept": |
| 162 | raise config_values.ConfigError( | 162 | raise config_values.ExperimentConfigError( |
| 163 | "visualization is only supported for model.framework pointcept; " | 163 | "visualization is only supported for model.framework pointcept; " |
| 164 | f"{config.model.framework} configs must omit the block" | 164 | f"{config.model.framework} configs must omit the block" |
| 165 | ) | 165 | ) |
| 166 | 166 |
| 170 | ) -> None: | 170 | ) -> None: |
| 171 | """Check every declared metric tag against the ontology's namespace.""" | 171 | """Check every declared metric tag against the ontology's namespace.""" |
| 172 | for metric in config.experiment.deciding_metrics: | 172 | for metric in config.experiment.deciding_metrics: |
| 173 | if not is_canonical_metric(metric, ontology=ontology): | 173 | if not is_canonical_metric(metric, ontology=ontology): |
| 174 | raise config_values.ConfigError( | 174 | raise config_values.ExperimentConfigError( |
| 175 | "experiment.deciding_metrics contains non-canonical metric " | 175 | "experiment.deciding_metrics contains non-canonical metric " |
| 176 | f"{metric!r} for ontology {ontology.name}" | 176 | f"{metric!r} for ontology {ontology.name}" |
| 177 | ) | 177 | ) |
| 178 | for metric in (config.train.monitor, config.train.early_stop_monitor): | 178 | for metric in (config.train.monitor, config.train.early_stop_monitor): |
| 179 | if not is_canonical_metric(metric, ontology=ontology): | 179 | if not is_canonical_metric(metric, ontology=ontology): |
| 180 | raise config_values.ConfigError( | 180 | raise config_values.ExperimentConfigError( |
| 181 | f"train monitor {metric!r} is outside the canonical namespace " | 181 | f"train monitor {metric!r} is outside the canonical namespace " |
| 182 | f"of ontology {ontology.name}" | 182 | f"of ontology {ontology.name}" |
| 183 | ) | 183 | ) |
| 184 | unknown_floor_classes = sorted( | 184 | unknown_floor_classes = sorted( |
| 185 | set(config.evaluation.precision_floors) - set(ontology.class_names) | 185 | set(config.evaluation.precision_floors) - set(ontology.class_names) |
| 186 | ) | 186 | ) |
| 187 | if unknown_floor_classes: | 187 | if unknown_floor_classes: |
| 188 | raise config_values.ConfigError( | 188 | raise config_values.ExperimentConfigError( |
| 189 | "evaluation.precision_floors has unknown classes " | 189 | "evaluation.precision_floors has unknown classes " |
| 190 | f"{unknown_floor_classes} for ontology {ontology.name}" | 190 | f"{unknown_floor_classes} for ontology {ontology.name}" |
| 191 | ) | 191 | ) |
| 192 | for contrast in config.study.contrasts: | 192 | for contrast in config.study.contrasts: |
| 193 | if not is_canonical_metric(contrast.metric, ontology=ontology): | 193 | if not is_canonical_metric(contrast.metric, ontology=ontology): |
| 194 | raise config_values.ConfigError( | 194 | raise config_values.ExperimentConfigError( |
| 195 | f"study contrast metric {contrast.metric!r} is not canonical " | 195 | f"study contrast metric {contrast.metric!r} is not canonical " |
| 196 | f"for ontology {ontology.name}" | 196 | f"for ontology {ontology.name}" |
| 197 | ) | 197 | ) |
| 198 | if config.study.sweep is not None and not is_canonical_metric( | 198 | if config.study.sweep is not None and not is_canonical_metric( |
| 199 | config.study.sweep.objective, ontology=ontology | 199 | config.study.sweep.objective, ontology=ontology |
| 200 | ): | 200 | ): |
| 201 | raise config_values.ConfigError( | 201 | raise config_values.ExperimentConfigError( |
| 202 | f"study.sweep.objective must be canonical for ontology {ontology.name}" | 202 | f"study.sweep.objective must be canonical for ontology {ontology.name}" |
| 203 | ) | 203 | ) |
| 204 | 204 | ||
| 205 | 205 |
| 248 | Returns: | 248 | Returns: |
| 249 | The validated ontology every other contract is checked against. | 249 | The validated ontology every other contract is checked against. |
| 250 | 250 | ||
| 251 | Raises: | 251 | Raises: |
| 252 | ConfigError: If the ontology cannot be located, loaded, or is invalid. | 252 | ExperimentConfigError: If the ontology cannot be located, loaded, or is invalid. |
| 253 | """ | 253 | """ |
| 254 | resolved = resolve_ontology_path(config) | 254 | resolved = resolve_ontology_path(config) |
| 255 | try: | 255 | try: |
| 256 | return load_ontology(resolved) | 256 | return load_ontology(resolved) |
| 257 | except OntologyError as exc: | 257 | except OntologyError as exc: |
| 258 | raise config_values.ConfigError( | 258 | raise config_values.ExperimentConfigError( |
| 259 | f"task.ontology {config.task.ontology.as_posix()} is not a valid " | 259 | f"task.ontology {config.task.ontology.as_posix()} is not a valid " |
| 260 | f"ontology: {exc}" | 260 | f"ontology: {exc}" |
| 261 | ) from exc | 261 | ) from exc |
| 262 | 262 |
| 270 | config: Parsed configuration. | 270 | config: Parsed configuration. |
| 271 | ontology: Ontology loaded from ``task.ontology``. | 271 | ontology: Ontology loaded from ``task.ontology``. |
| 272 | 272 | ||
| 273 | Raises: | 273 | Raises: |
| 274 | ConfigError: If any task, evaluation, model, or loss class contract | 274 | ExperimentConfigError: If any task, evaluation, model, or loss class contract |
| 275 | disagrees with the loaded ontology. | 275 | disagrees with the loaded ontology. |
| 276 | """ | 276 | """ |
| 277 | task = config.task | 277 | task = config.task |
| 278 | if task.num_classes != ontology.num_predicted_classes: | 278 | if task.num_classes != ontology.num_predicted_classes: |
| 279 | raise config_values.ConfigError( | 279 | raise config_values.ExperimentConfigError( |
| 280 | f"task.num_classes {task.num_classes} must equal ontology " | 280 | f"task.num_classes {task.num_classes} must equal ontology " |
| 281 | f"{ontology.name} num_predicted_classes " | 281 | f"{ontology.name} num_predicted_classes " |
| 282 | f"{ontology.num_predicted_classes}" | 282 | f"{ontology.num_predicted_classes}" |
| 283 | ) | 283 | ) |
| 284 | if task.ignore_index != ontology.void_id: | 284 | if task.ignore_index != ontology.void_id: |
| 285 | raise config_values.ConfigError( | 285 | raise config_values.ExperimentConfigError( |
| 286 | f"task.ignore_index {task.ignore_index} must equal ontology " | 286 | f"task.ignore_index {task.ignore_index} must equal ontology " |
| 287 | f"{ontology.name} void ID {ontology.void_id}" | 287 | f"{ontology.name} void ID {ontology.void_id}" |
| 288 | ) | 288 | ) |
| 289 | if task.classes_of_interest != ontology.interest_ids: | 289 | if task.classes_of_interest != ontology.interest_ids: |
| 290 | raise config_values.ConfigError( | 290 | raise config_values.ExperimentConfigError( |
| 291 | f"task.classes_of_interest {list(task.classes_of_interest)} must " | 291 | f"task.classes_of_interest {list(task.classes_of_interest)} must " |
| 292 | f"equal ontology {ontology.name} interest IDs " | 292 | f"equal ontology {ontology.name} interest IDs " |
| 293 | f"{list(ontology.interest_ids)}" | 293 | f"{list(ontology.interest_ids)}" |
| 294 | ) | 294 | ) |
| 297 | ) | 297 | ) |
| 298 | if len(set(task.linear_classes)) != len(task.linear_classes) or set( | 298 | if len(set(task.linear_classes)) != len(task.linear_classes) or set( |
| 299 | task.linear_classes | 299 | task.linear_classes |
| 300 | ) != set(linear_names): | 300 | ) != set(linear_names): |
| 301 | raise config_values.ConfigError( | 301 | raise config_values.ExperimentConfigError( |
| 302 | f"task.linear_classes {list(task.linear_classes)} must be exactly " | 302 | f"task.linear_classes {list(task.linear_classes)} must be exactly " |
| 303 | f"the linear classes {list(linear_names)} of ontology " | 303 | f"the linear classes {list(linear_names)} of ontology " |
| 304 | f"{ontology.name}" | 304 | f"{ontology.name}" |
| 305 | ) | 305 | ) |
| 306 | profiles = config.evaluation.object_matching.cluster_profiles | 306 | profiles = config.evaluation.object_matching.cluster_profiles |
| 307 | if profiles != task.ontology: | 307 | if profiles != task.ontology: |
| 308 | raise config_values.ConfigError( | 308 | raise config_values.ExperimentConfigError( |
| 309 | "evaluation.object_matching.cluster_profiles " | 309 | "evaluation.object_matching.cluster_profiles " |
| 310 | f"{profiles.as_posix()} must be the task ontology " | 310 | f"{profiles.as_posix()} must be the task ontology " |
| 311 | f"{task.ontology.as_posix()}" | 311 | f"{task.ontology.as_posix()}" |
| 312 | ) | 312 | ) |
| 313 | if "num_classes" in config.model.args: | 313 | if "num_classes" in config.model.args: |
| 314 | declared = config.model.args["num_classes"] | 314 | declared = config.model.args["num_classes"] |
| 315 | if declared != task.num_classes: | 315 | if declared != task.num_classes: |
| 316 | raise config_values.ConfigError( | 316 | raise config_values.ExperimentConfigError( |
| 317 | f"model.args.num_classes {declared!r} must equal " | 317 | f"model.args.num_classes {declared!r} must equal " |
| 318 | f"task.num_classes {task.num_classes}" | 318 | f"task.num_classes {task.num_classes}" |
| 319 | ) | 319 | ) |
| 320 | if "ignore_index" in config.loss.args: | 320 | if "ignore_index" in config.loss.args: |
| 321 | declared = config.loss.args["ignore_index"] | 321 | declared = config.loss.args["ignore_index"] |
| 322 | if declared != task.ignore_index: | 322 | if declared != task.ignore_index: |
| 323 | raise config_values.ConfigError( | 323 | raise config_values.ExperimentConfigError( |
| 324 | f"loss.args.ignore_index {declared!r} must equal " | 324 | f"loss.args.ignore_index {declared!r} must equal " |
| 325 | f"task.ignore_index {task.ignore_index}" | 325 | f"task.ignore_index {task.ignore_index}" |
| 326 | ) | 326 | ) |
| 327 | 327 |
| 352 | ) | 352 | ) |
| 353 | for overrides in override_groups: | 353 | for overrides in override_groups: |
| 354 | for path, value in overrides.items(): | 354 | for path, value in overrides.items(): |
| 355 | if path.startswith("study.") or path not in leaves: | 355 | if path.startswith("study.") or path not in leaves: |
| 356 | raise config_values.ConfigError( | 356 | raise config_values.ExperimentConfigError( |
| 357 | f"Study override path {path!r} is not a declared scalar/list leaf" | 357 | f"Study override path {path!r} is not a declared scalar/list leaf" |
| 358 | ) | 358 | ) |
| 359 | expected = leaves[path] | 359 | expected = leaves[path] |
| 360 | if not config_values.same_leaf_type(expected, value): | 360 | if not config_values.same_leaf_type(expected, value): |
| 361 | raise config_values.ConfigError( | 361 | raise config_values.ExperimentConfigError( |
| 362 | f"Study override {path!r} has incompatible value {value!r}; " | 362 | f"Study override {path!r} has incompatible value {value!r}; " |
| 363 | f"expected type of {expected!r}" | 363 | f"expected type of {expected!r}" |
| 364 | ) | 364 | ) |
| 250 | ) -> Mapping[str, tuple[config_values.OverrideValue, ...]]: | 250 | ) -> Mapping[str, tuple[config_values.OverrideValue, ...]]: |
| 251 | """Reject an axis that declares no value.""" | 251 | """Reject an axis that declares no value.""" |
| 252 | for path, values in value.items(): | 252 | for path, values in value.items(): |
| 253 | if not values: | 253 | if not values: |
| 254 | raise config_values.ConfigError( | 254 | raise config_values.ExperimentConfigError( |
| 255 | f"study.matrix.axes.{path} cannot be empty" | 255 | f"study.matrix.axes.{path} cannot be empty" |
| 256 | ) | 256 | ) |
| 257 | return value | 257 | return value |
| 258 | 258 |
| 268 | @pydantic.model_validator(mode="after") | 268 | @pydantic.model_validator(mode="after") |
| 269 | def _bounds_are_complete(self) -> SweepParameterConfig: | 269 | def _bounds_are_complete(self) -> SweepParameterConfig: |
| 270 | """Reject a parameter that is neither finite nor fully bounded.""" | 270 | """Reject a parameter that is neither finite nor fully bounded.""" |
| 271 | if self.values is not None and not self.values: | 271 | if self.values is not None and not self.values: |
| 272 | raise config_values.ConfigError("values cannot be empty") | 272 | raise config_values.ExperimentConfigError("values cannot be empty") |
| 273 | if self.values is None and ( | 273 | if self.values is None and ( |
| 274 | self.minimum is None or self.maximum is None or self.distribution is None | 274 | self.minimum is None or self.maximum is None or self.distribution is None |
| 275 | ): | 275 | ): |
| 276 | raise config_values.ConfigError( | 276 | raise config_values.ExperimentConfigError( |
| 277 | "requires values or minimum/maximum/distribution" | 277 | "requires values or minimum/maximum/distribution" |
| 278 | ) | 278 | ) |
| 279 | if ( | 279 | if ( |
| 280 | self.minimum is not None | 280 | self.minimum is not None |
| 281 | and self.maximum is not None | 281 | and self.maximum is not None |
| 282 | and self.minimum >= self.maximum | 282 | and self.minimum >= self.maximum |
| 283 | ): | 283 | ): |
| 284 | raise config_values.ConfigError("minimum must be smaller than maximum") | 284 | raise config_values.ExperimentConfigError( |
| 285 | "minimum must be smaller than maximum" | ||
| 286 | ) | ||
| 285 | return self | 287 | return self |
| 286 | 288 | ||
| 287 | 289 | ||
| 288 | class SweepConfig(StrictConfigModel): | 290 | class SweepConfig(StrictConfigModel): |
| 299 | cls, value: Mapping[str, SweepParameterConfig] | 301 | cls, value: Mapping[str, SweepParameterConfig] |
| 300 | ) -> Mapping[str, SweepParameterConfig]: | 302 | ) -> Mapping[str, SweepParameterConfig]: |
| 301 | """Reject a sweep that declares no parameter.""" | 303 | """Reject a sweep that declares no parameter.""" |
| 302 | if not value: | 304 | if not value: |
| 303 | raise config_values.ConfigError("study.sweep.parameters cannot be empty") | 305 | raise config_values.ExperimentConfigError( |
| 306 | "study.sweep.parameters cannot be empty" | ||
| 307 | ) | ||
| 304 | return value | 308 | return value |
| 305 | 309 | ||
| 306 | 310 | ||
| 307 | class ContrastConfig(StrictConfigModel): | 311 | class ContrastConfig(StrictConfigModel): |
| 327 | """Reject a study whose kind and payload disagree.""" | 331 | """Reject a study whose kind and payload disagree.""" |
| 328 | if self.kind == "single" and ( | 332 | if self.kind == "single" and ( |
| 329 | self.variants or self.matrix is not None or self.sweep is not None | 333 | self.variants or self.matrix is not None or self.sweep is not None |
| 330 | ): | 334 | ): |
| 331 | raise config_values.ConfigError( | 335 | raise config_values.ExperimentConfigError( |
| 332 | "study.kind single cannot carry variants, matrix, or sweep" | 336 | "study.kind single cannot carry variants, matrix, or sweep" |
| 333 | ) | 337 | ) |
| 334 | if self.kind == "variants" and ( | 338 | if self.kind == "variants" and ( |
| 335 | not self.variants or self.matrix is not None or self.sweep is not None | 339 | not self.variants or self.matrix is not None or self.sweep is not None |
| 336 | ): | 340 | ): |
| 337 | raise config_values.ConfigError( | 341 | raise config_values.ExperimentConfigError( |
| 338 | "study.kind variants requires only a non-empty variants payload" | 342 | "study.kind variants requires only a non-empty variants payload" |
| 339 | ) | 343 | ) |
| 340 | if self.kind == "matrix" and ( | 344 | if self.kind == "matrix" and ( |
| 341 | self.variants or self.matrix is None or self.sweep is not None | 345 | self.variants or self.matrix is None or self.sweep is not None |
| 342 | ): | 346 | ): |
| 343 | raise config_values.ConfigError("study.kind matrix requires only matrix") | 347 | raise config_values.ExperimentConfigError( |
| 348 | "study.kind matrix requires only matrix" | ||
| 349 | ) | ||
| 344 | if self.kind == "sweep" and ( | 350 | if self.kind == "sweep" and ( |
| 345 | self.variants or self.matrix is not None or self.sweep is None | 351 | self.variants or self.matrix is not None or self.sweep is None |
| 346 | ): | 352 | ): |
| 347 | raise config_values.ConfigError("study.kind sweep requires only sweep") | 353 | raise config_values.ExperimentConfigError( |
| 354 | "study.kind sweep requires only sweep" | ||
| 355 | ) | ||
| 348 | identities = [item.id for item in self.variants] | 356 | identities = [item.id for item in self.variants] |
| 349 | if len(set(identities)) != len(identities): | 357 | if len(set(identities)) != len(identities): |
| 350 | raise config_values.ConfigError("study.variants contains duplicate IDs") | 358 | raise config_values.ExperimentConfigError( |
| 359 | "study.variants contains duplicate IDs" | ||
| 360 | ) | ||
| 351 | return self | 361 | return self |
| 65 | @pydantic.model_validator(mode="after") | 65 | @pydantic.model_validator(mode="after") |
| 66 | def _fuse_config_matches_mode(self) -> LabelSourceConfig: | 66 | def _fuse_config_matches_mode(self) -> LabelSourceConfig: |
| 67 | """Tie the fusion config to the declared label mode.""" | 67 | """Tie the fusion config to the declared label mode.""" |
| 68 | if self.mode == "regenerate_full_resolution" and self.fuse_config is None: | 68 | if self.mode == "regenerate_full_resolution" and self.fuse_config is None: |
| 69 | raise config_values.ConfigError( | 69 | raise config_values.ExperimentConfigError( |
| 70 | "regenerate_full_resolution requires data.label_source.fuse_config" | 70 | "regenerate_full_resolution requires data.label_source.fuse_config" |
| 71 | ) | 71 | ) |
| 72 | if self.mode == "artifact" and self.fuse_config is not None: | 72 | if self.mode == "artifact" and self.fuse_config is not None: |
| 73 | raise config_values.ConfigError( | 73 | raise config_values.ExperimentConfigError( |
| 74 | "artifact label mode requires data.label_source.fuse_config: null" | 74 | "artifact label mode requires data.label_source.fuse_config: null" |
| 75 | ) | 75 | ) |
| 76 | return self | 76 | return self |
| 77 | 77 |
| 104 | @pydantic.model_validator(mode="after") | 104 | @pydantic.model_validator(mode="after") |
| 105 | def _overlap_fits_in_a_tile(self) -> TilingConfig: | 105 | def _overlap_fits_in_a_tile(self) -> TilingConfig: |
| 106 | """Reject an overlap that is not shorter than the tile.""" | 106 | """Reject an overlap that is not shorter than the tile.""" |
| 107 | if self.overlap_m >= self.length_m: | 107 | if self.overlap_m >= self.length_m: |
| 108 | raise config_values.ConfigError("tiling requires 0 <= overlap_m < length_m") | 108 | raise config_values.ExperimentConfigError( |
| 109 | "tiling requires 0 <= overlap_m < length_m" | ||
| 110 | ) | ||
| 109 | return self | 111 | return self |
| 110 | 112 | ||
| 111 | 113 | ||
| 112 | class DataConfig(config_schema.StrictConfigModel): | 114 | class DataConfig(config_schema.StrictConfigModel): |
| 204 | def _external_models_pin_their_checkout(self) -> ModelConfig: | 210 | def _external_models_pin_their_checkout(self) -> ModelConfig: |
| 205 | """Tie the external checkout variables to the declared framework.""" | 211 | """Tie the external checkout variables to the declared framework.""" |
| 206 | if self.framework == "cpu": | 212 | if self.framework == "cpu": |
| 207 | if self.checkout_env is not None or self.commit_env is not None: | 213 | if self.checkout_env is not None or self.commit_env is not None: |
| 208 | raise config_values.ConfigError( | 214 | raise config_values.ExperimentConfigError( |
| 209 | "CPU experiments cannot declare external checkout variables" | 215 | "CPU experiments cannot declare external checkout variables" |
| 210 | ) | 216 | ) |
| 211 | elif not (self.base_config and self.checkout_env and self.commit_env): | 217 | elif not (self.base_config and self.checkout_env and self.commit_env): |
| 212 | raise config_values.ConfigError( | 218 | raise config_values.ExperimentConfigError( |
| 213 | "External models require base_config, checkout_env, and commit_env" | 219 | "External models require base_config, checkout_env, and commit_env" |
| 214 | ) | 220 | ) |
| 215 | return self | 221 | return self |
| 216 | 222 |
| 300 | self.delta_quality is None | 306 | self.delta_quality is None |
| 301 | or self.delta_fp_per_km is None | 307 | or self.delta_fp_per_km is None |
| 302 | or not self.superiority_conditions | 308 | or not self.superiority_conditions |
| 303 | ): | 309 | ): |
| 304 | raise config_values.ConfigError( | 310 | raise config_values.ExperimentConfigError( |
| 305 | "enabled promotion requires non-null margins and superiority " | 311 | "enabled promotion requires non-null margins and superiority " |
| 306 | "conditions" | 312 | "conditions" |
| 307 | ) | 313 | ) |
| 308 | return self | 314 | return self |
| 325 | def _floors_are_fractions(cls, value: Mapping[str, float]) -> Mapping[str, float]: | 331 | def _floors_are_fractions(cls, value: Mapping[str, float]) -> Mapping[str, float]: |
| 326 | """Reject a precision floor outside the unit interval.""" | 332 | """Reject a precision floor outside the unit interval.""" |
| 327 | for name, floor in value.items(): | 333 | for name, floor in value.items(): |
| 328 | if not 0.0 <= floor <= 1.0: | 334 | if not 0.0 <= floor <= 1.0: |
| 329 | raise config_values.ConfigError( | 335 | raise config_values.ExperimentConfigError( |
| 330 | f"evaluation precision floor for {name} must be in [0, 1]" | 336 | f"evaluation precision floor for {name} must be in [0, 1]" |
| 331 | ) | 337 | ) |
| 332 | return value | 338 | return value |
| 333 | 339 | ||
| 334 | @pydantic.model_validator(mode="after") | 340 | @pydantic.model_validator(mode="after") |
| 335 | def _promotion_test_is_promotable(self) -> EvaluationConfig: | 341 | def _promotion_test_is_promotable(self) -> EvaluationConfig: |
| 336 | """Reject a locked-test split whose promotion protocol is disabled.""" | 342 | """Reject a locked-test split whose promotion protocol is disabled.""" |
| 337 | if self.split == "promotion_test" and not self.promotion.enabled: | 343 | if self.split == "promotion_test" and not self.promotion.enabled: |
| 338 | raise config_values.ConfigError( | 344 | raise config_values.ExperimentConfigError( |
| 339 | "promotion_test evaluation requires evaluation.promotion.enabled" | 345 | "promotion_test evaluation requires evaluation.promotion.enabled" |
| 340 | ) | 346 | ) |
| 341 | return self | 347 | return self |
| 342 | 348 |
| 376 | def _tiles_select_something(self) -> VisualizationConfig: | 382 | def _tiles_select_something(self) -> VisualizationConfig: |
| 377 | """Reject an empty or non-positive tile selection.""" | 383 | """Reject an empty or non-positive tile selection.""" |
| 378 | if isinstance(self.masks_tiles, int): | 384 | if isinstance(self.masks_tiles, int): |
| 379 | if self.masks_tiles < 1: | 385 | if self.masks_tiles < 1: |
| 380 | raise config_values.ConfigError( | 386 | raise config_values.ExperimentConfigError( |
| 381 | "visualization.masks_tiles must be >= 1" | 387 | "visualization.masks_tiles must be >= 1" |
| 382 | ) | 388 | ) |
| 383 | elif not self.masks_tiles: | 389 | elif not self.masks_tiles: |
| 384 | raise config_values.ConfigError("visualization.masks_tiles cannot be empty") | 390 | raise config_values.ExperimentConfigError( |
| 391 | "visualization.masks_tiles cannot be empty" | ||
| 392 | ) | ||
| 385 | return self | 393 | return self |
| 386 | 394 | ||
| 387 | 395 | ||
| 388 | class HarnessConfig(config_schema.StrictConfigModel): | 396 | class HarnessConfig(config_schema.StrictConfigModel): |
| 414 | if isinstance(data, Mapping) and data.get("visualization", False) is None: | 422 | if isinstance(data, Mapping) and data.get("visualization", False) is None: |
| 415 | raise ValueError("visualization must be a mapping") | 423 | raise ValueError("visualization must be a mapping") |
| 416 | return data | 424 | return data |
| 417 | 425 | ||
| 426 | @classmethod | ||
| 427 | def from_yaml(cls, path: str | Path) -> HarnessConfig: | ||
| 428 | """Load and strictly validate one E1--E14 experiment YAML. | ||
| 429 | |||
| 430 | This is the harness entry point every script and test uses; the | ||
| 431 | document read, strict validation, and the cross-section rules of | ||
| 432 | :mod:`src.train.config_rules` all run here. | ||
| 433 | |||
| 434 | Args: | ||
| 435 | path: Repository-relative or absolute YAML path. | ||
| 436 | |||
| 437 | Returns: | ||
| 438 | Fully typed immutable configuration carrying its own SHA-256. | ||
| 439 | |||
| 440 | Raises: | ||
| 441 | ExperimentConfigError: If loading, strict parsing, or validation | ||
| 442 | fails. | ||
| 443 | """ | ||
| 444 | # Deferred: the loading pipeline lives in the entry-point module, which | ||
| 445 | # imports this one for its models. | ||
| 446 | from src.train import config | ||
| 447 | |||
| 448 | return config._load_yaml_document(path) | ||
| 449 | |||
| 418 | @property | 450 | @property |
| 419 | def source_path(self) -> Path: | 451 | def source_path(self) -> Path: |
| 420 | """Path of the YAML document this configuration was parsed from.""" | 452 | """Path of the YAML document this configuration was parsed from.""" |
| 421 | return self._source_path | 453 | return self._source_path |
| 1 | """Strict YAML value typing shared by the experiment configuration models. | 1 | """Strict YAML value typing shared by the experiment configuration models. |
| 2 | 2 | ||
| 3 | The experiment YAML is a fail-closed contract: a value must already carry its | 3 | The experiment YAML is a fail-closed contract: a value must already carry its |
| 4 | declared type, so ``"50"`` is not a float, ``1`` is not a boolean, and ``3.0`` | 4 | declared type, so ``"50"`` is not a float, ``1`` is not a boolean, and ``3.0`` |
| 5 | is not an integer. That is deliberately stricter than the fleet coercion | 5 | is not an integer. That is deliberately stricter than the fleet coercion matrix |
| 6 | matrix of :func:`iolabs.common.config_loader.coerce_config_value`, so the | 6 | of :func:`iolabs.common.config_loader.coerce_to_field_type`, and it is the one |
| 7 | models in :mod:`src.train.config_schema` route every field through | 7 | sanctioned opt-out from it: the models in :mod:`src.train.config_schema` route |
| 8 | :func:`typed_value` instead of the inherited coercion. | 8 | every field through :func:`typed_value` instead of the inherited coercion. Do |
| 9 | not copy this into a packaged pipeline config -- it holds only for this | ||
| 10 | YAML experiment contract. | ||
| 9 | """ | 11 | """ |
| 10 | 12 | ||
| 11 | from __future__ import annotations | 13 | from __future__ import annotations |
| 12 | 14 |
| 23 | Scalar: TypeAlias = str | int | float | bool | None | 25 | Scalar: TypeAlias = str | int | float | bool | None |
| 24 | OverrideValue: TypeAlias = Scalar | list[Scalar] | 26 | OverrideValue: TypeAlias = Scalar | list[Scalar] |
| 25 | 27 | ||
| 26 | 28 | ||
| 27 | class ConfigError(config_loader.ConfigError): | 29 | class ExperimentConfigError(config_loader.ConfigError): |
| 28 | """Raised when an experiment configuration violates its strict schema.""" | 30 | """Raised when experiment config contains unsupported keys or values.""" |
| 29 | 31 | ||
| 30 | 32 | ||
| 31 | def freeze_value(value: Any) -> Any: | 33 | def freeze_value(value: Any) -> Any: |
| 32 | """Return *value* with every nested mapping wrapped read-only. | 34 | """Return *value* with every nested mapping wrapped read-only. |
| 89 | Returns: | 91 | Returns: |
| 90 | The value converted to the declared type. | 92 | The value converted to the declared type. |
| 91 | 93 | ||
| 92 | Raises: | 94 | Raises: |
| 93 | ConfigError: If the value does not match the declared type. | 95 | ExperimentConfigError: If the value does not match the declared type. |
| 94 | """ | 96 | """ |
| 95 | if annotation is None or annotation is Any: | 97 | if annotation is None or annotation is Any: |
| 96 | return value | 98 | return value |
| 97 | origin = get_origin(annotation) | 99 | origin = get_origin(annotation) |
| 135 | optional = type(None) in members | 137 | optional = type(None) in members |
| 136 | if value is None: | 138 | if value is None: |
| 137 | if optional: | 139 | if optional: |
| 138 | return None | 140 | return None |
| 139 | raise ConfigError(f"{where} must not be null") | 141 | raise ExperimentConfigError(f"{where} must not be null") |
| 140 | candidates = [item for item in members if item is not type(None)] | 142 | candidates = [item for item in members if item is not type(None)] |
| 141 | if len(candidates) == 1: | 143 | if len(candidates) == 1: |
| 142 | return typed_value(candidates[0], value, where) | 144 | return typed_value(candidates[0], value, where) |
| 143 | for candidate in candidates: | 145 | for candidate in candidates: |
| 144 | try: | 146 | try: |
| 145 | return typed_value(candidate, value, where) | 147 | return typed_value(candidate, value, where) |
| 146 | except ConfigError: | 148 | except ExperimentConfigError: |
| 147 | continue | 149 | continue |
| 148 | names = sorted(getattr(item, "__name__", str(item)) for item in candidates) | 150 | names = sorted(getattr(item, "__name__", str(item)) for item in candidates) |
| 149 | raise ConfigError( | 151 | raise ExperimentConfigError( |
| 150 | f"{where} must be one of {names}, got {type(value).__name__} {value!r}" | 152 | f"{where} must be one of {names}, got {type(value).__name__} {value!r}" |
| 151 | ) | 153 | ) |
| 152 | 154 | ||
| 153 | 155 | ||
| 154 | def mapping(value: Any, where: str) -> Mapping[str, Any]: | 156 | def mapping(value: Any, where: str) -> Mapping[str, Any]: |
| 155 | """Return *value* as a string-keyed mapping or raise.""" | 157 | """Return *value* as a string-keyed mapping or raise.""" |
| 156 | if not isinstance(value, Mapping): | 158 | if not isinstance(value, Mapping): |
| 157 | raise ConfigError(f"{where} must be a mapping") | 159 | raise ExperimentConfigError(f"{where} must be a mapping") |
| 158 | if any(not isinstance(key, str) for key in value): | 160 | if any(not isinstance(key, str) for key in value): |
| 159 | raise ConfigError(f"{where} keys must be strings") | 161 | raise ExperimentConfigError(f"{where} keys must be strings") |
| 160 | return value | 162 | return value |
| 161 | 163 | ||
| 162 | 164 | ||
| 163 | def sequence(value: Any, where: str) -> Sequence[Any]: | 165 | def sequence(value: Any, where: str) -> Sequence[Any]: |
| 164 | """Return *value* as a non-string sequence or raise.""" | 166 | """Return *value* as a non-string sequence or raise.""" |
| 165 | if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): | 167 | if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): |
| 166 | raise ConfigError(f"{where} must be a sequence") | 168 | raise ExperimentConfigError(f"{where} must be a sequence") |
| 167 | return value | 169 | return value |
| 168 | 170 | ||
| 169 | 171 | ||
| 170 | def string(value: Any, where: str) -> str: | 172 | def string(value: Any, where: str) -> str: |
| 171 | """Return *value* as a non-empty string or raise.""" | 173 | """Return *value* as a non-empty string or raise.""" |
| 172 | if not isinstance(value, str) or not value.strip(): | 174 | if not isinstance(value, str) or not value.strip(): |
| 173 | raise ConfigError(f"{where} must be a non-empty string") | 175 | raise ExperimentConfigError(f"{where} must be a non-empty string") |
| 174 | return value | 176 | return value |
| 175 | 177 | ||
| 176 | 178 | ||
| 177 | def integer(value: Any, where: str) -> int: | 179 | def integer(value: Any, where: str) -> int: |
| 178 | """Return *value* as an integer, rejecting booleans, or raise.""" | 180 | """Return *value* as an integer, rejecting booleans, or raise.""" |
| 179 | if isinstance(value, bool) or not isinstance(value, int): | 181 | if isinstance(value, bool) or not isinstance(value, int): |
| 180 | raise ConfigError(f"{where} must be an integer") | 182 | raise ExperimentConfigError(f"{where} must be an integer") |
| 181 | return value | 183 | return value |
| 182 | 184 | ||
| 183 | 185 | ||
| 184 | def number(value: Any, where: str) -> float: | 186 | def number(value: Any, where: str) -> float: |
| 185 | """Return *value* as a float, rejecting booleans, or raise.""" | 187 | """Return *value* as a float, rejecting booleans, or raise.""" |
| 186 | if isinstance(value, bool) or not isinstance(value, (int, float)): | 188 | if isinstance(value, bool) or not isinstance(value, (int, float)): |
| 187 | raise ConfigError(f"{where} must be numeric") | 189 | raise ExperimentConfigError(f"{where} must be numeric") |
| 188 | return float(value) | 190 | return float(value) |
| 189 | 191 | ||
| 190 | 192 | ||
| 191 | def boolean(value: Any, where: str) -> bool: | 193 | def boolean(value: Any, where: str) -> bool: |
| 192 | """Return *value* as a boolean or raise.""" | 194 | """Return *value* as a boolean or raise.""" |
| 193 | if not isinstance(value, bool): | 195 | if not isinstance(value, bool): |
| 194 | raise ConfigError(f"{where} must be boolean") | 196 | raise ExperimentConfigError(f"{where} must be boolean") |
| 195 | return value | 197 | return value |
| 196 | 198 | ||
| 197 | 199 | ||
| 198 | def path(value: Any, where: str) -> Path: | 200 | def path(value: Any, where: str) -> Path: |
| 199 | """Return *value* as a repository-relative path or raise.""" | 201 | """Return *value* as a repository-relative path or raise.""" |
| 200 | text = string(value, where) | 202 | text = string(value, where) |
| 201 | parsed = Path(text) | 203 | parsed = Path(text) |
| 202 | if parsed.is_absolute(): | 204 | if parsed.is_absolute(): |
| 203 | raise ConfigError(f"{where} must be repository-relative, got {parsed}") | 205 | raise ExperimentConfigError( |
| 206 | f"{where} must be repository-relative, got {parsed}" | ||
| 207 | ) | ||
| 204 | return parsed | 208 | return parsed |
| 205 | 209 | ||
| 206 | 210 | ||
| 207 | def choice(value: Any, choices: set[str], where: str) -> str: | 211 | def choice(value: Any, choices: set[str], where: str) -> str: |
| 208 | """Return *value* when it is one of the allowed string choices.""" | 212 | """Return *value* when it is one of the allowed string choices.""" |
| 209 | text = string(value, where) | 213 | text = string(value, where) |
| 210 | if text not in choices: | 214 | if text not in choices: |
| 211 | raise ConfigError(f"{where} must be one of {sorted(choices)}, got {text!r}") | 215 | raise ExperimentConfigError( |
| 216 | f"{where} must be one of {sorted(choices)}, got {text!r}" | ||
| 217 | ) | ||
| 212 | return text | 218 | return text |
| 213 | 219 | ||
| 214 | 220 | ||
| 215 | def check_keys( | 221 | def check_keys( |
| 226 | allowed: Keys that may be present. | 232 | allowed: Keys that may be present. |
| 227 | where: Dotted key path used in error messages. | 233 | where: Dotted key path used in error messages. |
| 228 | 234 | ||
| 229 | Raises: | 235 | Raises: |
| 230 | ConfigError: If a key is missing or unknown. | 236 | ExperimentConfigError: If a key is missing or unknown. |
| 231 | """ | 237 | """ |
| 232 | missing = sorted(required - set(value)) | 238 | missing = sorted(required - set(value)) |
| 233 | unknown = sorted(set(value) - allowed) | 239 | unknown = sorted(set(value) - allowed) |
| 234 | if missing or unknown: | 240 | if missing or unknown: |
| 236 | if missing: | 242 | if missing: |
| 237 | details.append(f"missing {missing}") | 243 | details.append(f"missing {missing}") |
| 238 | if unknown: | 244 | if unknown: |
| 239 | details.append(f"unknown {unknown}") | 245 | details.append(f"unknown {unknown}") |
| 240 | raise ConfigError(f"{where} has " + " and ".join(details)) | 246 | raise ExperimentConfigError(f"{where} has " + " and ".join(details)) |
| 241 | 247 | ||
| 242 | 248 | ||
| 243 | def leaf_values(value: Any, prefix: str = "") -> dict[str, OverrideValue]: | 249 | def leaf_values(value: Any, prefix: str = "") -> dict[str, OverrideValue]: |
| 244 | """Return every dotted scalar/list leaf of a raw configuration mapping. | 250 | """Return every dotted scalar/list leaf of a raw configuration mapping. |
| 23 | from pathlib import Path | 23 | from pathlib import Path |
| 24 | from typing import Any | 24 | from typing import Any |
| 25 | 25 | ||
| 26 | from src.contracts.ontology import Ontology, OntologyError, load_ontology | 26 | from src.contracts.ontology import Ontology, OntologyError, load_ontology |
| 27 | from src.train.config import ConfigError, load_config, resolve_ontology_path | 27 | from src.train.config import ExperimentConfigError, HarnessConfig, resolve_ontology_path |
| 28 | 28 | ||
| 29 | POINTCEPT_FRAMEWORK = "pointcept" | 29 | POINTCEPT_FRAMEWORK = "pointcept" |
| 30 | SPT_FRAMEWORK = "spt" | 30 | SPT_FRAMEWORK = "spt" |
| 31 | SUPPORTED_FRAMEWORKS = (POINTCEPT_FRAMEWORK, SPT_FRAMEWORK) | 31 | SUPPORTED_FRAMEWORKS = (POINTCEPT_FRAMEWORK, SPT_FRAMEWORK) |
| 171 | 171 | ||
| 172 | Raises: | 172 | Raises: |
| 173 | EmissionCheckError: If the framework is unsupported, a narrowing is | 173 | EmissionCheckError: If the framework is unsupported, a narrowing is |
| 174 | requested for SPT, or the emission disagrees with the ontology. | 174 | requested for SPT, or the emission disagrees with the ontology. |
| 175 | ConfigError: If the harness YAML is malformed. | 175 | ExperimentConfigError: If the harness YAML is malformed. |
| 176 | OntologyError: If the declared ontology cannot be loaded. | 176 | OntologyError: If the declared ontology cannot be loaded. |
| 177 | """ | 177 | """ |
| 178 | if framework not in SUPPORTED_FRAMEWORKS: | 178 | if framework not in SUPPORTED_FRAMEWORKS: |
| 179 | raise EmissionCheckError( | 179 | raise EmissionCheckError( |
| 180 | f"{config_path}: unsupported framework {framework!r}; expected one " | 180 | f"{config_path}: unsupported framework {framework!r}; expected one " |
| 181 | f"of {list(SUPPORTED_FRAMEWORKS)}" | 181 | f"of {list(SUPPORTED_FRAMEWORKS)}" |
| 182 | ) | 182 | ) |
| 183 | config = load_config(config_path) | 183 | config = HarnessConfig.from_yaml(config_path) |
| 184 | ontology = load_ontology(resolve_ontology_path(config)) | 184 | ontology = load_ontology(resolve_ontology_path(config)) |
| 185 | lane = Path(processed_root) if processed_root else config.data.processed_root | 185 | lane = Path(processed_root) if processed_root else config.data.processed_root |
| 186 | if framework == SPT_FRAMEWORK: | 186 | if framework == SPT_FRAMEWORK: |
| 187 | if corridors: | 187 | if corridors: |
| 245 | processed_root=args.processed_root, | 245 | processed_root=args.processed_root, |
| 246 | corridors=tuple(args.corridor), | 246 | corridors=tuple(args.corridor), |
| 247 | ) | 247 | ) |
| 248 | except ( | 248 | except ( |
| 249 | ConfigError, | 249 | ExperimentConfigError, |
| 250 | EmissionCheckError, | 250 | EmissionCheckError, |
| 251 | OntologyError, | 251 | OntologyError, |
| 252 | OSError, | 252 | OSError, |
| 253 | ) as exc: | 253 | ) as exc: |
| 33 | SplitTier, | 33 | SplitTier, |
| 34 | authorize_split_access, | 34 | authorize_split_access, |
| 35 | load_split_manifest, | 35 | load_split_manifest, |
| 36 | ) | 36 | ) |
| 37 | from src.train.config import HarnessConfig, load_config, resolve_ontology_path | 37 | from src.train.config import HarnessConfig, resolve_ontology_path |
| 38 | 38 | ||
| 39 | STARTED = "started" | 39 | STARTED = "started" |
| 40 | COMPLETED = "completed" | 40 | COMPLETED = "completed" |
| 41 | PROVENANCE_FILENAMES: dict[str, str] = { | 41 | PROVENANCE_FILENAMES: dict[str, str] = { |
| 417 | Raises: | 417 | Raises: |
| 418 | RunProvenanceError: If the run cannot prove its required evidence. | 418 | RunProvenanceError: If the run cannot prove its required evidence. |
| 419 | """ | 419 | """ |
| 420 | args = build_parser().parse_args(argv) | 420 | args = build_parser().parse_args(argv) |
| 421 | config = load_config(args.config) | 421 | config = HarnessConfig.from_yaml(args.config) |
| 422 | split_manifest = load_split_manifest( | 422 | split_manifest = load_split_manifest( |
| 423 | config.data.split_manifest, | 423 | config.data.split_manifest, |
| 424 | ontology=load_ontology(resolve_ontology_path(config)), | 424 | ontology=load_ontology(resolve_ontology_path(config)), |
| 425 | ) | 425 | ) |
| 1 | """Emit ontology-derived CLI overrides for the SPT and Pointcept runners. | 1 | """Emit ontology-derived CLI overrides for the SPT and Pointcept runners. |
| 2 | 2 | ||
| 3 | The harness experiment YAML is the source of truth for class count, void ID, | 3 | The harness experiment YAML is the source of truth for class count, void ID, |
| 4 | and predicted class names. This module loads that YAML with the same | 4 | and predicted class names. This module loads that YAML with the same |
| 5 | :func:`src.train.config.load_config` / :func:`src.contracts.ontology.load_ontology` | 5 | :meth:`src.train.config.HarnessConfig.from_yaml` / |
| 6 | :func:`src.contracts.ontology.load_ontology` | ||
| 6 | path the rest of the harness uses, then prints one ``KEY=VALUE`` override per | 7 | path the rest of the harness uses, then prints one ``KEY=VALUE`` override per |
| 7 | line. The shell runners append those lines to the framework command: | 8 | line. The shell runners append those lines to the framework command: |
| 8 | 9 | ||
| 9 | * Pointcept consumes them as ``--options`` tokens (``DictAction`` then | 10 | * Pointcept consumes them as ``--options`` tokens (``DictAction`` then |
| 28 | from collections.abc import Sequence | 29 | from collections.abc import Sequence |
| 29 | from pathlib import Path | 30 | from pathlib import Path |
| 30 | 31 | ||
| 31 | from src.contracts.ontology import Ontology, OntologyError, load_ontology | 32 | from src.contracts.ontology import Ontology, OntologyError, load_ontology |
| 32 | from src.train.config import ConfigError, load_config, resolve_ontology_path | 33 | from src.train.config import ExperimentConfigError, HarnessConfig, resolve_ontology_path |
| 33 | 34 | ||
| 34 | POINTCEPT_FRAMEWORK = "pointcept" | 35 | POINTCEPT_FRAMEWORK = "pointcept" |
| 35 | SPT_FRAMEWORK = "spt" | 36 | SPT_FRAMEWORK = "spt" |
| 36 | SUPPORTED_FRAMEWORKS = (POINTCEPT_FRAMEWORK, SPT_FRAMEWORK) | 37 | SUPPORTED_FRAMEWORKS = (POINTCEPT_FRAMEWORK, SPT_FRAMEWORK) |
| 91 | One ``KEY=VALUE`` override per tuple element. | 92 | One ``KEY=VALUE`` override per tuple element. |
| 92 | 93 | ||
| 93 | Raises: | 94 | Raises: |
| 94 | RunnerOptionsError: If ``framework`` is unsupported. | 95 | RunnerOptionsError: If ``framework`` is unsupported. |
| 95 | ConfigError: If the harness YAML is malformed. | 96 | ExperimentConfigError: If the harness YAML is malformed. |
| 96 | OntologyError: If the declared ontology cannot be loaded. | 97 | OntologyError: If the declared ontology cannot be loaded. |
| 97 | """ | 98 | """ |
| 98 | if framework not in SUPPORTED_FRAMEWORKS: | 99 | if framework not in SUPPORTED_FRAMEWORKS: |
| 99 | raise RunnerOptionsError( | 100 | raise RunnerOptionsError( |
| 100 | f"{config_path}: unsupported framework {framework!r}; expected " | 101 | f"{config_path}: unsupported framework {framework!r}; expected " |
| 101 | f"one of {list(SUPPORTED_FRAMEWORKS)}" | 102 | f"one of {list(SUPPORTED_FRAMEWORKS)}" |
| 102 | ) | 103 | ) |
| 103 | config = load_config(config_path) | 104 | config = HarnessConfig.from_yaml(config_path) |
| 104 | ontology = load_ontology(resolve_ontology_path(config)) | 105 | ontology = load_ontology(resolve_ontology_path(config)) |
| 105 | if framework == SPT_FRAMEWORK: | 106 | if framework == SPT_FRAMEWORK: |
| 106 | return spt_overrides(ontology) | 107 | return spt_overrides(ontology) |
| 107 | return pointcept_overrides(ontology) | 108 | return pointcept_overrides(ontology) |
| 139 | """ | 140 | """ |
| 140 | args = build_parser().parse_args(argv) | 141 | args = build_parser().parse_args(argv) |
| 141 | try: | 142 | try: |
| 142 | lines = collect_overrides(args.config, args.framework) | 143 | lines = collect_overrides(args.config, args.framework) |
| 143 | except (ConfigError, OntologyError, RunnerOptionsError, OSError) as exc: | 144 | except (ExperimentConfigError, OntologyError, RunnerOptionsError, OSError) as exc: |
| 144 | print(f"{args.config}: {exc}", file=sys.stderr) | 145 | print(f"{args.config}: {exc}", file=sys.stderr) |
| 145 | return 1 | 146 | return 1 |
| 146 | if not lines: | 147 | if not lines: |
| 147 | print( | 148 | print( |
| 40 | continuity_metrics, | 40 | continuity_metrics, |
| 41 | object_metrics, | 41 | object_metrics, |
| 42 | ) | 42 | ) |
| 43 | from src.train.config import ( | 43 | from src.train.config import ( |
| 44 | ConfigError, | 44 | ExperimentConfigError, |
| 45 | HarnessConfig, | 45 | HarnessConfig, |
| 46 | is_canonical_metric, | 46 | is_canonical_metric, |
| 47 | load_config, | ||
| 48 | resolve_ontology_path, | 47 | resolve_ontology_path, |
| 49 | ) | 48 | ) |
| 50 | 49 | ||
| 51 | 50 |
| 127 | Zero after reports and finalized provenance are written. | 126 | Zero after reports and finalized provenance are written. |
| 128 | """ | 127 | """ |
| 129 | args = build_parser().parse_args(argv) | 128 | args = build_parser().parse_args(argv) |
| 130 | try: | 129 | try: |
| 131 | config = load_config(args.config) | 130 | config = HarnessConfig.from_yaml(args.config) |
| 132 | if config.model.framework not in {"spt", "pointcept"}: | 131 | if config.model.framework not in {"spt", "pointcept"}: |
| 133 | raise EvaluationError( | 132 | raise EvaluationError( |
| 134 | f"{config.experiment.id} framework {config.model.framework} " | 133 | f"{config.experiment.id} framework {config.model.framework} " |
| 135 | "has no checkpoint evaluator" | 134 | "has no checkpoint evaluator" |
| 313 | }, | 312 | }, |
| 314 | ) | 313 | ) |
| 315 | write_corridor_run_provenance(run_dir / "provenance.json", final_provenance) | 314 | write_corridor_run_provenance(run_dir / "provenance.json", final_provenance) |
| 316 | return 0 | 315 | return 0 |
| 317 | except (ConfigError, EvaluationError, OSError, RuntimeError, ValueError) as exc: | 316 | except ( |
| 317 | ExperimentConfigError, | ||
| 318 | EvaluationError, | ||
| 319 | OSError, | ||
| 320 | RuntimeError, | ||
| 321 | ValueError, | ||
| 322 | ) as exc: | ||
| 318 | raise SystemExit(str(exc)) from exc | 323 | raise SystemExit(str(exc)) from exc |
| 319 | 324 | ||
| 320 | 325 | ||
| 321 | @dataclass(frozen=True) | 326 | @dataclass(frozen=True) |
| 24 | PreannotationError, | 24 | PreannotationError, |
| 25 | load_preannotation, | 25 | load_preannotation, |
| 26 | select_preannotation, | 26 | select_preannotation, |
| 27 | ) | 27 | ) |
| 28 | from src.train.config import load_config, resolve_ontology_path | 28 | from src.train.config import HarnessConfig, resolve_ontology_path |
| 29 | 29 | ||
| 30 | 30 | ||
| 31 | def build_parser() -> argparse.ArgumentParser: | 31 | def build_parser() -> argparse.ArgumentParser: |
| 32 | """Build the frozen preannotation-ingest CLI parser. | 32 | """Build the frozen preannotation-ingest CLI parser. |
| 57 | Process exit status. | 57 | Process exit status. |
| 58 | """ | 58 | """ |
| 59 | args = build_parser().parse_args(argv) | 59 | args = build_parser().parse_args(argv) |
| 60 | try: | 60 | try: |
| 61 | config = load_config(args.config) | 61 | config = HarnessConfig.from_yaml(args.config) |
| 62 | ontology = load_ontology(resolve_ontology_path(config)) | 62 | ontology = load_ontology(resolve_ontology_path(config)) |
| 63 | split_manifest = load_split_manifest( | 63 | split_manifest = load_split_manifest( |
| 64 | config.data.split_manifest, ontology=ontology | 64 | config.data.split_manifest, ontology=ontology |
| 65 | ) | 65 | ) |
| 50 | validate_boundary_las_codes, | 50 | validate_boundary_las_codes, |
| 51 | write_normalization, | 51 | write_normalization, |
| 52 | ) | 52 | ) |
| 53 | from src.dataset.features import expand_feature_columns | 53 | from src.dataset.features import expand_feature_columns |
| 54 | from src.train.config import load_config, resolve_ontology_path | 54 | from src.train.config import HarnessConfig, resolve_ontology_path |
| 55 | 55 | ||
| 56 | 56 | ||
| 57 | def build_parser() -> argparse.ArgumentParser: | 57 | def build_parser() -> argparse.ArgumentParser: |
| 58 | """Build the frozen dataset-preparation CLI parser. | 58 | """Build the frozen dataset-preparation CLI parser. |
| 88 | Process exit status. | 88 | Process exit status. |
| 89 | """ | 89 | """ |
| 90 | args = build_parser().parse_args(argv) | 90 | args = build_parser().parse_args(argv) |
| 91 | try: | 91 | try: |
| 92 | config = load_config(args.config) | 92 | config = HarnessConfig.from_yaml(args.config) |
| 93 | ontology = load_ontology(resolve_ontology_path(config)) | 93 | ontology = load_ontology(resolve_ontology_path(config)) |
| 94 | split_manifest = load_split_manifest( | 94 | split_manifest = load_split_manifest( |
| 95 | config.data.split_manifest, ontology=ontology | 95 | config.data.split_manifest, ontology=ontology |
| 96 | ) | 96 | ) |
| 7 | import logging | 7 | import logging |
| 8 | from collections.abc import Sequence | 8 | from collections.abc import Sequence |
| 9 | from pathlib import Path | 9 | from pathlib import Path |
| 10 | 10 | ||
| 11 | from src.train.config import ConfigError, load_config | 11 | from src.train.config import ExperimentConfigError, HarnessConfig |
| 12 | from src.train.dispatch import ( | 12 | from src.train.dispatch import ( |
| 13 | DispatchError, | 13 | DispatchError, |
| 14 | DispatchRequest, | 14 | DispatchRequest, |
| 15 | dispatch_experiment, | 15 | dispatch_experiment, |
| 55 | """ | 55 | """ |
| 56 | logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") | 56 | logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") |
| 57 | args = build_parser().parse_args(argv) | 57 | args = build_parser().parse_args(argv) |
| 58 | try: | 58 | try: |
| 59 | config = load_config(args.config) | 59 | config = HarnessConfig.from_yaml(args.config) |
| 60 | cells = plan_experiment(config) | 60 | cells = plan_experiment(config) |
| 61 | LOGGER.info( | 61 | LOGGER.info( |
| 62 | "%s study kind %s expands to %d cell(s): %s", | 62 | "%s study kind %s expands to %d cell(s): %s", |
| 63 | config.experiment.id, | 63 | config.experiment.id, |
| 73 | log_dir=args.log_dir, | 73 | log_dir=args.log_dir, |
| 74 | num_workers=args.num_workers, | 74 | num_workers=args.num_workers, |
| 75 | ) | 75 | ) |
| 76 | return dispatch_experiment(config, request) | 76 | return dispatch_experiment(config, request) |
| 77 | except (ConfigError, DispatchError) as exc: | 77 | except (ExperimentConfigError, DispatchError) as exc: |
| 78 | raise SystemExit(str(exc)) from exc | 78 | raise SystemExit(str(exc)) from exc |
| 79 | 79 | ||
| 80 | 80 | ||
| 81 | if __name__ == "__main__": | 81 | if __name__ == "__main__": |
| 14 | from src.adapters.pointcept import load_pointcept_identity | 14 | from src.adapters.pointcept import load_pointcept_identity |
| 15 | from src.adapters.remap import TilePrediction, blend_and_remap | 15 | from src.adapters.remap import TilePrediction, blend_and_remap |
| 16 | from src.adapters.spt import load_spt_identity | 16 | from src.adapters.spt import load_spt_identity |
| 17 | from src.dataset.canonical import CanonicalCloud, read_canonical_tile | 17 | from src.dataset.canonical import CanonicalCloud, read_canonical_tile |
| 18 | from src.train.config import load_config | 18 | from src.train.config import HarnessConfig |
| 19 | 19 | ||
| 20 | DEFAULT_FRAME_TOLERANCE_MM = 0.6 | 20 | DEFAULT_FRAME_TOLERANCE_MM = 0.6 |
| 21 | """Tolerated frame reconstruction error. | 21 | """Tolerated frame reconstruction error. |
| 22 | 22 |
| 61 | if not np.isfinite(args.frame_tolerance_mm) or args.frame_tolerance_mm <= 0.0: | 61 | if not np.isfinite(args.frame_tolerance_mm) or args.frame_tolerance_mm <= 0.0: |
| 62 | raise ValueError( | 62 | raise ValueError( |
| 63 | f"--frame-tolerance-mm must be positive, got {args.frame_tolerance_mm}" | 63 | f"--frame-tolerance-mm must be positive, got {args.frame_tolerance_mm}" |
| 64 | ) | 64 | ) |
| 65 | config = load_config(args.config) | 65 | config = HarnessConfig.from_yaml(args.config) |
| 66 | # Per-corridor adapter manifests live in the canonical lane | 66 | # Per-corridor adapter manifests live in the canonical lane |
| 67 | # (<canonical_root>/canonical/<corridor>.adapter.json); the legacy | 67 | # (<canonical_root>/canonical/<corridor>.adapter.json); the legacy |
| 68 | # top-level location is still accepted so older lanes verify. | 68 | # top-level location is still accepted so older lanes verify. |
| 69 | canonical_lane = config.data.canonical_root / "canonical" | 69 | canonical_lane = config.data.canonical_root / "canonical" |
| 32 | validate_full_resolution_manifest, | 32 | validate_full_resolution_manifest, |
| 33 | write_oracle_outputs, | 33 | write_oracle_outputs, |
| 34 | ) | 34 | ) |
| 35 | from src.train.config import ( | 35 | from src.train.config import ( |
| 36 | ConfigError, | 36 | ExperimentConfigError, |
| 37 | HarnessConfig, | 37 | HarnessConfig, |
| 38 | load_config, | ||
| 39 | resolve_ontology_path, | 38 | resolve_ontology_path, |
| 40 | ) | 39 | ) |
| 41 | 40 | ||
| 42 | LOGGER = logging.getLogger(__name__) | 41 | LOGGER = logging.getLogger(__name__) |
| 82 | """ | 81 | """ |
| 83 | args = build_parser().parse_args(argv) | 82 | args = build_parser().parse_args(argv) |
| 84 | logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") | 83 | logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") |
| 85 | try: | 84 | try: |
| 86 | config = load_config(args.config) | 85 | config = HarnessConfig.from_yaml(args.config) |
| 87 | if config.experiment.id != "E2": | 86 | if config.experiment.id != "E2": |
| 88 | raise VoxelOracleError( | 87 | raise VoxelOracleError( |
| 89 | f"scripts/voxel_oracle.py accepts E2 only, got {config.experiment.id}" | 88 | f"scripts/voxel_oracle.py accepts E2 only, got {config.experiment.id}" |
| 90 | ) | 89 | ) |
| 159 | ) | 158 | ) |
| 160 | LOGGER.info("Wrote E2 products: %s", ", ".join(str(path) for path in paths)) | 159 | LOGGER.info("Wrote E2 products: %s", ", ".join(str(path) for path in paths)) |
| 161 | return 0 | 160 | return 0 |
| 162 | except ( | 161 | except ( |
| 163 | ConfigError, | 162 | ExperimentConfigError, |
| 164 | SplitAccessError, | 163 | SplitAccessError, |
| 165 | VoxelOracleError, | 164 | VoxelOracleError, |
| 166 | OSError, | 165 | OSError, |
| 167 | ValueError, | 166 | ValueError, |
| 1 | """Public strict configuration and experiment-dispatch interfaces.""" | 1 | """Public strict configuration and experiment-dispatch interfaces.""" |
| 2 | 2 | ||
| 3 | from src.train.config import ConfigError, HarnessConfig, TrainConfig, load_config | 3 | from src.train.config import ExperimentConfigError, HarnessConfig, TrainConfig |
| 4 | from src.train.dispatch import DispatchError, dispatch_experiment | 4 | from src.train.dispatch import DispatchError, dispatch_experiment |
| 5 | 5 | ||
| 6 | __all__ = [ | 6 | __all__ = [ |
| 7 | "ConfigError", | ||
| 8 | "DispatchError", | 7 | "DispatchError", |
| 8 | "ExperimentConfigError", | ||
| 9 | "HarnessConfig", | 9 | "HarnessConfig", |
| 10 | "TrainConfig", | 10 | "TrainConfig", |
| 11 | "dispatch_experiment", | 11 | "dispatch_experiment", |
| 12 | "load_config", | ||
| 13 | ] | 12 | ] |
| 36 | Returns: | 36 | Returns: |
| 37 | The deterministic cell definitions of the declared study kind. | 37 | The deterministic cell definitions of the declared study kind. |
| 38 | 38 | ||
| 39 | Raises: | 39 | Raises: |
| 40 | ConfigError: If the study payload of the declared kind is absent or | 40 | ExperimentConfigError: If the study payload of the declared kind is absent or |
| 41 | expands to nothing. | 41 | expands to nothing. |
| 42 | """ | 42 | """ |
| 43 | study = config.study | 43 | study = config.study |
| 44 | if study.kind == "single": | 44 | if study.kind == "single": |
| 45 | return ((config.experiment.id, MappingProxyType({})),) | 45 | return ((config.experiment.id, MappingProxyType({})),) |
| 46 | if study.kind == "variants": | 46 | if study.kind == "variants": |
| 47 | if not study.variants: | 47 | if not study.variants: |
| 48 | raise config_values.ConfigError( | 48 | raise config_values.ExperimentConfigError( |
| 49 | f"{config.experiment.id} study.kind variants has no variants payload" | 49 | f"{config.experiment.id} study.kind variants has no variants payload" |
| 50 | ) | 50 | ) |
| 51 | return tuple((item.id, item.overrides) for item in study.variants) | 51 | return tuple((item.id, item.overrides) for item in study.variants) |
| 52 | if study.kind == "matrix": | 52 | if study.kind == "matrix": |
| 53 | if study.matrix is None: | 53 | if study.matrix is None: |
| 54 | raise config_values.ConfigError( | 54 | raise config_values.ExperimentConfigError( |
| 55 | f"{config.experiment.id} study.kind matrix has no matrix payload" | 55 | f"{config.experiment.id} study.kind matrix has no matrix payload" |
| 56 | ) | 56 | ) |
| 57 | return matrix_cells(study.matrix) | 57 | return matrix_cells(study.matrix) |
| 58 | if study.sweep is None: | 58 | if study.sweep is None: |
| 59 | raise config_values.ConfigError( | 59 | raise config_values.ExperimentConfigError( |
| 60 | f"{config.experiment.id} study.kind sweep has no sweep payload" | 60 | f"{config.experiment.id} study.kind sweep has no sweep payload" |
| 61 | ) | 61 | ) |
| 62 | return sweep_cells(study.sweep, config.seed) | 62 | return sweep_cells(study.sweep, config.seed) |
| 63 | 63 |
| 68 | unknown = sorted( | 68 | unknown = sorted( |
| 69 | {path for item in matrix.exclude for path in item} - set(axis_paths) | 69 | {path for item in matrix.exclude for path in item} - set(axis_paths) |
| 70 | ) | 70 | ) |
| 71 | if unknown: | 71 | if unknown: |
| 72 | raise config_values.ConfigError( | 72 | raise config_values.ExperimentConfigError( |
| 73 | f"study.matrix.exclude references non-axis paths {unknown}" | 73 | f"study.matrix.exclude references non-axis paths {unknown}" |
| 74 | ) | 74 | ) |
| 75 | definitions: list[CellDefinition] = [] | 75 | definitions: list[CellDefinition] = [] |
| 76 | excluded = [0] * len(matrix.exclude) | 76 | excluded = [0] * len(matrix.exclude) |
| 84 | if not dropped: | 84 | if not dropped: |
| 85 | definitions.append((cell_id(overrides), MappingProxyType(overrides))) | 85 | definitions.append((cell_id(overrides), MappingProxyType(overrides))) |
| 86 | for index, count in enumerate(excluded): | 86 | for index, count in enumerate(excluded): |
| 87 | if not count: | 87 | if not count: |
| 88 | raise config_values.ConfigError( | 88 | raise config_values.ExperimentConfigError( |
| 89 | f"study.matrix.exclude[{index}] matches no matrix cell" | 89 | f"study.matrix.exclude[{index}] matches no matrix cell" |
| 90 | ) | 90 | ) |
| 91 | for index, item in enumerate(matrix.include): | 91 | for index, item in enumerate(matrix.include): |
| 92 | if not item: | 92 | if not item: |
| 93 | raise config_values.ConfigError( | 93 | raise config_values.ExperimentConfigError( |
| 94 | f"study.matrix.include[{index}] cannot be empty" | 94 | f"study.matrix.include[{index}] cannot be empty" |
| 95 | ) | 95 | ) |
| 96 | definitions.append((cell_id(item), item)) | 96 | definitions.append((cell_id(item), item)) |
| 97 | if not definitions: | 97 | if not definitions: |
| 98 | raise config_values.ConfigError("study.matrix excludes every cell") | 98 | raise config_values.ExperimentConfigError("study.matrix excludes every cell") |
| 99 | return tuple(definitions) | 99 | return tuple(definitions) |
| 100 | 100 | ||
| 101 | 101 | ||
| 102 | def sweep_cells( | 102 | def sweep_cells( |
| 108 | unbounded = sorted( | 108 | unbounded = sorted( |
| 109 | path for path, item in sweep.parameters.items() if item.values is None | 109 | path for path, item in sweep.parameters.items() if item.values is None |
| 110 | ) | 110 | ) |
| 111 | if unbounded: | 111 | if unbounded: |
| 112 | raise config_values.ConfigError( | 112 | raise config_values.ExperimentConfigError( |
| 113 | f"study.sweep.method grid requires explicit values for {unbounded}" | 113 | f"study.sweep.method grid requires explicit values for {unbounded}" |
| 114 | ) | 114 | ) |
| 115 | definitions: list[CellDefinition] = [] | 115 | definitions: list[CellDefinition] = [] |
| 116 | for combination in itertools.product( | 116 | for combination in itertools.product( |
| 136 | ), | 136 | ), |
| 137 | ) | 137 | ) |
| 138 | for index in range(sweep.budget) | 138 | for index in range(sweep.budget) |
| 139 | ) | 139 | ) |
| 140 | raise config_values.ConfigError( | 140 | raise config_values.ExperimentConfigError( |
| 141 | f"study.sweep.method {sweep.method!r} is not implemented; supported " | 141 | f"study.sweep.method {sweep.method!r} is not implemented; supported " |
| 142 | "methods are grid and random" | 142 | "methods are grid and random" |
| 143 | ) | 143 | ) |
| 144 | 144 |
| 151 | """Draw one deterministic value for a sweep parameter.""" | 151 | """Draw one deterministic value for a sweep parameter.""" |
| 152 | if parameter.values is not None: | 152 | if parameter.values is not None: |
| 153 | return parameter.values[generator.randrange(len(parameter.values))] | 153 | return parameter.values[generator.randrange(len(parameter.values))] |
| 154 | if parameter.minimum is None or parameter.maximum is None: | 154 | if parameter.minimum is None or parameter.maximum is None: |
| 155 | raise config_values.ConfigError( | 155 | raise config_values.ExperimentConfigError( |
| 156 | f"{where} requires minimum and maximum for sampling" | 156 | f"{where} requires minimum and maximum for sampling" |
| 157 | ) | 157 | ) |
| 158 | if parameter.distribution == "uniform": | 158 | if parameter.distribution == "uniform": |
| 159 | drawn = generator.uniform(parameter.minimum, parameter.maximum) | 159 | drawn = generator.uniform(parameter.minimum, parameter.maximum) |
| 160 | elif parameter.distribution == "log_uniform": | 160 | elif parameter.distribution == "log_uniform": |
| 161 | if parameter.minimum <= 0.0: | 161 | if parameter.minimum <= 0.0: |
| 162 | raise config_values.ConfigError( | 162 | raise config_values.ExperimentConfigError( |
| 163 | f"{where}.minimum must be positive for log_uniform" | 163 | f"{where}.minimum must be positive for log_uniform" |
| 164 | ) | 164 | ) |
| 165 | drawn = math.exp( | 165 | drawn = math.exp( |
| 166 | generator.uniform(math.log(parameter.minimum), math.log(parameter.maximum)) | 166 | generator.uniform(math.log(parameter.minimum), math.log(parameter.maximum)) |
| 167 | ) | 167 | ) |
| 168 | else: | 168 | else: |
| 169 | raise config_values.ConfigError( | 169 | raise config_values.ExperimentConfigError( |
| 170 | f"{where}.distribution {parameter.distribution!r} is not implemented; " | 170 | f"{where}.distribution {parameter.distribution!r} is not implemented; " |
| 171 | "supported distributions are uniform and log_uniform" | 171 | "supported distributions are uniform and log_uniform" |
| 172 | ) | 172 | ) |
| 173 | return float(f"{drawn:.6g}") | 173 | return float(f"{drawn:.6g}") |
| 175 | 175 | ||
| 176 | def cell_id(overrides: Mapping[str, config_values.OverrideValue]) -> str: | 176 | def cell_id(overrides: Mapping[str, config_values.OverrideValue]) -> str: |
| 177 | """Derive a stable, filesystem-safe identity from a cell's overrides.""" | 177 | """Derive a stable, filesystem-safe identity from a cell's overrides.""" |
| 178 | if not overrides: | 178 | if not overrides: |
| 179 | raise config_values.ConfigError("A study cell requires at least one override") | 179 | raise config_values.ExperimentConfigError( |
| 180 | "A study cell requires at least one override" | ||
| 181 | ) | ||
| 180 | names = [path.rsplit(".", 1)[-1] for path in overrides] | 182 | names = [path.rsplit(".", 1)[-1] for path in overrides] |
| 181 | if len(set(names)) != len(names): | 183 | if len(set(names)) != len(names): |
| 182 | names = [path.replace(".", "_") for path in overrides] | 184 | names = [path.replace(".", "_") for path in overrides] |
| 183 | return "__".join( | 185 | return "__".join( |
| 216 | Returns: | 218 | Returns: |
| 217 | A deep copy of ``raw`` carrying the overridden leaves. | 219 | A deep copy of ``raw`` carrying the overridden leaves. |
| 218 | 220 | ||
| 219 | Raises: | 221 | Raises: |
| 220 | ConfigError: If a path is not a declared leaf or the value type differs. | 222 | ExperimentConfigError: If a path is not a declared leaf or the value |
| 223 | type differs. | ||
| 221 | """ | 224 | """ |
| 222 | result = copy.deepcopy(dict(raw)) | 225 | result = copy.deepcopy(dict(raw)) |
| 223 | leaves = config_values.leaf_values(raw) | 226 | leaves = config_values.leaf_values(raw) |
| 224 | for path in sorted(overrides): | 227 | for path in sorted(overrides): |
| 225 | value = overrides[path] | 228 | value = overrides[path] |
| 226 | if path.startswith("study.") or path not in leaves: | 229 | if path.startswith("study.") or path not in leaves: |
| 227 | raise config_values.ConfigError( | 230 | raise config_values.ExperimentConfigError( |
| 228 | f"{where} override path {path!r} is not a declared scalar/list leaf" | 231 | f"{where} override path {path!r} is not a declared scalar/list leaf" |
| 229 | ) | 232 | ) |
| 230 | expected = leaves[path] | 233 | expected = leaves[path] |
| 231 | if not config_values.same_leaf_type(expected, value): | 234 | if not config_values.same_leaf_type(expected, value): |
| 232 | raise config_values.ConfigError( | 235 | raise config_values.ExperimentConfigError( |
| 233 | f"{where} override {path!r} has incompatible value {value!r}; " | 236 | f"{where} override {path!r} has incompatible value {value!r}; " |
| 234 | f"expected type of {expected!r}" | 237 | f"expected type of {expected!r}" |
| 235 | ) | 238 | ) |
| 236 | _set_leaf(result, path, config_values.coerce_leaf(expected, value), where) | 239 | _set_leaf(result, path, config_values.coerce_leaf(expected, value), where) |
| 247 | segments = path.split(".") | 250 | segments = path.split(".") |
| 248 | node: Any = target | 251 | node: Any = target |
| 249 | for segment in segments[:-1]: | 252 | for segment in segments[:-1]: |
| 250 | if not isinstance(node, dict) or segment not in node: | 253 | if not isinstance(node, dict) or segment not in node: |
| 251 | raise config_values.ConfigError( | 254 | raise config_values.ExperimentConfigError( |
| 252 | f"{where} override path {path!r} is not addressable" | 255 | f"{where} override path {path!r} is not addressable" |
| 253 | ) | 256 | ) |
| 254 | node = node[segment] | 257 | node = node[segment] |
| 255 | if not isinstance(node, dict) or segments[-1] not in node: | 258 | if not isinstance(node, dict) or segments[-1] not in node: |
| 256 | raise config_values.ConfigError( | 259 | raise config_values.ExperimentConfigError( |
| 257 | f"{where} override path {path!r} is not addressable" | 260 | f"{where} override path {path!r} is not addressable" |
| 258 | ) | 261 | ) |
| 259 | node[segments[-1]] = value | 262 | node[segments[-1]] = value |
| 14 | 14 | ||
| 15 | from src.train.config import ( | 15 | from src.train.config import ( |
| 16 | ArtifactExistsParams, | 16 | ArtifactExistsParams, |
| 17 | CheckpointPolicyParams, | 17 | CheckpointPolicyParams, |
| 18 | ConfigError, | ||
| 19 | DataAvailableParams, | 18 | DataAvailableParams, |
| 19 | ExperimentConfigError, | ||
| 20 | GateConfig, | 20 | GateConfig, |
| 21 | HarnessConfig, | 21 | HarnessConfig, |
| 22 | HumanWorkflowParams, | 22 | HumanWorkflowParams, |
| 23 | ImplementationTicketParams, | 23 | ImplementationTicketParams, |
| 105 | DispatchError: If the study definition cannot be expanded strictly. | 105 | DispatchError: If the study definition cannot be expanded strictly. |
| 106 | """ | 106 | """ |
| 107 | try: | 107 | try: |
| 108 | return expand_study(config) | 108 | return expand_study(config) |
| 109 | except ConfigError as exc: | 109 | except ExperimentConfigError as exc: |
| 110 | raise DispatchError( | 110 | raise DispatchError( |
| 111 | f"{config.experiment.id} study expansion failed: {exc}" | 111 | f"{config.experiment.id} study expansion failed: {exc}" |
| 112 | ) from exc | 112 | ) from exc |
| 113 | 113 |
| 5 | from pathlib import Path | 5 | from pathlib import Path |
| 6 | 6 | ||
| 7 | from src.contracts.ontology import load_ontology | 7 | from src.contracts.ontology import load_ontology |
| 8 | from src.contracts.splits import SplitTier, load_split_manifest | 8 | from src.contracts.splits import SplitTier, load_split_manifest |
| 9 | from src.train.config import load_config | 9 | from src.train.config import HarnessConfig |
| 10 | 10 | ||
| 11 | CONFIG_PATH = Path("configs/dev/a1_recap_segment_085.yaml") | 11 | CONFIG_PATH = Path("configs/dev/a1_recap_segment_085.yaml") |
| 12 | ONTOLOGY_PATH = Path("configs/contracts/ontology_v2.yaml") | 12 | ONTOLOGY_PATH = Path("configs/contracts/ontology_v2.yaml") |
| 13 | SPLIT_PATH = Path("configs/contracts/corridor_splits_a1_recap_v1.yaml") | 13 | SPLIT_PATH = Path("configs/contracts/corridor_splits_a1_recap_v1.yaml") |
| 30 | assert manifest.supported_interest_ids[SplitTier.TRAIN] == tuple(range(9)) | 30 | assert manifest.supported_interest_ids[SplitTier.TRAIN] == tuple(range(9)) |
| 31 | 31 | ||
| 32 | 32 | ||
| 33 | def test_recap_config_points_at_the_recap_lanes_with_rgb_and_intensity() -> None: | 33 | def test_recap_config_points_at_the_recap_lanes_with_rgb_and_intensity() -> None: |
| 34 | config = load_config(CONFIG_PATH) | 34 | config = HarnessConfig.from_yaml(CONFIG_PATH) |
| 35 | 35 | ||
| 36 | assert config.data.split_manifest == SPLIT_PATH | 36 | assert config.data.split_manifest == SPLIT_PATH |
| 37 | assert config.data.root == Path("data/00_external/a1_recap_v1") | 37 | assert config.data.root == Path("data/00_external/a1_recap_v1") |
| 38 | assert config.data.corridors.include == (CORRIDOR_ID,) | 38 | assert config.data.corridors.include == (CORRIDOR_ID,) |
| 41 | SplitTier, | 41 | SplitTier, |
| 42 | authorize_split_access, | 42 | authorize_split_access, |
| 43 | load_split_manifest, | 43 | load_split_manifest, |
| 44 | ) | 44 | ) |
| 45 | from src.train.config import HarnessConfig, load_config | 45 | from src.train.config import HarnessConfig |
| 46 | 46 | ||
| 47 | CONFIG_PATH = Path("configs/e01_spt_pilot.yaml") | 47 | CONFIG_PATH = Path("configs/e01_spt_pilot.yaml") |
| 48 | POLICY_PATH = Path("configs/contracts/checkpoint_policy.yaml") | 48 | POLICY_PATH = Path("configs/contracts/checkpoint_policy.yaml") |
| 49 | GRID_ORIGIN = [10.0, -20.0, 5.0] | 49 | GRID_ORIGIN = [10.0, -20.0, 5.0] |
| 145 | 145 | ||
| 146 | Returns: | 146 | Returns: |
| 147 | The E1 configuration with a writable canonical root. | 147 | The E1 configuration with a writable canonical root. |
| 148 | """ | 148 | """ |
| 149 | config = load_config(CONFIG_PATH) | 149 | config = HarnessConfig.from_yaml(CONFIG_PATH) |
| 150 | canonical_root = tmp_path / "canonical_root" | 150 | canonical_root = tmp_path / "canonical_root" |
| 151 | (canonical_root / "canonical").mkdir(parents=True, exist_ok=True) | 151 | (canonical_root / "canonical").mkdir(parents=True, exist_ok=True) |
| 152 | for corridor_id in config.data.corridors.include: | 152 | for corridor_id in config.data.corridors.include: |
| 153 | (canonical_root / "canonical" / f"{corridor_id}.adapter.json").write_text( | 153 | (canonical_root / "canonical" / f"{corridor_id}.adapter.json").write_text( |
| 6 | from typing import Any | 6 | from typing import Any |
| 7 | 7 | ||
| 8 | import pytest | 8 | import pytest |
| 9 | import yaml | 9 | import yaml |
| 10 | from iolabs.common import config_loader | ||
| 10 | 11 | ||
| 11 | from src.contracts.ontology import load_ontology | 12 | from src.contracts.ontology import load_ontology |
| 12 | from src.train.config import ( | 13 | from src.train.config import ( |
| 13 | ConfigError, | 14 | ExperimentConfigError, |
| 15 | HarnessConfig, | ||
| 14 | is_canonical_metric, | 16 | is_canonical_metric, |
| 15 | load_config, | ||
| 16 | resolve_ontology_path, | 17 | resolve_ontology_path, |
| 17 | ) | 18 | ) |
| 18 | from src.train.dispatch import DispatchError, dispatch_experiment | 19 | from src.train.dispatch import DispatchError, dispatch_experiment |
| 19 | 20 |
| 24 | 25 | ||
| 25 | 26 | ||
| 26 | def test_all_fourteen_configs_parse_with_frozen_statuses() -> None: | 27 | def test_all_fourteen_configs_parse_with_frozen_statuses() -> None: |
| 27 | """Every experiment parses and matches the section 6 status matrix.""" | 28 | """Every experiment parses and matches the section 6 status matrix.""" |
| 28 | configs = [load_config(path) for path in CONFIGS] | 29 | configs = [HarnessConfig.from_yaml(path) for path in CONFIGS] |
| 29 | assert [item.experiment.id for item in configs] == [ | 30 | assert [item.experiment.id for item in configs] == [ |
| 30 | f"E{index}" for index in range(1, 15) | 31 | f"E{index}" for index in range(1, 15) |
| 31 | ] | 32 | ] |
| 32 | assert [item.experiment.status for item in configs] == [ | 33 | assert [item.experiment.status for item in configs] == [ |
| 52 | "sweep", | 53 | "sweep", |
| 53 | } | 54 | } |
| 54 | 55 | ||
| 55 | 56 | ||
| 57 | def test_error_class_is_config_error() -> None: | ||
| 58 | """The package error class is a fleet config error and a ValueError.""" | ||
| 59 | assert issubclass(ExperimentConfigError, config_loader.ConfigError) | ||
| 60 | assert issubclass(ExperimentConfigError, ValueError) | ||
| 61 | |||
| 62 | |||
| 63 | def test_unknown_top_level_key_is_rejected(tmp_path: Path) -> None: | ||
| 64 | """Unknown keys fail at the document root.""" | ||
| 65 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) | ||
| 66 | raw["surprise_section"] = {"enabled": True} | ||
| 67 | path = tmp_path / "unknown_root.yaml" | ||
| 68 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | ||
| 69 | with pytest.raises(ExperimentConfigError, match="surprise_section"): | ||
| 70 | HarnessConfig.from_yaml(path) | ||
| 71 | |||
| 72 | |||
| 56 | def test_unknown_nested_key_is_rejected(tmp_path: Path) -> None: | 73 | def test_unknown_nested_key_is_rejected(tmp_path: Path) -> None: |
| 57 | """Unknown keys fail at nested section boundaries.""" | 74 | """Unknown keys fail at nested section boundaries.""" |
| 58 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) | 75 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) |
| 59 | raw["train"]["surprise_callback"] = True | 76 | raw["train"]["surprise_callback"] = True |
| 60 | path = tmp_path / "unknown.yaml" | 77 | path = tmp_path / "unknown.yaml" |
| 61 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 78 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 62 | with pytest.raises(ConfigError, match="surprise_callback"): | 79 | with pytest.raises(ExperimentConfigError, match="surprise_callback"): |
| 63 | load_config(path) | 80 | HarnessConfig.from_yaml(path) |
| 64 | 81 | ||
| 65 | 82 | ||
| 66 | def test_incompatible_study_override_is_rejected(tmp_path: Path) -> None: | 83 | def test_incompatible_study_override_is_rejected(tmp_path: Path) -> None: |
| 67 | """Dotted overrides must name an existing leaf and preserve its type.""" | 84 | """Dotted overrides must name an existing leaf and preserve its type.""" |
| 70 | "adapter.spt.voxel_m": "two centimetres" | 87 | "adapter.spt.voxel_m": "two centimetres" |
| 71 | } | 88 | } |
| 72 | path = tmp_path / "bad_override.yaml" | 89 | path = tmp_path / "bad_override.yaml" |
| 73 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 90 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 74 | with pytest.raises(ConfigError, match="incompatible value"): | 91 | with pytest.raises(ExperimentConfigError, match="incompatible value"): |
| 75 | load_config(path) | 92 | HarnessConfig.from_yaml(path) |
| 76 | 93 | ||
| 77 | 94 | ||
| 78 | def test_external_framework_visualizer_fields_are_forbidden(tmp_path: Path) -> None: | 95 | def test_external_framework_visualizer_fields_are_forbidden(tmp_path: Path) -> None: |
| 79 | """SPT/Pointcept YAML cannot configure image-loop visualization callbacks.""" | 96 | """SPT/Pointcept YAML cannot configure image-loop visualization callbacks.""" |
| 80 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) | 97 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) |
| 81 | raw["train"]["viz_samples"] = 4 | 98 | raw["train"]["viz_samples"] = 4 |
| 82 | path = tmp_path / "external_viz.yaml" | 99 | path = tmp_path / "external_viz.yaml" |
| 83 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 100 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 84 | with pytest.raises(ConfigError, match="forbidden"): | 101 | with pytest.raises(ExperimentConfigError, match="forbidden"): |
| 85 | load_config(path) | 102 | HarnessConfig.from_yaml(path) |
| 86 | 103 | ||
| 87 | 104 | ||
| 88 | def test_visualization_block_parses_int_and_named_tiles(tmp_path: Path) -> None: | 105 | def test_visualization_block_parses_int_and_named_tiles(tmp_path: Path) -> None: |
| 89 | """A valid visualization block is optional and strictly typed.""" | 106 | """A valid visualization block is optional and strictly typed.""" |
| 90 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) | 107 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) |
| 91 | raw["visualization"] = {"masks_every_n_epochs": 2, "masks_tiles": 3} | 108 | raw["visualization"] = {"masks_every_n_epochs": 2, "masks_tiles": 3} |
| 92 | path = tmp_path / "viz.yaml" | 109 | path = tmp_path / "viz.yaml" |
| 93 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 110 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 94 | config = load_config(path) | 111 | config = HarnessConfig.from_yaml(path) |
| 95 | assert config.visualization is not None | 112 | assert config.visualization is not None |
| 96 | assert config.visualization.masks_every_n_epochs == 2 | 113 | assert config.visualization.masks_every_n_epochs == 2 |
| 97 | assert config.visualization.masks_tiles == 3 | 114 | assert config.visualization.masks_tiles == 3 |
| 98 | 115 |
| 100 | "masks_every_n_epochs": 1, | 117 | "masks_every_n_epochs": 1, |
| 101 | "masks_tiles": ["tile_a", "tile_b"], | 118 | "masks_tiles": ["tile_a", "tile_b"], |
| 102 | } | 119 | } |
| 103 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 120 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 104 | named = load_config(path) | 121 | named = HarnessConfig.from_yaml(path) |
| 105 | assert named.visualization is not None | 122 | assert named.visualization is not None |
| 106 | assert named.visualization.masks_tiles == ("tile_a", "tile_b") | 123 | assert named.visualization.masks_tiles == ("tile_a", "tile_b") |
| 107 | 124 | ||
| 108 | raw["visualization"] = {"masks_every_n_epochs": 4} | 125 | raw["visualization"] = {"masks_every_n_epochs": 4} |
| 109 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 126 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 110 | defaulted = load_config(path) | 127 | defaulted = HarnessConfig.from_yaml(path) |
| 111 | assert defaulted.visualization is not None | 128 | assert defaulted.visualization is not None |
| 112 | assert defaulted.visualization.masks_tiles == 2 | 129 | assert defaulted.visualization.masks_tiles == 2 |
| 113 | 130 | ||
| 114 | 131 |
| 121 | "surprise": True, | 138 | "surprise": True, |
| 122 | } | 139 | } |
| 123 | path = tmp_path / "viz_unknown.yaml" | 140 | path = tmp_path / "viz_unknown.yaml" |
| 124 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 141 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 125 | with pytest.raises(ConfigError, match="visualization.*surprise"): | 142 | with pytest.raises(ExperimentConfigError, match="visualization.*surprise"): |
| 126 | load_config(path) | 143 | HarnessConfig.from_yaml(path) |
| 127 | 144 | ||
| 128 | 145 | ||
| 129 | def test_visualization_every_n_epochs_zero_is_rejected(tmp_path: Path) -> None: | 146 | def test_visualization_every_n_epochs_zero_is_rejected(tmp_path: Path) -> None: |
| 130 | """masks_every_n_epochs must be >= 1 when the block is present.""" | 147 | """masks_every_n_epochs must be >= 1 when the block is present.""" |
| 131 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) | 148 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) |
| 132 | raw["visualization"] = {"masks_every_n_epochs": 0, "masks_tiles": 2} | 149 | raw["visualization"] = {"masks_every_n_epochs": 0, "masks_tiles": 2} |
| 133 | path = tmp_path / "viz_zero.yaml" | 150 | path = tmp_path / "viz_zero.yaml" |
| 134 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 151 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 135 | with pytest.raises(ConfigError, match="masks_every_n_epochs"): | 152 | with pytest.raises(ExperimentConfigError, match="masks_every_n_epochs"): |
| 136 | load_config(path) | 153 | HarnessConfig.from_yaml(path) |
| 137 | 154 | ||
| 138 | 155 | ||
| 139 | def test_visualization_empty_tiles_are_rejected(tmp_path: Path) -> None: | 156 | def test_visualization_empty_tiles_are_rejected(tmp_path: Path) -> None: |
| 140 | """An empty tile list is not a valid selection.""" | 157 | """An empty tile list is not a valid selection.""" |
| 141 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) | 158 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) |
| 142 | raw["visualization"] = {"masks_every_n_epochs": 1, "masks_tiles": []} | 159 | raw["visualization"] = {"masks_every_n_epochs": 1, "masks_tiles": []} |
| 143 | path = tmp_path / "viz_empty.yaml" | 160 | path = tmp_path / "viz_empty.yaml" |
| 144 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 161 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 145 | with pytest.raises(ConfigError, match="masks_tiles"): | 162 | with pytest.raises(ExperimentConfigError, match="masks_tiles"): |
| 146 | load_config(path) | 163 | HarnessConfig.from_yaml(path) |
| 147 | 164 | ||
| 148 | 165 | ||
| 149 | def test_absent_visualization_block_is_none() -> None: | 166 | def test_absent_visualization_block_is_none() -> None: |
| 150 | """Omitting visualization leaves the feature off.""" | 167 | """Omitting visualization leaves the feature off.""" |
| 151 | config = load_config(CONFIGS[1]) | 168 | config = HarnessConfig.from_yaml(CONFIGS[1]) |
| 152 | assert config.visualization is None | 169 | assert config.visualization is None |
| 153 | 170 | ||
| 154 | 171 | ||
| 155 | def test_visualization_requires_pointcept_framework(tmp_path: Path) -> None: | 172 | def test_visualization_requires_pointcept_framework(tmp_path: Path) -> None: |
| 158 | raw = yaml.safe_load(CONFIGS[index].read_text(encoding="utf-8")) | 175 | raw = yaml.safe_load(CONFIGS[index].read_text(encoding="utf-8")) |
| 159 | raw["visualization"] = {"masks_every_n_epochs": 2} | 176 | raw["visualization"] = {"masks_every_n_epochs": 2} |
| 160 | path = tmp_path / f"viz_wrong_framework_{index}.yaml" | 177 | path = tmp_path / f"viz_wrong_framework_{index}.yaml" |
| 161 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 178 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 162 | with pytest.raises(ConfigError, match="only supported for.*pointcept"): | 179 | with pytest.raises( |
| 163 | load_config(path) | 180 | ExperimentConfigError, match="only supported for.*pointcept" |
| 181 | ): | ||
| 182 | HarnessConfig.from_yaml(path) | ||
| 164 | 183 | ||
| 165 | 184 | ||
| 166 | def test_noncanonical_deciding_metric_is_rejected(tmp_path: Path) -> None: | 185 | def test_noncanonical_deciding_metric_is_rejected(tmp_path: Path) -> None: |
| 167 | """Decision metrics cannot silently depend on an unlogged tag.""" | 186 | """Decision metrics cannot silently depend on an unlogged tag.""" |
| 168 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) | 187 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) |
| 169 | raw["experiment"]["deciding_metrics"] = ["val/mystery"] | 188 | raw["experiment"]["deciding_metrics"] = ["val/mystery"] |
| 170 | path = tmp_path / "metric.yaml" | 189 | path = tmp_path / "metric.yaml" |
| 171 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 190 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 172 | with pytest.raises(ConfigError, match="non-canonical"): | 191 | with pytest.raises(ExperimentConfigError, match="non-canonical"): |
| 173 | load_config(path) | 192 | HarnessConfig.from_yaml(path) |
| 174 | 193 | ||
| 175 | 194 | ||
| 176 | @pytest.mark.parametrize( | 195 | @pytest.mark.parametrize( |
| 177 | ("section", "key", "value", "expected"), | 196 | ("section", "key", "value", "expected"), |
| 209 | node[key] = value | 228 | node[key] = value |
| 210 | path = tmp_path / "scalar.yaml" | 229 | path = tmp_path / "scalar.yaml" |
| 211 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 230 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 212 | dotted = ".".join((*section, key)) | 231 | dotted = ".".join((*section, key)) |
| 213 | with pytest.raises(ConfigError) as failure: | 232 | with pytest.raises(ExperimentConfigError) as failure: |
| 214 | load_config(path) | 233 | HarnessConfig.from_yaml(path) |
| 215 | message = str(failure.value) | 234 | message = str(failure.value) |
| 216 | assert str(path) in message | 235 | assert str(path) in message |
| 217 | assert dotted in message | 236 | assert dotted in message |
| 218 | assert expected in message | 237 | assert expected in message |
| 232 | pytest.skip("The reference gate declares no string parameter") | 251 | pytest.skip("The reference gate declares no string parameter") |
| 233 | params[key] = 3 | 252 | params[key] = 3 |
| 234 | path = tmp_path / "gate.yaml" | 253 | path = tmp_path / "gate.yaml" |
| 235 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 254 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 236 | with pytest.raises(ConfigError, match=f"params.{key}"): | 255 | with pytest.raises(ExperimentConfigError, match=f"params.{key}"): |
| 237 | load_config(path) | 256 | HarnessConfig.from_yaml(path) |
| 238 | 257 | ||
| 239 | 258 | ||
| 240 | def test_template_dispatch_names_implementation_ticket() -> None: | 259 | def test_template_dispatch_names_implementation_ticket() -> None: |
| 241 | """A template fails before runner work and identifies its exact ticket.""" | 260 | """A template fails before runner work and identifies its exact ticket.""" |
| 242 | config = load_config("configs/e03_feature_ablation.yaml") | 261 | config = HarnessConfig.from_yaml("configs/e03_feature_ablation.yaml") |
| 243 | with pytest.raises(DispatchError, match="AI3D-MLSEG-E3"): | 262 | with pytest.raises(DispatchError, match="AI3D-MLSEG-E3"): |
| 244 | dispatch_experiment(config) | 263 | dispatch_experiment(config) |
| 245 | 264 | ||
| 246 | 265 | ||
| 247 | def test_gated_dispatch_names_missing_artifact() -> None: | 266 | def test_gated_dispatch_names_missing_artifact() -> None: |
| 248 | """A gated experiment reports its first unmet prerequisite.""" | 267 | """A gated experiment reports its first unmet prerequisite.""" |
| 249 | config = load_config("configs/e07_ezsp_scale.yaml") | 268 | config = HarnessConfig.from_yaml("configs/e07_ezsp_scale.yaml") |
| 250 | with pytest.raises(DispatchError, match="data_contract_acceptance.json"): | 269 | with pytest.raises(DispatchError, match="data_contract_acceptance.json"): |
| 251 | dispatch_experiment(config) | 270 | dispatch_experiment(config) |
| 252 | 271 | ||
| 253 | 272 | ||
| 254 | def test_e1_dispatch_names_partition_oracle_report() -> None: | 273 | def test_e1_dispatch_names_partition_oracle_report() -> None: |
| 255 | """E1 cannot launch without the partition-purity report.""" | 274 | """E1 cannot launch without the partition-purity report.""" |
| 256 | config = load_config("configs/e01_spt_pilot.yaml") | 275 | config = HarnessConfig.from_yaml("configs/e01_spt_pilot.yaml") |
| 257 | with pytest.raises(DispatchError, match="partition_oracle.json"): | 276 | with pytest.raises(DispatchError, match="partition_oracle.json"): |
| 258 | dispatch_experiment(config) | 277 | dispatch_experiment(config) |
| 259 | 278 | ||
| 260 | 279 |
| 263 | ) -> None: | 282 | ) -> None: |
| 264 | """An external experiment names every unset framework variable.""" | 283 | """An external experiment names every unset framework variable.""" |
| 265 | for name in ("SPT_ROOT", "SPT_COMMIT", "SPT_PYTHON"): | 284 | for name in ("SPT_ROOT", "SPT_COMMIT", "SPT_PYTHON"): |
| 266 | monkeypatch.delenv(name, raising=False) | 285 | monkeypatch.delenv(name, raising=False) |
| 267 | config = load_config("configs/e01_spt_pilot.yaml") | 286 | config = HarnessConfig.from_yaml("configs/e01_spt_pilot.yaml") |
| 268 | with pytest.raises(DispatchError, match="SPT_ROOT, SPT_COMMIT, SPT_PYTHON"): | 287 | with pytest.raises(DispatchError, match="SPT_ROOT, SPT_COMMIT, SPT_PYTHON"): |
| 269 | dispatch_experiment(config) | 288 | dispatch_experiment(config) |
| 270 | 289 | ||
| 271 | 290 |
| 276 | checkout = _fake_spt_checkout(tmp_path) | 295 | checkout = _fake_spt_checkout(tmp_path) |
| 277 | monkeypatch.setenv("SPT_ROOT", str(checkout)) | 296 | monkeypatch.setenv("SPT_ROOT", str(checkout)) |
| 278 | monkeypatch.setenv("SPT_COMMIT", "0" * 40) | 297 | monkeypatch.setenv("SPT_COMMIT", "0" * 40) |
| 279 | monkeypatch.setenv("SPT_PYTHON", str(_fake_interpreter(tmp_path))) | 298 | monkeypatch.setenv("SPT_PYTHON", str(_fake_interpreter(tmp_path))) |
| 280 | config = load_config("configs/e01_spt_pilot.yaml") | 299 | config = HarnessConfig.from_yaml("configs/e01_spt_pilot.yaml") |
| 281 | with pytest.raises(DispatchError) as failure: | 300 | with pytest.raises(DispatchError) as failure: |
| 282 | dispatch_experiment(config) | 301 | dispatch_experiment(config) |
| 283 | assert "framework_environment" not in str(failure.value) | 302 | assert "framework_environment" not in str(failure.value) |
| 284 | 303 |
| 305 | ) -> None: | 324 | ) -> None: |
| 306 | """CPU experiments never demand external checkout variables.""" | 325 | """CPU experiments never demand external checkout variables.""" |
| 307 | for name in ("SPT_ROOT", "SPT_COMMIT", "SPT_PYTHON"): | 326 | for name in ("SPT_ROOT", "SPT_COMMIT", "SPT_PYTHON"): |
| 308 | monkeypatch.delenv(name, raising=False) | 327 | monkeypatch.delenv(name, raising=False) |
| 309 | config = load_config("configs/e02_voxel_survival_oracle.yaml") | 328 | config = HarnessConfig.from_yaml("configs/e02_voxel_survival_oracle.yaml") |
| 310 | with pytest.raises(DispatchError) as failure: | 329 | with pytest.raises(DispatchError) as failure: |
| 311 | dispatch_experiment(config) | 330 | dispatch_experiment(config) |
| 312 | assert "framework_environment" not in str(failure.value) | 331 | assert "framework_environment" not in str(failure.value) |
| 313 | 332 |
| 354 | 373 | ||
| 355 | def test_v2_recap_config_loads_against_ontology_v2(tmp_path: Path) -> None: | 374 | def test_v2_recap_config_loads_against_ontology_v2(tmp_path: Path) -> None: |
| 356 | """A v2 task block aligned with recap_semantics_v2 is accepted.""" | 375 | """A v2 task block aligned with recap_semantics_v2 is accepted.""" |
| 357 | path = _write_config(tmp_path, _v2_recap_document(), "v2.yaml") | 376 | path = _write_config(tmp_path, _v2_recap_document(), "v2.yaml") |
| 358 | config = load_config(path) | 377 | config = HarnessConfig.from_yaml(path) |
| 359 | assert config.task.ontology == ONTOLOGY_V2_PATH | 378 | assert config.task.ontology == ONTOLOGY_V2_PATH |
| 360 | assert config.task.num_classes == 11 | 379 | assert config.task.num_classes == 11 |
| 361 | assert config.task.ignore_index == 11 | 380 | assert config.task.ignore_index == 11 |
| 362 | assert config.task.classes_of_interest == tuple(range(9)) | 381 | assert config.task.classes_of_interest == tuple(range(9)) |
| 378 | """task.num_classes must equal ontology.num_predicted_classes.""" | 397 | """task.num_classes must equal ontology.num_predicted_classes.""" |
| 379 | raw = _v2_recap_document() | 398 | raw = _v2_recap_document() |
| 380 | raw["task"]["num_classes"] = 9 | 399 | raw["task"]["num_classes"] = 9 |
| 381 | path = _write_config(tmp_path, raw, "v2_num_classes.yaml") | 400 | path = _write_config(tmp_path, raw, "v2_num_classes.yaml") |
| 382 | with pytest.raises(ConfigError, match="num_predicted_classes"): | 401 | with pytest.raises(ExperimentConfigError, match="num_predicted_classes"): |
| 383 | load_config(path) | 402 | HarnessConfig.from_yaml(path) |
| 384 | 403 | ||
| 385 | 404 | ||
| 386 | def test_v2_config_rejects_incomplete_classes_of_interest( | 405 | def test_v2_config_rejects_incomplete_classes_of_interest( |
| 387 | tmp_path: Path, | 406 | tmp_path: Path, |
| 389 | """task.classes_of_interest must equal ontology.interest_ids.""" | 408 | """task.classes_of_interest must equal ontology.interest_ids.""" |
| 390 | raw = _v2_recap_document() | 409 | raw = _v2_recap_document() |
| 391 | raw["task"]["classes_of_interest"] = list(range(8)) | 410 | raw["task"]["classes_of_interest"] = list(range(8)) |
| 392 | path = _write_config(tmp_path, raw, "v2_interest.yaml") | 411 | path = _write_config(tmp_path, raw, "v2_interest.yaml") |
| 393 | with pytest.raises(ConfigError, match="classes_of_interest"): | 412 | with pytest.raises(ExperimentConfigError, match="classes_of_interest"): |
| 394 | load_config(path) | 413 | HarnessConfig.from_yaml(path) |
| 395 | 414 | ||
| 396 | 415 | ||
| 397 | def test_v2_config_rejects_v1_linear_class_names(tmp_path: Path) -> None: | 416 | def test_v2_config_rejects_v1_linear_class_names(tmp_path: Path) -> None: |
| 398 | """task.linear_classes must be the linear names of the loaded ontology.""" | 417 | """task.linear_classes must be the linear names of the loaded ontology.""" |
| 402 | "wall_noise_barrier", | 421 | "wall_noise_barrier", |
| 403 | "fence_gate", | 422 | "fence_gate", |
| 404 | ] | 423 | ] |
| 405 | path = _write_config(tmp_path, raw, "v2_linear.yaml") | 424 | path = _write_config(tmp_path, raw, "v2_linear.yaml") |
| 406 | with pytest.raises(ConfigError, match="linear_classes"): | 425 | with pytest.raises(ExperimentConfigError, match="linear_classes"): |
| 407 | load_config(path) | 426 | HarnessConfig.from_yaml(path) |
| 408 | 427 | ||
| 409 | 428 | ||
| 410 | def test_v2_config_rejects_v1_precision_floor_class(tmp_path: Path) -> None: | 429 | def test_v2_config_rejects_v1_precision_floor_class(tmp_path: Path) -> None: |
| 411 | """precision_floors keys must be predicted names of the loaded ontology.""" | 430 | """precision_floors keys must be predicted names of the loaded ontology.""" |
| 412 | raw = _v2_recap_document() | 431 | raw = _v2_recap_document() |
| 413 | raw["evaluation"]["precision_floors"] = {"sign_gantry": 0.9} | 432 | raw["evaluation"]["precision_floors"] = {"sign_gantry": 0.9} |
| 414 | path = _write_config(tmp_path, raw, "v2_floors.yaml") | 433 | path = _write_config(tmp_path, raw, "v2_floors.yaml") |
| 415 | with pytest.raises(ConfigError, match="sign_gantry"): | 434 | with pytest.raises(ExperimentConfigError, match="sign_gantry"): |
| 416 | load_config(path) | 435 | HarnessConfig.from_yaml(path) |
| 417 | 436 | ||
| 418 | 437 | ||
| 419 | def test_v2_config_rejects_model_num_classes_mismatch(tmp_path: Path) -> None: | 438 | def test_v2_config_rejects_model_num_classes_mismatch(tmp_path: Path) -> None: |
| 420 | """model.args.num_classes must equal task.num_classes when present.""" | 439 | """model.args.num_classes must equal task.num_classes when present.""" |
| 421 | raw = _v2_recap_document() | 440 | raw = _v2_recap_document() |
| 422 | raw["model"]["args"]["num_classes"] = 9 | 441 | raw["model"]["args"]["num_classes"] = 9 |
| 423 | path = _write_config(tmp_path, raw, "v2_model_classes.yaml") | 442 | path = _write_config(tmp_path, raw, "v2_model_classes.yaml") |
| 424 | with pytest.raises(ConfigError, match="model.args.num_classes"): | 443 | with pytest.raises(ExperimentConfigError, match="model.args.num_classes"): |
| 425 | load_config(path) | 444 | HarnessConfig.from_yaml(path) |
| 426 | 445 | ||
| 427 | 446 | ||
| 428 | def test_macro_interest_all9_is_canonical_only_under_v2() -> None: | 447 | def test_macro_interest_all9_is_canonical_only_under_v2() -> None: |
| 429 | """v2 macros use all9; the frozen v1 all8 tag is not canonical under v2.""" | 448 | """v2 macros use all9; the frozen v1 all8 tag is not canonical under v2.""" |
| 442 | """A v1 ontology has no delineator class, so the monitor is non-canonical.""" | 461 | """A v1 ontology has no delineator class, so the monitor is non-canonical.""" |
| 443 | raw = yaml.safe_load(A1_SMOKE_CONFIG.read_text(encoding="utf-8")) | 462 | raw = yaml.safe_load(A1_SMOKE_CONFIG.read_text(encoding="utf-8")) |
| 444 | raw["train"]["monitor"] = "val/iou_delineator" | 463 | raw["train"]["monitor"] = "val/iou_delineator" |
| 445 | path = _write_config(tmp_path, raw, "v1_delineator.yaml") | 464 | path = _write_config(tmp_path, raw, "v1_delineator.yaml") |
| 446 | with pytest.raises(ConfigError, match="delineator"): | 465 | with pytest.raises(ExperimentConfigError, match="delineator"): |
| 447 | load_config(path) | 466 | HarnessConfig.from_yaml(path) |
| 448 | 467 | ||
| 449 | 468 | ||
| 450 | def test_v2_config_rejects_loss_ignore_index_mismatch(tmp_path: Path) -> None: | 469 | def test_v2_config_rejects_loss_ignore_index_mismatch(tmp_path: Path) -> None: |
| 451 | """loss.args.ignore_index must equal task.ignore_index when present.""" | 470 | """loss.args.ignore_index must equal task.ignore_index when present.""" |
| 452 | raw = _v2_recap_document() | 471 | raw = _v2_recap_document() |
| 453 | raw["loss"]["args"]["ignore_index"] = 9 | 472 | raw["loss"]["args"]["ignore_index"] = 9 |
| 454 | path = _write_config(tmp_path, raw, "v2_loss_ignore_index.yaml") | 473 | path = _write_config(tmp_path, raw, "v2_loss_ignore_index.yaml") |
| 455 | with pytest.raises(ConfigError, match="loss.args.ignore_index"): | 474 | with pytest.raises(ExperimentConfigError, match="loss.args.ignore_index"): |
| 456 | load_config(path) | 475 | HarnessConfig.from_yaml(path) |
| 457 | 476 | ||
| 458 | 477 | ||
| 459 | def test_v2_config_rejects_cluster_profiles_from_another_ontology( | 478 | def test_v2_config_rejects_cluster_profiles_from_another_ontology( |
| 460 | tmp_path: Path, | 479 | tmp_path: Path, |
| 464 | raw["evaluation"]["object_matching"]["cluster_profiles"] = ( | 483 | raw["evaluation"]["object_matching"]["cluster_profiles"] = ( |
| 465 | "configs/contracts/ontology_v1.yaml" | 484 | "configs/contracts/ontology_v1.yaml" |
| 466 | ) | 485 | ) |
| 467 | path = _write_config(tmp_path, raw, "v2_cluster_profiles.yaml") | 486 | path = _write_config(tmp_path, raw, "v2_cluster_profiles.yaml") |
| 468 | with pytest.raises(ConfigError, match="cluster_profiles"): | 487 | with pytest.raises(ExperimentConfigError, match="cluster_profiles"): |
| 469 | load_config(path) | 488 | HarnessConfig.from_yaml(path) |
| 470 | 489 | ||
| 471 | 490 | ||
| 472 | def test_v2_config_accepts_all9_deciding_metric(tmp_path: Path) -> None: | 491 | def test_v2_config_accepts_all9_deciding_metric(tmp_path: Path) -> None: |
| 473 | """The structural macro suffix follows the ontology's interest count.""" | 492 | """The structural macro suffix follows the ontology's interest count.""" |
| 474 | raw = _v2_recap_document() | 493 | raw = _v2_recap_document() |
| 475 | raw["experiment"]["deciding_metrics"] = ["val/iou_macro_interest_all9"] | 494 | raw["experiment"]["deciding_metrics"] = ["val/iou_macro_interest_all9"] |
| 476 | path = _write_config(tmp_path, raw, "v2_all9_metric.yaml") | 495 | path = _write_config(tmp_path, raw, "v2_all9_metric.yaml") |
| 477 | 496 | ||
| 478 | config = load_config(path) | 497 | config = HarnessConfig.from_yaml(path) |
| 479 | 498 | ||
| 480 | assert config.experiment.deciding_metrics == ("val/iou_macro_interest_all9",) | 499 | assert config.experiment.deciding_metrics == ("val/iou_macro_interest_all9",) |
| 481 | 500 | ||
| 482 | 501 |
| 484 | """The frozen v1 macro tag is not canonical for a nine-interest ontology.""" | 503 | """The frozen v1 macro tag is not canonical for a nine-interest ontology.""" |
| 485 | raw = _v2_recap_document() | 504 | raw = _v2_recap_document() |
| 486 | raw["experiment"]["deciding_metrics"] = ["val/iou_macro_interest_all8"] | 505 | raw["experiment"]["deciding_metrics"] = ["val/iou_macro_interest_all8"] |
| 487 | path = _write_config(tmp_path, raw, "v2_all8_metric.yaml") | 506 | path = _write_config(tmp_path, raw, "v2_all8_metric.yaml") |
| 488 | with pytest.raises(ConfigError, match="deciding_metrics"): | 507 | with pytest.raises(ExperimentConfigError, match="deciding_metrics"): |
| 489 | load_config(path) | 508 | HarnessConfig.from_yaml(path) |
| 490 | 509 | ||
| 491 | 510 | ||
| 492 | def test_v2_gate_rejects_purity_class_from_another_ontology( | 511 | def test_v2_gate_rejects_purity_class_from_another_ontology( |
| 493 | tmp_path: Path, | 512 | tmp_path: Path, |
| 505 | }, | 524 | }, |
| 506 | } | 525 | } |
| 507 | ] | 526 | ] |
| 508 | path = _write_config(tmp_path, raw, "v2_gate_purity.yaml") | 527 | path = _write_config(tmp_path, raw, "v2_gate_purity.yaml") |
| 509 | with pytest.raises(ConfigError, match="sign_gantry"): | 528 | with pytest.raises(ExperimentConfigError, match="sign_gantry"): |
| 510 | load_config(path) | 529 | HarnessConfig.from_yaml(path) |
| 511 | 530 | ||
| 512 | 531 | ||
| 513 | def test_v2_gate_accepts_purity_class_of_the_task_ontology( | 532 | def test_v2_gate_accepts_purity_class_of_the_task_ontology( |
| 514 | tmp_path: Path, | 533 | tmp_path: Path, |
| 527 | } | 546 | } |
| 528 | ] | 547 | ] |
| 529 | path = _write_config(tmp_path, raw, "v2_gate_purity_ok.yaml") | 548 | path = _write_config(tmp_path, raw, "v2_gate_purity_ok.yaml") |
| 530 | 549 | ||
| 531 | config = load_config(path) | 550 | config = HarnessConfig.from_yaml(path) |
| 532 | 551 | ||
| 533 | assert config.experiment.gates[0].params.minimum_purity_by_class == { | 552 | assert config.experiment.gates[0].params.minimum_purity_by_class == { |
| 534 | "delineator": 0.8 | 553 | "delineator": 0.8 |
| 535 | } | 554 | } |
| 545 | raw["task"]["ontology"] = broken.name | 564 | raw["task"]["ontology"] = broken.name |
| 546 | raw["evaluation"]["object_matching"]["cluster_profiles"] = broken.name | 565 | raw["evaluation"]["object_matching"]["cluster_profiles"] = broken.name |
| 547 | path = _write_config(tmp_path, raw, "broken_ontology_config.yaml") | 566 | path = _write_config(tmp_path, raw, "broken_ontology_config.yaml") |
| 548 | with pytest.raises( | 567 | with pytest.raises( |
| 549 | ConfigError, match=r"task\.ontology .* is not a valid ontology" | 568 | ExperimentConfigError, match=r"task\.ontology .* is not a valid ontology" |
| 550 | ): | 569 | ): |
| 551 | load_config(path) | 570 | HarnessConfig.from_yaml(path) |
| 552 | 571 | ||
| 553 | 572 | ||
| 554 | def test_missing_task_ontology_names_the_working_directory( | 573 | def test_missing_task_ontology_names_the_working_directory( |
| 555 | tmp_path: Path, | 574 | tmp_path: Path, |
| 560 | raw["evaluation"]["object_matching"]["cluster_profiles"] = ( | 579 | raw["evaluation"]["object_matching"]["cluster_profiles"] = ( |
| 561 | "configs/contracts/ontology_absent.yaml" | 580 | "configs/contracts/ontology_absent.yaml" |
| 562 | ) | 581 | ) |
| 563 | path = _write_config(tmp_path, raw, "absent_ontology_config.yaml") | 582 | path = _write_config(tmp_path, raw, "absent_ontology_config.yaml") |
| 564 | with pytest.raises(ConfigError, match="was not found relative to"): | 583 | with pytest.raises(ExperimentConfigError, match="was not found relative to"): |
| 565 | load_config(path) | 584 | HarnessConfig.from_yaml(path) |
| 566 | 585 | ||
| 567 | 586 | ||
| 568 | def test_task_ontology_resolves_inside_the_configs_own_repository( | 587 | def test_task_ontology_resolves_inside_the_configs_own_repository( |
| 569 | tmp_path: Path, monkeypatch: pytest.MonkeyPatch | 588 | tmp_path: Path, monkeypatch: pytest.MonkeyPatch |
| 589 | ONTOLOGY_V2_PATH.read_text(encoding="utf-8"), encoding="utf-8" | 608 | ONTOLOGY_V2_PATH.read_text(encoding="utf-8"), encoding="utf-8" |
| 590 | ) | 609 | ) |
| 591 | monkeypatch.chdir(tmp_path / "elsewhere") | 610 | monkeypatch.chdir(tmp_path / "elsewhere") |
| 592 | 611 | ||
| 593 | config = load_config(config_path) | 612 | config = HarnessConfig.from_yaml(config_path) |
| 594 | 613 | ||
| 595 | assert resolve_ontology_path(config) == repo_contracts / "ontology_v1.yaml" | 614 | assert resolve_ontology_path(config) == repo_contracts / "ontology_v1.yaml" |
| 596 | assert config.task.num_classes == 9 | 615 | assert config.task.num_classes == 9 |
| 597 | assert config.task.ignore_index == 9 | 616 | assert config.task.ignore_index == 9 |
| 603 | """task.ontology also resolves against the config's own repository root.""" | 622 | """task.ontology also resolves against the config's own repository root.""" |
| 604 | config_path = Path("configs/e01_spt_pilot.yaml").resolve() | 623 | config_path = Path("configs/e01_spt_pilot.yaml").resolve() |
| 605 | monkeypatch.chdir(tmp_path) | 624 | monkeypatch.chdir(tmp_path) |
| 606 | 625 | ||
| 607 | config = load_config(config_path) | 626 | config = HarnessConfig.from_yaml(config_path) |
| 608 | 627 | ||
| 609 | assert config.task.ontology == Path("configs/contracts/ontology_v1.yaml") | 628 | assert config.task.ontology == Path("configs/contracts/ontology_v1.yaml") |
| 610 | assert config.task.num_classes == 9 | 629 | assert config.task.num_classes == 9 |
| 611 | 630 |
| 617 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) | 636 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) |
| 618 | raw["__sha256__"] = "deadbeef" | 637 | raw["__sha256__"] = "deadbeef" |
| 619 | path = tmp_path / "smuggled.yaml" | 638 | path = tmp_path / "smuggled.yaml" |
| 620 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 639 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 621 | with pytest.raises(ConfigError, match="Unknown.*__sha256__"): | 640 | with pytest.raises(ExperimentConfigError, match="Unknown.*__sha256__"): |
| 622 | load_config(path) | 641 | HarnessConfig.from_yaml(path) |
| 623 | 642 | ||
| 624 | 643 | ||
| 625 | def test_null_visualization_block_is_rejected(tmp_path: Path) -> None: | 644 | def test_null_visualization_block_is_rejected(tmp_path: Path) -> None: |
| 626 | """An explicitly null visualization block is a typo, not an omission.""" | 645 | """An explicitly null visualization block is a typo, not an omission.""" |
| 627 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) | 646 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) |
| 628 | raw["visualization"] = None | 647 | raw["visualization"] = None |
| 629 | path = tmp_path / "viz_null.yaml" | 648 | path = tmp_path / "viz_null.yaml" |
| 630 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 649 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 631 | with pytest.raises(ConfigError, match="visualization must be a mapping"): | 650 | with pytest.raises(ExperimentConfigError, match="visualization must be a mapping"): |
| 632 | load_config(path) | 651 | HarnessConfig.from_yaml(path) |
| 633 | 652 | ||
| 634 | 653 | ||
| 635 | def test_mapping_sections_are_read_only() -> None: | 654 | def test_mapping_sections_are_read_only() -> None: |
| 636 | """Frozen configs hand out read-only mappings, not mutable dicts.""" | 655 | """Frozen configs hand out read-only mappings, not mutable dicts.""" |
| 637 | config = load_config(CONFIGS[0]) | 656 | config = HarnessConfig.from_yaml(CONFIGS[0]) |
| 638 | with pytest.raises(TypeError): | 657 | with pytest.raises(TypeError): |
| 639 | config.model.args["num_classes"] = 99 # type: ignore[index] | 658 | config.model.args["num_classes"] = 99 # type: ignore[index] |
| 640 | with pytest.raises(TypeError): | 659 | with pytest.raises(TypeError): |
| 641 | config.loss.args["gamma"] = 0.0 # type: ignore[index] | 660 | config.loss.args["gamma"] = 0.0 # type: ignore[index] |
| 10 | 10 | ||
| 11 | import pytest | 11 | import pytest |
| 12 | import yaml | 12 | import yaml |
| 13 | 13 | ||
| 14 | from src.train.config import load_config | 14 | from src.train.config import HarnessConfig |
| 15 | from src.train.dispatch import DispatchError, DispatchRequest, dispatch_experiment | 15 | from src.train.dispatch import DispatchError, DispatchRequest, dispatch_experiment |
| 16 | 16 | ||
| 17 | BASE_CONFIG = Path("configs/e02_voxel_survival_oracle.yaml") | 17 | BASE_CONFIG = Path("configs/e02_voxel_survival_oracle.yaml") |
| 18 | CONTRACTS_DIR = Path("configs/contracts") | 18 | CONTRACTS_DIR = Path("configs/contracts") |
| 39 | raw["experiment"]["gates"] = [] | 39 | raw["experiment"]["gates"] = [] |
| 40 | raw["model"]["runner"] = "stub_runner.py" | 40 | raw["model"]["runner"] = "stub_runner.py" |
| 41 | raw["study"] = study | 41 | raw["study"] = study |
| 42 | (tmp_path / "stub_runner.py").write_text(STUB_RUNNER, encoding="utf-8") | 42 | (tmp_path / "stub_runner.py").write_text(STUB_RUNNER, encoding="utf-8") |
| 43 | # load_config resolves task.ontology against the working directory, so the | 43 | # HarnessConfig.from_yaml resolves task.ontology against the working |
| 44 | # directory, so the | ||
| 44 | # throwaway root needs the frozen contracts the config names. | 45 | # throwaway root needs the frozen contracts the config names. |
| 45 | shutil.copytree(CONTRACTS_DIR, tmp_path / CONTRACTS_DIR, dirs_exist_ok=True) | 46 | shutil.copytree(CONTRACTS_DIR, tmp_path / CONTRACTS_DIR, dirs_exist_ok=True) |
| 46 | config_path = tmp_path / "experiment.yaml" | 47 | config_path = tmp_path / "experiment.yaml" |
| 47 | config_path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 48 | config_path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 81 | log_path = tmp_path / "invocations.jsonl" | 82 | log_path = tmp_path / "invocations.jsonl" |
| 82 | monkeypatch.setenv("STUB_LOG", str(log_path)) | 83 | monkeypatch.setenv("STUB_LOG", str(log_path)) |
| 83 | request = DispatchRequest(log_dir=tmp_path / "runs") | 84 | request = DispatchRequest(log_dir=tmp_path / "runs") |
| 84 | assert dispatch_experiment( | 85 | assert dispatch_experiment( |
| 85 | load_config(config_path), request, repository_root=root | 86 | HarnessConfig.from_yaml(config_path), request, repository_root=root |
| 86 | ) == 0 | 87 | ) == 0 |
| 87 | records = _records(log_path) | 88 | records = _records(log_path) |
| 88 | assert [record["cell"] for record in records] == ["coarse", "fine"] | 89 | assert [record["cell"] for record in records] == ["coarse", "fine"] |
| 89 | assert [record["index"] for record in records] == ["0", "1"] | 90 | assert [record["index"] for record in records] == ["0", "1"] |
| 106 | """Every cell run directory carries identity, overrides, and its SHA-256.""" | 107 | """Every cell run directory carries identity, overrides, and its SHA-256.""" |
| 107 | root, config_path = _repository(tmp_path, _variants_study()) | 108 | root, config_path = _repository(tmp_path, _variants_study()) |
| 108 | monkeypatch.setenv("STUB_LOG", str(tmp_path / "invocations.jsonl")) | 109 | monkeypatch.setenv("STUB_LOG", str(tmp_path / "invocations.jsonl")) |
| 109 | dispatch_experiment( | 110 | dispatch_experiment( |
| 110 | load_config(config_path), | 111 | HarnessConfig.from_yaml(config_path), |
| 111 | DispatchRequest(log_dir=tmp_path / "runs"), | 112 | DispatchRequest(log_dir=tmp_path / "runs"), |
| 112 | repository_root=root, | 113 | repository_root=root, |
| 113 | ) | 114 | ) |
| 114 | provenance = json.loads( | 115 | provenance = json.loads( |
| 138 | root, config_path = _repository(tmp_path, study) | 139 | root, config_path = _repository(tmp_path, study) |
| 139 | log_path = tmp_path / "invocations.jsonl" | 140 | log_path = tmp_path / "invocations.jsonl" |
| 140 | monkeypatch.setenv("STUB_LOG", str(log_path)) | 141 | monkeypatch.setenv("STUB_LOG", str(log_path)) |
| 141 | dispatch_experiment( | 142 | dispatch_experiment( |
| 142 | load_config(config_path), | 143 | HarnessConfig.from_yaml(config_path), |
| 143 | DispatchRequest(log_dir=tmp_path / "runs"), | 144 | DispatchRequest(log_dir=tmp_path / "runs"), |
| 144 | repository_root=root, | 145 | repository_root=root, |
| 145 | ) | 146 | ) |
| 146 | records = _records(log_path) | 147 | records = _records(log_path) |
| 158 | monkeypatch.setenv("STUB_LOG", str(log_path)) | 159 | monkeypatch.setenv("STUB_LOG", str(log_path)) |
| 159 | monkeypatch.setenv("STUB_EXIT_CODE", "3") | 160 | monkeypatch.setenv("STUB_EXIT_CODE", "3") |
| 160 | with pytest.raises(DispatchError, match="cell coarse"): | 161 | with pytest.raises(DispatchError, match="cell coarse"): |
| 161 | dispatch_experiment( | 162 | dispatch_experiment( |
| 162 | load_config(config_path), | 163 | HarnessConfig.from_yaml(config_path), |
| 163 | DispatchRequest(log_dir=tmp_path / "runs"), | 164 | DispatchRequest(log_dir=tmp_path / "runs"), |
| 164 | repository_root=root, | 165 | repository_root=root, |
| 165 | ) | 166 | ) |
| 166 | assert len(_records(log_path)) == 1 | 167 | assert len(_records(log_path)) == 1 |
| 172 | """DispatchRequest narrowing is forwarded unchanged to each cell.""" | 173 | """DispatchRequest narrowing is forwarded unchanged to each cell.""" |
| 173 | root, config_path = _repository(tmp_path, _variants_study()) | 174 | root, config_path = _repository(tmp_path, _variants_study()) |
| 174 | log_path = tmp_path / "invocations.jsonl" | 175 | log_path = tmp_path / "invocations.jsonl" |
| 175 | monkeypatch.setenv("STUB_LOG", str(log_path)) | 176 | monkeypatch.setenv("STUB_LOG", str(log_path)) |
| 176 | config = load_config(config_path) | 177 | config = HarnessConfig.from_yaml(config_path) |
| 177 | request = DispatchRequest( | 178 | request = DispatchRequest( |
| 178 | corridors=(config.data.corridors.include[0],), | 179 | corridors=(config.data.corridors.include[0],), |
| 179 | fast_dev_run=True, | 180 | fast_dev_run=True, |
| 180 | num_workers=2, | 181 | num_workers=2, |
| 194 | """A training run needs both folds, so --split fails early and by name.""" | 195 | """A training run needs both folds, so --split fails early and by name.""" |
| 195 | root, config_path = _repository(tmp_path, _variants_study()) | 196 | root, config_path = _repository(tmp_path, _variants_study()) |
| 196 | with pytest.raises(DispatchError, match="refuses --split"): | 197 | with pytest.raises(DispatchError, match="refuses --split"): |
| 197 | dispatch_experiment( | 198 | dispatch_experiment( |
| 198 | load_config(config_path), | 199 | HarnessConfig.from_yaml(config_path), |
| 199 | DispatchRequest(splits=("train",)), | 200 | DispatchRequest(splits=("train",)), |
| 200 | repository_root=root, | 201 | repository_root=root, |
| 201 | ) | 202 | ) |
| 202 | 203 |
| 236 | """Study expansion never relaxes the narrowing contract.""" | 237 | """Study expansion never relaxes the narrowing contract.""" |
| 237 | root, config_path = _repository(tmp_path, _variants_study()) | 238 | root, config_path = _repository(tmp_path, _variants_study()) |
| 238 | with pytest.raises(DispatchError, match="promotion"): | 239 | with pytest.raises(DispatchError, match="promotion"): |
| 239 | dispatch_experiment( | 240 | dispatch_experiment( |
| 240 | load_config(config_path), | 241 | HarnessConfig.from_yaml(config_path), |
| 241 | DispatchRequest(splits=("promotion_test",)), | 242 | DispatchRequest(splits=("promotion_test",)), |
| 242 | repository_root=root, | 243 | repository_root=root, |
| 243 | ) | 244 | ) |
| 5 | from pathlib import Path | 5 | from pathlib import Path |
| 6 | 6 | ||
| 7 | import pytest | 7 | import pytest |
| 8 | 8 | ||
| 9 | from src.train.config import load_config | 9 | from src.train.config import HarnessConfig |
| 10 | from src.train.dispatch import DispatchError, dispatch_experiment | 10 | from src.train.dispatch import DispatchError, dispatch_experiment |
| 11 | 11 | ||
| 12 | SPT_CONFIG = Path("configs/e01_spt_pilot.yaml") | 12 | SPT_CONFIG = Path("configs/e01_spt_pilot.yaml") |
| 13 | CPU_CONFIG = Path("configs/e02_voxel_survival_oracle.yaml") | 13 | CPU_CONFIG = Path("configs/e02_voxel_survival_oracle.yaml") |
| 32 | 32 | ||
| 33 | 33 | ||
| 34 | def _dispatch_message() -> str: | 34 | def _dispatch_message() -> str: |
| 35 | """Dispatch the SPT pilot and return its refusal message.""" | 35 | """Dispatch the SPT pilot and return its refusal message.""" |
| 36 | config = load_config(SPT_CONFIG) | 36 | config = HarnessConfig.from_yaml(SPT_CONFIG) |
| 37 | with pytest.raises(DispatchError) as failure: | 37 | with pytest.raises(DispatchError) as failure: |
| 38 | dispatch_experiment(config) | 38 | dispatch_experiment(config) |
| 39 | return str(failure.value) | 39 | return str(failure.value) |
| 40 | 40 |
| 157 | monkeypatch: pytest.MonkeyPatch, | 157 | monkeypatch: pytest.MonkeyPatch, |
| 158 | ) -> None: | 158 | ) -> None: |
| 159 | """CPU experiments never consult the external framework environment.""" | 159 | """CPU experiments never consult the external framework environment.""" |
| 160 | monkeypatch.setenv("SPT_ROOT", "/definitely/not/a/checkout") | 160 | monkeypatch.setenv("SPT_ROOT", "/definitely/not/a/checkout") |
| 161 | config = load_config(CPU_CONFIG) | 161 | config = HarnessConfig.from_yaml(CPU_CONFIG) |
| 162 | with pytest.raises(DispatchError) as failure: | 162 | with pytest.raises(DispatchError) as failure: |
| 163 | dispatch_experiment(config) | 163 | dispatch_experiment(config) |
| 164 | assert "framework_environment" not in str(failure.value) | 164 | assert "framework_environment" not in str(failure.value) |
| 24 | SplitTier, | 24 | SplitTier, |
| 25 | authorize_split_access, | 25 | authorize_split_access, |
| 26 | load_split_manifest, | 26 | load_split_manifest, |
| 27 | ) | 27 | ) |
| 28 | from src.train.config import HarnessConfig, load_config | 28 | from src.train.config import HarnessConfig |
| 29 | 29 | ||
| 30 | CONFIG_PATH = Path("configs/e01_spt_pilot.yaml") | 30 | CONFIG_PATH = Path("configs/e01_spt_pilot.yaml") |
| 31 | GRID_ORIGIN = [10.0, -20.0, 5.0] | 31 | GRID_ORIGIN = [10.0, -20.0, 5.0] |
| 32 | 32 |
| 59 | 59 | ||
| 60 | 60 | ||
| 61 | def _config_with_prepared_corridors(tmp_path: Path) -> HarnessConfig: | 61 | def _config_with_prepared_corridors(tmp_path: Path) -> HarnessConfig: |
| 62 | """Return the E1 config pointed at synthetic adapter manifests.""" | 62 | """Return the E1 config pointed at synthetic adapter manifests.""" |
| 63 | config = load_config(CONFIG_PATH) | 63 | config = HarnessConfig.from_yaml(CONFIG_PATH) |
| 64 | canonical_root = tmp_path / "canonical_root" | 64 | canonical_root = tmp_path / "canonical_root" |
| 65 | (canonical_root / "canonical").mkdir(parents=True) | 65 | (canonical_root / "canonical").mkdir(parents=True) |
| 66 | for corridor_id in config.data.corridors.include: | 66 | for corridor_id in config.data.corridors.include: |
| 67 | (canonical_root / "canonical" / f"{corridor_id}.adapter.json").write_text( | 67 | (canonical_root / "canonical" / f"{corridor_id}.adapter.json").write_text( |
| 7 | 7 | ||
| 8 | import pytest | 8 | import pytest |
| 9 | import yaml | 9 | import yaml |
| 10 | 10 | ||
| 11 | from src.train.config import ConfigError, expand_study, load_config | 11 | from src.train.config import ExperimentConfigError, HarnessConfig, expand_study |
| 12 | 12 | ||
| 13 | BASE_CONFIG = Path("configs/e02_voxel_survival_oracle.yaml") | 13 | BASE_CONFIG = Path("configs/e02_voxel_survival_oracle.yaml") |
| 14 | SINGLE_CONFIG = Path("configs/e01_spt_pilot.yaml") | 14 | SINGLE_CONFIG = Path("configs/e01_spt_pilot.yaml") |
| 15 | 15 |
| 34 | 34 | ||
| 35 | 35 | ||
| 36 | def test_variants_study_expands_to_one_cell_per_variant() -> None: | 36 | def test_variants_study_expands_to_one_cell_per_variant() -> None: |
| 37 | """Every declared variant becomes its own resolved, hashed configuration.""" | 37 | """Every declared variant becomes its own resolved, hashed configuration.""" |
| 38 | config = load_config(BASE_CONFIG) | 38 | config = HarnessConfig.from_yaml(BASE_CONFIG) |
| 39 | cells = expand_study(config) | 39 | cells = expand_study(config) |
| 40 | assert [cell.id for cell in cells] == [ | 40 | assert [cell.id for cell in cells] == [ |
| 41 | variant.id for variant in config.study.variants | 41 | variant.id for variant in config.study.variants |
| 42 | ] | 42 | ] |
| 49 | 49 | ||
| 50 | 50 | ||
| 51 | def test_single_study_expands_to_exactly_one_identity_cell() -> None: | 51 | def test_single_study_expands_to_exactly_one_identity_cell() -> None: |
| 52 | """A single study runs once and changes nothing in the resolved config.""" | 52 | """A single study runs once and changes nothing in the resolved config.""" |
| 53 | config = load_config(SINGLE_CONFIG) | 53 | config = HarnessConfig.from_yaml(SINGLE_CONFIG) |
| 54 | cells = expand_study(config) | 54 | cells = expand_study(config) |
| 55 | assert len(cells) == 1 | 55 | assert len(cells) == 1 |
| 56 | assert cells[0].overrides == {} | 56 | assert cells[0].overrides == {} |
| 57 | assert cells[0].id == config.experiment.id | 57 | assert cells[0].id == config.experiment.id |
| 65 | 65 | ||
| 66 | 66 | ||
| 67 | def test_cell_order_and_hashes_are_deterministic() -> None: | 67 | def test_cell_order_and_hashes_are_deterministic() -> None: |
| 68 | """Repeated expansion yields identical identities, order, and hashes.""" | 68 | """Repeated expansion yields identical identities, order, and hashes.""" |
| 69 | config = load_config(BASE_CONFIG) | 69 | config = HarnessConfig.from_yaml(BASE_CONFIG) |
| 70 | first = expand_study(config) | 70 | first = expand_study(config) |
| 71 | second = expand_study(load_config(BASE_CONFIG)) | 71 | second = expand_study(HarnessConfig.from_yaml(BASE_CONFIG)) |
| 72 | assert [cell.id for cell in first] == [cell.id for cell in second] | 72 | assert [cell.id for cell in first] == [cell.id for cell in second] |
| 73 | assert [cell.config.sha256 for cell in first] == [ | 73 | assert [cell.config.sha256 for cell in first] == [ |
| 74 | cell.config.sha256 for cell in second | 74 | cell.config.sha256 for cell in second |
| 75 | ] | 75 | ] |
| 82 | {"id": "typo", "overrides": {"adapter.spt.voxel_metres": 0.02}} | 82 | {"id": "typo", "overrides": {"adapter.spt.voxel_metres": 0.02}} |
| 83 | ] | 83 | ] |
| 84 | raw["study"]["contrasts"] = [] | 84 | raw["study"]["contrasts"] = [] |
| 85 | path = _write(raw, tmp_path / "unknown_override.yaml") | 85 | path = _write(raw, tmp_path / "unknown_override.yaml") |
| 86 | with pytest.raises(ConfigError, match="adapter.spt.voxel_metres"): | 86 | with pytest.raises(ExperimentConfigError, match="adapter.spt.voxel_metres"): |
| 87 | load_config(path) | 87 | HarnessConfig.from_yaml(path) |
| 88 | 88 | ||
| 89 | 89 | ||
| 90 | def test_duplicate_variant_cell_ids_are_rejected(tmp_path: Path) -> None: | 90 | def test_duplicate_variant_cell_ids_are_rejected(tmp_path: Path) -> None: |
| 91 | """Two cells cannot share an identity.""" | 91 | """Two cells cannot share an identity.""" |
| 95 | {"id": "twin", "overrides": {"data.tiling.min_points": 2}}, | 95 | {"id": "twin", "overrides": {"data.tiling.min_points": 2}}, |
| 96 | ] | 96 | ] |
| 97 | raw["study"]["contrasts"] = [] | 97 | raw["study"]["contrasts"] = [] |
| 98 | path = _write(raw, tmp_path / "duplicate_ids.yaml") | 98 | path = _write(raw, tmp_path / "duplicate_ids.yaml") |
| 99 | with pytest.raises(ConfigError, match="duplicate"): | 99 | with pytest.raises(ExperimentConfigError, match="duplicate"): |
| 100 | load_config(path) | 100 | HarnessConfig.from_yaml(path) |
| 101 | 101 | ||
| 102 | 102 | ||
| 103 | def test_kind_without_its_payload_is_rejected(tmp_path: Path) -> None: | 103 | def test_kind_without_its_payload_is_rejected(tmp_path: Path) -> None: |
| 104 | """A discriminated kind whose payload is absent never expands silently.""" | 104 | """A discriminated kind whose payload is absent never expands silently.""" |
| 110 | "sweep": None, | 110 | "sweep": None, |
| 111 | "contrasts": [], | 111 | "contrasts": [], |
| 112 | } | 112 | } |
| 113 | path = _write(raw, tmp_path / "empty_variants.yaml") | 113 | path = _write(raw, tmp_path / "empty_variants.yaml") |
| 114 | with pytest.raises(ConfigError, match="variants"): | 114 | with pytest.raises(ExperimentConfigError, match="variants"): |
| 115 | load_config(path) | 115 | HarnessConfig.from_yaml(path) |
| 116 | 116 | ||
| 117 | 117 | ||
| 118 | def test_matrix_study_expands_cross_product_minus_exclusions( | 118 | def test_matrix_study_expands_cross_product_minus_exclusions( |
| 119 | tmp_path: Path, | 119 | tmp_path: Path, |
| 135 | }, | 135 | }, |
| 136 | "sweep": None, | 136 | "sweep": None, |
| 137 | "contrasts": [], | 137 | "contrasts": [], |
| 138 | } | 138 | } |
| 139 | config = load_config(_write(raw, tmp_path / "matrix.yaml")) | 139 | config = HarnessConfig.from_yaml(_write(raw, tmp_path / "matrix.yaml")) |
| 140 | cells = expand_study(config) | 140 | cells = expand_study(config) |
| 141 | assert [cell.id for cell in cells] == [ | 141 | assert [cell.id for cell in cells] == [ |
| 142 | "min_points-1__samples-500", | 142 | "min_points-1__samples-500", |
| 143 | "min_points-2__samples-500", | 143 | "min_points-2__samples-500", |
| 161 | }, | 161 | }, |
| 162 | "sweep": None, | 162 | "sweep": None, |
| 163 | "contrasts": [], | 163 | "contrasts": [], |
| 164 | } | 164 | } |
| 165 | config = load_config(_write(raw, tmp_path / "stale_exclude.yaml")) | 165 | config = HarnessConfig.from_yaml(_write(raw, tmp_path / "stale_exclude.yaml")) |
| 166 | with pytest.raises(ConfigError, match="matches no matrix cell"): | 166 | with pytest.raises(ExperimentConfigError, match="matches no matrix cell"): |
| 167 | expand_study(config) | 167 | expand_study(config) |
| 168 | 168 | ||
| 169 | 169 | ||
| 170 | def test_grid_sweep_expands_deterministically_within_budget( | 170 | def test_grid_sweep_expands_deterministically_within_budget( |
| 183 | "objective": "val/iou_macro_interest", | 183 | "objective": "val/iou_macro_interest", |
| 184 | }, | 184 | }, |
| 185 | "contrasts": [], | 185 | "contrasts": [], |
| 186 | } | 186 | } |
| 187 | config = load_config(_write(raw, tmp_path / "grid_sweep.yaml")) | 187 | config = HarnessConfig.from_yaml(_write(raw, tmp_path / "grid_sweep.yaml")) |
| 188 | cells = expand_study(config) | 188 | cells = expand_study(config) |
| 189 | assert [cell.id for cell in cells] == ["min_points-1", "min_points-2"] | 189 | assert [cell.id for cell in cells] == ["min_points-1", "min_points-2"] |
| 190 | assert [cell.config.data.tiling.min_points for cell in cells] == [1, 2] | 190 | assert [cell.config.data.tiling.min_points for cell in cells] == [1, 2] |
| 191 | 191 |
| 212 | "objective": "val/iou_macro_interest", | 212 | "objective": "val/iou_macro_interest", |
| 213 | }, | 213 | }, |
| 214 | "contrasts": [], | 214 | "contrasts": [], |
| 215 | } | 215 | } |
| 216 | config = load_config(_write(raw, tmp_path / "random_sweep.yaml")) | 216 | config = HarnessConfig.from_yaml(_write(raw, tmp_path / "random_sweep.yaml")) |
| 217 | first = expand_study(config) | 217 | first = expand_study(config) |
| 218 | second = expand_study(config) | 218 | second = expand_study(config) |
| 219 | assert [cell.id for cell in first] == ["sample_000", "sample_001", "sample_002"] | 219 | assert [cell.id for cell in first] == ["sample_000", "sample_001", "sample_002"] |
| 220 | assert [ | 220 | assert [ |
| 237 | "objective": "val/iou_macro_interest", | 237 | "objective": "val/iou_macro_interest", |
| 238 | }, | 238 | }, |
| 239 | "contrasts": [], | 239 | "contrasts": [], |
| 240 | } | 240 | } |
| 241 | config = load_config(_write(raw, tmp_path / "bayes_sweep.yaml")) | 241 | config = HarnessConfig.from_yaml(_write(raw, tmp_path / "bayes_sweep.yaml")) |
| 242 | with pytest.raises(ConfigError, match="bayesian"): | 242 | with pytest.raises(ExperimentConfigError, match="bayesian"): |
| 243 | expand_study(config) | 243 | expand_study(config) |
| 244 | 244 | ||
| 245 | 245 | ||
| 246 | def test_every_repository_experiment_expands() -> None: | 246 | def test_every_repository_experiment_expands() -> None: |
| 247 | """All E1-E14 studies expand to at least one strictly validated cell.""" | 247 | """All E1-E14 studies expand to at least one strictly validated cell.""" |
| 248 | for path in sorted(Path("configs").glob("e*.yaml")): | 248 | for path in sorted(Path("configs").glob("e*.yaml")): |
| 249 | config = load_config(path) | 249 | config = HarnessConfig.from_yaml(path) |
| 250 | cells = expand_study(config) | 250 | cells = expand_study(config) |
| 251 | assert cells | 251 | assert cells |
| 252 | assert len({cell.id for cell in cells}) == len(cells) | 252 | assert len({cell.id for cell in cells}) == len(cells) |
| 253 | if config.study.kind == "single": | 253 | if config.study.kind == "single": |
| 760 | 760 | ||
| 761 | 761 | ||
| 762 | def test_voxel_sizes_come_from_the_study_variants() -> None: | 762 | def test_voxel_sizes_come_from_the_study_variants() -> None: |
| 763 | """The study variants are the single source of the analysed grid sizes.""" | 763 | """The study variants are the single source of the analysed grid sizes.""" |
| 764 | from src.train.config import load_config | 764 | from src.train.config import HarnessConfig |
| 765 | 765 | ||
| 766 | module = _load_oracle_script() | 766 | module = _load_oracle_script() |
| 767 | 767 | ||
| 768 | assert module._voxel_sizes(load_config(E2_CONFIG), Path(E2_CONFIG)) == ( | 768 | assert module._voxel_sizes(HarnessConfig.from_yaml(E2_CONFIG), Path(E2_CONFIG)) == ( |
| 769 | 0.02, | 769 | 0.02, |
| 770 | 0.03, | 770 | 0.03, |
| 771 | 0.05, | 771 | 0.05, |
| 772 | ) | 772 | ) |
| 773 | 773 | ||
| 774 | 774 | ||
| 775 | def test_config_without_variant_voxel_sizes_fails_closed(tmp_path: Path) -> None: | 775 | def test_config_without_variant_voxel_sizes_fails_closed(tmp_path: Path) -> None: |
| 776 | """A config declaring no variant voxel size refuses to run E2.""" | 776 | """A config declaring no variant voxel size refuses to run E2.""" |
| 777 | from src.train.config import load_config | 777 | from src.train.config import HarnessConfig |
| 778 | 778 | ||
| 779 | module = _load_oracle_script() | 779 | module = _load_oracle_script() |
| 780 | 780 | ||
| 781 | def _drop_variants(raw: dict[str, Any]) -> None: | 781 | def _drop_variants(raw: dict[str, Any]) -> None: |
| 785 | 785 | ||
| 786 | path = _config_variant(tmp_path, _drop_variants) | 786 | path = _config_variant(tmp_path, _drop_variants) |
| 787 | 787 | ||
| 788 | with pytest.raises(VoxelOracleError, match="adapter.spt.voxel_m"): | 788 | with pytest.raises(VoxelOracleError, match="adapter.spt.voxel_m"): |
| 789 | module._voxel_sizes(load_config(path), path) | 789 | module._voxel_sizes(HarnessConfig.from_yaml(path), path) |
| 790 | 790 | ||
| 791 | 791 | ||
| 792 | def test_restated_model_voxel_sizes_must_agree(tmp_path: Path) -> None: | 792 | def test_restated_model_voxel_sizes_must_agree(tmp_path: Path) -> None: |
| 793 | """A duplicated ``model.args.voxel_sizes_m`` must match or fail closed.""" | 793 | """A duplicated ``model.args.voxel_sizes_m`` must match or fail closed.""" |
| 794 | from src.train.config import load_config | 794 | from src.train.config import HarnessConfig |
| 795 | 795 | ||
| 796 | module = _load_oracle_script() | 796 | module = _load_oracle_script() |
| 797 | 797 | ||
| 798 | def _restate(raw: dict[str, Any]) -> None: | 798 | def _restate(raw: dict[str, Any]) -> None: |
| 800 | 800 | ||
| 801 | path = _config_variant(tmp_path, _restate) | 801 | path = _config_variant(tmp_path, _restate) |
| 802 | 802 | ||
| 803 | with pytest.raises(VoxelOracleError, match="voxel_sizes_m"): | 803 | with pytest.raises(VoxelOracleError, match="voxel_sizes_m"): |
| 804 | module._voxel_sizes(load_config(path), path) | 804 | module._voxel_sizes(HarnessConfig.from_yaml(path), path) |
| 805 | 805 | ||
| 806 | 806 | ||
| 807 | def test_required_output_formats_are_written(tmp_path: Path) -> None: | 807 | def test_required_output_formats_are_written(tmp_path: Path) -> None: |
| 808 | """E2 always writes CSV, JSON, and Markdown products.""" | 808 | """E2 always writes CSV, JSON, and Markdown products.""" |
| 852 | 852 | ||
| 853 | 853 | ||
| 854 | def test_dispatched_cell_analyses_only_its_own_voxel_size() -> None: | 854 | def test_dispatched_cell_analyses_only_its_own_voxel_size() -> None: |
| 855 | """Dispatch runs E2 once per cell, so a cell must not re-run the sweep.""" | 855 | """Dispatch runs E2 once per cell, so a cell must not re-run the sweep.""" |
| 856 | from src.train.config import load_config | 856 | from src.train.config import HarnessConfig |
| 857 | 857 | ||
| 858 | module = _load_oracle_script() | 858 | module = _load_oracle_script() |
| 859 | config = load_config(E2_CONFIG) | 859 | config = HarnessConfig.from_yaml(E2_CONFIG) |
| 860 | 860 | ||
| 861 | assert module._voxel_sizes(config, Path(E2_CONFIG), cell=("voxel_2cm", 0)) == ( | 861 | assert module._voxel_sizes(config, Path(E2_CONFIG), cell=("voxel_2cm", 0)) == ( |
| 862 | 0.02, | 862 | 0.02, |
| 863 | ) | 863 | ) |
| 872 | 872 | ||
| 873 | 873 | ||
| 874 | def test_unknown_dispatched_cell_fails_closed() -> None: | 874 | def test_unknown_dispatched_cell_fails_closed() -> None: |
| 875 | """An unknown or misplaced study cell is refused by name.""" | 875 | """An unknown or misplaced study cell is refused by name.""" |
| 876 | from src.train.config import load_config | 876 | from src.train.config import HarnessConfig |
| 877 | 877 | ||
| 878 | module = _load_oracle_script() | 878 | module = _load_oracle_script() |
| 879 | config = load_config(E2_CONFIG) | 879 | config = HarnessConfig.from_yaml(E2_CONFIG) |
| 880 | 880 | ||
| 881 | with pytest.raises(VoxelOracleError, match="voxel_9cm"): | 881 | with pytest.raises(VoxelOracleError, match="voxel_9cm"): |
| 882 | module._voxel_sizes(config, Path(E2_CONFIG), cell=("voxel_9cm", 0)) | 882 | module._voxel_sizes(config, Path(E2_CONFIG), cell=("voxel_9cm", 0)) |
| 883 | with pytest.raises(VoxelOracleError, match="position 1"): | 883 | with pytest.raises(VoxelOracleError, match="position 1"): |
| 18 | ## Data and experiments | 18 | ## Data and experiments |
| 19 | 19 | ||
| 20 | - Keep source and generated datasets under the DVC lanes in `data/`; never commit data products directly. | 20 | - Keep source and generated datasets under the DVC lanes in `data/`; never commit data products directly. |
| 21 | - Every experiment is configured by one strict YAML file in `configs/`. | 21 | - Every experiment is configured by one strict YAML file in `configs/`. |
| 22 | - The configuration schema is a pydantic model tree on `iolabs.common.config_loader.ConfigModel`: `src/train/config_schema.py` (experiment, gates, study), `src/train/config_sections.py` (task, data, adapter, model, train, evaluation, runtime, provenance). Adding a key means adding a typed field to its model and to the YAML documents under `configs/`; nothing else. Cross-section and ontology-dependent rules live in `src/train/config_rules.py`, study expansion in `src/train/config_study.py`, and the strict YAML leaf typing in `src/train/config_values.py`. `src/train/config.py` stays the only import site (`load_config`, `expand_study`, `ConfigError`, the model classes). | 22 | - The configuration schema is a pydantic model tree on `iolabs.common.config_loader.ConfigModel`: `src/train/config_schema.py` (experiment, gates, study), `src/train/config_sections.py` (task, data, adapter, model, train, evaluation, runtime, provenance). Adding a key means adding a typed field to its model and to the YAML documents under `configs/`; nothing else. Cross-section and ontology-dependent rules live in `src/train/config_rules.py`, study expansion in `src/train/config_study.py`, and the strict YAML leaf typing in `src/train/config_values.py`. `src/train/config.py` stays the only import site (`HarnessConfig.from_yaml`, `expand_study`, `ExperimentConfigError`, the model classes). |
| 23 | - Class counts come from the experiment's ontology (`task.ontology`): the model predicts train IDs `0..N-1` and ID `N` is void with no output channel (v1: N=9, roadside classes; v2: N=11, the ReCap annotation classes with junk merged into unclassified/void). Never hard-code a class count. | 23 | - Class counts come from the experiment's ontology (`task.ontology`): the model predicts train IDs `0..N-1` and ID `N` is void with no output channel (v1: N=9, roadside classes; v2: N=11, the ReCap annotation classes with junk merged into unclassified/void). Never hard-code a class count. |
| 24 | - Boundary LAS codes belong to the code space the ontology, export map, and artifact all declare (`las_code_space`): `seg3d` is the legacy upstream space (v1), `asprs` is ASPRS LAS 1.4 and the ReCap lane (v2). Never read a LAS code without its space; resolve it through `registry_for(space)` in `src/contracts/las_codes.py`. | 24 | - Boundary LAS codes belong to the code space the ontology, export map, and artifact all declare (`las_code_space`): `seg3d` is the legacy upstream space (v1), `asprs` is ASPRS LAS 1.4 and the ReCap lane (v2). Never read a LAS code without its space; resolve it through `registry_for(space)` in `src/contracts/las_codes.py`. |
| 25 | - Never train on, evaluate on, or inspect `promotion_test` data without the command's explicit promotion authorization arguments. | 25 | - Never train on, evaluate on, or inspect `promotion_test` data without the command's explicit promotion authorization arguments. |
| 26 | - E2 accepts only canonical data whose manifest proves full-resolution, in-memory `FuseResult.classification` provenance. | 26 | - E2 accepts only canonical data whose manifest proves full-resolution, in-memory `FuseResult.classification` provenance. |
| 14 | ``` | 14 | ``` |
| 15 | 15 | ||
| 16 | The first command installs the CPU preparation/E2 environment. The `ml` extra adds Torch, Lightning, TensorBoard, torchmetrics, and `iolabs-ml-harness`; it still does not install SPT, Pointcept, spconv, or FlashAttention. For shared-harness development, run `scripts/dev/use_local_mlharness.sh`; a later `uv sync --extra ml` restores the locked source. | 16 | The first command installs the CPU preparation/E2 environment. The `ml` extra adds Torch, Lightning, TensorBoard, torchmetrics, and `iolabs-ml-harness`; it still does not install SPT, Pointcept, spconv, or FlashAttention. For shared-harness development, run `scripts/dev/use_local_mlharness.sh`; a later `uv sync --extra ml` restores the locked source. |
| 17 | 17 | ||
| 18 | ## Configuration | ||
| 19 | |||
| 20 | Every experiment is one strict YAML document under `configs/`. The schema is | ||
| 21 | `HarnessConfig` in `src/train/config.py` (a `config_loader.ConfigModel` through the | ||
| 22 | repository-local `StrictConfigModel`); nested YAML blocks are nested models and unknown | ||
| 23 | keys are rejected. **To add a config key: add the field (with its type, default and any | ||
| 24 | `Field` range) to its model and the same key to the YAML documents under `configs/` -- | ||
| 25 | nothing else.** `HarnessConfig.from_yaml(path)` returns the frozen `HarnessConfig`, | ||
| 26 | `expand_study(config)` returns its re-validated study cells, and both raise | ||
| 27 | `ExperimentConfigError`. | ||
| 28 | |||
| 29 | The schema exceeds one module, so it is split by section (fleet rule "field declarations | ||
| 30 | only"): `src/train/config_schema.py` (experiment, gates, study), `src/train/config_sections.py` | ||
| 31 | (task, data, adapter, model, train, evaluation, runtime, provenance), `src/train/config_values.py` | ||
| 32 | (strict YAML leaf typing), `src/train/config_rules.py` (cross-section and ontology rules), | ||
| 33 | `src/train/config_study.py` (study algebra). `src/train/config.py` stays the only import | ||
| 34 | site. Leaf typing is deliberately stricter than the fleet coercion matrix -- `"50"` is not | ||
| 35 | a float and `1` is not a boolean -- the sanctioned opt-out for this YAML experiment | ||
| 36 | contract, documented in `src/train/config_values.py`. Runtime narrowing comes from CLI | ||
| 37 | arguments and study overrides, never from repo-local JSON. | ||
| 38 | |||
| 18 | ## E2: voxel-survival oracle | 39 | ## E2: voxel-survival oracle |
| 19 | 40 | ||
| 20 | E2 is local-first and CPU-capable. It must run only after canonical preparation regenerates and stages the full-resolution in-memory `FuseResult.classification`; written seg3d/ReCap representative clouds are rejected as oracle input. | 41 | E2 is local-first and CPU-capable. It must run only after canonical preparation regenerates and stages the full-resolution in-memory `FuseResult.classification`; written seg3d/ReCap representative clouds are rejected as oracle input. |
| 21 | 42 |
| 14 | ``` | 14 | ``` |
| 15 | 15 | ||
| 16 | The first command installs the CPU preparation/E2 environment. The `ml` extra adds Torch, Lightning, TensorBoard, torchmetrics, and `iolabs-ml-harness`; it still does not install SPT, Pointcept, spconv, or FlashAttention. For shared-harness development, run `scripts/dev/use_local_mlharness.sh`; a later `uv sync --extra ml` restores the locked source. | 16 | The first command installs the CPU preparation/E2 environment. The `ml` extra adds Torch, Lightning, TensorBoard, torchmetrics, and `iolabs-ml-harness`; it still does not install SPT, Pointcept, spconv, or FlashAttention. For shared-harness development, run `scripts/dev/use_local_mlharness.sh`; a later `uv sync --extra ml` restores the locked source. |
| 17 | 17 | ||
| 18 | ## Configuration | ||
| 19 | |||
| 20 | Every experiment is one strict YAML document under `configs/`. The schema is | ||
| 21 | `HarnessConfig` in `src/train/config.py` (a `config_loader.ConfigModel` through the | ||
| 22 | repository-local `StrictConfigModel`); nested YAML blocks are nested models and unknown | ||
| 23 | keys are rejected. **To add a config key: add the field (with its type, default and any | ||
| 24 | `Field` range) to its model and the same key to the YAML documents under `configs/` -- | ||
| 25 | nothing else.** `HarnessConfig.from_yaml(path)` returns the frozen `HarnessConfig`, | ||
| 26 | `expand_study(config)` returns its re-validated study cells, and both raise | ||
| 27 | `ExperimentConfigError`. | ||
| 28 | |||
| 29 | The schema exceeds one module, so it is split by section (fleet rule "field declarations | ||
| 30 | only"): `src/train/config_schema.py` (experiment, gates, study), `src/train/config_sections.py` | ||
| 31 | (task, data, adapter, model, train, evaluation, runtime, provenance), `src/train/config_values.py` | ||
| 32 | (strict YAML leaf typing), `src/train/config_rules.py` (cross-section and ontology rules), | ||
| 33 | `src/train/config_study.py` (study algebra). `src/train/config.py` stays the only import | ||
| 34 | site. Leaf typing is deliberately stricter than the fleet coercion matrix -- `"50"` is not | ||
| 35 | a float and `1` is not a boolean -- the sanctioned opt-out for this YAML experiment | ||
| 36 | contract, documented in `src/train/config_values.py`. Runtime narrowing comes from CLI | ||
| 37 | arguments and study overrides, never from repo-local JSON. | ||
| 38 | |||
| 18 | ## E2: voxel-survival oracle | 39 | ## E2: voxel-survival oracle |
| 19 | 40 | ||
| 20 | E2 is local-first and CPU-capable. It must run only after canonical preparation regenerates and stages the full-resolution in-memory `FuseResult.classification`; written seg3d/ReCap representative clouds are rejected as oracle input. | 41 | E2 is local-first and CPU-capable. It must run only after canonical preparation regenerates and stages the full-resolution in-memory `FuseResult.classification`; written seg3d/ReCap representative clouds are rejected as oracle input. |
| 21 | 42 |
| 23 | from pathlib import Path | 23 | from pathlib import Path |
| 24 | from typing import Any | 24 | from typing import Any |
| 25 | 25 | ||
| 26 | from src.contracts.ontology import Ontology, OntologyError, load_ontology | 26 | from src.contracts.ontology import Ontology, OntologyError, load_ontology |
| 27 | from src.train.config import ConfigError, load_config, resolve_ontology_path | 27 | from src.train.config import ExperimentConfigError, HarnessConfig, resolve_ontology_path |
| 28 | 28 | ||
| 29 | POINTCEPT_FRAMEWORK = "pointcept" | 29 | POINTCEPT_FRAMEWORK = "pointcept" |
| 30 | SPT_FRAMEWORK = "spt" | 30 | SPT_FRAMEWORK = "spt" |
| 31 | SUPPORTED_FRAMEWORKS = (POINTCEPT_FRAMEWORK, SPT_FRAMEWORK) | 31 | SUPPORTED_FRAMEWORKS = (POINTCEPT_FRAMEWORK, SPT_FRAMEWORK) |
| 171 | 171 | ||
| 172 | Raises: | 172 | Raises: |
| 173 | EmissionCheckError: If the framework is unsupported, a narrowing is | 173 | EmissionCheckError: If the framework is unsupported, a narrowing is |
| 174 | requested for SPT, or the emission disagrees with the ontology. | 174 | requested for SPT, or the emission disagrees with the ontology. |
| 175 | ConfigError: If the harness YAML is malformed. | 175 | ExperimentConfigError: If the harness YAML is malformed. |
| 176 | OntologyError: If the declared ontology cannot be loaded. | 176 | OntologyError: If the declared ontology cannot be loaded. |
| 177 | """ | 177 | """ |
| 178 | if framework not in SUPPORTED_FRAMEWORKS: | 178 | if framework not in SUPPORTED_FRAMEWORKS: |
| 179 | raise EmissionCheckError( | 179 | raise EmissionCheckError( |
| 180 | f"{config_path}: unsupported framework {framework!r}; expected one " | 180 | f"{config_path}: unsupported framework {framework!r}; expected one " |
| 181 | f"of {list(SUPPORTED_FRAMEWORKS)}" | 181 | f"of {list(SUPPORTED_FRAMEWORKS)}" |
| 182 | ) | 182 | ) |
| 183 | config = load_config(config_path) | 183 | config = HarnessConfig.from_yaml(config_path) |
| 184 | ontology = load_ontology(resolve_ontology_path(config)) | 184 | ontology = load_ontology(resolve_ontology_path(config)) |
| 185 | lane = Path(processed_root) if processed_root else config.data.processed_root | 185 | lane = Path(processed_root) if processed_root else config.data.processed_root |
| 186 | if framework == SPT_FRAMEWORK: | 186 | if framework == SPT_FRAMEWORK: |
| 187 | if corridors: | 187 | if corridors: |
| 245 | processed_root=args.processed_root, | 245 | processed_root=args.processed_root, |
| 246 | corridors=tuple(args.corridor), | 246 | corridors=tuple(args.corridor), |
| 247 | ) | 247 | ) |
| 248 | except ( | 248 | except ( |
| 249 | ConfigError, | 249 | ExperimentConfigError, |
| 250 | EmissionCheckError, | 250 | EmissionCheckError, |
| 251 | OntologyError, | 251 | OntologyError, |
| 252 | OSError, | 252 | OSError, |
| 253 | ) as exc: | 253 | ) as exc: |
| 33 | SplitTier, | 33 | SplitTier, |
| 34 | authorize_split_access, | 34 | authorize_split_access, |
| 35 | load_split_manifest, | 35 | load_split_manifest, |
| 36 | ) | 36 | ) |
| 37 | from src.train.config import HarnessConfig, load_config, resolve_ontology_path | 37 | from src.train.config import HarnessConfig, resolve_ontology_path |
| 38 | 38 | ||
| 39 | STARTED = "started" | 39 | STARTED = "started" |
| 40 | COMPLETED = "completed" | 40 | COMPLETED = "completed" |
| 41 | PROVENANCE_FILENAMES: dict[str, str] = { | 41 | PROVENANCE_FILENAMES: dict[str, str] = { |
| 417 | Raises: | 417 | Raises: |
| 418 | RunProvenanceError: If the run cannot prove its required evidence. | 418 | RunProvenanceError: If the run cannot prove its required evidence. |
| 419 | """ | 419 | """ |
| 420 | args = build_parser().parse_args(argv) | 420 | args = build_parser().parse_args(argv) |
| 421 | config = load_config(args.config) | 421 | config = HarnessConfig.from_yaml(args.config) |
| 422 | split_manifest = load_split_manifest( | 422 | split_manifest = load_split_manifest( |
| 423 | config.data.split_manifest, | 423 | config.data.split_manifest, |
| 424 | ontology=load_ontology(resolve_ontology_path(config)), | 424 | ontology=load_ontology(resolve_ontology_path(config)), |
| 425 | ) | 425 | ) |
| 1 | """Emit ontology-derived CLI overrides for the SPT and Pointcept runners. | 1 | """Emit ontology-derived CLI overrides for the SPT and Pointcept runners. |
| 2 | 2 | ||
| 3 | The harness experiment YAML is the source of truth for class count, void ID, | 3 | The harness experiment YAML is the source of truth for class count, void ID, |
| 4 | and predicted class names. This module loads that YAML with the same | 4 | and predicted class names. This module loads that YAML with the same |
| 5 | :func:`src.train.config.load_config` / :func:`src.contracts.ontology.load_ontology` | 5 | :meth:`src.train.config.HarnessConfig.from_yaml` / |
| 6 | :func:`src.contracts.ontology.load_ontology` | ||
| 6 | path the rest of the harness uses, then prints one ``KEY=VALUE`` override per | 7 | path the rest of the harness uses, then prints one ``KEY=VALUE`` override per |
| 7 | line. The shell runners append those lines to the framework command: | 8 | line. The shell runners append those lines to the framework command: |
| 8 | 9 | ||
| 9 | * Pointcept consumes them as ``--options`` tokens (``DictAction`` then | 10 | * Pointcept consumes them as ``--options`` tokens (``DictAction`` then |
| 28 | from collections.abc import Sequence | 29 | from collections.abc import Sequence |
| 29 | from pathlib import Path | 30 | from pathlib import Path |
| 30 | 31 | ||
| 31 | from src.contracts.ontology import Ontology, OntologyError, load_ontology | 32 | from src.contracts.ontology import Ontology, OntologyError, load_ontology |
| 32 | from src.train.config import ConfigError, load_config, resolve_ontology_path | 33 | from src.train.config import ExperimentConfigError, HarnessConfig, resolve_ontology_path |
| 33 | 34 | ||
| 34 | POINTCEPT_FRAMEWORK = "pointcept" | 35 | POINTCEPT_FRAMEWORK = "pointcept" |
| 35 | SPT_FRAMEWORK = "spt" | 36 | SPT_FRAMEWORK = "spt" |
| 36 | SUPPORTED_FRAMEWORKS = (POINTCEPT_FRAMEWORK, SPT_FRAMEWORK) | 37 | SUPPORTED_FRAMEWORKS = (POINTCEPT_FRAMEWORK, SPT_FRAMEWORK) |
| 91 | One ``KEY=VALUE`` override per tuple element. | 92 | One ``KEY=VALUE`` override per tuple element. |
| 92 | 93 | ||
| 93 | Raises: | 94 | Raises: |
| 94 | RunnerOptionsError: If ``framework`` is unsupported. | 95 | RunnerOptionsError: If ``framework`` is unsupported. |
| 95 | ConfigError: If the harness YAML is malformed. | 96 | ExperimentConfigError: If the harness YAML is malformed. |
| 96 | OntologyError: If the declared ontology cannot be loaded. | 97 | OntologyError: If the declared ontology cannot be loaded. |
| 97 | """ | 98 | """ |
| 98 | if framework not in SUPPORTED_FRAMEWORKS: | 99 | if framework not in SUPPORTED_FRAMEWORKS: |
| 99 | raise RunnerOptionsError( | 100 | raise RunnerOptionsError( |
| 100 | f"{config_path}: unsupported framework {framework!r}; expected " | 101 | f"{config_path}: unsupported framework {framework!r}; expected " |
| 101 | f"one of {list(SUPPORTED_FRAMEWORKS)}" | 102 | f"one of {list(SUPPORTED_FRAMEWORKS)}" |
| 102 | ) | 103 | ) |
| 103 | config = load_config(config_path) | 104 | config = HarnessConfig.from_yaml(config_path) |
| 104 | ontology = load_ontology(resolve_ontology_path(config)) | 105 | ontology = load_ontology(resolve_ontology_path(config)) |
| 105 | if framework == SPT_FRAMEWORK: | 106 | if framework == SPT_FRAMEWORK: |
| 106 | return spt_overrides(ontology) | 107 | return spt_overrides(ontology) |
| 107 | return pointcept_overrides(ontology) | 108 | return pointcept_overrides(ontology) |
| 139 | """ | 140 | """ |
| 140 | args = build_parser().parse_args(argv) | 141 | args = build_parser().parse_args(argv) |
| 141 | try: | 142 | try: |
| 142 | lines = collect_overrides(args.config, args.framework) | 143 | lines = collect_overrides(args.config, args.framework) |
| 143 | except (ConfigError, OntologyError, RunnerOptionsError, OSError) as exc: | 144 | except (ExperimentConfigError, OntologyError, RunnerOptionsError, OSError) as exc: |
| 144 | print(f"{args.config}: {exc}", file=sys.stderr) | 145 | print(f"{args.config}: {exc}", file=sys.stderr) |
| 145 | return 1 | 146 | return 1 |
| 146 | if not lines: | 147 | if not lines: |
| 147 | print( | 148 | print( |
| 40 | continuity_metrics, | 40 | continuity_metrics, |
| 41 | object_metrics, | 41 | object_metrics, |
| 42 | ) | 42 | ) |
| 43 | from src.train.config import ( | 43 | from src.train.config import ( |
| 44 | ConfigError, | 44 | ExperimentConfigError, |
| 45 | HarnessConfig, | 45 | HarnessConfig, |
| 46 | is_canonical_metric, | 46 | is_canonical_metric, |
| 47 | load_config, | ||
| 48 | resolve_ontology_path, | 47 | resolve_ontology_path, |
| 49 | ) | 48 | ) |
| 50 | 49 | ||
| 51 | 50 |
| 127 | Zero after reports and finalized provenance are written. | 126 | Zero after reports and finalized provenance are written. |
| 128 | """ | 127 | """ |
| 129 | args = build_parser().parse_args(argv) | 128 | args = build_parser().parse_args(argv) |
| 130 | try: | 129 | try: |
| 131 | config = load_config(args.config) | 130 | config = HarnessConfig.from_yaml(args.config) |
| 132 | if config.model.framework not in {"spt", "pointcept"}: | 131 | if config.model.framework not in {"spt", "pointcept"}: |
| 133 | raise EvaluationError( | 132 | raise EvaluationError( |
| 134 | f"{config.experiment.id} framework {config.model.framework} " | 133 | f"{config.experiment.id} framework {config.model.framework} " |
| 135 | "has no checkpoint evaluator" | 134 | "has no checkpoint evaluator" |
| 313 | }, | 312 | }, |
| 314 | ) | 313 | ) |
| 315 | write_corridor_run_provenance(run_dir / "provenance.json", final_provenance) | 314 | write_corridor_run_provenance(run_dir / "provenance.json", final_provenance) |
| 316 | return 0 | 315 | return 0 |
| 317 | except (ConfigError, EvaluationError, OSError, RuntimeError, ValueError) as exc: | 316 | except ( |
| 317 | ExperimentConfigError, | ||
| 318 | EvaluationError, | ||
| 319 | OSError, | ||
| 320 | RuntimeError, | ||
| 321 | ValueError, | ||
| 322 | ) as exc: | ||
| 318 | raise SystemExit(str(exc)) from exc | 323 | raise SystemExit(str(exc)) from exc |
| 319 | 324 | ||
| 320 | 325 | ||
| 321 | @dataclass(frozen=True) | 326 | @dataclass(frozen=True) |
| 24 | PreannotationError, | 24 | PreannotationError, |
| 25 | load_preannotation, | 25 | load_preannotation, |
| 26 | select_preannotation, | 26 | select_preannotation, |
| 27 | ) | 27 | ) |
| 28 | from src.train.config import load_config, resolve_ontology_path | 28 | from src.train.config import HarnessConfig, resolve_ontology_path |
| 29 | 29 | ||
| 30 | 30 | ||
| 31 | def build_parser() -> argparse.ArgumentParser: | 31 | def build_parser() -> argparse.ArgumentParser: |
| 32 | """Build the frozen preannotation-ingest CLI parser. | 32 | """Build the frozen preannotation-ingest CLI parser. |
| 57 | Process exit status. | 57 | Process exit status. |
| 58 | """ | 58 | """ |
| 59 | args = build_parser().parse_args(argv) | 59 | args = build_parser().parse_args(argv) |
| 60 | try: | 60 | try: |
| 61 | config = load_config(args.config) | 61 | config = HarnessConfig.from_yaml(args.config) |
| 62 | ontology = load_ontology(resolve_ontology_path(config)) | 62 | ontology = load_ontology(resolve_ontology_path(config)) |
| 63 | split_manifest = load_split_manifest( | 63 | split_manifest = load_split_manifest( |
| 64 | config.data.split_manifest, ontology=ontology | 64 | config.data.split_manifest, ontology=ontology |
| 65 | ) | 65 | ) |
| 50 | validate_boundary_las_codes, | 50 | validate_boundary_las_codes, |
| 51 | write_normalization, | 51 | write_normalization, |
| 52 | ) | 52 | ) |
| 53 | from src.dataset.features import expand_feature_columns | 53 | from src.dataset.features import expand_feature_columns |
| 54 | from src.train.config import load_config, resolve_ontology_path | 54 | from src.train.config import HarnessConfig, resolve_ontology_path |
| 55 | 55 | ||
| 56 | 56 | ||
| 57 | def build_parser() -> argparse.ArgumentParser: | 57 | def build_parser() -> argparse.ArgumentParser: |
| 58 | """Build the frozen dataset-preparation CLI parser. | 58 | """Build the frozen dataset-preparation CLI parser. |
| 88 | Process exit status. | 88 | Process exit status. |
| 89 | """ | 89 | """ |
| 90 | args = build_parser().parse_args(argv) | 90 | args = build_parser().parse_args(argv) |
| 91 | try: | 91 | try: |
| 92 | config = load_config(args.config) | 92 | config = HarnessConfig.from_yaml(args.config) |
| 93 | ontology = load_ontology(resolve_ontology_path(config)) | 93 | ontology = load_ontology(resolve_ontology_path(config)) |
| 94 | split_manifest = load_split_manifest( | 94 | split_manifest = load_split_manifest( |
| 95 | config.data.split_manifest, ontology=ontology | 95 | config.data.split_manifest, ontology=ontology |
| 96 | ) | 96 | ) |
| 7 | import logging | 7 | import logging |
| 8 | from collections.abc import Sequence | 8 | from collections.abc import Sequence |
| 9 | from pathlib import Path | 9 | from pathlib import Path |
| 10 | 10 | ||
| 11 | from src.train.config import ConfigError, load_config | 11 | from src.train.config import ExperimentConfigError, HarnessConfig |
| 12 | from src.train.dispatch import ( | 12 | from src.train.dispatch import ( |
| 13 | DispatchError, | 13 | DispatchError, |
| 14 | DispatchRequest, | 14 | DispatchRequest, |
| 15 | dispatch_experiment, | 15 | dispatch_experiment, |
| 55 | """ | 55 | """ |
| 56 | logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") | 56 | logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") |
| 57 | args = build_parser().parse_args(argv) | 57 | args = build_parser().parse_args(argv) |
| 58 | try: | 58 | try: |
| 59 | config = load_config(args.config) | 59 | config = HarnessConfig.from_yaml(args.config) |
| 60 | cells = plan_experiment(config) | 60 | cells = plan_experiment(config) |
| 61 | LOGGER.info( | 61 | LOGGER.info( |
| 62 | "%s study kind %s expands to %d cell(s): %s", | 62 | "%s study kind %s expands to %d cell(s): %s", |
| 63 | config.experiment.id, | 63 | config.experiment.id, |
| 73 | log_dir=args.log_dir, | 73 | log_dir=args.log_dir, |
| 74 | num_workers=args.num_workers, | 74 | num_workers=args.num_workers, |
| 75 | ) | 75 | ) |
| 76 | return dispatch_experiment(config, request) | 76 | return dispatch_experiment(config, request) |
| 77 | except (ConfigError, DispatchError) as exc: | 77 | except (ExperimentConfigError, DispatchError) as exc: |
| 78 | raise SystemExit(str(exc)) from exc | 78 | raise SystemExit(str(exc)) from exc |
| 79 | 79 | ||
| 80 | 80 | ||
| 81 | if __name__ == "__main__": | 81 | if __name__ == "__main__": |
| 14 | from src.adapters.pointcept import load_pointcept_identity | 14 | from src.adapters.pointcept import load_pointcept_identity |
| 15 | from src.adapters.remap import TilePrediction, blend_and_remap | 15 | from src.adapters.remap import TilePrediction, blend_and_remap |
| 16 | from src.adapters.spt import load_spt_identity | 16 | from src.adapters.spt import load_spt_identity |
| 17 | from src.dataset.canonical import CanonicalCloud, read_canonical_tile | 17 | from src.dataset.canonical import CanonicalCloud, read_canonical_tile |
| 18 | from src.train.config import load_config | 18 | from src.train.config import HarnessConfig |
| 19 | 19 | ||
| 20 | DEFAULT_FRAME_TOLERANCE_MM = 0.6 | 20 | DEFAULT_FRAME_TOLERANCE_MM = 0.6 |
| 21 | """Tolerated frame reconstruction error. | 21 | """Tolerated frame reconstruction error. |
| 22 | 22 |
| 61 | if not np.isfinite(args.frame_tolerance_mm) or args.frame_tolerance_mm <= 0.0: | 61 | if not np.isfinite(args.frame_tolerance_mm) or args.frame_tolerance_mm <= 0.0: |
| 62 | raise ValueError( | 62 | raise ValueError( |
| 63 | f"--frame-tolerance-mm must be positive, got {args.frame_tolerance_mm}" | 63 | f"--frame-tolerance-mm must be positive, got {args.frame_tolerance_mm}" |
| 64 | ) | 64 | ) |
| 65 | config = load_config(args.config) | 65 | config = HarnessConfig.from_yaml(args.config) |
| 66 | # Per-corridor adapter manifests live in the canonical lane | 66 | # Per-corridor adapter manifests live in the canonical lane |
| 67 | # (<canonical_root>/canonical/<corridor>.adapter.json); the legacy | 67 | # (<canonical_root>/canonical/<corridor>.adapter.json); the legacy |
| 68 | # top-level location is still accepted so older lanes verify. | 68 | # top-level location is still accepted so older lanes verify. |
| 69 | canonical_lane = config.data.canonical_root / "canonical" | 69 | canonical_lane = config.data.canonical_root / "canonical" |
| 32 | validate_full_resolution_manifest, | 32 | validate_full_resolution_manifest, |
| 33 | write_oracle_outputs, | 33 | write_oracle_outputs, |
| 34 | ) | 34 | ) |
| 35 | from src.train.config import ( | 35 | from src.train.config import ( |
| 36 | ConfigError, | 36 | ExperimentConfigError, |
| 37 | HarnessConfig, | 37 | HarnessConfig, |
| 38 | load_config, | ||
| 39 | resolve_ontology_path, | 38 | resolve_ontology_path, |
| 40 | ) | 39 | ) |
| 41 | 40 | ||
| 42 | LOGGER = logging.getLogger(__name__) | 41 | LOGGER = logging.getLogger(__name__) |
| 82 | """ | 81 | """ |
| 83 | args = build_parser().parse_args(argv) | 82 | args = build_parser().parse_args(argv) |
| 84 | logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") | 83 | logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") |
| 85 | try: | 84 | try: |
| 86 | config = load_config(args.config) | 85 | config = HarnessConfig.from_yaml(args.config) |
| 87 | if config.experiment.id != "E2": | 86 | if config.experiment.id != "E2": |
| 88 | raise VoxelOracleError( | 87 | raise VoxelOracleError( |
| 89 | f"scripts/voxel_oracle.py accepts E2 only, got {config.experiment.id}" | 88 | f"scripts/voxel_oracle.py accepts E2 only, got {config.experiment.id}" |
| 90 | ) | 89 | ) |
| 159 | ) | 158 | ) |
| 160 | LOGGER.info("Wrote E2 products: %s", ", ".join(str(path) for path in paths)) | 159 | LOGGER.info("Wrote E2 products: %s", ", ".join(str(path) for path in paths)) |
| 161 | return 0 | 160 | return 0 |
| 162 | except ( | 161 | except ( |
| 163 | ConfigError, | 162 | ExperimentConfigError, |
| 164 | SplitAccessError, | 163 | SplitAccessError, |
| 165 | VoxelOracleError, | 164 | VoxelOracleError, |
| 166 | OSError, | 165 | OSError, |
| 167 | ValueError, | 166 | ValueError, |
| 1 | """Public strict configuration and experiment-dispatch interfaces.""" | 1 | """Public strict configuration and experiment-dispatch interfaces.""" |
| 2 | 2 | ||
| 3 | from src.train.config import ConfigError, HarnessConfig, TrainConfig, load_config | 3 | from src.train.config import ExperimentConfigError, HarnessConfig, TrainConfig |
| 4 | from src.train.dispatch import DispatchError, dispatch_experiment | 4 | from src.train.dispatch import DispatchError, dispatch_experiment |
| 5 | 5 | ||
| 6 | __all__ = [ | 6 | __all__ = [ |
| 7 | "ConfigError", | ||
| 8 | "DispatchError", | 7 | "DispatchError", |
| 8 | "ExperimentConfigError", | ||
| 9 | "HarnessConfig", | 9 | "HarnessConfig", |
| 10 | "TrainConfig", | 10 | "TrainConfig", |
| 11 | "dispatch_experiment", | 11 | "dispatch_experiment", |
| 12 | "load_config", | ||
| 13 | ] | 12 | ] |
| 1 | """Strict experiment configuration schema for corridor segmentation studies. | 1 | """Strict experiment configuration schema for corridor segmentation studies. |
| 2 | 2 | ||
| 3 | The schema itself is a pydantic model tree built on | 3 | The schema is a pydantic model tree on |
| 4 | :class:`iolabs.common.config_loader.ConfigModel`: :mod:`src.train.config_schema` | 4 | :class:`iolabs.common.config_loader.ConfigModel`, split by section because it |
| 5 | holds the experiment/gate/study models, :mod:`src.train.config_sections` the | 5 | exceeds one module: :mod:`src.train.config_schema` holds the experiment, gate, |
| 6 | data, model, training, and evaluation blocks. This module is the entry point | 6 | and study models, :mod:`src.train.config_sections` the data, model, training, |
| 7 | every script imports: it loads one YAML document, validates it, applies the | 7 | and evaluation blocks, :mod:`src.train.config_values` the strict YAML leaf |
| 8 | rules of :mod:`src.train.config_rules`, and expands a study into re-validated | 8 | typing, :mod:`src.train.config_rules` the cross-section rules, and |
| 9 | cells with :mod:`src.train.config_study`. | 9 | :mod:`src.train.config_study` the study algebra. This module is the entry point |
| 10 | 10 | every script imports: :meth:`HarnessConfig.from_yaml` loads and validates one | |
| 11 | Adding a configuration key means adding a field to its model (and to the YAML | 11 | YAML document, :func:`expand_study` expands a study into re-validated cells, |
| 12 | documents under ``configs/``); nothing else has to be touched. | 12 | and both raise :class:`ExperimentConfigError`. |
| 13 | |||
| 14 | Adding a configuration key means adding the field to its model and the same key | ||
| 15 | to the YAML documents under ``configs/`` -- nothing else. Unknown keys are | ||
| 16 | rejected. | ||
| 13 | """ | 17 | """ |
| 14 | 18 | ||
| 15 | from __future__ import annotations | 19 | from __future__ import annotations |
| 16 | 20 |
| 85 | TilingConfig, | 89 | TilingConfig, |
| 86 | TrainConfig, | 90 | TrainConfig, |
| 87 | VisualizationConfig, | 91 | VisualizationConfig, |
| 88 | ) | 92 | ) |
| 89 | from src.train.config_values import ConfigError, OverrideValue, Scalar | 93 | from src.train.config_values import ExperimentConfigError, OverrideValue, Scalar |
| 90 | 94 | ||
| 91 | logger = logging.getLogger(__name__) | 95 | logger = logging.getLogger(__name__) |
| 92 | 96 | ||
| 93 | _CONTEXT = "experiment config" | 97 | _CONTEXT = "experiment config" |
| 98 | "ArtifactExistsParams", | 102 | "ArtifactExistsParams", |
| 99 | "BootstrapConfig", | 103 | "BootstrapConfig", |
| 100 | "CheckpointPolicyGate", | 104 | "CheckpointPolicyGate", |
| 101 | "CheckpointPolicyParams", | 105 | "CheckpointPolicyParams", |
| 102 | "ConfigError", | ||
| 103 | "ContinuityConfig", | 106 | "ContinuityConfig", |
| 104 | "ContrastConfig", | 107 | "ContrastConfig", |
| 105 | "CorridorSelectionConfig", | 108 | "CorridorSelectionConfig", |
| 106 | "DataAvailableGate", | 109 | "DataAvailableGate", |
| 107 | "DataAvailableParams", | 110 | "DataAvailableParams", |
| 108 | "DataConfig", | 111 | "DataConfig", |
| 109 | "EvaluationConfig", | 112 | "EvaluationConfig", |
| 110 | "ExperimentConfig", | 113 | "ExperimentConfig", |
| 114 | "ExperimentConfigError", | ||
| 111 | "ExperimentStatus", | 115 | "ExperimentStatus", |
| 112 | "FeatureConfig", | 116 | "FeatureConfig", |
| 113 | "GateBase", | 117 | "GateBase", |
| 114 | "GateConfig", | 118 | "GateConfig", |
| 148 | "VariantConfig", | 152 | "VariantConfig", |
| 149 | "VisualizationConfig", | 153 | "VisualizationConfig", |
| 150 | "expand_study", | 154 | "expand_study", |
| 151 | "is_canonical_metric", | 155 | "is_canonical_metric", |
| 152 | "load_config", | ||
| 153 | "resolve_ontology_path", | 156 | "resolve_ontology_path", |
| 154 | ] | 157 | ] |
| 155 | 158 | ||
| 156 | 159 |
| 174 | document: str | 177 | document: str |
| 175 | config: HarnessConfig | 178 | config: HarnessConfig |
| 176 | 179 | ||
| 177 | 180 | ||
| 178 | def load_config(path: str | Path) -> HarnessConfig: | 181 | def _load_yaml_document(path: str | Path) -> HarnessConfig: |
| 179 | """Load and strictly validate one E1--E14 experiment YAML. | 182 | """Read, strictly validate, and rule-check one experiment YAML. |
| 183 | |||
| 184 | Implementation of :meth:`HarnessConfig.from_yaml`, which is the public | ||
| 185 | entry point; it lives here because the loading pipeline needs | ||
| 186 | :mod:`src.train.config_rules` and :mod:`src.train.config_study`. | ||
| 180 | 187 | ||
| 181 | Args: | 188 | Args: |
| 182 | path: Repository-relative or absolute YAML path. | 189 | path: Repository-relative or absolute YAML path. |
| 183 | 190 | ||
| 184 | Returns: | 191 | Returns: |
| 185 | Fully typed immutable configuration. | 192 | Fully typed immutable configuration. |
| 186 | 193 | ||
| 187 | Raises: | 194 | Raises: |
| 188 | ConfigError: If loading, strict parsing, or validation fails. | 195 | ExperimentConfigError: If loading, strict parsing, or validation fails. |
| 189 | """ | 196 | """ |
| 190 | config_path = Path(path) | 197 | config_path = Path(path) |
| 191 | try: | 198 | try: |
| 192 | payload = config_path.read_bytes() | 199 | payload = config_path.read_bytes() |
| 193 | raw = yaml.safe_load(payload.decode("utf-8")) | 200 | raw = yaml.safe_load(payload.decode("utf-8")) |
| 194 | except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: | 201 | except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: |
| 195 | raise ConfigError(f"Cannot load config {config_path}: {exc}") from exc | 202 | raise ExperimentConfigError( |
| 203 | f"Cannot load config {config_path}: {exc}" | ||
| 204 | ) from exc | ||
| 196 | try: | 205 | try: |
| 197 | return _build_config(raw, config_path, hashlib.sha256(payload).hexdigest()) | 206 | return _build_config(raw, config_path, hashlib.sha256(payload).hexdigest()) |
| 198 | except ConfigError as exc: | 207 | except ExperimentConfigError as exc: |
| 199 | raise ConfigError(f"{config_path}: {exc}") from exc | 208 | raise ExperimentConfigError(f"{config_path}: {exc}") from exc |
| 200 | 209 | ||
| 201 | 210 | ||
| 202 | def expand_study(config: HarnessConfig) -> tuple[StudyCell, ...]: | 211 | def expand_study(config: HarnessConfig) -> tuple[StudyCell, ...]: |
| 203 | """Expand a typed study definition into ordered, re-validated cells. | 212 | """Expand a typed study definition into ordered, re-validated cells. |
| 206 | per declared variant, ``matrix`` the deterministic cross product of its | 215 | per declared variant, ``matrix`` the deterministic cross product of its |
| 207 | axes after ``include``/``exclude``, and ``sweep`` the deterministic | 216 | axes after ``include``/``exclude``, and ``sweep`` the deterministic |
| 208 | enumeration of its typed parameters under the study ``seed``. Every cell's | 217 | enumeration of its typed parameters under the study ``seed``. Every cell's |
| 209 | overrides are re-applied to the raw mapping and re-validated through | 218 | overrides are re-applied to the raw mapping and re-validated through |
| 210 | :func:`load_config`'s machinery, so each cell carries its own SHA-256. | 219 | :meth:`HarnessConfig.from_yaml`'s machinery, so each cell carries its own SHA-256. |
| 211 | 220 | ||
| 212 | Args: | 221 | Args: |
| 213 | config: Strictly parsed experiment configuration. | 222 | config: Strictly parsed experiment configuration. |
| 214 | 223 | ||
| 215 | Returns: | 224 | Returns: |
| 216 | Deterministically ordered study cells, never empty. | 225 | Deterministically ordered study cells, never empty. |
| 217 | 226 | ||
| 218 | Raises: | 227 | Raises: |
| 219 | ConfigError: If the study payload, an override, a cell identity, or a | 228 | ExperimentConfigError: If the study payload, an override, a cell identity, or a |
| 220 | resolved cell configuration violates the strict schema. | 229 | resolved cell configuration violates the strict schema. |
| 221 | """ | 230 | """ |
| 222 | raw = _raw_document(config) | 231 | raw = _raw_document(config) |
| 223 | definitions = config_study.cell_definitions(config) | 232 | definitions = config_study.cell_definitions(config) |
| 224 | if not definitions: | 233 | if not definitions: |
| 225 | raise ConfigError( | 234 | raise ExperimentConfigError( |
| 226 | f"{config.experiment.id} study kind {config.study.kind} expanded to " | 235 | f"{config.experiment.id} study kind {config.study.kind} expanded to " |
| 227 | "no cells" | 236 | "no cells" |
| 228 | ) | 237 | ) |
| 229 | identities = [cell_id for cell_id, _ in definitions] | 238 | identities = [cell_id for cell_id, _ in definitions] |
| 230 | duplicates = sorted({item for item in identities if identities.count(item) > 1}) | 239 | duplicates = sorted({item for item in identities if identities.count(item) > 1}) |
| 231 | if duplicates: | 240 | if duplicates: |
| 232 | raise ConfigError( | 241 | raise ExperimentConfigError( |
| 233 | f"{config.experiment.id} study has duplicate cell IDs {duplicates}" | 242 | f"{config.experiment.id} study has duplicate cell IDs {duplicates}" |
| 234 | ) | 243 | ) |
| 235 | cells: list[StudyCell] = [] | 244 | cells: list[StudyCell] = [] |
| 236 | for index, (cell_id, overrides) in enumerate(definitions): | 245 | for index, (cell_id, overrides) in enumerate(definitions): |
| 239 | document = yaml.safe_dump(cell_raw, sort_keys=True, default_flow_style=False) | 248 | document = yaml.safe_dump(cell_raw, sort_keys=True, default_flow_style=False) |
| 240 | digest = hashlib.sha256(document.encode("utf-8")).hexdigest() | 249 | digest = hashlib.sha256(document.encode("utf-8")).hexdigest() |
| 241 | try: | 250 | try: |
| 242 | cell_config = _build_config(cell_raw, config.source_path, digest) | 251 | cell_config = _build_config(cell_raw, config.source_path, digest) |
| 243 | except ConfigError as exc: | 252 | except ExperimentConfigError as exc: |
| 244 | raise ConfigError(f"{config.source_path} cell {cell_id!r}: {exc}") from exc | 253 | raise ExperimentConfigError( |
| 254 | f"{config.source_path} cell {cell_id!r}: {exc}" | ||
| 255 | ) from exc | ||
| 245 | run_name = ( | 256 | run_name = ( |
| 246 | config.experiment.id | 257 | config.experiment.id |
| 247 | if config.study.kind == "single" | 258 | if config.study.kind == "single" |
| 248 | else f"{config.experiment.id}_{cell_id}" | 259 | else f"{config.experiment.id}_{cell_id}" |
| 263 | def _build_config(raw: Any, config_path: Path, sha256: str) -> HarnessConfig: | 274 | def _build_config(raw: Any, config_path: Path, sha256: str) -> HarnessConfig: |
| 264 | """Validate one raw document into a configuration carrying its digest.""" | 275 | """Validate one raw document into a configuration carrying its digest.""" |
| 265 | root = config_values.mapping(raw, str(config_path)) | 276 | root = config_values.mapping(raw, str(config_path)) |
| 266 | config = config_loader.validate_config( | 277 | config = config_loader.validate_config( |
| 267 | HarnessConfig, dict(root), context=_CONTEXT, error_cls=ConfigError | 278 | HarnessConfig, dict(root), context=_CONTEXT, error_cls=ExperimentConfigError |
| 268 | ) | 279 | ) |
| 269 | config.attach_provenance(config_path, sha256, copy.deepcopy(dict(root))) | 280 | config.attach_provenance(config_path, sha256, copy.deepcopy(dict(root))) |
| 270 | config_rules.validate_config(config, root) | 281 | config_rules.validate_config(config, root) |
| 271 | return config | 282 | return config |
| 274 | def _raw_document(config: HarnessConfig) -> Mapping[str, Any]: | 285 | def _raw_document(config: HarnessConfig) -> Mapping[str, Any]: |
| 275 | """Return the raw mapping a configuration was validated from. | 286 | """Return the raw mapping a configuration was validated from. |
| 276 | 287 | ||
| 277 | Args: | 288 | Args: |
| 278 | config: Configuration produced by :func:`load_config` or | 289 | config: Configuration produced by :meth:`HarnessConfig.from_yaml` |
| 279 | :func:`expand_study`. | 290 | or :func:`expand_study`. |
| 280 | 291 | ||
| 281 | Returns: | 292 | Returns: |
| 282 | The stashed raw mapping, or a fresh parse of ``config.source_path``. | 293 | The stashed raw mapping, or a fresh parse of ``config.source_path``. |
| 283 | 294 | ||
| 284 | Raises: | 295 | Raises: |
| 285 | ConfigError: If the source file must be re-read and cannot be parsed. | 296 | ExperimentConfigError: If the source file must be re-read and cannot be parsed. |
| 286 | """ | 297 | """ |
| 287 | stashed = config._raw_document | 298 | stashed = config._raw_document |
| 288 | if stashed is not None: | 299 | if stashed is not None: |
| 289 | return stashed | 300 | return stashed |
| 290 | try: | 301 | try: |
| 291 | payload = config.source_path.read_bytes() | 302 | payload = config.source_path.read_bytes() |
| 292 | raw = yaml.safe_load(payload.decode("utf-8")) | 303 | raw = yaml.safe_load(payload.decode("utf-8")) |
| 293 | except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: | 304 | except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: |
| 294 | raise ConfigError( | 305 | raise ExperimentConfigError( |
| 295 | f"Cannot re-read config {config.source_path}: {exc}" | 306 | f"Cannot re-read config {config.source_path}: {exc}" |
| 296 | ) from exc | 307 | ) from exc |
| 297 | return config_values.mapping(raw, str(config.source_path)) | 308 | return config_values.mapping(raw, str(config.source_path)) |
| 88 | Returns: | 88 | Returns: |
| 89 | An absolute, existing ontology path. | 89 | An absolute, existing ontology path. |
| 90 | 90 | ||
| 91 | Raises: | 91 | Raises: |
| 92 | ConfigError: If no candidate path exists. | 92 | ExperimentConfigError: If no candidate path exists. |
| 93 | """ | 93 | """ |
| 94 | declared = config.task.ontology | 94 | declared = config.task.ontology |
| 95 | if declared.is_absolute(): | 95 | if declared.is_absolute(): |
| 96 | if not declared.is_file(): | 96 | if not declared.is_file(): |
| 97 | raise config_values.ConfigError( | 97 | raise config_values.ExperimentConfigError( |
| 98 | f"{config.source_path}: task.ontology {declared.as_posix()} " | 98 | f"{config.source_path}: task.ontology {declared.as_posix()} " |
| 99 | f"does not exist" | 99 | f"does not exist" |
| 100 | ) | 100 | ) |
| 101 | return declared | 101 | return declared |
| 108 | return candidate.resolve() | 108 | return candidate.resolve() |
| 109 | searched = ", ".join( | 109 | searched = ", ".join( |
| 110 | sorted({candidate.parent.as_posix() for candidate in candidates}) | 110 | sorted({candidate.parent.as_posix() for candidate in candidates}) |
| 111 | ) | 111 | ) |
| 112 | raise config_values.ConfigError( | 112 | raise config_values.ExperimentConfigError( |
| 113 | f"{config.source_path}: task.ontology {declared.as_posix()} was not " | 113 | f"{config.source_path}: task.ontology {declared.as_posix()} was not " |
| 114 | f"found relative to any parent of the config or to the working " | 114 | f"found relative to any parent of the config or to the working " |
| 115 | f"directory {Path.cwd().as_posix()}; searched {searched}" | 115 | f"directory {Path.cwd().as_posix()}; searched {searched}" |
| 116 | ) | 116 | ) |
| 126 | raw: Raw mapping the configuration was parsed from, used to check the | 126 | raw: Raw mapping the configuration was parsed from, used to check the |
| 127 | declared study overrides against the document's own leaves. | 127 | declared study overrides against the document's own leaves. |
| 128 | 128 | ||
| 129 | Raises: | 129 | Raises: |
| 130 | ConfigError: If a cross-section, identity, or ontology rule fails. | 130 | ExperimentConfigError: If a cross-section, identity, or ontology rule fails. |
| 131 | """ | 131 | """ |
| 132 | if config.schema_version != 1: | 132 | if config.schema_version != 1: |
| 133 | raise config_values.ConfigError( | 133 | raise config_values.ExperimentConfigError( |
| 134 | f"Unsupported schema_version {config.schema_version}" | 134 | f"Unsupported schema_version {config.schema_version}" |
| 135 | ) | 135 | ) |
| 136 | if not _EXPERIMENT_ID_PATTERN.fullmatch(config.experiment.id): | 136 | if not _EXPERIMENT_ID_PATTERN.fullmatch(config.experiment.id): |
| 137 | raise config_values.ConfigError( | 137 | raise config_values.ExperimentConfigError( |
| 138 | f"Invalid experiment.id {config.experiment.id!r}" | 138 | f"Invalid experiment.id {config.experiment.id!r}" |
| 139 | ) | 139 | ) |
| 140 | ontology = _load_task_ontology(config) | 140 | ontology = _load_task_ontology(config) |
| 141 | _validate_task_against_ontology(config, ontology) | 141 | _validate_task_against_ontology(config, ontology) |
| 149 | """Check the rules that tie a section to the declared model framework.""" | 149 | """Check the rules that tie a section to the declared model framework.""" |
| 150 | if config.model.framework in {"spt", "pointcept"} and ( | 150 | if config.model.framework in {"spt", "pointcept"} and ( |
| 151 | _VIZ_TRAIN_KEYS & config.train.model_fields_set | 151 | _VIZ_TRAIN_KEYS & config.train.model_fields_set |
| 152 | ): | 152 | ): |
| 153 | raise config_values.ConfigError( | 153 | raise config_values.ExperimentConfigError( |
| 154 | "train.viz_every_n_epochs and train.viz_samples are forbidden for " | 154 | "train.viz_every_n_epochs and train.viz_samples are forbidden for " |
| 155 | "external frameworks" | 155 | "external frameworks" |
| 156 | ) | 156 | ) |
| 157 | if config.model.framework == "pointcept" and config.runtime.flash_attention: | 157 | if config.model.framework == "pointcept" and config.runtime.flash_attention: |
| 158 | raise config_values.ConfigError( | 158 | raise config_values.ExperimentConfigError( |
| 159 | "Pointcept PTv3/LitePT configurations must keep FlashAttention disabled" | 159 | "Pointcept PTv3/LitePT configurations must keep FlashAttention disabled" |
| 160 | ) | 160 | ) |
| 161 | if config.visualization is not None and config.model.framework != "pointcept": | 161 | if config.visualization is not None and config.model.framework != "pointcept": |
| 162 | raise config_values.ConfigError( | 162 | raise config_values.ExperimentConfigError( |
| 163 | "visualization is only supported for model.framework pointcept; " | 163 | "visualization is only supported for model.framework pointcept; " |
| 164 | f"{config.model.framework} configs must omit the block" | 164 | f"{config.model.framework} configs must omit the block" |
| 165 | ) | 165 | ) |
| 166 | 166 |
| 170 | ) -> None: | 170 | ) -> None: |
| 171 | """Check every declared metric tag against the ontology's namespace.""" | 171 | """Check every declared metric tag against the ontology's namespace.""" |
| 172 | for metric in config.experiment.deciding_metrics: | 172 | for metric in config.experiment.deciding_metrics: |
| 173 | if not is_canonical_metric(metric, ontology=ontology): | 173 | if not is_canonical_metric(metric, ontology=ontology): |
| 174 | raise config_values.ConfigError( | 174 | raise config_values.ExperimentConfigError( |
| 175 | "experiment.deciding_metrics contains non-canonical metric " | 175 | "experiment.deciding_metrics contains non-canonical metric " |
| 176 | f"{metric!r} for ontology {ontology.name}" | 176 | f"{metric!r} for ontology {ontology.name}" |
| 177 | ) | 177 | ) |
| 178 | for metric in (config.train.monitor, config.train.early_stop_monitor): | 178 | for metric in (config.train.monitor, config.train.early_stop_monitor): |
| 179 | if not is_canonical_metric(metric, ontology=ontology): | 179 | if not is_canonical_metric(metric, ontology=ontology): |
| 180 | raise config_values.ConfigError( | 180 | raise config_values.ExperimentConfigError( |
| 181 | f"train monitor {metric!r} is outside the canonical namespace " | 181 | f"train monitor {metric!r} is outside the canonical namespace " |
| 182 | f"of ontology {ontology.name}" | 182 | f"of ontology {ontology.name}" |
| 183 | ) | 183 | ) |
| 184 | unknown_floor_classes = sorted( | 184 | unknown_floor_classes = sorted( |
| 185 | set(config.evaluation.precision_floors) - set(ontology.class_names) | 185 | set(config.evaluation.precision_floors) - set(ontology.class_names) |
| 186 | ) | 186 | ) |
| 187 | if unknown_floor_classes: | 187 | if unknown_floor_classes: |
| 188 | raise config_values.ConfigError( | 188 | raise config_values.ExperimentConfigError( |
| 189 | "evaluation.precision_floors has unknown classes " | 189 | "evaluation.precision_floors has unknown classes " |
| 190 | f"{unknown_floor_classes} for ontology {ontology.name}" | 190 | f"{unknown_floor_classes} for ontology {ontology.name}" |
| 191 | ) | 191 | ) |
| 192 | for contrast in config.study.contrasts: | 192 | for contrast in config.study.contrasts: |
| 193 | if not is_canonical_metric(contrast.metric, ontology=ontology): | 193 | if not is_canonical_metric(contrast.metric, ontology=ontology): |
| 194 | raise config_values.ConfigError( | 194 | raise config_values.ExperimentConfigError( |
| 195 | f"study contrast metric {contrast.metric!r} is not canonical " | 195 | f"study contrast metric {contrast.metric!r} is not canonical " |
| 196 | f"for ontology {ontology.name}" | 196 | f"for ontology {ontology.name}" |
| 197 | ) | 197 | ) |
| 198 | if config.study.sweep is not None and not is_canonical_metric( | 198 | if config.study.sweep is not None and not is_canonical_metric( |
| 199 | config.study.sweep.objective, ontology=ontology | 199 | config.study.sweep.objective, ontology=ontology |
| 200 | ): | 200 | ): |
| 201 | raise config_values.ConfigError( | 201 | raise config_values.ExperimentConfigError( |
| 202 | f"study.sweep.objective must be canonical for ontology {ontology.name}" | 202 | f"study.sweep.objective must be canonical for ontology {ontology.name}" |
| 203 | ) | 203 | ) |
| 204 | 204 | ||
| 205 | 205 |
| 212 | unknown = sorted( | 212 | unknown = sorted( |
| 213 | set(gate.params.minimum_purity_by_class) - set(ontology.class_names) | 213 | set(gate.params.minimum_purity_by_class) - set(ontology.class_names) |
| 214 | ) | 214 | ) |
| 215 | if unknown: | 215 | if unknown: |
| 216 | raise config_values.ConfigError( | 216 | raise config_values.ExperimentConfigError( |
| 217 | f"gate {gate.name}.minimum_purity_by_class contains unknown " | 217 | f"gate {gate.name}.minimum_purity_by_class contains unknown " |
| 218 | f"ontology classes {unknown} for ontology {ontology.name}" | 218 | f"ontology classes {unknown} for ontology {ontology.name}" |
| 219 | ) | 219 | ) |
| 220 | gate_types = {gate.type for gate in config.experiment.gates if gate.required} | 220 | gate_types = {gate.type for gate in config.experiment.gates if gate.required} |
| 221 | if ( | 221 | if ( |
| 222 | config.experiment.status == "template-only" | 222 | config.experiment.status == "template-only" |
| 223 | and "implementation_ticket" not in gate_types | 223 | and "implementation_ticket" not in gate_types |
| 224 | ): | 224 | ): |
| 225 | raise config_values.ConfigError( | 225 | raise config_values.ExperimentConfigError( |
| 226 | "template-only experiments require an implementation_ticket gate" | 226 | "template-only experiments require an implementation_ticket gate" |
| 227 | ) | 227 | ) |
| 228 | if config.experiment.status == "gated-later" and not gate_types: | 228 | if config.experiment.status == "gated-later" and not gate_types: |
| 229 | raise config_values.ConfigError( | 229 | raise config_values.ExperimentConfigError( |
| 230 | "gated-later experiments require at least one required gate" | 230 | "gated-later experiments require at least one required gate" |
| 231 | ) | 231 | ) |
| 232 | if config.experiment.status == "implement-now" and config.experiment.id not in { | 232 | if config.experiment.status == "implement-now" and config.experiment.id not in { |
| 233 | "E1", | 233 | "E1", |
| 234 | "E2", | 234 | "E2", |
| 235 | }: | 235 | }: |
| 236 | raise config_values.ConfigError( | 236 | raise config_values.ExperimentConfigError( |
| 237 | "Only E1 and E2 are implement-now in schema version 1" | 237 | "Only E1 and E2 are implement-now in schema version 1" |
| 238 | ) | 238 | ) |
| 239 | 239 | ||
| 240 | 240 | ||
| 241 | def _load_task_ontology(config: config_sections.HarnessConfig) -> Ontology: | 241 | def _load_task_ontology(config: config_sections.HarnessConfig) -> Ontology: |
| 242 | """Load the ontology the config declares, failing closed as a ConfigError. | 242 | """Load the ontology the config declares, failing closed as a ExperimentConfigError. |
| 243 | 243 | ||
| 244 | Args: | 244 | Args: |
| 245 | config: Parsed configuration whose ``task.ontology`` path is resolved | 245 | config: Parsed configuration whose ``task.ontology`` path is resolved |
| 246 | relative to the repository root. | 246 | relative to the repository root. |
| 248 | Returns: | 248 | Returns: |
| 249 | The validated ontology every other contract is checked against. | 249 | The validated ontology every other contract is checked against. |
| 250 | 250 | ||
| 251 | Raises: | 251 | Raises: |
| 252 | ConfigError: If the ontology cannot be located, loaded, or is invalid. | 252 | ExperimentConfigError: If the ontology cannot be located, loaded, or is invalid. |
| 253 | """ | 253 | """ |
| 254 | resolved = resolve_ontology_path(config) | 254 | resolved = resolve_ontology_path(config) |
| 255 | try: | 255 | try: |
| 256 | return load_ontology(resolved) | 256 | return load_ontology(resolved) |
| 257 | except OntologyError as exc: | 257 | except OntologyError as exc: |
| 258 | raise config_values.ConfigError( | 258 | raise config_values.ExperimentConfigError( |
| 259 | f"task.ontology {config.task.ontology.as_posix()} is not a valid " | 259 | f"task.ontology {config.task.ontology.as_posix()} is not a valid " |
| 260 | f"ontology: {exc}" | 260 | f"ontology: {exc}" |
| 261 | ) from exc | 261 | ) from exc |
| 262 | 262 |
| 270 | config: Parsed configuration. | 270 | config: Parsed configuration. |
| 271 | ontology: Ontology loaded from ``task.ontology``. | 271 | ontology: Ontology loaded from ``task.ontology``. |
| 272 | 272 | ||
| 273 | Raises: | 273 | Raises: |
| 274 | ConfigError: If any task, evaluation, model, or loss class contract | 274 | ExperimentConfigError: If any task, evaluation, model, or loss class contract |
| 275 | disagrees with the loaded ontology. | 275 | disagrees with the loaded ontology. |
| 276 | """ | 276 | """ |
| 277 | task = config.task | 277 | task = config.task |
| 278 | if task.num_classes != ontology.num_predicted_classes: | 278 | if task.num_classes != ontology.num_predicted_classes: |
| 279 | raise config_values.ConfigError( | 279 | raise config_values.ExperimentConfigError( |
| 280 | f"task.num_classes {task.num_classes} must equal ontology " | 280 | f"task.num_classes {task.num_classes} must equal ontology " |
| 281 | f"{ontology.name} num_predicted_classes " | 281 | f"{ontology.name} num_predicted_classes " |
| 282 | f"{ontology.num_predicted_classes}" | 282 | f"{ontology.num_predicted_classes}" |
| 283 | ) | 283 | ) |
| 284 | if task.ignore_index != ontology.void_id: | 284 | if task.ignore_index != ontology.void_id: |
| 285 | raise config_values.ConfigError( | 285 | raise config_values.ExperimentConfigError( |
| 286 | f"task.ignore_index {task.ignore_index} must equal ontology " | 286 | f"task.ignore_index {task.ignore_index} must equal ontology " |
| 287 | f"{ontology.name} void ID {ontology.void_id}" | 287 | f"{ontology.name} void ID {ontology.void_id}" |
| 288 | ) | 288 | ) |
| 289 | if task.classes_of_interest != ontology.interest_ids: | 289 | if task.classes_of_interest != ontology.interest_ids: |
| 290 | raise config_values.ConfigError( | 290 | raise config_values.ExperimentConfigError( |
| 291 | f"task.classes_of_interest {list(task.classes_of_interest)} must " | 291 | f"task.classes_of_interest {list(task.classes_of_interest)} must " |
| 292 | f"equal ontology {ontology.name} interest IDs " | 292 | f"equal ontology {ontology.name} interest IDs " |
| 293 | f"{list(ontology.interest_ids)}" | 293 | f"{list(ontology.interest_ids)}" |
| 294 | ) | 294 | ) |
| 297 | ) | 297 | ) |
| 298 | if len(set(task.linear_classes)) != len(task.linear_classes) or set( | 298 | if len(set(task.linear_classes)) != len(task.linear_classes) or set( |
| 299 | task.linear_classes | 299 | task.linear_classes |
| 300 | ) != set(linear_names): | 300 | ) != set(linear_names): |
| 301 | raise config_values.ConfigError( | 301 | raise config_values.ExperimentConfigError( |
| 302 | f"task.linear_classes {list(task.linear_classes)} must be exactly " | 302 | f"task.linear_classes {list(task.linear_classes)} must be exactly " |
| 303 | f"the linear classes {list(linear_names)} of ontology " | 303 | f"the linear classes {list(linear_names)} of ontology " |
| 304 | f"{ontology.name}" | 304 | f"{ontology.name}" |
| 305 | ) | 305 | ) |
| 306 | profiles = config.evaluation.object_matching.cluster_profiles | 306 | profiles = config.evaluation.object_matching.cluster_profiles |
| 307 | if profiles != task.ontology: | 307 | if profiles != task.ontology: |
| 308 | raise config_values.ConfigError( | 308 | raise config_values.ExperimentConfigError( |
| 309 | "evaluation.object_matching.cluster_profiles " | 309 | "evaluation.object_matching.cluster_profiles " |
| 310 | f"{profiles.as_posix()} must be the task ontology " | 310 | f"{profiles.as_posix()} must be the task ontology " |
| 311 | f"{task.ontology.as_posix()}" | 311 | f"{task.ontology.as_posix()}" |
| 312 | ) | 312 | ) |
| 313 | if "num_classes" in config.model.args: | 313 | if "num_classes" in config.model.args: |
| 314 | declared = config.model.args["num_classes"] | 314 | declared = config.model.args["num_classes"] |
| 315 | if declared != task.num_classes: | 315 | if declared != task.num_classes: |
| 316 | raise config_values.ConfigError( | 316 | raise config_values.ExperimentConfigError( |
| 317 | f"model.args.num_classes {declared!r} must equal " | 317 | f"model.args.num_classes {declared!r} must equal " |
| 318 | f"task.num_classes {task.num_classes}" | 318 | f"task.num_classes {task.num_classes}" |
| 319 | ) | 319 | ) |
| 320 | if "ignore_index" in config.loss.args: | 320 | if "ignore_index" in config.loss.args: |
| 321 | declared = config.loss.args["ignore_index"] | 321 | declared = config.loss.args["ignore_index"] |
| 322 | if declared != task.ignore_index: | 322 | if declared != task.ignore_index: |
| 323 | raise config_values.ConfigError( | 323 | raise config_values.ExperimentConfigError( |
| 324 | f"loss.args.ignore_index {declared!r} must equal " | 324 | f"loss.args.ignore_index {declared!r} must equal " |
| 325 | f"task.ignore_index {task.ignore_index}" | 325 | f"task.ignore_index {task.ignore_index}" |
| 326 | ) | 326 | ) |
| 327 | 327 |
| 352 | ) | 352 | ) |
| 353 | for overrides in override_groups: | 353 | for overrides in override_groups: |
| 354 | for path, value in overrides.items(): | 354 | for path, value in overrides.items(): |
| 355 | if path.startswith("study.") or path not in leaves: | 355 | if path.startswith("study.") or path not in leaves: |
| 356 | raise config_values.ConfigError( | 356 | raise config_values.ExperimentConfigError( |
| 357 | f"Study override path {path!r} is not a declared scalar/list leaf" | 357 | f"Study override path {path!r} is not a declared scalar/list leaf" |
| 358 | ) | 358 | ) |
| 359 | expected = leaves[path] | 359 | expected = leaves[path] |
| 360 | if not config_values.same_leaf_type(expected, value): | 360 | if not config_values.same_leaf_type(expected, value): |
| 361 | raise config_values.ConfigError( | 361 | raise config_values.ExperimentConfigError( |
| 362 | f"Study override {path!r} has incompatible value {value!r}; " | 362 | f"Study override {path!r} has incompatible value {value!r}; " |
| 363 | f"expected type of {expected!r}" | 363 | f"expected type of {expected!r}" |
| 364 | ) | 364 | ) |
| 250 | ) -> Mapping[str, tuple[config_values.OverrideValue, ...]]: | 250 | ) -> Mapping[str, tuple[config_values.OverrideValue, ...]]: |
| 251 | """Reject an axis that declares no value.""" | 251 | """Reject an axis that declares no value.""" |
| 252 | for path, values in value.items(): | 252 | for path, values in value.items(): |
| 253 | if not values: | 253 | if not values: |
| 254 | raise config_values.ConfigError( | 254 | raise config_values.ExperimentConfigError( |
| 255 | f"study.matrix.axes.{path} cannot be empty" | 255 | f"study.matrix.axes.{path} cannot be empty" |
| 256 | ) | 256 | ) |
| 257 | return value | 257 | return value |
| 258 | 258 |
| 268 | @pydantic.model_validator(mode="after") | 268 | @pydantic.model_validator(mode="after") |
| 269 | def _bounds_are_complete(self) -> SweepParameterConfig: | 269 | def _bounds_are_complete(self) -> SweepParameterConfig: |
| 270 | """Reject a parameter that is neither finite nor fully bounded.""" | 270 | """Reject a parameter that is neither finite nor fully bounded.""" |
| 271 | if self.values is not None and not self.values: | 271 | if self.values is not None and not self.values: |
| 272 | raise config_values.ConfigError("values cannot be empty") | 272 | raise config_values.ExperimentConfigError("values cannot be empty") |
| 273 | if self.values is None and ( | 273 | if self.values is None and ( |
| 274 | self.minimum is None or self.maximum is None or self.distribution is None | 274 | self.minimum is None or self.maximum is None or self.distribution is None |
| 275 | ): | 275 | ): |
| 276 | raise config_values.ConfigError( | 276 | raise config_values.ExperimentConfigError( |
| 277 | "requires values or minimum/maximum/distribution" | 277 | "requires values or minimum/maximum/distribution" |
| 278 | ) | 278 | ) |
| 279 | if ( | 279 | if ( |
| 280 | self.minimum is not None | 280 | self.minimum is not None |
| 281 | and self.maximum is not None | 281 | and self.maximum is not None |
| 282 | and self.minimum >= self.maximum | 282 | and self.minimum >= self.maximum |
| 283 | ): | 283 | ): |
| 284 | raise config_values.ConfigError("minimum must be smaller than maximum") | 284 | raise config_values.ExperimentConfigError( |
| 285 | "minimum must be smaller than maximum" | ||
| 286 | ) | ||
| 285 | return self | 287 | return self |
| 286 | 288 | ||
| 287 | 289 | ||
| 288 | class SweepConfig(StrictConfigModel): | 290 | class SweepConfig(StrictConfigModel): |
| 299 | cls, value: Mapping[str, SweepParameterConfig] | 301 | cls, value: Mapping[str, SweepParameterConfig] |
| 300 | ) -> Mapping[str, SweepParameterConfig]: | 302 | ) -> Mapping[str, SweepParameterConfig]: |
| 301 | """Reject a sweep that declares no parameter.""" | 303 | """Reject a sweep that declares no parameter.""" |
| 302 | if not value: | 304 | if not value: |
| 303 | raise config_values.ConfigError("study.sweep.parameters cannot be empty") | 305 | raise config_values.ExperimentConfigError( |
| 306 | "study.sweep.parameters cannot be empty" | ||
| 307 | ) | ||
| 304 | return value | 308 | return value |
| 305 | 309 | ||
| 306 | 310 | ||
| 307 | class ContrastConfig(StrictConfigModel): | 311 | class ContrastConfig(StrictConfigModel): |
| 327 | """Reject a study whose kind and payload disagree.""" | 331 | """Reject a study whose kind and payload disagree.""" |
| 328 | if self.kind == "single" and ( | 332 | if self.kind == "single" and ( |
| 329 | self.variants or self.matrix is not None or self.sweep is not None | 333 | self.variants or self.matrix is not None or self.sweep is not None |
| 330 | ): | 334 | ): |
| 331 | raise config_values.ConfigError( | 335 | raise config_values.ExperimentConfigError( |
| 332 | "study.kind single cannot carry variants, matrix, or sweep" | 336 | "study.kind single cannot carry variants, matrix, or sweep" |
| 333 | ) | 337 | ) |
| 334 | if self.kind == "variants" and ( | 338 | if self.kind == "variants" and ( |
| 335 | not self.variants or self.matrix is not None or self.sweep is not None | 339 | not self.variants or self.matrix is not None or self.sweep is not None |
| 336 | ): | 340 | ): |
| 337 | raise config_values.ConfigError( | 341 | raise config_values.ExperimentConfigError( |
| 338 | "study.kind variants requires only a non-empty variants payload" | 342 | "study.kind variants requires only a non-empty variants payload" |
| 339 | ) | 343 | ) |
| 340 | if self.kind == "matrix" and ( | 344 | if self.kind == "matrix" and ( |
| 341 | self.variants or self.matrix is None or self.sweep is not None | 345 | self.variants or self.matrix is None or self.sweep is not None |
| 342 | ): | 346 | ): |
| 343 | raise config_values.ConfigError("study.kind matrix requires only matrix") | 347 | raise config_values.ExperimentConfigError( |
| 348 | "study.kind matrix requires only matrix" | ||
| 349 | ) | ||
| 344 | if self.kind == "sweep" and ( | 350 | if self.kind == "sweep" and ( |
| 345 | self.variants or self.matrix is not None or self.sweep is None | 351 | self.variants or self.matrix is not None or self.sweep is None |
| 346 | ): | 352 | ): |
| 347 | raise config_values.ConfigError("study.kind sweep requires only sweep") | 353 | raise config_values.ExperimentConfigError( |
| 354 | "study.kind sweep requires only sweep" | ||
| 355 | ) | ||
| 348 | identities = [item.id for item in self.variants] | 356 | identities = [item.id for item in self.variants] |
| 349 | if len(set(identities)) != len(identities): | 357 | if len(set(identities)) != len(identities): |
| 350 | raise config_values.ConfigError("study.variants contains duplicate IDs") | 358 | raise config_values.ExperimentConfigError( |
| 359 | "study.variants contains duplicate IDs" | ||
| 360 | ) | ||
| 351 | return self | 361 | return self |
| 65 | @pydantic.model_validator(mode="after") | 65 | @pydantic.model_validator(mode="after") |
| 66 | def _fuse_config_matches_mode(self) -> LabelSourceConfig: | 66 | def _fuse_config_matches_mode(self) -> LabelSourceConfig: |
| 67 | """Tie the fusion config to the declared label mode.""" | 67 | """Tie the fusion config to the declared label mode.""" |
| 68 | if self.mode == "regenerate_full_resolution" and self.fuse_config is None: | 68 | if self.mode == "regenerate_full_resolution" and self.fuse_config is None: |
| 69 | raise config_values.ConfigError( | 69 | raise config_values.ExperimentConfigError( |
| 70 | "regenerate_full_resolution requires data.label_source.fuse_config" | 70 | "regenerate_full_resolution requires data.label_source.fuse_config" |
| 71 | ) | 71 | ) |
| 72 | if self.mode == "artifact" and self.fuse_config is not None: | 72 | if self.mode == "artifact" and self.fuse_config is not None: |
| 73 | raise config_values.ConfigError( | 73 | raise config_values.ExperimentConfigError( |
| 74 | "artifact label mode requires data.label_source.fuse_config: null" | 74 | "artifact label mode requires data.label_source.fuse_config: null" |
| 75 | ) | 75 | ) |
| 76 | return self | 76 | return self |
| 77 | 77 |
| 104 | @pydantic.model_validator(mode="after") | 104 | @pydantic.model_validator(mode="after") |
| 105 | def _overlap_fits_in_a_tile(self) -> TilingConfig: | 105 | def _overlap_fits_in_a_tile(self) -> TilingConfig: |
| 106 | """Reject an overlap that is not shorter than the tile.""" | 106 | """Reject an overlap that is not shorter than the tile.""" |
| 107 | if self.overlap_m >= self.length_m: | 107 | if self.overlap_m >= self.length_m: |
| 108 | raise config_values.ConfigError("tiling requires 0 <= overlap_m < length_m") | 108 | raise config_values.ExperimentConfigError( |
| 109 | "tiling requires 0 <= overlap_m < length_m" | ||
| 110 | ) | ||
| 109 | return self | 111 | return self |
| 110 | 112 | ||
| 111 | 113 | ||
| 112 | class DataConfig(config_schema.StrictConfigModel): | 114 | class DataConfig(config_schema.StrictConfigModel): |
| 154 | @classmethod | 156 | @classmethod |
| 155 | def _emit_is_supported(cls, value: tuple[str, ...]) -> tuple[str, ...]: | 157 | def _emit_is_supported(cls, value: tuple[str, ...]) -> tuple[str, ...]: |
| 156 | """Reject unknown output formats and a missing canonical emission.""" | 158 | """Reject unknown output formats and a missing canonical emission.""" |
| 157 | if set(value) - {"canonical", "spt", "pointcept"}: | 159 | if set(value) - {"canonical", "spt", "pointcept"}: |
| 158 | raise config_values.ConfigError( | 160 | raise config_values.ExperimentConfigError( |
| 159 | "adapter.emit contains an unsupported output format" | 161 | "adapter.emit contains an unsupported output format" |
| 160 | ) | 162 | ) |
| 161 | if "canonical" not in value: | 163 | if "canonical" not in value: |
| 162 | raise config_values.ConfigError("adapter.emit must include canonical") | 164 | raise config_values.ExperimentConfigError( |
| 165 | "adapter.emit must include canonical" | ||
| 166 | ) | ||
| 163 | return value | 167 | return value |
| 164 | 168 | ||
| 165 | @pydantic.field_validator("canonical_version") | 169 | @pydantic.field_validator("canonical_version") |
| 166 | @classmethod | 170 | @classmethod |
| 167 | def _canonical_version_is_one(cls, value: int) -> int: | 171 | def _canonical_version_is_one(cls, value: int) -> int: |
| 168 | """Freeze the canonical dataset version at 1.""" | 172 | """Freeze the canonical dataset version at 1.""" |
| 169 | if value != 1: | 173 | if value != 1: |
| 170 | raise config_values.ConfigError("adapter.canonical_version must be 1") | 174 | raise config_values.ExperimentConfigError( |
| 175 | "adapter.canonical_version must be 1" | ||
| 176 | ) | ||
| 171 | return value | 177 | return value |
| 172 | 178 | ||
| 173 | 179 | ||
| 174 | class ModelConfig(config_schema.StrictConfigModel): | 180 | class ModelConfig(config_schema.StrictConfigModel): |
| 204 | def _external_models_pin_their_checkout(self) -> ModelConfig: | 210 | def _external_models_pin_their_checkout(self) -> ModelConfig: |
| 205 | """Tie the external checkout variables to the declared framework.""" | 211 | """Tie the external checkout variables to the declared framework.""" |
| 206 | if self.framework == "cpu": | 212 | if self.framework == "cpu": |
| 207 | if self.checkout_env is not None or self.commit_env is not None: | 213 | if self.checkout_env is not None or self.commit_env is not None: |
| 208 | raise config_values.ConfigError( | 214 | raise config_values.ExperimentConfigError( |
| 209 | "CPU experiments cannot declare external checkout variables" | 215 | "CPU experiments cannot declare external checkout variables" |
| 210 | ) | 216 | ) |
| 211 | elif not (self.base_config and self.checkout_env and self.commit_env): | 217 | elif not (self.base_config and self.checkout_env and self.commit_env): |
| 212 | raise config_values.ConfigError( | 218 | raise config_values.ExperimentConfigError( |
| 213 | "External models require base_config, checkout_env, and commit_env" | 219 | "External models require base_config, checkout_env, and commit_env" |
| 214 | ) | 220 | ) |
| 215 | return self | 221 | return self |
| 216 | 222 |
| 300 | self.delta_quality is None | 306 | self.delta_quality is None |
| 301 | or self.delta_fp_per_km is None | 307 | or self.delta_fp_per_km is None |
| 302 | or not self.superiority_conditions | 308 | or not self.superiority_conditions |
| 303 | ): | 309 | ): |
| 304 | raise config_values.ConfigError( | 310 | raise config_values.ExperimentConfigError( |
| 305 | "enabled promotion requires non-null margins and superiority " | 311 | "enabled promotion requires non-null margins and superiority " |
| 306 | "conditions" | 312 | "conditions" |
| 307 | ) | 313 | ) |
| 308 | return self | 314 | return self |
| 325 | def _floors_are_fractions(cls, value: Mapping[str, float]) -> Mapping[str, float]: | 331 | def _floors_are_fractions(cls, value: Mapping[str, float]) -> Mapping[str, float]: |
| 326 | """Reject a precision floor outside the unit interval.""" | 332 | """Reject a precision floor outside the unit interval.""" |
| 327 | for name, floor in value.items(): | 333 | for name, floor in value.items(): |
| 328 | if not 0.0 <= floor <= 1.0: | 334 | if not 0.0 <= floor <= 1.0: |
| 329 | raise config_values.ConfigError( | 335 | raise config_values.ExperimentConfigError( |
| 330 | f"evaluation precision floor for {name} must be in [0, 1]" | 336 | f"evaluation precision floor for {name} must be in [0, 1]" |
| 331 | ) | 337 | ) |
| 332 | return value | 338 | return value |
| 333 | 339 | ||
| 334 | @pydantic.model_validator(mode="after") | 340 | @pydantic.model_validator(mode="after") |
| 335 | def _promotion_test_is_promotable(self) -> EvaluationConfig: | 341 | def _promotion_test_is_promotable(self) -> EvaluationConfig: |
| 336 | """Reject a locked-test split whose promotion protocol is disabled.""" | 342 | """Reject a locked-test split whose promotion protocol is disabled.""" |
| 337 | if self.split == "promotion_test" and not self.promotion.enabled: | 343 | if self.split == "promotion_test" and not self.promotion.enabled: |
| 338 | raise config_values.ConfigError( | 344 | raise config_values.ExperimentConfigError( |
| 339 | "promotion_test evaluation requires evaluation.promotion.enabled" | 345 | "promotion_test evaluation requires evaluation.promotion.enabled" |
| 340 | ) | 346 | ) |
| 341 | return self | 347 | return self |
| 342 | 348 |
| 376 | def _tiles_select_something(self) -> VisualizationConfig: | 382 | def _tiles_select_something(self) -> VisualizationConfig: |
| 377 | """Reject an empty or non-positive tile selection.""" | 383 | """Reject an empty or non-positive tile selection.""" |
| 378 | if isinstance(self.masks_tiles, int): | 384 | if isinstance(self.masks_tiles, int): |
| 379 | if self.masks_tiles < 1: | 385 | if self.masks_tiles < 1: |
| 380 | raise config_values.ConfigError( | 386 | raise config_values.ExperimentConfigError( |
| 381 | "visualization.masks_tiles must be >= 1" | 387 | "visualization.masks_tiles must be >= 1" |
| 382 | ) | 388 | ) |
| 383 | elif not self.masks_tiles: | 389 | elif not self.masks_tiles: |
| 384 | raise config_values.ConfigError("visualization.masks_tiles cannot be empty") | 390 | raise config_values.ExperimentConfigError( |
| 391 | "visualization.masks_tiles cannot be empty" | ||
| 392 | ) | ||
| 385 | return self | 393 | return self |
| 386 | 394 | ||
| 387 | 395 | ||
| 388 | class HarnessConfig(config_schema.StrictConfigModel): | 396 | class HarnessConfig(config_schema.StrictConfigModel): |
| 414 | if isinstance(data, Mapping) and data.get("visualization", False) is None: | 422 | if isinstance(data, Mapping) and data.get("visualization", False) is None: |
| 415 | raise ValueError("visualization must be a mapping") | 423 | raise ValueError("visualization must be a mapping") |
| 416 | return data | 424 | return data |
| 417 | 425 | ||
| 426 | @classmethod | ||
| 427 | def from_yaml(cls, path: str | Path) -> HarnessConfig: | ||
| 428 | """Load and strictly validate one E1--E14 experiment YAML. | ||
| 429 | |||
| 430 | This is the harness entry point every script and test uses; the | ||
| 431 | document read, strict validation, and the cross-section rules of | ||
| 432 | :mod:`src.train.config_rules` all run here. | ||
| 433 | |||
| 434 | Args: | ||
| 435 | path: Repository-relative or absolute YAML path. | ||
| 436 | |||
| 437 | Returns: | ||
| 438 | Fully typed immutable configuration carrying its own SHA-256. | ||
| 439 | |||
| 440 | Raises: | ||
| 441 | ExperimentConfigError: If loading, strict parsing, or validation | ||
| 442 | fails. | ||
| 443 | """ | ||
| 444 | # Deferred: the loading pipeline lives in the entry-point module, which | ||
| 445 | # imports this one for its models. | ||
| 446 | from src.train import config | ||
| 447 | |||
| 448 | return config._load_yaml_document(path) | ||
| 449 | |||
| 418 | @property | 450 | @property |
| 419 | def source_path(self) -> Path: | 451 | def source_path(self) -> Path: |
| 420 | """Path of the YAML document this configuration was parsed from.""" | 452 | """Path of the YAML document this configuration was parsed from.""" |
| 421 | return self._source_path | 453 | return self._source_path |
| 36 | Returns: | 36 | Returns: |
| 37 | The deterministic cell definitions of the declared study kind. | 37 | The deterministic cell definitions of the declared study kind. |
| 38 | 38 | ||
| 39 | Raises: | 39 | Raises: |
| 40 | ConfigError: If the study payload of the declared kind is absent or | 40 | ExperimentConfigError: If the study payload of the declared kind is absent or |
| 41 | expands to nothing. | 41 | expands to nothing. |
| 42 | """ | 42 | """ |
| 43 | study = config.study | 43 | study = config.study |
| 44 | if study.kind == "single": | 44 | if study.kind == "single": |
| 45 | return ((config.experiment.id, MappingProxyType({})),) | 45 | return ((config.experiment.id, MappingProxyType({})),) |
| 46 | if study.kind == "variants": | 46 | if study.kind == "variants": |
| 47 | if not study.variants: | 47 | if not study.variants: |
| 48 | raise config_values.ConfigError( | 48 | raise config_values.ExperimentConfigError( |
| 49 | f"{config.experiment.id} study.kind variants has no variants payload" | 49 | f"{config.experiment.id} study.kind variants has no variants payload" |
| 50 | ) | 50 | ) |
| 51 | return tuple((item.id, item.overrides) for item in study.variants) | 51 | return tuple((item.id, item.overrides) for item in study.variants) |
| 52 | if study.kind == "matrix": | 52 | if study.kind == "matrix": |
| 53 | if study.matrix is None: | 53 | if study.matrix is None: |
| 54 | raise config_values.ConfigError( | 54 | raise config_values.ExperimentConfigError( |
| 55 | f"{config.experiment.id} study.kind matrix has no matrix payload" | 55 | f"{config.experiment.id} study.kind matrix has no matrix payload" |
| 56 | ) | 56 | ) |
| 57 | return matrix_cells(study.matrix) | 57 | return matrix_cells(study.matrix) |
| 58 | if study.sweep is None: | 58 | if study.sweep is None: |
| 59 | raise config_values.ConfigError( | 59 | raise config_values.ExperimentConfigError( |
| 60 | f"{config.experiment.id} study.kind sweep has no sweep payload" | 60 | f"{config.experiment.id} study.kind sweep has no sweep payload" |
| 61 | ) | 61 | ) |
| 62 | return sweep_cells(study.sweep, config.seed) | 62 | return sweep_cells(study.sweep, config.seed) |
| 63 | 63 |
| 68 | unknown = sorted( | 68 | unknown = sorted( |
| 69 | {path for item in matrix.exclude for path in item} - set(axis_paths) | 69 | {path for item in matrix.exclude for path in item} - set(axis_paths) |
| 70 | ) | 70 | ) |
| 71 | if unknown: | 71 | if unknown: |
| 72 | raise config_values.ConfigError( | 72 | raise config_values.ExperimentConfigError( |
| 73 | f"study.matrix.exclude references non-axis paths {unknown}" | 73 | f"study.matrix.exclude references non-axis paths {unknown}" |
| 74 | ) | 74 | ) |
| 75 | definitions: list[CellDefinition] = [] | 75 | definitions: list[CellDefinition] = [] |
| 76 | excluded = [0] * len(matrix.exclude) | 76 | excluded = [0] * len(matrix.exclude) |
| 84 | if not dropped: | 84 | if not dropped: |
| 85 | definitions.append((cell_id(overrides), MappingProxyType(overrides))) | 85 | definitions.append((cell_id(overrides), MappingProxyType(overrides))) |
| 86 | for index, count in enumerate(excluded): | 86 | for index, count in enumerate(excluded): |
| 87 | if not count: | 87 | if not count: |
| 88 | raise config_values.ConfigError( | 88 | raise config_values.ExperimentConfigError( |
| 89 | f"study.matrix.exclude[{index}] matches no matrix cell" | 89 | f"study.matrix.exclude[{index}] matches no matrix cell" |
| 90 | ) | 90 | ) |
| 91 | for index, item in enumerate(matrix.include): | 91 | for index, item in enumerate(matrix.include): |
| 92 | if not item: | 92 | if not item: |
| 93 | raise config_values.ConfigError( | 93 | raise config_values.ExperimentConfigError( |
| 94 | f"study.matrix.include[{index}] cannot be empty" | 94 | f"study.matrix.include[{index}] cannot be empty" |
| 95 | ) | 95 | ) |
| 96 | definitions.append((cell_id(item), item)) | 96 | definitions.append((cell_id(item), item)) |
| 97 | if not definitions: | 97 | if not definitions: |
| 98 | raise config_values.ConfigError("study.matrix excludes every cell") | 98 | raise config_values.ExperimentConfigError("study.matrix excludes every cell") |
| 99 | return tuple(definitions) | 99 | return tuple(definitions) |
| 100 | 100 | ||
| 101 | 101 | ||
| 102 | def sweep_cells( | 102 | def sweep_cells( |
| 108 | unbounded = sorted( | 108 | unbounded = sorted( |
| 109 | path for path, item in sweep.parameters.items() if item.values is None | 109 | path for path, item in sweep.parameters.items() if item.values is None |
| 110 | ) | 110 | ) |
| 111 | if unbounded: | 111 | if unbounded: |
| 112 | raise config_values.ConfigError( | 112 | raise config_values.ExperimentConfigError( |
| 113 | f"study.sweep.method grid requires explicit values for {unbounded}" | 113 | f"study.sweep.method grid requires explicit values for {unbounded}" |
| 114 | ) | 114 | ) |
| 115 | definitions: list[CellDefinition] = [] | 115 | definitions: list[CellDefinition] = [] |
| 116 | for combination in itertools.product( | 116 | for combination in itertools.product( |
| 136 | ), | 136 | ), |
| 137 | ) | 137 | ) |
| 138 | for index in range(sweep.budget) | 138 | for index in range(sweep.budget) |
| 139 | ) | 139 | ) |
| 140 | raise config_values.ConfigError( | 140 | raise config_values.ExperimentConfigError( |
| 141 | f"study.sweep.method {sweep.method!r} is not implemented; supported " | 141 | f"study.sweep.method {sweep.method!r} is not implemented; supported " |
| 142 | "methods are grid and random" | 142 | "methods are grid and random" |
| 143 | ) | 143 | ) |
| 144 | 144 |
| 151 | """Draw one deterministic value for a sweep parameter.""" | 151 | """Draw one deterministic value for a sweep parameter.""" |
| 152 | if parameter.values is not None: | 152 | if parameter.values is not None: |
| 153 | return parameter.values[generator.randrange(len(parameter.values))] | 153 | return parameter.values[generator.randrange(len(parameter.values))] |
| 154 | if parameter.minimum is None or parameter.maximum is None: | 154 | if parameter.minimum is None or parameter.maximum is None: |
| 155 | raise config_values.ConfigError( | 155 | raise config_values.ExperimentConfigError( |
| 156 | f"{where} requires minimum and maximum for sampling" | 156 | f"{where} requires minimum and maximum for sampling" |
| 157 | ) | 157 | ) |
| 158 | if parameter.distribution == "uniform": | 158 | if parameter.distribution == "uniform": |
| 159 | drawn = generator.uniform(parameter.minimum, parameter.maximum) | 159 | drawn = generator.uniform(parameter.minimum, parameter.maximum) |
| 160 | elif parameter.distribution == "log_uniform": | 160 | elif parameter.distribution == "log_uniform": |
| 161 | if parameter.minimum <= 0.0: | 161 | if parameter.minimum <= 0.0: |
| 162 | raise config_values.ConfigError( | 162 | raise config_values.ExperimentConfigError( |
| 163 | f"{where}.minimum must be positive for log_uniform" | 163 | f"{where}.minimum must be positive for log_uniform" |
| 164 | ) | 164 | ) |
| 165 | drawn = math.exp( | 165 | drawn = math.exp( |
| 166 | generator.uniform(math.log(parameter.minimum), math.log(parameter.maximum)) | 166 | generator.uniform(math.log(parameter.minimum), math.log(parameter.maximum)) |
| 167 | ) | 167 | ) |
| 168 | else: | 168 | else: |
| 169 | raise config_values.ConfigError( | 169 | raise config_values.ExperimentConfigError( |
| 170 | f"{where}.distribution {parameter.distribution!r} is not implemented; " | 170 | f"{where}.distribution {parameter.distribution!r} is not implemented; " |
| 171 | "supported distributions are uniform and log_uniform" | 171 | "supported distributions are uniform and log_uniform" |
| 172 | ) | 172 | ) |
| 173 | return float(f"{drawn:.6g}") | 173 | return float(f"{drawn:.6g}") |
| 175 | 175 | ||
| 176 | def cell_id(overrides: Mapping[str, config_values.OverrideValue]) -> str: | 176 | def cell_id(overrides: Mapping[str, config_values.OverrideValue]) -> str: |
| 177 | """Derive a stable, filesystem-safe identity from a cell's overrides.""" | 177 | """Derive a stable, filesystem-safe identity from a cell's overrides.""" |
| 178 | if not overrides: | 178 | if not overrides: |
| 179 | raise config_values.ConfigError("A study cell requires at least one override") | 179 | raise config_values.ExperimentConfigError( |
| 180 | "A study cell requires at least one override" | ||
| 181 | ) | ||
| 180 | names = [path.rsplit(".", 1)[-1] for path in overrides] | 182 | names = [path.rsplit(".", 1)[-1] for path in overrides] |
| 181 | if len(set(names)) != len(names): | 183 | if len(set(names)) != len(names): |
| 182 | names = [path.replace(".", "_") for path in overrides] | 184 | names = [path.replace(".", "_") for path in overrides] |
| 183 | return "__".join( | 185 | return "__".join( |
| 216 | Returns: | 218 | Returns: |
| 217 | A deep copy of ``raw`` carrying the overridden leaves. | 219 | A deep copy of ``raw`` carrying the overridden leaves. |
| 218 | 220 | ||
| 219 | Raises: | 221 | Raises: |
| 220 | ConfigError: If a path is not a declared leaf or the value type differs. | 222 | ExperimentConfigError: If a path is not a declared leaf or the value |
| 223 | type differs. | ||
| 221 | """ | 224 | """ |
| 222 | result = copy.deepcopy(dict(raw)) | 225 | result = copy.deepcopy(dict(raw)) |
| 223 | leaves = config_values.leaf_values(raw) | 226 | leaves = config_values.leaf_values(raw) |
| 224 | for path in sorted(overrides): | 227 | for path in sorted(overrides): |
| 225 | value = overrides[path] | 228 | value = overrides[path] |
| 226 | if path.startswith("study.") or path not in leaves: | 229 | if path.startswith("study.") or path not in leaves: |
| 227 | raise config_values.ConfigError( | 230 | raise config_values.ExperimentConfigError( |
| 228 | f"{where} override path {path!r} is not a declared scalar/list leaf" | 231 | f"{where} override path {path!r} is not a declared scalar/list leaf" |
| 229 | ) | 232 | ) |
| 230 | expected = leaves[path] | 233 | expected = leaves[path] |
| 231 | if not config_values.same_leaf_type(expected, value): | 234 | if not config_values.same_leaf_type(expected, value): |
| 232 | raise config_values.ConfigError( | 235 | raise config_values.ExperimentConfigError( |
| 233 | f"{where} override {path!r} has incompatible value {value!r}; " | 236 | f"{where} override {path!r} has incompatible value {value!r}; " |
| 234 | f"expected type of {expected!r}" | 237 | f"expected type of {expected!r}" |
| 235 | ) | 238 | ) |
| 236 | _set_leaf(result, path, config_values.coerce_leaf(expected, value), where) | 239 | _set_leaf(result, path, config_values.coerce_leaf(expected, value), where) |
| 247 | segments = path.split(".") | 250 | segments = path.split(".") |
| 248 | node: Any = target | 251 | node: Any = target |
| 249 | for segment in segments[:-1]: | 252 | for segment in segments[:-1]: |
| 250 | if not isinstance(node, dict) or segment not in node: | 253 | if not isinstance(node, dict) or segment not in node: |
| 251 | raise config_values.ConfigError( | 254 | raise config_values.ExperimentConfigError( |
| 252 | f"{where} override path {path!r} is not addressable" | 255 | f"{where} override path {path!r} is not addressable" |
| 253 | ) | 256 | ) |
| 254 | node = node[segment] | 257 | node = node[segment] |
| 255 | if not isinstance(node, dict) or segments[-1] not in node: | 258 | if not isinstance(node, dict) or segments[-1] not in node: |
| 256 | raise config_values.ConfigError( | 259 | raise config_values.ExperimentConfigError( |
| 257 | f"{where} override path {path!r} is not addressable" | 260 | f"{where} override path {path!r} is not addressable" |
| 258 | ) | 261 | ) |
| 259 | node[segments[-1]] = value | 262 | node[segments[-1]] = value |
| 1 | """Strict YAML value typing shared by the experiment configuration models. | 1 | """Strict YAML value typing shared by the experiment configuration models. |
| 2 | 2 | ||
| 3 | The experiment YAML is a fail-closed contract: a value must already carry its | 3 | The experiment YAML is a fail-closed contract: a value must already carry its |
| 4 | declared type, so ``"50"`` is not a float, ``1`` is not a boolean, and ``3.0`` | 4 | declared type, so ``"50"`` is not a float, ``1`` is not a boolean, and ``3.0`` |
| 5 | is not an integer. That is deliberately stricter than the fleet coercion | 5 | is not an integer. That is deliberately stricter than the fleet coercion matrix |
| 6 | matrix of :func:`iolabs.common.config_loader.coerce_config_value`, so the | 6 | of :func:`iolabs.common.config_loader.coerce_to_field_type`, and it is the one |
| 7 | models in :mod:`src.train.config_schema` route every field through | 7 | sanctioned opt-out from it: the models in :mod:`src.train.config_schema` route |
| 8 | :func:`typed_value` instead of the inherited coercion. | 8 | every field through :func:`typed_value` instead of the inherited coercion. Do |
| 9 | not copy this into a packaged pipeline config -- it holds only for this | ||
| 10 | YAML experiment contract. | ||
| 9 | """ | 11 | """ |
| 10 | 12 | ||
| 11 | from __future__ import annotations | 13 | from __future__ import annotations |
| 12 | 14 |
| 23 | Scalar: TypeAlias = str | int | float | bool | None | 25 | Scalar: TypeAlias = str | int | float | bool | None |
| 24 | OverrideValue: TypeAlias = Scalar | list[Scalar] | 26 | OverrideValue: TypeAlias = Scalar | list[Scalar] |
| 25 | 27 | ||
| 26 | 28 | ||
| 27 | class ConfigError(config_loader.ConfigError): | 29 | class ExperimentConfigError(config_loader.ConfigError): |
| 28 | """Raised when an experiment configuration violates its strict schema.""" | 30 | """Raised when experiment config contains unsupported keys or values.""" |
| 29 | 31 | ||
| 30 | 32 | ||
| 31 | def freeze_value(value: Any) -> Any: | 33 | def freeze_value(value: Any) -> Any: |
| 32 | """Return *value* with every nested mapping wrapped read-only. | 34 | """Return *value* with every nested mapping wrapped read-only. |
| 89 | Returns: | 91 | Returns: |
| 90 | The value converted to the declared type. | 92 | The value converted to the declared type. |
| 91 | 93 | ||
| 92 | Raises: | 94 | Raises: |
| 93 | ConfigError: If the value does not match the declared type. | 95 | ExperimentConfigError: If the value does not match the declared type. |
| 94 | """ | 96 | """ |
| 95 | if annotation is None or annotation is Any: | 97 | if annotation is None or annotation is Any: |
| 96 | return value | 98 | return value |
| 97 | origin = get_origin(annotation) | 99 | origin = get_origin(annotation) |
| 135 | optional = type(None) in members | 137 | optional = type(None) in members |
| 136 | if value is None: | 138 | if value is None: |
| 137 | if optional: | 139 | if optional: |
| 138 | return None | 140 | return None |
| 139 | raise ConfigError(f"{where} must not be null") | 141 | raise ExperimentConfigError(f"{where} must not be null") |
| 140 | candidates = [item for item in members if item is not type(None)] | 142 | candidates = [item for item in members if item is not type(None)] |
| 141 | if len(candidates) == 1: | 143 | if len(candidates) == 1: |
| 142 | return typed_value(candidates[0], value, where) | 144 | return typed_value(candidates[0], value, where) |
| 143 | for candidate in candidates: | 145 | for candidate in candidates: |
| 144 | try: | 146 | try: |
| 145 | return typed_value(candidate, value, where) | 147 | return typed_value(candidate, value, where) |
| 146 | except ConfigError: | 148 | except ExperimentConfigError: |
| 147 | continue | 149 | continue |
| 148 | names = sorted(getattr(item, "__name__", str(item)) for item in candidates) | 150 | names = sorted(getattr(item, "__name__", str(item)) for item in candidates) |
| 149 | raise ConfigError( | 151 | raise ExperimentConfigError( |
| 150 | f"{where} must be one of {names}, got {type(value).__name__} {value!r}" | 152 | f"{where} must be one of {names}, got {type(value).__name__} {value!r}" |
| 151 | ) | 153 | ) |
| 152 | 154 | ||
| 153 | 155 | ||
| 154 | def mapping(value: Any, where: str) -> Mapping[str, Any]: | 156 | def mapping(value: Any, where: str) -> Mapping[str, Any]: |
| 155 | """Return *value* as a string-keyed mapping or raise.""" | 157 | """Return *value* as a string-keyed mapping or raise.""" |
| 156 | if not isinstance(value, Mapping): | 158 | if not isinstance(value, Mapping): |
| 157 | raise ConfigError(f"{where} must be a mapping") | 159 | raise ExperimentConfigError(f"{where} must be a mapping") |
| 158 | if any(not isinstance(key, str) for key in value): | 160 | if any(not isinstance(key, str) for key in value): |
| 159 | raise ConfigError(f"{where} keys must be strings") | 161 | raise ExperimentConfigError(f"{where} keys must be strings") |
| 160 | return value | 162 | return value |
| 161 | 163 | ||
| 162 | 164 | ||
| 163 | def sequence(value: Any, where: str) -> Sequence[Any]: | 165 | def sequence(value: Any, where: str) -> Sequence[Any]: |
| 164 | """Return *value* as a non-string sequence or raise.""" | 166 | """Return *value* as a non-string sequence or raise.""" |
| 165 | if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): | 167 | if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): |
| 166 | raise ConfigError(f"{where} must be a sequence") | 168 | raise ExperimentConfigError(f"{where} must be a sequence") |
| 167 | return value | 169 | return value |
| 168 | 170 | ||
| 169 | 171 | ||
| 170 | def string(value: Any, where: str) -> str: | 172 | def string(value: Any, where: str) -> str: |
| 171 | """Return *value* as a non-empty string or raise.""" | 173 | """Return *value* as a non-empty string or raise.""" |
| 172 | if not isinstance(value, str) or not value.strip(): | 174 | if not isinstance(value, str) or not value.strip(): |
| 173 | raise ConfigError(f"{where} must be a non-empty string") | 175 | raise ExperimentConfigError(f"{where} must be a non-empty string") |
| 174 | return value | 176 | return value |
| 175 | 177 | ||
| 176 | 178 | ||
| 177 | def integer(value: Any, where: str) -> int: | 179 | def integer(value: Any, where: str) -> int: |
| 178 | """Return *value* as an integer, rejecting booleans, or raise.""" | 180 | """Return *value* as an integer, rejecting booleans, or raise.""" |
| 179 | if isinstance(value, bool) or not isinstance(value, int): | 181 | if isinstance(value, bool) or not isinstance(value, int): |
| 180 | raise ConfigError(f"{where} must be an integer") | 182 | raise ExperimentConfigError(f"{where} must be an integer") |
| 181 | return value | 183 | return value |
| 182 | 184 | ||
| 183 | 185 | ||
| 184 | def number(value: Any, where: str) -> float: | 186 | def number(value: Any, where: str) -> float: |
| 185 | """Return *value* as a float, rejecting booleans, or raise.""" | 187 | """Return *value* as a float, rejecting booleans, or raise.""" |
| 186 | if isinstance(value, bool) or not isinstance(value, (int, float)): | 188 | if isinstance(value, bool) or not isinstance(value, (int, float)): |
| 187 | raise ConfigError(f"{where} must be numeric") | 189 | raise ExperimentConfigError(f"{where} must be numeric") |
| 188 | return float(value) | 190 | return float(value) |
| 189 | 191 | ||
| 190 | 192 | ||
| 191 | def boolean(value: Any, where: str) -> bool: | 193 | def boolean(value: Any, where: str) -> bool: |
| 192 | """Return *value* as a boolean or raise.""" | 194 | """Return *value* as a boolean or raise.""" |
| 193 | if not isinstance(value, bool): | 195 | if not isinstance(value, bool): |
| 194 | raise ConfigError(f"{where} must be boolean") | 196 | raise ExperimentConfigError(f"{where} must be boolean") |
| 195 | return value | 197 | return value |
| 196 | 198 | ||
| 197 | 199 | ||
| 198 | def path(value: Any, where: str) -> Path: | 200 | def path(value: Any, where: str) -> Path: |
| 199 | """Return *value* as a repository-relative path or raise.""" | 201 | """Return *value* as a repository-relative path or raise.""" |
| 200 | text = string(value, where) | 202 | text = string(value, where) |
| 201 | parsed = Path(text) | 203 | parsed = Path(text) |
| 202 | if parsed.is_absolute(): | 204 | if parsed.is_absolute(): |
| 203 | raise ConfigError(f"{where} must be repository-relative, got {parsed}") | 205 | raise ExperimentConfigError( |
| 206 | f"{where} must be repository-relative, got {parsed}" | ||
| 207 | ) | ||
| 204 | return parsed | 208 | return parsed |
| 205 | 209 | ||
| 206 | 210 | ||
| 207 | def choice(value: Any, choices: set[str], where: str) -> str: | 211 | def choice(value: Any, choices: set[str], where: str) -> str: |
| 208 | """Return *value* when it is one of the allowed string choices.""" | 212 | """Return *value* when it is one of the allowed string choices.""" |
| 209 | text = string(value, where) | 213 | text = string(value, where) |
| 210 | if text not in choices: | 214 | if text not in choices: |
| 211 | raise ConfigError(f"{where} must be one of {sorted(choices)}, got {text!r}") | 215 | raise ExperimentConfigError( |
| 216 | f"{where} must be one of {sorted(choices)}, got {text!r}" | ||
| 217 | ) | ||
| 212 | return text | 218 | return text |
| 213 | 219 | ||
| 214 | 220 | ||
| 215 | def check_keys( | 221 | def check_keys( |
| 226 | allowed: Keys that may be present. | 232 | allowed: Keys that may be present. |
| 227 | where: Dotted key path used in error messages. | 233 | where: Dotted key path used in error messages. |
| 228 | 234 | ||
| 229 | Raises: | 235 | Raises: |
| 230 | ConfigError: If a key is missing or unknown. | 236 | ExperimentConfigError: If a key is missing or unknown. |
| 231 | """ | 237 | """ |
| 232 | missing = sorted(required - set(value)) | 238 | missing = sorted(required - set(value)) |
| 233 | unknown = sorted(set(value) - allowed) | 239 | unknown = sorted(set(value) - allowed) |
| 234 | if missing or unknown: | 240 | if missing or unknown: |
| 236 | if missing: | 242 | if missing: |
| 237 | details.append(f"missing {missing}") | 243 | details.append(f"missing {missing}") |
| 238 | if unknown: | 244 | if unknown: |
| 239 | details.append(f"unknown {unknown}") | 245 | details.append(f"unknown {unknown}") |
| 240 | raise ConfigError(f"{where} has " + " and ".join(details)) | 246 | raise ExperimentConfigError(f"{where} has " + " and ".join(details)) |
| 241 | 247 | ||
| 242 | 248 | ||
| 243 | def leaf_values(value: Any, prefix: str = "") -> dict[str, OverrideValue]: | 249 | def leaf_values(value: Any, prefix: str = "") -> dict[str, OverrideValue]: |
| 244 | """Return every dotted scalar/list leaf of a raw configuration mapping. | 250 | """Return every dotted scalar/list leaf of a raw configuration mapping. |
| 14 | 14 | ||
| 15 | from src.train.config import ( | 15 | from src.train.config import ( |
| 16 | ArtifactExistsParams, | 16 | ArtifactExistsParams, |
| 17 | CheckpointPolicyParams, | 17 | CheckpointPolicyParams, |
| 18 | ConfigError, | ||
| 19 | DataAvailableParams, | 18 | DataAvailableParams, |
| 19 | ExperimentConfigError, | ||
| 20 | GateConfig, | 20 | GateConfig, |
| 21 | HarnessConfig, | 21 | HarnessConfig, |
| 22 | HumanWorkflowParams, | 22 | HumanWorkflowParams, |
| 23 | ImplementationTicketParams, | 23 | ImplementationTicketParams, |
| 105 | DispatchError: If the study definition cannot be expanded strictly. | 105 | DispatchError: If the study definition cannot be expanded strictly. |
| 106 | """ | 106 | """ |
| 107 | try: | 107 | try: |
| 108 | return expand_study(config) | 108 | return expand_study(config) |
| 109 | except ConfigError as exc: | 109 | except ExperimentConfigError as exc: |
| 110 | raise DispatchError( | 110 | raise DispatchError( |
| 111 | f"{config.experiment.id} study expansion failed: {exc}" | 111 | f"{config.experiment.id} study expansion failed: {exc}" |
| 112 | ) from exc | 112 | ) from exc |
| 113 | 113 |
| 5 | from pathlib import Path | 5 | from pathlib import Path |
| 6 | 6 | ||
| 7 | from src.contracts.ontology import load_ontology | 7 | from src.contracts.ontology import load_ontology |
| 8 | from src.contracts.splits import SplitTier, load_split_manifest | 8 | from src.contracts.splits import SplitTier, load_split_manifest |
| 9 | from src.train.config import load_config | 9 | from src.train.config import HarnessConfig |
| 10 | 10 | ||
| 11 | CONFIG_PATH = Path("configs/dev/a1_recap_segment_085.yaml") | 11 | CONFIG_PATH = Path("configs/dev/a1_recap_segment_085.yaml") |
| 12 | ONTOLOGY_PATH = Path("configs/contracts/ontology_v2.yaml") | 12 | ONTOLOGY_PATH = Path("configs/contracts/ontology_v2.yaml") |
| 13 | SPLIT_PATH = Path("configs/contracts/corridor_splits_a1_recap_v1.yaml") | 13 | SPLIT_PATH = Path("configs/contracts/corridor_splits_a1_recap_v1.yaml") |
| 30 | assert manifest.supported_interest_ids[SplitTier.TRAIN] == tuple(range(9)) | 30 | assert manifest.supported_interest_ids[SplitTier.TRAIN] == tuple(range(9)) |
| 31 | 31 | ||
| 32 | 32 | ||
| 33 | def test_recap_config_points_at_the_recap_lanes_with_rgb_and_intensity() -> None: | 33 | def test_recap_config_points_at_the_recap_lanes_with_rgb_and_intensity() -> None: |
| 34 | config = load_config(CONFIG_PATH) | 34 | config = HarnessConfig.from_yaml(CONFIG_PATH) |
| 35 | 35 | ||
| 36 | assert config.data.split_manifest == SPLIT_PATH | 36 | assert config.data.split_manifest == SPLIT_PATH |
| 37 | assert config.data.root == Path("data/00_external/a1_recap_v1") | 37 | assert config.data.root == Path("data/00_external/a1_recap_v1") |
| 38 | assert config.data.corridors.include == (CORRIDOR_ID,) | 38 | assert config.data.corridors.include == (CORRIDOR_ID,) |
| 41 | SplitTier, | 41 | SplitTier, |
| 42 | authorize_split_access, | 42 | authorize_split_access, |
| 43 | load_split_manifest, | 43 | load_split_manifest, |
| 44 | ) | 44 | ) |
| 45 | from src.train.config import HarnessConfig, load_config | 45 | from src.train.config import HarnessConfig |
| 46 | 46 | ||
| 47 | CONFIG_PATH = Path("configs/e01_spt_pilot.yaml") | 47 | CONFIG_PATH = Path("configs/e01_spt_pilot.yaml") |
| 48 | POLICY_PATH = Path("configs/contracts/checkpoint_policy.yaml") | 48 | POLICY_PATH = Path("configs/contracts/checkpoint_policy.yaml") |
| 49 | GRID_ORIGIN = [10.0, -20.0, 5.0] | 49 | GRID_ORIGIN = [10.0, -20.0, 5.0] |
| 145 | 145 | ||
| 146 | Returns: | 146 | Returns: |
| 147 | The E1 configuration with a writable canonical root. | 147 | The E1 configuration with a writable canonical root. |
| 148 | """ | 148 | """ |
| 149 | config = load_config(CONFIG_PATH) | 149 | config = HarnessConfig.from_yaml(CONFIG_PATH) |
| 150 | canonical_root = tmp_path / "canonical_root" | 150 | canonical_root = tmp_path / "canonical_root" |
| 151 | (canonical_root / "canonical").mkdir(parents=True, exist_ok=True) | 151 | (canonical_root / "canonical").mkdir(parents=True, exist_ok=True) |
| 152 | for corridor_id in config.data.corridors.include: | 152 | for corridor_id in config.data.corridors.include: |
| 153 | (canonical_root / "canonical" / f"{corridor_id}.adapter.json").write_text( | 153 | (canonical_root / "canonical" / f"{corridor_id}.adapter.json").write_text( |
| 6 | from typing import Any | 6 | from typing import Any |
| 7 | 7 | ||
| 8 | import pytest | 8 | import pytest |
| 9 | import yaml | 9 | import yaml |
| 10 | from iolabs.common import config_loader | ||
| 10 | 11 | ||
| 11 | from src.contracts.ontology import load_ontology | 12 | from src.contracts.ontology import load_ontology |
| 12 | from src.train.config import ( | 13 | from src.train.config import ( |
| 13 | ConfigError, | 14 | ExperimentConfigError, |
| 15 | HarnessConfig, | ||
| 14 | is_canonical_metric, | 16 | is_canonical_metric, |
| 15 | load_config, | ||
| 16 | resolve_ontology_path, | 17 | resolve_ontology_path, |
| 17 | ) | 18 | ) |
| 18 | from src.train.dispatch import DispatchError, dispatch_experiment | 19 | from src.train.dispatch import DispatchError, dispatch_experiment |
| 19 | 20 |
| 24 | 25 | ||
| 25 | 26 | ||
| 26 | def test_all_fourteen_configs_parse_with_frozen_statuses() -> None: | 27 | def test_all_fourteen_configs_parse_with_frozen_statuses() -> None: |
| 27 | """Every experiment parses and matches the section 6 status matrix.""" | 28 | """Every experiment parses and matches the section 6 status matrix.""" |
| 28 | configs = [load_config(path) for path in CONFIGS] | 29 | configs = [HarnessConfig.from_yaml(path) for path in CONFIGS] |
| 29 | assert [item.experiment.id for item in configs] == [ | 30 | assert [item.experiment.id for item in configs] == [ |
| 30 | f"E{index}" for index in range(1, 15) | 31 | f"E{index}" for index in range(1, 15) |
| 31 | ] | 32 | ] |
| 32 | assert [item.experiment.status for item in configs] == [ | 33 | assert [item.experiment.status for item in configs] == [ |
| 52 | "sweep", | 53 | "sweep", |
| 53 | } | 54 | } |
| 54 | 55 | ||
| 55 | 56 | ||
| 57 | def test_error_class_is_config_error() -> None: | ||
| 58 | """The package error class is a fleet config error and a ValueError.""" | ||
| 59 | assert issubclass(ExperimentConfigError, config_loader.ConfigError) | ||
| 60 | assert issubclass(ExperimentConfigError, ValueError) | ||
| 61 | |||
| 62 | |||
| 63 | def test_unknown_top_level_key_is_rejected(tmp_path: Path) -> None: | ||
| 64 | """Unknown keys fail at the document root.""" | ||
| 65 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) | ||
| 66 | raw["surprise_section"] = {"enabled": True} | ||
| 67 | path = tmp_path / "unknown_root.yaml" | ||
| 68 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | ||
| 69 | with pytest.raises(ExperimentConfigError, match="surprise_section"): | ||
| 70 | HarnessConfig.from_yaml(path) | ||
| 71 | |||
| 72 | |||
| 56 | def test_unknown_nested_key_is_rejected(tmp_path: Path) -> None: | 73 | def test_unknown_nested_key_is_rejected(tmp_path: Path) -> None: |
| 57 | """Unknown keys fail at nested section boundaries.""" | 74 | """Unknown keys fail at nested section boundaries.""" |
| 58 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) | 75 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) |
| 59 | raw["train"]["surprise_callback"] = True | 76 | raw["train"]["surprise_callback"] = True |
| 60 | path = tmp_path / "unknown.yaml" | 77 | path = tmp_path / "unknown.yaml" |
| 61 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 78 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 62 | with pytest.raises(ConfigError, match="surprise_callback"): | 79 | with pytest.raises(ExperimentConfigError, match="surprise_callback"): |
| 63 | load_config(path) | 80 | HarnessConfig.from_yaml(path) |
| 64 | 81 | ||
| 65 | 82 | ||
| 66 | def test_incompatible_study_override_is_rejected(tmp_path: Path) -> None: | 83 | def test_incompatible_study_override_is_rejected(tmp_path: Path) -> None: |
| 67 | """Dotted overrides must name an existing leaf and preserve its type.""" | 84 | """Dotted overrides must name an existing leaf and preserve its type.""" |
| 70 | "adapter.spt.voxel_m": "two centimetres" | 87 | "adapter.spt.voxel_m": "two centimetres" |
| 71 | } | 88 | } |
| 72 | path = tmp_path / "bad_override.yaml" | 89 | path = tmp_path / "bad_override.yaml" |
| 73 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 90 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 74 | with pytest.raises(ConfigError, match="incompatible value"): | 91 | with pytest.raises(ExperimentConfigError, match="incompatible value"): |
| 75 | load_config(path) | 92 | HarnessConfig.from_yaml(path) |
| 76 | 93 | ||
| 77 | 94 | ||
| 78 | def test_external_framework_visualizer_fields_are_forbidden(tmp_path: Path) -> None: | 95 | def test_external_framework_visualizer_fields_are_forbidden(tmp_path: Path) -> None: |
| 79 | """SPT/Pointcept YAML cannot configure image-loop visualization callbacks.""" | 96 | """SPT/Pointcept YAML cannot configure image-loop visualization callbacks.""" |
| 80 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) | 97 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) |
| 81 | raw["train"]["viz_samples"] = 4 | 98 | raw["train"]["viz_samples"] = 4 |
| 82 | path = tmp_path / "external_viz.yaml" | 99 | path = tmp_path / "external_viz.yaml" |
| 83 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 100 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 84 | with pytest.raises(ConfigError, match="forbidden"): | 101 | with pytest.raises(ExperimentConfigError, match="forbidden"): |
| 85 | load_config(path) | 102 | HarnessConfig.from_yaml(path) |
| 86 | 103 | ||
| 87 | 104 | ||
| 88 | def test_visualization_block_parses_int_and_named_tiles(tmp_path: Path) -> None: | 105 | def test_visualization_block_parses_int_and_named_tiles(tmp_path: Path) -> None: |
| 89 | """A valid visualization block is optional and strictly typed.""" | 106 | """A valid visualization block is optional and strictly typed.""" |
| 90 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) | 107 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) |
| 91 | raw["visualization"] = {"masks_every_n_epochs": 2, "masks_tiles": 3} | 108 | raw["visualization"] = {"masks_every_n_epochs": 2, "masks_tiles": 3} |
| 92 | path = tmp_path / "viz.yaml" | 109 | path = tmp_path / "viz.yaml" |
| 93 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 110 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 94 | config = load_config(path) | 111 | config = HarnessConfig.from_yaml(path) |
| 95 | assert config.visualization is not None | 112 | assert config.visualization is not None |
| 96 | assert config.visualization.masks_every_n_epochs == 2 | 113 | assert config.visualization.masks_every_n_epochs == 2 |
| 97 | assert config.visualization.masks_tiles == 3 | 114 | assert config.visualization.masks_tiles == 3 |
| 98 | 115 |
| 100 | "masks_every_n_epochs": 1, | 117 | "masks_every_n_epochs": 1, |
| 101 | "masks_tiles": ["tile_a", "tile_b"], | 118 | "masks_tiles": ["tile_a", "tile_b"], |
| 102 | } | 119 | } |
| 103 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 120 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 104 | named = load_config(path) | 121 | named = HarnessConfig.from_yaml(path) |
| 105 | assert named.visualization is not None | 122 | assert named.visualization is not None |
| 106 | assert named.visualization.masks_tiles == ("tile_a", "tile_b") | 123 | assert named.visualization.masks_tiles == ("tile_a", "tile_b") |
| 107 | 124 | ||
| 108 | raw["visualization"] = {"masks_every_n_epochs": 4} | 125 | raw["visualization"] = {"masks_every_n_epochs": 4} |
| 109 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 126 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 110 | defaulted = load_config(path) | 127 | defaulted = HarnessConfig.from_yaml(path) |
| 111 | assert defaulted.visualization is not None | 128 | assert defaulted.visualization is not None |
| 112 | assert defaulted.visualization.masks_tiles == 2 | 129 | assert defaulted.visualization.masks_tiles == 2 |
| 113 | 130 | ||
| 114 | 131 |
| 121 | "surprise": True, | 138 | "surprise": True, |
| 122 | } | 139 | } |
| 123 | path = tmp_path / "viz_unknown.yaml" | 140 | path = tmp_path / "viz_unknown.yaml" |
| 124 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 141 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 125 | with pytest.raises(ConfigError, match="visualization.*surprise"): | 142 | with pytest.raises(ExperimentConfigError, match="visualization.*surprise"): |
| 126 | load_config(path) | 143 | HarnessConfig.from_yaml(path) |
| 127 | 144 | ||
| 128 | 145 | ||
| 129 | def test_visualization_every_n_epochs_zero_is_rejected(tmp_path: Path) -> None: | 146 | def test_visualization_every_n_epochs_zero_is_rejected(tmp_path: Path) -> None: |
| 130 | """masks_every_n_epochs must be >= 1 when the block is present.""" | 147 | """masks_every_n_epochs must be >= 1 when the block is present.""" |
| 131 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) | 148 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) |
| 132 | raw["visualization"] = {"masks_every_n_epochs": 0, "masks_tiles": 2} | 149 | raw["visualization"] = {"masks_every_n_epochs": 0, "masks_tiles": 2} |
| 133 | path = tmp_path / "viz_zero.yaml" | 150 | path = tmp_path / "viz_zero.yaml" |
| 134 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 151 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 135 | with pytest.raises(ConfigError, match="masks_every_n_epochs"): | 152 | with pytest.raises(ExperimentConfigError, match="masks_every_n_epochs"): |
| 136 | load_config(path) | 153 | HarnessConfig.from_yaml(path) |
| 137 | 154 | ||
| 138 | 155 | ||
| 139 | def test_visualization_empty_tiles_are_rejected(tmp_path: Path) -> None: | 156 | def test_visualization_empty_tiles_are_rejected(tmp_path: Path) -> None: |
| 140 | """An empty tile list is not a valid selection.""" | 157 | """An empty tile list is not a valid selection.""" |
| 141 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) | 158 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) |
| 142 | raw["visualization"] = {"masks_every_n_epochs": 1, "masks_tiles": []} | 159 | raw["visualization"] = {"masks_every_n_epochs": 1, "masks_tiles": []} |
| 143 | path = tmp_path / "viz_empty.yaml" | 160 | path = tmp_path / "viz_empty.yaml" |
| 144 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 161 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 145 | with pytest.raises(ConfigError, match="masks_tiles"): | 162 | with pytest.raises(ExperimentConfigError, match="masks_tiles"): |
| 146 | load_config(path) | 163 | HarnessConfig.from_yaml(path) |
| 147 | 164 | ||
| 148 | 165 | ||
| 149 | def test_absent_visualization_block_is_none() -> None: | 166 | def test_absent_visualization_block_is_none() -> None: |
| 150 | """Omitting visualization leaves the feature off.""" | 167 | """Omitting visualization leaves the feature off.""" |
| 151 | config = load_config(CONFIGS[1]) | 168 | config = HarnessConfig.from_yaml(CONFIGS[1]) |
| 152 | assert config.visualization is None | 169 | assert config.visualization is None |
| 153 | 170 | ||
| 154 | 171 | ||
| 155 | def test_visualization_requires_pointcept_framework(tmp_path: Path) -> None: | 172 | def test_visualization_requires_pointcept_framework(tmp_path: Path) -> None: |
| 158 | raw = yaml.safe_load(CONFIGS[index].read_text(encoding="utf-8")) | 175 | raw = yaml.safe_load(CONFIGS[index].read_text(encoding="utf-8")) |
| 159 | raw["visualization"] = {"masks_every_n_epochs": 2} | 176 | raw["visualization"] = {"masks_every_n_epochs": 2} |
| 160 | path = tmp_path / f"viz_wrong_framework_{index}.yaml" | 177 | path = tmp_path / f"viz_wrong_framework_{index}.yaml" |
| 161 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 178 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 162 | with pytest.raises(ConfigError, match="only supported for.*pointcept"): | 179 | with pytest.raises( |
| 163 | load_config(path) | 180 | ExperimentConfigError, match="only supported for.*pointcept" |
| 181 | ): | ||
| 182 | HarnessConfig.from_yaml(path) | ||
| 164 | 183 | ||
| 165 | 184 | ||
| 166 | def test_noncanonical_deciding_metric_is_rejected(tmp_path: Path) -> None: | 185 | def test_noncanonical_deciding_metric_is_rejected(tmp_path: Path) -> None: |
| 167 | """Decision metrics cannot silently depend on an unlogged tag.""" | 186 | """Decision metrics cannot silently depend on an unlogged tag.""" |
| 168 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) | 187 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) |
| 169 | raw["experiment"]["deciding_metrics"] = ["val/mystery"] | 188 | raw["experiment"]["deciding_metrics"] = ["val/mystery"] |
| 170 | path = tmp_path / "metric.yaml" | 189 | path = tmp_path / "metric.yaml" |
| 171 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 190 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 172 | with pytest.raises(ConfigError, match="non-canonical"): | 191 | with pytest.raises(ExperimentConfigError, match="non-canonical"): |
| 173 | load_config(path) | 192 | HarnessConfig.from_yaml(path) |
| 174 | 193 | ||
| 175 | 194 | ||
| 176 | @pytest.mark.parametrize( | 195 | @pytest.mark.parametrize( |
| 177 | ("section", "key", "value", "expected"), | 196 | ("section", "key", "value", "expected"), |
| 209 | node[key] = value | 228 | node[key] = value |
| 210 | path = tmp_path / "scalar.yaml" | 229 | path = tmp_path / "scalar.yaml" |
| 211 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 230 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 212 | dotted = ".".join((*section, key)) | 231 | dotted = ".".join((*section, key)) |
| 213 | with pytest.raises(ConfigError) as failure: | 232 | with pytest.raises(ExperimentConfigError) as failure: |
| 214 | load_config(path) | 233 | HarnessConfig.from_yaml(path) |
| 215 | message = str(failure.value) | 234 | message = str(failure.value) |
| 216 | assert str(path) in message | 235 | assert str(path) in message |
| 217 | assert dotted in message | 236 | assert dotted in message |
| 218 | assert expected in message | 237 | assert expected in message |
| 232 | pytest.skip("The reference gate declares no string parameter") | 251 | pytest.skip("The reference gate declares no string parameter") |
| 233 | params[key] = 3 | 252 | params[key] = 3 |
| 234 | path = tmp_path / "gate.yaml" | 253 | path = tmp_path / "gate.yaml" |
| 235 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 254 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 236 | with pytest.raises(ConfigError, match=f"params.{key}"): | 255 | with pytest.raises(ExperimentConfigError, match=f"params.{key}"): |
| 237 | load_config(path) | 256 | HarnessConfig.from_yaml(path) |
| 238 | 257 | ||
| 239 | 258 | ||
| 240 | def test_template_dispatch_names_implementation_ticket() -> None: | 259 | def test_template_dispatch_names_implementation_ticket() -> None: |
| 241 | """A template fails before runner work and identifies its exact ticket.""" | 260 | """A template fails before runner work and identifies its exact ticket.""" |
| 242 | config = load_config("configs/e03_feature_ablation.yaml") | 261 | config = HarnessConfig.from_yaml("configs/e03_feature_ablation.yaml") |
| 243 | with pytest.raises(DispatchError, match="AI3D-MLSEG-E3"): | 262 | with pytest.raises(DispatchError, match="AI3D-MLSEG-E3"): |
| 244 | dispatch_experiment(config) | 263 | dispatch_experiment(config) |
| 245 | 264 | ||
| 246 | 265 | ||
| 247 | def test_gated_dispatch_names_missing_artifact() -> None: | 266 | def test_gated_dispatch_names_missing_artifact() -> None: |
| 248 | """A gated experiment reports its first unmet prerequisite.""" | 267 | """A gated experiment reports its first unmet prerequisite.""" |
| 249 | config = load_config("configs/e07_ezsp_scale.yaml") | 268 | config = HarnessConfig.from_yaml("configs/e07_ezsp_scale.yaml") |
| 250 | with pytest.raises(DispatchError, match="data_contract_acceptance.json"): | 269 | with pytest.raises(DispatchError, match="data_contract_acceptance.json"): |
| 251 | dispatch_experiment(config) | 270 | dispatch_experiment(config) |
| 252 | 271 | ||
| 253 | 272 | ||
| 254 | def test_e1_dispatch_names_partition_oracle_report() -> None: | 273 | def test_e1_dispatch_names_partition_oracle_report() -> None: |
| 255 | """E1 cannot launch without the partition-purity report.""" | 274 | """E1 cannot launch without the partition-purity report.""" |
| 256 | config = load_config("configs/e01_spt_pilot.yaml") | 275 | config = HarnessConfig.from_yaml("configs/e01_spt_pilot.yaml") |
| 257 | with pytest.raises(DispatchError, match="partition_oracle.json"): | 276 | with pytest.raises(DispatchError, match="partition_oracle.json"): |
| 258 | dispatch_experiment(config) | 277 | dispatch_experiment(config) |
| 259 | 278 | ||
| 260 | 279 |
| 263 | ) -> None: | 282 | ) -> None: |
| 264 | """An external experiment names every unset framework variable.""" | 283 | """An external experiment names every unset framework variable.""" |
| 265 | for name in ("SPT_ROOT", "SPT_COMMIT", "SPT_PYTHON"): | 284 | for name in ("SPT_ROOT", "SPT_COMMIT", "SPT_PYTHON"): |
| 266 | monkeypatch.delenv(name, raising=False) | 285 | monkeypatch.delenv(name, raising=False) |
| 267 | config = load_config("configs/e01_spt_pilot.yaml") | 286 | config = HarnessConfig.from_yaml("configs/e01_spt_pilot.yaml") |
| 268 | with pytest.raises(DispatchError, match="SPT_ROOT, SPT_COMMIT, SPT_PYTHON"): | 287 | with pytest.raises(DispatchError, match="SPT_ROOT, SPT_COMMIT, SPT_PYTHON"): |
| 269 | dispatch_experiment(config) | 288 | dispatch_experiment(config) |
| 270 | 289 | ||
| 271 | 290 |
| 276 | checkout = _fake_spt_checkout(tmp_path) | 295 | checkout = _fake_spt_checkout(tmp_path) |
| 277 | monkeypatch.setenv("SPT_ROOT", str(checkout)) | 296 | monkeypatch.setenv("SPT_ROOT", str(checkout)) |
| 278 | monkeypatch.setenv("SPT_COMMIT", "0" * 40) | 297 | monkeypatch.setenv("SPT_COMMIT", "0" * 40) |
| 279 | monkeypatch.setenv("SPT_PYTHON", str(_fake_interpreter(tmp_path))) | 298 | monkeypatch.setenv("SPT_PYTHON", str(_fake_interpreter(tmp_path))) |
| 280 | config = load_config("configs/e01_spt_pilot.yaml") | 299 | config = HarnessConfig.from_yaml("configs/e01_spt_pilot.yaml") |
| 281 | with pytest.raises(DispatchError) as failure: | 300 | with pytest.raises(DispatchError) as failure: |
| 282 | dispatch_experiment(config) | 301 | dispatch_experiment(config) |
| 283 | assert "framework_environment" not in str(failure.value) | 302 | assert "framework_environment" not in str(failure.value) |
| 284 | 303 |
| 305 | ) -> None: | 324 | ) -> None: |
| 306 | """CPU experiments never demand external checkout variables.""" | 325 | """CPU experiments never demand external checkout variables.""" |
| 307 | for name in ("SPT_ROOT", "SPT_COMMIT", "SPT_PYTHON"): | 326 | for name in ("SPT_ROOT", "SPT_COMMIT", "SPT_PYTHON"): |
| 308 | monkeypatch.delenv(name, raising=False) | 327 | monkeypatch.delenv(name, raising=False) |
| 309 | config = load_config("configs/e02_voxel_survival_oracle.yaml") | 328 | config = HarnessConfig.from_yaml("configs/e02_voxel_survival_oracle.yaml") |
| 310 | with pytest.raises(DispatchError) as failure: | 329 | with pytest.raises(DispatchError) as failure: |
| 311 | dispatch_experiment(config) | 330 | dispatch_experiment(config) |
| 312 | assert "framework_environment" not in str(failure.value) | 331 | assert "framework_environment" not in str(failure.value) |
| 313 | 332 |
| 354 | 373 | ||
| 355 | def test_v2_recap_config_loads_against_ontology_v2(tmp_path: Path) -> None: | 374 | def test_v2_recap_config_loads_against_ontology_v2(tmp_path: Path) -> None: |
| 356 | """A v2 task block aligned with recap_semantics_v2 is accepted.""" | 375 | """A v2 task block aligned with recap_semantics_v2 is accepted.""" |
| 357 | path = _write_config(tmp_path, _v2_recap_document(), "v2.yaml") | 376 | path = _write_config(tmp_path, _v2_recap_document(), "v2.yaml") |
| 358 | config = load_config(path) | 377 | config = HarnessConfig.from_yaml(path) |
| 359 | assert config.task.ontology == ONTOLOGY_V2_PATH | 378 | assert config.task.ontology == ONTOLOGY_V2_PATH |
| 360 | assert config.task.num_classes == 11 | 379 | assert config.task.num_classes == 11 |
| 361 | assert config.task.ignore_index == 11 | 380 | assert config.task.ignore_index == 11 |
| 362 | assert config.task.classes_of_interest == tuple(range(9)) | 381 | assert config.task.classes_of_interest == tuple(range(9)) |
| 378 | """task.num_classes must equal ontology.num_predicted_classes.""" | 397 | """task.num_classes must equal ontology.num_predicted_classes.""" |
| 379 | raw = _v2_recap_document() | 398 | raw = _v2_recap_document() |
| 380 | raw["task"]["num_classes"] = 9 | 399 | raw["task"]["num_classes"] = 9 |
| 381 | path = _write_config(tmp_path, raw, "v2_num_classes.yaml") | 400 | path = _write_config(tmp_path, raw, "v2_num_classes.yaml") |
| 382 | with pytest.raises(ConfigError, match="num_predicted_classes"): | 401 | with pytest.raises(ExperimentConfigError, match="num_predicted_classes"): |
| 383 | load_config(path) | 402 | HarnessConfig.from_yaml(path) |
| 384 | 403 | ||
| 385 | 404 | ||
| 386 | def test_v2_config_rejects_incomplete_classes_of_interest( | 405 | def test_v2_config_rejects_incomplete_classes_of_interest( |
| 387 | tmp_path: Path, | 406 | tmp_path: Path, |
| 389 | """task.classes_of_interest must equal ontology.interest_ids.""" | 408 | """task.classes_of_interest must equal ontology.interest_ids.""" |
| 390 | raw = _v2_recap_document() | 409 | raw = _v2_recap_document() |
| 391 | raw["task"]["classes_of_interest"] = list(range(8)) | 410 | raw["task"]["classes_of_interest"] = list(range(8)) |
| 392 | path = _write_config(tmp_path, raw, "v2_interest.yaml") | 411 | path = _write_config(tmp_path, raw, "v2_interest.yaml") |
| 393 | with pytest.raises(ConfigError, match="classes_of_interest"): | 412 | with pytest.raises(ExperimentConfigError, match="classes_of_interest"): |
| 394 | load_config(path) | 413 | HarnessConfig.from_yaml(path) |
| 395 | 414 | ||
| 396 | 415 | ||
| 397 | def test_v2_config_rejects_v1_linear_class_names(tmp_path: Path) -> None: | 416 | def test_v2_config_rejects_v1_linear_class_names(tmp_path: Path) -> None: |
| 398 | """task.linear_classes must be the linear names of the loaded ontology.""" | 417 | """task.linear_classes must be the linear names of the loaded ontology.""" |
| 402 | "wall_noise_barrier", | 421 | "wall_noise_barrier", |
| 403 | "fence_gate", | 422 | "fence_gate", |
| 404 | ] | 423 | ] |
| 405 | path = _write_config(tmp_path, raw, "v2_linear.yaml") | 424 | path = _write_config(tmp_path, raw, "v2_linear.yaml") |
| 406 | with pytest.raises(ConfigError, match="linear_classes"): | 425 | with pytest.raises(ExperimentConfigError, match="linear_classes"): |
| 407 | load_config(path) | 426 | HarnessConfig.from_yaml(path) |
| 408 | 427 | ||
| 409 | 428 | ||
| 410 | def test_v2_config_rejects_v1_precision_floor_class(tmp_path: Path) -> None: | 429 | def test_v2_config_rejects_v1_precision_floor_class(tmp_path: Path) -> None: |
| 411 | """precision_floors keys must be predicted names of the loaded ontology.""" | 430 | """precision_floors keys must be predicted names of the loaded ontology.""" |
| 412 | raw = _v2_recap_document() | 431 | raw = _v2_recap_document() |
| 413 | raw["evaluation"]["precision_floors"] = {"sign_gantry": 0.9} | 432 | raw["evaluation"]["precision_floors"] = {"sign_gantry": 0.9} |
| 414 | path = _write_config(tmp_path, raw, "v2_floors.yaml") | 433 | path = _write_config(tmp_path, raw, "v2_floors.yaml") |
| 415 | with pytest.raises(ConfigError, match="sign_gantry"): | 434 | with pytest.raises(ExperimentConfigError, match="sign_gantry"): |
| 416 | load_config(path) | 435 | HarnessConfig.from_yaml(path) |
| 417 | 436 | ||
| 418 | 437 | ||
| 419 | def test_v2_config_rejects_model_num_classes_mismatch(tmp_path: Path) -> None: | 438 | def test_v2_config_rejects_model_num_classes_mismatch(tmp_path: Path) -> None: |
| 420 | """model.args.num_classes must equal task.num_classes when present.""" | 439 | """model.args.num_classes must equal task.num_classes when present.""" |
| 421 | raw = _v2_recap_document() | 440 | raw = _v2_recap_document() |
| 422 | raw["model"]["args"]["num_classes"] = 9 | 441 | raw["model"]["args"]["num_classes"] = 9 |
| 423 | path = _write_config(tmp_path, raw, "v2_model_classes.yaml") | 442 | path = _write_config(tmp_path, raw, "v2_model_classes.yaml") |
| 424 | with pytest.raises(ConfigError, match="model.args.num_classes"): | 443 | with pytest.raises(ExperimentConfigError, match="model.args.num_classes"): |
| 425 | load_config(path) | 444 | HarnessConfig.from_yaml(path) |
| 426 | 445 | ||
| 427 | 446 | ||
| 428 | def test_macro_interest_all9_is_canonical_only_under_v2() -> None: | 447 | def test_macro_interest_all9_is_canonical_only_under_v2() -> None: |
| 429 | """v2 macros use all9; the frozen v1 all8 tag is not canonical under v2.""" | 448 | """v2 macros use all9; the frozen v1 all8 tag is not canonical under v2.""" |
| 442 | """A v1 ontology has no delineator class, so the monitor is non-canonical.""" | 461 | """A v1 ontology has no delineator class, so the monitor is non-canonical.""" |
| 443 | raw = yaml.safe_load(A1_SMOKE_CONFIG.read_text(encoding="utf-8")) | 462 | raw = yaml.safe_load(A1_SMOKE_CONFIG.read_text(encoding="utf-8")) |
| 444 | raw["train"]["monitor"] = "val/iou_delineator" | 463 | raw["train"]["monitor"] = "val/iou_delineator" |
| 445 | path = _write_config(tmp_path, raw, "v1_delineator.yaml") | 464 | path = _write_config(tmp_path, raw, "v1_delineator.yaml") |
| 446 | with pytest.raises(ConfigError, match="delineator"): | 465 | with pytest.raises(ExperimentConfigError, match="delineator"): |
| 447 | load_config(path) | 466 | HarnessConfig.from_yaml(path) |
| 448 | 467 | ||
| 449 | 468 | ||
| 450 | def test_v2_config_rejects_loss_ignore_index_mismatch(tmp_path: Path) -> None: | 469 | def test_v2_config_rejects_loss_ignore_index_mismatch(tmp_path: Path) -> None: |
| 451 | """loss.args.ignore_index must equal task.ignore_index when present.""" | 470 | """loss.args.ignore_index must equal task.ignore_index when present.""" |
| 452 | raw = _v2_recap_document() | 471 | raw = _v2_recap_document() |
| 453 | raw["loss"]["args"]["ignore_index"] = 9 | 472 | raw["loss"]["args"]["ignore_index"] = 9 |
| 454 | path = _write_config(tmp_path, raw, "v2_loss_ignore_index.yaml") | 473 | path = _write_config(tmp_path, raw, "v2_loss_ignore_index.yaml") |
| 455 | with pytest.raises(ConfigError, match="loss.args.ignore_index"): | 474 | with pytest.raises(ExperimentConfigError, match="loss.args.ignore_index"): |
| 456 | load_config(path) | 475 | HarnessConfig.from_yaml(path) |
| 457 | 476 | ||
| 458 | 477 | ||
| 459 | def test_v2_config_rejects_cluster_profiles_from_another_ontology( | 478 | def test_v2_config_rejects_cluster_profiles_from_another_ontology( |
| 460 | tmp_path: Path, | 479 | tmp_path: Path, |
| 464 | raw["evaluation"]["object_matching"]["cluster_profiles"] = ( | 483 | raw["evaluation"]["object_matching"]["cluster_profiles"] = ( |
| 465 | "configs/contracts/ontology_v1.yaml" | 484 | "configs/contracts/ontology_v1.yaml" |
| 466 | ) | 485 | ) |
| 467 | path = _write_config(tmp_path, raw, "v2_cluster_profiles.yaml") | 486 | path = _write_config(tmp_path, raw, "v2_cluster_profiles.yaml") |
| 468 | with pytest.raises(ConfigError, match="cluster_profiles"): | 487 | with pytest.raises(ExperimentConfigError, match="cluster_profiles"): |
| 469 | load_config(path) | 488 | HarnessConfig.from_yaml(path) |
| 470 | 489 | ||
| 471 | 490 | ||
| 472 | def test_v2_config_accepts_all9_deciding_metric(tmp_path: Path) -> None: | 491 | def test_v2_config_accepts_all9_deciding_metric(tmp_path: Path) -> None: |
| 473 | """The structural macro suffix follows the ontology's interest count.""" | 492 | """The structural macro suffix follows the ontology's interest count.""" |
| 474 | raw = _v2_recap_document() | 493 | raw = _v2_recap_document() |
| 475 | raw["experiment"]["deciding_metrics"] = ["val/iou_macro_interest_all9"] | 494 | raw["experiment"]["deciding_metrics"] = ["val/iou_macro_interest_all9"] |
| 476 | path = _write_config(tmp_path, raw, "v2_all9_metric.yaml") | 495 | path = _write_config(tmp_path, raw, "v2_all9_metric.yaml") |
| 477 | 496 | ||
| 478 | config = load_config(path) | 497 | config = HarnessConfig.from_yaml(path) |
| 479 | 498 | ||
| 480 | assert config.experiment.deciding_metrics == ("val/iou_macro_interest_all9",) | 499 | assert config.experiment.deciding_metrics == ("val/iou_macro_interest_all9",) |
| 481 | 500 | ||
| 482 | 501 |
| 484 | """The frozen v1 macro tag is not canonical for a nine-interest ontology.""" | 503 | """The frozen v1 macro tag is not canonical for a nine-interest ontology.""" |
| 485 | raw = _v2_recap_document() | 504 | raw = _v2_recap_document() |
| 486 | raw["experiment"]["deciding_metrics"] = ["val/iou_macro_interest_all8"] | 505 | raw["experiment"]["deciding_metrics"] = ["val/iou_macro_interest_all8"] |
| 487 | path = _write_config(tmp_path, raw, "v2_all8_metric.yaml") | 506 | path = _write_config(tmp_path, raw, "v2_all8_metric.yaml") |
| 488 | with pytest.raises(ConfigError, match="deciding_metrics"): | 507 | with pytest.raises(ExperimentConfigError, match="deciding_metrics"): |
| 489 | load_config(path) | 508 | HarnessConfig.from_yaml(path) |
| 490 | 509 | ||
| 491 | 510 | ||
| 492 | def test_v2_gate_rejects_purity_class_from_another_ontology( | 511 | def test_v2_gate_rejects_purity_class_from_another_ontology( |
| 493 | tmp_path: Path, | 512 | tmp_path: Path, |
| 505 | }, | 524 | }, |
| 506 | } | 525 | } |
| 507 | ] | 526 | ] |
| 508 | path = _write_config(tmp_path, raw, "v2_gate_purity.yaml") | 527 | path = _write_config(tmp_path, raw, "v2_gate_purity.yaml") |
| 509 | with pytest.raises(ConfigError, match="sign_gantry"): | 528 | with pytest.raises(ExperimentConfigError, match="sign_gantry"): |
| 510 | load_config(path) | 529 | HarnessConfig.from_yaml(path) |
| 511 | 530 | ||
| 512 | 531 | ||
| 513 | def test_v2_gate_accepts_purity_class_of_the_task_ontology( | 532 | def test_v2_gate_accepts_purity_class_of_the_task_ontology( |
| 514 | tmp_path: Path, | 533 | tmp_path: Path, |
| 527 | } | 546 | } |
| 528 | ] | 547 | ] |
| 529 | path = _write_config(tmp_path, raw, "v2_gate_purity_ok.yaml") | 548 | path = _write_config(tmp_path, raw, "v2_gate_purity_ok.yaml") |
| 530 | 549 | ||
| 531 | config = load_config(path) | 550 | config = HarnessConfig.from_yaml(path) |
| 532 | 551 | ||
| 533 | assert config.experiment.gates[0].params.minimum_purity_by_class == { | 552 | assert config.experiment.gates[0].params.minimum_purity_by_class == { |
| 534 | "delineator": 0.8 | 553 | "delineator": 0.8 |
| 535 | } | 554 | } |
| 545 | raw["task"]["ontology"] = broken.name | 564 | raw["task"]["ontology"] = broken.name |
| 546 | raw["evaluation"]["object_matching"]["cluster_profiles"] = broken.name | 565 | raw["evaluation"]["object_matching"]["cluster_profiles"] = broken.name |
| 547 | path = _write_config(tmp_path, raw, "broken_ontology_config.yaml") | 566 | path = _write_config(tmp_path, raw, "broken_ontology_config.yaml") |
| 548 | with pytest.raises( | 567 | with pytest.raises( |
| 549 | ConfigError, match=r"task\.ontology .* is not a valid ontology" | 568 | ExperimentConfigError, match=r"task\.ontology .* is not a valid ontology" |
| 550 | ): | 569 | ): |
| 551 | load_config(path) | 570 | HarnessConfig.from_yaml(path) |
| 552 | 571 | ||
| 553 | 572 | ||
| 554 | def test_missing_task_ontology_names_the_working_directory( | 573 | def test_missing_task_ontology_names_the_working_directory( |
| 555 | tmp_path: Path, | 574 | tmp_path: Path, |
| 560 | raw["evaluation"]["object_matching"]["cluster_profiles"] = ( | 579 | raw["evaluation"]["object_matching"]["cluster_profiles"] = ( |
| 561 | "configs/contracts/ontology_absent.yaml" | 580 | "configs/contracts/ontology_absent.yaml" |
| 562 | ) | 581 | ) |
| 563 | path = _write_config(tmp_path, raw, "absent_ontology_config.yaml") | 582 | path = _write_config(tmp_path, raw, "absent_ontology_config.yaml") |
| 564 | with pytest.raises(ConfigError, match="was not found relative to"): | 583 | with pytest.raises(ExperimentConfigError, match="was not found relative to"): |
| 565 | load_config(path) | 584 | HarnessConfig.from_yaml(path) |
| 566 | 585 | ||
| 567 | 586 | ||
| 568 | def test_task_ontology_resolves_inside_the_configs_own_repository( | 587 | def test_task_ontology_resolves_inside_the_configs_own_repository( |
| 569 | tmp_path: Path, monkeypatch: pytest.MonkeyPatch | 588 | tmp_path: Path, monkeypatch: pytest.MonkeyPatch |
| 589 | ONTOLOGY_V2_PATH.read_text(encoding="utf-8"), encoding="utf-8" | 608 | ONTOLOGY_V2_PATH.read_text(encoding="utf-8"), encoding="utf-8" |
| 590 | ) | 609 | ) |
| 591 | monkeypatch.chdir(tmp_path / "elsewhere") | 610 | monkeypatch.chdir(tmp_path / "elsewhere") |
| 592 | 611 | ||
| 593 | config = load_config(config_path) | 612 | config = HarnessConfig.from_yaml(config_path) |
| 594 | 613 | ||
| 595 | assert resolve_ontology_path(config) == repo_contracts / "ontology_v1.yaml" | 614 | assert resolve_ontology_path(config) == repo_contracts / "ontology_v1.yaml" |
| 596 | assert config.task.num_classes == 9 | 615 | assert config.task.num_classes == 9 |
| 597 | assert config.task.ignore_index == 9 | 616 | assert config.task.ignore_index == 9 |
| 603 | """task.ontology also resolves against the config's own repository root.""" | 622 | """task.ontology also resolves against the config's own repository root.""" |
| 604 | config_path = Path("configs/e01_spt_pilot.yaml").resolve() | 623 | config_path = Path("configs/e01_spt_pilot.yaml").resolve() |
| 605 | monkeypatch.chdir(tmp_path) | 624 | monkeypatch.chdir(tmp_path) |
| 606 | 625 | ||
| 607 | config = load_config(config_path) | 626 | config = HarnessConfig.from_yaml(config_path) |
| 608 | 627 | ||
| 609 | assert config.task.ontology == Path("configs/contracts/ontology_v1.yaml") | 628 | assert config.task.ontology == Path("configs/contracts/ontology_v1.yaml") |
| 610 | assert config.task.num_classes == 9 | 629 | assert config.task.num_classes == 9 |
| 611 | 630 |
| 617 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) | 636 | raw = yaml.safe_load(CONFIGS[0].read_text(encoding="utf-8")) |
| 618 | raw["__sha256__"] = "deadbeef" | 637 | raw["__sha256__"] = "deadbeef" |
| 619 | path = tmp_path / "smuggled.yaml" | 638 | path = tmp_path / "smuggled.yaml" |
| 620 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 639 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 621 | with pytest.raises(ConfigError, match="Unknown.*__sha256__"): | 640 | with pytest.raises(ExperimentConfigError, match="Unknown.*__sha256__"): |
| 622 | load_config(path) | 641 | HarnessConfig.from_yaml(path) |
| 623 | 642 | ||
| 624 | 643 | ||
| 625 | def test_null_visualization_block_is_rejected(tmp_path: Path) -> None: | 644 | def test_null_visualization_block_is_rejected(tmp_path: Path) -> None: |
| 626 | """An explicitly null visualization block is a typo, not an omission.""" | 645 | """An explicitly null visualization block is a typo, not an omission.""" |
| 627 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) | 646 | raw = yaml.safe_load(CONFIGS[8].read_text(encoding="utf-8")) |
| 628 | raw["visualization"] = None | 647 | raw["visualization"] = None |
| 629 | path = tmp_path / "viz_null.yaml" | 648 | path = tmp_path / "viz_null.yaml" |
| 630 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 649 | path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 631 | with pytest.raises(ConfigError, match="visualization must be a mapping"): | 650 | with pytest.raises(ExperimentConfigError, match="visualization must be a mapping"): |
| 632 | load_config(path) | 651 | HarnessConfig.from_yaml(path) |
| 633 | 652 | ||
| 634 | 653 | ||
| 635 | def test_mapping_sections_are_read_only() -> None: | 654 | def test_mapping_sections_are_read_only() -> None: |
| 636 | """Frozen configs hand out read-only mappings, not mutable dicts.""" | 655 | """Frozen configs hand out read-only mappings, not mutable dicts.""" |
| 637 | config = load_config(CONFIGS[0]) | 656 | config = HarnessConfig.from_yaml(CONFIGS[0]) |
| 638 | with pytest.raises(TypeError): | 657 | with pytest.raises(TypeError): |
| 639 | config.model.args["num_classes"] = 99 # type: ignore[index] | 658 | config.model.args["num_classes"] = 99 # type: ignore[index] |
| 640 | with pytest.raises(TypeError): | 659 | with pytest.raises(TypeError): |
| 641 | config.loss.args["gamma"] = 0.0 # type: ignore[index] | 660 | config.loss.args["gamma"] = 0.0 # type: ignore[index] |
| 10 | 10 | ||
| 11 | import pytest | 11 | import pytest |
| 12 | import yaml | 12 | import yaml |
| 13 | 13 | ||
| 14 | from src.train.config import load_config | 14 | from src.train.config import HarnessConfig |
| 15 | from src.train.dispatch import DispatchError, DispatchRequest, dispatch_experiment | 15 | from src.train.dispatch import DispatchError, DispatchRequest, dispatch_experiment |
| 16 | 16 | ||
| 17 | BASE_CONFIG = Path("configs/e02_voxel_survival_oracle.yaml") | 17 | BASE_CONFIG = Path("configs/e02_voxel_survival_oracle.yaml") |
| 18 | CONTRACTS_DIR = Path("configs/contracts") | 18 | CONTRACTS_DIR = Path("configs/contracts") |
| 39 | raw["experiment"]["gates"] = [] | 39 | raw["experiment"]["gates"] = [] |
| 40 | raw["model"]["runner"] = "stub_runner.py" | 40 | raw["model"]["runner"] = "stub_runner.py" |
| 41 | raw["study"] = study | 41 | raw["study"] = study |
| 42 | (tmp_path / "stub_runner.py").write_text(STUB_RUNNER, encoding="utf-8") | 42 | (tmp_path / "stub_runner.py").write_text(STUB_RUNNER, encoding="utf-8") |
| 43 | # load_config resolves task.ontology against the working directory, so the | 43 | # HarnessConfig.from_yaml resolves task.ontology against the working |
| 44 | # directory, so the | ||
| 44 | # throwaway root needs the frozen contracts the config names. | 45 | # throwaway root needs the frozen contracts the config names. |
| 45 | shutil.copytree(CONTRACTS_DIR, tmp_path / CONTRACTS_DIR, dirs_exist_ok=True) | 46 | shutil.copytree(CONTRACTS_DIR, tmp_path / CONTRACTS_DIR, dirs_exist_ok=True) |
| 46 | config_path = tmp_path / "experiment.yaml" | 47 | config_path = tmp_path / "experiment.yaml" |
| 47 | config_path.write_text(yaml.safe_dump(raw), encoding="utf-8") | 48 | config_path.write_text(yaml.safe_dump(raw), encoding="utf-8") |
| 81 | log_path = tmp_path / "invocations.jsonl" | 82 | log_path = tmp_path / "invocations.jsonl" |
| 82 | monkeypatch.setenv("STUB_LOG", str(log_path)) | 83 | monkeypatch.setenv("STUB_LOG", str(log_path)) |
| 83 | request = DispatchRequest(log_dir=tmp_path / "runs") | 84 | request = DispatchRequest(log_dir=tmp_path / "runs") |
| 84 | assert dispatch_experiment( | 85 | assert dispatch_experiment( |
| 85 | load_config(config_path), request, repository_root=root | 86 | HarnessConfig.from_yaml(config_path), request, repository_root=root |
| 86 | ) == 0 | 87 | ) == 0 |
| 87 | records = _records(log_path) | 88 | records = _records(log_path) |
| 88 | assert [record["cell"] for record in records] == ["coarse", "fine"] | 89 | assert [record["cell"] for record in records] == ["coarse", "fine"] |
| 89 | assert [record["index"] for record in records] == ["0", "1"] | 90 | assert [record["index"] for record in records] == ["0", "1"] |
| 106 | """Every cell run directory carries identity, overrides, and its SHA-256.""" | 107 | """Every cell run directory carries identity, overrides, and its SHA-256.""" |
| 107 | root, config_path = _repository(tmp_path, _variants_study()) | 108 | root, config_path = _repository(tmp_path, _variants_study()) |
| 108 | monkeypatch.setenv("STUB_LOG", str(tmp_path / "invocations.jsonl")) | 109 | monkeypatch.setenv("STUB_LOG", str(tmp_path / "invocations.jsonl")) |
| 109 | dispatch_experiment( | 110 | dispatch_experiment( |
| 110 | load_config(config_path), | 111 | HarnessConfig.from_yaml(config_path), |
| 111 | DispatchRequest(log_dir=tmp_path / "runs"), | 112 | DispatchRequest(log_dir=tmp_path / "runs"), |
| 112 | repository_root=root, | 113 | repository_root=root, |
| 113 | ) | 114 | ) |
| 114 | provenance = json.loads( | 115 | provenance = json.loads( |
| 138 | root, config_path = _repository(tmp_path, study) | 139 | root, config_path = _repository(tmp_path, study) |
| 139 | log_path = tmp_path / "invocations.jsonl" | 140 | log_path = tmp_path / "invocations.jsonl" |
| 140 | monkeypatch.setenv("STUB_LOG", str(log_path)) | 141 | monkeypatch.setenv("STUB_LOG", str(log_path)) |
| 141 | dispatch_experiment( | 142 | dispatch_experiment( |
| 142 | load_config(config_path), | 143 | HarnessConfig.from_yaml(config_path), |
| 143 | DispatchRequest(log_dir=tmp_path / "runs"), | 144 | DispatchRequest(log_dir=tmp_path / "runs"), |
| 144 | repository_root=root, | 145 | repository_root=root, |
| 145 | ) | 146 | ) |
| 146 | records = _records(log_path) | 147 | records = _records(log_path) |
| 158 | monkeypatch.setenv("STUB_LOG", str(log_path)) | 159 | monkeypatch.setenv("STUB_LOG", str(log_path)) |
| 159 | monkeypatch.setenv("STUB_EXIT_CODE", "3") | 160 | monkeypatch.setenv("STUB_EXIT_CODE", "3") |
| 160 | with pytest.raises(DispatchError, match="cell coarse"): | 161 | with pytest.raises(DispatchError, match="cell coarse"): |
| 161 | dispatch_experiment( | 162 | dispatch_experiment( |
| 162 | load_config(config_path), | 163 | HarnessConfig.from_yaml(config_path), |
| 163 | DispatchRequest(log_dir=tmp_path / "runs"), | 164 | DispatchRequest(log_dir=tmp_path / "runs"), |
| 164 | repository_root=root, | 165 | repository_root=root, |
| 165 | ) | 166 | ) |
| 166 | assert len(_records(log_path)) == 1 | 167 | assert len(_records(log_path)) == 1 |
| 172 | """DispatchRequest narrowing is forwarded unchanged to each cell.""" | 173 | """DispatchRequest narrowing is forwarded unchanged to each cell.""" |
| 173 | root, config_path = _repository(tmp_path, _variants_study()) | 174 | root, config_path = _repository(tmp_path, _variants_study()) |
| 174 | log_path = tmp_path / "invocations.jsonl" | 175 | log_path = tmp_path / "invocations.jsonl" |
| 175 | monkeypatch.setenv("STUB_LOG", str(log_path)) | 176 | monkeypatch.setenv("STUB_LOG", str(log_path)) |
| 176 | config = load_config(config_path) | 177 | config = HarnessConfig.from_yaml(config_path) |
| 177 | request = DispatchRequest( | 178 | request = DispatchRequest( |
| 178 | corridors=(config.data.corridors.include[0],), | 179 | corridors=(config.data.corridors.include[0],), |
| 179 | fast_dev_run=True, | 180 | fast_dev_run=True, |
| 180 | num_workers=2, | 181 | num_workers=2, |
| 194 | """A training run needs both folds, so --split fails early and by name.""" | 195 | """A training run needs both folds, so --split fails early and by name.""" |
| 195 | root, config_path = _repository(tmp_path, _variants_study()) | 196 | root, config_path = _repository(tmp_path, _variants_study()) |
| 196 | with pytest.raises(DispatchError, match="refuses --split"): | 197 | with pytest.raises(DispatchError, match="refuses --split"): |
| 197 | dispatch_experiment( | 198 | dispatch_experiment( |
| 198 | load_config(config_path), | 199 | HarnessConfig.from_yaml(config_path), |
| 199 | DispatchRequest(splits=("train",)), | 200 | DispatchRequest(splits=("train",)), |
| 200 | repository_root=root, | 201 | repository_root=root, |
| 201 | ) | 202 | ) |
| 202 | 203 |
| 236 | """Study expansion never relaxes the narrowing contract.""" | 237 | """Study expansion never relaxes the narrowing contract.""" |
| 237 | root, config_path = _repository(tmp_path, _variants_study()) | 238 | root, config_path = _repository(tmp_path, _variants_study()) |
| 238 | with pytest.raises(DispatchError, match="promotion"): | 239 | with pytest.raises(DispatchError, match="promotion"): |
| 239 | dispatch_experiment( | 240 | dispatch_experiment( |
| 240 | load_config(config_path), | 241 | HarnessConfig.from_yaml(config_path), |
| 241 | DispatchRequest(splits=("promotion_test",)), | 242 | DispatchRequest(splits=("promotion_test",)), |
| 242 | repository_root=root, | 243 | repository_root=root, |
| 243 | ) | 244 | ) |
| 5 | from pathlib import Path | 5 | from pathlib import Path |
| 6 | 6 | ||
| 7 | import pytest | 7 | import pytest |
| 8 | 8 | ||
| 9 | from src.train.config import load_config | 9 | from src.train.config import HarnessConfig |
| 10 | from src.train.dispatch import DispatchError, dispatch_experiment | 10 | from src.train.dispatch import DispatchError, dispatch_experiment |
| 11 | 11 | ||
| 12 | SPT_CONFIG = Path("configs/e01_spt_pilot.yaml") | 12 | SPT_CONFIG = Path("configs/e01_spt_pilot.yaml") |
| 13 | CPU_CONFIG = Path("configs/e02_voxel_survival_oracle.yaml") | 13 | CPU_CONFIG = Path("configs/e02_voxel_survival_oracle.yaml") |
| 32 | 32 | ||
| 33 | 33 | ||
| 34 | def _dispatch_message() -> str: | 34 | def _dispatch_message() -> str: |
| 35 | """Dispatch the SPT pilot and return its refusal message.""" | 35 | """Dispatch the SPT pilot and return its refusal message.""" |
| 36 | config = load_config(SPT_CONFIG) | 36 | config = HarnessConfig.from_yaml(SPT_CONFIG) |
| 37 | with pytest.raises(DispatchError) as failure: | 37 | with pytest.raises(DispatchError) as failure: |
| 38 | dispatch_experiment(config) | 38 | dispatch_experiment(config) |
| 39 | return str(failure.value) | 39 | return str(failure.value) |
| 40 | 40 |
| 157 | monkeypatch: pytest.MonkeyPatch, | 157 | monkeypatch: pytest.MonkeyPatch, |
| 158 | ) -> None: | 158 | ) -> None: |
| 159 | """CPU experiments never consult the external framework environment.""" | 159 | """CPU experiments never consult the external framework environment.""" |
| 160 | monkeypatch.setenv("SPT_ROOT", "/definitely/not/a/checkout") | 160 | monkeypatch.setenv("SPT_ROOT", "/definitely/not/a/checkout") |
| 161 | config = load_config(CPU_CONFIG) | 161 | config = HarnessConfig.from_yaml(CPU_CONFIG) |
| 162 | with pytest.raises(DispatchError) as failure: | 162 | with pytest.raises(DispatchError) as failure: |
| 163 | dispatch_experiment(config) | 163 | dispatch_experiment(config) |
| 164 | assert "framework_environment" not in str(failure.value) | 164 | assert "framework_environment" not in str(failure.value) |
| 24 | SplitTier, | 24 | SplitTier, |
| 25 | authorize_split_access, | 25 | authorize_split_access, |
| 26 | load_split_manifest, | 26 | load_split_manifest, |
| 27 | ) | 27 | ) |
| 28 | from src.train.config import HarnessConfig, load_config | 28 | from src.train.config import HarnessConfig |
| 29 | 29 | ||
| 30 | CONFIG_PATH = Path("configs/e01_spt_pilot.yaml") | 30 | CONFIG_PATH = Path("configs/e01_spt_pilot.yaml") |
| 31 | GRID_ORIGIN = [10.0, -20.0, 5.0] | 31 | GRID_ORIGIN = [10.0, -20.0, 5.0] |
| 32 | 32 |
| 59 | 59 | ||
| 60 | 60 | ||
| 61 | def _config_with_prepared_corridors(tmp_path: Path) -> HarnessConfig: | 61 | def _config_with_prepared_corridors(tmp_path: Path) -> HarnessConfig: |
| 62 | """Return the E1 config pointed at synthetic adapter manifests.""" | 62 | """Return the E1 config pointed at synthetic adapter manifests.""" |
| 63 | config = load_config(CONFIG_PATH) | 63 | config = HarnessConfig.from_yaml(CONFIG_PATH) |
| 64 | canonical_root = tmp_path / "canonical_root" | 64 | canonical_root = tmp_path / "canonical_root" |
| 65 | (canonical_root / "canonical").mkdir(parents=True) | 65 | (canonical_root / "canonical").mkdir(parents=True) |
| 66 | for corridor_id in config.data.corridors.include: | 66 | for corridor_id in config.data.corridors.include: |
| 67 | (canonical_root / "canonical" / f"{corridor_id}.adapter.json").write_text( | 67 | (canonical_root / "canonical" / f"{corridor_id}.adapter.json").write_text( |
| 7 | 7 | ||
| 8 | import pytest | 8 | import pytest |
| 9 | import yaml | 9 | import yaml |
| 10 | 10 | ||
| 11 | from src.train.config import ConfigError, expand_study, load_config | 11 | from src.train.config import ExperimentConfigError, HarnessConfig, expand_study |
| 12 | 12 | ||
| 13 | BASE_CONFIG = Path("configs/e02_voxel_survival_oracle.yaml") | 13 | BASE_CONFIG = Path("configs/e02_voxel_survival_oracle.yaml") |
| 14 | SINGLE_CONFIG = Path("configs/e01_spt_pilot.yaml") | 14 | SINGLE_CONFIG = Path("configs/e01_spt_pilot.yaml") |
| 15 | 15 |
| 34 | 34 | ||
| 35 | 35 | ||
| 36 | def test_variants_study_expands_to_one_cell_per_variant() -> None: | 36 | def test_variants_study_expands_to_one_cell_per_variant() -> None: |
| 37 | """Every declared variant becomes its own resolved, hashed configuration.""" | 37 | """Every declared variant becomes its own resolved, hashed configuration.""" |
| 38 | config = load_config(BASE_CONFIG) | 38 | config = HarnessConfig.from_yaml(BASE_CONFIG) |
| 39 | cells = expand_study(config) | 39 | cells = expand_study(config) |
| 40 | assert [cell.id for cell in cells] == [ | 40 | assert [cell.id for cell in cells] == [ |
| 41 | variant.id for variant in config.study.variants | 41 | variant.id for variant in config.study.variants |
| 42 | ] | 42 | ] |
| 49 | 49 | ||
| 50 | 50 | ||
| 51 | def test_single_study_expands_to_exactly_one_identity_cell() -> None: | 51 | def test_single_study_expands_to_exactly_one_identity_cell() -> None: |
| 52 | """A single study runs once and changes nothing in the resolved config.""" | 52 | """A single study runs once and changes nothing in the resolved config.""" |
| 53 | config = load_config(SINGLE_CONFIG) | 53 | config = HarnessConfig.from_yaml(SINGLE_CONFIG) |
| 54 | cells = expand_study(config) | 54 | cells = expand_study(config) |
| 55 | assert len(cells) == 1 | 55 | assert len(cells) == 1 |
| 56 | assert cells[0].overrides == {} | 56 | assert cells[0].overrides == {} |
| 57 | assert cells[0].id == config.experiment.id | 57 | assert cells[0].id == config.experiment.id |
| 65 | 65 | ||
| 66 | 66 | ||
| 67 | def test_cell_order_and_hashes_are_deterministic() -> None: | 67 | def test_cell_order_and_hashes_are_deterministic() -> None: |
| 68 | """Repeated expansion yields identical identities, order, and hashes.""" | 68 | """Repeated expansion yields identical identities, order, and hashes.""" |
| 69 | config = load_config(BASE_CONFIG) | 69 | config = HarnessConfig.from_yaml(BASE_CONFIG) |
| 70 | first = expand_study(config) | 70 | first = expand_study(config) |
| 71 | second = expand_study(load_config(BASE_CONFIG)) | 71 | second = expand_study(HarnessConfig.from_yaml(BASE_CONFIG)) |
| 72 | assert [cell.id for cell in first] == [cell.id for cell in second] | 72 | assert [cell.id for cell in first] == [cell.id for cell in second] |
| 73 | assert [cell.config.sha256 for cell in first] == [ | 73 | assert [cell.config.sha256 for cell in first] == [ |
| 74 | cell.config.sha256 for cell in second | 74 | cell.config.sha256 for cell in second |
| 75 | ] | 75 | ] |
| 82 | {"id": "typo", "overrides": {"adapter.spt.voxel_metres": 0.02}} | 82 | {"id": "typo", "overrides": {"adapter.spt.voxel_metres": 0.02}} |
| 83 | ] | 83 | ] |
| 84 | raw["study"]["contrasts"] = [] | 84 | raw["study"]["contrasts"] = [] |
| 85 | path = _write(raw, tmp_path / "unknown_override.yaml") | 85 | path = _write(raw, tmp_path / "unknown_override.yaml") |
| 86 | with pytest.raises(ConfigError, match="adapter.spt.voxel_metres"): | 86 | with pytest.raises(ExperimentConfigError, match="adapter.spt.voxel_metres"): |
| 87 | load_config(path) | 87 | HarnessConfig.from_yaml(path) |
| 88 | 88 | ||
| 89 | 89 | ||
| 90 | def test_duplicate_variant_cell_ids_are_rejected(tmp_path: Path) -> None: | 90 | def test_duplicate_variant_cell_ids_are_rejected(tmp_path: Path) -> None: |
| 91 | """Two cells cannot share an identity.""" | 91 | """Two cells cannot share an identity.""" |
| 95 | {"id": "twin", "overrides": {"data.tiling.min_points": 2}}, | 95 | {"id": "twin", "overrides": {"data.tiling.min_points": 2}}, |
| 96 | ] | 96 | ] |
| 97 | raw["study"]["contrasts"] = [] | 97 | raw["study"]["contrasts"] = [] |
| 98 | path = _write(raw, tmp_path / "duplicate_ids.yaml") | 98 | path = _write(raw, tmp_path / "duplicate_ids.yaml") |
| 99 | with pytest.raises(ConfigError, match="duplicate"): | 99 | with pytest.raises(ExperimentConfigError, match="duplicate"): |
| 100 | load_config(path) | 100 | HarnessConfig.from_yaml(path) |
| 101 | 101 | ||
| 102 | 102 | ||
| 103 | def test_kind_without_its_payload_is_rejected(tmp_path: Path) -> None: | 103 | def test_kind_without_its_payload_is_rejected(tmp_path: Path) -> None: |
| 104 | """A discriminated kind whose payload is absent never expands silently.""" | 104 | """A discriminated kind whose payload is absent never expands silently.""" |
| 110 | "sweep": None, | 110 | "sweep": None, |
| 111 | "contrasts": [], | 111 | "contrasts": [], |
| 112 | } | 112 | } |
| 113 | path = _write(raw, tmp_path / "empty_variants.yaml") | 113 | path = _write(raw, tmp_path / "empty_variants.yaml") |
| 114 | with pytest.raises(ConfigError, match="variants"): | 114 | with pytest.raises(ExperimentConfigError, match="variants"): |
| 115 | load_config(path) | 115 | HarnessConfig.from_yaml(path) |
| 116 | 116 | ||
| 117 | 117 | ||
| 118 | def test_matrix_study_expands_cross_product_minus_exclusions( | 118 | def test_matrix_study_expands_cross_product_minus_exclusions( |
| 119 | tmp_path: Path, | 119 | tmp_path: Path, |
| 135 | }, | 135 | }, |
| 136 | "sweep": None, | 136 | "sweep": None, |
| 137 | "contrasts": [], | 137 | "contrasts": [], |
| 138 | } | 138 | } |
| 139 | config = load_config(_write(raw, tmp_path / "matrix.yaml")) | 139 | config = HarnessConfig.from_yaml(_write(raw, tmp_path / "matrix.yaml")) |
| 140 | cells = expand_study(config) | 140 | cells = expand_study(config) |
| 141 | assert [cell.id for cell in cells] == [ | 141 | assert [cell.id for cell in cells] == [ |
| 142 | "min_points-1__samples-500", | 142 | "min_points-1__samples-500", |
| 143 | "min_points-2__samples-500", | 143 | "min_points-2__samples-500", |
| 161 | }, | 161 | }, |
| 162 | "sweep": None, | 162 | "sweep": None, |
| 163 | "contrasts": [], | 163 | "contrasts": [], |
| 164 | } | 164 | } |
| 165 | config = load_config(_write(raw, tmp_path / "stale_exclude.yaml")) | 165 | config = HarnessConfig.from_yaml(_write(raw, tmp_path / "stale_exclude.yaml")) |
| 166 | with pytest.raises(ConfigError, match="matches no matrix cell"): | 166 | with pytest.raises(ExperimentConfigError, match="matches no matrix cell"): |
| 167 | expand_study(config) | 167 | expand_study(config) |
| 168 | 168 | ||
| 169 | 169 | ||
| 170 | def test_grid_sweep_expands_deterministically_within_budget( | 170 | def test_grid_sweep_expands_deterministically_within_budget( |
| 183 | "objective": "val/iou_macro_interest", | 183 | "objective": "val/iou_macro_interest", |
| 184 | }, | 184 | }, |
| 185 | "contrasts": [], | 185 | "contrasts": [], |
| 186 | } | 186 | } |
| 187 | config = load_config(_write(raw, tmp_path / "grid_sweep.yaml")) | 187 | config = HarnessConfig.from_yaml(_write(raw, tmp_path / "grid_sweep.yaml")) |
| 188 | cells = expand_study(config) | 188 | cells = expand_study(config) |
| 189 | assert [cell.id for cell in cells] == ["min_points-1", "min_points-2"] | 189 | assert [cell.id for cell in cells] == ["min_points-1", "min_points-2"] |
| 190 | assert [cell.config.data.tiling.min_points for cell in cells] == [1, 2] | 190 | assert [cell.config.data.tiling.min_points for cell in cells] == [1, 2] |
| 191 | 191 |
| 212 | "objective": "val/iou_macro_interest", | 212 | "objective": "val/iou_macro_interest", |
| 213 | }, | 213 | }, |
| 214 | "contrasts": [], | 214 | "contrasts": [], |
| 215 | } | 215 | } |
| 216 | config = load_config(_write(raw, tmp_path / "random_sweep.yaml")) | 216 | config = HarnessConfig.from_yaml(_write(raw, tmp_path / "random_sweep.yaml")) |
| 217 | first = expand_study(config) | 217 | first = expand_study(config) |
| 218 | second = expand_study(config) | 218 | second = expand_study(config) |
| 219 | assert [cell.id for cell in first] == ["sample_000", "sample_001", "sample_002"] | 219 | assert [cell.id for cell in first] == ["sample_000", "sample_001", "sample_002"] |
| 220 | assert [ | 220 | assert [ |
| 237 | "objective": "val/iou_macro_interest", | 237 | "objective": "val/iou_macro_interest", |
| 238 | }, | 238 | }, |
| 239 | "contrasts": [], | 239 | "contrasts": [], |
| 240 | } | 240 | } |
| 241 | config = load_config(_write(raw, tmp_path / "bayes_sweep.yaml")) | 241 | config = HarnessConfig.from_yaml(_write(raw, tmp_path / "bayes_sweep.yaml")) |
| 242 | with pytest.raises(ConfigError, match="bayesian"): | 242 | with pytest.raises(ExperimentConfigError, match="bayesian"): |
| 243 | expand_study(config) | 243 | expand_study(config) |
| 244 | 244 | ||
| 245 | 245 | ||
| 246 | def test_every_repository_experiment_expands() -> None: | 246 | def test_every_repository_experiment_expands() -> None: |
| 247 | """All E1-E14 studies expand to at least one strictly validated cell.""" | 247 | """All E1-E14 studies expand to at least one strictly validated cell.""" |
| 248 | for path in sorted(Path("configs").glob("e*.yaml")): | 248 | for path in sorted(Path("configs").glob("e*.yaml")): |
| 249 | config = load_config(path) | 249 | config = HarnessConfig.from_yaml(path) |
| 250 | cells = expand_study(config) | 250 | cells = expand_study(config) |
| 251 | assert cells | 251 | assert cells |
| 252 | assert len({cell.id for cell in cells}) == len(cells) | 252 | assert len({cell.id for cell in cells}) == len(cells) |
| 253 | if config.study.kind == "single": | 253 | if config.study.kind == "single": |
| 760 | 760 | ||
| 761 | 761 | ||
| 762 | def test_voxel_sizes_come_from_the_study_variants() -> None: | 762 | def test_voxel_sizes_come_from_the_study_variants() -> None: |
| 763 | """The study variants are the single source of the analysed grid sizes.""" | 763 | """The study variants are the single source of the analysed grid sizes.""" |
| 764 | from src.train.config import load_config | 764 | from src.train.config import HarnessConfig |
| 765 | 765 | ||
| 766 | module = _load_oracle_script() | 766 | module = _load_oracle_script() |
| 767 | 767 | ||
| 768 | assert module._voxel_sizes(load_config(E2_CONFIG), Path(E2_CONFIG)) == ( | 768 | assert module._voxel_sizes(HarnessConfig.from_yaml(E2_CONFIG), Path(E2_CONFIG)) == ( |
| 769 | 0.02, | 769 | 0.02, |
| 770 | 0.03, | 770 | 0.03, |
| 771 | 0.05, | 771 | 0.05, |
| 772 | ) | 772 | ) |
| 773 | 773 | ||
| 774 | 774 | ||
| 775 | def test_config_without_variant_voxel_sizes_fails_closed(tmp_path: Path) -> None: | 775 | def test_config_without_variant_voxel_sizes_fails_closed(tmp_path: Path) -> None: |
| 776 | """A config declaring no variant voxel size refuses to run E2.""" | 776 | """A config declaring no variant voxel size refuses to run E2.""" |
| 777 | from src.train.config import load_config | 777 | from src.train.config import HarnessConfig |
| 778 | 778 | ||
| 779 | module = _load_oracle_script() | 779 | module = _load_oracle_script() |
| 780 | 780 | ||
| 781 | def _drop_variants(raw: dict[str, Any]) -> None: | 781 | def _drop_variants(raw: dict[str, Any]) -> None: |
| 785 | 785 | ||
| 786 | path = _config_variant(tmp_path, _drop_variants) | 786 | path = _config_variant(tmp_path, _drop_variants) |
| 787 | 787 | ||
| 788 | with pytest.raises(VoxelOracleError, match="adapter.spt.voxel_m"): | 788 | with pytest.raises(VoxelOracleError, match="adapter.spt.voxel_m"): |
| 789 | module._voxel_sizes(load_config(path), path) | 789 | module._voxel_sizes(HarnessConfig.from_yaml(path), path) |
| 790 | 790 | ||
| 791 | 791 | ||
| 792 | def test_restated_model_voxel_sizes_must_agree(tmp_path: Path) -> None: | 792 | def test_restated_model_voxel_sizes_must_agree(tmp_path: Path) -> None: |
| 793 | """A duplicated ``model.args.voxel_sizes_m`` must match or fail closed.""" | 793 | """A duplicated ``model.args.voxel_sizes_m`` must match or fail closed.""" |
| 794 | from src.train.config import load_config | 794 | from src.train.config import HarnessConfig |
| 795 | 795 | ||
| 796 | module = _load_oracle_script() | 796 | module = _load_oracle_script() |
| 797 | 797 | ||
| 798 | def _restate(raw: dict[str, Any]) -> None: | 798 | def _restate(raw: dict[str, Any]) -> None: |
| 800 | 800 | ||
| 801 | path = _config_variant(tmp_path, _restate) | 801 | path = _config_variant(tmp_path, _restate) |
| 802 | 802 | ||
| 803 | with pytest.raises(VoxelOracleError, match="voxel_sizes_m"): | 803 | with pytest.raises(VoxelOracleError, match="voxel_sizes_m"): |
| 804 | module._voxel_sizes(load_config(path), path) | 804 | module._voxel_sizes(HarnessConfig.from_yaml(path), path) |
| 805 | 805 | ||
| 806 | 806 | ||
| 807 | def test_required_output_formats_are_written(tmp_path: Path) -> None: | 807 | def test_required_output_formats_are_written(tmp_path: Path) -> None: |
| 808 | """E2 always writes CSV, JSON, and Markdown products.""" | 808 | """E2 always writes CSV, JSON, and Markdown products.""" |
| 852 | 852 | ||
| 853 | 853 | ||
| 854 | def test_dispatched_cell_analyses_only_its_own_voxel_size() -> None: | 854 | def test_dispatched_cell_analyses_only_its_own_voxel_size() -> None: |
| 855 | """Dispatch runs E2 once per cell, so a cell must not re-run the sweep.""" | 855 | """Dispatch runs E2 once per cell, so a cell must not re-run the sweep.""" |
| 856 | from src.train.config import load_config | 856 | from src.train.config import HarnessConfig |
| 857 | 857 | ||
| 858 | module = _load_oracle_script() | 858 | module = _load_oracle_script() |
| 859 | config = load_config(E2_CONFIG) | 859 | config = HarnessConfig.from_yaml(E2_CONFIG) |
| 860 | 860 | ||
| 861 | assert module._voxel_sizes(config, Path(E2_CONFIG), cell=("voxel_2cm", 0)) == ( | 861 | assert module._voxel_sizes(config, Path(E2_CONFIG), cell=("voxel_2cm", 0)) == ( |
| 862 | 0.02, | 862 | 0.02, |
| 863 | ) | 863 | ) |
| 872 | 872 | ||
| 873 | 873 | ||
| 874 | def test_unknown_dispatched_cell_fails_closed() -> None: | 874 | def test_unknown_dispatched_cell_fails_closed() -> None: |
| 875 | """An unknown or misplaced study cell is refused by name.""" | 875 | """An unknown or misplaced study cell is refused by name.""" |
| 876 | from src.train.config import load_config | 876 | from src.train.config import HarnessConfig |
| 877 | 877 | ||
| 878 | module = _load_oracle_script() | 878 | module = _load_oracle_script() |
| 879 | config = load_config(E2_CONFIG) | 879 | config = HarnessConfig.from_yaml(E2_CONFIG) |
| 880 | 880 | ||
| 881 | with pytest.raises(VoxelOracleError, match="voxel_9cm"): | 881 | with pytest.raises(VoxelOracleError, match="voxel_9cm"): |
| 882 | module._voxel_sizes(config, Path(E2_CONFIG), cell=("voxel_9cm", 0)) | 882 | module._voxel_sizes(config, Path(E2_CONFIG), cell=("voxel_9cm", 0)) |
| 883 | with pytest.raises(VoxelOracleError, match="position 1"): | 883 | with pytest.raises(VoxelOracleError, match="position 1"): |
<Name><Section>Config), module constants, keyword-only entry points, canonical test names, README config section. No behaviour change intended.