Fail fast on invalid SWE prerequisite tasks

scripts/validate_swe_population_prerequisites.py

@@ -9,15 +9,18 @@ from pathlib import Path
 import subprocess
 
 from messageboardbench.swe_board import load_records, plan_hash
-from messageboardbench.swe_prerequisites import validate_environment_index_for_records
+from messageboardbench.swe_prerequisites import (
+    validate_environment_index_for_records,
+    validate_task_manifest,
+)
 from messageboardbench.swe_validation import (
     ValidationError,
     docker_preflight,
-    load_pair,
     manifest as trial_manifest,
     run_trial,
     swebench_spec,
     validate_expected_matrix,
+    validate_pair,
 )
 
 
@@ -40,6 +43,51 @@ def pull_image_once(image: str, pulled: set[str], environ, run=subprocess.run) -
     pulled.add(image)
 
 
+def load_selected_pairs(plan, loader=None):
+    """Load each pinned dataset split once, then validate all selected pairs."""
+    loader = load_records if loader is None else loader
+    revision = plan["dataset"]["revision"]
+    original_records = loader(revision, "original")
+    conflicting_records = loader(revision, "conflicting")
+    pairs = {}
+    for instance_id in plan["selection"]["instance_ids"]:
+        try:
+            original = original_records[instance_id]
+            conflicting = conflicting_records[instance_id]
+        except KeyError as exc:
+            raise ValidationError(
+                f"selected task is absent from a pinned dataset split: {instance_id}"
+            ) from exc
+        validate_pair(original, conflicting)
+        pairs[instance_id] = (original, conflicting)
+    return pairs, conflicting_records
+
+
+def require_resolved_targets(instance_id: str, result) -> None:
+    """Reject an unusable validation cell before another cell is run."""
+    statuses = result.target_statuses
+    if not statuses or any(status in {"MISSING", "ERROR"} for status in statuses.values()):
+        raise ValidationError(
+            f"validation contains missing/error targets: {instance_id} "
+            f"{result.split}/{result.mode}"
+        )
+
+
+def validate_existing_manifests(plan, out: Path, conflicting_records) -> None:
+    """Fail on the first invalid resume artifact before Docker work continues."""
+    selected = plan["selection"]["instance_ids"]
+    for position, instance_id in enumerate(selected, 1):
+        manifest_path = out / instance_id.replace("/", "_") / "manifest.json"
+        if manifest_path.exists():
+            print(
+                f"[{position}/{len(selected)}] {instance_id}: validating existing manifest",
+                flush=True,
+            )
+            validate_task_manifest(
+                plan, instance_id, manifest_path, conflicting_records[instance_id]
+            )
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--plan", type=Path, required=True)
@@ -52,30 +100,39 @@ def main() -> int:
     out = args.out.resolve()
     if declared != out / "index.json":
         raise SystemExit("--out does not match the frozen validation index location")
+    pairs, conflicting_records = load_selected_pairs(plan)
     if declared.is_file():
-        records = load_records(plan["dataset"]["revision"], "conflicting")
-        result = validate_environment_index_for_records(plan, ROOT, records)
+        result = validate_environment_index_for_records(plan, ROOT, conflicting_records)
         print(json.dumps({"status": "already validated", **result}, indent=2))
         return 0
 
+    validate_existing_manifests(plan, out, conflicting_records)
     docker_preflight(os.environ)
     out.mkdir(parents=True, exist_ok=True)
     entries = {}
     pulled_images: set[str] = set()
-    for instance_id in plan["selection"]["instance_ids"]:
+    selected = plan["selection"]["instance_ids"]
+    for position, instance_id in enumerate(selected, 1):
+        prefix = f"[{position}/{len(selected)}] {instance_id}"
         task_dir = out / instance_id.replace("/", "_")
         manifest_path = task_dir / "manifest.json"
-        if not manifest_path.exists():
-            original, conflicting = load_pair(plan["dataset"]["revision"], instance_id)
+        original, conflicting = pairs[instance_id]
+        if manifest_path.exists():
+            print(f"{prefix}: reusing validated manifest", flush=True)
+        else:
+            print(f"{prefix}: pulling exact image", flush=True)
             pull_image_once(swebench_spec(original)[0], pulled_images, os.environ)
-            results = [
-                run_trial(record, split=split, mode=mode, out_dir=task_dir,
-                          environ=os.environ,
-                          memory=plan["parameters"]["memory"],
-                          timeout_seconds=plan["parameters"]["scorer_timeout_seconds"])
-                for split, record in (("original", original), ("conflicting", conflicting))
-                for mode in ("nochange", "oracle")
-            ]
+            results = []
+            for split, record in (("original", original), ("conflicting", conflicting)):
+                for mode in ("nochange", "oracle"):
+                    print(f"{prefix}: running {split}/{mode}", flush=True)
+                    result = run_trial(
+                        record, split=split, mode=mode, out_dir=task_dir,
+                        environ=os.environ, memory=plan["parameters"]["memory"],
+                        timeout_seconds=plan["parameters"]["scorer_timeout_seconds"],
+                    )
+                    require_resolved_targets(instance_id, result)
+                    results.append(result)
             validate_expected_matrix(results)
             value = trial_manifest(
                 plan["dataset"]["revision"], instance_id, original, conflicting, results
@@ -83,6 +140,7 @@ def main() -> int:
             with manifest_path.open("x") as handle:
                 json.dump(value, handle, indent=2, sort_keys=True)
                 handle.write("\n")
+            print(f"{prefix}: validation passed", flush=True)
         entries[instance_id] = {
             "path": str(manifest_path.relative_to(ROOT)), "sha256": sha(manifest_path)
         }
@@ -92,8 +150,7 @@ def main() -> int:
     with declared.open("x") as handle:
         json.dump(index, handle, indent=2, sort_keys=True)
         handle.write("\n")
-    records = load_records(plan["dataset"]["revision"], "conflicting")
-    result = validate_environment_index_for_records(plan, ROOT, records)
+    result = validate_environment_index_for_records(plan, ROOT, conflicting_records)
     print(json.dumps({"status": "validated", **result}, indent=2))
     return 0