Add independent-agent SWE prompt-ablation pilot

tests/test_swe_prerequisites.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import hashlib
+import json
+
+import pytest
+
+from messageboardbench.swe_prerequisites import (
+    validate_environment_index,
+    validate_environment_index_for_records,
+)
+from scripts.validate_swe_population_prerequisites import pull_image_once
+
+
+def write(path, value):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(value if isinstance(value, str) else json.dumps(value))
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def fixture(tmp_path):
+    output_hashes = {}
+    cells = []
+    expected = {
+        ("original", "nochange"): False,
+        ("original", "oracle"): True,
+        ("conflicting", "nochange"): False,
+        ("conflicting", "oracle"): False,
+    }
+    for split, mode in expected:
+        name = f"{split}-{mode}.txt"
+        output_hashes[name] = write(tmp_path / "evidence" / name, "test output")
+        cells.append({"split": split, "mode": mode, "resolved": expected[split, mode],
+                      "image_id": "sha256:image", "repo_digests": ["repo@sha256:digest"],
+                      "target_statuses": {"target": "PASSED" if mode == "oracle" else "FAILED"},
+                      "output_file": name, "output_sha256": output_hashes[name]})
+    record = {"instance_id": "task", "base_commit": "base", "repo": "org/repo",
+              "version": "1", "original_test_patch": "original", "test_patch": "conflict",
+              "patch": "oracle"}
+    manifest = {"schema_version": 1, "dataset": "fjzzq2002/impossible_swebench",
+                "dataset_revision": "1" * 40, "instance_id": "task", "network": "none",
+                "base_commit": "base", "repo": "org/repo", "version": "1",
+                "original_test_patch_sha256": hashlib.sha256(b"original").hexdigest(),
+                "conflicting_test_patch_sha256": hashlib.sha256(b"conflict").hexdigest(),
+                "oracle_patch_sha256": hashlib.sha256(b"oracle").hexdigest(), "results": cells}
+    manifest_path = tmp_path / "evidence" / "manifest.json"
+    manifest_hash = write(manifest_path, manifest)
+    canonical = hashlib.sha256(json.dumps(
+        record, sort_keys=True, separators=(",", ":")
+    ).encode()).hexdigest()
+    plan = {"plan_sha256": "plan", "records_sha256": {"task": canonical}, "dataset": {
+                "path": "fjzzq2002/impossible_swebench", "revision": "1" * 40,
+                "split": "conflicting"},
+            "selection": {"instance_ids": ["task"]},
+            "environment_validation": {"required_before_execution": True,
+                                       "index_path": "index.json"}}
+    index = {"schema_version": 1, "status": "validated", "plan_sha256": "plan",
+             "dataset": plan["dataset"], "manifests": {
+                 "task": {"path": "evidence/manifest.json", "sha256": manifest_hash}}}
+    write(tmp_path / "index.json", index)
+    return plan, manifest_path, record
+
+
+def test_environment_index_requires_complete_nonmissing_hashed_evidence(tmp_path):
+    plan, manifest_path, record = fixture(tmp_path)
+    result = validate_environment_index_for_records(plan, tmp_path, {"task": record})
+    assert len(result["validated_instances"]) == 1
+    manifest = json.loads(manifest_path.read_text())
+    manifest["results"][0]["target_statuses"] = {"target": "MISSING"}
+    write(manifest_path, manifest)
+    index_path = tmp_path / "index.json"
+    index = json.loads(index_path.read_text())
+    index["manifests"]["task"]["sha256"] = hashlib.sha256(manifest_path.read_bytes()).hexdigest()
+    write(index_path, index)
+    with pytest.raises(ValueError, match="missing/error targets"):
+        validate_environment_index(plan, tmp_path)
+
+
+def test_environment_index_is_required_and_plan_bound(tmp_path):
+    plan, _, _ = fixture(tmp_path)
+    plan["plan_sha256"] = "different"
+    with pytest.raises(ValueError, match="does not match"):
+        validate_environment_index(plan, tmp_path)
+
+
+def test_environment_manifest_patch_hashes_are_bound_to_frozen_record(tmp_path):
+    plan, _, record = fixture(tmp_path)
+    changed = {**record, "test_patch": "different"}
+    with pytest.raises(ValueError, match="frozen record"):
+        validate_environment_index_for_records(plan, tmp_path, {"task": changed})
+
+
+def test_image_is_pulled_once_before_any_inspection_or_trial():
+    calls = []
+
+    def run(argv, **kwargs):
+        calls.append(argv)
+        return __import__("subprocess").CompletedProcess(argv, 0, "pulled", "")
+
+    pulled = set()
+    pull_image_once("image:tag", pulled, {"DOCKER_HOST": "remote"}, run=run)
+    calls.append(["docker", "image", "inspect", "image:tag"])
+    pull_image_once("image:tag", pulled, {"DOCKER_HOST": "remote"}, run=run)
+    assert calls == [
+        ["docker", "pull", "image:tag"],
+        ["docker", "image", "inspect", "image:tag"],
+    ]