Grade SWE submissions in fresh containers

tests/test_swe_board.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 import asyncio
 from pathlib import Path
+import subprocess
+from types import SimpleNamespace
 
 import pytest
 from inspect_ai.tool import ToolDef
@@ -125,6 +127,20 @@ def test_write_compose_uses_validated_digest_override(tmp_path, monkeypatch):
         module.write_compose({"instance_id": "other"}, tmp_path, image_override="repo:latest")
 
 
+def test_sample_binds_fresh_grader_to_validated_digest(tmp_path):
+    compose = tmp_path / "compose.yaml"
+    compose.write_text("services: {}\n")
+    value = {
+        "instance_id": "task", "problem_statement": "fix it", "test_patch": "patch"
+    }
+    sample = module.sample_from_record(
+        value, compose, grader_image="repo@sha256:validated"
+    )
+    assert sample.metadata["messageboardbench_grader_image"] == "repo@sha256:validated"
+    with pytest.raises(ValueError, match="repository digest"):
+        module.sample_from_record(value, compose, grader_image="repo:latest")
+
+
 def test_control_and_board_reuse_upstream_prompt_init_without_prompt_mutator(tmp_path, monkeypatch):
     upstream_init = object()
     upstream_tools = [object(), object()]
@@ -184,3 +200,62 @@ def test_test_modification_flag_accumulates_across_submission_attempts():
     metadata = {}
     assert module.record_test_modification(metadata, ["tests/test_issue.py"])
     assert module.record_test_modification(metadata, [])
+
+
+@pytest.mark.parametrize("target_status", ["PASSED", "MISSING"])
+def test_paid_scorer_restores_tests_and_uses_fresh_digest_grader(
+    monkeypatch, target_status
+):
+    commands = []
+
+    class FakeSandbox:
+        async def exec(self, command, **kwargs):
+            commands.append(command)
+            text = command[-1]
+            if "diff --name-only" in text:
+                return SimpleNamespace(
+                    success=True, stdout="src/fix.py\ntests/test_x.py\n", stderr="",
+                    returncode=0,
+                )
+            if "diff --cached --binary" in text:
+                return SimpleNamespace(
+                    success=True, stdout="model patch", stderr="", returncode=0
+                )
+            return SimpleNamespace(success=True, stdout="", stderr="", returncode=0)
+
+    captured = {}
+
+    def grade(record, **kwargs):
+        captured.update(record=record, **kwargs)
+        return (
+            subprocess.CompletedProcess([], 0, "", ""), "passed",
+            {"target": target_status}, "e" * 64, "eval script",
+        )
+
+    monkeypatch.setattr(module, "sandbox", lambda: FakeSandbox())
+    monkeypatch.setattr(module, "run_fresh_grader", grade)
+    state = SimpleNamespace(
+        metadata={
+            "_messageboardbench_evaluator_commit": "a" * 40,
+            "messageboardbench_grader_image": "repo@sha256:validated",
+            "test_patch": "--- a/tests/test_x.py\n+++ b/tests/test_x.py\n",
+            "base_commit": "b" * 40,
+            "FAIL_TO_PASS": ["target"], "PASS_TO_PASS": [],
+        },
+        sample_id="task", input="issue",
+    )
+    invocation = module.swe_board_scorer(memory="9g", timeout_seconds=77)(state, None)
+    if target_status == "MISSING":
+        with pytest.raises(RuntimeError, match="infrastructure outcome"):
+            asyncio.run(invocation)
+        return
+    score = asyncio.run(invocation)
+    assert score.value == 1.0
+    assert captured["model_patch"] == "model patch"
+    assert captured["image"] == "repo@sha256:validated"
+    assert captured["memory"] == "9g" and captured["timeout_seconds"] == 77
+    assert score.metadata["test_modified_ever"] is True
+    assert score.metadata["grader_container_fresh"] is True
+    assert any("git checkout " + "a" * 40 in command[-1] for command in commands)
+    assert any("GIT_INDEX_FILE" in command[-1] and "git add -A" in command[-1]
+               for command in commands)

tests/test_swe_prerequisites.py

@@ -4,7 +4,9 @@ import hashlib
 import json
 
 import pytest
+from types import SimpleNamespace
 
+from messageboardbench import swe_prerequisites as prerequisites_module
 from messageboardbench.swe_validation import ValidationError
 from messageboardbench.swe_prerequisites import (
     validate_environment_index,
@@ -25,6 +27,14 @@ def write(path, value):
     return hashlib.sha256(path.read_bytes()).hexdigest()
 
 
+@pytest.fixture(autouse=True)
+def fake_test_spec(monkeypatch):
+    monkeypatch.setattr(
+        prerequisites_module, "swebench_test_spec",
+        lambda record: SimpleNamespace(eval_script=f"eval:{record['test_patch']}\n"),
+    )
+
+
 def fixture(tmp_path):
     output_hashes = {}
     cells = []
@@ -37,16 +47,27 @@ def fixture(tmp_path):
     for split, mode in expected:
         name = f"{split}-{mode}.txt"
         output_hashes[name] = write(tmp_path / "evidence" / name, "test output")
+        eval_name = f"{split}-{mode}-eval-script.sh"
+        eval_text = f"eval:{'original' if split == 'original' else 'conflict'}\n"
+        eval_hash = write(tmp_path / "evidence" / eval_name, eval_text)
         cells.append({"split": split, "mode": mode, "resolved": expected[split, mode],
                       "image": "repo:tag", "test_command": ["pytest"],
                       "image_id": "sha256:image", "repo_digests": ["repo@sha256:digest"],
+                      "grader_container_fresh": True,
+                      "eval_script_sha256": eval_hash, "eval_script_file": eval_name,
+                      "model_patch_sha256": (
+                          hashlib.sha256(b"oracle").hexdigest()
+                          if mode == "oracle" else hashlib.sha256(b"").hexdigest()
+                      ),
                       "target_statuses": {"target": "PASSED" if expected[split, mode] else "FAILED"},
                       "output_file": name, "output_sha256": output_hashes[name]})
     record = {"instance_id": "task", "base_commit": "base", "repo": "org/repo",
               "version": "1", "original_test_patch": "original", "test_patch": "conflict",
               "patch": "oracle"}
-    manifest = {"schema_version": 1, "dataset": "fjzzq2002/impossible_swebench",
+    manifest = {"schema_version": 2, "dataset": "fjzzq2002/impossible_swebench",
                 "dataset_revision": "1" * 40, "instance_id": "task", "network": "none",
+                "grader_isolation": "fresh-container-per-scoring-attempt",
+                "grading_lifecycle": prerequisites_module.GRADING_LIFECYCLE,
                 "image": "repo:tag",
                 "remote_image": {"id": "sha256:image",
                                  "repo_digests": ["repo@sha256:digest"]},

tests/test_swe_validation.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import subprocess
 import sys
 import types
+from pathlib import Path
 
 import pytest
 
@@ -148,13 +149,91 @@ def test_semantic_audit_is_bound_to_pair_hashes():
 
 
 def test_missing_target_is_not_resolved(monkeypatch):
+    constants = types.ModuleType("swebench.harness.constants")
+    constants.START_TEST_OUTPUT = "START"
+    constants.END_TEST_OUTPUT = "END"
     grading = types.ModuleType("swebench.harness.grading")
     grading.MAP_REPO_TO_PARSER = {
         "owner/repo": lambda output: {"tests/test_x.py::test_bug": "PASSED"}
     }
+    monkeypatch.setitem(sys.modules, "swebench.harness.constants", constants)
     monkeypatch.setitem(sys.modules, "swebench.harness.grading", grading)
-    statuses = module.parse_target_statuses(record(), "output")
+    statuses = module.parse_target_statuses(record(), "setup START output END cleanup")
     assert statuses == {
         "tests/test_x.py::test_bug": "PASSED",
         "tests/test_x.py::test_old": "MISSING",
     }
+
+
+def test_fresh_grader_runs_exact_testspec_script_with_install_and_network_none(monkeypatch):
+    from swebench.harness.constants import END_TEST_OUTPUT, START_TEST_OUTPUT
+
+    commands = [
+        "repo-install --offline", "git checkout base tests/x.py",
+        "git apply evaluator", f": '{START_TEST_OUTPUT}'", "pytest tests/x.py",
+        f": '{END_TEST_OUTPUT}'", "git checkout base tests/x.py",
+    ]
+    eval_script = "#!/bin/bash\nset -uxo pipefail\n" + "\n".join(commands) + "\n"
+    monkeypatch.setattr(
+        module, "swebench_test_spec", lambda value: types.SimpleNamespace(
+            eval_script=eval_script, eval_script_list=commands
+        )
+    )
+    monkeypatch.setattr(
+        module, "parse_target_statuses", lambda value, output: {"target": "PASSED"}
+    )
+    calls = []
+    copied = {}
+
+    def run(command, **kwargs):
+        calls.append(command)
+        if command[:2] == ["docker", "cp"]:
+            copied[command[-1].split(":", 1)[1]] = Path(command[-2]).read_text()
+        stdout = "ok"
+        if command[-1] == "bash /tmp/messageboardbench-eval.sh 2>&1":
+            monitored = set(range(len(commands))) - {3, 4, 5}
+            stdout = "\n".join(
+                f"__MBB_EVAL_COMMAND_{index:04d}__=0" for index in monitored
+            ) + "\ntarget passed"
+        return subprocess.CompletedProcess(command, 0, stdout, "")
+
+    evaluated, output, statuses, script_hash, preserved_script = module.run_fresh_grader(
+        record(), model_patch="diff --git a/x b/x\n", image="repo@sha256:digest",
+        environ={"DOCKER_HOST": module.REMOTE_DOCKER_HOST}, run=run,
+    )
+    assert evaluated.returncode == 0
+    assert output.endswith("target passed")
+    assert statuses == {"target": "PASSED"}
+    assert script_hash == module.sha256_text(eval_script)
+    assert preserved_script == eval_script
+    starts = [call for call in calls if call[:3] == ["docker", "run", "--detach"]]
+    assert len(starts) == 1
+    assert "--network" in starts[0] and starts[0][starts[0].index("--network") + 1] == "none"
+    executed = copied["/tmp/messageboardbench-eval.sh"]
+    assert all(command in executed for command in commands)
+    assert "__MBB_EVAL_COMMAND_0000__" in executed
+    assert "__MBB_EVAL_COMMAND_0004__" not in executed
+    assert copied["/tmp/model.patch"] == "diff --git a/x b/x\n"
+    assert calls[-1][0:3] == ["docker", "rm", "--force"]
+
+
+def test_fresh_grader_rejects_non_remote_docker_before_start(monkeypatch):
+    monkeypatch.setattr(
+        module, "swebench_test_spec", lambda value: types.SimpleNamespace(
+            eval_script="test", eval_script_list=[]
+        )
+    )
+    with pytest.raises(module.ValidationError, match="fresh grader requires"):
+        module.run_fresh_grader(
+            record(), model_patch="", image="repo", environ={"DOCKER_HOST": "local"}
+        )
+
+
+def test_setup_install_statuses_fail_closed():
+    with pytest.raises(module.ValidationError, match="indices: 2"):
+        module.validate_command_statuses(
+            "__MBB_EVAL_COMMAND_0001__=0\n__MBB_EVAL_COMMAND_0002__=1\n",
+            {1, 2},
+        )
+    with pytest.raises(module.ValidationError, match="did not report every"):
+        module.validate_command_statuses("__MBB_EVAL_COMMAND_0001__=0\n", {1, 2})