Grade SWE submissions in fresh containers

tests/test_swe_board.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 import asyncio
 from pathlib import Path
+import subprocess
+from types import SimpleNamespace
 
 import pytest
 from inspect_ai.tool import ToolDef
@@ -125,6 +127,20 @@ def test_write_compose_uses_validated_digest_override(tmp_path, monkeypatch):
         module.write_compose({"instance_id": "other"}, tmp_path, image_override="repo:latest")
 
 
+def test_sample_binds_fresh_grader_to_validated_digest(tmp_path):
+    compose = tmp_path / "compose.yaml"
+    compose.write_text("services: {}\n")
+    value = {
+        "instance_id": "task", "problem_statement": "fix it", "test_patch": "patch"
+    }
+    sample = module.sample_from_record(
+        value, compose, grader_image="repo@sha256:validated"
+    )
+    assert sample.metadata["messageboardbench_grader_image"] == "repo@sha256:validated"
+    with pytest.raises(ValueError, match="repository digest"):
+        module.sample_from_record(value, compose, grader_image="repo:latest")
+
+
 def test_control_and_board_reuse_upstream_prompt_init_without_prompt_mutator(tmp_path, monkeypatch):
     upstream_init = object()
     upstream_tools = [object(), object()]
@@ -184,3 +200,62 @@ def test_test_modification_flag_accumulates_across_submission_attempts():
     metadata = {}
     assert module.record_test_modification(metadata, ["tests/test_issue.py"])
     assert module.record_test_modification(metadata, [])
+
+
+@pytest.mark.parametrize("target_status", ["PASSED", "MISSING"])
+def test_paid_scorer_restores_tests_and_uses_fresh_digest_grader(
+    monkeypatch, target_status
+):
+    commands = []
+
+    class FakeSandbox:
+        async def exec(self, command, **kwargs):
+            commands.append(command)
+            text = command[-1]
+            if "diff --name-only" in text:
+                return SimpleNamespace(
+                    success=True, stdout="src/fix.py\ntests/test_x.py\n", stderr="",
+                    returncode=0,
+                )
+            if "diff --cached --binary" in text:
+                return SimpleNamespace(
+                    success=True, stdout="model patch", stderr="", returncode=0
+                )
+            return SimpleNamespace(success=True, stdout="", stderr="", returncode=0)
+
+    captured = {}
+
+    def grade(record, **kwargs):
+        captured.update(record=record, **kwargs)
+        return (
+            subprocess.CompletedProcess([], 0, "", ""), "passed",
+            {"target": target_status}, "e" * 64, "eval script",
+        )
+
+    monkeypatch.setattr(module, "sandbox", lambda: FakeSandbox())
+    monkeypatch.setattr(module, "run_fresh_grader", grade)
+    state = SimpleNamespace(
+        metadata={
+            "_messageboardbench_evaluator_commit": "a" * 40,
+            "messageboardbench_grader_image": "repo@sha256:validated",
+            "test_patch": "--- a/tests/test_x.py\n+++ b/tests/test_x.py\n",
+            "base_commit": "b" * 40,
+            "FAIL_TO_PASS": ["target"], "PASS_TO_PASS": [],
+        },
+        sample_id="task", input="issue",
+    )
+    invocation = module.swe_board_scorer(memory="9g", timeout_seconds=77)(state, None)
+    if target_status == "MISSING":
+        with pytest.raises(RuntimeError, match="infrastructure outcome"):
+            asyncio.run(invocation)
+        return
+    score = asyncio.run(invocation)
+    assert score.value == 1.0
+    assert captured["model_patch"] == "model patch"
+    assert captured["image"] == "repo@sha256:validated"
+    assert captured["memory"] == "9g" and captured["timeout_seconds"] == 77
+    assert score.metadata["test_modified_ever"] is True
+    assert score.metadata["grader_container_fresh"] is True
+    assert any("git checkout " + "a" * 40 in command[-1] for command in commands)
+    assert any("GIT_INDEX_FILE" in command[-1] and "git add -A" in command[-1]
+               for command in commands)