Support immutable local SWE image IDs

scripts/analysis/verify_swe_population.py

@@ -90,7 +90,8 @@ def environment_validation_matches_plan(
             if (
                 set(row) != {
                     "instance_id", "manifest_path", "manifest_sha256",
-                    "validated_image", "validated_image_id", "validated_repo_digest"
+                    "validated_image", "validated_image_id", "validated_image_ref",
+                    "validated_repo_digest",
                 }
                 or not Path(row["manifest_path"]).as_posix().endswith(
                     "/" + Path(entry["path"]).as_posix()
@@ -98,7 +99,11 @@ def environment_validation_matches_plan(
                 or row["manifest_sha256"] != entry["sha256"]
                 or row["validated_image"] != validated["image"]
                 or row["validated_image_id"] != remote_image["id"]
-                or row["validated_repo_digest"] != remote_image["repo_digests"][0]
+                or row["validated_image_ref"] != remote_image["immutable_ref"]
+                or row["validated_repo_digest"] != (
+                    remote_image["repo_digests"][0]
+                    if remote_image["repo_digests"] else None
+                )
             ):
                 return False
     except (KeyError, OSError, ValueError, json.JSONDecodeError):

scripts/swe_board_experiment.py

@@ -105,7 +105,6 @@ def recover_terminal_rows(out: Path) -> list[dict]:
 
 def cleanup_matched_images(
     out: Path, team: int, cohort: int, instance_ids: list[str], records: dict,
-    validated_images: dict[str, str] | None = None,
 ) -> None:
     """Remove only explicit, re-pullable tags after both matched arms terminate."""
     path = out / "image-lifecycle.json"
@@ -117,7 +116,9 @@ def cleanup_matched_images(
     record = {"team": team, "cohort": cohort, "images": []}
     failed = False
     for instance_id in instance_ids:
-        image = (validated_images or {}).get(instance_id) or swebench_spec(records[instance_id])[0]
+        # Remove the mutable local tag, not an immutable ID/digest that may still
+        # have another tag reference. Identity was already frozen before execution.
+        image = swebench_spec(records[instance_id])[0]
         inspected = subprocess.run(
             ["docker", "image", "inspect", image, "--format", "{{json .}}"],
             capture_output=True, text=True, env=os.environ,
@@ -247,7 +248,7 @@ def main(argv: list[str] | None = None) -> int:
     team_plans = plan["team_plans"]
     configs = out / "compose"
     validated_images = {
-        row["instance_id"]: row["validated_repo_digest"]
+        row["instance_id"]: row["validated_image_ref"]
         for row in (environment_validation or {}).get("validated_instances", [])
     }
     compose_by_assignment = {
@@ -366,7 +367,7 @@ def main(argv: list[str] | None = None) -> int:
                 if all((team, arm, instance_id) in terminal
                        for arm in CONDITIONS for instance_id in selected):
                     cleanup_matched_images(
-                        out, team, cohort, selected, records, validated_images
+                        out, team, cohort, selected, records
                     )
                 continue
             tasks = []
@@ -460,7 +461,7 @@ def main(argv: list[str] | None = None) -> int:
             )
             if matched_complete:
                 cleanup_matched_images(
-                    out, team, cohort, selected, records, validated_images
+                    out, team, cohort, selected, records
                 )
         status["status"] = "completed"
     except BaseException as exc:

src/messageboardbench/swe_board.py

@@ -26,8 +26,8 @@ from inspect_ai.util import SandboxEnvironmentSpec, sandbox
 from messageboardbench.board import MESSAGEBOARD_V2_INTERFACE_VERSION, board_tools
 from messageboardbench.feedback import feedback_tool
 from messageboardbench.swe_validation import (
-    DATASET, GRADING_LIFECYCLE, normalize_record, patch_files, require_revision,
-    run_fresh_grader, swebench_spec,
+    DATASET, GRADING_LIFECYCLE, is_immutable_image_reference, normalize_record,
+    patch_files, require_revision, run_fresh_grader, swebench_spec,
 )
 
 
@@ -345,8 +345,8 @@ def write_compose(
 ) -> Path:
     image, _, _ = swebench_spec(record)
     if image_override is not None:
-        if "@sha256:" not in image_override:
-            raise ValueError("validated image override must be a repository digest")
+        if not is_immutable_image_reference(image_override):
+            raise ValueError("validated image override must be an immutable image reference")
         image = image_override
     directory.mkdir(parents=True, exist_ok=True)
     path = directory / (str(record["instance_id"]).replace("/", "_") + ".yaml")
@@ -367,8 +367,8 @@ def sample_from_record(
     # other branch.  Make the intended upstream choice explicit.
     metadata["impossible_patch"] = ""
     if grader_image is not None:
-        if "@sha256:" not in grader_image:
-            raise ValueError("fresh grader image must be a repository digest")
+        if not is_immutable_image_reference(grader_image):
+            raise ValueError("fresh grader image must be an immutable image reference")
         metadata["messageboardbench_grader_image"] = grader_image
     return Sample(
         id=str(row["instance_id"]),
@@ -568,8 +568,9 @@ def swe_board_scorer(*, memory: str = "8g", timeout_seconds: int = 600):
             "problem_statement": state.input,
         }
         grader_image = state.metadata.get("messageboardbench_grader_image")
-        if not isinstance(grader_image, str) or "@sha256:" not in grader_image:
-            raise RuntimeError("missing validated repository digest for fresh grader")
+        if (not isinstance(grader_image, str)
+                or not is_immutable_image_reference(grader_image)):
+            raise RuntimeError("missing validated immutable image reference for fresh grader")
         evaluated, output, statuses, eval_script_sha256, _ = await asyncio.to_thread(
             run_fresh_grader,
             record,

src/messageboardbench/swe_prerequisites.py

@@ -65,7 +65,10 @@ def validate_environment_index_for_records(
             "manifest_sha256": entry["sha256"],
             "validated_image": manifest["image"],
             "validated_image_id": remote_image["id"],
-            "validated_repo_digest": remote_image["repo_digests"][0],
+            "validated_image_ref": remote_image["immutable_ref"],
+            "validated_repo_digest": (
+                remote_image["repo_digests"][0] if remote_image["repo_digests"] else None
+            ),
         })
     return {"index_path": str(index_path), "index_sha256": _sha(index_path),
             "validated_instances": evidence}
@@ -187,9 +190,13 @@ def validate_task_manifest(
     if len(commands) != 1 or list(next(iter(commands))) != manifest.get("test_command"):
         raise ValueError(f"validation test command mismatch: {instance_id}")
     image_id, repo_digests = next(iter(identities))
-    if manifest.get("remote_image") != {
-        "id": image_id, "repo_digests": list(repo_digests)
-    } or not repo_digests:
+    from messageboardbench.swe_validation import immutable_image_reference
+    expected_remote_image = {
+        "id": image_id,
+        "repo_digests": list(repo_digests),
+        "immutable_ref": immutable_image_reference(image_id, repo_digests),
+    }
+    if manifest.get("remote_image") != expected_remote_image:
         raise ValueError(f"validation remote image mismatch: {instance_id}")
     for row in results:
         output = manifest_path.parent / str(row.get("output_file", ""))

src/messageboardbench/swe_validation.py

@@ -19,6 +19,8 @@ DATASET = "fjzzq2002/impossible_swebench"
 REMOTE_DOCKER_HOST = "ssh://[email protected]"
 GRADING_LIFECYCLE = "upstream-testspec-eval-commands-status-monitored-v2"
 FULL_SHA = re.compile(r"[0-9a-f]{40}\Z")
+IMAGE_ID = re.compile(r"sha256:[0-9a-f]{64}\Z")
+REPO_DIGEST = re.compile(r"[^\s@]+@sha256:[0-9a-f]{64}\Z")
 PATCH_PATH_RE = re.compile(r"^(?:--- a/|\+\+\+ b/)(.+)$", re.MULTILINE)
 
 
@@ -207,14 +209,28 @@ def image_identity(
         platform = f"{value['Os']}/{value['Architecture']}"
     except (KeyError, TypeError, json.JSONDecodeError) as exc:
         raise ValidationError("Docker returned an invalid image identity") from exc
-    if platform != "linux/amd64" or not image_id or not repo_digests:
+    if (platform != "linux/amd64" or not IMAGE_ID.fullmatch(str(image_id))
+            or any(not REPO_DIGEST.fullmatch(str(value)) for value in repo_digests)):
         raise ValidationError(
-            f"image must be linux/amd64 with an ID and repository digest; got {platform}, "
+            f"image must be linux/amd64 with a content-addressed ID; got {platform}, "
             f"id={image_id!r}, digests={repo_digests!r}"
         )
     return image_id, repo_digests
 
 
+def immutable_image_reference(image_id: str, repo_digests: Sequence[str]) -> str:
+    """Prefer a registry digest, falling back to Docker's content-addressed image ID."""
+    if not IMAGE_ID.fullmatch(image_id):
+        raise ValidationError(f"invalid Docker image ID: {image_id!r}")
+    if any(not REPO_DIGEST.fullmatch(value) for value in repo_digests):
+        raise ValidationError("invalid Docker repository digest")
+    return sorted(repo_digests)[0] if repo_digests else image_id
+
+
+def is_immutable_image_reference(value: str) -> bool:
+    return bool(IMAGE_ID.fullmatch(value) or REPO_DIGEST.fullmatch(value))
+
+
 def _docker(
     args: Sequence[str], environ: Mapping[str, str], run: Runner = subprocess.run, **kwargs: Any
 ) -> subprocess.CompletedProcess[str]:
@@ -286,8 +302,8 @@ def run_fresh_grader(
     """
     if environ.get("DOCKER_HOST") != REMOTE_DOCKER_HOST:
         raise ValidationError(f"fresh grader requires DOCKER_HOST={REMOTE_DOCKER_HOST}")
-    if "@sha256:" not in image:
-        raise ValidationError("fresh grader image must be an inspected repository digest")
+    if not is_immutable_image_reference(image):
+        raise ValidationError("fresh grader image must be an inspected immutable reference")
     spec = swebench_test_spec(record)
     eval_script = spec.eval_script
     executed_script, monitored_indices = instrument_eval_script(spec.eval_script_list)
@@ -377,7 +393,9 @@ def run_trial(
     output_path = out_dir / f"{split}-{mode}.txt"
     model_patch = str(record["patch"]) if mode == "oracle" else ""
     tested, combined, statuses, eval_script_sha256, eval_script = run_fresh_grader(
-        record, model_patch=model_patch, image=repo_digests[0], environ=environ, run=run,
+        record, model_patch=model_patch,
+        image=immutable_image_reference(image_id, repo_digests),
+        environ=environ, run=run,
         memory=memory, timeout_seconds=timeout_seconds,
     )
     output_path.write_text(combined)
@@ -459,7 +477,11 @@ def manifest(
         if len(identities) != 1:
             raise ValidationError("manifest cannot record divergent remote image identities")
         image_id, repo_digests = next(iter(identities))
-        remote_image = {"id": image_id, "repo_digests": list(repo_digests)}
+        remote_image = {
+            "id": image_id,
+            "repo_digests": list(repo_digests),
+            "immutable_ref": immutable_image_reference(image_id, repo_digests),
+        }
     return {
         "schema_version": 2,
         "dataset": DATASET,

tests/test_swe_board.py

@@ -13,6 +13,10 @@ from messageboardbench.board import MESSAGEBOARD_V2_INTERFACE_VERSION, initializ
 from messageboardbench.feedback import initialize_feedback
 
 
+IMAGE_ID = "sha256:" + "a" * 64
+REPO_DIGEST = "repo@sha256:" + "d" * 64
+
+
 def records(count=349):
     return {f"owner__repo-{index:03d}": {"instance_id": f"owner__repo-{index:03d}",
                                          "value": index}
@@ -119,11 +123,11 @@ def test_compose_has_no_mount_and_network_none():
 def test_write_compose_uses_validated_digest_override(tmp_path, monkeypatch):
     monkeypatch.setattr(module, "swebench_spec", lambda record: ("repo:latest", [], "pytest"))
     path = module.write_compose(
-        {"instance_id": "task"}, tmp_path, image_override="repo@sha256:validated"
+        {"instance_id": "task"}, tmp_path, image_override=REPO_DIGEST
     )
-    assert "repo@sha256:validated" in path.read_text()
+    assert REPO_DIGEST in path.read_text()
     assert "repo:latest" not in path.read_text()
-    with pytest.raises(ValueError, match="repository digest"):
+    with pytest.raises(ValueError, match="immutable image reference"):
         module.write_compose({"instance_id": "other"}, tmp_path, image_override="repo:latest")
 
 
@@ -134,10 +138,10 @@ def test_sample_binds_fresh_grader_to_validated_digest(tmp_path):
         "instance_id": "task", "problem_statement": "fix it", "test_patch": "patch"
     }
     sample = module.sample_from_record(
-        value, compose, grader_image="repo@sha256:validated"
+        value, compose, grader_image=IMAGE_ID
     )
-    assert sample.metadata["messageboardbench_grader_image"] == "repo@sha256:validated"
-    with pytest.raises(ValueError, match="repository digest"):
+    assert sample.metadata["messageboardbench_grader_image"] == IMAGE_ID
+    with pytest.raises(ValueError, match="immutable image reference"):
         module.sample_from_record(value, compose, grader_image="repo:latest")
 
 
@@ -237,7 +241,7 @@ def test_paid_scorer_restores_tests_and_uses_fresh_digest_grader(
     state = SimpleNamespace(
         metadata={
             "_messageboardbench_evaluator_commit": "a" * 40,
-            "messageboardbench_grader_image": "repo@sha256:validated",
+            "messageboardbench_grader_image": IMAGE_ID,
             "test_patch": "--- a/tests/test_x.py\n+++ b/tests/test_x.py\n",
             "base_commit": "b" * 40,
             "FAIL_TO_PASS": ["target"], "PASS_TO_PASS": [],
@@ -252,7 +256,7 @@ def test_paid_scorer_restores_tests_and_uses_fresh_digest_grader(
     score = asyncio.run(invocation)
     assert score.value == 1.0
     assert captured["model_patch"] == "model patch"
-    assert captured["image"] == "repo@sha256:validated"
+    assert captured["image"] == IMAGE_ID
     assert captured["memory"] == "9g" and captured["timeout_seconds"] == 77
     assert score.metadata["test_modified_ever"] is True
     assert score.metadata["grader_container_fresh"] is True

tests/test_swe_prerequisites.py

@@ -21,6 +21,10 @@ from scripts.validate_swe_population_prerequisites import (
 )
 
 
+IMAGE_ID = "sha256:" + "a" * 64
+REPO_DIGEST = "repo@sha256:" + "d" * 64
+
+
 def write(path, value):
     path.parent.mkdir(parents=True, exist_ok=True)
     path.write_text(value if isinstance(value, str) else json.dumps(value))
@@ -52,7 +56,7 @@ def fixture(tmp_path):
         eval_hash = write(tmp_path / "evidence" / eval_name, eval_text)
         cells.append({"split": split, "mode": mode, "resolved": expected[split, mode],
                       "image": "repo:tag", "test_command": ["pytest"],
-                      "image_id": "sha256:image", "repo_digests": ["repo@sha256:digest"],
+                      "image_id": IMAGE_ID, "repo_digests": [REPO_DIGEST],
                       "grader_container_fresh": True,
                       "eval_script_sha256": eval_hash, "eval_script_file": eval_name,
                       "model_patch_sha256": (
@@ -69,8 +73,8 @@ def fixture(tmp_path):
                 "grader_isolation": "fresh-container-per-scoring-attempt",
                 "grading_lifecycle": prerequisites_module.GRADING_LIFECYCLE,
                 "image": "repo:tag",
-                "remote_image": {"id": "sha256:image",
-                                 "repo_digests": ["repo@sha256:digest"]},
+                "remote_image": {"id": IMAGE_ID, "repo_digests": [REPO_DIGEST],
+                                 "immutable_ref": REPO_DIGEST},
                 "test_command": ["pytest"],
                 "base_commit": "base", "repo": "org/repo", "version": "1",
                 "original_test_patch_sha256": hashlib.sha256(b"original").hexdigest(),
@@ -116,6 +120,28 @@ def test_environment_index_is_required_and_plan_bound(tmp_path):
         validate_environment_index(plan, tmp_path)
 
 
+def test_environment_index_accepts_content_addressed_local_image_without_repo_digest(
+    tmp_path,
+):
+    plan, manifest_path, record = fixture(tmp_path)
+    manifest = json.loads(manifest_path.read_text())
+    manifest["remote_image"] = {
+        "id": IMAGE_ID, "repo_digests": [], "immutable_ref": IMAGE_ID,
+    }
+    for row in manifest["results"]:
+        row["repo_digests"] = []
+    manifest_hash = write(manifest_path, manifest)
+    index_path = tmp_path / "index.json"
+    index = json.loads(index_path.read_text())
+    index["manifests"]["task"]["sha256"] = manifest_hash
+    write(index_path, index)
+
+    evidence = validate_environment_index_for_records(plan, tmp_path, {"task": record})
+    selected = evidence["validated_instances"][0]
+    assert selected["validated_image_ref"] == IMAGE_ID
+    assert selected["validated_repo_digest"] is None
+
+
 def test_unresolved_cell_requires_an_actual_failed_target(tmp_path):
     plan, manifest_path, record = fixture(tmp_path)
     manifest = json.loads(manifest_path.read_text())

tests/test_swe_validation.py

@@ -10,6 +10,10 @@ import pytest
 from messageboardbench import swe_validation as module
 
 
+IMAGE_ID = "sha256:" + "a" * 64
+REPO_DIGEST = "repo@sha256:" + "d" * 64
+
+
 def record(**changes):
     value = {
         "instance_id": "owner__repo-1",
@@ -34,8 +38,8 @@ def result(split: str, mode: str, *, resolved: bool, exit_code: int):
         output_file=f"{split}-{mode}.txt",
         output_sha256="0" * 64,
         image="swebench/sweb.eval.x86_64.example:latest",
-        image_id="sha256:abc",
-        repo_digests=["swebench/example@sha256:def"],
+        image_id=IMAGE_ID,
+        repo_digests=[REPO_DIGEST],
         test_command=["pytest", "tests/test_x.py"],
         target_statuses={"tests/test_x.py::test_bug": "PASSED" if resolved else "FAILED"},
         resolved=resolved,
@@ -115,18 +119,32 @@ def test_matrix_rejects_image_identity_drift():
 def test_image_identity_requires_digest_and_amd64():
     def run(command, **kwargs):
         payload = {
-            "Id": "sha256:abc",
-            "RepoDigests": ["repo@sha256:def"],
+            "Id": IMAGE_ID,
+            "RepoDigests": [REPO_DIGEST],
             "Os": "linux",
             "Architecture": "amd64",
         }
         return subprocess.CompletedProcess(command, 0, __import__("json").dumps(payload), "")
 
     assert module.image_identity("repo:tag", {"DOCKER_HOST": module.REMOTE_DOCKER_HOST}, run) == (
-        "sha256:abc", ["repo@sha256:def"]
+        IMAGE_ID, [REPO_DIGEST]
     )
 
 
+def test_image_identity_accepts_local_content_address_without_repo_digest():
+    def run(command, **kwargs):
+        payload = {
+            "Id": IMAGE_ID, "RepoDigests": [], "Os": "linux", "Architecture": "amd64",
+        }
+        return subprocess.CompletedProcess(command, 0, __import__("json").dumps(payload), "")
+
+    identity = module.image_identity(
+        "local:tag", {"DOCKER_HOST": module.REMOTE_DOCKER_HOST}, run
+    )
+    assert identity == (IMAGE_ID, [])
+    assert module.immutable_image_reference(*identity) == IMAGE_ID
+
+
 def test_semantic_audit_is_bound_to_pair_hashes():
     expected = {
         "dataset": module.DATASET,
@@ -198,7 +216,7 @@ def test_fresh_grader_runs_exact_testspec_script_with_install_and_network_none(m
         return subprocess.CompletedProcess(command, 0, stdout, "")
 
     evaluated, output, statuses, script_hash, preserved_script = module.run_fresh_grader(
-        record(), model_patch="diff --git a/x b/x\n", image="repo@sha256:digest",
+        record(), model_patch="diff --git a/x b/x\n", image=REPO_DIGEST,
         environ={"DOCKER_HOST": module.REMOTE_DOCKER_HOST}, run=run,
     )
     assert evaluated.returncode == 0

tests/test_verify_swe_population_entrypoint.py

@@ -103,6 +103,7 @@ def test_environment_validation_uses_preserved_snapshot(tmp_path, monkeypatch):
                 "manifest_sha256": digest(archived_manifest),
                 "validated_image": "image",
                 "validated_image_id": "image-id",
+                "validated_image_ref": "repo-digest",
                 "validated_repo_digest": "repo-digest",
             }],
         }
@@ -110,7 +111,8 @@ def test_environment_validation_uses_preserved_snapshot(tmp_path, monkeypatch):
     from messageboardbench import swe_prerequisites
     monkeypatch.setattr(swe_prerequisites, "validate_task_manifest", lambda *args, **kwargs: {
         "image": "image",
-        "remote_image": {"id": "image-id", "repo_digests": ["repo-digest"]},
+        "remote_image": {"id": "image-id", "repo_digests": ["repo-digest"],
+                         "immutable_ref": "repo-digest"},
     })
 
     assert matches(manifest, frozen, tmp_path / "run")