diff --git a/scripts/analysis/verify_swe_population.py b/scripts/analysis/verify_swe_population.py index 7af5d26..f764f69 100644 --- a/scripts/analysis/verify_swe_population.py +++ b/scripts/analysis/verify_swe_population.py @@ -90,7 +90,8 @@ def environment_validation_matches_plan( if ( set(row) != { "instance_id", "manifest_path", "manifest_sha256", - "validated_image", "validated_image_id", "validated_repo_digest" + "validated_image", "validated_image_id", "validated_image_ref", + "validated_repo_digest", } or not Path(row["manifest_path"]).as_posix().endswith( "/" + Path(entry["path"]).as_posix() @@ -98,7 +99,11 @@ def environment_validation_matches_plan( or row["manifest_sha256"] != entry["sha256"] or row["validated_image"] != validated["image"] or row["validated_image_id"] != remote_image["id"] - or row["validated_repo_digest"] != remote_image["repo_digests"][0] + or row["validated_image_ref"] != remote_image["immutable_ref"] + or row["validated_repo_digest"] != ( + remote_image["repo_digests"][0] + if remote_image["repo_digests"] else None + ) ): return False except (KeyError, OSError, ValueError, json.JSONDecodeError): diff --git a/scripts/swe_board_experiment.py b/scripts/swe_board_experiment.py index 7967f2c..e3e1cfe 100644 --- a/scripts/swe_board_experiment.py +++ b/scripts/swe_board_experiment.py @@ -105,7 +105,6 @@ def recover_terminal_rows(out: Path) -> list[dict]: def cleanup_matched_images( out: Path, team: int, cohort: int, instance_ids: list[str], records: dict, - validated_images: dict[str, str] | None = None, ) -> None: """Remove only explicit, re-pullable tags after both matched arms terminate.""" path = out / "image-lifecycle.json" @@ -117,7 +116,9 @@ def cleanup_matched_images( record = {"team": team, "cohort": cohort, "images": []} failed = False for instance_id in instance_ids: - image = (validated_images or {}).get(instance_id) or swebench_spec(records[instance_id])[0] + # Remove the mutable local tag, not an immutable ID/digest that may still + # have another tag reference. Identity was already frozen before execution. + image = swebench_spec(records[instance_id])[0] inspected = subprocess.run( ["docker", "image", "inspect", image, "--format", "{{json .}}"], capture_output=True, text=True, env=os.environ, @@ -247,7 +248,7 @@ def main(argv: list[str] | None = None) -> int: team_plans = plan["team_plans"] configs = out / "compose" validated_images = { - row["instance_id"]: row["validated_repo_digest"] + row["instance_id"]: row["validated_image_ref"] for row in (environment_validation or {}).get("validated_instances", []) } compose_by_assignment = { @@ -366,7 +367,7 @@ def main(argv: list[str] | None = None) -> int: if all((team, arm, instance_id) in terminal for arm in CONDITIONS for instance_id in selected): cleanup_matched_images( - out, team, cohort, selected, records, validated_images + out, team, cohort, selected, records ) continue tasks = [] @@ -460,7 +461,7 @@ def main(argv: list[str] | None = None) -> int: ) if matched_complete: cleanup_matched_images( - out, team, cohort, selected, records, validated_images + out, team, cohort, selected, records ) status["status"] = "completed" except BaseException as exc: diff --git a/src/messageboardbench/swe_board.py b/src/messageboardbench/swe_board.py index 4d56dba..96dff30 100644 --- a/src/messageboardbench/swe_board.py +++ b/src/messageboardbench/swe_board.py @@ -26,8 +26,8 @@ from inspect_ai.util import SandboxEnvironmentSpec, sandbox from messageboardbench.board import MESSAGEBOARD_V2_INTERFACE_VERSION, board_tools from messageboardbench.feedback import feedback_tool from messageboardbench.swe_validation import ( - DATASET, GRADING_LIFECYCLE, normalize_record, patch_files, require_revision, - run_fresh_grader, swebench_spec, + DATASET, GRADING_LIFECYCLE, is_immutable_image_reference, normalize_record, + patch_files, require_revision, run_fresh_grader, swebench_spec, ) @@ -345,8 +345,8 @@ def write_compose( ) -> Path: image, _, _ = swebench_spec(record) if image_override is not None: - if "@sha256:" not in image_override: - raise ValueError("validated image override must be a repository digest") + if not is_immutable_image_reference(image_override): + raise ValueError("validated image override must be an immutable image reference") image = image_override directory.mkdir(parents=True, exist_ok=True) path = directory / (str(record["instance_id"]).replace("/", "_") + ".yaml") @@ -367,8 +367,8 @@ def sample_from_record( # other branch. Make the intended upstream choice explicit. metadata["impossible_patch"] = "" if grader_image is not None: - if "@sha256:" not in grader_image: - raise ValueError("fresh grader image must be a repository digest") + if not is_immutable_image_reference(grader_image): + raise ValueError("fresh grader image must be an immutable image reference") metadata["messageboardbench_grader_image"] = grader_image return Sample( id=str(row["instance_id"]), @@ -568,8 +568,9 @@ def swe_board_scorer(*, memory: str = "8g", timeout_seconds: int = 600): "problem_statement": state.input, } grader_image = state.metadata.get("messageboardbench_grader_image") - if not isinstance(grader_image, str) or "@sha256:" not in grader_image: - raise RuntimeError("missing validated repository digest for fresh grader") + if (not isinstance(grader_image, str) + or not is_immutable_image_reference(grader_image)): + raise RuntimeError("missing validated immutable image reference for fresh grader") evaluated, output, statuses, eval_script_sha256, _ = await asyncio.to_thread( run_fresh_grader, record, diff --git a/src/messageboardbench/swe_prerequisites.py b/src/messageboardbench/swe_prerequisites.py index a52517e..2bcfeca 100644 --- a/src/messageboardbench/swe_prerequisites.py +++ b/src/messageboardbench/swe_prerequisites.py @@ -65,7 +65,10 @@ def validate_environment_index_for_records( "manifest_sha256": entry["sha256"], "validated_image": manifest["image"], "validated_image_id": remote_image["id"], - "validated_repo_digest": remote_image["repo_digests"][0], + "validated_image_ref": remote_image["immutable_ref"], + "validated_repo_digest": ( + remote_image["repo_digests"][0] if remote_image["repo_digests"] else None + ), }) return {"index_path": str(index_path), "index_sha256": _sha(index_path), "validated_instances": evidence} @@ -187,9 +190,13 @@ def validate_task_manifest( if len(commands) != 1 or list(next(iter(commands))) != manifest.get("test_command"): raise ValueError(f"validation test command mismatch: {instance_id}") image_id, repo_digests = next(iter(identities)) - if manifest.get("remote_image") != { - "id": image_id, "repo_digests": list(repo_digests) - } or not repo_digests: + from messageboardbench.swe_validation import immutable_image_reference + expected_remote_image = { + "id": image_id, + "repo_digests": list(repo_digests), + "immutable_ref": immutable_image_reference(image_id, repo_digests), + } + if manifest.get("remote_image") != expected_remote_image: raise ValueError(f"validation remote image mismatch: {instance_id}") for row in results: output = manifest_path.parent / str(row.get("output_file", "")) diff --git a/src/messageboardbench/swe_validation.py b/src/messageboardbench/swe_validation.py index cc6a376..c566c33 100644 --- a/src/messageboardbench/swe_validation.py +++ b/src/messageboardbench/swe_validation.py @@ -19,6 +19,8 @@ DATASET = "fjzzq2002/impossible_swebench" REMOTE_DOCKER_HOST = "ssh://pj@100.68.126.75" GRADING_LIFECYCLE = "upstream-testspec-eval-commands-status-monitored-v2" FULL_SHA = re.compile(r"[0-9a-f]{40}\Z") +IMAGE_ID = re.compile(r"sha256:[0-9a-f]{64}\Z") +REPO_DIGEST = re.compile(r"[^\s@]+@sha256:[0-9a-f]{64}\Z") PATCH_PATH_RE = re.compile(r"^(?:--- a/|\+\+\+ b/)(.+)$", re.MULTILINE) @@ -207,14 +209,28 @@ def image_identity( platform = f"{value['Os']}/{value['Architecture']}" except (KeyError, TypeError, json.JSONDecodeError) as exc: raise ValidationError("Docker returned an invalid image identity") from exc - if platform != "linux/amd64" or not image_id or not repo_digests: + if (platform != "linux/amd64" or not IMAGE_ID.fullmatch(str(image_id)) + or any(not REPO_DIGEST.fullmatch(str(value)) for value in repo_digests)): raise ValidationError( - f"image must be linux/amd64 with an ID and repository digest; got {platform}, " + f"image must be linux/amd64 with a content-addressed ID; got {platform}, " f"id={image_id!r}, digests={repo_digests!r}" ) return image_id, repo_digests +def immutable_image_reference(image_id: str, repo_digests: Sequence[str]) -> str: + """Prefer a registry digest, falling back to Docker's content-addressed image ID.""" + if not IMAGE_ID.fullmatch(image_id): + raise ValidationError(f"invalid Docker image ID: {image_id!r}") + if any(not REPO_DIGEST.fullmatch(value) for value in repo_digests): + raise ValidationError("invalid Docker repository digest") + return sorted(repo_digests)[0] if repo_digests else image_id + + +def is_immutable_image_reference(value: str) -> bool: + return bool(IMAGE_ID.fullmatch(value) or REPO_DIGEST.fullmatch(value)) + + def _docker( args: Sequence[str], environ: Mapping[str, str], run: Runner = subprocess.run, **kwargs: Any ) -> subprocess.CompletedProcess[str]: @@ -286,8 +302,8 @@ def run_fresh_grader( """ if environ.get("DOCKER_HOST") != REMOTE_DOCKER_HOST: raise ValidationError(f"fresh grader requires DOCKER_HOST={REMOTE_DOCKER_HOST}") - if "@sha256:" not in image: - raise ValidationError("fresh grader image must be an inspected repository digest") + if not is_immutable_image_reference(image): + raise ValidationError("fresh grader image must be an inspected immutable reference") spec = swebench_test_spec(record) eval_script = spec.eval_script executed_script, monitored_indices = instrument_eval_script(spec.eval_script_list) @@ -377,7 +393,9 @@ def run_trial( output_path = out_dir / f"{split}-{mode}.txt" model_patch = str(record["patch"]) if mode == "oracle" else "" tested, combined, statuses, eval_script_sha256, eval_script = run_fresh_grader( - record, model_patch=model_patch, image=repo_digests[0], environ=environ, run=run, + record, model_patch=model_patch, + image=immutable_image_reference(image_id, repo_digests), + environ=environ, run=run, memory=memory, timeout_seconds=timeout_seconds, ) output_path.write_text(combined) @@ -459,7 +477,11 @@ def manifest( if len(identities) != 1: raise ValidationError("manifest cannot record divergent remote image identities") image_id, repo_digests = next(iter(identities)) - remote_image = {"id": image_id, "repo_digests": list(repo_digests)} + remote_image = { + "id": image_id, + "repo_digests": list(repo_digests), + "immutable_ref": immutable_image_reference(image_id, repo_digests), + } return { "schema_version": 2, "dataset": DATASET, diff --git a/tests/test_swe_board.py b/tests/test_swe_board.py index 59bc24c..c49297a 100644 --- a/tests/test_swe_board.py +++ b/tests/test_swe_board.py @@ -13,6 +13,10 @@ from messageboardbench.board import MESSAGEBOARD_V2_INTERFACE_VERSION, initializ from messageboardbench.feedback import initialize_feedback +IMAGE_ID = "sha256:" + "a" * 64 +REPO_DIGEST = "repo@sha256:" + "d" * 64 + + def records(count=349): return {f"owner__repo-{index:03d}": {"instance_id": f"owner__repo-{index:03d}", "value": index} @@ -119,11 +123,11 @@ def test_compose_has_no_mount_and_network_none(): def test_write_compose_uses_validated_digest_override(tmp_path, monkeypatch): monkeypatch.setattr(module, "swebench_spec", lambda record: ("repo:latest", [], "pytest")) path = module.write_compose( - {"instance_id": "task"}, tmp_path, image_override="repo@sha256:validated" + {"instance_id": "task"}, tmp_path, image_override=REPO_DIGEST ) - assert "repo@sha256:validated" in path.read_text() + assert REPO_DIGEST in path.read_text() assert "repo:latest" not in path.read_text() - with pytest.raises(ValueError, match="repository digest"): + with pytest.raises(ValueError, match="immutable image reference"): module.write_compose({"instance_id": "other"}, tmp_path, image_override="repo:latest") @@ -134,10 +138,10 @@ def test_sample_binds_fresh_grader_to_validated_digest(tmp_path): "instance_id": "task", "problem_statement": "fix it", "test_patch": "patch" } sample = module.sample_from_record( - value, compose, grader_image="repo@sha256:validated" + value, compose, grader_image=IMAGE_ID ) - assert sample.metadata["messageboardbench_grader_image"] == "repo@sha256:validated" - with pytest.raises(ValueError, match="repository digest"): + assert sample.metadata["messageboardbench_grader_image"] == IMAGE_ID + with pytest.raises(ValueError, match="immutable image reference"): module.sample_from_record(value, compose, grader_image="repo:latest") @@ -237,7 +241,7 @@ def test_paid_scorer_restores_tests_and_uses_fresh_digest_grader( state = SimpleNamespace( metadata={ "_messageboardbench_evaluator_commit": "a" * 40, - "messageboardbench_grader_image": "repo@sha256:validated", + "messageboardbench_grader_image": IMAGE_ID, "test_patch": "--- a/tests/test_x.py\n+++ b/tests/test_x.py\n", "base_commit": "b" * 40, "FAIL_TO_PASS": ["target"], "PASS_TO_PASS": [], @@ -252,7 +256,7 @@ def test_paid_scorer_restores_tests_and_uses_fresh_digest_grader( score = asyncio.run(invocation) assert score.value == 1.0 assert captured["model_patch"] == "model patch" - assert captured["image"] == "repo@sha256:validated" + assert captured["image"] == IMAGE_ID assert captured["memory"] == "9g" and captured["timeout_seconds"] == 77 assert score.metadata["test_modified_ever"] is True assert score.metadata["grader_container_fresh"] is True diff --git a/tests/test_swe_prerequisites.py b/tests/test_swe_prerequisites.py index 78e0dcc..e71e7c5 100644 --- a/tests/test_swe_prerequisites.py +++ b/tests/test_swe_prerequisites.py @@ -21,6 +21,10 @@ from scripts.validate_swe_population_prerequisites import ( ) +IMAGE_ID = "sha256:" + "a" * 64 +REPO_DIGEST = "repo@sha256:" + "d" * 64 + + def write(path, value): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(value if isinstance(value, str) else json.dumps(value)) @@ -52,7 +56,7 @@ def fixture(tmp_path): eval_hash = write(tmp_path / "evidence" / eval_name, eval_text) cells.append({"split": split, "mode": mode, "resolved": expected[split, mode], "image": "repo:tag", "test_command": ["pytest"], - "image_id": "sha256:image", "repo_digests": ["repo@sha256:digest"], + "image_id": IMAGE_ID, "repo_digests": [REPO_DIGEST], "grader_container_fresh": True, "eval_script_sha256": eval_hash, "eval_script_file": eval_name, "model_patch_sha256": ( @@ -69,8 +73,8 @@ def fixture(tmp_path): "grader_isolation": "fresh-container-per-scoring-attempt", "grading_lifecycle": prerequisites_module.GRADING_LIFECYCLE, "image": "repo:tag", - "remote_image": {"id": "sha256:image", - "repo_digests": ["repo@sha256:digest"]}, + "remote_image": {"id": IMAGE_ID, "repo_digests": [REPO_DIGEST], + "immutable_ref": REPO_DIGEST}, "test_command": ["pytest"], "base_commit": "base", "repo": "org/repo", "version": "1", "original_test_patch_sha256": hashlib.sha256(b"original").hexdigest(), @@ -116,6 +120,28 @@ def test_environment_index_is_required_and_plan_bound(tmp_path): validate_environment_index(plan, tmp_path) +def test_environment_index_accepts_content_addressed_local_image_without_repo_digest( + tmp_path, +): + plan, manifest_path, record = fixture(tmp_path) + manifest = json.loads(manifest_path.read_text()) + manifest["remote_image"] = { + "id": IMAGE_ID, "repo_digests": [], "immutable_ref": IMAGE_ID, + } + for row in manifest["results"]: + row["repo_digests"] = [] + manifest_hash = write(manifest_path, manifest) + index_path = tmp_path / "index.json" + index = json.loads(index_path.read_text()) + index["manifests"]["task"]["sha256"] = manifest_hash + write(index_path, index) + + evidence = validate_environment_index_for_records(plan, tmp_path, {"task": record}) + selected = evidence["validated_instances"][0] + assert selected["validated_image_ref"] == IMAGE_ID + assert selected["validated_repo_digest"] is None + + def test_unresolved_cell_requires_an_actual_failed_target(tmp_path): plan, manifest_path, record = fixture(tmp_path) manifest = json.loads(manifest_path.read_text()) diff --git a/tests/test_swe_validation.py b/tests/test_swe_validation.py index 904d854..c15403e 100644 --- a/tests/test_swe_validation.py +++ b/tests/test_swe_validation.py @@ -10,6 +10,10 @@ import pytest from messageboardbench import swe_validation as module +IMAGE_ID = "sha256:" + "a" * 64 +REPO_DIGEST = "repo@sha256:" + "d" * 64 + + def record(**changes): value = { "instance_id": "owner__repo-1", @@ -34,8 +38,8 @@ def result(split: str, mode: str, *, resolved: bool, exit_code: int): output_file=f"{split}-{mode}.txt", output_sha256="0" * 64, image="swebench/sweb.eval.x86_64.example:latest", - image_id="sha256:abc", - repo_digests=["swebench/example@sha256:def"], + image_id=IMAGE_ID, + repo_digests=[REPO_DIGEST], test_command=["pytest", "tests/test_x.py"], target_statuses={"tests/test_x.py::test_bug": "PASSED" if resolved else "FAILED"}, resolved=resolved, @@ -115,18 +119,32 @@ def test_matrix_rejects_image_identity_drift(): def test_image_identity_requires_digest_and_amd64(): def run(command, **kwargs): payload = { - "Id": "sha256:abc", - "RepoDigests": ["repo@sha256:def"], + "Id": IMAGE_ID, + "RepoDigests": [REPO_DIGEST], "Os": "linux", "Architecture": "amd64", } return subprocess.CompletedProcess(command, 0, __import__("json").dumps(payload), "") assert module.image_identity("repo:tag", {"DOCKER_HOST": module.REMOTE_DOCKER_HOST}, run) == ( - "sha256:abc", ["repo@sha256:def"] + IMAGE_ID, [REPO_DIGEST] ) +def test_image_identity_accepts_local_content_address_without_repo_digest(): + def run(command, **kwargs): + payload = { + "Id": IMAGE_ID, "RepoDigests": [], "Os": "linux", "Architecture": "amd64", + } + return subprocess.CompletedProcess(command, 0, __import__("json").dumps(payload), "") + + identity = module.image_identity( + "local:tag", {"DOCKER_HOST": module.REMOTE_DOCKER_HOST}, run + ) + assert identity == (IMAGE_ID, []) + assert module.immutable_image_reference(*identity) == IMAGE_ID + + def test_semantic_audit_is_bound_to_pair_hashes(): expected = { "dataset": module.DATASET, @@ -198,7 +216,7 @@ def test_fresh_grader_runs_exact_testspec_script_with_install_and_network_none(m return subprocess.CompletedProcess(command, 0, stdout, "") evaluated, output, statuses, script_hash, preserved_script = module.run_fresh_grader( - record(), model_patch="diff --git a/x b/x\n", image="repo@sha256:digest", + record(), model_patch="diff --git a/x b/x\n", image=REPO_DIGEST, environ={"DOCKER_HOST": module.REMOTE_DOCKER_HOST}, run=run, ) assert evaluated.returncode == 0 diff --git a/tests/test_verify_swe_population_entrypoint.py b/tests/test_verify_swe_population_entrypoint.py index 1b0ff7c..e141dc7 100644 --- a/tests/test_verify_swe_population_entrypoint.py +++ b/tests/test_verify_swe_population_entrypoint.py @@ -103,6 +103,7 @@ def test_environment_validation_uses_preserved_snapshot(tmp_path, monkeypatch): "manifest_sha256": digest(archived_manifest), "validated_image": "image", "validated_image_id": "image-id", + "validated_image_ref": "repo-digest", "validated_repo_digest": "repo-digest", }], } @@ -110,7 +111,8 @@ def test_environment_validation_uses_preserved_snapshot(tmp_path, monkeypatch): from messageboardbench import swe_prerequisites monkeypatch.setattr(swe_prerequisites, "validate_task_manifest", lambda *args, **kwargs: { "image": "image", - "remote_image": {"id": "image-id", "repo_digests": ["repo-digest"]}, + "remote_image": {"id": "image-id", "repo_digests": ["repo-digest"], + "immutable_ref": "repo-digest"}, }) assert matches(manifest, frozen, tmp_path / "run")