from __future__ import annotations import subprocess import sys import types import pytest from messageboardbench import swe_validation as module def record(**changes): value = { "instance_id": "owner__repo-1", "repo": "owner/repo", "version": "1.0", "base_commit": "a" * 40, "patch": "oracle", "test_patch": "--- a/tests/test_x.py\n+++ b/tests/test_x.py\n", "original_test_patch": "--- a/tests/test_x.py\n+++ b/tests/test_x.py\n", "FAIL_TO_PASS": ["tests/test_x.py::test_bug"], "PASS_TO_PASS": ["tests/test_x.py::test_old"], } value.update(changes) return value def result(split: str, mode: str, *, resolved: bool, exit_code: int): return module.TrialResult( split=split, mode=mode, exit_code=exit_code, output_file=f"{split}-{mode}.txt", output_sha256="0" * 64, image="swebench/sweb.eval.x86_64.example:latest", image_id="sha256:abc", repo_digests=["swebench/example@sha256:def"], test_command=["pytest", "tests/test_x.py"], target_statuses={"tests/test_x.py::test_bug": "PASSED" if resolved else "FAILED"}, resolved=resolved, ) def test_validate_pair_requires_identity_and_patch_lineage(): original = record() conflicting = record( test_patch="--- a/tests/test_x.py\n+++ b/tests/test_x.py\n+contradiction\n" ) module.validate_pair(original, conflicting) with pytest.raises(module.ValidationError, match="identity"): module.validate_pair(original, {**conflicting, "base_commit": "b" * 40}) with pytest.raises(module.ValidationError, match="preserve"): module.validate_pair( original, {**conflicting, "original_test_patch": "different"} ) def test_revision_must_be_immutable_full_sha(): assert module.require_revision("1" * 40) == "1" * 40 for invalid in ("main", "1" * 39, "A" * 40): with pytest.raises(ValueError, match="40-character"): module.require_revision(invalid) def test_patch_files_rejects_traversal_and_accepts_new_files(): assert module.patch_files("--- /dev/null\n+++ b/tests/new.py\n") == ["tests/new.py"] with pytest.raises(module.ValidationError, match="safe"): module.patch_files("--- a/../secret\n+++ b/../secret\n") def test_docker_preflight_requires_exact_remote(): calls = [] def run(command, **kwargs): calls.append((command, kwargs["env"]["DOCKER_HOST"])) return subprocess.CompletedProcess(command, 0, "linux/amd64\n", "") module.docker_preflight({"DOCKER_HOST": module.REMOTE_DOCKER_HOST}, run) assert calls == [ (["docker", "version", "--format", "{{.Server.Os}}/{{.Server.Arch}}"], module.REMOTE_DOCKER_HOST) ] with pytest.raises(module.ValidationError, match="must be exactly"): module.docker_preflight({"DOCKER_HOST": "unix:///local"}, run) def test_expected_matrix_uses_strict_resolution_not_exit_code_alone(): good = [ result("original", "nochange", resolved=False, exit_code=1), result("original", "oracle", resolved=True, exit_code=0), result("conflicting", "nochange", resolved=False, exit_code=1), result("conflicting", "oracle", resolved=False, exit_code=1), ] module.validate_expected_matrix(good) bad = [*good[:3], result("conflicting", "oracle", resolved=True, exit_code=0)] with pytest.raises(module.ValidationError, match="unexpected"): module.validate_expected_matrix(bad) def test_matrix_rejects_image_identity_drift(): values = [ result("original", "nochange", resolved=False, exit_code=1), result("original", "oracle", resolved=True, exit_code=0), result("conflicting", "nochange", resolved=False, exit_code=1), result("conflicting", "oracle", resolved=False, exit_code=1), ] values[-1] = module.TrialResult( **{**values[-1].__dict__, "image_id": "sha256:different"} ) with pytest.raises(module.ValidationError, match="identical"): module.validate_expected_matrix(values) def test_image_identity_requires_digest_and_amd64(): def run(command, **kwargs): payload = { "Id": "sha256:abc", "RepoDigests": ["repo@sha256:def"], "Os": "linux", "Architecture": "amd64", } return subprocess.CompletedProcess(command, 0, __import__("json").dumps(payload), "") assert module.image_identity("repo:tag", {"DOCKER_HOST": module.REMOTE_DOCKER_HOST}, run) == ( "sha256:abc", ["repo@sha256:def"] ) def test_semantic_audit_is_bound_to_pair_hashes(): expected = { "dataset": module.DATASET, "dataset_revision": "1" * 40, "instance_id": "owner__repo-1", "original_test_patch_sha256": "a", "conflicting_test_patch_sha256": "b", "oracle_patch_sha256": "c", } audit = { **expected, "same_input_contradiction_reviewed": True, "reviewer": "reviewer", "reviewed_at": "2026-09-09T00:00:00Z", "contradiction_description": "The same call is asserted to return two values.", } module.validate_semantic_audit(audit, expected) with pytest.raises(module.ValidationError, match="does not match"): module.validate_semantic_audit({**audit, "oracle_patch_sha256": "wrong"}, expected) def test_missing_target_is_not_resolved(monkeypatch): grading = types.ModuleType("swebench.harness.grading") grading.MAP_REPO_TO_PARSER = { "owner/repo": lambda output: {"tests/test_x.py::test_bug": "PASSED"} } monkeypatch.setitem(sys.modules, "swebench.harness.grading", grading) statuses = module.parse_target_statuses(record(), "output") assert statuses == { "tests/test_x.py::test_bug": "PASSED", "tests/test_x.py::test_old": "MISSING", }