messageboardbench/tests/test_swe_validation.py

from __future__ import annotations

import subprocess
import sys
import types

import pytest

from messageboardbench import swe_validation as module


def record(**changes):
    value = {
        "instance_id": "owner__repo-1",
        "repo": "owner/repo",
        "version": "1.0",
        "base_commit": "a" * 40,
        "patch": "oracle",
        "test_patch": "--- a/tests/test_x.py\n+++ b/tests/test_x.py\n",
        "original_test_patch": "--- a/tests/test_x.py\n+++ b/tests/test_x.py\n",
        "FAIL_TO_PASS": ["tests/test_x.py::test_bug"],
        "PASS_TO_PASS": ["tests/test_x.py::test_old"],
    }
    value.update(changes)
    return value


def result(split: str, mode: str, *, resolved: bool, exit_code: int):
    return module.TrialResult(
        split=split,
        mode=mode,
        exit_code=exit_code,
        output_file=f"{split}-{mode}.txt",
        output_sha256="0" * 64,
        image="swebench/sweb.eval.x86_64.example:latest",
        image_id="sha256:abc",
        repo_digests=["swebench/example@sha256:def"],
        test_command=["pytest", "tests/test_x.py"],
        target_statuses={"tests/test_x.py::test_bug": "PASSED" if resolved else "FAILED"},
        resolved=resolved,
    )


def test_validate_pair_requires_identity_and_patch_lineage():
    original = record()
    conflicting = record(
        test_patch="--- a/tests/test_x.py\n+++ b/tests/test_x.py\n+contradiction\n"
    )
    module.validate_pair(original, conflicting)
    with pytest.raises(module.ValidationError, match="identity"):
        module.validate_pair(original, {**conflicting, "base_commit": "b" * 40})
    with pytest.raises(module.ValidationError, match="preserve"):
        module.validate_pair(
            original, {**conflicting, "original_test_patch": "different"}
        )


def test_revision_must_be_immutable_full_sha():
    assert module.require_revision("1" * 40) == "1" * 40
    for invalid in ("main", "1" * 39, "A" * 40):
        with pytest.raises(ValueError, match="40-character"):
            module.require_revision(invalid)


def test_patch_files_rejects_traversal_and_accepts_new_files():
    assert module.patch_files("--- /dev/null\n+++ b/tests/new.py\n") == ["tests/new.py"]
    with pytest.raises(module.ValidationError, match="safe"):
        module.patch_files("--- a/../secret\n+++ b/../secret\n")


def test_docker_preflight_requires_exact_remote():
    calls = []

    def run(command, **kwargs):
        calls.append((command, kwargs["env"]["DOCKER_HOST"]))
        return subprocess.CompletedProcess(command, 0, "linux/amd64\n", "")

    module.docker_preflight({"DOCKER_HOST": module.REMOTE_DOCKER_HOST}, run)
    assert calls == [
        (["docker", "version", "--format", "{{.Server.Os}}/{{.Server.Arch}}"],
         module.REMOTE_DOCKER_HOST)
    ]
    with pytest.raises(module.ValidationError, match="must be exactly"):
        module.docker_preflight({"DOCKER_HOST": "unix:///local"}, run)


def test_expected_matrix_uses_strict_resolution_not_exit_code_alone():
    good = [
        result("original", "nochange", resolved=False, exit_code=1),
        result("original", "oracle", resolved=True, exit_code=0),
        result("conflicting", "nochange", resolved=False, exit_code=1),
        result("conflicting", "oracle", resolved=False, exit_code=1),
    ]
    module.validate_expected_matrix(good)
    bad = [*good[:3], result("conflicting", "oracle", resolved=True, exit_code=0)]
    with pytest.raises(module.ValidationError, match="unexpected"):
        module.validate_expected_matrix(bad)


def test_matrix_rejects_image_identity_drift():
    values = [
        result("original", "nochange", resolved=False, exit_code=1),
        result("original", "oracle", resolved=True, exit_code=0),
        result("conflicting", "nochange", resolved=False, exit_code=1),
        result("conflicting", "oracle", resolved=False, exit_code=1),
    ]
    values[-1] = module.TrialResult(
        **{**values[-1].__dict__, "image_id": "sha256:different"}
    )
    with pytest.raises(module.ValidationError, match="identical"):
        module.validate_expected_matrix(values)


def test_image_identity_requires_digest_and_amd64():
    def run(command, **kwargs):
        payload = {
            "Id": "sha256:abc",
            "RepoDigests": ["repo@sha256:def"],
            "Os": "linux",
            "Architecture": "amd64",
        }
        return subprocess.CompletedProcess(command, 0, __import__("json").dumps(payload), "")

    assert module.image_identity("repo:tag", {"DOCKER_HOST": module.REMOTE_DOCKER_HOST}, run) == (
        "sha256:abc", ["repo@sha256:def"]
    )


def test_semantic_audit_is_bound_to_pair_hashes():
    expected = {
        "dataset": module.DATASET,
        "dataset_revision": "1" * 40,
        "instance_id": "owner__repo-1",
        "original_test_patch_sha256": "a",
        "conflicting_test_patch_sha256": "b",
        "oracle_patch_sha256": "c",
    }
    audit = {
        **expected,
        "same_input_contradiction_reviewed": True,
        "reviewer": "reviewer",
        "reviewed_at": "2026-09-09T00:00:00Z",
        "contradiction_description": "The same call is asserted to return two values.",
    }
    module.validate_semantic_audit(audit, expected)
    with pytest.raises(module.ValidationError, match="does not match"):
        module.validate_semantic_audit({**audit, "oracle_patch_sha256": "wrong"}, expected)


def test_missing_target_is_not_resolved(monkeypatch):
    grading = types.ModuleType("swebench.harness.grading")
    grading.MAP_REPO_TO_PARSER = {
        "owner/repo": lambda output: {"tests/test_x.py::test_bug": "PASSED"}
    }
    monkeypatch.setitem(sys.modules, "swebench.harness.grading", grading)
    statuses = module.parse_target_statuses(record(), "output")
    assert statuses == {
        "tests/test_x.py::test_bug": "PASSED",
        "tests/test_x.py::test_old": "MISSING",
    }