messageboardbench/tests/test_verify_swe_population_entrypoint.py

import runpy
import hashlib
import json
from pathlib import Path
import subprocess
import sys


def test_verify_swe_population_script_imports_when_invoked_by_path_from_repo_root():
    root = Path(__file__).resolve().parents[1]
    result = subprocess.run(
        [sys.executable, "scripts/analysis/verify_swe_population.py", "--help"],
        cwd=root, capture_output=True, text=True,
    )
    assert result.returncode == 0, result.stderr
    assert "--run" in result.stdout and "--export" in result.stdout


def test_board_operations_are_linked_to_board_episodes_without_condition_field():
    root = Path(__file__).resolve().parents[1]
    namespace = runpy.run_path(root / "scripts/analysis/verify_swe_population.py")
    board_operations_are_board_only = namespace["board_operations_are_board_only"]
    rows = [
        {"episode_id": "control-1", "condition": "control"},
        {"episode_id": "board-1", "condition": "board"},
    ]
    assert board_operations_are_board_only(rows, [{"episode_id": "board-1"}])
    assert not board_operations_are_board_only(rows, [{"episode_id": "control-1"}])
    assert not board_operations_are_board_only(rows, [{"episode_id": "unknown"}])


def test_enriched_environment_validation_is_compared_separately_from_plan_fields():
    root = Path(__file__).resolve().parents[1]
    namespace = runpy.run_path(root / "scripts/analysis/verify_swe_population.py")
    matches = namespace["manifest_matches_frozen_plan"]
    frozen = {
        "model": "openrouter/example",
        "environment_validation": {
            "index_path": "work/example/index.json",
            "required_before_execution": True,
        },
    }
    manifest = {
        "model": "openrouter/example",
        "environment_validation": {
            "index_path": str(root / "work/example/index.json"),
            "index_sha256": "abc",
            "validated_instances": [],
        },
    }

    assert matches(manifest, frozen)
    assert not matches({**manifest, "model": "openrouter/changed"}, frozen)


def test_environment_validation_uses_preserved_snapshot(tmp_path, monkeypatch):
    root = Path(__file__).resolve().parents[1]
    namespace = runpy.run_path(root / "scripts/analysis/verify_swe_population.py")
    matches = namespace["environment_validation_matches_plan"]
    snapshot = tmp_path / "run/environment-validation"
    relative_manifest = Path("decisions/001-task/attempt-001/manifest.json")
    archived_manifest = snapshot / relative_manifest
    archived_manifest.parent.mkdir(parents=True)
    archived_manifest.write_text("{}\n")
    ledger_path = snapshot / "ledger.json"
    ledger_path.write_text("{}\n")

    digest = lambda path: hashlib.sha256(path.read_bytes()).hexdigest()
    frozen = {
        "dataset": {"path": "dataset", "revision": "revision", "split": "conflicting"},
        "plan_sha256": "plan-hash",
        "environment_validation": {
            "index_path": "work/example/index.json",
            "required_before_execution": True,
        },
        "selection": {
            "instance_ids": ["task"],
            "screening_ledger": {"file_sha256": digest(ledger_path)},
            "selected_manifest_sha256": {"task": digest(archived_manifest)},
        },
    }
    index_path = snapshot / "index.json"
    index_path.write_text(json.dumps({
        "schema_version": 1,
        "status": "validated",
        "plan_sha256": "plan-hash",
        "dataset": frozen["dataset"],
        "manifests": {
            "task": {
                "path": "work/example/" + relative_manifest.as_posix(),
                "sha256": digest(archived_manifest),
            }
        },
    }))
    manifest = {
        "environment_validation": {
            "index_path": "/unused/repository/work/example/index.json",
            "index_sha256": digest(index_path),
            "snapshot_path": str(snapshot),
            "validated_instances": [{
                "instance_id": "task",
                "manifest_path": "/unused/repository/work/example/" + relative_manifest.as_posix(),
                "manifest_sha256": digest(archived_manifest),
                "validated_image": "image",
                "validated_image_id": "image-id",
                "validated_repo_digest": "repo-digest",
            }],
        }
    }
    from messageboardbench import swe_prerequisites
    monkeypatch.setattr(swe_prerequisites, "validate_task_manifest", lambda *args, **kwargs: {
        "image": "image",
        "remote_image": {"id": "image-id", "repo_digests": ["repo-digest"]},
    })

    assert matches(manifest, frozen, tmp_path / "run")
    frozen["selection"]["selected_manifest_sha256"]["task"] = "wrong"
    assert not matches(manifest, frozen, tmp_path / "run")
    frozen["selection"]["selected_manifest_sha256"]["task"] = digest(archived_manifest)
    index_path.write_text(index_path.read_text() + "\n")
    assert not matches(manifest, frozen, tmp_path / "run")