messageboardbench/tests/test_prompt_validation.py

from __future__ import annotations

import hashlib
import importlib.util
import json
from pathlib import Path
from types import SimpleNamespace

import pytest
from inspect_ai.dataset import Sample

from messageboardbench.calibration_run import prepare_validation_samples
from messageboardbench.prompt_calibration import TaskPartitions, build_manifest, render_tools_instruction


REVISION = "e" * 40
PROMPT = "def candidate(x): pass"
TEST = "def check(candidate): pass"


def load_runner():
    spec = importlib.util.spec_from_file_location(
        "run_prompt_validation", Path(__file__).parents[1] / "scripts/run_prompt_validation.py"
    )
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def plan_fixture(tmp_path: Path):
    partitions = TaskPartitions(
        development=("dev",), validation=("val",), communication_holdout=("hold",),
    )
    manifest = build_manifest(dataset_revision=REVISION, partitions=partitions)
    path = tmp_path / "plan.json"
    path.write_text(json.dumps(manifest))
    audit = tmp_path / "validation-audit.json"
    pairs = {(row["task_id"], row["split"]) for row in manifest["validation_assignments"]}
    audit.write_text(json.dumps({
        "schema_version": 2, "status": "ready", "partition": "validation",
        "dataset": {"path": manifest["benchmark"]["dataset"], "revision": REVISION},
        "review": {"reviewer_type": "human", "reviewer": "Test reviewer",
                   "no_model_outcomes_inspected": True},
        "approved_pairs": [{
            "task_id": task_id, "split": split, "task_validated": True,
            "evaluator_validated": True,
            "task_prompt_sha256": hashlib.sha256(PROMPT.encode()).hexdigest(),
            "test_sha256": hashlib.sha256(TEST.encode()).hexdigest(),
        } for task_id, split in sorted(pairs)],
    }))
    return manifest, path, audit


def sample_loader(revision):
    base = render_tools_instruction("A")
    return {split: {"val": Sample(
        id="val", input=base, metadata={"instruction_prompt": base, "prompt": PROMPT,
                                        "test": TEST, "entry_point": "candidate"},
    )} for split in ("original", "conflicting")}


def test_prepare_validation_uses_only_validation_d():
    manifest = build_manifest(
        dataset_revision=REVISION,
        partitions=TaskPartitions(development=("dev",), validation=("val",),
                                  communication_holdout=("hold",)),
    )
    source = {"path": "/plan", "file_sha256": "a" * 64,
              "manifest_sha256": manifest["manifest_sha256"]}
    prepared = prepare_validation_samples(manifest, source, loader=sample_loader)
    assert {row["assignment"]["task_id"] for row in prepared} == {"val"}
    assert {row["assignment"]["prompt_variant"] for row in prepared} == {"D"}
    assert all(row["sample"].metadata["calibration"]["phase"] == "validation"
               for row in prepared)


def test_preview_is_offline_and_does_not_consume(tmp_path, monkeypatch, capsys):
    runner = load_runner()
    _, plan, audit = plan_fixture(tmp_path)
    out = tmp_path / "run"
    monkeypatch.setattr(runner, "prepare_validation_samples",
                        lambda *args: pytest.fail("preview loaded dataset"))
    monkeypatch.setattr(runner, "LEDGER_DIR", tmp_path / "ledger")
    assert runner.main(["--manifest", str(plan), "--validation-audit", str(audit),
                        "--out", str(out)]) == 0
    assert "Preview only" in capsys.readouterr().out
    assert not out.exists() and not (tmp_path / "ledger").exists()


def test_execute_requires_remote_docker_before_dataset(tmp_path, monkeypatch):
    runner = load_runner()
    _, plan, audit = plan_fixture(tmp_path)
    monkeypatch.delenv("DOCKER_HOST", raising=False)
    monkeypatch.setattr(runner, "prepare_validation_samples",
                        lambda *args: pytest.fail("wrong host loaded dataset"))
    with pytest.raises(RuntimeError, match="remote Docker daemon"):
        runner.main(["--manifest", str(plan), "--validation-audit", str(audit),
                     "--out", str(tmp_path / "run"), "--execute"])


def test_mock_execution_writes_gate_compatible_run_and_is_one_shot(tmp_path, monkeypatch):
    runner = load_runner()
    manifest, plan, audit = plan_fixture(tmp_path)
    monkeypatch.setattr(
        runner, "prepare_validation_samples",
        lambda loaded, source: prepare_validation_samples(loaded, source, loader=sample_loader),
    )
    monkeypatch.setattr(runner, "LEDGER_DIR", tmp_path / "ledger")
    monkeypatch.setenv("DOCKER_HOST", runner.REMOTE_DOCKER_HOST)
    budgets = iter([{"usage": 1, "limit": 5, "limit_remaining": 4},
                    {"usage": 1.1, "limit": 5, "limit_remaining": 3.9}])
    monkeypatch.setattr(runner, "budget", lambda: next(budgets))
    import inspect_ai
    import messageboardbench.board_task as board_task
    import messageboardbench.task as task_module
    monkeypatch.setattr(inspect_ai, "Task", lambda **kwargs: SimpleNamespace(**kwargs))
    monkeypatch.setattr(board_task, "episode_solver", lambda *args, **kwargs: "solver")
    monkeypatch.setattr(task_module, "scratch_scorer", lambda split: "scorer")
    out = tmp_path / "run"

    eval_calls = []
    def fake_eval(tasks, **kwargs):
        eval_calls.append(tasks)
        log_path = out / "evals" / f"mock-{len(eval_calls)}.eval"
        log_path.parent.mkdir(exist_ok=True)
        log_path.write_bytes(b"eval")
        sample = tasks[0].dataset[0]
        score = SimpleNamespace(value="I", metadata={})
        returned = SimpleNamespace(id=sample.id, scores={"score": score}, messages=[],
                                   model_usage={}, limit=None, error=None, metadata=sample.metadata)
        return [SimpleNamespace(location=str(log_path), status="success", samples=[returned])]

    monkeypatch.setattr(inspect_ai, "eval", fake_eval)
    assert runner.main(["--manifest", str(plan), "--validation-audit", str(audit),
                        "--out", str(out), "--execute"]) == 0
    run_manifest = json.loads((out / "run-manifest.json").read_text())
    assert run_manifest["phase"] == "validation"
    assert run_manifest["development_assignments_executed"] is False
    assert run_manifest["communication_holdout_assignments_executed"] is False
    assert json.loads((out / "status.json").read_text())["status"] == "completed"
    from messageboardbench.confirmation import verify_completed_prompt_d_validation
    evidence = verify_completed_prompt_d_validation(plan, out)
    assert evidence["completed_assignments"] == len(manifest["validation_assignments"])
    with pytest.raises(ValueError, match="already consumed"):
        runner.consume_once(manifest, plan, tmp_path / "other")