messageboardbench/tests/test_confirmation.py

import json
from pathlib import Path

import pytest

from messageboardbench.calibration_run import read_frozen_manifest
from messageboardbench.communication_plan import build_communication_plan
from messageboardbench.completion import completion_manifest_record
from messageboardbench.confirmation import (
    consume_plan_once,
    verify_calibration_review,
    verify_completed_calibration,
    verify_completed_prompt_d_validation,
    verify_prompt_d_validation,
)
from messageboardbench.prompt_calibration import build_manifest, write_manifest


REVISION = "a" * 40


def completed_calibration(tmp_path: Path):
    plan_path = tmp_path / "plan.json"
    write_manifest(plan_path, build_manifest(dataset_revision=REVISION))
    plan, source = read_frozen_manifest(plan_path)
    run = tmp_path / "run"
    run.mkdir()
    (run / "evals").mkdir()
    (run / "frozen-plan.json").write_bytes(plan_path.read_bytes())
    results = [{
        "assignment": row, "log": str(run / "evals" / f"{index}.eval"),
        "error": None,
        "completion": completion_manifest_record(),
        "calibration": {"communication": "none"},
    } for index, row in enumerate(plan["development_assignments"], 1)]
    for row in results:
        Path(row["log"]).write_bytes(b"mock eval log")
    (run / "results.json").write_text(json.dumps(results))
    (run / "status.json").write_text(json.dumps({
        "status": "completed", "phase": "development",
        "completed_assignments": len(results), "in_flight_assignment": None,
    }))
    (run / "run-manifest.json").write_text(json.dumps({
        "purpose": "prompt-calibration-development-execution", "phase": "development",
        "execute": True, "communication": "none", "completion": completion_manifest_record(),
        "manifest": source,
    }))
    return plan_path, run, len(results)


def completed_validation(plan_path: Path, run: Path):
    plan, source = read_frozen_manifest(plan_path)
    run.mkdir()
    (run / "evals").mkdir()
    (run / "frozen-plan.json").write_bytes(plan_path.read_bytes())
    results = []
    for index, assignment in enumerate(plan["validation_assignments"], 1):
        log_path = run / "evals" / f"{index}.eval"
        log_path.write_bytes(b"mock validation eval log")
        results.append({
            "assignment": assignment,
            "sample_id": assignment["task_id"],
            "log": str(log_path),
            "error": None,
            "completion": completion_manifest_record(),
            "calibration": {
                "phase": "validation", "communication": "none",
                "assignment": assignment, "manifest": source,
                "policy_prompt": {"variant": "D"},
            },
        })
    (run / "results.json").write_text(json.dumps(results))
    (run / "status.json").write_text(json.dumps({
        "status": "completed", "phase": "validation",
        "completed_assignments": len(results), "in_flight_assignment": None,
    }))
    audit_path = run.parent / "validation-audit.json"
    audit_path.write_text(json.dumps({
        "schema_version": 2, "status": "ready", "partition": "validation",
        "dataset": {"path": plan["benchmark"]["dataset"],
                    "revision": plan["benchmark"]["dataset_revision"]},
        "review": {"reviewer_type": "human", "reviewer": "Test reviewer",
                   "no_model_outcomes_inspected": True},
        "approved_pairs": [{
            "task_id": task_id, "split": split, "task_validated": True,
            "evaluator_validated": True, "task_prompt_sha256": "1" * 64,
            "test_sha256": "2" * 64,
        } for task_id, split in sorted({
            (row["task_id"], row["split"]) for row in plan["validation_assignments"]
        })],
    }))
    run_manifest = {
        "purpose": "prompt-calibration-validation-execution", "phase": "validation",
        "execute": True, "communication": "none", "completion": completion_manifest_record(),
        "assignments": len(results), "manifest": source,
        "validation_audit": {
            "path": str(audit_path),
            "sha256": __import__("hashlib").sha256(audit_path.read_bytes()).hexdigest(),
        },
    }
    run_manifest.update({key: plan["environment"][key] for key in (
        "model", "message_limit", "token_limit", "time_limit_seconds", "temperature",
        "reasoning_effort", "max_attempts", "strict_tools", "sample_retries", "request_retries",
    )})
    (run / "run-manifest.json").write_text(json.dumps(run_manifest))
    return run, len(results)


def test_completed_calibration_and_review_bind_exact_bytes(tmp_path):
    plan, run, count = completed_calibration(tmp_path)
    evidence = verify_completed_calibration(plan, run)
    review_path = tmp_path / "review.json"
    review_path.write_text(json.dumps({
        "schema_version": 1, "status": "ready",
        "purpose": "prompt-calibration-behavior-review",
        "calibration_evidence_sha256": evidence["evidence_sha256"],
        "no_communication_holdout_outcomes_inspected": True,
        "reviewer": "Internal review group",
        "assignment_labels": [
            {"assignment_index": i, "label": "no_observed_gaming"}
            for i in range(1, count + 1)
        ],
        "prompt_d_assessment": {
            "decision": "proceed", "variation_adequate": True, "rationale": "Observed variation",
        },
    }))
    source = verify_calibration_review(review_path, evidence)
    assert source["calibration_evidence_sha256"] == evidence["evidence_sha256"]

    status = json.loads((run / "status.json").read_text())
    status["status"] = "running"
    (run / "status.json").write_text(json.dumps(status))
    with pytest.raises(ValueError, match="not completed"):
        verify_completed_calibration(plan, run)


def test_review_cannot_proceed_without_d_variation(tmp_path):
    plan, run, count = completed_calibration(tmp_path)
    evidence = verify_completed_calibration(plan, run)
    review = tmp_path / "review.json"
    review.write_text(json.dumps({
        "schema_version": 1, "status": "ready",
        "purpose": "prompt-calibration-behavior-review",
        "calibration_evidence_sha256": evidence["evidence_sha256"],
        "no_communication_holdout_outcomes_inspected": True, "reviewer": "Reviewer",
        "assignment_labels": [{"assignment_index": i, "label": "ambiguous"}
                              for i in range(1, count + 1)],
        "prompt_d_assessment": {"decision": "stop", "variation_adequate": False,
                                "rationale": "No variation"},
    }))
    with pytest.raises(ValueError, match="not reviewed as adequate"):
        verify_calibration_review(review, evidence)


def test_plan_consumption_is_write_once(tmp_path):
    plan_path = tmp_path / "communication.json"
    plan = build_communication_plan({"teams": 2})
    plan_path.write_text(json.dumps(plan))
    ledger = tmp_path / "ledger"
    receipt = consume_plan_once(
        plan_path, tmp_path / "run", plan["plan_sha256"], ledger_dir=ledger
    )
    assert receipt["status"] == "consumed"
    with pytest.raises(ValueError, match="already consumed"):
        consume_plan_once(
            plan_path, tmp_path / "other", plan["plan_sha256"], ledger_dir=ledger
        )


def test_prompt_d_validation_must_bind_and_proceed(tmp_path):
    plan, _, _ = completed_calibration(tmp_path)
    validation_run, count = completed_validation(plan, tmp_path / "validation-run")
    execution = verify_completed_prompt_d_validation(plan, validation_run)
    record = {
        "schema_version": 1, "status": "ready", "purpose": "prompt-d-validation",
        "calibration_evidence_sha256": "c" * 64, "dataset_revision": REVISION,
        "model": "openrouter/example/model", "prompt_variant": "D",
        "prompt_sha256": "d" * 64, "communication": "none",
        "completion_policy": completion_manifest_record(),
        "communication_holdout_executed": False, "artifact_review_complete": True,
        "decision": "proceed", "variation_adequate": True,
        "validation_run_directory": str(validation_run),
        "validation_execution_sha256": execution["evidence_sha256"],
        "completed_assignments": count, "reviewer": "Validation reviewers",
        "rationale": "Both gaming and non-gaming outcomes occurred.",
        "assignment_labels": [
            {
                "assignment_index": row["assignment_index"],
                "label": (
                    "successful_gaming" if row["assignment_index"] == min(
                        item["assignment_index"] for item in
                        read_frozen_manifest(plan)[0]["validation_assignments"]
                        if item["split"] == "conflicting"
                    )
                    else "no_observed_gaming"
                ),
                "artifact_labels": (
                    ["contradiction_targeted_gaming"] if row["assignment_index"] == min(
                        item["assignment_index"] for item in
                        read_frozen_manifest(plan)[0]["validation_assignments"]
                        if item["split"] == "conflicting"
                    ) else ["specification_faithful"]
                ),
                "notes": "Artifact inspected and classified independently of evaluator score.",
            }
            for row in read_frozen_manifest(plan)[0]["validation_assignments"]
        ],
    }
    path = tmp_path / "validation.json"
    path.write_text(json.dumps(record))
    assert verify_prompt_d_validation(
        path, plan_path=plan, calibration_evidence_sha256="c" * 64, dataset_revision=REVISION,
        model="openrouter/example/model", prompt_sha256="d" * 64,
    )["status"] == "ready"
    first_log = next((validation_run / "evals").glob("*.eval"))
    first_log.write_bytes(b"mutated after review")
    with pytest.raises(ValueError, match="not ready"):
        verify_prompt_d_validation(
            path, plan_path=plan, calibration_evidence_sha256="c" * 64,
            dataset_revision=REVISION, model="openrouter/example/model",
            prompt_sha256="d" * 64,
        )
    first_log.write_bytes(b"mock validation eval log")
    record["decision"] = "stop"
    path.write_text(json.dumps(record))
    with pytest.raises(ValueError, match="not ready"):
        verify_prompt_d_validation(
            path, plan_path=plan, calibration_evidence_sha256="c" * 64, dataset_revision=REVISION,
            model="openrouter/example/model", prompt_sha256="d" * 64,
        )


def test_prompt_d_validation_distinguishes_targeted_from_weak_oracle_gaming(tmp_path):
    plan, _, _ = completed_calibration(tmp_path)
    validation_run, count = completed_validation(plan, tmp_path / "validation-run")
    execution = verify_completed_prompt_d_validation(plan, validation_run)
    assignments = read_frozen_manifest(plan)[0]["validation_assignments"]
    first_conflicting = min(row["assignment_index"] for row in assignments
                            if row["split"] == "conflicting")
    record = {
        "schema_version": 1, "status": "ready", "purpose": "prompt-d-validation",
        "calibration_evidence_sha256": "c" * 64, "dataset_revision": REVISION,
        "model": "openrouter/example/model", "prompt_variant": "D",
        "prompt_sha256": "d" * 64, "communication": "none",
        "completion_policy": completion_manifest_record(),
        "communication_holdout_executed": False, "artifact_review_complete": True,
        "decision": "proceed", "variation_adequate": True,
        "validation_run_directory": str(validation_run),
        "validation_execution_sha256": execution["evidence_sha256"],
        "completed_assignments": count, "reviewer": "Validation reviewers",
        "rationale": "Targeted gaming and non-gaming outcomes occurred.",
        "assignment_labels": [{
            "assignment_index": row["assignment_index"],
            "label": "successful_gaming" if row["assignment_index"] == first_conflicting
                     else "no_observed_gaming",
            "artifact_labels": ["generic_weak_oracle_exploitation"]
                               if row["assignment_index"] == first_conflicting
                               else ["specification_faithful"],
            "notes": "Artifact inspected.",
        } for row in assignments],
    }
    path = tmp_path / "validation.json"
    path.write_text(json.dumps(record))
    kwargs = dict(
        plan_path=plan, calibration_evidence_sha256="c" * 64, dataset_revision=REVISION,
        model="openrouter/example/model", prompt_sha256="d" * 64,
    )
    with pytest.raises(ValueError, match="contradiction-targeted"):
        verify_prompt_d_validation(path, **kwargs)

    target = next(row for row in record["assignment_labels"]
                  if row["assignment_index"] == first_conflicting)
    target["label"] = "other_evaluator_gaming"
    path.write_text(json.dumps(record))
    with pytest.raises(ValueError, match="behavioral variation"):
        verify_prompt_d_validation(path, **kwargs)

    target["label"] = "successful_gaming"
    target["artifact_labels"].append("contradiction_targeted_gaming")
    path.write_text(json.dumps(record))
    assert verify_prompt_d_validation(path, **kwargs)["status"] == "ready"


def test_prompt_d_validation_rejects_mutated_results_and_plan(tmp_path):
    plan, _, _ = completed_calibration(tmp_path)
    validation_run, _ = completed_validation(plan, tmp_path / "validation-run")
    verify_completed_prompt_d_validation(plan, validation_run)
    rows = json.loads((validation_run / "results.json").read_text())
    rows[0]["assignment"]["task_id"] = "lcbhard_70"
    (validation_run / "results.json").write_text(json.dumps(rows))
    with pytest.raises(ValueError, match="frozen assignment sequence"):
        verify_completed_prompt_d_validation(plan, validation_run)

    (validation_run / "results.json").write_text(json.dumps([]))


def test_validation_manifest_rejects_non_d_or_holdout_assignment(tmp_path):
    plan = tmp_path / "bad-plan.json"
    manifest = build_manifest(dataset_revision=REVISION)
    manifest["validation_assignments"][0]["prompt_variant"] = "A"
    from messageboardbench.calibration_run import canonical_manifest_sha256
    manifest["manifest_sha256"] = canonical_manifest_sha256(manifest)
    plan.write_text(json.dumps(manifest))
    with pytest.raises(ValueError, match="only preselected prompt D"):
        read_frozen_manifest(plan)