Checkpoint experiments through SWE pilot v2

tests/test_prompt_validation.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+from inspect_ai.dataset import Sample
+
+from messageboardbench.calibration_run import prepare_validation_samples
+from messageboardbench.prompt_calibration import TaskPartitions, build_manifest, render_tools_instruction
+
+
+REVISION = "e" * 40
+PROMPT = "def candidate(x): pass"
+TEST = "def check(candidate): pass"
+
+
+def load_runner():
+    spec = importlib.util.spec_from_file_location(
+        "run_prompt_validation", Path(__file__).parents[1] / "scripts/run_prompt_validation.py"
+    )
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def plan_fixture(tmp_path: Path):
+    partitions = TaskPartitions(
+        development=("dev",), validation=("val",), communication_holdout=("hold",),
+    )
+    manifest = build_manifest(dataset_revision=REVISION, partitions=partitions)
+    path = tmp_path / "plan.json"
+    path.write_text(json.dumps(manifest))
+    audit = tmp_path / "validation-audit.json"
+    pairs = {(row["task_id"], row["split"]) for row in manifest["validation_assignments"]}
+    audit.write_text(json.dumps({
+        "schema_version": 2, "status": "ready", "partition": "validation",
+        "dataset": {"path": manifest["benchmark"]["dataset"], "revision": REVISION},
+        "review": {"reviewer_type": "human", "reviewer": "Test reviewer",
+                   "no_model_outcomes_inspected": True},
+        "approved_pairs": [{
+            "task_id": task_id, "split": split, "task_validated": True,
+            "evaluator_validated": True,
+            "task_prompt_sha256": hashlib.sha256(PROMPT.encode()).hexdigest(),
+            "test_sha256": hashlib.sha256(TEST.encode()).hexdigest(),
+        } for task_id, split in sorted(pairs)],
+    }))
+    return manifest, path, audit
+
+
+def sample_loader(revision):
+    base = render_tools_instruction("A")
+    return {split: {"val": Sample(
+        id="val", input=base, metadata={"instruction_prompt": base, "prompt": PROMPT,
+                                        "test": TEST, "entry_point": "candidate"},
+    )} for split in ("original", "conflicting")}
+
+
+def test_prepare_validation_uses_only_validation_d():
+    manifest = build_manifest(
+        dataset_revision=REVISION,
+        partitions=TaskPartitions(development=("dev",), validation=("val",),
+                                  communication_holdout=("hold",)),
+    )
+    source = {"path": "/plan", "file_sha256": "a" * 64,
+              "manifest_sha256": manifest["manifest_sha256"]}
+    prepared = prepare_validation_samples(manifest, source, loader=sample_loader)
+    assert {row["assignment"]["task_id"] for row in prepared} == {"val"}
+    assert {row["assignment"]["prompt_variant"] for row in prepared} == {"D"}
+    assert all(row["sample"].metadata["calibration"]["phase"] == "validation"
+               for row in prepared)
+
+
+def test_preview_is_offline_and_does_not_consume(tmp_path, monkeypatch, capsys):
+    runner = load_runner()
+    _, plan, audit = plan_fixture(tmp_path)
+    out = tmp_path / "run"
+    monkeypatch.setattr(runner, "prepare_validation_samples",
+                        lambda *args: pytest.fail("preview loaded dataset"))
+    monkeypatch.setattr(runner, "LEDGER_DIR", tmp_path / "ledger")
+    assert runner.main(["--manifest", str(plan), "--validation-audit", str(audit),
+                        "--out", str(out)]) == 0
+    assert "Preview only" in capsys.readouterr().out
+    assert not out.exists() and not (tmp_path / "ledger").exists()
+
+
+def test_execute_requires_remote_docker_before_dataset(tmp_path, monkeypatch):
+    runner = load_runner()
+    _, plan, audit = plan_fixture(tmp_path)
+    monkeypatch.delenv("DOCKER_HOST", raising=False)
+    monkeypatch.setattr(runner, "prepare_validation_samples",
+                        lambda *args: pytest.fail("wrong host loaded dataset"))
+    with pytest.raises(RuntimeError, match="remote Docker daemon"):
+        runner.main(["--manifest", str(plan), "--validation-audit", str(audit),
+                     "--out", str(tmp_path / "run"), "--execute"])
+
+
+def test_mock_execution_writes_gate_compatible_run_and_is_one_shot(tmp_path, monkeypatch):
+    runner = load_runner()
+    manifest, plan, audit = plan_fixture(tmp_path)
+    monkeypatch.setattr(
+        runner, "prepare_validation_samples",
+        lambda loaded, source: prepare_validation_samples(loaded, source, loader=sample_loader),
+    )
+    monkeypatch.setattr(runner, "LEDGER_DIR", tmp_path / "ledger")
+    monkeypatch.setenv("DOCKER_HOST", runner.REMOTE_DOCKER_HOST)
+    budgets = iter([{"usage": 1, "limit": 5, "limit_remaining": 4},
+                    {"usage": 1.1, "limit": 5, "limit_remaining": 3.9}])
+    monkeypatch.setattr(runner, "budget", lambda: next(budgets))
+    import inspect_ai
+    import messageboardbench.board_task as board_task
+    import messageboardbench.task as task_module
+    monkeypatch.setattr(inspect_ai, "Task", lambda **kwargs: SimpleNamespace(**kwargs))
+    monkeypatch.setattr(board_task, "episode_solver", lambda *args, **kwargs: "solver")
+    monkeypatch.setattr(task_module, "scratch_scorer", lambda split: "scorer")
+    out = tmp_path / "run"
+
+    eval_calls = []
+    def fake_eval(tasks, **kwargs):
+        eval_calls.append(tasks)
+        log_path = out / "evals" / f"mock-{len(eval_calls)}.eval"
+        log_path.parent.mkdir(exist_ok=True)
+        log_path.write_bytes(b"eval")
+        sample = tasks[0].dataset[0]
+        score = SimpleNamespace(value="I", metadata={})
+        returned = SimpleNamespace(id=sample.id, scores={"score": score}, messages=[],
+                                   model_usage={}, limit=None, error=None, metadata=sample.metadata)
+        return [SimpleNamespace(location=str(log_path), status="success", samples=[returned])]
+
+    monkeypatch.setattr(inspect_ai, "eval", fake_eval)
+    assert runner.main(["--manifest", str(plan), "--validation-audit", str(audit),
+                        "--out", str(out), "--execute"]) == 0
+    run_manifest = json.loads((out / "run-manifest.json").read_text())
+    assert run_manifest["phase"] == "validation"
+    assert run_manifest["development_assignments_executed"] is False
+    assert run_manifest["communication_holdout_assignments_executed"] is False
+    assert json.loads((out / "status.json").read_text())["status"] == "completed"
+    from messageboardbench.confirmation import verify_completed_prompt_d_validation
+    evidence = verify_completed_prompt_d_validation(plan, out)
+    assert evidence["completed_assignments"] == len(manifest["validation_assignments"])
+    with pytest.raises(ValueError, match="already consumed"):
+        runner.consume_once(manifest, plan, tmp_path / "other")