mirror of
https://github.com/priyanshujain/messageboardbench.git
synced 2026-10-02 11:07:07 +00:00
287 lines
11 KiB
Python
287 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
from copy import deepcopy
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
from inspect_ai.dataset import Sample
|
|
|
|
from messageboardbench.calibration_run import (
|
|
canonical_manifest_sha256,
|
|
prepare_development_samples,
|
|
read_frozen_manifest,
|
|
)
|
|
from messageboardbench.prompt_calibration import (
|
|
DEFAULT_PARTITIONS,
|
|
TaskPartitions,
|
|
build_manifest,
|
|
render_tools_instruction,
|
|
)
|
|
|
|
|
|
REVISION = "c" * 40
|
|
|
|
|
|
def write_manifest(path: Path, manifest: dict) -> Path:
|
|
path.write_text(json.dumps(manifest, indent=2) + "\n")
|
|
return path
|
|
|
|
|
|
def test_reads_exact_self_hashed_immutable_manifest(tmp_path):
|
|
manifest = build_manifest(dataset_revision=REVISION)
|
|
path = write_manifest(tmp_path / "plan.json", manifest)
|
|
loaded, source = read_frozen_manifest(path)
|
|
assert loaded == manifest
|
|
assert source["manifest_sha256"] == canonical_manifest_sha256(manifest)
|
|
assert source["file_sha256"]
|
|
|
|
tampered = deepcopy(manifest)
|
|
tampered["environment"]["temperature"] = 0
|
|
write_manifest(tmp_path / "tampered.json", tampered)
|
|
with pytest.raises(ValueError, match="self-hash mismatch"):
|
|
read_frozen_manifest(tmp_path / "tampered.json")
|
|
|
|
mutable = deepcopy(manifest)
|
|
mutable["benchmark"]["dataset_revision"] = "main"
|
|
mutable["manifest_sha256"] = canonical_manifest_sha256(mutable)
|
|
write_manifest(tmp_path / "mutable.json", mutable)
|
|
with pytest.raises(ValueError, match="40-character"):
|
|
read_frozen_manifest(tmp_path / "mutable.json")
|
|
|
|
|
|
def test_rejects_default_communication_holdout_even_if_redeclared(tmp_path):
|
|
partitions = TaskPartitions(
|
|
development=(DEFAULT_PARTITIONS.communication_holdout[0],),
|
|
validation=("validation-x",),
|
|
communication_holdout=("holdout-x",),
|
|
)
|
|
manifest = build_manifest(dataset_revision=REVISION, partitions=partitions)
|
|
path = write_manifest(tmp_path / "bad-plan.json", manifest)
|
|
with pytest.raises(ValueError, match="communication holdout"):
|
|
read_frozen_manifest(path)
|
|
|
|
|
|
def test_manifest_binds_generation_and_retry_settings(tmp_path):
|
|
manifest = build_manifest(dataset_revision=REVISION)
|
|
environment = manifest["environment"]
|
|
assert environment["temperature"] == 1
|
|
assert environment["reasoning_effort"] == "high"
|
|
assert environment["strict_tools"] is False
|
|
assert environment["sample_retries"] == 0
|
|
assert environment["request_retries"] == 1
|
|
assert environment["assignment_concurrency"] == 1
|
|
|
|
for field, value in (
|
|
("strict_tools", True),
|
|
("sample_retries", 1),
|
|
("request_retries", 2),
|
|
("assignment_concurrency", 2),
|
|
):
|
|
changed = deepcopy(manifest)
|
|
changed["environment"][field] = value
|
|
changed["manifest_sha256"] = canonical_manifest_sha256(changed)
|
|
path = write_manifest(tmp_path / f"bad-{field}.json", changed)
|
|
with pytest.raises(ValueError, match=field):
|
|
read_frozen_manifest(path)
|
|
|
|
|
|
def test_prepared_samples_preserve_prompt_provenance_and_never_load_holdout():
|
|
partitions = TaskPartitions(
|
|
development=("dev-1",), validation=("val-1",), communication_holdout=("hold-1",)
|
|
)
|
|
manifest = build_manifest(dataset_revision=REVISION, partitions=partitions)
|
|
calls = []
|
|
|
|
def loader(revision):
|
|
calls.append(revision)
|
|
base = render_tools_instruction("A")
|
|
return {
|
|
split: {
|
|
"dev-1": Sample(
|
|
id="dev-1",
|
|
input=base,
|
|
metadata={
|
|
"instruction_prompt": base,
|
|
"prompt": "def candidate(x):",
|
|
"test": "def check(candidate): pass",
|
|
"entry_point": "candidate",
|
|
"impossible_type": split,
|
|
},
|
|
)
|
|
}
|
|
for split in ("original", "conflicting")
|
|
}
|
|
|
|
source = {"path": "/plan.json", "file_sha256": "f" * 64,
|
|
"manifest_sha256": manifest["manifest_sha256"]}
|
|
prepared = prepare_development_samples(manifest, source, loader=loader)
|
|
assert calls == [REVISION]
|
|
assert len(prepared) == 8
|
|
assert {row["assignment"]["task_id"] for row in prepared} == {"dev-1"}
|
|
for row in prepared:
|
|
metadata = row["sample"].metadata
|
|
provenance = metadata["calibration"]
|
|
assert provenance["communication"] == "none"
|
|
assert provenance["manifest"] == source
|
|
assert provenance["completion"] == metadata["completion"]
|
|
assert provenance["policy_prompt"]["rendered_instruction_prompt"] == row["sample"].input
|
|
assert provenance["task_prompt_sha256"]
|
|
assert provenance["test_sha256"]
|
|
|
|
|
|
def load_runner():
|
|
spec = importlib.util.spec_from_file_location(
|
|
"run_prompt_calibration", Path(__file__).parents[1] / "scripts/run_prompt_calibration.py"
|
|
)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def test_runner_preview_does_not_load_dataset_create_output_or_execute(tmp_path, monkeypatch, capsys):
|
|
runner = load_runner()
|
|
plan = write_manifest(tmp_path / "plan.json", build_manifest(dataset_revision=REVISION))
|
|
out = tmp_path / "run"
|
|
monkeypatch.setattr(
|
|
runner,
|
|
"prepare_development_samples",
|
|
lambda *args, **kwargs: pytest.fail("preview must not load the dataset"),
|
|
)
|
|
assert runner.main(["--manifest", str(plan), "--out", str(out)]) == 0
|
|
printed = capsys.readouterr().out
|
|
assert '"execute": false' in printed
|
|
assert "Preview only" in printed
|
|
assert not out.exists()
|
|
|
|
|
|
def test_execute_requires_remote_docker_wrapper_before_loading_dataset(tmp_path, monkeypatch):
|
|
runner = load_runner()
|
|
plan = write_manifest(tmp_path / "plan.json", build_manifest(dataset_revision=REVISION))
|
|
monkeypatch.delenv("DOCKER_HOST", raising=False)
|
|
monkeypatch.setattr(
|
|
runner,
|
|
"prepare_development_samples",
|
|
lambda *args, **kwargs: pytest.fail("wrong Docker host must fail before dataset loading"),
|
|
)
|
|
with pytest.raises(RuntimeError, match="remote Docker daemon"):
|
|
runner.main([
|
|
"--manifest", str(plan), "--out", str(tmp_path / "run"), "--execute"
|
|
])
|
|
|
|
|
|
def test_resume_requires_execute(tmp_path):
|
|
runner = load_runner()
|
|
plan = write_manifest(tmp_path / "plan.json", build_manifest(dataset_revision=REVISION))
|
|
with pytest.raises(SystemExit):
|
|
runner.main([
|
|
"--manifest", str(plan), "--out", str(tmp_path / "run"), "--resume"
|
|
])
|
|
|
|
|
|
def test_resume_refuses_an_in_flight_assignment(tmp_path, monkeypatch):
|
|
runner = load_runner()
|
|
manifest = build_manifest(dataset_revision=REVISION)
|
|
plan = write_manifest(tmp_path / "plan.json", manifest)
|
|
_, source = read_frozen_manifest(plan)
|
|
out = tmp_path / "run"
|
|
out.mkdir()
|
|
(out / "frozen-plan.json").write_bytes(plan.read_bytes())
|
|
(out / "run-manifest.json").write_text(json.dumps({"manifest": source}))
|
|
(out / "status.json").write_text(json.dumps({
|
|
"status": "interrupted", "phase": "development",
|
|
"completed_assignments": 0, "in_flight_assignment": 1,
|
|
}))
|
|
(out / "budget-before.json").write_text(json.dumps({"usage": 0.0}))
|
|
monkeypatch.setattr(runner, "prepare_development_samples",
|
|
lambda *args: [None] * len(manifest["development_assignments"]))
|
|
monkeypatch.setattr(runner, "budget", lambda: {
|
|
"usage": 0.0, "limit": 5.0, "limit_remaining": 5.0
|
|
})
|
|
monkeypatch.setenv("DOCKER_HOST", runner.REMOTE_DOCKER_HOST)
|
|
with pytest.raises(ValueError, match="implicit sample retry"):
|
|
runner.main([
|
|
"--manifest", str(plan), "--out", str(out), "--execute", "--resume"
|
|
])
|
|
|
|
|
|
def test_mock_execution_uses_only_frozen_settings_and_preserves_results(
|
|
tmp_path, monkeypatch
|
|
):
|
|
runner = load_runner()
|
|
manifest = build_manifest(dataset_revision=REVISION, temperature=0.4,
|
|
reasoning_effort="low")
|
|
assignment = manifest["development_assignments"][0]
|
|
manifest["development_assignments"] = [assignment]
|
|
plan = write_manifest(tmp_path / "plan.json", manifest)
|
|
source = {"path": str(plan.resolve()), "file_sha256": "e" * 64,
|
|
"manifest_sha256": manifest["manifest_sha256"]}
|
|
monkeypatch.setattr(runner, "read_frozen_manifest", lambda path: (manifest, source))
|
|
base = render_tools_instruction(assignment["prompt_variant"])
|
|
sample = Sample(
|
|
id=assignment["task_id"], input=base,
|
|
metadata={
|
|
"instruction_prompt": base,
|
|
"prompt": "def candidate(x):",
|
|
"test": "def check(candidate): pass",
|
|
"entry_point": "candidate",
|
|
"calibration": {"communication": "none", "completion":
|
|
manifest["environment"]["completion_policy"]},
|
|
"completion": manifest["environment"]["completion_policy"],
|
|
},
|
|
)
|
|
monkeypatch.setattr(runner, "prepare_development_samples", lambda *args: [{
|
|
"assignment": assignment, "sample": sample, "provenance": sample.metadata["calibration"]
|
|
}])
|
|
budget_values = iter([
|
|
{"usage": 1.0, "limit": 5.0, "limit_remaining": 4.0},
|
|
{"usage": 1.1, "limit": 5.0, "limit_remaining": 3.9},
|
|
])
|
|
monkeypatch.setattr(runner, "budget", lambda: next(budget_values))
|
|
monkeypatch.setenv("DOCKER_HOST", runner.REMOTE_DOCKER_HOST)
|
|
|
|
import inspect_ai
|
|
import messageboardbench.board_task as board_task
|
|
import messageboardbench.task as task_module
|
|
monkeypatch.setattr(inspect_ai, "Task", lambda **kwargs: SimpleNamespace(**kwargs))
|
|
solver_calls = []
|
|
monkeypatch.setattr(
|
|
board_task, "episode_solver",
|
|
lambda *args, **kwargs: solver_calls.append((args, kwargs)) or "private-solver",
|
|
)
|
|
monkeypatch.setattr(task_module, "scratch_scorer", lambda split: f"scorer-{split}")
|
|
calls = []
|
|
|
|
def fake_eval(tasks, **kwargs):
|
|
calls.append((tasks, kwargs))
|
|
score = SimpleNamespace(value="C", metadata={"scratch_files": {},
|
|
"test_modified_ever": False})
|
|
returned = SimpleNamespace(
|
|
id=sample.id, scores={"score": score}, messages=[], model_usage={},
|
|
limit=None, error=None, metadata=sample.metadata,
|
|
)
|
|
return [SimpleNamespace(location="mock.eval", status="success", samples=[returned])]
|
|
|
|
monkeypatch.setattr(inspect_ai, "eval", fake_eval)
|
|
out = tmp_path / "run"
|
|
assert runner.main([
|
|
"--manifest", str(plan), "--out", str(out), "--execute"
|
|
]) == 0
|
|
assert len(calls) == 1
|
|
task, kwargs = calls[0][0][0], calls[0][1]
|
|
assert task.solver == "private-solver"
|
|
assert solver_calls[0][1] == {"completion_mode": "plain-final"}
|
|
assert kwargs["temperature"] == 0.4
|
|
assert kwargs["reasoning_effort"] == "low"
|
|
assert kwargs["model_args"] == {"strict_tools": False}
|
|
assert kwargs["retry_on_error"] == 0 and kwargs["max_retries"] == 1
|
|
result = json.loads((out / "results.json").read_text())[0]
|
|
assert result["assignment"] == assignment
|
|
assert result["calibration"]["communication"] == "none"
|
|
assert result["completion"] == manifest["environment"]["completion_policy"]
|
|
status = json.loads((out / "status.json").read_text())
|
|
assert status["status"] == "completed"
|
|
assert status["in_flight_assignment"] is None
|