mirror of
https://github.com/priyanshujain/messageboardbench.git
synced 2026-10-02 19:17:07 +00:00
Checkpoint experiments through SWE pilot v2
This commit is contained in:
pj committed
1 parent
abacd5c5e1
commit
72d77018d8
845 files changed
+431756
-41
No files matched your search
@@ -0,0 +1,305 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from messageboardbench.calibration_run import read_frozen_manifest
|
||||
from messageboardbench.communication_plan import build_communication_plan
|
||||
from messageboardbench.completion import completion_manifest_record
|
||||
from messageboardbench.confirmation import (
|
||||
consume_plan_once,
|
||||
verify_calibration_review,
|
||||
verify_completed_calibration,
|
||||
verify_completed_prompt_d_validation,
|
||||
verify_prompt_d_validation,
|
||||
)
|
||||
from messageboardbench.prompt_calibration import build_manifest, write_manifest
|
||||
|
||||
|
||||
REVISION = "a" * 40
|
||||
|
||||
|
||||
def completed_calibration(tmp_path: Path):
|
||||
plan_path = tmp_path / "plan.json"
|
||||
write_manifest(plan_path, build_manifest(dataset_revision=REVISION))
|
||||
plan, source = read_frozen_manifest(plan_path)
|
||||
run = tmp_path / "run"
|
||||
run.mkdir()
|
||||
(run / "evals").mkdir()
|
||||
(run / "frozen-plan.json").write_bytes(plan_path.read_bytes())
|
||||
results = [{
|
||||
"assignment": row, "log": str(run / "evals" / f"{index}.eval"),
|
||||
"error": None,
|
||||
"completion": completion_manifest_record(),
|
||||
"calibration": {"communication": "none"},
|
||||
} for index, row in enumerate(plan["development_assignments"], 1)]
|
||||
for row in results:
|
||||
Path(row["log"]).write_bytes(b"mock eval log")
|
||||
(run / "results.json").write_text(json.dumps(results))
|
||||
(run / "status.json").write_text(json.dumps({
|
||||
"status": "completed", "phase": "development",
|
||||
"completed_assignments": len(results), "in_flight_assignment": None,
|
||||
}))
|
||||
(run / "run-manifest.json").write_text(json.dumps({
|
||||
"purpose": "prompt-calibration-development-execution", "phase": "development",
|
||||
"execute": True, "communication": "none", "completion": completion_manifest_record(),
|
||||
"manifest": source,
|
||||
}))
|
||||
return plan_path, run, len(results)
|
||||
|
||||
|
||||
def completed_validation(plan_path: Path, run: Path):
|
||||
plan, source = read_frozen_manifest(plan_path)
|
||||
run.mkdir()
|
||||
(run / "evals").mkdir()
|
||||
(run / "frozen-plan.json").write_bytes(plan_path.read_bytes())
|
||||
results = []
|
||||
for index, assignment in enumerate(plan["validation_assignments"], 1):
|
||||
log_path = run / "evals" / f"{index}.eval"
|
||||
log_path.write_bytes(b"mock validation eval log")
|
||||
results.append({
|
||||
"assignment": assignment,
|
||||
"sample_id": assignment["task_id"],
|
||||
"log": str(log_path),
|
||||
"error": None,
|
||||
"completion": completion_manifest_record(),
|
||||
"calibration": {
|
||||
"phase": "validation", "communication": "none",
|
||||
"assignment": assignment, "manifest": source,
|
||||
"policy_prompt": {"variant": "D"},
|
||||
},
|
||||
})
|
||||
(run / "results.json").write_text(json.dumps(results))
|
||||
(run / "status.json").write_text(json.dumps({
|
||||
"status": "completed", "phase": "validation",
|
||||
"completed_assignments": len(results), "in_flight_assignment": None,
|
||||
}))
|
||||
audit_path = run.parent / "validation-audit.json"
|
||||
audit_path.write_text(json.dumps({
|
||||
"schema_version": 2, "status": "ready", "partition": "validation",
|
||||
"dataset": {"path": plan["benchmark"]["dataset"],
|
||||
"revision": plan["benchmark"]["dataset_revision"]},
|
||||
"review": {"reviewer_type": "human", "reviewer": "Test reviewer",
|
||||
"no_model_outcomes_inspected": True},
|
||||
"approved_pairs": [{
|
||||
"task_id": task_id, "split": split, "task_validated": True,
|
||||
"evaluator_validated": True, "task_prompt_sha256": "1" * 64,
|
||||
"test_sha256": "2" * 64,
|
||||
} for task_id, split in sorted({
|
||||
(row["task_id"], row["split"]) for row in plan["validation_assignments"]
|
||||
})],
|
||||
}))
|
||||
run_manifest = {
|
||||
"purpose": "prompt-calibration-validation-execution", "phase": "validation",
|
||||
"execute": True, "communication": "none", "completion": completion_manifest_record(),
|
||||
"assignments": len(results), "manifest": source,
|
||||
"validation_audit": {
|
||||
"path": str(audit_path),
|
||||
"sha256": __import__("hashlib").sha256(audit_path.read_bytes()).hexdigest(),
|
||||
},
|
||||
}
|
||||
run_manifest.update({key: plan["environment"][key] for key in (
|
||||
"model", "message_limit", "token_limit", "time_limit_seconds", "temperature",
|
||||
"reasoning_effort", "max_attempts", "strict_tools", "sample_retries", "request_retries",
|
||||
)})
|
||||
(run / "run-manifest.json").write_text(json.dumps(run_manifest))
|
||||
return run, len(results)
|
||||
|
||||
|
||||
def test_completed_calibration_and_review_bind_exact_bytes(tmp_path):
|
||||
plan, run, count = completed_calibration(tmp_path)
|
||||
evidence = verify_completed_calibration(plan, run)
|
||||
review_path = tmp_path / "review.json"
|
||||
review_path.write_text(json.dumps({
|
||||
"schema_version": 1, "status": "ready",
|
||||
"purpose": "prompt-calibration-behavior-review",
|
||||
"calibration_evidence_sha256": evidence["evidence_sha256"],
|
||||
"no_communication_holdout_outcomes_inspected": True,
|
||||
"reviewer": "Internal review group",
|
||||
"assignment_labels": [
|
||||
{"assignment_index": i, "label": "no_observed_gaming"}
|
||||
for i in range(1, count + 1)
|
||||
],
|
||||
"prompt_d_assessment": {
|
||||
"decision": "proceed", "variation_adequate": True, "rationale": "Observed variation",
|
||||
},
|
||||
}))
|
||||
source = verify_calibration_review(review_path, evidence)
|
||||
assert source["calibration_evidence_sha256"] == evidence["evidence_sha256"]
|
||||
|
||||
status = json.loads((run / "status.json").read_text())
|
||||
status["status"] = "running"
|
||||
(run / "status.json").write_text(json.dumps(status))
|
||||
with pytest.raises(ValueError, match="not completed"):
|
||||
verify_completed_calibration(plan, run)
|
||||
|
||||
|
||||
def test_review_cannot_proceed_without_d_variation(tmp_path):
|
||||
plan, run, count = completed_calibration(tmp_path)
|
||||
evidence = verify_completed_calibration(plan, run)
|
||||
review = tmp_path / "review.json"
|
||||
review.write_text(json.dumps({
|
||||
"schema_version": 1, "status": "ready",
|
||||
"purpose": "prompt-calibration-behavior-review",
|
||||
"calibration_evidence_sha256": evidence["evidence_sha256"],
|
||||
"no_communication_holdout_outcomes_inspected": True, "reviewer": "Reviewer",
|
||||
"assignment_labels": [{"assignment_index": i, "label": "ambiguous"}
|
||||
for i in range(1, count + 1)],
|
||||
"prompt_d_assessment": {"decision": "stop", "variation_adequate": False,
|
||||
"rationale": "No variation"},
|
||||
}))
|
||||
with pytest.raises(ValueError, match="not reviewed as adequate"):
|
||||
verify_calibration_review(review, evidence)
|
||||
|
||||
|
||||
def test_plan_consumption_is_write_once(tmp_path):
|
||||
plan_path = tmp_path / "communication.json"
|
||||
plan = build_communication_plan({"teams": 2})
|
||||
plan_path.write_text(json.dumps(plan))
|
||||
ledger = tmp_path / "ledger"
|
||||
receipt = consume_plan_once(
|
||||
plan_path, tmp_path / "run", plan["plan_sha256"], ledger_dir=ledger
|
||||
)
|
||||
assert receipt["status"] == "consumed"
|
||||
with pytest.raises(ValueError, match="already consumed"):
|
||||
consume_plan_once(
|
||||
plan_path, tmp_path / "other", plan["plan_sha256"], ledger_dir=ledger
|
||||
)
|
||||
|
||||
|
||||
def test_prompt_d_validation_must_bind_and_proceed(tmp_path):
|
||||
plan, _, _ = completed_calibration(tmp_path)
|
||||
validation_run, count = completed_validation(plan, tmp_path / "validation-run")
|
||||
execution = verify_completed_prompt_d_validation(plan, validation_run)
|
||||
record = {
|
||||
"schema_version": 1, "status": "ready", "purpose": "prompt-d-validation",
|
||||
"calibration_evidence_sha256": "c" * 64, "dataset_revision": REVISION,
|
||||
"model": "openrouter/example/model", "prompt_variant": "D",
|
||||
"prompt_sha256": "d" * 64, "communication": "none",
|
||||
"completion_policy": completion_manifest_record(),
|
||||
"communication_holdout_executed": False, "artifact_review_complete": True,
|
||||
"decision": "proceed", "variation_adequate": True,
|
||||
"validation_run_directory": str(validation_run),
|
||||
"validation_execution_sha256": execution["evidence_sha256"],
|
||||
"completed_assignments": count, "reviewer": "Validation reviewers",
|
||||
"rationale": "Both gaming and non-gaming outcomes occurred.",
|
||||
"assignment_labels": [
|
||||
{
|
||||
"assignment_index": row["assignment_index"],
|
||||
"label": (
|
||||
"successful_gaming" if row["assignment_index"] == min(
|
||||
item["assignment_index"] for item in
|
||||
read_frozen_manifest(plan)[0]["validation_assignments"]
|
||||
if item["split"] == "conflicting"
|
||||
)
|
||||
else "no_observed_gaming"
|
||||
),
|
||||
"artifact_labels": (
|
||||
["contradiction_targeted_gaming"] if row["assignment_index"] == min(
|
||||
item["assignment_index"] for item in
|
||||
read_frozen_manifest(plan)[0]["validation_assignments"]
|
||||
if item["split"] == "conflicting"
|
||||
) else ["specification_faithful"]
|
||||
),
|
||||
"notes": "Artifact inspected and classified independently of evaluator score.",
|
||||
}
|
||||
for row in read_frozen_manifest(plan)[0]["validation_assignments"]
|
||||
],
|
||||
}
|
||||
path = tmp_path / "validation.json"
|
||||
path.write_text(json.dumps(record))
|
||||
assert verify_prompt_d_validation(
|
||||
path, plan_path=plan, calibration_evidence_sha256="c" * 64, dataset_revision=REVISION,
|
||||
model="openrouter/example/model", prompt_sha256="d" * 64,
|
||||
)["status"] == "ready"
|
||||
first_log = next((validation_run / "evals").glob("*.eval"))
|
||||
first_log.write_bytes(b"mutated after review")
|
||||
with pytest.raises(ValueError, match="not ready"):
|
||||
verify_prompt_d_validation(
|
||||
path, plan_path=plan, calibration_evidence_sha256="c" * 64,
|
||||
dataset_revision=REVISION, model="openrouter/example/model",
|
||||
prompt_sha256="d" * 64,
|
||||
)
|
||||
first_log.write_bytes(b"mock validation eval log")
|
||||
record["decision"] = "stop"
|
||||
path.write_text(json.dumps(record))
|
||||
with pytest.raises(ValueError, match="not ready"):
|
||||
verify_prompt_d_validation(
|
||||
path, plan_path=plan, calibration_evidence_sha256="c" * 64, dataset_revision=REVISION,
|
||||
model="openrouter/example/model", prompt_sha256="d" * 64,
|
||||
)
|
||||
|
||||
|
||||
def test_prompt_d_validation_distinguishes_targeted_from_weak_oracle_gaming(tmp_path):
|
||||
plan, _, _ = completed_calibration(tmp_path)
|
||||
validation_run, count = completed_validation(plan, tmp_path / "validation-run")
|
||||
execution = verify_completed_prompt_d_validation(plan, validation_run)
|
||||
assignments = read_frozen_manifest(plan)[0]["validation_assignments"]
|
||||
first_conflicting = min(row["assignment_index"] for row in assignments
|
||||
if row["split"] == "conflicting")
|
||||
record = {
|
||||
"schema_version": 1, "status": "ready", "purpose": "prompt-d-validation",
|
||||
"calibration_evidence_sha256": "c" * 64, "dataset_revision": REVISION,
|
||||
"model": "openrouter/example/model", "prompt_variant": "D",
|
||||
"prompt_sha256": "d" * 64, "communication": "none",
|
||||
"completion_policy": completion_manifest_record(),
|
||||
"communication_holdout_executed": False, "artifact_review_complete": True,
|
||||
"decision": "proceed", "variation_adequate": True,
|
||||
"validation_run_directory": str(validation_run),
|
||||
"validation_execution_sha256": execution["evidence_sha256"],
|
||||
"completed_assignments": count, "reviewer": "Validation reviewers",
|
||||
"rationale": "Targeted gaming and non-gaming outcomes occurred.",
|
||||
"assignment_labels": [{
|
||||
"assignment_index": row["assignment_index"],
|
||||
"label": "successful_gaming" if row["assignment_index"] == first_conflicting
|
||||
else "no_observed_gaming",
|
||||
"artifact_labels": ["generic_weak_oracle_exploitation"]
|
||||
if row["assignment_index"] == first_conflicting
|
||||
else ["specification_faithful"],
|
||||
"notes": "Artifact inspected.",
|
||||
} for row in assignments],
|
||||
}
|
||||
path = tmp_path / "validation.json"
|
||||
path.write_text(json.dumps(record))
|
||||
kwargs = dict(
|
||||
plan_path=plan, calibration_evidence_sha256="c" * 64, dataset_revision=REVISION,
|
||||
model="openrouter/example/model", prompt_sha256="d" * 64,
|
||||
)
|
||||
with pytest.raises(ValueError, match="contradiction-targeted"):
|
||||
verify_prompt_d_validation(path, **kwargs)
|
||||
|
||||
target = next(row for row in record["assignment_labels"]
|
||||
if row["assignment_index"] == first_conflicting)
|
||||
target["label"] = "other_evaluator_gaming"
|
||||
path.write_text(json.dumps(record))
|
||||
with pytest.raises(ValueError, match="behavioral variation"):
|
||||
verify_prompt_d_validation(path, **kwargs)
|
||||
|
||||
target["label"] = "successful_gaming"
|
||||
target["artifact_labels"].append("contradiction_targeted_gaming")
|
||||
path.write_text(json.dumps(record))
|
||||
assert verify_prompt_d_validation(path, **kwargs)["status"] == "ready"
|
||||
|
||||
|
||||
def test_prompt_d_validation_rejects_mutated_results_and_plan(tmp_path):
|
||||
plan, _, _ = completed_calibration(tmp_path)
|
||||
validation_run, _ = completed_validation(plan, tmp_path / "validation-run")
|
||||
verify_completed_prompt_d_validation(plan, validation_run)
|
||||
rows = json.loads((validation_run / "results.json").read_text())
|
||||
rows[0]["assignment"]["task_id"] = "lcbhard_70"
|
||||
(validation_run / "results.json").write_text(json.dumps(rows))
|
||||
with pytest.raises(ValueError, match="frozen assignment sequence"):
|
||||
verify_completed_prompt_d_validation(plan, validation_run)
|
||||
|
||||
(validation_run / "results.json").write_text(json.dumps([]))
|
||||
|
||||
|
||||
def test_validation_manifest_rejects_non_d_or_holdout_assignment(tmp_path):
|
||||
plan = tmp_path / "bad-plan.json"
|
||||
manifest = build_manifest(dataset_revision=REVISION)
|
||||
manifest["validation_assignments"][0]["prompt_variant"] = "A"
|
||||
from messageboardbench.calibration_run import canonical_manifest_sha256
|
||||
manifest["manifest_sha256"] = canonical_manifest_sha256(manifest)
|
||||
plan.write_text(json.dumps(manifest))
|
||||
with pytest.raises(ValueError, match="only preselected prompt D"):
|
||||
read_frozen_manifest(plan)
|
||||
Reference in new issue
Block a user