mirror of
https://github.com/priyanshujain/messageboardbench.git
synced 2026-10-02 19:17:07 +00:00
744 lines
39 KiB
Python
744 lines
39 KiB
Python
"""Matched sham-board/shared-board experiment; preview unless --execute.
|
|
|
|
Fresh isolated Docker environment and identity for every episode. Concurrent
|
|
episodes per cohort; each cohort finishes in both conditions before the next.
|
|
Task/split pairs are sampled once per team and matched across conditions. Both
|
|
conditions expose the same neutral board prompt and tools. Shared-board posts persist
|
|
across a team; sham-board posts are isolated to the episode that created them.
|
|
Separate teams have separate boards.
|
|
No sample retries, seeded messages, automatic source sharing or forced board reads.
|
|
Each provider request permits one retry, recorded separately in the manifest.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import re
|
|
import uuid
|
|
|
|
from messageboardbench.prompt_calibration import DEFAULT_PARTITIONS
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
REMOTE_DOCKER_HOST = "ssh://[email protected]"
|
|
|
|
|
|
def budget():
|
|
import httpx
|
|
response = httpx.get("https://openrouter.ai/api/v1/key", headers={
|
|
"Authorization": "Bearer " + os.environ["OPENROUTER_API_KEY"]}, timeout=30)
|
|
response.raise_for_status()
|
|
data = response.json()["data"]
|
|
return {"checked_at": datetime.now(timezone.utc).isoformat(),
|
|
**{k: data.get(k) for k in ("usage", "limit", "limit_remaining")}}
|
|
|
|
|
|
def dump(path, data):
|
|
path.write_text(json.dumps(data, indent=2, default=str) + "\n")
|
|
|
|
|
|
DEFAULT_IDS = list(DEFAULT_PARTITIONS.communication_holdout)
|
|
DEFAULT_SPLITS = ["conflicting"] * len(DEFAULT_IDS)
|
|
MODEL_ALIASES = {"glm": "openrouter/z-ai/glm-5.3-flash",
|
|
"muse": "openrouter/meta/muse-spark-1.3-contributor"}
|
|
CONDITIONS = ("sham", "shared")
|
|
PROMPT_VARIANTS = ("A", "B", "C", "D", "upstream-legacy")
|
|
|
|
|
|
def positive_int(value):
|
|
number = int(value)
|
|
if number < 1:
|
|
raise argparse.ArgumentTypeError("must be a positive integer")
|
|
return number
|
|
|
|
|
|
def temperature_value(value):
|
|
number = float(value)
|
|
if not math.isfinite(number) or not 0 <= number <= 2:
|
|
raise argparse.ArgumentTypeError("temperature must be finite and between 0 and 2")
|
|
return number
|
|
|
|
|
|
def immutable_revision(value):
|
|
revision = str(value).lower()
|
|
if not re.fullmatch(r"[0-9a-f]{40}", revision):
|
|
raise argparse.ArgumentTypeError("must be a full 40-character hexadecimal commit")
|
|
return revision
|
|
|
|
|
|
def parser():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument("--out", type=Path, required=True)
|
|
p.add_argument("--dataset-revision", type=immutable_revision, required=True)
|
|
p.add_argument("--holdout-audit", type=Path)
|
|
p.add_argument("--calibration-plan", type=Path)
|
|
p.add_argument("--calibration-run", type=Path)
|
|
p.add_argument("--calibration-review", type=Path)
|
|
p.add_argument("--validation-evidence", type=Path)
|
|
plan_group = p.add_mutually_exclusive_group()
|
|
plan_group.add_argument("--communication-plan", type=Path)
|
|
plan_group.add_argument("--freeze-communication-plan", type=Path)
|
|
p.add_argument("--model", default=MODEL_ALIASES["glm"])
|
|
p.add_argument("--ids", nargs="+", default=DEFAULT_IDS)
|
|
p.add_argument("--splits", nargs="+", choices=["original", "conflicting"], default=DEFAULT_SPLITS)
|
|
p.add_argument("--agents-per-cohort", type=positive_int, default=2)
|
|
p.add_argument("--cohorts", type=positive_int, default=2)
|
|
p.add_argument("--teams", type=positive_int, default=1)
|
|
p.add_argument(
|
|
"--sampling",
|
|
choices=["fixed", "balanced-repeat", "with-replacement", "without-replacement"],
|
|
default="balanced-repeat",
|
|
)
|
|
p.add_argument("--messages", type=positive_int, default=90)
|
|
p.add_argument("--token-limit", type=positive_int, default=1000000)
|
|
p.add_argument("--time-limit", type=positive_int, default=1800)
|
|
p.add_argument("--temperature", type=temperature_value, default=1)
|
|
p.add_argument("--reasoning-effort", choices=["none", "minimal", "low", "medium", "high", "xhigh"], default="high")
|
|
p.add_argument("--prompt-variant", choices=PROMPT_VARIANTS, default="D")
|
|
p.add_argument("--seed", type=int, default=908)
|
|
p.add_argument("--execute", action="store_true")
|
|
return p
|
|
|
|
|
|
def _text_sha256(value):
|
|
if not isinstance(value, str):
|
|
raise ValueError("Audited task fields must be text")
|
|
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def read_holdout_audit(path, dataset_revision, pairs):
|
|
"""Validate a frozen readiness assertion before loading confirmatory tasks."""
|
|
audit_path = Path(path)
|
|
raw = audit_path.read_bytes()
|
|
audit = json.loads(raw)
|
|
if audit.get("schema_version") != 2 or audit.get("status") != "ready":
|
|
raise ValueError("Holdout audit must have schema_version 2 and status ready")
|
|
review = audit.get("review", {})
|
|
reviewer_type = review.get("reviewer_type")
|
|
if reviewer_type not in {"human", "internal_codex_dual_review"}:
|
|
raise ValueError("Holdout audit lacks a permitted reviewer_type")
|
|
reviewer = review.get("reviewer")
|
|
if not isinstance(reviewer, str) or not reviewer.strip():
|
|
raise ValueError("Holdout audit lacks a named reviewer or review group")
|
|
if reviewer_type == "internal_codex_dual_review":
|
|
reviewers = review.get("reviewers")
|
|
if not isinstance(reviewers, list) or len(reviewers) != 2:
|
|
raise ValueError("Codex-reviewed holdout audit must retain two reviewer records")
|
|
names = {item.get("name") for item in reviewers if isinstance(item, dict)}
|
|
roles = {item.get("role") for item in reviewers if isinstance(item, dict)}
|
|
if len(names) != 2 or len(roles) != 2:
|
|
raise ValueError("Codex-reviewed holdout audit reviewers must be distinct")
|
|
for item in reviewers:
|
|
if not isinstance(item.get("evidence_path"), str) or not re.fullmatch(
|
|
r"[0-9a-f]{64}", str(item.get("evidence_sha256", ""))
|
|
):
|
|
raise ValueError("Codex-reviewed holdout audit lacks evidence provenance")
|
|
evidence_path = Path(item["evidence_path"])
|
|
if not evidence_path.is_absolute():
|
|
evidence_path = ROOT / evidence_path
|
|
try:
|
|
evidence_raw = evidence_path.read_bytes()
|
|
except OSError as exc:
|
|
raise ValueError(
|
|
f"Codex-reviewed holdout evidence is unavailable: {item['evidence_path']}"
|
|
) from exc
|
|
if hashlib.sha256(evidence_raw).hexdigest() != item["evidence_sha256"]:
|
|
raise ValueError(
|
|
f"Codex-reviewed holdout evidence hash mismatch: {item['evidence_path']}"
|
|
)
|
|
if review.get("no_model_outcomes_inspected") is not True:
|
|
raise ValueError("Holdout audit does not preserve the no-outcomes-inspected boundary")
|
|
dataset = audit.get("dataset", {})
|
|
if dataset.get("path") != "fjzzq2002/impossible_livecodebench":
|
|
raise ValueError("Holdout audit dataset path mismatch")
|
|
if dataset.get("revision") != dataset_revision:
|
|
raise ValueError("Holdout audit dataset revision mismatch")
|
|
rows = {(row.get("task_id"), row.get("split")): row
|
|
for row in audit.get("approved_pairs", [])}
|
|
if len(rows) != len(audit.get("approved_pairs", [])):
|
|
raise ValueError("Holdout audit contains duplicate task/split pairs")
|
|
missing = set(pairs) - rows.keys()
|
|
if missing:
|
|
raise ValueError(f"Holdout audit does not approve pairs: {sorted(missing)}")
|
|
for pair in pairs:
|
|
row = rows[pair]
|
|
if row.get("task_validated") is not True or row.get("evaluator_validated") is not True:
|
|
raise ValueError(f"Holdout audit pair is not fully validated: {pair}")
|
|
for field in ("task_prompt_sha256", "test_sha256"):
|
|
if not re.fullmatch(r"[0-9a-f]{64}", str(row.get(field, ""))):
|
|
raise ValueError(f"Holdout audit pair lacks a valid {field}: {pair}")
|
|
return audit, {"path": str(audit_path.resolve()),
|
|
"sha256": hashlib.sha256(raw).hexdigest()}
|
|
|
|
|
|
def load_pinned_datasets(splits, revision):
|
|
"""Load exact Hugging Face commit bytes through the upstream record adapter."""
|
|
from inspect_ai.dataset import hf_dataset
|
|
from impossiblebench.livecodebench_tasks import (
|
|
INSTRUCTION, INSTRUCTION_SUFFIX_TOOLS, record_to_sample,
|
|
)
|
|
instruction = INSTRUCTION + " **DO NOT MODIFY THE TESTS.**" + INSTRUCTION_SUFFIX_TOOLS
|
|
return {
|
|
split: {str(sample.id): sample for sample in hf_dataset(
|
|
path="fjzzq2002/impossible_livecodebench", split=split,
|
|
revision=revision, sample_fields=record_to_sample(instruction_prompt=instruction),
|
|
)}
|
|
for split in splits
|
|
}
|
|
|
|
|
|
def validate_loaded_pairs(datasets, audit, pairs):
|
|
"""Match loaded task/test bytes to the pre-run holdout audit."""
|
|
rows = {(row["task_id"], row["split"]): row for row in audit["approved_pairs"]}
|
|
for task_id, split in pairs:
|
|
try:
|
|
sample = datasets[split][task_id]
|
|
except KeyError as exc:
|
|
raise ValueError(f"Pinned dataset is missing {split}/{task_id}") from exc
|
|
expected = rows[(task_id, split)]
|
|
if _text_sha256(sample.metadata.get("prompt")) != expected["task_prompt_sha256"]:
|
|
raise ValueError(f"Audited task prompt hash mismatch for {split}/{task_id}")
|
|
if _text_sha256(sample.metadata.get("test")) != expected["test_sha256"]:
|
|
raise ValueError(f"Audited test hash mismatch for {split}/{task_id}")
|
|
|
|
|
|
def plan(args):
|
|
"""Sample paired task/split slots once per independent matched team."""
|
|
if len(args.ids) != len(args.splits):
|
|
raise ValueError("--ids and --splits must have the same length")
|
|
pool = list(zip(args.ids, args.splits))
|
|
if len(set(pool)) != len(pool):
|
|
raise ValueError(
|
|
"Task/split pool pairs must be distinct; use balanced-repeat or "
|
|
"with-replacement to repeat them"
|
|
)
|
|
slots = args.agents_per_cohort * args.cohorts
|
|
if args.sampling == "fixed" and len(pool) != slots:
|
|
raise ValueError("fixed sampling requires exactly agents-per-cohort * cohorts task/split pairs")
|
|
if args.sampling == "without-replacement" and slots > len(pool):
|
|
raise ValueError("without-replacement requires at least agents-per-cohort * cohorts pool pairs")
|
|
# Separate RNGs prevent sampling settings from changing condition-order randomness.
|
|
sampling_rng, schedule_rng = random.Random(args.seed), random.Random(args.seed)
|
|
teams = []
|
|
for team in range(args.teams):
|
|
if args.sampling == "fixed":
|
|
selected = pool[:]
|
|
elif args.sampling == "with-replacement":
|
|
selected = sampling_rng.choices(pool, k=slots)
|
|
elif args.sampling == "without-replacement":
|
|
selected = sampling_rng.sample(pool, k=slots)
|
|
else:
|
|
# Balance each cohort independently. Exact multiples give every task
|
|
# equal representation; otherwise task counts differ by at most one.
|
|
selected = []
|
|
complete_repeats, remainder = divmod(args.agents_per_cohort, len(pool))
|
|
for _cohort in range(args.cohorts):
|
|
cohort_pairs = pool * complete_repeats
|
|
cohort_pairs.extend(sampling_rng.sample(pool, k=remainder))
|
|
sampling_rng.shuffle(cohort_pairs)
|
|
selected.extend(cohort_pairs)
|
|
teams.append({"team": team + 1, "ids": [p[0] for p in selected],
|
|
"splits": [p[1] for p in selected]})
|
|
|
|
# Run randomized matched condition blocks round-robin by cohort. Every team's
|
|
# preceding cohort completes before its later cohort begins, while independent
|
|
# teams are spread across wall-clock time.
|
|
schedule = []
|
|
for cohort in range(args.cohorts):
|
|
team_order = list(range(1, args.teams + 1))
|
|
schedule_rng.shuffle(team_order)
|
|
for team in team_order:
|
|
order = list(CONDITIONS)
|
|
schedule_rng.shuffle(order)
|
|
schedule.extend({"team": team, "cohort": cohort + 1, "condition": c} for c in order)
|
|
return teams, schedule
|
|
|
|
|
|
def export_boards(board_bindings, exporter):
|
|
"""Export every isolated store into the reporter's flat, run-keyed format."""
|
|
snapshots = [
|
|
{**exporter(binding["path"], binding["run_id"]),
|
|
"condition": binding["condition"], "team": binding["team"],
|
|
"slot": binding["slot"]}
|
|
for binding in board_bindings
|
|
]
|
|
return {"run_ids": [s["run_id"] for s in snapshots],
|
|
"stores": [{k: s[k] for k in ("run_id", "condition", "team", "slot")}
|
|
for s in snapshots],
|
|
"posts": [p for s in snapshots for p in s["posts"]],
|
|
"audit": [a for s in snapshots for a in s["audit"]]}
|
|
|
|
|
|
def main():
|
|
p = parser()
|
|
args = p.parse_args()
|
|
if args.execute and args.freeze_communication_plan is not None:
|
|
p.error("--freeze-communication-plan cannot be combined with --execute")
|
|
if args.execute and os.environ.get("DOCKER_HOST") != REMOTE_DOCKER_HOST:
|
|
p.error(
|
|
f"execution requires DOCKER_HOST={REMOTE_DOCKER_HOST}; use `just board-run`"
|
|
)
|
|
args.model = MODEL_ALIASES.get(args.model, args.model)
|
|
if not args.model.startswith("openrouter/") or len(args.model.split("/")) < 3:
|
|
p.error("--model must be glm, muse, or an openrouter/provider/model identifier")
|
|
try:
|
|
team_plans, schedule = plan(args)
|
|
except ValueError as exc:
|
|
p.error(str(exc))
|
|
pairs = list(zip(args.ids, args.splits))
|
|
holdout_policy = args.prompt_variant == "D"
|
|
reserved_holdout = set(DEFAULT_PARTITIONS.communication_holdout)
|
|
if not holdout_policy:
|
|
if args.communication_plan or args.freeze_communication_plan or args.calibration_plan:
|
|
p.error("frozen calibration/communication plans are only valid with prompt D")
|
|
development_ids = set(DEFAULT_PARTITIONS.development)
|
|
outside_development = sorted(set(args.ids) - development_ids)
|
|
if outside_development:
|
|
p.error(
|
|
"nonconfirmatory board runs may use only development tasks; "
|
|
f"reserved or unknown IDs: {outside_development}"
|
|
)
|
|
if holdout_policy:
|
|
if any(split != "conflicting" for split in args.splits):
|
|
p.error("confirmatory communication holdout pairs must all use the conflicting split")
|
|
outside_holdout = sorted(set(args.ids) - reserved_holdout)
|
|
if outside_holdout:
|
|
p.error("confirmatory runs may use only the frozen communication holdout; "
|
|
f"outside IDs: {outside_holdout}")
|
|
if args.holdout_audit is None:
|
|
preflight = {
|
|
"purpose": "preconfirmatory-board-task-audit",
|
|
"confirmatory_ready": False,
|
|
"dataset": {"path": "fjzzq2002/impossible_livecodebench",
|
|
"revision": args.dataset_revision},
|
|
"prompt_variant": args.prompt_variant,
|
|
"task_pairs": [{"task_id": task_id, "split": split}
|
|
for task_id, split in pairs],
|
|
"blockers": [
|
|
"A reviewed --holdout-audit is required before loading or executing confirmatory tasks."
|
|
],
|
|
}
|
|
if args.execute:
|
|
p.error(preflight["blockers"][0])
|
|
print(json.dumps(preflight, indent=2), flush=True)
|
|
return
|
|
if args.execute and args.communication_plan is None:
|
|
p.error(
|
|
"holdout execution requires --communication-plan; freeze and inspect a plan first"
|
|
)
|
|
if args.freeze_communication_plan is not None and not all(
|
|
(args.calibration_plan, args.calibration_run, args.calibration_review,
|
|
args.validation_evidence)
|
|
):
|
|
p.error(
|
|
"freezing a communication plan requires --calibration-plan, "
|
|
"--calibration-run, --calibration-review, and --validation-evidence"
|
|
)
|
|
if args.execute and not all(
|
|
(args.calibration_plan, args.calibration_run, args.calibration_review,
|
|
args.validation_evidence)
|
|
):
|
|
p.error(
|
|
"confirmatory execution requires --calibration-plan, --calibration-run, "
|
|
"--calibration-review, and --validation-evidence"
|
|
)
|
|
os.chdir(ROOT)
|
|
from dotenv import load_dotenv
|
|
from inspect_ai import Task, eval as inspect_eval
|
|
from messageboardbench.board import BOARD_INTERFACE_VERSION, initialize_board, export_board
|
|
from messageboardbench.board_task import episode_solver, availability
|
|
from messageboardbench.board_task import NEUTRAL_BOARD_TOOL_LIST
|
|
from messageboardbench.completion import completion_manifest_record
|
|
from messageboardbench.prompt_calibration import apply_variant_to_sample, render_tools_instruction
|
|
from messageboardbench.task import scratch_scorer
|
|
import impossiblebench.livecodebench_scorers as upstream
|
|
import impossiblebench.livecodebench_agent_full as upstream_agent
|
|
import impossiblebench.livecodebench_tasks as upstream_tasks
|
|
audit = audit_source = None
|
|
if holdout_policy:
|
|
try:
|
|
audit, audit_source = read_holdout_audit(
|
|
args.holdout_audit, args.dataset_revision, pairs
|
|
)
|
|
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
|
p.error(f"holdout audit is not ready: {exc}")
|
|
datasets = load_pinned_datasets(set(args.splits), args.dataset_revision)
|
|
for task_id, split in zip(args.ids,args.splits):
|
|
if task_id not in datasets[split]: raise ValueError(f"Missing {split}/{task_id}")
|
|
if holdout_policy:
|
|
try:
|
|
validate_loaded_pairs(datasets, audit, pairs)
|
|
except ValueError as exc:
|
|
p.error(str(exc))
|
|
if args.prompt_variant == "upstream-legacy":
|
|
rendered_prompts = {
|
|
sample.metadata.get("instruction_prompt")
|
|
for split in datasets.values() for sample in split.values()
|
|
}
|
|
if len(rendered_prompts) != 1 or not isinstance(next(iter(rendered_prompts)), str):
|
|
raise ValueError("Expected one exact upstream tools instruction across the task pool")
|
|
rendered_prompt = next(iter(rendered_prompts))
|
|
prompt_source = "upstream dataset sample metadata.instruction_prompt"
|
|
prompt_source_reference = "installed impossiblebench tools-scaffold dataset sample"
|
|
else:
|
|
rendered_prompt = render_tools_instruction(args.prompt_variant)
|
|
prompt_source = "src/messageboardbench/prompt_calibration.py:PROMPT_VARIANTS"
|
|
prompt_source_reference = "ImpossibleBench paper Table 1; https://arxiv.org/abs/2510.20270v1"
|
|
prompt_bytes = rendered_prompt.encode("utf-8")
|
|
prompt_record = {
|
|
"variant": args.prompt_variant,
|
|
"source": prompt_source,
|
|
"source_reference": prompt_source_reference,
|
|
"rendered_instruction_prompt": rendered_prompt,
|
|
"rendered_instruction_prompt_encoding": "utf-8",
|
|
"rendered_instruction_prompt_base64": base64.b64encode(prompt_bytes).decode("ascii"),
|
|
"rendered_instruction_prompt_bytes": len(prompt_bytes),
|
|
"rendered_instruction_prompt_sha256": hashlib.sha256(prompt_bytes).hexdigest(),
|
|
}
|
|
approved_pair_hashes = ([row for row in audit["approved_pairs"]
|
|
if (row["task_id"], row["split"]) in set(pairs)]
|
|
if audit is not None else None)
|
|
completion_record = completion_manifest_record()
|
|
sham_availability = availability("sham", "{episode_id}")
|
|
shared_availability = availability("shared", "{episode_id}")
|
|
if sham_availability != shared_availability:
|
|
raise RuntimeError("sham/shared model-visible board interfaces differ")
|
|
interface_payload = {
|
|
"availability_template": sham_availability,
|
|
"tools_list_insertion": NEUTRAL_BOARD_TOOL_LIST,
|
|
"board_tool_names": ["board_post", "board_read"],
|
|
}
|
|
neutral_interface_record = {
|
|
"conditions": list(CONDITIONS),
|
|
"episode_identity_is_fresh_but_template_is_shared": True,
|
|
**interface_payload,
|
|
"interface_payload_sha256": _text_sha256(
|
|
json.dumps(interface_payload, sort_keys=True, separators=(",", ":"))
|
|
),
|
|
"host_side_difference_only": "board-store persistence scope",
|
|
}
|
|
calibration_source = None
|
|
calibration_execution = None
|
|
calibration_review_source = None
|
|
validation_source = None
|
|
if args.calibration_plan is not None:
|
|
calibration_bytes = args.calibration_plan.read_bytes()
|
|
calibration = json.loads(calibration_bytes)
|
|
claimed_hash = calibration.get("manifest_sha256")
|
|
unhashed = dict(calibration)
|
|
unhashed.pop("manifest_sha256", None)
|
|
actual_self_hash = _text_sha256(
|
|
json.dumps(unhashed, sort_keys=True, separators=(",", ":"))
|
|
)
|
|
if claimed_hash != actual_self_hash:
|
|
p.error("calibration plan self-hash mismatch")
|
|
if calibration.get("purpose") != "prompt-calibration-development-only":
|
|
p.error("calibration plan has the wrong purpose")
|
|
if calibration.get("benchmark", {}).get("dataset_revision") != args.dataset_revision:
|
|
p.error("calibration plan dataset revision mismatch")
|
|
calibration_environment = calibration.get("environment", {})
|
|
expected_calibration_environment = {
|
|
"model": args.model,
|
|
"message_limit": args.messages,
|
|
"token_limit": args.token_limit,
|
|
"time_limit_seconds": args.time_limit,
|
|
"max_attempts": 3,
|
|
"temperature": args.temperature,
|
|
"reasoning_effort": args.reasoning_effort,
|
|
"strict_tools": False,
|
|
"sample_retries": 0,
|
|
"request_retries": 1,
|
|
}
|
|
mismatched_environment = {
|
|
key: (calibration_environment.get(key), expected)
|
|
for key, expected in expected_calibration_environment.items()
|
|
if calibration_environment.get(key) != expected
|
|
}
|
|
if mismatched_environment:
|
|
p.error(
|
|
"calibration plan model/budgets differ from the communication run: "
|
|
+ repr(mismatched_environment)
|
|
)
|
|
frozen_d = next((row for row in calibration.get("prompt_variants", [])
|
|
if row.get("variant_id") == "D"), None)
|
|
if not frozen_d or frozen_d.get("rendered_tools_instruction_sha256") != prompt_record["rendered_instruction_prompt_sha256"]:
|
|
p.error("calibration plan does not bind the exact prompt D bytes")
|
|
calibration_source = {
|
|
"path": str(args.calibration_plan.resolve()),
|
|
"sha256": hashlib.sha256(calibration_bytes).hexdigest(),
|
|
"manifest_sha256": claimed_hash,
|
|
}
|
|
if args.calibration_run is not None or args.calibration_review is not None:
|
|
if args.calibration_plan is None or args.calibration_run is None or args.calibration_review is None:
|
|
p.error(
|
|
"calibration evidence requires --calibration-plan, --calibration-run, "
|
|
"and --calibration-review together"
|
|
)
|
|
from messageboardbench.confirmation import (
|
|
verify_calibration_review,
|
|
verify_completed_calibration,
|
|
)
|
|
try:
|
|
calibration_execution = verify_completed_calibration(
|
|
args.calibration_plan, args.calibration_run
|
|
)
|
|
calibration_review_source = verify_calibration_review(
|
|
args.calibration_review, calibration_execution
|
|
)
|
|
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
|
p.error(f"calibration evidence is not ready: {exc}")
|
|
if args.validation_evidence is not None:
|
|
if calibration_execution is None:
|
|
p.error("prompt-D validation requires completed calibration evidence")
|
|
from messageboardbench.confirmation import verify_prompt_d_validation
|
|
try:
|
|
validation_source = verify_prompt_d_validation(
|
|
args.validation_evidence,
|
|
plan_path=args.calibration_plan,
|
|
calibration_evidence_sha256=calibration_execution["evidence_sha256"],
|
|
dataset_revision=args.dataset_revision,
|
|
model=args.model,
|
|
prompt_sha256=prompt_record["rendered_instruction_prompt_sha256"],
|
|
)
|
|
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
|
p.error(f"prompt-D validation evidence is not ready: {exc}")
|
|
plan_binding = {
|
|
"dataset": {"path": "fjzzq2002/impossible_livecodebench",
|
|
"revision": args.dataset_revision,
|
|
"holdout_audit_sha256": audit_source["sha256"] if audit_source else None},
|
|
"calibration_plan_sha256": calibration_source["sha256"] if calibration_source else None,
|
|
"calibration_execution_sha256": (
|
|
calibration_execution["evidence_sha256"] if calibration_execution else None
|
|
),
|
|
"calibration_review_sha256": (
|
|
calibration_review_source["sha256"] if calibration_review_source else None
|
|
),
|
|
"prompt_d_validation_sha256": validation_source["sha256"] if validation_source else None,
|
|
"prompt_variant": args.prompt_variant,
|
|
"prompt_sha256": prompt_record["rendered_instruction_prompt_sha256"],
|
|
"model": args.model,
|
|
"task_pairs": [{"task_id": x, "split": y} for x, y in pairs],
|
|
"agents_per_cohort": args.agents_per_cohort,
|
|
"cohorts": args.cohorts,
|
|
"teams": args.teams,
|
|
"sampling": args.sampling,
|
|
"seed": args.seed,
|
|
"team_plans": team_plans,
|
|
"schedule": schedule,
|
|
"message_limit": args.messages,
|
|
"token_limit": args.token_limit,
|
|
"time_limit": args.time_limit,
|
|
"max_attempts": 3,
|
|
"reasoning_effort": args.reasoning_effort,
|
|
"temperature": args.temperature,
|
|
"strict_tools": False,
|
|
"sample_retries": 0,
|
|
"request_retries": 1,
|
|
"completion_policy": completion_record,
|
|
"neutral_board_interface": neutral_interface_record,
|
|
}
|
|
communication_source = None
|
|
confirmatory = False
|
|
if args.freeze_communication_plan is not None:
|
|
from messageboardbench.communication_plan import build_communication_plan, write_plan
|
|
try:
|
|
frozen_plan = build_communication_plan(plan_binding)
|
|
write_plan(args.freeze_communication_plan, frozen_plan)
|
|
except (OSError, ValueError) as exc:
|
|
p.error(f"could not freeze communication plan: {exc}")
|
|
communication_source = {
|
|
"path": str(args.freeze_communication_plan.resolve()),
|
|
"plan_sha256": frozen_plan["plan_sha256"],
|
|
"status": "newly-frozen-not-executed",
|
|
}
|
|
elif args.communication_plan is not None:
|
|
from messageboardbench.communication_plan import verify_communication_plan
|
|
plan_bytes = args.communication_plan.read_bytes()
|
|
frozen_plan = json.loads(plan_bytes)
|
|
try:
|
|
verify_communication_plan(frozen_plan, plan_binding)
|
|
except ValueError as exc:
|
|
p.error(f"communication plan is invalid: {exc}")
|
|
communication_source = {
|
|
"path": str(args.communication_plan.resolve()),
|
|
"sha256": hashlib.sha256(plan_bytes).hexdigest(),
|
|
"plan_sha256": frozen_plan["plan_sha256"],
|
|
"status": "verified-for-execution",
|
|
}
|
|
confirmatory = True
|
|
config = {"purpose":("confirmatory-neutral-sham-shared-board" if confirmatory
|
|
else ("preconfirmatory-holdout-plan" if holdout_policy
|
|
else "nonconfirmatory-neutral-sham-shared-board")), "model":args.model,
|
|
"confirmatory":confirmatory, "confirmatory_ready":confirmatory,
|
|
"calibration_plan":calibration_source,
|
|
"calibration_execution":calibration_execution,
|
|
"calibration_review":calibration_review_source,
|
|
"prompt_d_validation":validation_source,
|
|
"communication_plan":communication_source,
|
|
"dataset":{"path":"fjzzq2002/impossible_livecodebench",
|
|
"revision":args.dataset_revision,
|
|
"revision_kind":"immutable_commit",
|
|
"holdout_audit":audit_source,
|
|
"approved_pair_hashes":approved_pair_hashes},
|
|
"board_interface_version":BOARD_INTERFACE_VERSION,
|
|
"neutral_board_interface":neutral_interface_record,
|
|
"policy_prompt":prompt_record,
|
|
"completion":completion_record,
|
|
"conditions":list(CONDITIONS), "ids":args.ids, "splits":args.splits,
|
|
"agents_per_cohort":args.agents_per_cohort, "cohorts":args.cohorts, "teams":args.teams,
|
|
"sampling":args.sampling, "team_plans":team_plans, "schedule":schedule,
|
|
"planned_episodes":2*args.agents_per_cohort*args.cohorts*args.teams,
|
|
"message_limit":args.messages, "token_limit":args.token_limit, "time_limit":args.time_limit,
|
|
"max_attempts":3, "reasoning_effort":args.reasoning_effort, "temperature":args.temperature,
|
|
"strict_tools":False, "sample_retries":0, "request_retries":1,
|
|
"fresh_identity_and_private_files_every_episode":True,
|
|
"identical_board_prompt_and_tools_both_conditions":True,
|
|
"shared_posts_persist_within_team":True,
|
|
"sham_posts_isolated_per_episode":True,
|
|
"boards_isolated_between_teams":True, "forced_reads":False,
|
|
"automatic_publication":False, "seeded_posts":False,
|
|
"condition_specific_integrity_instruction":False, "schedule_seed":args.seed,
|
|
"limitations":[("Frozen communication holdout; repeats are not new independent task instances."
|
|
if holdout_policy else
|
|
"Nonconfirmatory prompt/task selection; do not report as a confirmatory result."),
|
|
f"Message limit {args.messages}; preliminary model diagnostic used 60.",
|
|
"Both conditions receive the same neutral board descriptions and tools.",
|
|
"The sham control retains only posts made by the current episode.",
|
|
"Private scratch snapshots are at submissions; transient files rely on action logs."]}
|
|
print(json.dumps(config,indent=2),flush=True)
|
|
if args.freeze_communication_plan is not None:
|
|
print(f"Frozen communication plan to {args.freeze_communication_plan}; no model request was made.", flush=True)
|
|
return
|
|
if not args.execute: return
|
|
consumption = None
|
|
if confirmatory:
|
|
from messageboardbench.confirmation import assert_plan_unconsumed, consume_plan_once
|
|
try:
|
|
assert_plan_unconsumed(frozen_plan["plan_sha256"])
|
|
except ValueError as exc:
|
|
p.error(str(exc))
|
|
load_dotenv(ROOT/".env")
|
|
out=args.out.resolve(); out.mkdir(parents=True,exist_ok=False)
|
|
if confirmatory:
|
|
consumption = consume_plan_once(
|
|
args.communication_plan, out, frozen_plan["plan_sha256"]
|
|
)
|
|
config["communication_plan_consumption"] = consumption
|
|
before = budget()
|
|
if before["limit_remaining"] is None or before["limit_remaining"] < 0.5:
|
|
raise RuntimeError("Insufficient remaining key budget for this run; cap not changed")
|
|
dump(out/"manifest.json",config); dump(out/"budget-before.json",before)
|
|
sources=[Path(__file__),*sorted((ROOT/"src/messageboardbench").glob("*.py")),
|
|
Path(upstream.__file__),Path(upstream_agent.__file__),Path(upstream_tasks.__file__),
|
|
ROOT/"compose.yaml"]
|
|
archive=out/"source-snapshot";archive.mkdir();index=[]
|
|
for i,source in enumerate(sources):
|
|
data=source.read_bytes();name=f"{i}-{source.name}";(archive/name).write_bytes(data)
|
|
index.append({"source":str(source),"archived":name,"sha256":hashlib.sha256(data).hexdigest()})
|
|
dump(archive/"index.json",index)
|
|
identities=[]; board_bindings=[]; binding_by_episode={}
|
|
for team in team_plans:
|
|
run_ids={"shared":uuid.uuid4().hex, "sham":{}}
|
|
episode_ids={c:["worker-"+uuid.uuid4().hex[:12] for _ in team['ids']] for c in config['conditions']}
|
|
identities.append({"team":team['team'], "run_ids":run_ids,"episode_ids":episode_ids})
|
|
shared_path=out/("board.sqlite" if args.teams == 1 else f"board-team-{team['team']}.sqlite")
|
|
initialize_board(shared_path,run_ids['shared'])
|
|
shared_binding={"condition":"shared", "team":team["team"], "slot":None,
|
|
"path":shared_path, "run_id":run_ids["shared"]}
|
|
board_bindings.append(shared_binding)
|
|
for pos, episode_id in enumerate(episode_ids["shared"]):
|
|
binding_by_episode[episode_id]=shared_binding
|
|
for pos, episode_id in enumerate(episode_ids["sham"], start=1):
|
|
sham_run_id=uuid.uuid4().hex
|
|
run_ids["sham"][str(pos)]=sham_run_id
|
|
sham_path=out/f"sham-board-team-{team['team']}-slot-{pos}.sqlite"
|
|
initialize_board(sham_path,sham_run_id)
|
|
sham_binding={"condition":"sham", "team":team["team"], "slot":pos,
|
|
"path":sham_path, "run_id":sham_run_id}
|
|
board_bindings.append(sham_binding)
|
|
binding_by_episode[episode_id]=sham_binding
|
|
dump(out/"identities.json", {**(identities[0] if args.teams == 1 else {}), "teams":identities})
|
|
# Preserve the original one-team schedule artifact for historical consumers.
|
|
dump(out/"schedule.json", [[s['cohort']-1,s['condition']] for s in schedule] if args.teams == 1 else schedule)
|
|
table=[]; status={"status":"running","completed_phases":0}
|
|
try:
|
|
for phase,entry in enumerate(schedule):
|
|
team=entry["team"];cohort=entry["cohort"]-1;condition=entry["condition"]
|
|
selected=team_plans[team-1];identity=identities[team-1]
|
|
episode_ids=identity["episode_ids"]
|
|
tasks=[];inputs=[]
|
|
for j in range(args.agents_per_cohort):
|
|
pos=cohort*args.agents_per_cohort+j;task_id=selected["ids"][pos];split=selected["splits"][pos]
|
|
episode_id=episode_ids[condition][pos]
|
|
binding=binding_by_episode[episode_id]
|
|
run_id=binding["run_id"]
|
|
board_path=binding["path"]
|
|
source_sample=datasets[split][task_id]
|
|
sample=(source_sample.model_copy(deep=True)
|
|
if args.prompt_variant == "upstream-legacy"
|
|
else apply_variant_to_sample(source_sample, args.prompt_variant))
|
|
sample.metadata=dict(sample.metadata or {})
|
|
if sample.input != rendered_prompt or sample.metadata.get("instruction_prompt") != rendered_prompt:
|
|
raise RuntimeError("Rendered instruction prompt differs from the frozen run prompt")
|
|
sample.metadata.update(run_id=run_id,episode_id=episode_id,cohort=cohort+1,condition=condition,team=team,slot=pos+1,cohort_slot=j+1,
|
|
completion=completion_record)
|
|
inputs.append({"sample":sample.model_dump(mode="json"),
|
|
"availability":availability(condition,episode_id),
|
|
"policy_prompt":prompt_record})
|
|
tasks.append(Task(name=f"board_pilot_t{team}_{condition}_c{cohort+1}_p{j+1}",dataset=[sample],
|
|
solver=episode_solver(condition,episode_id,task_id,run_id,board_path),
|
|
scorer=scratch_scorer(split),sandbox=("docker",str(ROOT/"compose.yaml")),
|
|
message_limit=args.messages,metadata={**config,"condition":condition,"cohort":cohort+1,"team":team,"slot":pos+1,"cohort_slot":j+1,"split":split}))
|
|
dump(out/f"phase-{phase+1}-inputs.json",inputs)
|
|
print(f"Starting phase {phase+1}: team {team} {condition} cohort {cohort+1}",flush=True)
|
|
logs=inspect_eval(tasks,model=args.model,model_args={"strict_tools":False},log_dir=str(out/"evals"),
|
|
max_tasks=args.agents_per_cohort,max_samples=args.agents_per_cohort,max_sandboxes=args.agents_per_cohort,max_connections=args.agents_per_cohort,max_retries=1,timeout=300,
|
|
retry_on_error=0,fail_on_error=False,time_limit=args.time_limit,token_limit=args.token_limit,
|
|
reasoning_effort=args.reasoning_effort,temperature=args.temperature)
|
|
new=[]
|
|
for log in logs:
|
|
for s in log.samples or []:
|
|
score=next(iter(s.scores.values())) if s.scores else None
|
|
new.append({"log":log.location,"sample_id":str(s.id),"condition":condition,
|
|
"cohort":cohort+1,"team":team,"slot":s.metadata["slot"],"episode_id":s.metadata['episode_id'],"split":log.eval.metadata['split'],
|
|
"score":score.value if score else None,"messages":len(s.messages),
|
|
"model_calls":sum(m.role=='assistant' for m in s.messages),
|
|
"tool_calls":sum(len(getattr(m,'tool_calls',[]) or []) for m in s.messages),
|
|
"usage":{k:v.model_dump(mode='json') for k,v in s.model_usage.items()},
|
|
"limit":s.limit.model_dump(mode='json') if s.limit else None,
|
|
"error":s.error.message if s.error else None,
|
|
"unsuccessful_completion":s.metadata.get("unsuccessful_completion"),
|
|
"plain_text_completion":s.metadata.get("plain_text_completion"),
|
|
"completion":completion_record,
|
|
"scratch_files":list((score.metadata or {}).get('scratch_files',{})) if score else [],
|
|
"test_modified_ever":(score.metadata or {}).get('test_modified_ever') if score else None,
|
|
"manual_behavior_review":"pending"})
|
|
table.extend(new);dump(out/'results.json',table);dump(out/f'board-after-phase-{phase+1}.json',export_boards(board_bindings,export_board))
|
|
after=budget();dump(out/f'budget-after-phase-{phase+1}.json',after)
|
|
print(json.dumps({"phase":phase+1,"results":new,"budget":after},indent=2),flush=True)
|
|
if len(new)!=args.agents_per_cohort or any(r['error'] for r in new) or any(l.status!='success' for l in logs):
|
|
raise RuntimeError("Incomplete cohort; inspect before starting fresh episodes")
|
|
status['completed_phases']=phase+1
|
|
if after['limit_remaining'] is None or after['limit_remaining'] < 0.25: raise RuntimeError("Low key budget; stopping at cohort boundary")
|
|
status['status']='completed'
|
|
except BaseException as exc:
|
|
status.update(status='interrupted',error=repr(exc));raise
|
|
finally:
|
|
dump(out/'status.json',status);dump(out/'board-final.json',export_boards(board_bindings,export_board))
|
|
# Preserve the original run outcome if the final accounting request fails.
|
|
try:
|
|
after=budget();after['usage_delta']=after['usage']-before['usage']
|
|
except Exception as accounting_error:
|
|
after={"accounting_error":repr(accounting_error),"usage_delta":None}
|
|
dump(out/'budget-after.json',after)
|
|
print(json.dumps({"status":status,"budget":after},indent=2),flush=True)
|
|
|
|
|
|
if __name__ == '__main__': main()
|