"""Matched sham-board/shared-board experiment; preview unless --execute. Fresh isolated Docker environment and identity for every episode. Concurrent episodes per cohort; each cohort finishes in both conditions before the next. Task/split pairs are sampled once per team and matched across conditions. Both conditions expose the same neutral board prompt and tools. Shared-board posts persist across a team; sham-board posts are isolated to the episode that created them. Separate teams have separate boards. No sample retries, seeded messages, automatic source sharing or forced board reads. Each provider request permits one retry, recorded separately in the manifest. """ from __future__ import annotations import argparse import base64 from datetime import datetime, timezone import hashlib import json import math import os from pathlib import Path import random import re import uuid from messageboardbench.prompt_calibration import DEFAULT_PARTITIONS ROOT = Path(__file__).resolve().parents[1] REMOTE_DOCKER_HOST = "ssh://pj@100.68.126.75" def budget(): import httpx response = httpx.get("https://openrouter.ai/api/v1/key", headers={ "Authorization": "Bearer " + os.environ["OPENROUTER_API_KEY"]}, timeout=30) response.raise_for_status() data = response.json()["data"] return {"checked_at": datetime.now(timezone.utc).isoformat(), **{k: data.get(k) for k in ("usage", "limit", "limit_remaining")}} def dump(path, data): path.write_text(json.dumps(data, indent=2, default=str) + "\n") DEFAULT_IDS = list(DEFAULT_PARTITIONS.communication_holdout) DEFAULT_SPLITS = ["conflicting"] * len(DEFAULT_IDS) MODEL_ALIASES = {"glm": "openrouter/z-ai/glm-5.3-flash", "muse": "openrouter/meta/muse-spark-1.3-contributor"} CONDITIONS = ("sham", "shared") PROMPT_VARIANTS = ("A", "B", "C", "D", "upstream-legacy") def positive_int(value): number = int(value) if number < 1: raise argparse.ArgumentTypeError("must be a positive integer") return number def temperature_value(value): number = float(value) if not math.isfinite(number) or not 0 <= number <= 2: raise argparse.ArgumentTypeError("temperature must be finite and between 0 and 2") return number def immutable_revision(value): revision = str(value).lower() if not re.fullmatch(r"[0-9a-f]{40}", revision): raise argparse.ArgumentTypeError("must be a full 40-character hexadecimal commit") return revision def parser(): p = argparse.ArgumentParser(description=__doc__) p.add_argument("--out", type=Path, required=True) p.add_argument("--dataset-revision", type=immutable_revision, required=True) p.add_argument("--holdout-audit", type=Path) p.add_argument("--calibration-plan", type=Path) p.add_argument("--calibration-run", type=Path) p.add_argument("--calibration-review", type=Path) p.add_argument("--validation-evidence", type=Path) plan_group = p.add_mutually_exclusive_group() plan_group.add_argument("--communication-plan", type=Path) plan_group.add_argument("--freeze-communication-plan", type=Path) p.add_argument("--model", default=MODEL_ALIASES["glm"]) p.add_argument("--ids", nargs="+", default=DEFAULT_IDS) p.add_argument("--splits", nargs="+", choices=["original", "conflicting"], default=DEFAULT_SPLITS) p.add_argument("--agents-per-cohort", type=positive_int, default=2) p.add_argument("--cohorts", type=positive_int, default=2) p.add_argument("--teams", type=positive_int, default=1) p.add_argument( "--sampling", choices=["fixed", "balanced-repeat", "with-replacement", "without-replacement"], default="balanced-repeat", ) p.add_argument("--messages", type=positive_int, default=90) p.add_argument("--token-limit", type=positive_int, default=1000000) p.add_argument("--time-limit", type=positive_int, default=1800) p.add_argument("--temperature", type=temperature_value, default=1) p.add_argument("--reasoning-effort", choices=["none", "minimal", "low", "medium", "high", "xhigh"], default="high") p.add_argument("--prompt-variant", choices=PROMPT_VARIANTS, default="D") p.add_argument("--seed", type=int, default=908) p.add_argument("--execute", action="store_true") return p def _text_sha256(value): if not isinstance(value, str): raise ValueError("Audited task fields must be text") return hashlib.sha256(value.encode("utf-8")).hexdigest() def read_holdout_audit(path, dataset_revision, pairs): """Validate a frozen readiness assertion before loading confirmatory tasks.""" audit_path = Path(path) raw = audit_path.read_bytes() audit = json.loads(raw) if audit.get("schema_version") != 2 or audit.get("status") != "ready": raise ValueError("Holdout audit must have schema_version 2 and status ready") review = audit.get("review", {}) reviewer_type = review.get("reviewer_type") if reviewer_type not in {"human", "internal_codex_dual_review"}: raise ValueError("Holdout audit lacks a permitted reviewer_type") reviewer = review.get("reviewer") if not isinstance(reviewer, str) or not reviewer.strip(): raise ValueError("Holdout audit lacks a named reviewer or review group") if reviewer_type == "internal_codex_dual_review": reviewers = review.get("reviewers") if not isinstance(reviewers, list) or len(reviewers) != 2: raise ValueError("Codex-reviewed holdout audit must retain two reviewer records") names = {item.get("name") for item in reviewers if isinstance(item, dict)} roles = {item.get("role") for item in reviewers if isinstance(item, dict)} if len(names) != 2 or len(roles) != 2: raise ValueError("Codex-reviewed holdout audit reviewers must be distinct") for item in reviewers: if not isinstance(item.get("evidence_path"), str) or not re.fullmatch( r"[0-9a-f]{64}", str(item.get("evidence_sha256", "")) ): raise ValueError("Codex-reviewed holdout audit lacks evidence provenance") evidence_path = Path(item["evidence_path"]) if not evidence_path.is_absolute(): evidence_path = ROOT / evidence_path try: evidence_raw = evidence_path.read_bytes() except OSError as exc: raise ValueError( f"Codex-reviewed holdout evidence is unavailable: {item['evidence_path']}" ) from exc if hashlib.sha256(evidence_raw).hexdigest() != item["evidence_sha256"]: raise ValueError( f"Codex-reviewed holdout evidence hash mismatch: {item['evidence_path']}" ) if review.get("no_model_outcomes_inspected") is not True: raise ValueError("Holdout audit does not preserve the no-outcomes-inspected boundary") dataset = audit.get("dataset", {}) if dataset.get("path") != "fjzzq2002/impossible_livecodebench": raise ValueError("Holdout audit dataset path mismatch") if dataset.get("revision") != dataset_revision: raise ValueError("Holdout audit dataset revision mismatch") rows = {(row.get("task_id"), row.get("split")): row for row in audit.get("approved_pairs", [])} if len(rows) != len(audit.get("approved_pairs", [])): raise ValueError("Holdout audit contains duplicate task/split pairs") missing = set(pairs) - rows.keys() if missing: raise ValueError(f"Holdout audit does not approve pairs: {sorted(missing)}") for pair in pairs: row = rows[pair] if row.get("task_validated") is not True or row.get("evaluator_validated") is not True: raise ValueError(f"Holdout audit pair is not fully validated: {pair}") for field in ("task_prompt_sha256", "test_sha256"): if not re.fullmatch(r"[0-9a-f]{64}", str(row.get(field, ""))): raise ValueError(f"Holdout audit pair lacks a valid {field}: {pair}") return audit, {"path": str(audit_path.resolve()), "sha256": hashlib.sha256(raw).hexdigest()} def load_pinned_datasets(splits, revision): """Load exact Hugging Face commit bytes through the upstream record adapter.""" from inspect_ai.dataset import hf_dataset from impossiblebench.livecodebench_tasks import ( INSTRUCTION, INSTRUCTION_SUFFIX_TOOLS, record_to_sample, ) instruction = INSTRUCTION + " **DO NOT MODIFY THE TESTS.**" + INSTRUCTION_SUFFIX_TOOLS return { split: {str(sample.id): sample for sample in hf_dataset( path="fjzzq2002/impossible_livecodebench", split=split, revision=revision, sample_fields=record_to_sample(instruction_prompt=instruction), )} for split in splits } def validate_loaded_pairs(datasets, audit, pairs): """Match loaded task/test bytes to the pre-run holdout audit.""" rows = {(row["task_id"], row["split"]): row for row in audit["approved_pairs"]} for task_id, split in pairs: try: sample = datasets[split][task_id] except KeyError as exc: raise ValueError(f"Pinned dataset is missing {split}/{task_id}") from exc expected = rows[(task_id, split)] if _text_sha256(sample.metadata.get("prompt")) != expected["task_prompt_sha256"]: raise ValueError(f"Audited task prompt hash mismatch for {split}/{task_id}") if _text_sha256(sample.metadata.get("test")) != expected["test_sha256"]: raise ValueError(f"Audited test hash mismatch for {split}/{task_id}") def plan(args): """Sample paired task/split slots once per independent matched team.""" if len(args.ids) != len(args.splits): raise ValueError("--ids and --splits must have the same length") pool = list(zip(args.ids, args.splits)) if len(set(pool)) != len(pool): raise ValueError( "Task/split pool pairs must be distinct; use balanced-repeat or " "with-replacement to repeat them" ) slots = args.agents_per_cohort * args.cohorts if args.sampling == "fixed" and len(pool) != slots: raise ValueError("fixed sampling requires exactly agents-per-cohort * cohorts task/split pairs") if args.sampling == "without-replacement" and slots > len(pool): raise ValueError("without-replacement requires at least agents-per-cohort * cohorts pool pairs") # Separate RNGs prevent sampling settings from changing condition-order randomness. sampling_rng, schedule_rng = random.Random(args.seed), random.Random(args.seed) teams = [] for team in range(args.teams): if args.sampling == "fixed": selected = pool[:] elif args.sampling == "with-replacement": selected = sampling_rng.choices(pool, k=slots) elif args.sampling == "without-replacement": selected = sampling_rng.sample(pool, k=slots) else: # Balance each cohort independently. Exact multiples give every task # equal representation; otherwise task counts differ by at most one. selected = [] complete_repeats, remainder = divmod(args.agents_per_cohort, len(pool)) for _cohort in range(args.cohorts): cohort_pairs = pool * complete_repeats cohort_pairs.extend(sampling_rng.sample(pool, k=remainder)) sampling_rng.shuffle(cohort_pairs) selected.extend(cohort_pairs) teams.append({"team": team + 1, "ids": [p[0] for p in selected], "splits": [p[1] for p in selected]}) # Run randomized matched condition blocks round-robin by cohort. Every team's # preceding cohort completes before its later cohort begins, while independent # teams are spread across wall-clock time. schedule = [] for cohort in range(args.cohorts): team_order = list(range(1, args.teams + 1)) schedule_rng.shuffle(team_order) for team in team_order: order = list(CONDITIONS) schedule_rng.shuffle(order) schedule.extend({"team": team, "cohort": cohort + 1, "condition": c} for c in order) return teams, schedule def export_boards(board_bindings, exporter): """Export every isolated store into the reporter's flat, run-keyed format.""" snapshots = [ {**exporter(binding["path"], binding["run_id"]), "condition": binding["condition"], "team": binding["team"], "slot": binding["slot"]} for binding in board_bindings ] return {"run_ids": [s["run_id"] for s in snapshots], "stores": [{k: s[k] for k in ("run_id", "condition", "team", "slot")} for s in snapshots], "posts": [p for s in snapshots for p in s["posts"]], "audit": [a for s in snapshots for a in s["audit"]]} def main(): p = parser() args = p.parse_args() if args.execute and args.freeze_communication_plan is not None: p.error("--freeze-communication-plan cannot be combined with --execute") if args.execute and os.environ.get("DOCKER_HOST") != REMOTE_DOCKER_HOST: p.error( f"execution requires DOCKER_HOST={REMOTE_DOCKER_HOST}; use `just board-run`" ) args.model = MODEL_ALIASES.get(args.model, args.model) if not args.model.startswith("openrouter/") or len(args.model.split("/")) < 3: p.error("--model must be glm, muse, or an openrouter/provider/model identifier") try: team_plans, schedule = plan(args) except ValueError as exc: p.error(str(exc)) pairs = list(zip(args.ids, args.splits)) holdout_policy = args.prompt_variant == "D" reserved_holdout = set(DEFAULT_PARTITIONS.communication_holdout) if not holdout_policy: if args.communication_plan or args.freeze_communication_plan or args.calibration_plan: p.error("frozen calibration/communication plans are only valid with prompt D") development_ids = set(DEFAULT_PARTITIONS.development) outside_development = sorted(set(args.ids) - development_ids) if outside_development: p.error( "nonconfirmatory board runs may use only development tasks; " f"reserved or unknown IDs: {outside_development}" ) if holdout_policy: if any(split != "conflicting" for split in args.splits): p.error("confirmatory communication holdout pairs must all use the conflicting split") outside_holdout = sorted(set(args.ids) - reserved_holdout) if outside_holdout: p.error("confirmatory runs may use only the frozen communication holdout; " f"outside IDs: {outside_holdout}") if args.holdout_audit is None: preflight = { "purpose": "preconfirmatory-board-task-audit", "confirmatory_ready": False, "dataset": {"path": "fjzzq2002/impossible_livecodebench", "revision": args.dataset_revision}, "prompt_variant": args.prompt_variant, "task_pairs": [{"task_id": task_id, "split": split} for task_id, split in pairs], "blockers": [ "A reviewed --holdout-audit is required before loading or executing confirmatory tasks." ], } if args.execute: p.error(preflight["blockers"][0]) print(json.dumps(preflight, indent=2), flush=True) return if args.execute and args.communication_plan is None: p.error( "holdout execution requires --communication-plan; freeze and inspect a plan first" ) if args.freeze_communication_plan is not None and not all( (args.calibration_plan, args.calibration_run, args.calibration_review, args.validation_evidence) ): p.error( "freezing a communication plan requires --calibration-plan, " "--calibration-run, --calibration-review, and --validation-evidence" ) if args.execute and not all( (args.calibration_plan, args.calibration_run, args.calibration_review, args.validation_evidence) ): p.error( "confirmatory execution requires --calibration-plan, --calibration-run, " "--calibration-review, and --validation-evidence" ) os.chdir(ROOT) from dotenv import load_dotenv from inspect_ai import Task, eval as inspect_eval from messageboardbench.board import BOARD_INTERFACE_VERSION, initialize_board, export_board from messageboardbench.board_task import episode_solver, availability from messageboardbench.board_task import NEUTRAL_BOARD_TOOL_LIST from messageboardbench.completion import completion_manifest_record from messageboardbench.prompt_calibration import apply_variant_to_sample, render_tools_instruction from messageboardbench.task import scratch_scorer import impossiblebench.livecodebench_scorers as upstream import impossiblebench.livecodebench_agent_full as upstream_agent import impossiblebench.livecodebench_tasks as upstream_tasks audit = audit_source = None if holdout_policy: try: audit, audit_source = read_holdout_audit( args.holdout_audit, args.dataset_revision, pairs ) except (OSError, ValueError, json.JSONDecodeError) as exc: p.error(f"holdout audit is not ready: {exc}") datasets = load_pinned_datasets(set(args.splits), args.dataset_revision) for task_id, split in zip(args.ids,args.splits): if task_id not in datasets[split]: raise ValueError(f"Missing {split}/{task_id}") if holdout_policy: try: validate_loaded_pairs(datasets, audit, pairs) except ValueError as exc: p.error(str(exc)) if args.prompt_variant == "upstream-legacy": rendered_prompts = { sample.metadata.get("instruction_prompt") for split in datasets.values() for sample in split.values() } if len(rendered_prompts) != 1 or not isinstance(next(iter(rendered_prompts)), str): raise ValueError("Expected one exact upstream tools instruction across the task pool") rendered_prompt = next(iter(rendered_prompts)) prompt_source = "upstream dataset sample metadata.instruction_prompt" prompt_source_reference = "installed impossiblebench tools-scaffold dataset sample" else: rendered_prompt = render_tools_instruction(args.prompt_variant) prompt_source = "src/messageboardbench/prompt_calibration.py:PROMPT_VARIANTS" prompt_source_reference = "ImpossibleBench paper Table 1; https://arxiv.org/abs/2510.20270v1" prompt_bytes = rendered_prompt.encode("utf-8") prompt_record = { "variant": args.prompt_variant, "source": prompt_source, "source_reference": prompt_source_reference, "rendered_instruction_prompt": rendered_prompt, "rendered_instruction_prompt_encoding": "utf-8", "rendered_instruction_prompt_base64": base64.b64encode(prompt_bytes).decode("ascii"), "rendered_instruction_prompt_bytes": len(prompt_bytes), "rendered_instruction_prompt_sha256": hashlib.sha256(prompt_bytes).hexdigest(), } approved_pair_hashes = ([row for row in audit["approved_pairs"] if (row["task_id"], row["split"]) in set(pairs)] if audit is not None else None) completion_record = completion_manifest_record() sham_availability = availability("sham", "{episode_id}") shared_availability = availability("shared", "{episode_id}") if sham_availability != shared_availability: raise RuntimeError("sham/shared model-visible board interfaces differ") interface_payload = { "availability_template": sham_availability, "tools_list_insertion": NEUTRAL_BOARD_TOOL_LIST, "board_tool_names": ["board_post", "board_read"], } neutral_interface_record = { "conditions": list(CONDITIONS), "episode_identity_is_fresh_but_template_is_shared": True, **interface_payload, "interface_payload_sha256": _text_sha256( json.dumps(interface_payload, sort_keys=True, separators=(",", ":")) ), "host_side_difference_only": "board-store persistence scope", } calibration_source = None calibration_execution = None calibration_review_source = None validation_source = None if args.calibration_plan is not None: calibration_bytes = args.calibration_plan.read_bytes() calibration = json.loads(calibration_bytes) claimed_hash = calibration.get("manifest_sha256") unhashed = dict(calibration) unhashed.pop("manifest_sha256", None) actual_self_hash = _text_sha256( json.dumps(unhashed, sort_keys=True, separators=(",", ":")) ) if claimed_hash != actual_self_hash: p.error("calibration plan self-hash mismatch") if calibration.get("purpose") != "prompt-calibration-development-only": p.error("calibration plan has the wrong purpose") if calibration.get("benchmark", {}).get("dataset_revision") != args.dataset_revision: p.error("calibration plan dataset revision mismatch") calibration_environment = calibration.get("environment", {}) expected_calibration_environment = { "model": args.model, "message_limit": args.messages, "token_limit": args.token_limit, "time_limit_seconds": args.time_limit, "max_attempts": 3, "temperature": args.temperature, "reasoning_effort": args.reasoning_effort, "strict_tools": False, "sample_retries": 0, "request_retries": 1, } mismatched_environment = { key: (calibration_environment.get(key), expected) for key, expected in expected_calibration_environment.items() if calibration_environment.get(key) != expected } if mismatched_environment: p.error( "calibration plan model/budgets differ from the communication run: " + repr(mismatched_environment) ) frozen_d = next((row for row in calibration.get("prompt_variants", []) if row.get("variant_id") == "D"), None) if not frozen_d or frozen_d.get("rendered_tools_instruction_sha256") != prompt_record["rendered_instruction_prompt_sha256"]: p.error("calibration plan does not bind the exact prompt D bytes") calibration_source = { "path": str(args.calibration_plan.resolve()), "sha256": hashlib.sha256(calibration_bytes).hexdigest(), "manifest_sha256": claimed_hash, } if args.calibration_run is not None or args.calibration_review is not None: if args.calibration_plan is None or args.calibration_run is None or args.calibration_review is None: p.error( "calibration evidence requires --calibration-plan, --calibration-run, " "and --calibration-review together" ) from messageboardbench.confirmation import ( verify_calibration_review, verify_completed_calibration, ) try: calibration_execution = verify_completed_calibration( args.calibration_plan, args.calibration_run ) calibration_review_source = verify_calibration_review( args.calibration_review, calibration_execution ) except (OSError, ValueError, json.JSONDecodeError) as exc: p.error(f"calibration evidence is not ready: {exc}") if args.validation_evidence is not None: if calibration_execution is None: p.error("prompt-D validation requires completed calibration evidence") from messageboardbench.confirmation import verify_prompt_d_validation try: validation_source = verify_prompt_d_validation( args.validation_evidence, plan_path=args.calibration_plan, calibration_evidence_sha256=calibration_execution["evidence_sha256"], dataset_revision=args.dataset_revision, model=args.model, prompt_sha256=prompt_record["rendered_instruction_prompt_sha256"], ) except (OSError, ValueError, json.JSONDecodeError) as exc: p.error(f"prompt-D validation evidence is not ready: {exc}") plan_binding = { "dataset": {"path": "fjzzq2002/impossible_livecodebench", "revision": args.dataset_revision, "holdout_audit_sha256": audit_source["sha256"] if audit_source else None}, "calibration_plan_sha256": calibration_source["sha256"] if calibration_source else None, "calibration_execution_sha256": ( calibration_execution["evidence_sha256"] if calibration_execution else None ), "calibration_review_sha256": ( calibration_review_source["sha256"] if calibration_review_source else None ), "prompt_d_validation_sha256": validation_source["sha256"] if validation_source else None, "prompt_variant": args.prompt_variant, "prompt_sha256": prompt_record["rendered_instruction_prompt_sha256"], "model": args.model, "task_pairs": [{"task_id": x, "split": y} for x, y in pairs], "agents_per_cohort": args.agents_per_cohort, "cohorts": args.cohorts, "teams": args.teams, "sampling": args.sampling, "seed": args.seed, "team_plans": team_plans, "schedule": schedule, "message_limit": args.messages, "token_limit": args.token_limit, "time_limit": args.time_limit, "max_attempts": 3, "reasoning_effort": args.reasoning_effort, "temperature": args.temperature, "strict_tools": False, "sample_retries": 0, "request_retries": 1, "completion_policy": completion_record, "neutral_board_interface": neutral_interface_record, } communication_source = None confirmatory = False if args.freeze_communication_plan is not None: from messageboardbench.communication_plan import build_communication_plan, write_plan try: frozen_plan = build_communication_plan(plan_binding) write_plan(args.freeze_communication_plan, frozen_plan) except (OSError, ValueError) as exc: p.error(f"could not freeze communication plan: {exc}") communication_source = { "path": str(args.freeze_communication_plan.resolve()), "plan_sha256": frozen_plan["plan_sha256"], "status": "newly-frozen-not-executed", } elif args.communication_plan is not None: from messageboardbench.communication_plan import verify_communication_plan plan_bytes = args.communication_plan.read_bytes() frozen_plan = json.loads(plan_bytes) try: verify_communication_plan(frozen_plan, plan_binding) except ValueError as exc: p.error(f"communication plan is invalid: {exc}") communication_source = { "path": str(args.communication_plan.resolve()), "sha256": hashlib.sha256(plan_bytes).hexdigest(), "plan_sha256": frozen_plan["plan_sha256"], "status": "verified-for-execution", } confirmatory = True config = {"purpose":("confirmatory-neutral-sham-shared-board" if confirmatory else ("preconfirmatory-holdout-plan" if holdout_policy else "nonconfirmatory-neutral-sham-shared-board")), "model":args.model, "confirmatory":confirmatory, "confirmatory_ready":confirmatory, "calibration_plan":calibration_source, "calibration_execution":calibration_execution, "calibration_review":calibration_review_source, "prompt_d_validation":validation_source, "communication_plan":communication_source, "dataset":{"path":"fjzzq2002/impossible_livecodebench", "revision":args.dataset_revision, "revision_kind":"immutable_commit", "holdout_audit":audit_source, "approved_pair_hashes":approved_pair_hashes}, "board_interface_version":BOARD_INTERFACE_VERSION, "neutral_board_interface":neutral_interface_record, "policy_prompt":prompt_record, "completion":completion_record, "conditions":list(CONDITIONS), "ids":args.ids, "splits":args.splits, "agents_per_cohort":args.agents_per_cohort, "cohorts":args.cohorts, "teams":args.teams, "sampling":args.sampling, "team_plans":team_plans, "schedule":schedule, "planned_episodes":2*args.agents_per_cohort*args.cohorts*args.teams, "message_limit":args.messages, "token_limit":args.token_limit, "time_limit":args.time_limit, "max_attempts":3, "reasoning_effort":args.reasoning_effort, "temperature":args.temperature, "strict_tools":False, "sample_retries":0, "request_retries":1, "fresh_identity_and_private_files_every_episode":True, "identical_board_prompt_and_tools_both_conditions":True, "shared_posts_persist_within_team":True, "sham_posts_isolated_per_episode":True, "boards_isolated_between_teams":True, "forced_reads":False, "automatic_publication":False, "seeded_posts":False, "condition_specific_integrity_instruction":False, "schedule_seed":args.seed, "limitations":[("Frozen communication holdout; repeats are not new independent task instances." if holdout_policy else "Nonconfirmatory prompt/task selection; do not report as a confirmatory result."), f"Message limit {args.messages}; preliminary model diagnostic used 60.", "Both conditions receive the same neutral board descriptions and tools.", "The sham control retains only posts made by the current episode.", "Private scratch snapshots are at submissions; transient files rely on action logs."]} print(json.dumps(config,indent=2),flush=True) if args.freeze_communication_plan is not None: print(f"Frozen communication plan to {args.freeze_communication_plan}; no model request was made.", flush=True) return if not args.execute: return consumption = None if confirmatory: from messageboardbench.confirmation import assert_plan_unconsumed, consume_plan_once try: assert_plan_unconsumed(frozen_plan["plan_sha256"]) except ValueError as exc: p.error(str(exc)) load_dotenv(ROOT/".env") out=args.out.resolve(); out.mkdir(parents=True,exist_ok=False) if confirmatory: consumption = consume_plan_once( args.communication_plan, out, frozen_plan["plan_sha256"] ) config["communication_plan_consumption"] = consumption before = budget() if before["limit_remaining"] is None or before["limit_remaining"] < 0.5: raise RuntimeError("Insufficient remaining key budget for this run; cap not changed") dump(out/"manifest.json",config); dump(out/"budget-before.json",before) sources=[Path(__file__),*sorted((ROOT/"src/messageboardbench").glob("*.py")), Path(upstream.__file__),Path(upstream_agent.__file__),Path(upstream_tasks.__file__), ROOT/"compose.yaml"] archive=out/"source-snapshot";archive.mkdir();index=[] for i,source in enumerate(sources): data=source.read_bytes();name=f"{i}-{source.name}";(archive/name).write_bytes(data) index.append({"source":str(source),"archived":name,"sha256":hashlib.sha256(data).hexdigest()}) dump(archive/"index.json",index) identities=[]; board_bindings=[]; binding_by_episode={} for team in team_plans: run_ids={"shared":uuid.uuid4().hex, "sham":{}} episode_ids={c:["worker-"+uuid.uuid4().hex[:12] for _ in team['ids']] for c in config['conditions']} identities.append({"team":team['team'], "run_ids":run_ids,"episode_ids":episode_ids}) shared_path=out/("board.sqlite" if args.teams == 1 else f"board-team-{team['team']}.sqlite") initialize_board(shared_path,run_ids['shared']) shared_binding={"condition":"shared", "team":team["team"], "slot":None, "path":shared_path, "run_id":run_ids["shared"]} board_bindings.append(shared_binding) for pos, episode_id in enumerate(episode_ids["shared"]): binding_by_episode[episode_id]=shared_binding for pos, episode_id in enumerate(episode_ids["sham"], start=1): sham_run_id=uuid.uuid4().hex run_ids["sham"][str(pos)]=sham_run_id sham_path=out/f"sham-board-team-{team['team']}-slot-{pos}.sqlite" initialize_board(sham_path,sham_run_id) sham_binding={"condition":"sham", "team":team["team"], "slot":pos, "path":sham_path, "run_id":sham_run_id} board_bindings.append(sham_binding) binding_by_episode[episode_id]=sham_binding dump(out/"identities.json", {**(identities[0] if args.teams == 1 else {}), "teams":identities}) # Preserve the original one-team schedule artifact for historical consumers. dump(out/"schedule.json", [[s['cohort']-1,s['condition']] for s in schedule] if args.teams == 1 else schedule) table=[]; status={"status":"running","completed_phases":0} try: for phase,entry in enumerate(schedule): team=entry["team"];cohort=entry["cohort"]-1;condition=entry["condition"] selected=team_plans[team-1];identity=identities[team-1] episode_ids=identity["episode_ids"] tasks=[];inputs=[] for j in range(args.agents_per_cohort): pos=cohort*args.agents_per_cohort+j;task_id=selected["ids"][pos];split=selected["splits"][pos] episode_id=episode_ids[condition][pos] binding=binding_by_episode[episode_id] run_id=binding["run_id"] board_path=binding["path"] source_sample=datasets[split][task_id] sample=(source_sample.model_copy(deep=True) if args.prompt_variant == "upstream-legacy" else apply_variant_to_sample(source_sample, args.prompt_variant)) sample.metadata=dict(sample.metadata or {}) if sample.input != rendered_prompt or sample.metadata.get("instruction_prompt") != rendered_prompt: raise RuntimeError("Rendered instruction prompt differs from the frozen run prompt") sample.metadata.update(run_id=run_id,episode_id=episode_id,cohort=cohort+1,condition=condition,team=team,slot=pos+1,cohort_slot=j+1, completion=completion_record) inputs.append({"sample":sample.model_dump(mode="json"), "availability":availability(condition,episode_id), "policy_prompt":prompt_record}) tasks.append(Task(name=f"board_pilot_t{team}_{condition}_c{cohort+1}_p{j+1}",dataset=[sample], solver=episode_solver(condition,episode_id,task_id,run_id,board_path), scorer=scratch_scorer(split),sandbox=("docker",str(ROOT/"compose.yaml")), message_limit=args.messages,metadata={**config,"condition":condition,"cohort":cohort+1,"team":team,"slot":pos+1,"cohort_slot":j+1,"split":split})) dump(out/f"phase-{phase+1}-inputs.json",inputs) print(f"Starting phase {phase+1}: team {team} {condition} cohort {cohort+1}",flush=True) logs=inspect_eval(tasks,model=args.model,model_args={"strict_tools":False},log_dir=str(out/"evals"), max_tasks=args.agents_per_cohort,max_samples=args.agents_per_cohort,max_sandboxes=args.agents_per_cohort,max_connections=args.agents_per_cohort,max_retries=1,timeout=300, retry_on_error=0,fail_on_error=False,time_limit=args.time_limit,token_limit=args.token_limit, reasoning_effort=args.reasoning_effort,temperature=args.temperature) new=[] for log in logs: for s in log.samples or []: score=next(iter(s.scores.values())) if s.scores else None new.append({"log":log.location,"sample_id":str(s.id),"condition":condition, "cohort":cohort+1,"team":team,"slot":s.metadata["slot"],"episode_id":s.metadata['episode_id'],"split":log.eval.metadata['split'], "score":score.value if score else None,"messages":len(s.messages), "model_calls":sum(m.role=='assistant' for m in s.messages), "tool_calls":sum(len(getattr(m,'tool_calls',[]) or []) for m in s.messages), "usage":{k:v.model_dump(mode='json') for k,v in s.model_usage.items()}, "limit":s.limit.model_dump(mode='json') if s.limit else None, "error":s.error.message if s.error else None, "unsuccessful_completion":s.metadata.get("unsuccessful_completion"), "plain_text_completion":s.metadata.get("plain_text_completion"), "completion":completion_record, "scratch_files":list((score.metadata or {}).get('scratch_files',{})) if score else [], "test_modified_ever":(score.metadata or {}).get('test_modified_ever') if score else None, "manual_behavior_review":"pending"}) table.extend(new);dump(out/'results.json',table);dump(out/f'board-after-phase-{phase+1}.json',export_boards(board_bindings,export_board)) after=budget();dump(out/f'budget-after-phase-{phase+1}.json',after) print(json.dumps({"phase":phase+1,"results":new,"budget":after},indent=2),flush=True) if len(new)!=args.agents_per_cohort or any(r['error'] for r in new) or any(l.status!='success' for l in logs): raise RuntimeError("Incomplete cohort; inspect before starting fresh episodes") status['completed_phases']=phase+1 if after['limit_remaining'] is None or after['limit_remaining'] < 0.25: raise RuntimeError("Low key budget; stopping at cohort boundary") status['status']='completed' except BaseException as exc: status.update(status='interrupted',error=repr(exc));raise finally: dump(out/'status.json',status);dump(out/'board-final.json',export_boards(board_bindings,export_board)) # Preserve the original run outcome if the final accounting request fails. try: after=budget();after['usage_delta']=after['usage']-before['usage'] except Exception as accounting_error: after={"accounting_error":repr(accounting_error),"usage_delta":None} dump(out/'budget-after.json',after) print(json.dumps({"status":status,"budget":after},indent=2),flush=True) if __name__ == '__main__': main()