"""Offline planning and orchestration checks; never call a model provider.""" import asyncio import hashlib import importlib.util import json from pathlib import Path from types import SimpleNamespace import pytest from messageboardbench.prompt_calibration import render_tools_instruction from messageboardbench.prompt_calibration import build_manifest, write_manifest from messageboardbench.calibration_run import read_frozen_manifest from messageboardbench.completion import completion_manifest_record from messageboardbench.confirmation import ( verify_completed_calibration, verify_completed_prompt_d_validation, ) spec = importlib.util.spec_from_file_location("board_pilot", Path(__file__).parents[1] / "scripts/board_pilot.py") pilot = importlib.util.module_from_spec(spec) spec.loader.exec_module(pilot) REVISION = 'a' * 40 TASK_PROMPT = 'def f(): pass' TEST_CODE = 'def check(candidate): pass' def args(*extra): return pilot.parser().parse_args([ "--out", "unused", "--dataset-revision", REVISION, *extra ]) def make_sample(task_id): instruction = render_tools_instruction('A') from inspect_ai.dataset import Sample return Sample(id=task_id, input=instruction, metadata={ 'instruction_prompt': instruction, 'prompt': TASK_PROMPT, 'test': TEST_CODE, 'entry_point': 'f', }) def write_audit(path, pairs=None): pairs = pairs or list(zip(pilot.DEFAULT_IDS, pilot.DEFAULT_SPLITS)) data = { 'schema_version': 2, 'status': 'ready', 'dataset': {'path': 'fjzzq2002/impossible_livecodebench', 'revision': REVISION}, 'review': {'reviewer_type': 'human', 'reviewer': 'Test reviewer', 'reviewers': [], 'no_model_outcomes_inspected': True}, 'approved_pairs': [{ 'task_id': task_id, 'split': split, 'task_validated': True, 'evaluator_validated': True, 'task_prompt_sha256': hashlib.sha256(TASK_PROMPT.encode()).hexdigest(), 'test_sha256': hashlib.sha256(TEST_CODE.encode()).hexdigest(), } for task_id, split in pairs], } path.write_text(json.dumps(data)) return path def write_calibration(path, **kwargs): write_manifest(path, build_manifest(dataset_revision=REVISION, **kwargs)) return path def write_completed_calibration(plan_path, run_path): plan, source = read_frozen_manifest(plan_path) run_path.mkdir() (run_path / 'evals').mkdir() (run_path / 'frozen-plan.json').write_bytes(plan_path.read_bytes()) rows = [{ 'assignment': assignment, 'error': None, 'sample_id': assignment['task_id'], 'log': str(run_path / 'evals' / f'{index}.eval'), 'completion': completion_manifest_record(), 'calibration': {'communication': 'none'}, } for index, assignment in enumerate(plan['development_assignments'], 1)] for row in rows: Path(row['log']).write_bytes(b'mock eval log') (run_path / 'results.json').write_text(json.dumps(rows)) (run_path / 'status.json').write_text(json.dumps({ 'status': 'completed', 'phase': 'development', 'completed_assignments': len(rows), 'in_flight_assignment': None, })) (run_path / 'run-manifest.json').write_text(json.dumps({ 'purpose': 'prompt-calibration-development-execution', 'phase': 'development', 'execute': True, 'communication': 'none', 'completion': completion_manifest_record(), 'manifest': source, })) evidence = verify_completed_calibration(plan_path, run_path) review_path = run_path.parent / 'calibration-review.json' review_path.write_text(json.dumps({ 'schema_version': 1, 'status': 'ready', 'purpose': 'prompt-calibration-behavior-review', 'calibration_evidence_sha256': evidence['evidence_sha256'], 'no_communication_holdout_outcomes_inspected': True, 'reviewer': 'Test review group', 'assignment_labels': [ {'assignment_index': i, 'label': 'no_observed_gaming'} for i in range(1, len(rows) + 1) ], 'prompt_d_assessment': { 'decision': 'proceed', 'variation_adequate': True, 'rationale': 'Test fixture', }, })) validation_run = run_path.parent / 'prompt-d-validation-run' validation_run.mkdir() (validation_run / 'evals').mkdir() (validation_run / 'frozen-plan.json').write_bytes(plan_path.read_bytes()) validation_rows = [{ 'assignment': assignment, 'error': None, 'sample_id': assignment['task_id'], 'log': str(validation_run / 'evals' / f'{index}.eval'), 'completion': completion_manifest_record(), 'calibration': { 'phase': 'validation', 'communication': 'none', 'assignment': assignment, 'manifest': source, 'policy_prompt': {'variant': 'D'}, }, } for index, assignment in enumerate(plan['validation_assignments'], 1)] for row in validation_rows: Path(row['log']).write_bytes(b'mock validation eval log') (validation_run / 'results.json').write_text(json.dumps(validation_rows)) (validation_run / 'status.json').write_text(json.dumps({ 'status': 'completed', 'phase': 'validation', 'completed_assignments': len(validation_rows), 'in_flight_assignment': None, })) validation_manifest = { 'purpose': 'prompt-calibration-validation-execution', 'phase': 'validation', 'execute': True, 'communication': 'none', 'completion': completion_manifest_record(), 'assignments': len(validation_rows), 'manifest': source, } validation_audit = run_path.parent / 'validation-audit.json' validation_audit.write_text(json.dumps({ 'schema_version': 2, 'status': 'ready', 'partition': 'validation', 'dataset': {'path': plan['benchmark']['dataset'], 'revision': plan['benchmark']['dataset_revision']}, 'review': {'reviewer_type': 'human', 'reviewer': 'Test reviewer', 'no_model_outcomes_inspected': True}, 'approved_pairs': [{ 'task_id': task_id, 'split': split, 'task_validated': True, 'evaluator_validated': True, 'task_prompt_sha256': '1' * 64, 'test_sha256': '2' * 64, } for task_id, split in sorted({ (row['task_id'], row['split']) for row in plan['validation_assignments'] })], })) validation_manifest['validation_audit'] = { 'path': str(validation_audit), 'sha256': hashlib.sha256(validation_audit.read_bytes()).hexdigest(), } validation_manifest.update({key: plan['environment'][key] for key in ( 'model', 'message_limit', 'token_limit', 'time_limit_seconds', 'temperature', 'reasoning_effort', 'max_attempts', 'strict_tools', 'sample_retries', 'request_retries', )}) (validation_run / 'run-manifest.json').write_text(json.dumps(validation_manifest)) validation_execution = verify_completed_prompt_d_validation(plan_path, validation_run) validation_path = run_path.parent / 'prompt-d-validation.json' validation_path.write_text(json.dumps({ 'schema_version': 1, 'status': 'ready', 'purpose': 'prompt-d-validation', 'calibration_evidence_sha256': evidence['evidence_sha256'], 'dataset_revision': REVISION, 'model': plan['environment']['model'], 'prompt_variant': 'D', 'prompt_sha256': hashlib.sha256(render_tools_instruction('D').encode()).hexdigest(), 'communication': 'none', 'completion_policy': completion_manifest_record(), 'communication_holdout_executed': False, 'artifact_review_complete': True, 'validation_run_directory': str(validation_run), 'validation_execution_sha256': validation_execution['evidence_sha256'], 'decision': 'proceed', 'variation_adequate': True, 'completed_assignments': len(validation_rows), 'reviewer': 'Test review group', 'rationale': 'Test fixture validation has variation', 'assignment_labels': [{ 'assignment_index': row['assignment_index'], 'label': ('successful_gaming' if row['assignment_index'] == min( item['assignment_index'] for item in plan['validation_assignments'] if item['split'] == 'conflicting') else 'no_observed_gaming'), 'artifact_labels': (['contradiction_targeted_gaming'] if row['assignment_index'] == min( item['assignment_index'] for item in plan['validation_assignments'] if item['split'] == 'conflicting') else ['specification_faithful']), 'notes': 'Artifact inspected and classified independently of score.', } for row in plan['validation_assignments']], })) return run_path, review_path, validation_path def test_default_plan_uses_balanced_holdout_sampling(): options = args() assert options.prompt_variant == 'D' assert options.sampling == 'balanced-repeat' teams, schedule = pilot.plan(options) assert len(teams) == 1 assert len(teams[0]["ids"]) == 4 assert set(zip(teams[0]["ids"], teams[0]["splits"])) <= set( zip(pilot.DEFAULT_IDS, pilot.DEFAULT_SPLITS) ) assert [(s["cohort"], s["condition"]) for s in schedule] == [ (1, "shared"), (1, "sham"), (2, "sham"), (2, "shared")] def test_confirmatory_preview_fails_closed_before_dataset_load(monkeypatch, capsys): monkeypatch.setattr(pilot, 'load_pinned_datasets', lambda *values: pytest.fail('blocked preview must not load tasks')) monkeypatch.setattr('sys.argv', ['board_pilot', '--out', 'unused', '--dataset-revision', REVISION]) pilot.main() preview = json.loads(capsys.readouterr().out) assert preview['confirmatory_ready'] is False assert preview['dataset']['revision'] == REVISION assert 'holdout-audit' in preview['blockers'][0] def test_confirmatory_rejects_development_ids_and_unpinned_revision(monkeypatch): with pytest.raises(SystemExit): pilot.parser().parse_args(['--out', 'unused', '--dataset-revision', 'main']) monkeypatch.setattr('sys.argv', ['board_pilot', '--out', 'unused', '--dataset-revision', REVISION, '--ids', 'lcbhard_0', '--splits', 'conflicting', '--agents-per-cohort', '1', '--cohorts', '1']) with pytest.raises(SystemExit): pilot.main() def test_confirmatory_rejects_original_split_even_for_reserved_id(monkeypatch): monkeypatch.setattr('sys.argv', ['board_pilot', '--out', 'unused', '--dataset-revision', REVISION, '--ids', pilot.DEFAULT_IDS[0], '--splits', 'original', '--agents-per-cohort', '1', '--cohorts', '1']) with pytest.raises(SystemExit): pilot.main() def test_holdout_execution_requires_frozen_communication_plan(tmp_path, monkeypatch): audit = write_audit(tmp_path / 'holdout-audit.json') monkeypatch.setattr('sys.argv', ['board_pilot', '--out', 'unused', '--dataset-revision', REVISION, '--holdout-audit', str(audit), '--execute']) with pytest.raises(SystemExit): pilot.main() @pytest.mark.parametrize('variant', ['A', 'B', 'C', 'upstream-legacy']) def test_nonconfirmatory_prompts_cannot_consume_communication_holdout(variant, monkeypatch): monkeypatch.setattr('sys.argv', ['board_pilot', '--out', 'unused', '--dataset-revision', REVISION, '--prompt-variant', variant]) with pytest.raises(SystemExit, match='2'): pilot.main() def test_nonconfirmatory_prompts_cannot_consume_validation_reserve(monkeypatch): monkeypatch.setattr('sys.argv', ['board_pilot', '--out', 'unused', '--dataset-revision', REVISION, '--prompt-variant', 'A', '--ids', 'lcbhard_3', '--splits', 'conflicting', '--agents-per-cohort', '1', '--cohorts', '1']) with pytest.raises(SystemExit): pilot.main() def test_dataset_loader_passes_exact_immutable_revision(monkeypatch): import inspect_ai.dataset calls = [] def fake_hf_dataset(**kwargs): calls.append(kwargs) return [make_sample('lcbhard_7')] monkeypatch.setattr(inspect_ai.dataset, 'hf_dataset', fake_hf_dataset) loaded = pilot.load_pinned_datasets({'conflicting'}, REVISION) assert set(loaded['conflicting']) == {'lcbhard_7'} assert calls[0]['path'] == 'fjzzq2002/impossible_livecodebench' assert calls[0]['split'] == 'conflicting' assert calls[0]['revision'] == REVISION @pytest.mark.parametrize("sampling", ["with-replacement", "without-replacement"]) def test_sampling_reproducible_and_pool_pairs_preserved(sampling): options = args("--agents-per-cohort", "2", "--cohorts", "2", "--teams", "3", "--sampling", sampling) teams, schedule = pilot.plan(options) assert (teams, schedule) == pilot.plan(options) assert len(teams) == 3 and len(schedule) == 12 pool = set(zip(pilot.DEFAULT_IDS, pilot.DEFAULT_SPLITS)) for team in teams: pairs = list(zip(team["ids"], team["splits"])) assert len(pairs) == 4 and set(pairs) <= pool if sampling == "without-replacement": assert len(set(pairs)) == 4 phases = [s for s in schedule if s["team"] == team["team"]] assert {(s["cohort"], s["condition"]) for s in phases} == { (c, condition) for c in (1, 2) for condition in pilot.CONDITIONS} def test_balanced_repeat_balances_each_cohort_and_interleaves_teams(): options = args( "--agents-per-cohort", "22", "--cohorts", "3", "--teams", "4", "--sampling", "balanced-repeat", ) teams, schedule = pilot.plan(options) assert (teams, schedule) == pilot.plan(options) pool = set(zip(pilot.DEFAULT_IDS, pilot.DEFAULT_SPLITS)) for team in teams: pairs = list(zip(team["ids"], team["splits"])) assert len(pairs) == 66 for cohort in range(3): cohort_pairs = pairs[cohort * 22:(cohort + 1) * 22] assert set(cohort_pairs) == pool assert all(cohort_pairs.count(pair) == 2 for pair in pool) assert [row["cohort"] for row in schedule] == [1] * 8 + [2] * 8 + [3] * 8 for offset in range(0, len(schedule), 2): block = schedule[offset:offset + 2] assert len({row["team"] for row in block}) == 1 assert len({row["cohort"] for row in block}) == 1 assert {row["condition"] for row in block} == set(pilot.CONDITIONS) def test_balanced_repeat_generic_nondivisible_cohort(): options = args( "--agents-per-cohort", "7", "--cohorts", "2", "--teams", "2", "--sampling", "balanced-repeat", ) teams, _ = pilot.plan(options) pool = list(zip(pilot.DEFAULT_IDS, pilot.DEFAULT_SPLITS)) for team in teams: pairs = list(zip(team["ids"], team["splits"])) for cohort in range(2): counts = [pairs[cohort * 7:(cohort + 1) * 7].count(pair) for pair in pool] assert max(counts) - min(counts) <= 1 @pytest.mark.parametrize("flags", [ ("--agents-per-cohort", "1", "--sampling", "fixed"), ("--agents-per-cohort", "12", "--sampling", "without-replacement"), ("--ids", "lcbhard_0"), ("--ids", "lcbhard_0", "lcbhard_0", "--splits", "original", "original"), ]) def test_bad_plans_rejected(flags): with pytest.raises(ValueError): pilot.plan(args(*flags)) @pytest.mark.parametrize("flag", ["--agents-per-cohort", "--cohorts", "--teams", "--messages", "--token-limit", "--time-limit"]) def test_zero_budgets_and_sizes_rejected(flag): with pytest.raises(SystemExit): args(flag, "0") @pytest.mark.parametrize("value", ["nan", "inf", "-1", "2.1"]) def test_invalid_temperature_rejected(value): with pytest.raises(SystemExit): args("--temperature", value) def test_multiteam_execution_matches_conditions_and_isolates_boards(tmp_path, monkeypatch): import inspect_ai from inspect_ai.dataset import Sample import messageboardbench.board_task as board_task import messageboardbench.task as task_module calls, bindings = [], [] monkeypatch.setattr(pilot, "budget", lambda: {"usage": 0, "limit": 5, "limit_remaining": 5}) monkeypatch.setattr(pilot, 'load_pinned_datasets', lambda splits, revision: { split: {task_id: make_sample(task_id) for task_id in pilot.DEFAULT_IDS} for split in splits}) monkeypatch.setattr(inspect_ai, "Task", lambda **kw: SimpleNamespace(**kw)) monkeypatch.setattr(board_task, "episode_solver", lambda *values: bindings.append(values)) monkeypatch.setattr(task_module, "scratch_scorer", lambda split: None) def evaluate(tasks, **kwargs): calls.append((tasks, kwargs)) return [SimpleNamespace(location="mock.eval", status="success", eval=SimpleNamespace(metadata=t.metadata), samples=[SimpleNamespace(id=t.dataset[0].id, metadata=t.dataset[0].metadata, scores={}, messages=[], model_usage={}, limit=None, error=None)]) for t in tasks] monkeypatch.setattr(inspect_ai, "eval", evaluate) out = tmp_path / "run" audit = write_audit(tmp_path / 'holdout-audit.json') calibration = write_calibration( tmp_path / 'calibration.json', message_limit=117, token_limit=12345, time_limit=321, temperature=0.5, reasoning_effort='low', ) calibration_run, calibration_review, validation_evidence = write_completed_calibration( calibration, tmp_path / 'calibration-run' ) communication = tmp_path / 'communication.json' common = ["board_pilot", "--out", str(out), "--teams", "2", "--agents-per-cohort", "2", "--cohorts", "2", "--sampling", "with-replacement", "--messages", "117", "--token-limit", "12345", "--time-limit", "321", "--temperature", "0.5", "--reasoning-effort", "low", "--dataset-revision", REVISION, "--holdout-audit", str(audit), "--calibration-plan", str(calibration), "--calibration-run", str(calibration_run), "--calibration-review", str(calibration_review), "--validation-evidence", str(validation_evidence)] monkeypatch.setattr("sys.argv", [*common, '--freeze-communication-plan', str(communication)]) pilot.main() monkeypatch.setenv('DOCKER_HOST', pilot.REMOTE_DOCKER_HOST) monkeypatch.setattr("sys.argv", [*common, '--communication-plan', str(communication), "--execute"]) pilot.main() assert len(calls) == 8 for tasks, kwargs in calls: assert len(tasks) == 2 assert kwargs["max_tasks"] == kwargs["max_samples"] == kwargs["max_sandboxes"] == 2 assert kwargs["token_limit"] == 12345 and kwargs["time_limit"] == 321 assert kwargs["temperature"] == 0.5 and kwargs["reasoning_effort"] == "low" assert all(t.message_limit == 117 for t in tasks) by_team_condition = {} episode_ids = [] for tasks, _ in calls: for task in tasks: sample = task.dataset[0] meta = sample.metadata episode_ids.append(meta["episode_id"]) by_team_condition.setdefault((meta["team"], meta["condition"]), []).append((sample.id, task.metadata["split"], meta["slot"])) assert len(episode_ids) == len(set(episode_ids)) == 16 for team in (1, 2): assert by_team_condition[team, "sham"] == by_team_condition[team, "shared"] shared_boards = {v[4] for v in bindings if v[0] == "shared"} assert shared_boards == {out / "board-team-1.sqlite", out / "board-team-2.sqlite"} sham_boards = [v[4] for v in bindings if v[0] == "sham"] assert len(sham_boards) == len(set(sham_boards)) == 8 assert all(path.name.startswith('sham-board-team-') for path in sham_boards) assert all(v[4] is not None for v in bindings) snapshot = json.loads((out / "board-final.json").read_text()) assert len(set(snapshot["run_ids"])) == 10 assert len(snapshot['stores']) == 10 manifest = json.loads((out / "manifest.json").read_text()) assert manifest["planned_episodes"] == 16 assert manifest['conditions'] == ['sham', 'shared'] assert manifest['policy_prompt']['variant'] == 'D' assert manifest['policy_prompt']['rendered_instruction_prompt'] == render_tools_instruction('D') assert manifest['policy_prompt']['rendered_instruction_prompt_sha256'] assert manifest['policy_prompt']['rendered_instruction_prompt_base64'] assert manifest['dataset']['revision'] == REVISION assert manifest['dataset']['revision_kind'] == 'immutable_commit' assert manifest['dataset']['holdout_audit']['sha256'] assert manifest['dataset']['approved_pair_hashes'] assert manifest['confirmatory'] and manifest['confirmatory_ready'] assert manifest['communication_plan']['status'] == 'verified-for-execution' assert manifest['calibration_plan']['sha256'] assert manifest['calibration_execution']['evidence_sha256'] assert manifest['calibration_review']['status'] == 'ready' assert manifest['prompt_d_validation']['status'] == 'ready' assert manifest['communication_plan_consumption']['status'] == 'consumed' assert manifest['completion']['mode'] == 'plain-assistant-final-or-submit' assert manifest['completion']['adds_model_visible_tools'] is False assert manifest['completion']['installed_identically_across_conditions'] assert manifest['identical_board_prompt_and_tools_both_conditions'] assert manifest['sham_posts_isolated_per_episode'] assert json.loads((out / "status.json").read_text())["status"] == "completed" from messageboardbench.board import board_tools shared = [v for v in bindings if v[0] == 'shared'] shared_post, _ = board_tools(shared[0][4], shared[0][3], shared[0][1], shared[0][2]) _, shared_read = board_tools(shared[1][4], shared[1][3], shared[1][1], shared[1][2]) asyncio.run(shared_post('shared text')) assert json.loads(asyncio.run(shared_read()))['posts'][0]['text'] == 'shared text' sham = [v for v in bindings if v[0] == 'sham'] sham_post, sham_self_read = board_tools(sham[0][4], sham[0][3], sham[0][1], sham[0][2]) _, other_sham_read = board_tools(sham[1][4], sham[1][3], sham[1][1], sham[1][2]) asyncio.run(sham_post('isolated text')) assert json.loads(asyncio.run(sham_self_read()))['posts'][0]['text'] == 'isolated text' assert json.loads(asyncio.run(other_sham_read()))['posts'] == [] phase_inputs = [json.loads(path.read_text()) for path in sorted(out.glob('phase-*-inputs.json'))] prompt_rows = [row['policy_prompt'] for phase in phase_inputs for row in phase] assert prompt_rows and all(row == manifest['policy_prompt'] for row in prompt_rows) samples = [row['sample'] for phase in phase_inputs for row in phase] assert all(sample['input'] == render_tools_instruction('D') for sample in samples) assert all(sample['metadata']['instruction_prompt'] == render_tools_instruction('D') for sample in samples) assert all(sample['metadata']['completion'] == manifest['completion'] for sample in samples) by_team_slot = {} for sample in samples: metadata = sample['metadata'] by_team_slot.setdefault((metadata['team'], metadata['slot']), []).append(sample) assert all(len(pair) == 2 and pair[0]['input'] == pair[1]['input'] for pair in by_team_slot.values()) def test_upstream_legacy_prompt_path_is_explicit(tmp_path, monkeypatch): import inspect_ai from inspect_ai.dataset import Sample import messageboardbench.board_task as board_task import messageboardbench.task as task_module upstream_instruction = render_tools_instruction('A') development_ids = ['lcbhard_0', 'lcbhard_1', 'lcbhard_2', 'lcbhard_10'] monkeypatch.setattr(pilot, 'budget', lambda: {'usage': 0, 'limit': 5, 'limit_remaining': 5}) monkeypatch.setattr(pilot, 'load_pinned_datasets', lambda splits, revision: { split: {task_id: make_sample(task_id) for task_id in development_ids} for split in splits}) monkeypatch.setattr(inspect_ai, 'Task', lambda **kw: SimpleNamespace(**kw)) monkeypatch.setattr(board_task, 'episode_solver', lambda *values: None) monkeypatch.setattr(task_module, 'scratch_scorer', lambda split: None) monkeypatch.setattr(inspect_ai, 'eval', lambda tasks, **kwargs: [SimpleNamespace( location='mock.eval', status='success', eval=SimpleNamespace(metadata=t.metadata), samples=[SimpleNamespace(id=t.dataset[0].id, metadata=t.dataset[0].metadata, scores={}, messages=[], model_usage={}, limit=None, error=None)]) for t in tasks]) monkeypatch.setenv('DOCKER_HOST', pilot.REMOTE_DOCKER_HOST) out = tmp_path / 'legacy' monkeypatch.setattr('sys.argv', ['board_pilot', '--out', str(out), '--prompt-variant', 'upstream-legacy', '--dataset-revision', REVISION, '--ids', *development_ids, '--splits', 'conflicting', 'conflicting', 'conflicting', 'conflicting', '--execute']) pilot.main() manifest = json.loads((out / 'manifest.json').read_text()) assert manifest['policy_prompt']['variant'] == 'upstream-legacy' assert manifest['policy_prompt']['rendered_instruction_prompt'] == upstream_instruction assert manifest['policy_prompt']['source'].startswith('upstream dataset') assert not manifest['confirmatory'] and not manifest['confirmatory_ready']