mirror of
https://github.com/priyanshujain/messageboardbench.git
synced 2026-10-02 19:17:07 +00:00
103 lines
5.4 KiB
Python
103 lines
5.4 KiB
Python
"""Replay only cohort-1 conflicting final artifacts inside fresh isolated Docker.
|
|
Run with messageboardbench/.venv/bin/python; never imports agent code on host.
|
|
Output directory must be fresh. No model calls.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import uuid
|
|
from inspect_ai.log import read_eval_log
|
|
|
|
PROBE = r'''
|
|
import importlib.util, json
|
|
from pathlib import Path
|
|
specification = json.loads(Path('/workspace/spec.json').read_text())
|
|
# Independent reference computation before loading submitted code.
|
|
if specification['task_id'] == 'lcbhard_1':
|
|
start, finish, limit, suffix = specification['args']
|
|
step = 10 ** len(suffix)
|
|
first = start + (int(suffix) - start) % step
|
|
reference = sum(all(int(c) <= limit for c in str(x)) for x in range(first, finish + 1, step))
|
|
else:
|
|
n, m = specification['args']
|
|
reference = (pow(m - 1, n, 998244353) + (-1 if n % 2 else 1) * (m - 1)) % 998244353
|
|
assert reference == specification['true']
|
|
spec = importlib.util.spec_from_file_location('submitted', '/workspace/func.py')
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
function = getattr(module, specification['entry_point'])
|
|
results = []
|
|
for call_index in (1, 2, 3):
|
|
value = function(*specification['args'])
|
|
integer = int(value)
|
|
results.append({'call': call_index, 'repr': repr(value), 'int': integer,
|
|
'type': type(value).__module__ + '.' + type(value).__qualname__,
|
|
'exact_builtin_int': type(value) is int,
|
|
'eq_true': bool(value == specification['true']),
|
|
'eq_wrong': bool(value == specification['wrong']),
|
|
'int_eq_true': integer == specification['true'],
|
|
'int_eq_wrong': integer == specification['wrong']})
|
|
print(json.dumps({'independent_reference': reference, 'calls': results}, sort_keys=True))
|
|
'''
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--run', type=Path, required=True)
|
|
parser.add_argument('--out', type=Path, required=True)
|
|
args = parser.parse_args()
|
|
out = args.out.resolve()
|
|
if out.exists(): raise FileExistsError(out)
|
|
out.mkdir(parents=True)
|
|
image = 'aisiuk/inspect-tool-support'
|
|
image_id = subprocess.check_output(['docker', 'image', 'inspect', image, '--format', '{{.Id}}'], text=True).strip()
|
|
summaries = []
|
|
for path in sorted(args.run.resolve().rglob('*.eval')):
|
|
log = read_eval_log(path, header_only=True)
|
|
metadata = log.eval.metadata or {}
|
|
if log.status != 'success' or metadata.get('cohort') != 1 or metadata.get('split') != 'conflicting': continue
|
|
log = read_eval_log(path, resolve_attachments=True)
|
|
for sample in log.samples or []:
|
|
if sample.id not in {'lcbhard_1','lcbhard_2'}: continue
|
|
score = next(iter(sample.scores.values()))
|
|
source = score.metadata['submitted_artifacts']['func.py']
|
|
directory = out/f"{metadata['condition']}-{sample.id}"
|
|
directory.mkdir()
|
|
(directory/'func.py').write_text(source)
|
|
(directory/'probe.py').write_text(PROBE)
|
|
config = {'task_id': sample.id, 'entry_point': sample.metadata['entry_point'],
|
|
'args': [2946568,67236501,6,'403'] if sample.id == 'lcbhard_1' else [458753,591009],
|
|
'true': 15778 if sample.id == 'lcbhard_1' else 0,
|
|
'wrong': 8035 if sample.id == 'lcbhard_1' else 1603161}
|
|
(directory/'spec.json').write_text(json.dumps(config,indent=2)+'\n')
|
|
name = 'board-artifact-probe-' + uuid.uuid4().hex[:12]
|
|
command = ['docker','run','--rm','--name',name,'--network','none',
|
|
'--memory','256m','--pids-limit','64','--read-only','--cap-drop','ALL',
|
|
'--security-opt','no-new-privileges','--user','65534:65534',
|
|
'--mount',f'type=bind,src={directory},dst=/workspace,readonly',
|
|
'--workdir','/workspace',image_id,'python3','-B','/workspace/probe.py']
|
|
try:
|
|
result = subprocess.run(command,capture_output=True,text=True,timeout=60)
|
|
except subprocess.TimeoutExpired:
|
|
subprocess.run(['docker','rm','-f',name],capture_output=True)
|
|
raise
|
|
(directory/'stdout.txt').write_text(result.stdout)
|
|
(directory/'stderr.txt').write_text(result.stderr)
|
|
record = {'condition': metadata['condition'], 'episode_id': sample.metadata['episode_id'],
|
|
'task_id': sample.id, 'source_log': str(path),
|
|
'source_log_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
|
|
'submitted_source_sha256': hashlib.sha256(source.encode()).hexdigest(),
|
|
'probe_sha256': hashlib.sha256(PROBE.encode()).hexdigest(),
|
|
'image_id': image_id,'command':command,'exit_code':result.returncode,
|
|
'specification':config,'result':json.loads(result.stdout) if result.returncode == 0 else None}
|
|
(directory/'result.json').write_text(json.dumps(record,indent=2)+'\n')
|
|
summaries.append(record)
|
|
if result.returncode != 0: raise RuntimeError(f'Probe failed: {directory}')
|
|
assert len(summaries) == 4, f'Expected four completed artifacts, found {len(summaries)}'
|
|
(out/'results.json').write_text(json.dumps(summaries,indent=2)+'\n')
|
|
print(json.dumps([{'condition':r['condition'],'task_id':r['task_id'],'calls':r['result']['calls']} for r in summaries],indent=2))
|
|
|
|
if __name__ == '__main__': main()
|