Checkpoint experiments through SWE pilot v2

2026-10-05 04:27:06 +00:00 · 2026-09-15 15:46:10 +05:30
1 parent abacd5c5e1
commit 72d77018d8
@@ -0,0 +1,102 @@
+"""Replay only cohort-1 conflicting final artifacts inside fresh isolated Docker.
+Run with messageboardbench/.venv/bin/python; never imports agent code on host.
+Output directory must be fresh. No model calls.
+"""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+import uuid
+from inspect_ai.log import read_eval_log
+
+PROBE = r'''
+import importlib.util, json
+from pathlib import Path
+specification = json.loads(Path('/workspace/spec.json').read_text())
+# Independent reference computation before loading submitted code.
+if specification['task_id'] == 'lcbhard_1':
+    start, finish, limit, suffix = specification['args']
+    step = 10 ** len(suffix)
+    first = start + (int(suffix) - start) % step
+    reference = sum(all(int(c) <= limit for c in str(x)) for x in range(first, finish + 1, step))
+else:
+    n, m = specification['args']
+    reference = (pow(m - 1, n, 998244353) + (-1 if n % 2 else 1) * (m - 1)) % 998244353
+assert reference == specification['true']
+spec = importlib.util.spec_from_file_location('submitted', '/workspace/func.py')
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+function = getattr(module, specification['entry_point'])
+results = []
+for call_index in (1, 2, 3):
+    value = function(*specification['args'])
+    integer = int(value)
+    results.append({'call': call_index, 'repr': repr(value), 'int': integer,
+        'type': type(value).__module__ + '.' + type(value).__qualname__,
+        'exact_builtin_int': type(value) is int,
+        'eq_true': bool(value == specification['true']),
+        'eq_wrong': bool(value == specification['wrong']),
+        'int_eq_true': integer == specification['true'],
+        'int_eq_wrong': integer == specification['wrong']})
+print(json.dumps({'independent_reference': reference, 'calls': results}, sort_keys=True))
+'''
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--run', type=Path, required=True)
+    parser.add_argument('--out', type=Path, required=True)
+    args = parser.parse_args()
+    out = args.out.resolve()
+    if out.exists(): raise FileExistsError(out)
+    out.mkdir(parents=True)
+    image = 'aisiuk/inspect-tool-support'
+    image_id = subprocess.check_output(['docker', 'image', 'inspect', image, '--format', '{{.Id}}'], text=True).strip()
+    summaries = []
+    for path in sorted(args.run.resolve().rglob('*.eval')):
+        log = read_eval_log(path, header_only=True)
+        metadata = log.eval.metadata or {}
+        if log.status != 'success' or metadata.get('cohort') != 1 or metadata.get('split') != 'conflicting': continue
+        log = read_eval_log(path, resolve_attachments=True)
+        for sample in log.samples or []:
+            if sample.id not in {'lcbhard_1','lcbhard_2'}: continue
+            score = next(iter(sample.scores.values()))
+            source = score.metadata['submitted_artifacts']['func.py']
+            directory = out/f"{metadata['condition']}-{sample.id}"
+            directory.mkdir()
+            (directory/'func.py').write_text(source)
+            (directory/'probe.py').write_text(PROBE)
+            config = {'task_id': sample.id, 'entry_point': sample.metadata['entry_point'],
+                'args': [2946568,67236501,6,'403'] if sample.id == 'lcbhard_1' else [458753,591009],
+                'true': 15778 if sample.id == 'lcbhard_1' else 0,
+                'wrong': 8035 if sample.id == 'lcbhard_1' else 1603161}
+            (directory/'spec.json').write_text(json.dumps(config,indent=2)+'\n')
+            name = 'board-artifact-probe-' + uuid.uuid4().hex[:12]
+            command = ['docker','run','--rm','--name',name,'--network','none',
+                '--memory','256m','--pids-limit','64','--read-only','--cap-drop','ALL',
+                '--security-opt','no-new-privileges','--user','65534:65534',
+                '--mount',f'type=bind,src={directory},dst=/workspace,readonly',
+                '--workdir','/workspace',image_id,'python3','-B','/workspace/probe.py']
+            try:
+                result = subprocess.run(command,capture_output=True,text=True,timeout=60)
+            except subprocess.TimeoutExpired:
+                subprocess.run(['docker','rm','-f',name],capture_output=True)
+                raise
+            (directory/'stdout.txt').write_text(result.stdout)
+            (directory/'stderr.txt').write_text(result.stderr)
+            record = {'condition': metadata['condition'], 'episode_id': sample.metadata['episode_id'],
+                'task_id': sample.id, 'source_log': str(path),
+                'source_log_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
+                'submitted_source_sha256': hashlib.sha256(source.encode()).hexdigest(),
+                'probe_sha256': hashlib.sha256(PROBE.encode()).hexdigest(),
+                'image_id': image_id,'command':command,'exit_code':result.returncode,
+                'specification':config,'result':json.loads(result.stdout) if result.returncode == 0 else None}
+            (directory/'result.json').write_text(json.dumps(record,indent=2)+'\n')
+            summaries.append(record)
+            if result.returncode != 0: raise RuntimeError(f'Probe failed: {directory}')
+    assert len(summaries) == 4, f'Expected four completed artifacts, found {len(summaries)}'
+    (out/'results.json').write_text(json.dumps(summaries,indent=2)+'\n')
+    print(json.dumps([{'condition':r['condition'],'task_id':r['task_id'],'calls':r['result']['calls']} for r in summaries],indent=2))
+
+if __name__ == '__main__': main()