Checkpoint experiments through SWE pilot v2

scripts/analysis/replay_board_artifacts.py

@@ -0,0 +1,89 @@
+"""Replay final artifacts in Docker without networking or host mounts; no paid calls.
+
+Host code only copies artifact bytes. Each artifact executes in a separate container.
+"""
+import argparse
+import hashlib
+import io
+import json
+from pathlib import Path
+import subprocess
+import tarfile
+import uuid
+
+
+def replay(files, destination, image):
+    destination.mkdir(parents=True, exist_ok=False)
+    for name, content in files.items():
+        (destination / name).write_bytes(content)
+    archive = io.BytesIO()
+    with tarfile.open(fileobj=archive, mode='w') as tar:
+        directory = tarfile.TarInfo('workspace')
+        directory.type = tarfile.DIRTYPE
+        directory.mode = 0o755
+        tar.addfile(directory)
+        for name, content in files.items():
+            info = tarfile.TarInfo('workspace/' + name)
+            info.size = len(content)
+            info.mode = 0o444
+            tar.addfile(info, io.BytesIO(content))
+    name = 'board-artifact-validation-' + uuid.uuid4().hex[:12]
+    create = ['docker', 'create', '--name', name, '--network', 'none', '--memory', '512m',
+              '--pids-limit', '64', '--cap-drop', 'ALL', '--security-opt',
+              'no-new-privileges', '--user', '65534:65534', '--workdir', '/workspace',
+              image, 'python3', '-B', '/workspace/probe.py']
+    subprocess.run(create, check=True, capture_output=True)
+    try:
+        subprocess.run(['docker', 'cp', '-', name + ':/'], input=archive.getvalue(), check=True, capture_output=True)
+        configuration = json.loads(subprocess.check_output(['docker', 'inspect', name]))[0]
+        assert configuration['HostConfig']['NetworkMode'] == 'none'
+        assert not configuration['Mounts']
+        try:
+            result = subprocess.run(['docker', 'start', '-a', name], capture_output=True, timeout=120)
+            timed_out = False
+        except subprocess.TimeoutExpired as error:
+            result = subprocess.CompletedProcess([], 124, error.stdout or b'', error.stderr or b'')
+            timed_out = True
+        state = json.loads(subprocess.check_output(['docker', 'inspect', name]))[0]['State']
+        (destination / 'stdout.txt').write_bytes(result.stdout)
+        (destination / 'stderr.txt').write_bytes(result.stderr)
+        record = {'create_command': create, 'network_mode': configuration['HostConfig']['NetworkMode'],
+                  'mounts': configuration['Mounts'], 'image_id': configuration['Image'],
+                  'docker_start_exit_code': result.returncode, 'container_exit_code': state['ExitCode'],
+                  'timed_out': timed_out, 'files_sha256': {k: hashlib.sha256(v).hexdigest() for k, v in files.items()}}
+        (destination / 'result.json').write_text(json.dumps(record, indent=2) + '\n')
+        return record
+    finally:
+        subprocess.run(['docker', 'rm', '-f', name], capture_output=True, check=True)
+
+
+if __name__ == '__main__':
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument('--run', type=Path, required=True)
+    p.add_argument('--export', type=Path, required=True)
+    p.add_argument('--out', type=Path, required=True)
+    p.add_argument('--image', default='aisiuk/inspect-tool-support')
+    args = p.parse_args()
+    args.out.mkdir(parents=True, exist_ok=False)
+    inputs = {x['sample']['metadata']['episode_id']: x['sample']['metadata']
+              for path in args.run.glob('phase-*-inputs.json') for x in json.loads(path.read_text())}
+    rows = json.loads((args.export / 'episodes.json').read_text())
+    records = []
+    for row in rows:
+        final = json.loads((args.export / row['report_directory'] / 'final-artifacts.json').read_text())
+        artifacts = final.get('submitted_artifacts', {})
+        if not isinstance(artifacts.get('func.py'), str):
+            records.append({'episode_id': row['episode_id'], 'skipped': 'No captured source'})
+            continue
+        meta = inputs[row['episode_id']]
+        test = artifacts.get('test.py')
+        if not test:
+            raise ValueError('Missing captured test.py: ' + row['episode_id'])
+        # Execute the exact captured test file. runpy preserves filename and __main__.
+        probe = "import runpy\nrunpy.run_path('/workspace/test.py', run_name='__main__')\nprint('REPLAY_COMPLETED')\n"
+        files = {'func.py': artifacts['func.py'].encode(), 'test.py': test.encode(), 'probe.py': probe.encode()}
+        result = replay(files, args.out / row['report_directory'], args.image)
+        records.append({'episode_id': row['episode_id'], 'task_id': row['task_id'], 'condition': row['condition'],
+                        'split': row['split'], 'reported_score': row['score'], **result})
+        print(json.dumps({'episode_id': row['episode_id'], 'container_exit_code': result['container_exit_code']}), flush=True)
+    (args.out / 'summary.json').write_text(json.dumps(records, indent=2) + '\n')