Checkpoint experiments through SWE pilot v2

tests/test_experiment_bundle.py

@@ -0,0 +1,171 @@
+import json
+from pathlib import Path
+import subprocess
+
+import pytest
+
+from messageboardbench.experiment_bundle import (
+    REMOTE_DOCKER_HOST,
+    manifest_sha256,
+    run_bundle,
+    validate_manifest,
+)
+
+
+def script(root: Path, relative: str) -> None:
+    path = root / relative
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("# offline fixture\n")
+
+
+def ready_manifest(root: Path) -> dict:
+    for name in (
+        "scripts/fake_runner.py",
+        "scripts/board_report.py",
+        "scripts/analysis/validate_board_export.py",
+        "scripts/analysis/board_resources.py",
+        "scripts/remote_docker.py",
+    ):
+        script(root, name)
+    manifest = {
+        "schema_version": 1,
+        "status": "ready",
+        "experiment_id": "fixture",
+        "remote_docker_host": REMOTE_DOCKER_HOST,
+        "blockers": [],
+        "outputs": {
+            "run_dir": "logs/fixture/run",
+            "report_dir": "logs/fixture/report",
+            "verification_file": "logs/fixture/verification.json",
+            "resource_file": "logs/fixture/resources.json",
+            "state_file": "logs/fixture-status.json",
+        },
+        "execution": {
+            "argv": [".venv/bin/python", "scripts/fake_runner.py", "--out", "logs/fixture/run", "--execute"]
+        },
+        "postprocess": [
+            {
+                "name": "report",
+                "requires": ["logs/fixture/run/manifest.json"],
+                "argv": [".venv/bin/python", "scripts/board_report.py", "--run", "logs/fixture/run", "--out", "logs/fixture/report"],
+            },
+            {
+                "name": "verify",
+                "requires": ["logs/fixture/report/manifest.json"],
+                "argv": [".venv/bin/python", "scripts/analysis/validate_board_export.py", "--run", "logs/fixture/run", "--export", "logs/fixture/report", "--out", "logs/fixture/verification.json"],
+            },
+            {
+                "name": "resources",
+                "requires": ["logs/fixture/report/episodes.json"],
+                "argv": [".venv/bin/python", "scripts/analysis/board_resources.py", "--run", "logs/fixture/run", "--export", "logs/fixture/report", "--out", "logs/fixture/resources.json"],
+            },
+        ],
+    }
+    manifest["manifest_sha256"] = manifest_sha256(manifest)
+    return manifest
+
+
+def test_draft_fails_before_command_validation(tmp_path):
+    manifest = {"schema_version": 1, "status": "draft", "blockers": ["not frozen"]}
+    with pytest.raises(ValueError, match="not frozen"):
+        validate_manifest(manifest, tmp_path)
+
+
+def test_ready_manifest_rejects_hash_mutation_and_nonremote_host(tmp_path):
+    manifest = ready_manifest(tmp_path)
+    validate_manifest(manifest, tmp_path)
+    manifest["experiment_id"] = "mutated"
+    with pytest.raises(ValueError, match="self-hash"):
+        validate_manifest(manifest, tmp_path)
+    manifest["manifest_sha256"] = manifest_sha256(manifest)
+    manifest["remote_docker_host"] = "unix:///var/run/docker.sock"
+    manifest["manifest_sha256"] = manifest_sha256(manifest)
+    with pytest.raises(ValueError, match="remote_docker_host"):
+        validate_manifest(manifest, tmp_path)
+
+
+def test_nonzero_execution_still_runs_safe_available_postprocessing(tmp_path):
+    manifest = ready_manifest(tmp_path)
+    manifest_path = tmp_path / "experiment.json"
+    manifest_path.write_text(json.dumps(manifest))
+    calls = []
+
+    def fake_run(argv, **kwargs):
+        calls.append(argv)
+        if len(calls) == 1:
+            run_dir = tmp_path / "logs/fixture/run"
+            run_dir.mkdir(parents=True)
+            (run_dir / "manifest.json").write_text("{}")
+            return subprocess.CompletedProcess(argv, 9)
+        if len(calls) == 2:
+            report_dir = tmp_path / "logs/fixture/report"
+            report_dir.mkdir(parents=True)
+            (report_dir / "manifest.json").write_text("{}")
+            (report_dir / "episodes.json").write_text("[]")
+        return subprocess.CompletedProcess(argv, 0)
+
+    assert run_bundle(manifest_path, tmp_path, run=fake_run) == 9
+    assert calls[0][:3] == [".venv/bin/python", "scripts/remote_docker.py", "--"]
+    assert [call[1] for call in calls[1:]] == [
+        "scripts/board_report.py", "scripts/analysis/validate_board_export.py",
+        "scripts/analysis/board_resources.py",
+    ]
+    status = json.loads((tmp_path / "logs/fixture-status.json").read_text())
+    assert status["status"] == "partial"
+    assert status["execution"]["returncode"] == 9
+    assert [step["status"] for step in status["postprocess"]] == ["completed", "completed", "completed"]
+
+
+def test_missing_postprocess_input_is_recorded_without_running_step(tmp_path):
+    manifest = ready_manifest(tmp_path)
+    manifest_path = tmp_path / "experiment.json"
+    manifest_path.write_text(json.dumps(manifest))
+    calls = []
+
+    def fake_run(argv, **kwargs):
+        calls.append(argv)
+        return subprocess.CompletedProcess(argv, 4)
+
+    assert run_bundle(manifest_path, tmp_path, run=fake_run) == 4
+    assert len(calls) == 1
+    status = json.loads((tmp_path / "logs/fixture-status.json").read_text())
+    assert status["status"] == "partial"
+    assert [step["status"] for step in status["postprocess"]] == ["skipped", "skipped", "skipped"]
+
+
+def test_existing_state_refuses_a_second_start_before_execution(tmp_path):
+    manifest = ready_manifest(tmp_path)
+    manifest_path = tmp_path / "experiment.json"
+    manifest_path.write_text(json.dumps(manifest))
+    state = tmp_path / "logs/fixture-status.json"
+    state.parent.mkdir(parents=True)
+    state.write_text('{"status":"running"}\n')
+    calls = []
+    with pytest.raises(ValueError, match="fresh output already exists"):
+        run_bundle(manifest_path, tmp_path, run=lambda *args, **kwargs: calls.append(args))
+    assert calls == []
+
+
+def test_resumable_bundle_archives_derived_outputs_before_restart(tmp_path):
+    manifest = ready_manifest(tmp_path)
+    manifest["execution"]["resume"] = True
+    manifest["manifest_sha256"] = manifest_sha256(manifest)
+    manifest_path = tmp_path / "experiment.json"
+    manifest_path.write_text(json.dumps(manifest))
+    run_dir = tmp_path / "logs/fixture/run"
+    run_dir.mkdir(parents=True)
+    (run_dir / "manifest.json").write_text("{}")
+    report = tmp_path / "logs/fixture/report"
+    report.mkdir(parents=True)
+    (report / "old.txt").write_text("old")
+    state = tmp_path / "logs/fixture-status.json"
+    state.write_text(json.dumps({"experiment_id": "fixture", "status": "partial", "resume_count": 0}))
+
+    def fake_run(argv, **kwargs):
+        return subprocess.CompletedProcess(argv, 7)
+
+    assert run_bundle(manifest_path, tmp_path, run=fake_run) == 7
+    archived = tmp_path / "logs/fixture/resume-history/attempt-1/report/old.txt"
+    assert archived.read_text() == "old"
+    status = json.loads(state.read_text())
+    assert status["resume_count"] == 1