messageboardbench/tests/test_experiment_bundle.py

import json
from pathlib import Path
import subprocess

import pytest

from messageboardbench.experiment_bundle import (
    REMOTE_DOCKER_HOST,
    manifest_sha256,
    run_bundle,
    validate_manifest,
)


def script(root: Path, relative: str) -> None:
    path = root / relative
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text("# offline fixture\n")


def ready_manifest(root: Path) -> dict:
    for name in (
        "scripts/fake_runner.py",
        "scripts/board_report.py",
        "scripts/analysis/validate_board_export.py",
        "scripts/analysis/board_resources.py",
        "scripts/remote_docker.py",
    ):
        script(root, name)
    manifest = {
        "schema_version": 1,
        "status": "ready",
        "experiment_id": "fixture",
        "remote_docker_host": REMOTE_DOCKER_HOST,
        "blockers": [],
        "outputs": {
            "run_dir": "logs/fixture/run",
            "report_dir": "logs/fixture/report",
            "verification_file": "logs/fixture/verification.json",
            "resource_file": "logs/fixture/resources.json",
            "state_file": "logs/fixture-status.json",
        },
        "execution": {
            "argv": [".venv/bin/python", "scripts/fake_runner.py", "--out", "logs/fixture/run", "--execute"]
        },
        "postprocess": [
            {
                "name": "report",
                "requires": ["logs/fixture/run/manifest.json"],
                "argv": [".venv/bin/python", "scripts/board_report.py", "--run", "logs/fixture/run", "--out", "logs/fixture/report"],
            },
            {
                "name": "verify",
                "requires": ["logs/fixture/report/manifest.json"],
                "argv": [".venv/bin/python", "scripts/analysis/validate_board_export.py", "--run", "logs/fixture/run", "--export", "logs/fixture/report", "--out", "logs/fixture/verification.json"],
            },
            {
                "name": "resources",
                "requires": ["logs/fixture/report/episodes.json"],
                "argv": [".venv/bin/python", "scripts/analysis/board_resources.py", "--run", "logs/fixture/run", "--export", "logs/fixture/report", "--out", "logs/fixture/resources.json"],
            },
        ],
    }
    manifest["manifest_sha256"] = manifest_sha256(manifest)
    return manifest


def test_draft_fails_before_command_validation(tmp_path):
    manifest = {"schema_version": 1, "status": "draft", "blockers": ["not frozen"]}
    with pytest.raises(ValueError, match="not frozen"):
        validate_manifest(manifest, tmp_path)


def test_ready_manifest_rejects_hash_mutation_and_nonremote_host(tmp_path):
    manifest = ready_manifest(tmp_path)
    validate_manifest(manifest, tmp_path)
    manifest["experiment_id"] = "mutated"
    with pytest.raises(ValueError, match="self-hash"):
        validate_manifest(manifest, tmp_path)
    manifest["manifest_sha256"] = manifest_sha256(manifest)
    manifest["remote_docker_host"] = "unix:///var/run/docker.sock"
    manifest["manifest_sha256"] = manifest_sha256(manifest)
    with pytest.raises(ValueError, match="remote_docker_host"):
        validate_manifest(manifest, tmp_path)


def test_nonzero_execution_still_runs_safe_available_postprocessing(tmp_path):
    manifest = ready_manifest(tmp_path)
    manifest_path = tmp_path / "experiment.json"
    manifest_path.write_text(json.dumps(manifest))
    calls = []

    def fake_run(argv, **kwargs):
        calls.append(argv)
        if len(calls) == 1:
            run_dir = tmp_path / "logs/fixture/run"
            run_dir.mkdir(parents=True)
            (run_dir / "manifest.json").write_text("{}")
            return subprocess.CompletedProcess(argv, 9)
        if len(calls) == 2:
            report_dir = tmp_path / "logs/fixture/report"
            report_dir.mkdir(parents=True)
            (report_dir / "manifest.json").write_text("{}")
            (report_dir / "episodes.json").write_text("[]")
        return subprocess.CompletedProcess(argv, 0)

    assert run_bundle(manifest_path, tmp_path, run=fake_run) == 9
    assert calls[0][:3] == [".venv/bin/python", "scripts/remote_docker.py", "--"]
    assert [call[1] for call in calls[1:]] == [
        "scripts/board_report.py", "scripts/analysis/validate_board_export.py",
        "scripts/analysis/board_resources.py",
    ]
    status = json.loads((tmp_path / "logs/fixture-status.json").read_text())
    assert status["status"] == "partial"
    assert status["execution"]["returncode"] == 9
    assert [step["status"] for step in status["postprocess"]] == ["completed", "completed", "completed"]


def test_missing_postprocess_input_is_recorded_without_running_step(tmp_path):
    manifest = ready_manifest(tmp_path)
    manifest_path = tmp_path / "experiment.json"
    manifest_path.write_text(json.dumps(manifest))
    calls = []

    def fake_run(argv, **kwargs):
        calls.append(argv)
        return subprocess.CompletedProcess(argv, 4)

    assert run_bundle(manifest_path, tmp_path, run=fake_run) == 4
    assert len(calls) == 1
    status = json.loads((tmp_path / "logs/fixture-status.json").read_text())
    assert status["status"] == "partial"
    assert [step["status"] for step in status["postprocess"]] == ["skipped", "skipped", "skipped"]


def test_existing_state_refuses_a_second_start_before_execution(tmp_path):
    manifest = ready_manifest(tmp_path)
    manifest_path = tmp_path / "experiment.json"
    manifest_path.write_text(json.dumps(manifest))
    state = tmp_path / "logs/fixture-status.json"
    state.parent.mkdir(parents=True)
    state.write_text('{"status":"running"}\n')
    calls = []
    with pytest.raises(ValueError, match="fresh output already exists"):
        run_bundle(manifest_path, tmp_path, run=lambda *args, **kwargs: calls.append(args))
    assert calls == []


def test_resumable_bundle_archives_derived_outputs_before_restart(tmp_path):
    manifest = ready_manifest(tmp_path)
    manifest["execution"]["resume"] = True
    manifest["manifest_sha256"] = manifest_sha256(manifest)
    manifest_path = tmp_path / "experiment.json"
    manifest_path.write_text(json.dumps(manifest))
    run_dir = tmp_path / "logs/fixture/run"
    run_dir.mkdir(parents=True)
    (run_dir / "manifest.json").write_text("{}")
    report = tmp_path / "logs/fixture/report"
    report.mkdir(parents=True)
    (report / "old.txt").write_text("old")
    state = tmp_path / "logs/fixture-status.json"
    state.write_text(json.dumps({"experiment_id": "fixture", "status": "partial", "resume_count": 0}))

    def fake_run(argv, **kwargs):
        return subprocess.CompletedProcess(argv, 7)

    assert run_bundle(manifest_path, tmp_path, run=fake_run) == 7
    archived = tmp_path / "logs/fixture/resume-history/attempt-1/report/old.txt"
    assert archived.read_text() == "old"
    status = json.loads(state.read_text())
    assert status["resume_count"] == 1