mirror of
https://github.com/priyanshujain/messageboardbench.git
synced 2026-10-02 19:17:07 +00:00
103 lines
4.0 KiB
Python
103 lines
4.0 KiB
Python
"""Build or validate the no-model SWE readiness index for a frozen pilot."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
|
|
from messageboardbench.swe_board import load_records, plan_hash
|
|
from messageboardbench.swe_prerequisites import validate_environment_index_for_records
|
|
from messageboardbench.swe_validation import (
|
|
ValidationError,
|
|
docker_preflight,
|
|
load_pair,
|
|
manifest as trial_manifest,
|
|
run_trial,
|
|
swebench_spec,
|
|
validate_expected_matrix,
|
|
)
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def sha(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def pull_image_once(image: str, pulled: set[str], environ, run=subprocess.run) -> None:
|
|
"""Pull an exact image tag once before run_trial tries to inspect it."""
|
|
if image in pulled:
|
|
return
|
|
result = run(["docker", "pull", image], env=dict(environ), text=True,
|
|
capture_output=True)
|
|
if result.returncode:
|
|
detail = (result.stderr or result.stdout or "").strip()
|
|
raise ValidationError(f"image pull failed for {image}: {detail}")
|
|
pulled.add(image)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--plan", type=Path, required=True)
|
|
parser.add_argument("--out", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
plan = json.loads(args.plan.read_text())
|
|
if plan.get("plan_sha256") != plan_hash(plan):
|
|
raise SystemExit("frozen plan self-hash mismatch")
|
|
declared = (ROOT / plan["environment_validation"]["index_path"]).resolve()
|
|
out = args.out.resolve()
|
|
if declared != out / "index.json":
|
|
raise SystemExit("--out does not match the frozen validation index location")
|
|
if declared.is_file():
|
|
records = load_records(plan["dataset"]["revision"], "conflicting")
|
|
result = validate_environment_index_for_records(plan, ROOT, records)
|
|
print(json.dumps({"status": "already validated", **result}, indent=2))
|
|
return 0
|
|
|
|
docker_preflight(os.environ)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
entries = {}
|
|
pulled_images: set[str] = set()
|
|
for instance_id in plan["selection"]["instance_ids"]:
|
|
task_dir = out / instance_id.replace("/", "_")
|
|
manifest_path = task_dir / "manifest.json"
|
|
if not manifest_path.exists():
|
|
original, conflicting = load_pair(plan["dataset"]["revision"], instance_id)
|
|
pull_image_once(swebench_spec(original)[0], pulled_images, os.environ)
|
|
results = [
|
|
run_trial(record, split=split, mode=mode, out_dir=task_dir,
|
|
environ=os.environ,
|
|
memory=plan["parameters"]["memory"],
|
|
timeout_seconds=plan["parameters"]["scorer_timeout_seconds"])
|
|
for split, record in (("original", original), ("conflicting", conflicting))
|
|
for mode in ("nochange", "oracle")
|
|
]
|
|
validate_expected_matrix(results)
|
|
value = trial_manifest(
|
|
plan["dataset"]["revision"], instance_id, original, conflicting, results
|
|
)
|
|
with manifest_path.open("x") as handle:
|
|
json.dump(value, handle, indent=2, sort_keys=True)
|
|
handle.write("\n")
|
|
entries[instance_id] = {
|
|
"path": str(manifest_path.relative_to(ROOT)), "sha256": sha(manifest_path)
|
|
}
|
|
index = {"schema_version": 1, "status": "validated",
|
|
"plan_sha256": plan["plan_sha256"], "dataset": plan["dataset"],
|
|
"manifests": entries}
|
|
with declared.open("x") as handle:
|
|
json.dump(index, handle, indent=2, sort_keys=True)
|
|
handle.write("\n")
|
|
records = load_records(plan["dataset"]["revision"], "conflicting")
|
|
result = validate_environment_index_for_records(plan, ROOT, records)
|
|
print(json.dumps({"status": "validated", **result}, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|