mirror of
https://github.com/priyanshujain/messageboardbench.git
synced 2026-10-04 12:07:06 +00:00
Add deterministic SWE candidate screening
This commit is contained in:
1 parent
aeaae4291f
commit
175c9d48f4
13 files changed
+1292
-50
No files matched your search
@@ -38,14 +38,19 @@ def fixture(tmp_path):
|
||||
name = f"{split}-{mode}.txt"
|
||||
output_hashes[name] = write(tmp_path / "evidence" / name, "test output")
|
||||
cells.append({"split": split, "mode": mode, "resolved": expected[split, mode],
|
||||
"image": "repo:tag", "test_command": ["pytest"],
|
||||
"image_id": "sha256:image", "repo_digests": ["repo@sha256:digest"],
|
||||
"target_statuses": {"target": "PASSED" if mode == "oracle" else "FAILED"},
|
||||
"target_statuses": {"target": "PASSED" if expected[split, mode] else "FAILED"},
|
||||
"output_file": name, "output_sha256": output_hashes[name]})
|
||||
record = {"instance_id": "task", "base_commit": "base", "repo": "org/repo",
|
||||
"version": "1", "original_test_patch": "original", "test_patch": "conflict",
|
||||
"patch": "oracle"}
|
||||
manifest = {"schema_version": 1, "dataset": "fjzzq2002/impossible_swebench",
|
||||
"dataset_revision": "1" * 40, "instance_id": "task", "network": "none",
|
||||
"image": "repo:tag",
|
||||
"remote_image": {"id": "sha256:image",
|
||||
"repo_digests": ["repo@sha256:digest"]},
|
||||
"test_command": ["pytest"],
|
||||
"base_commit": "base", "repo": "org/repo", "version": "1",
|
||||
"original_test_patch_sha256": hashlib.sha256(b"original").hexdigest(),
|
||||
"conflicting_test_patch_sha256": hashlib.sha256(b"conflict").hexdigest(),
|
||||
@@ -90,6 +95,21 @@ def test_environment_index_is_required_and_plan_bound(tmp_path):
|
||||
validate_environment_index(plan, tmp_path)
|
||||
|
||||
|
||||
def test_unresolved_cell_requires_an_actual_failed_target(tmp_path):
|
||||
plan, manifest_path, record = fixture(tmp_path)
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
manifest["results"][0]["target_statuses"] = {"target": "PASSED"}
|
||||
write(manifest_path, manifest)
|
||||
index_path = tmp_path / "index.json"
|
||||
index = json.loads(index_path.read_text())
|
||||
index["manifests"]["task"]["sha256"] = hashlib.sha256(
|
||||
manifest_path.read_bytes()
|
||||
).hexdigest()
|
||||
write(index_path, index)
|
||||
with pytest.raises(ValueError, match="lacks a failed target"):
|
||||
validate_environment_index_for_records(plan, tmp_path, {"task": record})
|
||||
|
||||
|
||||
def test_environment_manifest_patch_hashes_are_bound_to_frozen_record(tmp_path):
|
||||
plan, _, record = fixture(tmp_path)
|
||||
changed = {**record, "test_patch": "different"}
|
||||
|
||||
Reference in new issue
Block a user