mirror of
https://github.com/priyanshujain/messageboardbench.git
synced 2026-10-04 03:57:06 +00:00
fix swe pilot v3 offline verification
This commit is contained in:
1 parent
08c8de15c2
commit
13d62ae7fc
6 files changed
+357
-31
No files matched your search
@@ -1,4 +1,9 @@
|
||||
from scripts.swe_population_report import binary_score, paired_analysis
|
||||
from messageboardbench.swe_reporting import (
|
||||
binary_score,
|
||||
paired_analysis,
|
||||
strict_analysis_rows,
|
||||
summarize,
|
||||
)
|
||||
|
||||
|
||||
def test_paired_analysis_uses_each_arm_not_late_bound_generator_variable():
|
||||
@@ -16,3 +21,37 @@ def test_paired_analysis_uses_each_arm_not_late_bound_generator_variable():
|
||||
|
||||
def test_binary_score_tolerates_partial_generic_episode_rows():
|
||||
assert binary_score({}) is None
|
||||
|
||||
|
||||
def test_missing_strict_targets_are_excluded_from_primary_and_paired_analysis():
|
||||
rows = [
|
||||
{"episode_id": "control-a", "team": 1, "task_id": "a", "condition": "control", "score": 1.0},
|
||||
{"episode_id": "board-a", "team": 1, "task_id": "a", "condition": "board", "score": 0.0},
|
||||
{"episode_id": "control-b", "team": 1, "task_id": "b", "condition": "control", "score": 0.0},
|
||||
{"episode_id": "board-b", "team": 1, "task_id": "b", "condition": "board", "score": 0.0},
|
||||
]
|
||||
artifacts = {
|
||||
"control-a": {"strict_target_statuses": {"test": "PASSED"}},
|
||||
"board-a": {"strict_target_statuses": {"test": "FAILED"}},
|
||||
"control-b": {"strict_target_statuses": {"test": "MISSING"}},
|
||||
"board-b": {"strict_target_statuses": {"test": "ERROR"}},
|
||||
}
|
||||
analysis_rows = strict_analysis_rows(rows, artifacts)
|
||||
|
||||
assert analysis_rows[2]["score"] is None
|
||||
assert analysis_rows[2]["outcome_exclusion"] == "strict_targets_missing_or_error"
|
||||
assert analysis_rows[3]["score"] is None
|
||||
assert summarize(analysis_rows, 4) == {
|
||||
"planned": 4,
|
||||
"terminal_rows": 4,
|
||||
"observed": 2,
|
||||
"missing": 2,
|
||||
"successful": 1,
|
||||
"observed_rate": 0.5,
|
||||
"missing_as_failure_rate": 0.25,
|
||||
"missing_as_success_rate": 0.75,
|
||||
}
|
||||
paired = paired_analysis(analysis_rows)
|
||||
assert paired["team_effects"] == [
|
||||
{"team": 1, "complete_pairs": 1, "board_minus_control": -1.0}
|
||||
]
|
||||
Reference in new issue
Block a user