mirror of
https://github.com/priyanshujain/messageboardbench.git
synced 2026-10-02 19:17:07 +00:00
97 lines
4.2 KiB
Python
97 lines
4.2 KiB
Python
"""Pure deterministic statistics shared by SWE reporting and verification."""
|
|
from __future__ import annotations
|
|
|
|
import itertools
|
|
import math
|
|
from typing import Mapping
|
|
|
|
|
|
def binary_score(row):
|
|
score = row.get("score")
|
|
return 1 if score in (1, 1.0, "C") else 0 if score is not None else None
|
|
|
|
|
|
def strict_analysis_rows(rows, artifacts_by_episode: Mapping[str, dict]):
|
|
"""Return copies whose scores are missing when strict targets are unavailable."""
|
|
result = []
|
|
for row in rows:
|
|
copied = dict(row)
|
|
statuses = artifacts_by_episode.get(row["episode_id"], {}).get(
|
|
"strict_target_statuses"
|
|
)
|
|
invalid = (
|
|
not isinstance(statuses, dict)
|
|
or not statuses
|
|
or any(value in {"MISSING", "ERROR"} for value in statuses.values())
|
|
)
|
|
if invalid:
|
|
copied["score"] = None
|
|
copied["outcome_exclusion"] = (
|
|
"strict_targets_missing_or_error"
|
|
if isinstance(statuses, dict) and statuses
|
|
else "strict_targets_unavailable"
|
|
)
|
|
result.append(copied)
|
|
return result
|
|
|
|
|
|
def summarize(rows, planned):
|
|
observed = [binary_score(row) for row in rows if binary_score(row) is not None]
|
|
missing = planned - len(observed)
|
|
return {
|
|
"planned": planned,
|
|
"terminal_rows": len(rows),
|
|
"observed": len(observed),
|
|
"missing": missing,
|
|
"successful": sum(observed),
|
|
"observed_rate": sum(observed) / len(observed) if observed else None,
|
|
"missing_as_failure_rate": sum(observed) / planned if planned else None,
|
|
"missing_as_success_rate": (
|
|
(sum(observed) + missing) / planned if planned else None
|
|
),
|
|
}
|
|
|
|
|
|
def paired_analysis(rows):
|
|
by_key = {(row["team"], row["task_id"], row["condition"]): row for row in rows}
|
|
teams = sorted({row["team"] for row in rows})
|
|
effects = []
|
|
discordant = {"board_only": 0, "control_only": 0}
|
|
for team in teams:
|
|
ids = sorted({row["task_id"] for row in rows if row["team"] == team})
|
|
pairs = [tuple(
|
|
binary_score(by_key[(team, task, arm)])
|
|
if (team, task, arm) in by_key else None
|
|
for arm in ("control", "board")
|
|
) for task in ids]
|
|
complete = [(control, board) for control, board in pairs
|
|
if control is not None and board is not None]
|
|
effects.append({
|
|
"team": team, "complete_pairs": len(complete),
|
|
"board_minus_control": (sum(board - control for control, board in complete) / len(complete)
|
|
if complete else None),
|
|
})
|
|
discordant["board_only"] += sum(control == 0 and board == 1 for control, board in complete)
|
|
discordant["control_only"] += sum(control == 1 and board == 0 for control, board in complete)
|
|
values = [row["board_minus_control"] for row in effects if row["board_minus_control"] is not None]
|
|
weights = [row["complete_pairs"] for row in effects if row["board_minus_control"] is not None]
|
|
observed_signed = (sum(value * weight for value, weight in zip(values, weights)) / sum(weights)
|
|
if weights else None)
|
|
observed = abs(observed_signed) if observed_signed is not None else None
|
|
sign_flip = None
|
|
if values and len(values) <= 20:
|
|
statistics = [abs(sum(sign * value * weight for sign, value, weight in zip(signs, values, weights)) / sum(weights))
|
|
for signs in itertools.product((-1, 1), repeat=len(values))]
|
|
sign_flip = sum(value >= observed - 1e-15 for value in statistics) / len(statistics)
|
|
discordant_total = sum(discordant.values())
|
|
mcnemar = None
|
|
if discordant_total:
|
|
low = min(discordant.values())
|
|
mcnemar = min(1.0, 2 * sum(math.comb(discordant_total, k)
|
|
for k in range(low + 1)) / (2 ** discordant_total))
|
|
return {"team_effects": effects,
|
|
"task_count_weighted_team_board_minus_control": observed_signed,
|
|
"exact_team_sign_flip_p_two_sided": sign_flip,
|
|
"task_pair_discordance": discordant,
|
|
"descriptive_task_level_mcnemar_p_two_sided_not_cluster_valid": mcnemar}
|