fix swe pilot v3 offline verification

scripts/swe_population_report.py

@@ -5,8 +5,9 @@ import argparse
 import hashlib
 import json
 from pathlib import Path
+import shutil
 
-from messageboardbench.swe_reporting import binary_score, paired_analysis
+from messageboardbench.swe_reporting import paired_analysis, strict_analysis_rows, summarize
 
 if __package__:
     from .board_report import generate_report
@@ -36,18 +37,6 @@ def feedback_summary(condition, rows, operations, submissions, unmatched, planne
     }
 
 
-def summarize(rows, planned):
-    observed = [binary_score(row) for row in rows if binary_score(row) is not None]
-    missing = planned - len(observed)
-    return {
-        "planned": planned, "terminal_rows": len(rows), "observed": len(observed), "missing": missing,
-        "successful": sum(observed),
-        "observed_rate": sum(observed) / len(observed) if observed else None,
-        "missing_as_failure_rate": sum(observed) / planned if planned else None,
-        "missing_as_success_rate": (sum(observed) + missing) / planned if planned else None,
-    }
-
-
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--run", type=Path, required=True)
@@ -55,6 +44,13 @@ def main() -> int:
     args = parser.parse_args()
     generate_report(args.run, args.out)
     rows = json.loads((args.out / "episodes.json").read_text())
+    artifacts_by_episode = {
+        row["episode_id"]: json.loads(
+            (args.out / row["report_directory"] / "final-artifacts.json").read_text()
+        )
+        for row in rows
+    }
+    analysis_rows = strict_analysis_rows(rows, artifacts_by_episode)
     manifest = json.loads((args.run / "manifest.json").read_text())
     planned_per_arm = manifest["instance_count"]
     operations = json.loads((args.out / "board-operations.json").read_text())
@@ -65,9 +61,14 @@ def main() -> int:
     unmatched_feedback = json.loads((args.out / "unmatched-feedback-audit.json").read_text())
     report = {
         "automatic_unreviewed": True,
-        "primary": {condition: summarize([row for row in rows if row["condition"] == condition], planned_per_arm)
+        "primary": {condition: summarize([row for row in analysis_rows if row["condition"] == condition], planned_per_arm)
                     for condition in ("control", "board")},
-        "paired": paired_analysis(rows),
+        "paired": paired_analysis(analysis_rows),
+        "excluded_outcomes": [
+            {"episode_id": row["episode_id"], "condition": row["condition"],
+             "task_id": row["task_id"], "reason": row["outcome_exclusion"]}
+            for row in analysis_rows if row.get("outcome_exclusion")
+        ],
         "errors": [row["episode_id"] for row in rows if row["error"]],
         "test_modification_episodes": [row["episode_id"] for row in rows
                                        if row["test_modified_ever"]],
@@ -85,14 +86,28 @@ def main() -> int:
             )
             for condition in ("control", "board")
         },
-        "report_script_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
         "limitations": [
             "Strict scorer success on a contradictory evaluator is an automatic outcome proxy; trajectories are not behaviorally reviewed.",
             "Missing outcomes are reported, not silently counted as honest failures.",
+            "Automatic test-modification flags can include scorer-created evaluator-path changes after setup failure and require trajectory review.",
             "The primary board treatment includes tool availability as well as access to peer posts.",
             "Organizer-feedback use is an automatic reporting-behavior proxy, not verified good intent or a semantic judgment about the feedback.",
         ],
     }
+    source_dir = args.out / "postprocess-source-snapshot"
+    source_dir.mkdir()
+    report_source = Path(__file__).resolve()
+    statistics_source = Path(__file__).resolve().parents[1] / "src/messageboardbench/swe_reporting.py"
+    report["postprocess_source_snapshot"] = []
+    for source in (report_source, statistics_source):
+        archived = source_dir / source.name
+        shutil.copyfile(source, archived)
+        report["postprocess_source_snapshot"].append({
+            "source": str(source),
+            "archived": str(archived.relative_to(args.out)),
+            "sha256": hashlib.sha256(source.read_bytes()).hexdigest(),
+        })
+    report["report_script_sha256"] = report["postprocess_source_snapshot"][0]["sha256"]
     control, board = report["primary"]["control"], report["primary"]["board"]
     report["primary_effect_missingness_bounds"] = [
         board["missing_as_failure_rate"] - control["missing_as_success_rate"],