From 749a173020ffd3b05c1bcdaa30abddaa30f522a8 Mon Sep 17 00:00:00 2001
From: PJ <ipriyanshujain@gmail.com>
Date: Sat, 22 Aug 2026 21:12:27 +0530
Subject: [PATCH] fix(confusion-matrix): the same exclusion, so silence is not
 scored as agreement

A run that spent its budget failing preconditions judged some other app than
the one under test, and its quiet checker was being scored as a true negative
against the reviewer.
---
 .../confusion-matrix/checker.go               | 26 ++++++++++++++-----
 .../confusion-matrix/fixtures_test.go         |  8 +++++-
 .../confusion-matrix/matrix_test.go           |  9 +++++++
 3 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/cmd/internal-tools/confusion-matrix/checker.go b/cmd/internal-tools/confusion-matrix/checker.go
index 7ca9a5e..1d62193 100644
--- a/cmd/internal-tools/confusion-matrix/checker.go
+++ b/cmd/internal-tools/confusion-matrix/checker.go
@@ -30,6 +30,11 @@ const (
 	reasonTimedOut    = "timed out"
 	reasonNonzeroExit = "nonzero exit"
 	reasonTraceError  = "unreadable trace"
+	// reasonPreconditionFailures is the run that exited cleanly having spent its
+	// budget failing preconditions, so its properties judged some other app than
+	// the one under test. Scoring the silence as a checker that fired on nothing
+	// reads a run that never explored as agreement with the reviewer.
+	reasonPreconditionFailures = "precondition failures"
 )
 
 type sweepManifest struct {
@@ -54,13 +59,18 @@ type sweepImplementationRecord struct {
 }
 
 type campaignRunRecord struct {
-	Seed               int64    `json:"seed"`
-	ExitCode           int      `json:"exit_code"`
-	LaunchError        string   `json:"launch_error"`
-	TimedOut           bool     `json:"timed_out"`
-	TraceError         string   `json:"trace_error"`
-	RunDirectory       string   `json:"run_directory"`
-	ViolatedProperties []string `json:"violated_properties"`
+	Seed        int64  `json:"seed"`
+	ExitCode    int    `json:"exit_code"`
+	LaunchError string `json:"launch_error"`
+	TimedOut    bool   `json:"timed_out"`
+	TraceError  string `json:"trace_error"`
+	Steps       int    `json:"steps"`
+	// PreconditionFailures is how many of the run's steps never had the app under
+	// test in front of them. The campaign omits the field when it is zero, so
+	// absence and zero mean the same thing here.
+	PreconditionFailures int      `json:"precondition_failures"`
+	RunDirectory         string   `json:"run_directory"`
+	ViolatedProperties   []string `json:"violated_properties"`
 }
 
 // checkerVerdict is one implementation's whole checker side, pooled across the
@@ -280,6 +290,8 @@ func excludedBecause(record campaignRunRecord) string {
 		return reasonNonzeroExit
 	case record.TraceError != "":
 		return reasonTraceError
+	case record.PreconditionFailures*2 > record.Steps:
+		return reasonPreconditionFailures
 	default:
 		return ""
 	}
diff --git a/cmd/internal-tools/confusion-matrix/fixtures_test.go b/cmd/internal-tools/confusion-matrix/fixtures_test.go
index 76e8e11..faeca2f 100644
--- a/cmd/internal-tools/confusion-matrix/fixtures_test.go
+++ b/cmd/internal-tools/confusion-matrix/fixtures_test.go
@@ -21,7 +21,10 @@ type fixtureRun struct {
 	// the run inside that campaign.
 	CampaignExitCode int
 	TimedOut         bool
-	Violated         []string
+	// PreconditionFailures is how many of the run's 400 steps never had the app
+	// under test in front of them.
+	PreconditionFailures int
+	Violated             []string
 	// Surfaces is the locatableSurfaces reading the trace records. A nil map
 	// with NoTrace false still writes a reading of every surface false.
 	Surfaces map[string]bool
@@ -136,6 +139,9 @@ func writeImplementation(t *testing.T, built fixture, implementation fixtureImpl
 		if run.TimedOut {
 			campaignRun["timed_out"] = true
 		}
+		if run.PreconditionFailures > 0 {
+			campaignRun["precondition_failures"] = run.PreconditionFailures
+		}
 		if len(run.Violated) > 0 {
 			campaignRun["violated_properties"] = run.Violated
 		}
diff --git a/cmd/internal-tools/confusion-matrix/matrix_test.go b/cmd/internal-tools/confusion-matrix/matrix_test.go
index fae3852..f3530c9 100644
--- a/cmd/internal-tools/confusion-matrix/matrix_test.go
+++ b/cmd/internal-tools/confusion-matrix/matrix_test.go
@@ -65,6 +65,15 @@ func TestCrossTabulateScoresEachImplementationIntoOneCell(t *testing.T) {
 			},
 			wantExcluded: missingNoUsableRun,
 		},
+		{
+			name: "every run spent its budget outside the app under test",
+			implementation: fixtureImplementation{
+				Name: "impl-10", Model: "Opus 5",
+				Runs:   []fixtureRun{{Seed: 1, PreconditionFailures: 380}},
+				Review: &fixtureReview{Overall: overallNotDefective},
+			},
+			wantExcluded: missingNoUsableRun,
+		},
 		{
 			name: "no verdict was filed",
 			implementation: fixtureImplementation{