fix(confusion-matrix): the same exclusion, so silence is not scored as agreement

A run that spent its budget failing preconditions judged some other app than
the one under test, and its quiet checker was being scored as a true negative
against the reviewer.

cmd/internal-tools/confusion-matrix/checker.go

@@ -30,6 +30,11 @@ const (
 	reasonTimedOut    = "timed out"
 	reasonNonzeroExit = "nonzero exit"
 	reasonTraceError  = "unreadable trace"
+	// reasonPreconditionFailures is the run that exited cleanly having spent its
+	// budget failing preconditions, so its properties judged some other app than
+	// the one under test. Scoring the silence as a checker that fired on nothing
+	// reads a run that never explored as agreement with the reviewer.
+	reasonPreconditionFailures = "precondition failures"
 )
 
 type sweepManifest struct {
@@ -59,6 +64,11 @@ type campaignRunRecord struct {
 	LaunchError string `json:"launch_error"`
 	TimedOut    bool   `json:"timed_out"`
 	TraceError  string `json:"trace_error"`
+	Steps       int    `json:"steps"`
+	// PreconditionFailures is how many of the run's steps never had the app under
+	// test in front of them. The campaign omits the field when it is zero, so
+	// absence and zero mean the same thing here.
+	PreconditionFailures int      `json:"precondition_failures"`
 	RunDirectory         string   `json:"run_directory"`
 	ViolatedProperties   []string `json:"violated_properties"`
 }
@@ -280,6 +290,8 @@ func excludedBecause(record campaignRunRecord) string {
 		return reasonNonzeroExit
 	case record.TraceError != "":
 		return reasonTraceError
+	case record.PreconditionFailures*2 > record.Steps:
+		return reasonPreconditionFailures
 	default:
 		return ""
 	}

cmd/internal-tools/confusion-matrix/fixtures_test.go

@@ -21,6 +21,9 @@ type fixtureRun struct {
 	// the run inside that campaign.
 	CampaignExitCode int
 	TimedOut         bool
+	// PreconditionFailures is how many of the run's 400 steps never had the app
+	// under test in front of them.
+	PreconditionFailures int
 	Violated             []string
 	// Surfaces is the locatableSurfaces reading the trace records. A nil map
 	// with NoTrace false still writes a reading of every surface false.
@@ -136,6 +139,9 @@ func writeImplementation(t *testing.T, built fixture, implementation fixtureImpl
 		if run.TimedOut {
 			campaignRun["timed_out"] = true
 		}
+		if run.PreconditionFailures > 0 {
+			campaignRun["precondition_failures"] = run.PreconditionFailures
+		}
 		if len(run.Violated) > 0 {
 			campaignRun["violated_properties"] = run.Violated
 		}

cmd/internal-tools/confusion-matrix/matrix_test.go

@@ -65,6 +65,15 @@ func TestCrossTabulateScoresEachImplementationIntoOneCell(t *testing.T) {
 			},
 			wantExcluded: missingNoUsableRun,
 		},
+		{
+			name: "every run spent its budget outside the app under test",
+			implementation: fixtureImplementation{
+				Name: "impl-10", Model: "Opus 5",
+				Runs:   []fixtureRun{{Seed: 1, PreconditionFailures: 380}},
+				Review: &fixtureReview{Overall: overallNotDefective},
+			},
+			wantExcluded: missingNoUsableRun,
+		},
 		{
 			name: "no verdict was filed",
 			implementation: fixtureImplementation{