mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 20:17:09 +00:00
fix(confusion-matrix): a campaign that died is missing data, not a true negative
The sweep-level loop excluded a run on launch_error alone, while excludedBecause already checked the campaign process's exit code. An interrupted campaign wrote exit_code -1 with an empty launch_error, so its one completed seed scored the implementation as a clean cell on a tenth of the planned evidence. The fixture builder wrote one exit code into both the sweep record and the campaign run record, which is why no test could tell the two levels apart.
This commit is contained in:
1 parent
1d684a98eb
commit
ff6c66a74b
3 files changed
+55
-5
No files matched your search
@@ -122,6 +122,36 @@ func TestCrossTabulateScoresEachImplementationIntoOneCell(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestACampaignThatDiedIsMissingDataNotATrueNegative covers the sweep it was
|
||||
// interrupted on: the one seed the campaign got through wrote a clean run
|
||||
// before the process died, and scoring the implementation on it reads the nine
|
||||
// seeds that never ran as agreement between the checker and the reviewer.
|
||||
func TestACampaignThatDiedIsMissingDataNotATrueNegative(t *testing.T) {
|
||||
interrupted := cleanRun(1)
|
||||
interrupted.CampaignExitCode = -1
|
||||
|
||||
emitted, stdout := runTool(t, writeFixture(t, []fixtureImplementation{{
|
||||
Name: "impl-07", Model: "Opus 5",
|
||||
Runs: []fixtureRun{interrupted},
|
||||
Review: &fixtureReview{Overall: overallNotDefective},
|
||||
}}, defaultMapping))
|
||||
|
||||
entry := exclusionFor(t, emitted, "impl-07")
|
||||
if entry.Reason != missingNoUsableRun {
|
||||
t.Fatalf("impl-07 excluded as %q, want %q", entry.Reason, missingNoUsableRun)
|
||||
}
|
||||
if !strings.Contains(entry.Detail, reasonNonzeroExit) {
|
||||
t.Errorf("exclusion detail %q does not name %q", entry.Detail, reasonNonzeroExit)
|
||||
}
|
||||
if emitted.Implementations.TrueNegative != 0 || emitted.Implementations.Scored != 0 {
|
||||
t.Errorf("implementation matrix = %+v, want no cell: a dead campaign is absent evidence",
|
||||
emitted.Implementations)
|
||||
}
|
||||
if !strings.Contains(stdout, "carry no cell") {
|
||||
t.Errorf("the report never separates the dead campaign from the matrix:\n%s", stdout)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnlocatableSurfaceIsNeitherAPositiveNorANegative pins the rule the
|
||||
// pre-registration turns on: a clause whose every covering property a
|
||||
// never-located surface left unrunnable is a portability miss. Counting it as a
|
||||
|
||||
Reference in new issue
Block a user