mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 12:07:09 +00:00
fix(confusion-matrix): the same exclusion, so silence is not scored as agreement
A run that spent its budget failing preconditions judged some other app than the one under test, and its quiet checker was being scored as a true negative against the reviewer.
This commit is contained in:
1 parent
3e4633f588
commit
749a173020
3 files changed
+35
-8
No files matched your search
@@ -65,6 +65,15 @@ func TestCrossTabulateScoresEachImplementationIntoOneCell(t *testing.T) {
|
||||
},
|
||||
wantExcluded: missingNoUsableRun,
|
||||
},
|
||||
{
|
||||
name: "every run spent its budget outside the app under test",
|
||||
implementation: fixtureImplementation{
|
||||
Name: "impl-10", Model: "Opus 5",
|
||||
Runs: []fixtureRun{{Seed: 1, PreconditionFailures: 380}},
|
||||
Review: &fixtureReview{Overall: overallNotDefective},
|
||||
},
|
||||
wantExcluded: missingNoUsableRun,
|
||||
},
|
||||
{
|
||||
name: "no verdict was filed",
|
||||
implementation: fixtureImplementation{
|
||||
|
||||
Reference in new issue
Block a user