mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
fix(confusion-matrix): the same exclusion, so silence is not scored as agreement
A run that spent its budget failing preconditions judged some other app than the one under test, and its quiet checker was being scored as a true negative against the reviewer.
This commit is contained in:
1 parent
3e4633f588
commit
749a173020
3 files changed
+35
-8
No files matched your search
@@ -21,7 +21,10 @@ type fixtureRun struct {
|
||||
// the run inside that campaign.
|
||||
CampaignExitCode int
|
||||
TimedOut bool
|
||||
Violated []string
|
||||
// PreconditionFailures is how many of the run's 400 steps never had the app
|
||||
// under test in front of them.
|
||||
PreconditionFailures int
|
||||
Violated []string
|
||||
// Surfaces is the locatableSurfaces reading the trace records. A nil map
|
||||
// with NoTrace false still writes a reading of every surface false.
|
||||
Surfaces map[string]bool
|
||||
@@ -136,6 +139,9 @@ func writeImplementation(t *testing.T, built fixture, implementation fixtureImpl
|
||||
if run.TimedOut {
|
||||
campaignRun["timed_out"] = true
|
||||
}
|
||||
if run.PreconditionFailures > 0 {
|
||||
campaignRun["precondition_failures"] = run.PreconditionFailures
|
||||
}
|
||||
if len(run.Violated) > 0 {
|
||||
campaignRun["violated_properties"] = run.Violated
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user