mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 12:07:09 +00:00
fix(confusion-matrix): a campaign that died is missing data, not a true negative
The sweep-level loop excluded a run on launch_error alone, while excludedBecause already checked the campaign process's exit code. An interrupted campaign wrote exit_code -1 with an empty launch_error, so its one completed seed scored the implementation as a clean cell on a tenth of the planned evidence. The fixture builder wrote one exit code into both the sweep record and the campaign run record, which is why no test could tell the two levels apart.
This commit is contained in:
1 parent
1d684a98eb
commit
ff6c66a74b
3 files changed
+55
-5
No files matched your search
@@ -170,8 +170,8 @@ func readImplementation(sweepDirectory string, record sweepImplementationRecord)
|
||||
fired := map[string]bool{}
|
||||
for _, run := range record.Runs {
|
||||
verdict.RunsRecorded++
|
||||
if run.LaunchError != "" {
|
||||
verdict.ExcludedByReason[reasonLaunchError]++
|
||||
if reason := sweepRunExcludedBecause(run); reason != "" {
|
||||
verdict.ExcludedByReason[reason]++
|
||||
continue
|
||||
}
|
||||
directory := resolveCampaignDirectory(sweepDirectory, record.Name, run)
|
||||
@@ -254,6 +254,22 @@ func readCampaignRuns(directory string) ([]campaignRunRecord, error) {
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// sweepRunExcludedBecause reads the outcome of the campaign process itself,
|
||||
// which the records inside its directory cannot report. One sweep run is one
|
||||
// campaign of one seed, so a campaign that died left a runs.jsonl that is
|
||||
// partial or empty, and scoring the runs it did write reads the seeds it never
|
||||
// reached as agreement.
|
||||
func sweepRunExcludedBecause(record sweepRunRecord) string {
|
||||
switch {
|
||||
case record.LaunchError != "":
|
||||
return reasonLaunchError
|
||||
case record.ExitCode != 0:
|
||||
return reasonNonzeroExit
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func excludedBecause(record campaignRunRecord) string {
|
||||
switch {
|
||||
case record.LaunchError != "":
|
||||
|
||||
Reference in new issue
Block a user