From 3e4633f588286c8dd0ef4e6c84edfe820952c9bb Mon Sep 17 00:00:00 2001 From: PJ Date: Sat, 22 Aug 2026 21:12:27 +0530 Subject: [PATCH] fix(analyze): a run that never explored is excluded, not counted as evidence The campaign has written precondition_failures since it learned to count them and neither reader declared it, so a run whose scope guard could not bring the app back entered the survival analysis with its whole budget as exposure the app survived. A strict majority of steps is the line: below it the failures were transient and the run still explored. --- cmd/internal-tools/analyze/load.go | 26 +++++++++++--- cmd/internal-tools/analyze/load_test.go | 46 +++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/cmd/internal-tools/analyze/load.go b/cmd/internal-tools/analyze/load.go index 2bb8796..965920f 100644 --- a/cmd/internal-tools/analyze/load.go +++ b/cmd/internal-tools/analyze/load.go @@ -65,17 +65,30 @@ type runRecord struct { // field, and reading that silence as none would let a denominator of unknown // provenance pass for one that excludes the setup's login. UnattributedActions *int `json:"unattributed_actions"` + // PreconditionFailures is how many of the run's steps never had the app under + // test in front of them: the startup gate's verdict and every later step the + // scope guard could not bring the app back for. The campaign omits the field + // when it is zero, so absence and zero mean the same thing here. + PreconditionFailures int `json:"precondition_failures"` } // Exclusion reasons. A run that failed or timed out is missing data, not a // censored observation: it broke off, so its step count is not exposure the app // survived and counting it as one would bias the survival estimate downward. const ( - reasonLaunchError = "launch error" - reasonTimedOut = "timed out" - reasonNonzeroExit = "nonzero exit" - reasonTraceError = "unreadable trace" - reasonMalformedStep = "violation step outside the budget" + reasonLaunchError = "launch error" + reasonTimedOut = "timed out" + reasonNonzeroExit = "nonzero exit" + reasonTraceError = "unreadable trace" + // reasonPreconditionFailures is the run that exited cleanly having spent its + // budget failing preconditions. The scope guard records one on every step it + // could not bring the app back for and lets the run finish, so nothing else + // here separates it from a run that explored the whole budget and found + // nothing. A majority is the line: below it the failures are transient and + // the run still explored, above it the step count the analysis would censor + // at is mostly steps the app was never there for. + reasonPreconditionFailures = "precondition failures" + reasonMalformedStep = "violation step outside the budget" ) type classifiedRun struct { @@ -201,6 +214,9 @@ func classify(record runRecord, budget int) classifiedRun { case record.TraceError != "": item.ExcludedBecause = reasonTraceError return item + case record.PreconditionFailures*2 > record.Steps: + item.ExcludedBecause = reasonPreconditionFailures + return item } if record.FirstViolationOriginStep == nil { if len(record.ViolatedProperties) > 0 { diff --git a/cmd/internal-tools/analyze/load_test.go b/cmd/internal-tools/analyze/load_test.go index f1ea053..14aa960 100644 --- a/cmd/internal-tools/analyze/load_test.go +++ b/cmd/internal-tools/analyze/load_test.go @@ -20,6 +20,8 @@ func TestClassify_FailedAndTimedOutRunsAreMissingDataNotCensored(t *testing.T) { {"timed out", runRecord{TimedOut: true, ExitCode: -1}, reasonTimedOut}, {"nonzero exit", runRecord{ExitCode: 3}, reasonNonzeroExit}, {"unreadable trace", runRecord{TraceError: "no run directory with meta.json"}, reasonTraceError}, + {"never started", runRecord{PreconditionFailures: 1}, reasonPreconditionFailures}, + {"most steps never saw the app", runRecord{Steps: 40, PreconditionFailures: 21}, reasonPreconditionFailures}, {"violation at step zero", runRecord{FirstViolationOriginStep: stepPointer(0)}, reasonMalformedStep}, {"violation without a step", runRecord{ViolatedProperties: []string{"cartTotal"}}, reasonMalformedStep}, } @@ -148,6 +150,50 @@ func TestGroupArms_PoolsDirectoriesSharingAnArmAndReportsMissingSeeds(t *testing } } +// The scope guard records a precondition failure on every step it could not +// bring the app back for and still lets the run exit cleanly, so a run that +// spent its budget outside the app under test arrives at the analysis looking +// like one that explored the whole budget and found nothing. +func TestGroupArms_ExcludesRunsThatSpentTheBudgetFailingPreconditions(t *testing.T) { + root := t.TempDir() + directory := filepath.Join(root, "escaped") + writeCampaign(t, directory, map[string]any{"arm": "seeded", "max_steps": 40, "seeds": []int{1, 2, 3}}, + []map[string]any{ + {"seed": 1, "exit_code": 0, "steps": 40, "actions": 40, "precondition_failures": 36}, + {"seed": 2, "exit_code": 0, "steps": 40, "actions": 40, "precondition_failures": 20}, + {"seed": 3, "exit_code": 0, "steps": 40, "actions": 40}, + }) + + arms, err := groupArms([]string{directory}) + if err != nil { + t.Fatal(err) + } + if len(arms) != 1 { + t.Fatalf("%d arms, want 1", len(arms)) + } + excludedBySeed := map[int64]string{} + for _, item := range arms[0].Runs { + excludedBySeed[item.Seed] = item.ExcludedBecause + } + if excludedBySeed[1] != reasonPreconditionFailures { + t.Errorf("the run that spent its budget failing preconditions is excluded as %q, want %q", + excludedBySeed[1], reasonPreconditionFailures) + } + if excludedBySeed[2] != "" || excludedBySeed[3] != "" { + t.Errorf("runs that explored are excluded as %q and %q, want both kept", + excludedBySeed[2], excludedBySeed[3]) + } + observations := arms[0].observations() + if len(observations) != 2 { + t.Fatalf("%d observations, want only the 2 runs that explored", len(observations)) + } + for _, item := range observations { + if item.Event || item.Steps != 40 { + t.Errorf("observation %+v, want a censored observation at 40", item) + } + } +} + func TestGroupArms_RejectsDisagreeingStepBudgets(t *testing.T) { root := t.TempDir() writeCampaign(t, filepath.Join(root, "a"), map[string]any{"arm": "seeded", "max_steps": 40, "seeds": []int{1}},