feat(campaign): count the runs that were never in the app

A run that failed its precondition has zero steps and no violations, which is
what a short clean run looks like too. The summary now counts the trace records
naming an unmet precondition, so a campaign directory answers "how many of
these were never in the app" without grepping any log.
This commit is contained in:
pj committed 2026-08-17 14:35:37 +05:30
1 parent ca8b517d98
commit ddf17129bd
2 files changed
+56

No files matched your search

+10
View File
@@ -32,6 +32,12 @@ type traceSummary struct {
FirstViolationReason string `json:"first_violation_reason,omitempty"` FirstViolationReason string `json:"first_violation_reason,omitempty"`
FirstViolationIsError bool `json:"first_violation_is_error,omitempty"` FirstViolationIsError bool `json:"first_violation_is_error,omitempty"`
ViolatedProperties []string `json:"violated_properties,omitempty"` ViolatedProperties []string `json:"violated_properties,omitempty"`
// PreconditionFailures counts the trace records naming a precondition the
// run could not meet: the startup gate's verdict at step 0, and every later
// step the scope guard could not bring the app back for. A run with one of
// these and no steps never started, and counting it as a run that explored
// and found nothing puts a harness failure in the same column as evidence.
PreconditionFailures int `json:"precondition_failures,omitempty"`
} }
type traceLine struct { type traceLine struct {
@@ -43,6 +49,7 @@ type traceLine struct {
// carries none at all. // carries none at all.
NextAction json.RawMessage `json:"next_action"` NextAction json.RawMessage `json:"next_action"`
ActionSkipped string `json:"action_skipped"` ActionSkipped string `json:"action_skipped"`
PreconditionFailure string `json:"precondition_failure"`
Violations []string `json:"violations"` Violations []string `json:"violations"`
Witnesses map[string]trace.Witness `json:"witnesses"` Witnesses map[string]trace.Witness `json:"witnesses"`
} }
@@ -123,6 +130,9 @@ func summarizeTrace(tracePath string) (traceSummary, error) {
if !synthetic && dispatchedAction(line) { if !synthetic && dispatchedAction(line) {
summary.Actions++ summary.Actions++
} }
if line.PreconditionFailure != "" {
summary.PreconditionFailures++
}
for _, property := range line.Violations { for _, property := range line.Violations {
violated[property] = true violated[property] = true
recordViolation(&summary, line, property) recordViolation(&summary, line, property)
@@ -292,3 +292,49 @@ func TestSummarizeTrace_MalformedLine(t *testing.T) {
t.Fatal("expected an error for a malformed trace line") t.Fatal("expected an error for a malformed trace line")
} }
} }
// A run whose app never came to the foreground has to be countable off the
// summary. Without it the campaign row for a run that never started is a row of
// zero steps and no violations, which is what a clean short run looks like too.
func TestSummarizeRun_CountsPreconditionFailures(t *testing.T) {
seedDirectory := t.TempDir()
writeRunDirectory(t, seedDirectory, "20260812-090000", []trace.Step{
{Index: 0, PreconditionFailure: "app_not_in_foreground"},
})
_, summary, err := summarizeRun(seedDirectory)
if err != nil {
t.Fatal(err)
}
if summary.PreconditionFailures != 1 {
t.Errorf("precondition failures: got %d, want 1", summary.PreconditionFailures)
}
if summary.Steps != 0 {
t.Errorf("steps: got %d, want 0; the run never observed anything", summary.Steps)
}
}
// The same fact mid-run: steps the scope guard could not bring the app back for
// are still steps, and they are counted separately from the ones that explored.
func TestSummarizeRun_CountsMidRunPreconditionFailures(t *testing.T) {
seedDirectory := t.TempDir()
outsideApp := func(index int) trace.Step {
step := actingStep(index)
step.PreconditionFailure = "app_not_in_foreground"
return step
}
writeRunDirectory(t, seedDirectory, "20260812-090000", []trace.Step{
actingStep(1), outsideApp(2), outsideApp(3), actingStep(4),
})
_, summary, err := summarizeRun(seedDirectory)
if err != nil {
t.Fatal(err)
}
if summary.PreconditionFailures != 2 {
t.Errorf("precondition failures: got %d, want 2", summary.PreconditionFailures)
}
if summary.Steps != 4 {
t.Errorf("steps: got %d, want 4", summary.Steps)
}
}