mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 11:07:10 +00:00
feat(campaign): count the runs that were never in the app
A run that failed its precondition has zero steps and no violations, which is what a short clean run looks like too. The summary now counts the trace records naming an unmet precondition, so a campaign directory answers "how many of these were never in the app" without grepping any log.
This commit is contained in:
1 parent
ca8b517d98
commit
ddf17129bd
2 files changed
+60
-4
No files matched your search
@@ -32,6 +32,12 @@ type traceSummary struct {
|
|||||||
FirstViolationReason string `json:"first_violation_reason,omitempty"`
|
FirstViolationReason string `json:"first_violation_reason,omitempty"`
|
||||||
FirstViolationIsError bool `json:"first_violation_is_error,omitempty"`
|
FirstViolationIsError bool `json:"first_violation_is_error,omitempty"`
|
||||||
ViolatedProperties []string `json:"violated_properties,omitempty"`
|
ViolatedProperties []string `json:"violated_properties,omitempty"`
|
||||||
|
// PreconditionFailures counts the trace records naming a precondition the
|
||||||
|
// run could not meet: the startup gate's verdict at step 0, and every later
|
||||||
|
// step the scope guard could not bring the app back for. A run with one of
|
||||||
|
// these and no steps never started, and counting it as a run that explored
|
||||||
|
// and found nothing puts a harness failure in the same column as evidence.
|
||||||
|
PreconditionFailures int `json:"precondition_failures,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type traceLine struct {
|
type traceLine struct {
|
||||||
@@ -41,10 +47,11 @@ type traceLine struct {
|
|||||||
Hierarchy json.RawMessage `json:"hierarchy"`
|
Hierarchy json.RawMessage `json:"hierarchy"`
|
||||||
// NextAction is read only for its presence: a step that chose no action
|
// NextAction is read only for its presence: a step that chose no action
|
||||||
// carries none at all.
|
// carries none at all.
|
||||||
NextAction json.RawMessage `json:"next_action"`
|
NextAction json.RawMessage `json:"next_action"`
|
||||||
ActionSkipped string `json:"action_skipped"`
|
ActionSkipped string `json:"action_skipped"`
|
||||||
Violations []string `json:"violations"`
|
PreconditionFailure string `json:"precondition_failure"`
|
||||||
Witnesses map[string]trace.Witness `json:"witnesses"`
|
Violations []string `json:"violations"`
|
||||||
|
Witnesses map[string]trace.Witness `json:"witnesses"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func dispatchedAction(line traceLine) bool {
|
func dispatchedAction(line traceLine) bool {
|
||||||
@@ -123,6 +130,9 @@ func summarizeTrace(tracePath string) (traceSummary, error) {
|
|||||||
if !synthetic && dispatchedAction(line) {
|
if !synthetic && dispatchedAction(line) {
|
||||||
summary.Actions++
|
summary.Actions++
|
||||||
}
|
}
|
||||||
|
if line.PreconditionFailure != "" {
|
||||||
|
summary.PreconditionFailures++
|
||||||
|
}
|
||||||
for _, property := range line.Violations {
|
for _, property := range line.Violations {
|
||||||
violated[property] = true
|
violated[property] = true
|
||||||
recordViolation(&summary, line, property)
|
recordViolation(&summary, line, property)
|
||||||
|
|||||||
@@ -292,3 +292,49 @@ func TestSummarizeTrace_MalformedLine(t *testing.T) {
|
|||||||
t.Fatal("expected an error for a malformed trace line")
|
t.Fatal("expected an error for a malformed trace line")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A run whose app never came to the foreground has to be countable off the
|
||||||
|
// summary. Without it the campaign row for a run that never started is a row of
|
||||||
|
// zero steps and no violations, which is what a clean short run looks like too.
|
||||||
|
func TestSummarizeRun_CountsPreconditionFailures(t *testing.T) {
|
||||||
|
seedDirectory := t.TempDir()
|
||||||
|
writeRunDirectory(t, seedDirectory, "20260812-090000", []trace.Step{
|
||||||
|
{Index: 0, PreconditionFailure: "app_not_in_foreground"},
|
||||||
|
})
|
||||||
|
|
||||||
|
_, summary, err := summarizeRun(seedDirectory)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if summary.PreconditionFailures != 1 {
|
||||||
|
t.Errorf("precondition failures: got %d, want 1", summary.PreconditionFailures)
|
||||||
|
}
|
||||||
|
if summary.Steps != 0 {
|
||||||
|
t.Errorf("steps: got %d, want 0; the run never observed anything", summary.Steps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The same fact mid-run: steps the scope guard could not bring the app back for
|
||||||
|
// are still steps, and they are counted separately from the ones that explored.
|
||||||
|
func TestSummarizeRun_CountsMidRunPreconditionFailures(t *testing.T) {
|
||||||
|
seedDirectory := t.TempDir()
|
||||||
|
outsideApp := func(index int) trace.Step {
|
||||||
|
step := actingStep(index)
|
||||||
|
step.PreconditionFailure = "app_not_in_foreground"
|
||||||
|
return step
|
||||||
|
}
|
||||||
|
writeRunDirectory(t, seedDirectory, "20260812-090000", []trace.Step{
|
||||||
|
actingStep(1), outsideApp(2), outsideApp(3), actingStep(4),
|
||||||
|
})
|
||||||
|
|
||||||
|
_, summary, err := summarizeRun(seedDirectory)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if summary.PreconditionFailures != 2 {
|
||||||
|
t.Errorf("precondition failures: got %d, want 2", summary.PreconditionFailures)
|
||||||
|
}
|
||||||
|
if summary.Steps != 4 {
|
||||||
|
t.Errorf("steps: got %d, want 4", summary.Steps)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user