refactor(runner): emit onset-only violations to trace and summary

Switch the per-step violation list from the sticky verdict map to the
verifier's onset set. Each property now appears exactly once across a
run: at the step it first violates, not on every subsequent step where
the residual stays false. Removes the dead violationNames helper.
This commit is contained in:
pj committed 2026-05-30 16:32:17 +05:30
1 parent b9fa41553f
commit 60c4ef7458
2 files changed
+74 -12

No files matched your search

+2 -12
View File
@@ -165,8 +165,8 @@ func Run(ctx context.Context, options Options) (Summary, error) {
screen = tree.Elements[0].Screen screen = tree.Elements[0].Screen
} }
logger.Info("step", "index", stepIndex, "screen", screen, "nodes", treeSize) logger.Info("step", "index", stepIndex, "screen", screen, "nodes", treeSize)
verdicts := options.Verifier.EvaluateProperties() options.Verifier.EvaluateProperties()
violations := violationNames(verdicts) violations := options.Verifier.NewlyViolatedProperties()
for _, name := range violations { for _, name := range violations {
if predicateErr := options.Verifier.PredicateError(name); predicateErr != nil { if predicateErr := options.Verifier.PredicateError(name); predicateErr != nil {
logger.Warn("predicate error", "step", stepIndex, "property", name, "err", predicateErr) logger.Warn("predicate error", "step", stepIndex, "property", name, "err", predicateErr)
@@ -266,16 +266,6 @@ func validate(options Options) error {
return nil return nil
} }
func violationNames(verdicts map[string]ltl.Verdict) []string {
var names []string
for name, verdict := range verdicts {
if verdict == ltl.VerdictViolated {
names = append(names, name)
}
}
return names
}
// ensureForeground keeps the app under test in the foreground. When the driver // ensureForeground keeps the app under test in the foreground. When the driver
// can report the foreground app and it no longer matches the bundle under test, // can report the foreground app and it no longer matches the bundle under test,
// the app is relaunched. Returns true when a relaunch happened so the caller // the app is relaunched. Returns true when a relaunch happened so the caller
+72
View File
@@ -1,8 +1,10 @@
package runner package runner
import ( import (
"bufio"
"bytes" "bytes"
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"log/slog" "log/slog"
@@ -118,6 +120,76 @@ func TestRunner_ViolationSurfacesInSummary(t *testing.T) {
} }
} }
func TestRunner_ViolationSurfacesOnlyOnOnsetStep(t *testing.T) {
// violationSpec uses always(() => false): onset fires on step 1 and the
// residual stays violated forever. The runner must record the violation
// exactly once (at the onset step) in both summary.Violations and trace
// lines, not on every subsequent step.
state := newHarnessWithSpec(t, violationSpec)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
summary, err := Run(ctx, Options{
Duration: 200 * time.Millisecond,
IdleTimeout: 20 * time.Millisecond,
Driver: state.mock,
Verifier: state.verifier,
TraceWriter: state.writer,
})
if err != nil {
t.Fatalf("Run: %v", err)
}
if summary.Steps < 2 {
t.Fatalf("need at least 2 steps to prove onset-only behavior, got %d", summary.Steps)
}
if len(summary.Violations) != 1 {
t.Fatalf("expected exactly one ViolationRecord (onset only), got %d: %v",
len(summary.Violations), summary.Violations)
}
if summary.Violations[0].StepIndex != 1 {
t.Errorf("onset step: got %d, want 1 (always(()=>false) violates immediately)",
summary.Violations[0].StepIndex)
}
if !slices.Equal(summary.Violations[0].Properties, []string{"balanceNonNegative"}) {
t.Errorf("onset properties: got %v, want [balanceNonNegative]",
summary.Violations[0].Properties)
}
file, err := os.Open(filepath.Join(state.writer.Directory(), "trace.jsonl"))
if err != nil {
t.Fatal(err)
}
defer file.Close()
type traceLine struct {
Step int `json:"step"`
Violations []string `json:"violations"`
}
linesWithViolations := 0
scanner := bufio.NewScanner(file)
scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
for scanner.Scan() {
var line traceLine
if err := json.Unmarshal(scanner.Bytes(), &line); err != nil {
t.Fatalf("trace line decode: %v", err)
}
if len(line.Violations) == 0 {
continue
}
linesWithViolations++
if line.Step != 1 {
t.Errorf("step %d unexpectedly emitted violations %v (should be onset-only at step 1)",
line.Step, line.Violations)
}
}
if err := scanner.Err(); err != nil {
t.Fatalf("scan trace: %v", err)
}
if linesWithViolations != 1 {
t.Errorf("expected exactly 1 trace line with violations, got %d", linesWithViolations)
}
}
func TestRunner_ThrowingPredicateIsLoggedNotPanic(t *testing.T) { func TestRunner_ThrowingPredicateIsLoggedNotPanic(t *testing.T) {
const throwingSpec = ` const throwingSpec = `
globalThis.properties = { globalThis.properties = {