From 60c4ef74583c27a14d3994790e8c65a3220c1b4b Mon Sep 17 00:00:00 2001 From: PJ Date: Sat, 30 May 2026 16:32:17 +0530 Subject: [PATCH] refactor(runner): emit onset-only violations to trace and summary Switch the per-step violation list from the sticky verdict map to the verifier's onset set. Each property now appears exactly once across a run: at the step it first violates, not on every subsequent step where the residual stays false. Removes the dead violationNames helper. --- internal/runner/runner.go | 14 +------ internal/runner/runner_test.go | 72 ++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 12 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index f4f166a..de944b7 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -165,8 +165,8 @@ func Run(ctx context.Context, options Options) (Summary, error) { screen = tree.Elements[0].Screen } logger.Info("step", "index", stepIndex, "screen", screen, "nodes", treeSize) - verdicts := options.Verifier.EvaluateProperties() - violations := violationNames(verdicts) + options.Verifier.EvaluateProperties() + violations := options.Verifier.NewlyViolatedProperties() for _, name := range violations { if predicateErr := options.Verifier.PredicateError(name); predicateErr != nil { logger.Warn("predicate error", "step", stepIndex, "property", name, "err", predicateErr) @@ -266,16 +266,6 @@ func validate(options Options) error { return nil } -func violationNames(verdicts map[string]ltl.Verdict) []string { - var names []string - for name, verdict := range verdicts { - if verdict == ltl.VerdictViolated { - names = append(names, name) - } - } - return names -} - // ensureForeground keeps the app under test in the foreground. When the driver // can report the foreground app and it no longer matches the bundle under test, // the app is relaunched. Returns true when a relaunch happened so the caller diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index cf19dca..842ee11 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -1,8 +1,10 @@ package runner import ( + "bufio" "bytes" "context" + "encoding/json" "errors" "fmt" "log/slog" @@ -118,6 +120,76 @@ func TestRunner_ViolationSurfacesInSummary(t *testing.T) { } } +func TestRunner_ViolationSurfacesOnlyOnOnsetStep(t *testing.T) { + // violationSpec uses always(() => false): onset fires on step 1 and the + // residual stays violated forever. The runner must record the violation + // exactly once (at the onset step) in both summary.Violations and trace + // lines, not on every subsequent step. + state := newHarnessWithSpec(t, violationSpec) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + summary, err := Run(ctx, Options{ + Duration: 200 * time.Millisecond, + IdleTimeout: 20 * time.Millisecond, + Driver: state.mock, + Verifier: state.verifier, + TraceWriter: state.writer, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + if summary.Steps < 2 { + t.Fatalf("need at least 2 steps to prove onset-only behavior, got %d", summary.Steps) + } + if len(summary.Violations) != 1 { + t.Fatalf("expected exactly one ViolationRecord (onset only), got %d: %v", + len(summary.Violations), summary.Violations) + } + if summary.Violations[0].StepIndex != 1 { + t.Errorf("onset step: got %d, want 1 (always(()=>false) violates immediately)", + summary.Violations[0].StepIndex) + } + if !slices.Equal(summary.Violations[0].Properties, []string{"balanceNonNegative"}) { + t.Errorf("onset properties: got %v, want [balanceNonNegative]", + summary.Violations[0].Properties) + } + + file, err := os.Open(filepath.Join(state.writer.Directory(), "trace.jsonl")) + if err != nil { + t.Fatal(err) + } + defer file.Close() + + type traceLine struct { + Step int `json:"step"` + Violations []string `json:"violations"` + } + linesWithViolations := 0 + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + for scanner.Scan() { + var line traceLine + if err := json.Unmarshal(scanner.Bytes(), &line); err != nil { + t.Fatalf("trace line decode: %v", err) + } + if len(line.Violations) == 0 { + continue + } + linesWithViolations++ + if line.Step != 1 { + t.Errorf("step %d unexpectedly emitted violations %v (should be onset-only at step 1)", + line.Step, line.Violations) + } + } + if err := scanner.Err(); err != nil { + t.Fatalf("scan trace: %v", err) + } + if linesWithViolations != 1 { + t.Errorf("expected exactly 1 trace line with violations, got %d", linesWithViolations) + } +} + func TestRunner_ThrowingPredicateIsLoggedNotPanic(t *testing.T) { const throwingSpec = ` globalThis.properties = {