From 39289f81df732025cc9262f45eaf8307be0eee1e Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 13 Aug 2026 00:33:57 +0530 Subject: [PATCH] feat(runner): stop the step loop at the first violation on request --- internal/runner/runner.go | 11 ++++ internal/runner/runner_test.go | 96 ++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index de98cb6..4eeb080 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -31,6 +31,11 @@ type Options struct { // positive value stops the loop once that many steps have run. MaxSteps int + // StopOnViolation ends the step loop as soon as a step records a + // violation, so a run that exists to find one bug stops at the evidence + // instead of spending the rest of its budget past it. + StopOnViolation bool + BundleID string Driver driver.DeviceDriver Verifier *verifier.Verifier @@ -296,6 +301,12 @@ func Run(ctx context.Context, options Options) (Summary, error) { summary.Steps = stepIndex if len(violations) > 0 { summary.Violations = append(summary.Violations, violationRecords(violations, witnesses, stepIndex)...) + // The step is already written, so the trace ends on the state that + // produced the violation. Finalize below still runs, so pending + // liveness obligations are reported alongside it. + if options.StopOnViolation { + break + } } // Wait actions are themselves a settling: skip the idle poll. Actions // that mutate the UI fall through to WaitForIdle so the next step's diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 0e884f4..749289d 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -1976,3 +1976,99 @@ func TestRunner_SkipsActionWhenOverlayStealsFocusAtApplyTime(t *testing.T) { t.Error("apply-time guard failed: a tap fired while a system overlay held focus") } } + +// TestRunner_StopOnViolationEndsAtTheFirstViolation pins the gate CI runs on: +// a step budget of 8 against a spec that only violates on the third step must +// end on step 3 and write nothing after it, so the trace's last state is the +// one that produced the violation. +func TestRunner_StopOnViolationEndsAtTheFirstViolation(t *testing.T) { + const thirdStepViolationSpec = ` +import { actions, always, extract } from "@sanderling/spec"; +let observed = 0; +const tick = extract(() => ++observed); +globalThis.properties = { + staysUnderThree: always(() => tick.current < 3), +}; +globalThis.actions = actions(() => []); +` + state := newHarnessWithSpec(t, thirdStepViolationSpec) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + summary, err := Run(ctx, Options{ + Duration: time.Hour, + IdleTimeout: 20 * time.Millisecond, + MaxSteps: 8, + StopOnViolation: true, + Driver: state.mock, + Verifier: state.verifier, + TraceWriter: state.writer, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + if !containsProperty(summary.Violations, "staysUnderThree") { + t.Fatalf("expected staysUnderThree to fire, got %v", summary.Violations) + } + if summary.Steps != 3 { + t.Errorf("steps: got %d, want 3 (the run must stop at the violating step, not run the 8-step budget)", + summary.Steps) + } + for _, step := range traceStepIndices(t, state.writer.Directory()) { + if step > summary.Steps { + t.Errorf("trace kept stepping after the violation: found step %d past step %d", + step, summary.Steps) + } + } +} + +// TestRunner_WithoutStopOnViolationRunsTheWholeBudget is the other half: the +// default must stay a full-budget fuzz run, so turning the flag on is the only +// thing that shortens a run. +func TestRunner_WithoutStopOnViolationRunsTheWholeBudget(t *testing.T) { + state := newHarnessWithSpec(t, violationSpec) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + summary, err := Run(ctx, Options{ + Duration: time.Hour, + IdleTimeout: 20 * time.Millisecond, + MaxSteps: 4, + Driver: state.mock, + Verifier: state.verifier, + TraceWriter: state.writer, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + if summary.Steps != 4 { + t.Errorf("steps: got %d, want 4; a violation must not shorten a default run", summary.Steps) + } +} + +// traceStepIndices reads every step index the trace recorded, so a test can +// assert on what the run actually wrote rather than on the summary alone. +func traceStepIndices(t *testing.T, directory string) []int { + t.Helper() + file, err := os.Open(filepath.Join(directory, "trace.jsonl")) + if err != nil { + t.Fatal(err) + } + defer file.Close() + var steps []int + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + for scanner.Scan() { + var line struct { + Step int `json:"step"` + } + if err := json.Unmarshal(scanner.Bytes(), &line); err != nil { + t.Fatalf("trace line decode: %v", err) + } + steps = append(steps, line.Step) + } + if err := scanner.Err(); err != nil { + t.Fatalf("scan trace: %v", err) + } + return steps +}