feat(runner): stop the step loop at the first violation on request

This commit is contained in:
pj committed 2026-08-13 00:33:57 +05:30
1 parent 1f71e052d7
commit 39289f81df
2 files changed
+107

No files matched your search

+11
View File
@@ -31,6 +31,11 @@ type Options struct {
// positive value stops the loop once that many steps have run. // positive value stops the loop once that many steps have run.
MaxSteps int MaxSteps int
// StopOnViolation ends the step loop as soon as a step records a
// violation, so a run that exists to find one bug stops at the evidence
// instead of spending the rest of its budget past it.
StopOnViolation bool
BundleID string BundleID string
Driver driver.DeviceDriver Driver driver.DeviceDriver
Verifier *verifier.Verifier Verifier *verifier.Verifier
@@ -296,6 +301,12 @@ func Run(ctx context.Context, options Options) (Summary, error) {
summary.Steps = stepIndex summary.Steps = stepIndex
if len(violations) > 0 { if len(violations) > 0 {
summary.Violations = append(summary.Violations, violationRecords(violations, witnesses, stepIndex)...) summary.Violations = append(summary.Violations, violationRecords(violations, witnesses, stepIndex)...)
// The step is already written, so the trace ends on the state that
// produced the violation. Finalize below still runs, so pending
// liveness obligations are reported alongside it.
if options.StopOnViolation {
break
}
} }
// Wait actions are themselves a settling: skip the idle poll. Actions // Wait actions are themselves a settling: skip the idle poll. Actions
// that mutate the UI fall through to WaitForIdle so the next step's // that mutate the UI fall through to WaitForIdle so the next step's
+96
View File
@@ -1976,3 +1976,99 @@ func TestRunner_SkipsActionWhenOverlayStealsFocusAtApplyTime(t *testing.T) {
t.Error("apply-time guard failed: a tap fired while a system overlay held focus") t.Error("apply-time guard failed: a tap fired while a system overlay held focus")
} }
} }
// TestRunner_StopOnViolationEndsAtTheFirstViolation pins the gate CI runs on:
// a step budget of 8 against a spec that only violates on the third step must
// end on step 3 and write nothing after it, so the trace's last state is the
// one that produced the violation.
func TestRunner_StopOnViolationEndsAtTheFirstViolation(t *testing.T) {
const thirdStepViolationSpec = `
import { actions, always, extract } from "@sanderling/spec";
let observed = 0;
const tick = extract(() => ++observed);
globalThis.properties = {
staysUnderThree: always(() => tick.current < 3),
};
globalThis.actions = actions(() => []);
`
state := newHarnessWithSpec(t, thirdStepViolationSpec)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
summary, err := Run(ctx, Options{
Duration: time.Hour,
IdleTimeout: 20 * time.Millisecond,
MaxSteps: 8,
StopOnViolation: true,
Driver: state.mock,
Verifier: state.verifier,
TraceWriter: state.writer,
})
if err != nil {
t.Fatalf("Run: %v", err)
}
if !containsProperty(summary.Violations, "staysUnderThree") {
t.Fatalf("expected staysUnderThree to fire, got %v", summary.Violations)
}
if summary.Steps != 3 {
t.Errorf("steps: got %d, want 3 (the run must stop at the violating step, not run the 8-step budget)",
summary.Steps)
}
for _, step := range traceStepIndices(t, state.writer.Directory()) {
if step > summary.Steps {
t.Errorf("trace kept stepping after the violation: found step %d past step %d",
step, summary.Steps)
}
}
}
// TestRunner_WithoutStopOnViolationRunsTheWholeBudget is the other half: the
// default must stay a full-budget fuzz run, so turning the flag on is the only
// thing that shortens a run.
func TestRunner_WithoutStopOnViolationRunsTheWholeBudget(t *testing.T) {
state := newHarnessWithSpec(t, violationSpec)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
summary, err := Run(ctx, Options{
Duration: time.Hour,
IdleTimeout: 20 * time.Millisecond,
MaxSteps: 4,
Driver: state.mock,
Verifier: state.verifier,
TraceWriter: state.writer,
})
if err != nil {
t.Fatalf("Run: %v", err)
}
if summary.Steps != 4 {
t.Errorf("steps: got %d, want 4; a violation must not shorten a default run", summary.Steps)
}
}
// traceStepIndices reads every step index the trace recorded, so a test can
// assert on what the run actually wrote rather than on the summary alone.
func traceStepIndices(t *testing.T, directory string) []int {
t.Helper()
file, err := os.Open(filepath.Join(directory, "trace.jsonl"))
if err != nil {
t.Fatal(err)
}
defer file.Close()
var steps []int
scanner := bufio.NewScanner(file)
scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
for scanner.Scan() {
var line struct {
Step int `json:"step"`
}
if err := json.Unmarshal(scanner.Bytes(), &line); err != nil {
t.Fatalf("trace line decode: %v", err)
}
steps = append(steps, line.Step)
}
if err := scanner.Err(); err != nil {
t.Fatalf("scan trace: %v", err)
}
return steps
}