mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 11:07:10 +00:00
feat(runner): stop the step loop at the first violation on request
This commit is contained in:
1 parent
1f71e052d7
commit
39289f81df
2 files changed
+107
No files matched your search
@@ -31,6 +31,11 @@ type Options struct {
|
||||
// positive value stops the loop once that many steps have run.
|
||||
MaxSteps int
|
||||
|
||||
// StopOnViolation ends the step loop as soon as a step records a
|
||||
// violation, so a run that exists to find one bug stops at the evidence
|
||||
// instead of spending the rest of its budget past it.
|
||||
StopOnViolation bool
|
||||
|
||||
BundleID string
|
||||
Driver driver.DeviceDriver
|
||||
Verifier *verifier.Verifier
|
||||
@@ -296,6 +301,12 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
||||
summary.Steps = stepIndex
|
||||
if len(violations) > 0 {
|
||||
summary.Violations = append(summary.Violations, violationRecords(violations, witnesses, stepIndex)...)
|
||||
// The step is already written, so the trace ends on the state that
|
||||
// produced the violation. Finalize below still runs, so pending
|
||||
// liveness obligations are reported alongside it.
|
||||
if options.StopOnViolation {
|
||||
break
|
||||
}
|
||||
}
|
||||
// Wait actions are themselves a settling: skip the idle poll. Actions
|
||||
// that mutate the UI fall through to WaitForIdle so the next step's
|
||||
|
||||
@@ -1976,3 +1976,99 @@ func TestRunner_SkipsActionWhenOverlayStealsFocusAtApplyTime(t *testing.T) {
|
||||
t.Error("apply-time guard failed: a tap fired while a system overlay held focus")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunner_StopOnViolationEndsAtTheFirstViolation pins the gate CI runs on:
|
||||
// a step budget of 8 against a spec that only violates on the third step must
|
||||
// end on step 3 and write nothing after it, so the trace's last state is the
|
||||
// one that produced the violation.
|
||||
func TestRunner_StopOnViolationEndsAtTheFirstViolation(t *testing.T) {
|
||||
const thirdStepViolationSpec = `
|
||||
import { actions, always, extract } from "@sanderling/spec";
|
||||
let observed = 0;
|
||||
const tick = extract(() => ++observed);
|
||||
globalThis.properties = {
|
||||
staysUnderThree: always(() => tick.current < 3),
|
||||
};
|
||||
globalThis.actions = actions(() => []);
|
||||
`
|
||||
state := newHarnessWithSpec(t, thirdStepViolationSpec)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
summary, err := Run(ctx, Options{
|
||||
Duration: time.Hour,
|
||||
IdleTimeout: 20 * time.Millisecond,
|
||||
MaxSteps: 8,
|
||||
StopOnViolation: true,
|
||||
Driver: state.mock,
|
||||
Verifier: state.verifier,
|
||||
TraceWriter: state.writer,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if !containsProperty(summary.Violations, "staysUnderThree") {
|
||||
t.Fatalf("expected staysUnderThree to fire, got %v", summary.Violations)
|
||||
}
|
||||
if summary.Steps != 3 {
|
||||
t.Errorf("steps: got %d, want 3 (the run must stop at the violating step, not run the 8-step budget)",
|
||||
summary.Steps)
|
||||
}
|
||||
for _, step := range traceStepIndices(t, state.writer.Directory()) {
|
||||
if step > summary.Steps {
|
||||
t.Errorf("trace kept stepping after the violation: found step %d past step %d",
|
||||
step, summary.Steps)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunner_WithoutStopOnViolationRunsTheWholeBudget is the other half: the
|
||||
// default must stay a full-budget fuzz run, so turning the flag on is the only
|
||||
// thing that shortens a run.
|
||||
func TestRunner_WithoutStopOnViolationRunsTheWholeBudget(t *testing.T) {
|
||||
state := newHarnessWithSpec(t, violationSpec)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
summary, err := Run(ctx, Options{
|
||||
Duration: time.Hour,
|
||||
IdleTimeout: 20 * time.Millisecond,
|
||||
MaxSteps: 4,
|
||||
Driver: state.mock,
|
||||
Verifier: state.verifier,
|
||||
TraceWriter: state.writer,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if summary.Steps != 4 {
|
||||
t.Errorf("steps: got %d, want 4; a violation must not shorten a default run", summary.Steps)
|
||||
}
|
||||
}
|
||||
|
||||
// traceStepIndices reads every step index the trace recorded, so a test can
|
||||
// assert on what the run actually wrote rather than on the summary alone.
|
||||
func traceStepIndices(t *testing.T, directory string) []int {
|
||||
t.Helper()
|
||||
file, err := os.Open(filepath.Join(directory, "trace.jsonl"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
var steps []int
|
||||
scanner := bufio.NewScanner(file)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
|
||||
for scanner.Scan() {
|
||||
var line struct {
|
||||
Step int `json:"step"`
|
||||
}
|
||||
if err := json.Unmarshal(scanner.Bytes(), &line); err != nil {
|
||||
t.Fatalf("trace line decode: %v", err)
|
||||
}
|
||||
steps = append(steps, line.Step)
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
t.Fatalf("scan trace: %v", err)
|
||||
}
|
||||
return steps
|
||||
}
|
||||
Reference in new issue
Block a user