mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
feat(runner): stop the step loop at the first violation on request
This commit is contained in:
1 parent
1f71e052d7
commit
39289f81df
2 files changed
+107
No files matched your search
@@ -31,6 +31,11 @@ type Options struct {
|
|||||||
// positive value stops the loop once that many steps have run.
|
// positive value stops the loop once that many steps have run.
|
||||||
MaxSteps int
|
MaxSteps int
|
||||||
|
|
||||||
|
// StopOnViolation ends the step loop as soon as a step records a
|
||||||
|
// violation, so a run that exists to find one bug stops at the evidence
|
||||||
|
// instead of spending the rest of its budget past it.
|
||||||
|
StopOnViolation bool
|
||||||
|
|
||||||
BundleID string
|
BundleID string
|
||||||
Driver driver.DeviceDriver
|
Driver driver.DeviceDriver
|
||||||
Verifier *verifier.Verifier
|
Verifier *verifier.Verifier
|
||||||
@@ -296,6 +301,12 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
|||||||
summary.Steps = stepIndex
|
summary.Steps = stepIndex
|
||||||
if len(violations) > 0 {
|
if len(violations) > 0 {
|
||||||
summary.Violations = append(summary.Violations, violationRecords(violations, witnesses, stepIndex)...)
|
summary.Violations = append(summary.Violations, violationRecords(violations, witnesses, stepIndex)...)
|
||||||
|
// The step is already written, so the trace ends on the state that
|
||||||
|
// produced the violation. Finalize below still runs, so pending
|
||||||
|
// liveness obligations are reported alongside it.
|
||||||
|
if options.StopOnViolation {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Wait actions are themselves a settling: skip the idle poll. Actions
|
// Wait actions are themselves a settling: skip the idle poll. Actions
|
||||||
// that mutate the UI fall through to WaitForIdle so the next step's
|
// that mutate the UI fall through to WaitForIdle so the next step's
|
||||||
|
|||||||
@@ -1976,3 +1976,99 @@ func TestRunner_SkipsActionWhenOverlayStealsFocusAtApplyTime(t *testing.T) {
|
|||||||
t.Error("apply-time guard failed: a tap fired while a system overlay held focus")
|
t.Error("apply-time guard failed: a tap fired while a system overlay held focus")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestRunner_StopOnViolationEndsAtTheFirstViolation pins the gate CI runs on:
|
||||||
|
// a step budget of 8 against a spec that only violates on the third step must
|
||||||
|
// end on step 3 and write nothing after it, so the trace's last state is the
|
||||||
|
// one that produced the violation.
|
||||||
|
func TestRunner_StopOnViolationEndsAtTheFirstViolation(t *testing.T) {
|
||||||
|
const thirdStepViolationSpec = `
|
||||||
|
import { actions, always, extract } from "@sanderling/spec";
|
||||||
|
let observed = 0;
|
||||||
|
const tick = extract(() => ++observed);
|
||||||
|
globalThis.properties = {
|
||||||
|
staysUnderThree: always(() => tick.current < 3),
|
||||||
|
};
|
||||||
|
globalThis.actions = actions(() => []);
|
||||||
|
`
|
||||||
|
state := newHarnessWithSpec(t, thirdStepViolationSpec)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
summary, err := Run(ctx, Options{
|
||||||
|
Duration: time.Hour,
|
||||||
|
IdleTimeout: 20 * time.Millisecond,
|
||||||
|
MaxSteps: 8,
|
||||||
|
StopOnViolation: true,
|
||||||
|
Driver: state.mock,
|
||||||
|
Verifier: state.verifier,
|
||||||
|
TraceWriter: state.writer,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run: %v", err)
|
||||||
|
}
|
||||||
|
if !containsProperty(summary.Violations, "staysUnderThree") {
|
||||||
|
t.Fatalf("expected staysUnderThree to fire, got %v", summary.Violations)
|
||||||
|
}
|
||||||
|
if summary.Steps != 3 {
|
||||||
|
t.Errorf("steps: got %d, want 3 (the run must stop at the violating step, not run the 8-step budget)",
|
||||||
|
summary.Steps)
|
||||||
|
}
|
||||||
|
for _, step := range traceStepIndices(t, state.writer.Directory()) {
|
||||||
|
if step > summary.Steps {
|
||||||
|
t.Errorf("trace kept stepping after the violation: found step %d past step %d",
|
||||||
|
step, summary.Steps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunner_WithoutStopOnViolationRunsTheWholeBudget is the other half: the
|
||||||
|
// default must stay a full-budget fuzz run, so turning the flag on is the only
|
||||||
|
// thing that shortens a run.
|
||||||
|
func TestRunner_WithoutStopOnViolationRunsTheWholeBudget(t *testing.T) {
|
||||||
|
state := newHarnessWithSpec(t, violationSpec)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
summary, err := Run(ctx, Options{
|
||||||
|
Duration: time.Hour,
|
||||||
|
IdleTimeout: 20 * time.Millisecond,
|
||||||
|
MaxSteps: 4,
|
||||||
|
Driver: state.mock,
|
||||||
|
Verifier: state.verifier,
|
||||||
|
TraceWriter: state.writer,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run: %v", err)
|
||||||
|
}
|
||||||
|
if summary.Steps != 4 {
|
||||||
|
t.Errorf("steps: got %d, want 4; a violation must not shorten a default run", summary.Steps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// traceStepIndices reads every step index the trace recorded, so a test can
|
||||||
|
// assert on what the run actually wrote rather than on the summary alone.
|
||||||
|
func traceStepIndices(t *testing.T, directory string) []int {
|
||||||
|
t.Helper()
|
||||||
|
file, err := os.Open(filepath.Join(directory, "trace.jsonl"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
var steps []int
|
||||||
|
scanner := bufio.NewScanner(file)
|
||||||
|
scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
|
||||||
|
for scanner.Scan() {
|
||||||
|
var line struct {
|
||||||
|
Step int `json:"step"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(scanner.Bytes(), &line); err != nil {
|
||||||
|
t.Fatalf("trace line decode: %v", err)
|
||||||
|
}
|
||||||
|
steps = append(steps, line.Step)
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
t.Fatalf("scan trace: %v", err)
|
||||||
|
}
|
||||||
|
return steps
|
||||||
|
}
|
||||||
Reference in new issue
Block a user