mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 11:07:10 +00:00
Attribute violations to the causing step and render witness evidence (#59)
* feat(ltl): attribute violations to the obligation origin step * feat(verifier): label evaluator observations with the runner step index * feat(trace): carry the causing step in violation witnesses and summary * feat(replay): move the violation marker to the causing step * feat(replay-ui): render witness evidence in the violations panel * feat(replay-ui): wire witnesses and step jump into violation panels * fix(ltl): treat next obligations as vacuous at run end * fix(runner): give the finalize trace record its own step index
This commit is contained in:
15 files changed
+719
-85
No files matched your search
+36
-14
@@ -8,6 +8,8 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -185,6 +187,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
||||
Tree: tree,
|
||||
LastAction: lastAction,
|
||||
StepTime: stepStart,
|
||||
StepIndex: stepIndex,
|
||||
RunStart: summary.StartTime,
|
||||
Logs: logs,
|
||||
}); err != nil {
|
||||
@@ -263,10 +266,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
||||
}
|
||||
summary.Steps = stepIndex
|
||||
if len(violations) > 0 {
|
||||
summary.Violations = append(summary.Violations, ViolationRecord{
|
||||
StepIndex: stepIndex,
|
||||
Properties: violations,
|
||||
})
|
||||
summary.Violations = append(summary.Violations, violationRecords(violations, witnesses, stepIndex)...)
|
||||
}
|
||||
// Wait actions are themselves a settling: skip the idle poll. Actions
|
||||
// that mutate the UI fall through to WaitForIdle so the next step's
|
||||
@@ -284,17 +284,17 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
||||
}
|
||||
|
||||
// Finalize each evaluator once the loop ends so liveness obligations that
|
||||
// never discharged (an unbounded eventually that never fired, a strong
|
||||
// next with no successor) are reported as violations rather than silently
|
||||
// left pending. Properties already violated mid-run are not re-reported.
|
||||
// never discharged (an eventually that never fired) are reported as
|
||||
// violations rather than silently left pending. Properties already
|
||||
// violated mid-run are not re-reported. The synthetic record gets its own
|
||||
// step index so no two trace lines share one; witnesses still attribute
|
||||
// the violation to the step that spawned the obligation.
|
||||
if ended := options.Verifier.Finalize(); len(ended) > 0 {
|
||||
witnesses := collectWitnesses(options.Verifier, ended, logger, stepIndex)
|
||||
summary.Violations = append(summary.Violations, ViolationRecord{
|
||||
StepIndex: stepIndex,
|
||||
Properties: ended,
|
||||
})
|
||||
finalIndex := stepIndex + 1
|
||||
witnesses := collectWitnesses(options.Verifier, ended, logger, finalIndex)
|
||||
summary.Violations = append(summary.Violations, violationRecords(ended, witnesses, finalIndex)...)
|
||||
finalStep := trace.Step{
|
||||
Index: stepIndex,
|
||||
Index: finalIndex,
|
||||
Timestamp: time.Now(),
|
||||
Violations: ended,
|
||||
Witnesses: witnesses,
|
||||
@@ -806,6 +806,26 @@ func captureMetrics(ctx context.Context, options Options, logger *slog.Logger, s
|
||||
}
|
||||
}
|
||||
|
||||
// violationRecords groups newly-violated properties by the step their witness
|
||||
// attributes the violation to (the causing step), falling back to the
|
||||
// detection step for properties without a witness. Records are ordered by
|
||||
// step; properties keep the sorted order NewlyViolatedProperties produced.
|
||||
func violationRecords(properties []string, witnesses map[string]trace.Witness, detectionStep int) []ViolationRecord {
|
||||
byStep := map[int][]string{}
|
||||
for _, name := range properties {
|
||||
step := detectionStep
|
||||
if witness, ok := witnesses[name]; ok && witness.Step > 0 {
|
||||
step = witness.Step
|
||||
}
|
||||
byStep[step] = append(byStep[step], name)
|
||||
}
|
||||
records := make([]ViolationRecord, 0, len(byStep))
|
||||
for _, step := range slices.Sorted(maps.Keys(byStep)) {
|
||||
records = append(records, ViolationRecord{StepIndex: step, Properties: byStep[step]})
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
// collectWitnesses gathers the violation witness for each newly-violated
|
||||
// property, logs its cause, and returns them keyed by property name for the
|
||||
// trace. Properties without a captured witness are skipped.
|
||||
@@ -820,10 +840,12 @@ func collectWitnesses(verifierInstance *verifier.Verifier, properties []string,
|
||||
continue
|
||||
}
|
||||
logger.Warn("property violated",
|
||||
"step", stepIndex, "property", name, "reason", witness.Reason, "error", witness.IsError)
|
||||
"step", witness.Step, "detected_step", stepIndex,
|
||||
"property", name, "reason", witness.Reason, "error", witness.IsError)
|
||||
witnesses[name] = trace.Witness{
|
||||
Reason: witness.Reason,
|
||||
IsError: witness.IsError,
|
||||
Step: witness.Step,
|
||||
Extractors: witness.Extractors,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -253,6 +253,186 @@ func TestRunner_ViolationSurfacesOnlyOnOnsetStep(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunner_NextViolationAttributedToCausingStep(t *testing.T) {
|
||||
// always(next(p)): the obligation spawned at step 2 fails against step 3's
|
||||
// state. The summary record and the trace witness must attribute the
|
||||
// violation to step 2 (the causing step); the trace line that carries it is
|
||||
// still step 3, where the failure was detected.
|
||||
const nextViolationSpec = `
|
||||
import { actions, always, next, extract } from "@sanderling/spec";
|
||||
let observed = 0;
|
||||
const tick = extract(() => ++observed);
|
||||
globalThis.properties = {
|
||||
nextHolds: always(next(() => tick.current < 3)),
|
||||
};
|
||||
globalThis.actions = actions(() => []);
|
||||
`
|
||||
state := newHarnessWithSpec(t, nextViolationSpec)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
summary, err := Run(ctx, Options{
|
||||
Duration: time.Hour,
|
||||
IdleTimeout: 20 * time.Millisecond,
|
||||
MaxSteps: 5,
|
||||
Driver: state.mock,
|
||||
Verifier: state.verifier,
|
||||
TraceWriter: state.writer,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if len(summary.Violations) != 1 {
|
||||
t.Fatalf("expected exactly one ViolationRecord, got %d: %v",
|
||||
len(summary.Violations), summary.Violations)
|
||||
}
|
||||
if summary.Violations[0].StepIndex != 2 {
|
||||
t.Errorf("summary step: got %d, want 2 (the step that spawned the next obligation)",
|
||||
summary.Violations[0].StepIndex)
|
||||
}
|
||||
if !slices.Equal(summary.Violations[0].Properties, []string{"nextHolds"}) {
|
||||
t.Errorf("properties: got %v, want [nextHolds]", summary.Violations[0].Properties)
|
||||
}
|
||||
|
||||
file, err := os.Open(filepath.Join(state.writer.Directory(), "trace.jsonl"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
type traceLine struct {
|
||||
Step int `json:"step"`
|
||||
Violations []string `json:"violations"`
|
||||
Witnesses map[string]trace.Witness `json:"witnesses"`
|
||||
}
|
||||
found := false
|
||||
scanner := bufio.NewScanner(file)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
|
||||
for scanner.Scan() {
|
||||
var line traceLine
|
||||
if err := json.Unmarshal(scanner.Bytes(), &line); err != nil {
|
||||
t.Fatalf("trace line decode: %v", err)
|
||||
}
|
||||
if len(line.Violations) == 0 {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if line.Step != 3 {
|
||||
t.Errorf("violation detected on step %d, want 3", line.Step)
|
||||
}
|
||||
witness, ok := line.Witnesses["nextHolds"]
|
||||
if !ok {
|
||||
t.Fatalf("step %d carries no witness for nextHolds", line.Step)
|
||||
}
|
||||
if witness.Step != 2 {
|
||||
t.Errorf("witness step: got %d, want 2 (causing step)", witness.Step)
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
t.Fatalf("scan trace: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Error("no trace line carried the violation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunner_AlwaysNextLeavesNoEndOfRunViolation(t *testing.T) {
|
||||
// always(next(p)) ends every run with a pending deferred check. That
|
||||
// residue is vacuous (no successor state to check), so neither the
|
||||
// summary nor the trace may report an end-of-run violation.
|
||||
const spec = `
|
||||
import { actions, always, next } from "@sanderling/spec";
|
||||
globalThis.properties = {
|
||||
nextHolds: always(next(() => true)),
|
||||
};
|
||||
globalThis.actions = actions(() => []);
|
||||
`
|
||||
state := newHarnessWithSpec(t, spec)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
summary, err := Run(ctx, Options{
|
||||
Duration: time.Hour,
|
||||
IdleTimeout: 20 * time.Millisecond,
|
||||
MaxSteps: 3,
|
||||
Driver: state.mock,
|
||||
Verifier: state.verifier,
|
||||
TraceWriter: state.writer,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if len(summary.Violations) != 0 {
|
||||
t.Errorf("expected no violations, got %v", summary.Violations)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunner_FinalizeRecordUsesDistinctStepIndex(t *testing.T) {
|
||||
// An eventually that never fires is reported at run end through a
|
||||
// synthetic trace record. That record must carry its own step index so no
|
||||
// two trace lines share one (duplicate indices made the replay UI select
|
||||
// two rows at once).
|
||||
const spec = `
|
||||
import { actions, eventually } from "@sanderling/spec";
|
||||
globalThis.properties = {
|
||||
neverFires: eventually(() => false),
|
||||
};
|
||||
globalThis.actions = actions(() => []);
|
||||
`
|
||||
state := newHarnessWithSpec(t, spec)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
summary, err := Run(ctx, Options{
|
||||
Duration: time.Hour,
|
||||
IdleTimeout: 20 * time.Millisecond,
|
||||
MaxSteps: 3,
|
||||
Driver: state.mock,
|
||||
Verifier: state.verifier,
|
||||
TraceWriter: state.writer,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if !containsProperty(summary.Violations, "neverFires") {
|
||||
t.Fatalf("expected neverFires in violations: %v", summary.Violations)
|
||||
}
|
||||
|
||||
file, err := os.Open(filepath.Join(state.writer.Directory(), "trace.jsonl"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
type traceLine struct {
|
||||
Step int `json:"step"`
|
||||
Violations []string `json:"violations"`
|
||||
}
|
||||
seen := map[int]bool{}
|
||||
finalizeStep := 0
|
||||
scanner := bufio.NewScanner(file)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
|
||||
for scanner.Scan() {
|
||||
var line traceLine
|
||||
if err := json.Unmarshal(scanner.Bytes(), &line); err != nil {
|
||||
t.Fatalf("trace line decode: %v", err)
|
||||
}
|
||||
if seen[line.Step] {
|
||||
t.Errorf("duplicate step index %d in trace", line.Step)
|
||||
}
|
||||
seen[line.Step] = true
|
||||
if slices.Contains(line.Violations, "neverFires") {
|
||||
finalizeStep = line.Step
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
t.Fatalf("scan trace: %v", err)
|
||||
}
|
||||
if finalizeStep != summary.Steps+1 {
|
||||
t.Errorf("finalize record step = %d, want %d (steps+1)", finalizeStep, summary.Steps+1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunner_ThrowingPredicateIsLoggedNotPanic(t *testing.T) {
|
||||
const throwingSpec = `
|
||||
import { actions, always, Tap } from "@sanderling/spec";
|
||||
|
||||
Reference in new issue
Block a user