mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 11:07:10 +00:00
Attribute violations to the causing step and render witness evidence (#59)
* feat(ltl): attribute violations to the obligation origin step * feat(verifier): label evaluator observations with the runner step index * feat(trace): carry the causing step in violation witnesses and summary * feat(replay): move the violation marker to the causing step * feat(replay-ui): render witness evidence in the violations panel * feat(replay-ui): wire witnesses and step jump into violation panels * fix(ltl): treat next obligations as vacuous at run end * fix(runner): give the finalize trace record its own step index
This commit is contained in:
15 files changed
+719
-85
No files matched your search
+77
-49
@@ -33,17 +33,27 @@ func (v Verdict) String() string {
|
|||||||
// violates, the overall verdict latches to Violated.
|
// violates, the overall verdict latches to Violated.
|
||||||
type Evaluator struct {
|
type Evaluator struct {
|
||||||
root Formula
|
root Formula
|
||||||
pending []Formula
|
pending []obligation
|
||||||
violated bool
|
violated bool
|
||||||
steps int
|
steps int
|
||||||
violation *Violation
|
violation *Violation
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// obligation pairs a residual formula with the step that spawned it, so a
|
||||||
|
// deferred check (a next, a pending eventually) that fails on a later step can
|
||||||
|
// be attributed to the step that created the obligation.
|
||||||
|
type obligation struct {
|
||||||
|
formula Formula
|
||||||
|
origin int
|
||||||
|
}
|
||||||
|
|
||||||
// Violation is the witness for a latched verdict: the failing sub-formula, a
|
// Violation is the witness for a latched verdict: the failing sub-formula, a
|
||||||
// human-readable reason, and the observation step it fired at. A thrown
|
// human-readable reason, and the step the failed obligation originated at. For
|
||||||
// predicate carries the goja error text as its reason and sets IsError; a plain
|
// an immediate predicate failure that is the observation step itself; for a
|
||||||
// false carries "predicate false"; Finalize fills it for liveness obligations
|
// deferred obligation (next, eventually) it is the earlier step that spawned
|
||||||
// that never discharged.
|
// it, the one that caused the violation. A thrown predicate carries the goja
|
||||||
|
// error text as its reason and sets IsError; a plain false carries "predicate
|
||||||
|
// false"; Finalize fills it for liveness obligations that never discharged.
|
||||||
type Violation struct {
|
type Violation struct {
|
||||||
Formula Formula
|
Formula Formula
|
||||||
Reason string
|
Reason string
|
||||||
@@ -62,19 +72,29 @@ func (e *Evaluator) Observe() Verdict {
|
|||||||
return e.ObserveAt(time.Now())
|
return e.ObserveAt(time.Now())
|
||||||
}
|
}
|
||||||
|
|
||||||
// ObserveAt is like Observe but takes the current step time explicitly.
|
// ObserveAt is like Observe but takes the current step time explicitly. Steps
|
||||||
|
// are numbered by an internal counter starting at 1; callers whose step
|
||||||
|
// numbering can skip observations should use ObserveAtStep instead.
|
||||||
func (e *Evaluator) ObserveAt(now time.Time) Verdict {
|
func (e *Evaluator) ObserveAt(now time.Time) Verdict {
|
||||||
|
return e.ObserveAtStep(now, e.steps+1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ObserveAtStep is like ObserveAt but labels the observation with the caller's
|
||||||
|
// step index, so violation witnesses carry the caller's numbering even when
|
||||||
|
// some steps were never observed (for example transitional steps the verifier
|
||||||
|
// skips).
|
||||||
|
func (e *Evaluator) ObserveAtStep(now time.Time, step int) Verdict {
|
||||||
if e.violated {
|
if e.violated {
|
||||||
return VerdictViolated
|
return VerdictViolated
|
||||||
}
|
}
|
||||||
e.steps++
|
e.steps = step
|
||||||
|
|
||||||
fresh := rootObligation(e.root)
|
fresh := obligation{formula: rootObligation(e.root), origin: step}
|
||||||
obligations := append(e.pending, fresh)
|
obligations := append(e.pending, fresh)
|
||||||
e.pending = e.pending[:0]
|
e.pending = e.pending[:0]
|
||||||
|
|
||||||
for _, obligation := range obligations {
|
for _, entry := range obligations {
|
||||||
result := reduce(obligation, now)
|
result := reduce(entry.formula, now)
|
||||||
switch result.status {
|
switch result.status {
|
||||||
case statusHolds:
|
case statusHolds:
|
||||||
// drop
|
// drop
|
||||||
@@ -83,11 +103,11 @@ func (e *Evaluator) ObserveAt(now time.Time) Verdict {
|
|||||||
e.pending = nil
|
e.pending = nil
|
||||||
e.violation = result.witness
|
e.violation = result.witness
|
||||||
if e.violation != nil {
|
if e.violation != nil {
|
||||||
e.violation.Step = e.steps
|
e.violation.Step = entry.origin
|
||||||
}
|
}
|
||||||
return VerdictViolated
|
return VerdictViolated
|
||||||
case statusPending:
|
case statusPending:
|
||||||
e.pending = append(e.pending, result.formula)
|
e.pending = append(e.pending, obligation{formula: result.formula, origin: entry.origin})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -100,41 +120,43 @@ func (e *Evaluator) ObserveAt(now time.Time) Verdict {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// collapse removes structurally-identical obligations, keeping the first
|
// collapse removes structurally-identical obligations, keeping the first
|
||||||
// occurrence in order. Distinct predicates never merge because ThunkFormula's
|
// occurrence in order so the surviving entry carries the earliest origin step.
|
||||||
// name participates in its describe() key, so deduping cannot hide a violation.
|
// Distinct predicates never merge because ThunkFormula's name participates in
|
||||||
func collapse(obligations []Formula) []Formula {
|
// its describe() key, so deduping cannot hide a violation.
|
||||||
|
func collapse(obligations []obligation) []obligation {
|
||||||
if len(obligations) < 2 {
|
if len(obligations) < 2 {
|
||||||
return obligations
|
return obligations
|
||||||
}
|
}
|
||||||
seen := make(map[string]struct{}, len(obligations))
|
seen := make(map[string]struct{}, len(obligations))
|
||||||
result := obligations[:0]
|
result := obligations[:0]
|
||||||
for _, obligation := range obligations {
|
for _, entry := range obligations {
|
||||||
key := obligation.describe()
|
key := entry.formula.describe()
|
||||||
if _, ok := seen[key]; ok {
|
if _, ok := seen[key]; ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
seen[key] = struct{}{}
|
seen[key] = struct{}{}
|
||||||
result = append(result, obligation)
|
result = append(result, entry)
|
||||||
}
|
}
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
// Finalize reports the terminal verdict for the run. Pending obligations that
|
// Finalize reports the terminal verdict for the run. A liveness promise that
|
||||||
// can never be discharged by a future step (an unbounded eventually that never
|
// never discharged (an eventually that never fired) resolves to Violated. A
|
||||||
// fired, a strong next with no successor) resolve to Violated; safety
|
// deferred state check (the residue of a next) has no successor state to
|
||||||
// obligations that were never breached resolve to Holds.
|
// evaluate against, so it is indefinite and resolves vacuously to Holds: the
|
||||||
|
// run ended before the obligation could be checked, which is not a failure.
|
||||||
func (e *Evaluator) Finalize() Verdict {
|
func (e *Evaluator) Finalize() Verdict {
|
||||||
if e.violated {
|
if e.violated {
|
||||||
return VerdictViolated
|
return VerdictViolated
|
||||||
}
|
}
|
||||||
for _, obligation := range e.pending {
|
for _, entry := range e.pending {
|
||||||
if finalize(obligation) == statusViolated {
|
if finalize(entry.formula) == statusViolated {
|
||||||
e.violated = true
|
e.violated = true
|
||||||
e.pending = nil
|
e.pending = nil
|
||||||
e.violation = &Violation{
|
e.violation = &Violation{
|
||||||
Formula: obligation,
|
Formula: entry.formula,
|
||||||
Reason: finalizeReason(obligation),
|
Reason: finalizeReason(entry.formula),
|
||||||
Step: e.steps,
|
Step: entry.origin,
|
||||||
}
|
}
|
||||||
return VerdictViolated
|
return VerdictViolated
|
||||||
}
|
}
|
||||||
@@ -156,17 +178,18 @@ func finalizeReason(formula Formula) string {
|
|||||||
switch formula.(type) {
|
switch formula.(type) {
|
||||||
case EventuallyFormula:
|
case EventuallyFormula:
|
||||||
return "eventually never satisfied"
|
return "eventually never satisfied"
|
||||||
case NextFormula:
|
|
||||||
return "next obligation unmet at run end"
|
|
||||||
case ThunkFormula:
|
|
||||||
return "obligation unmet at run end"
|
|
||||||
default:
|
default:
|
||||||
return "liveness obligation unmet at run end"
|
return "liveness obligation unmet at run end"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// finalize collapses a pending obligation to its terminal status assuming no
|
// finalize collapses a pending obligation to its terminal status assuming no
|
||||||
// further steps will occur.
|
// further steps will occur. Three-valued: a pending thunk or next is the
|
||||||
|
// residue of a deferred state check with no state left to check, so it is
|
||||||
|
// indefinite (statusPending) rather than violated; only a liveness promise
|
||||||
|
// (an eventually that never fired) is a definite end-of-run violation.
|
||||||
|
// Connectives combine with Kleene semantics so an indefinite sub-formula
|
||||||
|
// never manufactures a definite verdict.
|
||||||
func finalize(formula Formula) residualStatus {
|
func finalize(formula Formula) residualStatus {
|
||||||
switch concrete := formula.(type) {
|
switch concrete := formula.(type) {
|
||||||
case PureFormula:
|
case PureFormula:
|
||||||
@@ -175,11 +198,11 @@ func finalize(formula Formula) residualStatus {
|
|||||||
}
|
}
|
||||||
return statusViolated
|
return statusViolated
|
||||||
case ThunkFormula:
|
case ThunkFormula:
|
||||||
return statusViolated
|
return statusPending
|
||||||
case EventuallyFormula:
|
case EventuallyFormula:
|
||||||
return statusViolated
|
return statusViolated
|
||||||
case NextFormula:
|
case NextFormula:
|
||||||
return statusViolated
|
return statusPending
|
||||||
case AlwaysFormula:
|
case AlwaysFormula:
|
||||||
return statusHolds
|
return statusHolds
|
||||||
case NowFormula:
|
case NowFormula:
|
||||||
@@ -188,24 +211,34 @@ func finalize(formula Formula) residualStatus {
|
|||||||
switch finalize(concrete.Inner) {
|
switch finalize(concrete.Inner) {
|
||||||
case statusViolated:
|
case statusViolated:
|
||||||
return statusHolds
|
return statusHolds
|
||||||
default:
|
case statusHolds:
|
||||||
return statusViolated
|
return statusViolated
|
||||||
|
default:
|
||||||
|
return statusPending
|
||||||
}
|
}
|
||||||
case AndFormula:
|
case AndFormula:
|
||||||
if finalize(concrete.Left) == statusViolated || finalize(concrete.Right) == statusViolated {
|
left, right := finalize(concrete.Left), finalize(concrete.Right)
|
||||||
|
if left == statusViolated || right == statusViolated {
|
||||||
return statusViolated
|
return statusViolated
|
||||||
}
|
}
|
||||||
|
if left == statusPending || right == statusPending {
|
||||||
|
return statusPending
|
||||||
|
}
|
||||||
return statusHolds
|
return statusHolds
|
||||||
case OrFormula:
|
case OrFormula:
|
||||||
if finalize(concrete.Left) == statusHolds || finalize(concrete.Right) == statusHolds {
|
left, right := finalize(concrete.Left), finalize(concrete.Right)
|
||||||
|
if left == statusHolds || right == statusHolds {
|
||||||
return statusHolds
|
return statusHolds
|
||||||
}
|
}
|
||||||
|
if left == statusPending || right == statusPending {
|
||||||
|
return statusPending
|
||||||
|
}
|
||||||
return statusViolated
|
return statusViolated
|
||||||
case ImpliesFormula:
|
case ImpliesFormula:
|
||||||
if finalize(concrete.Antecedent) == statusViolated {
|
return finalize(OrFormula{
|
||||||
return statusHolds
|
Left: NotFormula{Inner: concrete.Antecedent},
|
||||||
}
|
Right: concrete.Consequent,
|
||||||
return finalize(concrete.Consequent)
|
})
|
||||||
default:
|
default:
|
||||||
return statusHolds
|
return statusHolds
|
||||||
}
|
}
|
||||||
@@ -224,9 +257,9 @@ func (e *Evaluator) Residual() Formula {
|
|||||||
if len(e.pending) == 0 {
|
if len(e.pending) == 0 {
|
||||||
return PureFormula{Value: true}
|
return PureFormula{Value: true}
|
||||||
}
|
}
|
||||||
combined := e.pending[0]
|
combined := e.pending[0].formula
|
||||||
for _, formula := range e.pending[1:] {
|
for _, entry := range e.pending[1:] {
|
||||||
combined = AndFormula{Left: combined, Right: formula}
|
combined = AndFormula{Left: combined, Right: entry.formula}
|
||||||
}
|
}
|
||||||
return combined
|
return combined
|
||||||
}
|
}
|
||||||
@@ -258,11 +291,6 @@ type reduceResult struct {
|
|||||||
|
|
||||||
func holds() reduceResult { return reduceResult{status: statusHolds} }
|
func holds() reduceResult { return reduceResult{status: statusHolds} }
|
||||||
|
|
||||||
// violated reports a violation without an attached witness. Used where the
|
|
||||||
// failing sub-formula is recovered from a child result whose own witness is
|
|
||||||
// carried up by violatedFrom.
|
|
||||||
func violated() reduceResult { return reduceResult{status: statusViolated} }
|
|
||||||
|
|
||||||
// violatedWith reports a violation that originates at the given sub-formula
|
// violatedWith reports a violation that originates at the given sub-formula
|
||||||
// with the given reason. The reason distinguishes a thrown predicate from a
|
// with the given reason. The reason distinguishes a thrown predicate from a
|
||||||
// plain false so callers (and the replay UI) can render the cause.
|
// plain false so callers (and the replay UI) can render the cause.
|
||||||
|
|||||||
@@ -18,13 +18,32 @@ func TestFinalize_UnboundedEventuallyUnmetIsViolated(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFinalize_FinalStepNextIsViolated(t *testing.T) {
|
func TestFinalize_FinalStepNextIsVacuouslyHolds(t *testing.T) {
|
||||||
|
// A next obligation pending at run end has no successor state to check;
|
||||||
|
// the run ending before the check is not a failure (weak next at the
|
||||||
|
// trace boundary).
|
||||||
evaluator := NewEvaluator(Next(ThunkNamed("p", func() (bool, error) { return true, nil })))
|
evaluator := NewEvaluator(Next(ThunkNamed("p", func() (bool, error) { return true, nil })))
|
||||||
if got := evaluator.Observe(); got != VerdictPending {
|
if got := evaluator.Observe(); got != VerdictPending {
|
||||||
t.Fatalf("step 1: got %v, want pending", got)
|
t.Fatalf("step 1: got %v, want pending", got)
|
||||||
}
|
}
|
||||||
if got := evaluator.Finalize(); got != VerdictViolated {
|
if got := evaluator.Finalize(); got != VerdictHolds {
|
||||||
t.Errorf("Finalize = %v, want violated", got)
|
t.Errorf("Finalize = %v, want holds", got)
|
||||||
|
}
|
||||||
|
if witness := evaluator.Violation(); witness != nil {
|
||||||
|
t.Errorf("Violation = %+v, want nil for a vacuous next", witness)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFinalize_AlwaysNextNeverReportsAtRunEnd(t *testing.T) {
|
||||||
|
// always(next(p)): every step spawns a deferred check and the last one is
|
||||||
|
// always pending when the run ends. That residue must not surface as an
|
||||||
|
// end-of-run violation.
|
||||||
|
evaluator := NewEvaluator(Always(Next(ThunkNamed("p", func() (bool, error) { return true, nil }))))
|
||||||
|
for index := range 3 {
|
||||||
|
evaluator.ObserveAt(time.Unix(int64(index), 0))
|
||||||
|
}
|
||||||
|
if got := evaluator.Finalize(); got != VerdictHolds {
|
||||||
|
t.Errorf("Finalize = %v, want holds", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,20 +147,23 @@ func TestViolationLatchIsMonotonic(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCollapse_IdenticalObligationsMerge(t *testing.T) {
|
func TestCollapse_IdenticalObligationsMerge(t *testing.T) {
|
||||||
merged := collapse([]Formula{
|
merged := collapse([]obligation{
|
||||||
Next(Pure(true)),
|
{formula: Next(Pure(true)), origin: 1},
|
||||||
Next(Pure(true)),
|
{formula: Next(Pure(true)), origin: 2},
|
||||||
Next(Pure(true)),
|
{formula: Next(Pure(true)), origin: 3},
|
||||||
})
|
})
|
||||||
if len(merged) != 1 {
|
if len(merged) != 1 {
|
||||||
t.Errorf("expected 1 obligation after collapse, got %d", len(merged))
|
t.Errorf("expected 1 obligation after collapse, got %d", len(merged))
|
||||||
}
|
}
|
||||||
|
if merged[0].origin != 1 {
|
||||||
|
t.Errorf("collapse must keep the earliest origin, got %d", merged[0].origin)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCollapse_DistinctPredicatesDoNotMerge(t *testing.T) {
|
func TestCollapse_DistinctPredicatesDoNotMerge(t *testing.T) {
|
||||||
merged := collapse([]Formula{
|
merged := collapse([]obligation{
|
||||||
Eventually(ThunkNamed("p3", func() (bool, error) { return false, nil })),
|
{formula: Eventually(ThunkNamed("p3", func() (bool, error) { return false, nil }))},
|
||||||
Eventually(ThunkNamed("p4", func() (bool, error) { return false, nil })),
|
{formula: Eventually(ThunkNamed("p4", func() (bool, error) { return false, nil }))},
|
||||||
})
|
})
|
||||||
if len(merged) != 2 {
|
if len(merged) != 2 {
|
||||||
t.Errorf("distinct predicates must not merge, got %d", len(merged))
|
t.Errorf("distinct predicates must not merge, got %d", len(merged))
|
||||||
|
|||||||
@@ -71,6 +71,83 @@ func TestViolation_FinalizeFillsWitness(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestViolation_NextAttributesOriginStep(t *testing.T) {
|
||||||
|
// always(next(p)): the obligation spawned at step 2 is checked against
|
||||||
|
// step 3's state; the violation belongs to step 2, the step that caused it.
|
||||||
|
values := []bool{true, false}
|
||||||
|
step := 0
|
||||||
|
evaluator := NewEvaluator(Always(Next(ThunkNamed("p", func() (bool, error) {
|
||||||
|
current := values[step]
|
||||||
|
step++
|
||||||
|
return current, nil
|
||||||
|
}))))
|
||||||
|
if got := evaluator.ObserveAt(time.Unix(0, 0)); got != VerdictPending {
|
||||||
|
t.Fatalf("step 1: got %v, want pending", got)
|
||||||
|
}
|
||||||
|
if got := evaluator.ObserveAt(time.Unix(1, 0)); got != VerdictPending {
|
||||||
|
t.Fatalf("step 2: got %v, want pending", got)
|
||||||
|
}
|
||||||
|
if got := evaluator.ObserveAt(time.Unix(2, 0)); got != VerdictViolated {
|
||||||
|
t.Fatalf("step 3: got %v, want violated", got)
|
||||||
|
}
|
||||||
|
witness := evaluator.Violation()
|
||||||
|
if witness == nil {
|
||||||
|
t.Fatal("Violation = nil, want non-nil")
|
||||||
|
}
|
||||||
|
if witness.Step != 2 {
|
||||||
|
t.Errorf("Step = %d, want 2 (the step that spawned the next obligation)", witness.Step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestViolation_FinalizeAttributesOriginStep(t *testing.T) {
|
||||||
|
// An eventually that never fires is reported by Finalize at the step the
|
||||||
|
// obligation was first spawned (collapse keeps the earliest origin).
|
||||||
|
evaluator := NewEvaluator(Eventually(ThunkNamed("p", func() (bool, error) {
|
||||||
|
return false, nil
|
||||||
|
})))
|
||||||
|
evaluator.ObserveAt(time.Unix(0, 0))
|
||||||
|
evaluator.ObserveAt(time.Unix(1, 0))
|
||||||
|
evaluator.ObserveAt(time.Unix(2, 0))
|
||||||
|
if got := evaluator.Finalize(); got != VerdictViolated {
|
||||||
|
t.Fatalf("Finalize = %v, want violated", got)
|
||||||
|
}
|
||||||
|
witness := evaluator.Violation()
|
||||||
|
if witness == nil {
|
||||||
|
t.Fatal("Violation = nil after Finalize, want non-nil")
|
||||||
|
}
|
||||||
|
if witness.Step != 1 {
|
||||||
|
t.Errorf("Step = %d, want 1 (the step the eventually obligation was spawned)", witness.Step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestViolation_ObserveAtStepUsesCallerNumbering(t *testing.T) {
|
||||||
|
// The caller skips step 5 (a transitional step the verifier never saw); the
|
||||||
|
// origin must carry the caller's labels, not a contiguous internal count.
|
||||||
|
values := []bool{true, false}
|
||||||
|
step := 0
|
||||||
|
evaluator := NewEvaluator(Always(Next(ThunkNamed("p", func() (bool, error) {
|
||||||
|
current := values[step]
|
||||||
|
step++
|
||||||
|
return current, nil
|
||||||
|
}))))
|
||||||
|
if got := evaluator.ObserveAtStep(time.Unix(0, 0), 3); got != VerdictPending {
|
||||||
|
t.Fatalf("step 3: got %v, want pending", got)
|
||||||
|
}
|
||||||
|
if got := evaluator.ObserveAtStep(time.Unix(1, 0), 4); got != VerdictPending {
|
||||||
|
t.Fatalf("step 4: got %v, want pending", got)
|
||||||
|
}
|
||||||
|
if got := evaluator.ObserveAtStep(time.Unix(2, 0), 6); got != VerdictViolated {
|
||||||
|
t.Fatalf("step 6: got %v, want violated", got)
|
||||||
|
}
|
||||||
|
witness := evaluator.Violation()
|
||||||
|
if witness == nil {
|
||||||
|
t.Fatal("Violation = nil, want non-nil")
|
||||||
|
}
|
||||||
|
if witness.Step != 4 {
|
||||||
|
t.Errorf("Step = %d, want 4 (caller-labeled origin, not detection step 6)", witness.Step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestViolation_NilBeforeViolation(t *testing.T) {
|
func TestViolation_NilBeforeViolation(t *testing.T) {
|
||||||
evaluator := NewEvaluator(Always(Pure(true)))
|
evaluator := NewEvaluator(Always(Pure(true)))
|
||||||
evaluator.Observe()
|
evaluator.Observe()
|
||||||
|
|||||||
@@ -91,7 +91,7 @@ func scanSteps(tracePath string) ([]StepSummary, []int64, int, time.Time, error)
|
|||||||
reader := bufio.NewReaderSize(file, 64*1024)
|
reader := bufio.NewReaderSize(file, 64*1024)
|
||||||
steps := []StepSummary{}
|
steps := []StepSummary{}
|
||||||
offsets := []int64{}
|
offsets := []int64{}
|
||||||
violationCount := 0
|
attributions := []violationAttribution{}
|
||||||
var offset int64
|
var offset int64
|
||||||
for {
|
for {
|
||||||
lineStart := offset
|
lineStart := offset
|
||||||
@@ -102,19 +102,20 @@ func scanSteps(tracePath string) ([]StepSummary, []int64, int, time.Time, error)
|
|||||||
trimmed = trimmed[:len(trimmed)-1]
|
trimmed = trimmed[:len(trimmed)-1]
|
||||||
}
|
}
|
||||||
if len(trimmed) > 0 {
|
if len(trimmed) > 0 {
|
||||||
summary, partial, decodeErr := decodeStepSummary(trimmed)
|
summary, lineAttributions, decodeErr := decodeStepSummary(trimmed)
|
||||||
if decodeErr != nil {
|
if decodeErr != nil {
|
||||||
return nil, nil, 0, time.Time{}, decodeErr
|
return nil, nil, 0, time.Time{}, decodeErr
|
||||||
}
|
}
|
||||||
steps = append(steps, summary)
|
steps = append(steps, summary)
|
||||||
offsets = append(offsets, lineStart)
|
offsets = append(offsets, lineStart)
|
||||||
violationCount += partial
|
attributions = append(attributions, lineAttributions...)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return steps, offsets, violationCount, info.ModTime(), nil
|
markViolations(steps, attributions)
|
||||||
|
return steps, offsets, len(attributions), info.ModTime(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Step decodes the full Step record at index n (1-based, matching trace.Step.Index).
|
// Step decodes the full Step record at index n (1-based, matching trace.Step.Index).
|
||||||
|
|||||||
@@ -56,7 +56,16 @@ func tallyTrace(tracePath string) (steps, violations int, err error) {
|
|||||||
return steps, violations, nil
|
return steps, violations, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func decodeStepSummary(line []byte) (StepSummary, int, error) {
|
// violationAttribution maps one recorded violation to the step the marker
|
||||||
|
// belongs on: the causing step its witness names, falling back to the step
|
||||||
|
// whose trace line carries it (the detection step) for witnesses written
|
||||||
|
// before the step field existed.
|
||||||
|
type violationAttribution struct {
|
||||||
|
attributedStep int
|
||||||
|
detectedStep int
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeStepSummary(line []byte) (StepSummary, []violationAttribution, error) {
|
||||||
var partial struct {
|
var partial struct {
|
||||||
Index int `json:"step"`
|
Index int `json:"step"`
|
||||||
Timestamp time.Time `json:"timestamp"`
|
Timestamp time.Time `json:"timestamp"`
|
||||||
@@ -76,15 +85,28 @@ func decodeStepSummary(line []byte) (StepSummary, int, error) {
|
|||||||
} `json:"next_action,omitempty"`
|
} `json:"next_action,omitempty"`
|
||||||
Exceptions []json.RawMessage `json:"exceptions,omitempty"`
|
Exceptions []json.RawMessage `json:"exceptions,omitempty"`
|
||||||
Violations []string `json:"violations,omitempty"`
|
Violations []string `json:"violations,omitempty"`
|
||||||
|
Witnesses map[string]struct {
|
||||||
|
Step int `json:"step"`
|
||||||
|
} `json:"witnesses,omitempty"`
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(line, &partial); err != nil {
|
if err := json.Unmarshal(line, &partial); err != nil {
|
||||||
return StepSummary{}, 0, fmt.Errorf("decode step: %w", err)
|
return StepSummary{}, nil, fmt.Errorf("decode step: %w", err)
|
||||||
|
}
|
||||||
|
var attributions []violationAttribution
|
||||||
|
for _, name := range partial.Violations {
|
||||||
|
attribution := violationAttribution{
|
||||||
|
attributedStep: partial.Index,
|
||||||
|
detectedStep: partial.Index,
|
||||||
|
}
|
||||||
|
if witness, ok := partial.Witnesses[name]; ok && witness.Step > 0 {
|
||||||
|
attribution.attributedStep = witness.Step
|
||||||
|
}
|
||||||
|
attributions = append(attributions, attribution)
|
||||||
}
|
}
|
||||||
summary := StepSummary{
|
summary := StepSummary{
|
||||||
Index: partial.Index,
|
Index: partial.Index,
|
||||||
Timestamp: partial.Timestamp,
|
Timestamp: partial.Timestamp,
|
||||||
Screen: partial.Screen,
|
Screen: partial.Screen,
|
||||||
HasViolations: len(partial.Violations) > 0,
|
|
||||||
HasExceptions: len(partial.Exceptions) > 0,
|
HasExceptions: len(partial.Exceptions) > 0,
|
||||||
}
|
}
|
||||||
if partial.NextAction != nil {
|
if partial.NextAction != nil {
|
||||||
@@ -113,7 +135,33 @@ func decodeStepSummary(line []byte) (StepSummary, int, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return summary, len(partial.Violations), nil
|
return summary, attributions, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// markViolations sets HasViolations on the step each attribution points at.
|
||||||
|
// The marker goes on the causing step; when that index is missing from the
|
||||||
|
// trace the detection step keeps it. Duplicate indices (a finalize line echoes
|
||||||
|
// the last step's index) resolve to the first occurrence, the real step.
|
||||||
|
func markViolations(steps []StepSummary, attributions []violationAttribution) {
|
||||||
|
if len(attributions) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
positionOf := make(map[int]int, len(steps))
|
||||||
|
for position, step := range steps {
|
||||||
|
if _, ok := positionOf[step.Index]; !ok {
|
||||||
|
positionOf[step.Index] = position
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, attribution := range attributions {
|
||||||
|
position, ok := positionOf[attribution.attributedStep]
|
||||||
|
if !ok {
|
||||||
|
position, ok = positionOf[attribution.detectedStep]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
steps[position].HasViolations = true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func swipeDirectionLabel(fromX, fromY, toX, toY int) string {
|
func swipeDirectionLabel(fromX, fromY, toX, toY int) string {
|
||||||
|
|||||||
@@ -138,6 +138,66 @@ func TestCacheStep_LazyDecodeReturnsFullStep(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCacheOpen_ViolationMarkerMovesToCausingStep(t *testing.T) {
|
||||||
|
// The violation is detected at step 3 but its witness attributes it to
|
||||||
|
// step 2 (the step that spawned the next obligation). The action-list
|
||||||
|
// marker belongs on step 2; the full step 3 payload keeps the violation
|
||||||
|
// record itself.
|
||||||
|
root := t.TempDir()
|
||||||
|
startedAt := time.Now().UTC()
|
||||||
|
steps := []trace.Step{
|
||||||
|
{Index: 1, Timestamp: startedAt},
|
||||||
|
{Index: 2, Timestamp: startedAt.Add(time.Second)},
|
||||||
|
{
|
||||||
|
Index: 3,
|
||||||
|
Timestamp: startedAt.Add(2 * time.Second),
|
||||||
|
Violations: []string{"prop1"},
|
||||||
|
Witnesses: map[string]trace.Witness{"prop1": {Reason: "predicate false", Step: 2}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
writeRun(t, root, "r1", trace.Meta{StartedAt: startedAt, EndedAt: timePointer(startedAt.Add(3 * time.Second))}, steps)
|
||||||
|
|
||||||
|
cache := NewCache(root)
|
||||||
|
run, err := cache.Open("r1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Open: %v", err)
|
||||||
|
}
|
||||||
|
if !run.Steps[1].HasViolations {
|
||||||
|
t.Error("step 2 (causing step) should carry the violation marker")
|
||||||
|
}
|
||||||
|
if run.Steps[2].HasViolations {
|
||||||
|
t.Error("step 3 (detection step) should not carry the marker")
|
||||||
|
}
|
||||||
|
full, err := cache.Step(run, 3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Step(3): %v", err)
|
||||||
|
}
|
||||||
|
if len(full.Violations) != 1 || full.Violations[0] != "prop1" {
|
||||||
|
t.Errorf("step 3 payload violations = %v, want [prop1]", full.Violations)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCacheOpen_ViolationWithoutWitnessStepKeepsDetectionStep(t *testing.T) {
|
||||||
|
// Traces written before witnesses carried a step field fall back to
|
||||||
|
// marking the detection step, the old behavior.
|
||||||
|
root := t.TempDir()
|
||||||
|
startedAt := time.Now().UTC()
|
||||||
|
steps := []trace.Step{
|
||||||
|
{Index: 1, Timestamp: startedAt},
|
||||||
|
{Index: 2, Timestamp: startedAt.Add(time.Second), Violations: []string{"prop1"}},
|
||||||
|
}
|
||||||
|
writeRun(t, root, "r1", trace.Meta{StartedAt: startedAt, EndedAt: timePointer(startedAt.Add(2 * time.Second))}, steps)
|
||||||
|
|
||||||
|
cache := NewCache(root)
|
||||||
|
run, err := cache.Open("r1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Open: %v", err)
|
||||||
|
}
|
||||||
|
if !run.Steps[1].HasViolations {
|
||||||
|
t.Error("step 2 should keep the marker when the witness has no step")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDecodeStepSummary_ActionLabelPerKind(t *testing.T) {
|
func TestDecodeStepSummary_ActionLabelPerKind(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
line string
|
line string
|
||||||
|
|||||||
+36
-14
@@ -8,6 +8,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"maps"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -185,6 +187,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
|||||||
Tree: tree,
|
Tree: tree,
|
||||||
LastAction: lastAction,
|
LastAction: lastAction,
|
||||||
StepTime: stepStart,
|
StepTime: stepStart,
|
||||||
|
StepIndex: stepIndex,
|
||||||
RunStart: summary.StartTime,
|
RunStart: summary.StartTime,
|
||||||
Logs: logs,
|
Logs: logs,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
@@ -263,10 +266,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
|||||||
}
|
}
|
||||||
summary.Steps = stepIndex
|
summary.Steps = stepIndex
|
||||||
if len(violations) > 0 {
|
if len(violations) > 0 {
|
||||||
summary.Violations = append(summary.Violations, ViolationRecord{
|
summary.Violations = append(summary.Violations, violationRecords(violations, witnesses, stepIndex)...)
|
||||||
StepIndex: stepIndex,
|
|
||||||
Properties: violations,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
// Wait actions are themselves a settling: skip the idle poll. Actions
|
// Wait actions are themselves a settling: skip the idle poll. Actions
|
||||||
// that mutate the UI fall through to WaitForIdle so the next step's
|
// that mutate the UI fall through to WaitForIdle so the next step's
|
||||||
@@ -284,17 +284,17 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Finalize each evaluator once the loop ends so liveness obligations that
|
// Finalize each evaluator once the loop ends so liveness obligations that
|
||||||
// never discharged (an unbounded eventually that never fired, a strong
|
// never discharged (an eventually that never fired) are reported as
|
||||||
// next with no successor) are reported as violations rather than silently
|
// violations rather than silently left pending. Properties already
|
||||||
// left pending. Properties already violated mid-run are not re-reported.
|
// violated mid-run are not re-reported. The synthetic record gets its own
|
||||||
|
// step index so no two trace lines share one; witnesses still attribute
|
||||||
|
// the violation to the step that spawned the obligation.
|
||||||
if ended := options.Verifier.Finalize(); len(ended) > 0 {
|
if ended := options.Verifier.Finalize(); len(ended) > 0 {
|
||||||
witnesses := collectWitnesses(options.Verifier, ended, logger, stepIndex)
|
finalIndex := stepIndex + 1
|
||||||
summary.Violations = append(summary.Violations, ViolationRecord{
|
witnesses := collectWitnesses(options.Verifier, ended, logger, finalIndex)
|
||||||
StepIndex: stepIndex,
|
summary.Violations = append(summary.Violations, violationRecords(ended, witnesses, finalIndex)...)
|
||||||
Properties: ended,
|
|
||||||
})
|
|
||||||
finalStep := trace.Step{
|
finalStep := trace.Step{
|
||||||
Index: stepIndex,
|
Index: finalIndex,
|
||||||
Timestamp: time.Now(),
|
Timestamp: time.Now(),
|
||||||
Violations: ended,
|
Violations: ended,
|
||||||
Witnesses: witnesses,
|
Witnesses: witnesses,
|
||||||
@@ -806,6 +806,26 @@ func captureMetrics(ctx context.Context, options Options, logger *slog.Logger, s
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// violationRecords groups newly-violated properties by the step their witness
|
||||||
|
// attributes the violation to (the causing step), falling back to the
|
||||||
|
// detection step for properties without a witness. Records are ordered by
|
||||||
|
// step; properties keep the sorted order NewlyViolatedProperties produced.
|
||||||
|
func violationRecords(properties []string, witnesses map[string]trace.Witness, detectionStep int) []ViolationRecord {
|
||||||
|
byStep := map[int][]string{}
|
||||||
|
for _, name := range properties {
|
||||||
|
step := detectionStep
|
||||||
|
if witness, ok := witnesses[name]; ok && witness.Step > 0 {
|
||||||
|
step = witness.Step
|
||||||
|
}
|
||||||
|
byStep[step] = append(byStep[step], name)
|
||||||
|
}
|
||||||
|
records := make([]ViolationRecord, 0, len(byStep))
|
||||||
|
for _, step := range slices.Sorted(maps.Keys(byStep)) {
|
||||||
|
records = append(records, ViolationRecord{StepIndex: step, Properties: byStep[step]})
|
||||||
|
}
|
||||||
|
return records
|
||||||
|
}
|
||||||
|
|
||||||
// collectWitnesses gathers the violation witness for each newly-violated
|
// collectWitnesses gathers the violation witness for each newly-violated
|
||||||
// property, logs its cause, and returns them keyed by property name for the
|
// property, logs its cause, and returns them keyed by property name for the
|
||||||
// trace. Properties without a captured witness are skipped.
|
// trace. Properties without a captured witness are skipped.
|
||||||
@@ -820,10 +840,12 @@ func collectWitnesses(verifierInstance *verifier.Verifier, properties []string,
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
logger.Warn("property violated",
|
logger.Warn("property violated",
|
||||||
"step", stepIndex, "property", name, "reason", witness.Reason, "error", witness.IsError)
|
"step", witness.Step, "detected_step", stepIndex,
|
||||||
|
"property", name, "reason", witness.Reason, "error", witness.IsError)
|
||||||
witnesses[name] = trace.Witness{
|
witnesses[name] = trace.Witness{
|
||||||
Reason: witness.Reason,
|
Reason: witness.Reason,
|
||||||
IsError: witness.IsError,
|
IsError: witness.IsError,
|
||||||
|
Step: witness.Step,
|
||||||
Extractors: witness.Extractors,
|
Extractors: witness.Extractors,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -253,6 +253,186 @@ func TestRunner_ViolationSurfacesOnlyOnOnsetStep(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunner_NextViolationAttributedToCausingStep(t *testing.T) {
|
||||||
|
// always(next(p)): the obligation spawned at step 2 fails against step 3's
|
||||||
|
// state. The summary record and the trace witness must attribute the
|
||||||
|
// violation to step 2 (the causing step); the trace line that carries it is
|
||||||
|
// still step 3, where the failure was detected.
|
||||||
|
const nextViolationSpec = `
|
||||||
|
import { actions, always, next, extract } from "@sanderling/spec";
|
||||||
|
let observed = 0;
|
||||||
|
const tick = extract(() => ++observed);
|
||||||
|
globalThis.properties = {
|
||||||
|
nextHolds: always(next(() => tick.current < 3)),
|
||||||
|
};
|
||||||
|
globalThis.actions = actions(() => []);
|
||||||
|
`
|
||||||
|
state := newHarnessWithSpec(t, nextViolationSpec)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
summary, err := Run(ctx, Options{
|
||||||
|
Duration: time.Hour,
|
||||||
|
IdleTimeout: 20 * time.Millisecond,
|
||||||
|
MaxSteps: 5,
|
||||||
|
Driver: state.mock,
|
||||||
|
Verifier: state.verifier,
|
||||||
|
TraceWriter: state.writer,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run: %v", err)
|
||||||
|
}
|
||||||
|
if len(summary.Violations) != 1 {
|
||||||
|
t.Fatalf("expected exactly one ViolationRecord, got %d: %v",
|
||||||
|
len(summary.Violations), summary.Violations)
|
||||||
|
}
|
||||||
|
if summary.Violations[0].StepIndex != 2 {
|
||||||
|
t.Errorf("summary step: got %d, want 2 (the step that spawned the next obligation)",
|
||||||
|
summary.Violations[0].StepIndex)
|
||||||
|
}
|
||||||
|
if !slices.Equal(summary.Violations[0].Properties, []string{"nextHolds"}) {
|
||||||
|
t.Errorf("properties: got %v, want [nextHolds]", summary.Violations[0].Properties)
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := os.Open(filepath.Join(state.writer.Directory(), "trace.jsonl"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
type traceLine struct {
|
||||||
|
Step int `json:"step"`
|
||||||
|
Violations []string `json:"violations"`
|
||||||
|
Witnesses map[string]trace.Witness `json:"witnesses"`
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
scanner := bufio.NewScanner(file)
|
||||||
|
scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
|
||||||
|
for scanner.Scan() {
|
||||||
|
var line traceLine
|
||||||
|
if err := json.Unmarshal(scanner.Bytes(), &line); err != nil {
|
||||||
|
t.Fatalf("trace line decode: %v", err)
|
||||||
|
}
|
||||||
|
if len(line.Violations) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
found = true
|
||||||
|
if line.Step != 3 {
|
||||||
|
t.Errorf("violation detected on step %d, want 3", line.Step)
|
||||||
|
}
|
||||||
|
witness, ok := line.Witnesses["nextHolds"]
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("step %d carries no witness for nextHolds", line.Step)
|
||||||
|
}
|
||||||
|
if witness.Step != 2 {
|
||||||
|
t.Errorf("witness step: got %d, want 2 (causing step)", witness.Step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
t.Fatalf("scan trace: %v", err)
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Error("no trace line carried the violation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunner_AlwaysNextLeavesNoEndOfRunViolation(t *testing.T) {
|
||||||
|
// always(next(p)) ends every run with a pending deferred check. That
|
||||||
|
// residue is vacuous (no successor state to check), so neither the
|
||||||
|
// summary nor the trace may report an end-of-run violation.
|
||||||
|
const spec = `
|
||||||
|
import { actions, always, next } from "@sanderling/spec";
|
||||||
|
globalThis.properties = {
|
||||||
|
nextHolds: always(next(() => true)),
|
||||||
|
};
|
||||||
|
globalThis.actions = actions(() => []);
|
||||||
|
`
|
||||||
|
state := newHarnessWithSpec(t, spec)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
summary, err := Run(ctx, Options{
|
||||||
|
Duration: time.Hour,
|
||||||
|
IdleTimeout: 20 * time.Millisecond,
|
||||||
|
MaxSteps: 3,
|
||||||
|
Driver: state.mock,
|
||||||
|
Verifier: state.verifier,
|
||||||
|
TraceWriter: state.writer,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run: %v", err)
|
||||||
|
}
|
||||||
|
if len(summary.Violations) != 0 {
|
||||||
|
t.Errorf("expected no violations, got %v", summary.Violations)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunner_FinalizeRecordUsesDistinctStepIndex(t *testing.T) {
|
||||||
|
// An eventually that never fires is reported at run end through a
|
||||||
|
// synthetic trace record. That record must carry its own step index so no
|
||||||
|
// two trace lines share one (duplicate indices made the replay UI select
|
||||||
|
// two rows at once).
|
||||||
|
const spec = `
|
||||||
|
import { actions, eventually } from "@sanderling/spec";
|
||||||
|
globalThis.properties = {
|
||||||
|
neverFires: eventually(() => false),
|
||||||
|
};
|
||||||
|
globalThis.actions = actions(() => []);
|
||||||
|
`
|
||||||
|
state := newHarnessWithSpec(t, spec)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
summary, err := Run(ctx, Options{
|
||||||
|
Duration: time.Hour,
|
||||||
|
IdleTimeout: 20 * time.Millisecond,
|
||||||
|
MaxSteps: 3,
|
||||||
|
Driver: state.mock,
|
||||||
|
Verifier: state.verifier,
|
||||||
|
TraceWriter: state.writer,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run: %v", err)
|
||||||
|
}
|
||||||
|
if !containsProperty(summary.Violations, "neverFires") {
|
||||||
|
t.Fatalf("expected neverFires in violations: %v", summary.Violations)
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := os.Open(filepath.Join(state.writer.Directory(), "trace.jsonl"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
type traceLine struct {
|
||||||
|
Step int `json:"step"`
|
||||||
|
Violations []string `json:"violations"`
|
||||||
|
}
|
||||||
|
seen := map[int]bool{}
|
||||||
|
finalizeStep := 0
|
||||||
|
scanner := bufio.NewScanner(file)
|
||||||
|
scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
|
||||||
|
for scanner.Scan() {
|
||||||
|
var line traceLine
|
||||||
|
if err := json.Unmarshal(scanner.Bytes(), &line); err != nil {
|
||||||
|
t.Fatalf("trace line decode: %v", err)
|
||||||
|
}
|
||||||
|
if seen[line.Step] {
|
||||||
|
t.Errorf("duplicate step index %d in trace", line.Step)
|
||||||
|
}
|
||||||
|
seen[line.Step] = true
|
||||||
|
if slices.Contains(line.Violations, "neverFires") {
|
||||||
|
finalizeStep = line.Step
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
t.Fatalf("scan trace: %v", err)
|
||||||
|
}
|
||||||
|
if finalizeStep != summary.Steps+1 {
|
||||||
|
t.Errorf("finalize record step = %d, want %d (steps+1)", finalizeStep, summary.Steps+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunner_ThrowingPredicateIsLoggedNotPanic(t *testing.T) {
|
func TestRunner_ThrowingPredicateIsLoggedNotPanic(t *testing.T) {
|
||||||
const throwingSpec = `
|
const throwingSpec = `
|
||||||
import { actions, always, Tap } from "@sanderling/spec";
|
import { actions, always, Tap } from "@sanderling/spec";
|
||||||
|
|||||||
@@ -43,8 +43,13 @@ type Step struct {
|
|||||||
// Witness is the trace-side record of a property violation: why it fired and a
|
// Witness is the trace-side record of a property violation: why it fired and a
|
||||||
// snapshot of every extractor's value at the violating step.
|
// snapshot of every extractor's value at the violating step.
|
||||||
type Witness struct {
|
type Witness struct {
|
||||||
Reason string `json:"reason,omitempty"`
|
Reason string `json:"reason,omitempty"`
|
||||||
IsError bool `json:"is_error,omitempty"`
|
IsError bool `json:"is_error,omitempty"`
|
||||||
|
// Step is the step the failed obligation originated at: the step that
|
||||||
|
// caused the violation. For a deferred obligation (a next, an eventually)
|
||||||
|
// this is earlier than the step whose record carries the witness, which is
|
||||||
|
// where the failure was detected.
|
||||||
|
Step int `json:"step,omitempty"`
|
||||||
Extractors map[string]json.RawMessage `json:"extractors,omitempty"`
|
Extractors map[string]json.RawMessage `json:"extractors,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -782,6 +782,46 @@ globalThis.properties = {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A next obligation spawned at step k and checked against step k+1's state is
|
||||||
|
// attributed to step k, the step that caused it. StepIndex from SnapshotInput
|
||||||
|
// labels the evaluator observations so the witness carries runner step numbers.
|
||||||
|
func TestEvaluateProperties_NextViolationCarriesOriginStepIndex(t *testing.T) {
|
||||||
|
const spec = `
|
||||||
|
globalThis.flag = __sanderling__.extract(state => state.snapshots["flag"] ?? false, "flag");
|
||||||
|
globalThis.properties = {
|
||||||
|
flagStaysTrue: __sanderling__.always(__sanderling__.next(() => flag.current === true)),
|
||||||
|
};
|
||||||
|
`
|
||||||
|
verifier := newVerifier(t)
|
||||||
|
mustLoad(t, verifier, spec)
|
||||||
|
|
||||||
|
push := func(stepIndex int, flag string) map[string]ltl.Verdict {
|
||||||
|
t.Helper()
|
||||||
|
if err := verifier.PushSnapshot(SnapshotInput{
|
||||||
|
Snapshots: Snapshots{"flag": json.RawMessage(flag)},
|
||||||
|
StepIndex: stepIndex,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return verifier.EvaluateProperties()
|
||||||
|
}
|
||||||
|
|
||||||
|
push(7, `true`)
|
||||||
|
push(8, `true`)
|
||||||
|
verdicts := push(9, `false`)
|
||||||
|
if got := verdicts["flagStaysTrue"]; got != ltl.VerdictViolated {
|
||||||
|
t.Fatalf("step 9: verdict = %v, want violated", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
witness := verifier.Witness("flagStaysTrue")
|
||||||
|
if witness == nil {
|
||||||
|
t.Fatal("Witness = nil, want non-nil")
|
||||||
|
}
|
||||||
|
if witness.Step != 8 {
|
||||||
|
t.Errorf("Witness.Step = %d, want 8 (the step that spawned the next obligation)", witness.Step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoad_AcceptsSpecWithoutPropertiesOrActions(t *testing.T) {
|
func TestLoad_AcceptsSpecWithoutPropertiesOrActions(t *testing.T) {
|
||||||
verifier := newVerifier(t)
|
verifier := newVerifier(t)
|
||||||
if err := verifier.Load(`const noop = 1;`); err != nil {
|
if err := verifier.Load(`const noop = 1;`); err != nil {
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ type Verifier struct {
|
|||||||
lastLogs []LogEntry
|
lastLogs []LogEntry
|
||||||
lastExceptions []Exception
|
lastExceptions []Exception
|
||||||
stepTime time.Time
|
stepTime time.Time
|
||||||
|
stepIndex int
|
||||||
runStart time.Time
|
runStart time.Time
|
||||||
|
|
||||||
appPackage string
|
appPackage string
|
||||||
@@ -259,6 +260,7 @@ func (v *Verifier) PushSnapshot(input SnapshotInput) error {
|
|||||||
v.lastLogs = input.Logs
|
v.lastLogs = input.Logs
|
||||||
v.lastExceptions = input.Exceptions
|
v.lastExceptions = input.Exceptions
|
||||||
v.stepTime = input.StepTime
|
v.stepTime = input.StepTime
|
||||||
|
v.stepIndex = input.StepIndex
|
||||||
if v.runStart.IsZero() {
|
if v.runStart.IsZero() {
|
||||||
v.runStart = input.RunStart
|
v.runStart = input.RunStart
|
||||||
}
|
}
|
||||||
@@ -385,6 +387,11 @@ type SnapshotInput struct {
|
|||||||
Tree *hierarchy.Tree
|
Tree *hierarchy.Tree
|
||||||
LastAction *Action
|
LastAction *Action
|
||||||
StepTime time.Time
|
StepTime time.Time
|
||||||
|
// StepIndex is the runner's step number for this snapshot. Evaluators label
|
||||||
|
// observations with it so violation witnesses carry runner step numbers even
|
||||||
|
// when transitional steps were skipped. Zero means unlabeled; evaluators
|
||||||
|
// then fall back to their internal counter.
|
||||||
|
StepIndex int
|
||||||
RunStart time.Time
|
RunStart time.Time
|
||||||
Logs []LogEntry
|
Logs []LogEntry
|
||||||
Exceptions []Exception
|
Exceptions []Exception
|
||||||
@@ -404,7 +411,11 @@ func (v *Verifier) EvaluateProperties() map[string]ltl.Verdict {
|
|||||||
stepTime = time.Now()
|
stepTime = time.Now()
|
||||||
}
|
}
|
||||||
for name, evaluator := range v.evaluators {
|
for name, evaluator := range v.evaluators {
|
||||||
verdicts[name] = evaluator.ObserveAt(stepTime)
|
if v.stepIndex > 0 {
|
||||||
|
verdicts[name] = evaluator.ObserveAtStep(stepTime, v.stepIndex)
|
||||||
|
} else {
|
||||||
|
verdicts[name] = evaluator.ObserveAt(stepTime)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var onset []string
|
var onset []string
|
||||||
|
|||||||
@@ -97,6 +97,57 @@
|
|||||||
font-size: 13px;
|
font-size: 13px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.violations-panel-witness {
|
||||||
|
margin-top: 6px;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 4px;
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.violations-panel-witness-line {
|
||||||
|
display: flex;
|
||||||
|
align-items: baseline;
|
||||||
|
gap: 8px;
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.violations-panel-witness-key {
|
||||||
|
flex-shrink: 0;
|
||||||
|
min-width: 96px;
|
||||||
|
color: var(--text-muted);
|
||||||
|
font-size: 11px;
|
||||||
|
text-transform: lowercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
.violations-panel-witness-value {
|
||||||
|
word-break: break-all;
|
||||||
|
white-space: pre-wrap;
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.violations-panel-witness-step {
|
||||||
|
font-family: var(--font-mono);
|
||||||
|
font-size: 11px;
|
||||||
|
padding: 1px 6px;
|
||||||
|
background: var(--surface);
|
||||||
|
color: var(--text-primary);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: 3px;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
|
||||||
|
.violations-panel-witness-step:hover {
|
||||||
|
border-color: var(--border-strong);
|
||||||
|
}
|
||||||
|
|
||||||
|
.violations-panel-witness-evidence {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 3px;
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
|
||||||
.violations-panel-residual {
|
.violations-panel-residual {
|
||||||
margin-top: 6px;
|
margin-top: 6px;
|
||||||
font-size: 12px;
|
font-size: 12px;
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { useMemo } from "react";
|
import { useMemo } from "react";
|
||||||
import type { ResidualNode } from "../types";
|
import type { ResidualNode, Witness } from "../types";
|
||||||
import ResidualNodeView from "../components/ResidualNode";
|
import ResidualNodeView from "../components/ResidualNode";
|
||||||
import "./ViolationsPanel.css";
|
import "./ViolationsPanel.css";
|
||||||
|
|
||||||
@@ -7,8 +7,10 @@ export interface ViolationsPanelProps {
|
|||||||
propertyNames: string[];
|
propertyNames: string[];
|
||||||
violations: string[];
|
violations: string[];
|
||||||
residuals?: Record<string, ResidualNode>;
|
residuals?: Record<string, ResidualNode>;
|
||||||
|
witnesses?: Record<string, Witness>;
|
||||||
onJumpToFirstViolation: () => void;
|
onJumpToFirstViolation: () => void;
|
||||||
hasFirstViolation: boolean;
|
hasFirstViolation: boolean;
|
||||||
|
onJumpToStep?: (step: number) => void;
|
||||||
/** When true, only render violated rows and hide the header button row. */
|
/** When true, only render violated rows and hide the header button row. */
|
||||||
violationsOnly?: boolean;
|
violationsOnly?: boolean;
|
||||||
}
|
}
|
||||||
@@ -36,12 +38,74 @@ function statusFor(
|
|||||||
return "pending";
|
return "pending";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatValue(value: unknown): string {
|
||||||
|
const encoded = JSON.stringify(value);
|
||||||
|
return encoded === undefined ? String(value) : encoded;
|
||||||
|
}
|
||||||
|
|
||||||
|
function WitnessView({
|
||||||
|
witness,
|
||||||
|
onJumpToStep,
|
||||||
|
open,
|
||||||
|
}: {
|
||||||
|
witness: Witness;
|
||||||
|
onJumpToStep?: (step: number) => void;
|
||||||
|
open: boolean;
|
||||||
|
}) {
|
||||||
|
const evidence = Object.entries(witness.extractors ?? {})
|
||||||
|
.filter(([, value]) => value !== null && value !== undefined)
|
||||||
|
.sort(([a], [b]) => a.localeCompare(b));
|
||||||
|
return (
|
||||||
|
<div className="violations-panel-witness">
|
||||||
|
{witness.reason ? (
|
||||||
|
<div className="violations-panel-witness-line">
|
||||||
|
<span className="violations-panel-witness-key">
|
||||||
|
{witness.is_error ? "error" : "reason"}
|
||||||
|
</span>
|
||||||
|
<span className="violations-panel-witness-value">{witness.reason}</span>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
{witness.step ? (
|
||||||
|
<div className="violations-panel-witness-line">
|
||||||
|
<span className="violations-panel-witness-key">caused at</span>
|
||||||
|
{onJumpToStep ? (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="violations-panel-witness-step"
|
||||||
|
onClick={() => onJumpToStep(witness.step as number)}
|
||||||
|
>
|
||||||
|
step {witness.step}
|
||||||
|
</button>
|
||||||
|
) : (
|
||||||
|
<span className="violations-panel-witness-value">step {witness.step}</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
{evidence.length > 0 ? (
|
||||||
|
<details className="violations-panel-residual" open={open}>
|
||||||
|
<summary>witness</summary>
|
||||||
|
<dl className="violations-panel-witness-evidence">
|
||||||
|
{evidence.map(([name, value]) => (
|
||||||
|
<div key={name} className="violations-panel-witness-line">
|
||||||
|
<dt className="violations-panel-witness-key">{name}</dt>
|
||||||
|
<dd className="violations-panel-witness-value">{formatValue(value)}</dd>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</dl>
|
||||||
|
</details>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export default function ViolationsPanel({
|
export default function ViolationsPanel({
|
||||||
propertyNames,
|
propertyNames,
|
||||||
violations,
|
violations,
|
||||||
residuals,
|
residuals,
|
||||||
|
witnesses,
|
||||||
onJumpToFirstViolation,
|
onJumpToFirstViolation,
|
||||||
hasFirstViolation,
|
hasFirstViolation,
|
||||||
|
onJumpToStep,
|
||||||
violationsOnly = false,
|
violationsOnly = false,
|
||||||
}: ViolationsPanelProps) {
|
}: ViolationsPanelProps) {
|
||||||
const violationSet = useMemo(() => new Set(violations), [violations]);
|
const violationSet = useMemo(() => new Set(violations), [violations]);
|
||||||
@@ -84,6 +148,7 @@ export default function ViolationsPanel({
|
|||||||
<ul className="violations-panel-list">
|
<ul className="violations-panel-list">
|
||||||
{rows.map(({ name, status }) => {
|
{rows.map(({ name, status }) => {
|
||||||
const residual = residuals?.[name];
|
const residual = residuals?.[name];
|
||||||
|
const witness = status === "violated" ? witnesses?.[name] : undefined;
|
||||||
return (
|
return (
|
||||||
<li key={name} className="violations-panel-row" data-status={status}>
|
<li key={name} className="violations-panel-row" data-status={status}>
|
||||||
<div className="violations-panel-row-head">
|
<div className="violations-panel-row-head">
|
||||||
@@ -96,6 +161,9 @@ export default function ViolationsPanel({
|
|||||||
</span>
|
</span>
|
||||||
<span className="violations-panel-name">{name}</span>
|
<span className="violations-panel-name">{name}</span>
|
||||||
</div>
|
</div>
|
||||||
|
{witness ? (
|
||||||
|
<WitnessView witness={witness} onJumpToStep={onJumpToStep} open={violationsOnly} />
|
||||||
|
) : null}
|
||||||
{residual ? (
|
{residual ? (
|
||||||
<details className="violations-panel-residual">
|
<details className="violations-panel-residual">
|
||||||
<summary>residual</summary>
|
<summary>residual</summary>
|
||||||
|
|||||||
@@ -126,6 +126,8 @@ export default function RunDetail() {
|
|||||||
const violationsAfter = nextStep?.violations ?? violationsBefore;
|
const violationsAfter = nextStep?.violations ?? violationsBefore;
|
||||||
const residualsBefore = currentStep?.residuals;
|
const residualsBefore = currentStep?.residuals;
|
||||||
const residualsAfter = nextStep?.residuals ?? residualsBefore;
|
const residualsAfter = nextStep?.residuals ?? residualsBefore;
|
||||||
|
const witnessesBefore = currentStep?.witnesses;
|
||||||
|
const witnessesAfter = nextStep?.witnesses ?? witnessesBefore;
|
||||||
const exceptionsForStep = currentStep?.exceptions;
|
const exceptionsForStep = currentStep?.exceptions;
|
||||||
|
|
||||||
const beforeTabs: TabDefinition[] = [
|
const beforeTabs: TabDefinition[] = [
|
||||||
@@ -157,8 +159,10 @@ export default function RunDetail() {
|
|||||||
propertyNames={history?.names ?? []}
|
propertyNames={history?.names ?? []}
|
||||||
violations={violationsBefore}
|
violations={violationsBefore}
|
||||||
residuals={residualsBefore}
|
residuals={residualsBefore}
|
||||||
|
witnesses={witnessesBefore}
|
||||||
onJumpToFirstViolation={jumpToFirstViolation}
|
onJumpToFirstViolation={jumpToFirstViolation}
|
||||||
hasFirstViolation={history?.firstViolationStep !== undefined}
|
hasFirstViolation={history?.firstViolationStep !== undefined}
|
||||||
|
onJumpToStep={goTo}
|
||||||
/>
|
/>
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
@@ -176,8 +180,10 @@ export default function RunDetail() {
|
|||||||
propertyNames={history?.names ?? []}
|
propertyNames={history?.names ?? []}
|
||||||
violations={violationsBefore}
|
violations={violationsBefore}
|
||||||
residuals={residualsBefore}
|
residuals={residualsBefore}
|
||||||
|
witnesses={witnessesBefore}
|
||||||
onJumpToFirstViolation={jumpToFirstViolation}
|
onJumpToFirstViolation={jumpToFirstViolation}
|
||||||
hasFirstViolation={history?.firstViolationStep !== undefined}
|
hasFirstViolation={history?.firstViolationStep !== undefined}
|
||||||
|
onJumpToStep={goTo}
|
||||||
violationsOnly
|
violationsOnly
|
||||||
/>
|
/>
|
||||||
),
|
),
|
||||||
@@ -213,8 +219,10 @@ export default function RunDetail() {
|
|||||||
propertyNames={history?.names ?? []}
|
propertyNames={history?.names ?? []}
|
||||||
violations={violationsAfter}
|
violations={violationsAfter}
|
||||||
residuals={residualsAfter}
|
residuals={residualsAfter}
|
||||||
|
witnesses={witnessesAfter}
|
||||||
onJumpToFirstViolation={jumpToFirstViolation}
|
onJumpToFirstViolation={jumpToFirstViolation}
|
||||||
hasFirstViolation={history?.firstViolationStep !== undefined}
|
hasFirstViolation={history?.firstViolationStep !== undefined}
|
||||||
|
onJumpToStep={goTo}
|
||||||
/>
|
/>
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
@@ -232,8 +240,10 @@ export default function RunDetail() {
|
|||||||
propertyNames={history?.names ?? []}
|
propertyNames={history?.names ?? []}
|
||||||
violations={violationsAfter}
|
violations={violationsAfter}
|
||||||
residuals={residualsAfter}
|
residuals={residualsAfter}
|
||||||
|
witnesses={witnessesAfter}
|
||||||
onJumpToFirstViolation={jumpToFirstViolation}
|
onJumpToFirstViolation={jumpToFirstViolation}
|
||||||
hasFirstViolation={history?.firstViolationStep !== undefined}
|
hasFirstViolation={history?.firstViolationStep !== undefined}
|
||||||
|
onJumpToStep={goTo}
|
||||||
violationsOnly
|
violationsOnly
|
||||||
/>
|
/>
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -121,6 +121,16 @@ export interface ExtractorChange {
|
|||||||
curr: unknown;
|
curr: unknown;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface Witness {
|
||||||
|
reason?: string;
|
||||||
|
is_error?: boolean;
|
||||||
|
// step is the step the failed obligation originated at: the causing step,
|
||||||
|
// which for deferred obligations (next, eventually) is earlier than the
|
||||||
|
// step whose record carries the witness.
|
||||||
|
step?: number;
|
||||||
|
extractors?: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
export interface Step {
|
export interface Step {
|
||||||
step: number;
|
step: number;
|
||||||
timestamp: string;
|
timestamp: string;
|
||||||
@@ -134,4 +144,5 @@ export interface Step {
|
|||||||
residuals?: Record<string, ResidualNode>;
|
residuals?: Record<string, ResidualNode>;
|
||||||
metrics?: Metrics;
|
metrics?: Metrics;
|
||||||
extractor_changes?: Record<string, ExtractorChange>;
|
extractor_changes?: Record<string, ExtractorChange>;
|
||||||
|
witnesses?: Record<string, Witness>;
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user