feat(verifier): label evaluator observations with the runner step index

This commit is contained in:
pj committed 2026-06-05 22:31:05 +05:30
1 parent dc7d23eaa2
commit ff06fd9b1d
3 files changed
+53 -1

No files matched your search

+1
View File
@@ -185,6 +185,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
Tree: tree, Tree: tree,
LastAction: lastAction, LastAction: lastAction,
StepTime: stepStart, StepTime: stepStart,
StepIndex: stepIndex,
RunStart: summary.StartTime, RunStart: summary.StartTime,
Logs: logs, Logs: logs,
}); err != nil { }); err != nil {
+40
View File
@@ -782,6 +782,46 @@ globalThis.properties = {
} }
} }
// A next obligation spawned at step k and checked against step k+1's state is
// attributed to step k, the step that caused it. StepIndex from SnapshotInput
// labels the evaluator observations so the witness carries runner step numbers.
func TestEvaluateProperties_NextViolationCarriesOriginStepIndex(t *testing.T) {
const spec = `
globalThis.flag = __sanderling__.extract(state => state.snapshots["flag"] ?? false, "flag");
globalThis.properties = {
flagStaysTrue: __sanderling__.always(__sanderling__.next(() => flag.current === true)),
};
`
verifier := newVerifier(t)
mustLoad(t, verifier, spec)
push := func(stepIndex int, flag string) map[string]ltl.Verdict {
t.Helper()
if err := verifier.PushSnapshot(SnapshotInput{
Snapshots: Snapshots{"flag": json.RawMessage(flag)},
StepIndex: stepIndex,
}); err != nil {
t.Fatal(err)
}
return verifier.EvaluateProperties()
}
push(7, `true`)
push(8, `true`)
verdicts := push(9, `false`)
if got := verdicts["flagStaysTrue"]; got != ltl.VerdictViolated {
t.Fatalf("step 9: verdict = %v, want violated", got)
}
witness := verifier.Witness("flagStaysTrue")
if witness == nil {
t.Fatal("Witness = nil, want non-nil")
}
if witness.Step != 8 {
t.Errorf("Witness.Step = %d, want 8 (the step that spawned the next obligation)", witness.Step)
}
}
func TestLoad_AcceptsSpecWithoutPropertiesOrActions(t *testing.T) { func TestLoad_AcceptsSpecWithoutPropertiesOrActions(t *testing.T) {
verifier := newVerifier(t) verifier := newVerifier(t)
if err := verifier.Load(`const noop = 1;`); err != nil { if err := verifier.Load(`const noop = 1;`); err != nil {
+12 -1
View File
@@ -39,6 +39,7 @@ type Verifier struct {
lastLogs []LogEntry lastLogs []LogEntry
lastExceptions []Exception lastExceptions []Exception
stepTime time.Time stepTime time.Time
stepIndex int
runStart time.Time runStart time.Time
appPackage string appPackage string
@@ -259,6 +260,7 @@ func (v *Verifier) PushSnapshot(input SnapshotInput) error {
v.lastLogs = input.Logs v.lastLogs = input.Logs
v.lastExceptions = input.Exceptions v.lastExceptions = input.Exceptions
v.stepTime = input.StepTime v.stepTime = input.StepTime
v.stepIndex = input.StepIndex
if v.runStart.IsZero() { if v.runStart.IsZero() {
v.runStart = input.RunStart v.runStart = input.RunStart
} }
@@ -385,6 +387,11 @@ type SnapshotInput struct {
Tree *hierarchy.Tree Tree *hierarchy.Tree
LastAction *Action LastAction *Action
StepTime time.Time StepTime time.Time
// StepIndex is the runner's step number for this snapshot. Evaluators label
// observations with it so violation witnesses carry runner step numbers even
// when transitional steps were skipped. Zero means unlabeled; evaluators
// then fall back to their internal counter.
StepIndex int
RunStart time.Time RunStart time.Time
Logs []LogEntry Logs []LogEntry
Exceptions []Exception Exceptions []Exception
@@ -404,7 +411,11 @@ func (v *Verifier) EvaluateProperties() map[string]ltl.Verdict {
stepTime = time.Now() stepTime = time.Now()
} }
for name, evaluator := range v.evaluators { for name, evaluator := range v.evaluators {
verdicts[name] = evaluator.ObserveAt(stepTime) if v.stepIndex > 0 {
verdicts[name] = evaluator.ObserveAtStep(stepTime, v.stepIndex)
} else {
verdicts[name] = evaluator.ObserveAt(stepTime)
}
} }
var onset []string var onset []string