mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
feat(verifier): label evaluator observations with the runner step index
This commit is contained in:
1 parent
dc7d23eaa2
commit
ff06fd9b1d
3 files changed
+53
-1
No files matched your search
@@ -185,6 +185,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
|||||||
Tree: tree,
|
Tree: tree,
|
||||||
LastAction: lastAction,
|
LastAction: lastAction,
|
||||||
StepTime: stepStart,
|
StepTime: stepStart,
|
||||||
|
StepIndex: stepIndex,
|
||||||
RunStart: summary.StartTime,
|
RunStart: summary.StartTime,
|
||||||
Logs: logs,
|
Logs: logs,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
|
|||||||
@@ -782,6 +782,46 @@ globalThis.properties = {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A next obligation spawned at step k and checked against step k+1's state is
|
||||||
|
// attributed to step k, the step that caused it. StepIndex from SnapshotInput
|
||||||
|
// labels the evaluator observations so the witness carries runner step numbers.
|
||||||
|
func TestEvaluateProperties_NextViolationCarriesOriginStepIndex(t *testing.T) {
|
||||||
|
const spec = `
|
||||||
|
globalThis.flag = __sanderling__.extract(state => state.snapshots["flag"] ?? false, "flag");
|
||||||
|
globalThis.properties = {
|
||||||
|
flagStaysTrue: __sanderling__.always(__sanderling__.next(() => flag.current === true)),
|
||||||
|
};
|
||||||
|
`
|
||||||
|
verifier := newVerifier(t)
|
||||||
|
mustLoad(t, verifier, spec)
|
||||||
|
|
||||||
|
push := func(stepIndex int, flag string) map[string]ltl.Verdict {
|
||||||
|
t.Helper()
|
||||||
|
if err := verifier.PushSnapshot(SnapshotInput{
|
||||||
|
Snapshots: Snapshots{"flag": json.RawMessage(flag)},
|
||||||
|
StepIndex: stepIndex,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return verifier.EvaluateProperties()
|
||||||
|
}
|
||||||
|
|
||||||
|
push(7, `true`)
|
||||||
|
push(8, `true`)
|
||||||
|
verdicts := push(9, `false`)
|
||||||
|
if got := verdicts["flagStaysTrue"]; got != ltl.VerdictViolated {
|
||||||
|
t.Fatalf("step 9: verdict = %v, want violated", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
witness := verifier.Witness("flagStaysTrue")
|
||||||
|
if witness == nil {
|
||||||
|
t.Fatal("Witness = nil, want non-nil")
|
||||||
|
}
|
||||||
|
if witness.Step != 8 {
|
||||||
|
t.Errorf("Witness.Step = %d, want 8 (the step that spawned the next obligation)", witness.Step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoad_AcceptsSpecWithoutPropertiesOrActions(t *testing.T) {
|
func TestLoad_AcceptsSpecWithoutPropertiesOrActions(t *testing.T) {
|
||||||
verifier := newVerifier(t)
|
verifier := newVerifier(t)
|
||||||
if err := verifier.Load(`const noop = 1;`); err != nil {
|
if err := verifier.Load(`const noop = 1;`); err != nil {
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ type Verifier struct {
|
|||||||
lastLogs []LogEntry
|
lastLogs []LogEntry
|
||||||
lastExceptions []Exception
|
lastExceptions []Exception
|
||||||
stepTime time.Time
|
stepTime time.Time
|
||||||
|
stepIndex int
|
||||||
runStart time.Time
|
runStart time.Time
|
||||||
|
|
||||||
appPackage string
|
appPackage string
|
||||||
@@ -259,6 +260,7 @@ func (v *Verifier) PushSnapshot(input SnapshotInput) error {
|
|||||||
v.lastLogs = input.Logs
|
v.lastLogs = input.Logs
|
||||||
v.lastExceptions = input.Exceptions
|
v.lastExceptions = input.Exceptions
|
||||||
v.stepTime = input.StepTime
|
v.stepTime = input.StepTime
|
||||||
|
v.stepIndex = input.StepIndex
|
||||||
if v.runStart.IsZero() {
|
if v.runStart.IsZero() {
|
||||||
v.runStart = input.RunStart
|
v.runStart = input.RunStart
|
||||||
}
|
}
|
||||||
@@ -385,6 +387,11 @@ type SnapshotInput struct {
|
|||||||
Tree *hierarchy.Tree
|
Tree *hierarchy.Tree
|
||||||
LastAction *Action
|
LastAction *Action
|
||||||
StepTime time.Time
|
StepTime time.Time
|
||||||
|
// StepIndex is the runner's step number for this snapshot. Evaluators label
|
||||||
|
// observations with it so violation witnesses carry runner step numbers even
|
||||||
|
// when transitional steps were skipped. Zero means unlabeled; evaluators
|
||||||
|
// then fall back to their internal counter.
|
||||||
|
StepIndex int
|
||||||
RunStart time.Time
|
RunStart time.Time
|
||||||
Logs []LogEntry
|
Logs []LogEntry
|
||||||
Exceptions []Exception
|
Exceptions []Exception
|
||||||
@@ -404,7 +411,11 @@ func (v *Verifier) EvaluateProperties() map[string]ltl.Verdict {
|
|||||||
stepTime = time.Now()
|
stepTime = time.Now()
|
||||||
}
|
}
|
||||||
for name, evaluator := range v.evaluators {
|
for name, evaluator := range v.evaluators {
|
||||||
verdicts[name] = evaluator.ObserveAt(stepTime)
|
if v.stepIndex > 0 {
|
||||||
|
verdicts[name] = evaluator.ObserveAtStep(stepTime, v.stepIndex)
|
||||||
|
} else {
|
||||||
|
verdicts[name] = evaluator.ObserveAt(stepTime)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var onset []string
|
var onset []string
|
||||||
|
|||||||
Reference in new issue
Block a user