mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
refactor(runner): emit onset-only violations to trace and summary
Switch the per-step violation list from the sticky verdict map to the verifier's onset set. Each property now appears exactly once across a run: at the step it first violates, not on every subsequent step where the residual stays false. Removes the dead violationNames helper.
This commit is contained in:
1 parent
b9fa41553f
commit
60c4ef7458
2 files changed
+74
-12
No files matched your search
@@ -165,8 +165,8 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
|||||||
screen = tree.Elements[0].Screen
|
screen = tree.Elements[0].Screen
|
||||||
}
|
}
|
||||||
logger.Info("step", "index", stepIndex, "screen", screen, "nodes", treeSize)
|
logger.Info("step", "index", stepIndex, "screen", screen, "nodes", treeSize)
|
||||||
verdicts := options.Verifier.EvaluateProperties()
|
options.Verifier.EvaluateProperties()
|
||||||
violations := violationNames(verdicts)
|
violations := options.Verifier.NewlyViolatedProperties()
|
||||||
for _, name := range violations {
|
for _, name := range violations {
|
||||||
if predicateErr := options.Verifier.PredicateError(name); predicateErr != nil {
|
if predicateErr := options.Verifier.PredicateError(name); predicateErr != nil {
|
||||||
logger.Warn("predicate error", "step", stepIndex, "property", name, "err", predicateErr)
|
logger.Warn("predicate error", "step", stepIndex, "property", name, "err", predicateErr)
|
||||||
@@ -266,16 +266,6 @@ func validate(options Options) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func violationNames(verdicts map[string]ltl.Verdict) []string {
|
|
||||||
var names []string
|
|
||||||
for name, verdict := range verdicts {
|
|
||||||
if verdict == ltl.VerdictViolated {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return names
|
|
||||||
}
|
|
||||||
|
|
||||||
// ensureForeground keeps the app under test in the foreground. When the driver
|
// ensureForeground keeps the app under test in the foreground. When the driver
|
||||||
// can report the foreground app and it no longer matches the bundle under test,
|
// can report the foreground app and it no longer matches the bundle under test,
|
||||||
// the app is relaunched. Returns true when a relaunch happened so the caller
|
// the app is relaunched. Returns true when a relaunch happened so the caller
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
package runner
|
package runner
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bufio"
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
@@ -118,6 +120,76 @@ func TestRunner_ViolationSurfacesInSummary(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunner_ViolationSurfacesOnlyOnOnsetStep(t *testing.T) {
|
||||||
|
// violationSpec uses always(() => false): onset fires on step 1 and the
|
||||||
|
// residual stays violated forever. The runner must record the violation
|
||||||
|
// exactly once (at the onset step) in both summary.Violations and trace
|
||||||
|
// lines, not on every subsequent step.
|
||||||
|
state := newHarnessWithSpec(t, violationSpec)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
summary, err := Run(ctx, Options{
|
||||||
|
Duration: 200 * time.Millisecond,
|
||||||
|
IdleTimeout: 20 * time.Millisecond,
|
||||||
|
Driver: state.mock,
|
||||||
|
Verifier: state.verifier,
|
||||||
|
TraceWriter: state.writer,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run: %v", err)
|
||||||
|
}
|
||||||
|
if summary.Steps < 2 {
|
||||||
|
t.Fatalf("need at least 2 steps to prove onset-only behavior, got %d", summary.Steps)
|
||||||
|
}
|
||||||
|
if len(summary.Violations) != 1 {
|
||||||
|
t.Fatalf("expected exactly one ViolationRecord (onset only), got %d: %v",
|
||||||
|
len(summary.Violations), summary.Violations)
|
||||||
|
}
|
||||||
|
if summary.Violations[0].StepIndex != 1 {
|
||||||
|
t.Errorf("onset step: got %d, want 1 (always(()=>false) violates immediately)",
|
||||||
|
summary.Violations[0].StepIndex)
|
||||||
|
}
|
||||||
|
if !slices.Equal(summary.Violations[0].Properties, []string{"balanceNonNegative"}) {
|
||||||
|
t.Errorf("onset properties: got %v, want [balanceNonNegative]",
|
||||||
|
summary.Violations[0].Properties)
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := os.Open(filepath.Join(state.writer.Directory(), "trace.jsonl"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
type traceLine struct {
|
||||||
|
Step int `json:"step"`
|
||||||
|
Violations []string `json:"violations"`
|
||||||
|
}
|
||||||
|
linesWithViolations := 0
|
||||||
|
scanner := bufio.NewScanner(file)
|
||||||
|
scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
|
||||||
|
for scanner.Scan() {
|
||||||
|
var line traceLine
|
||||||
|
if err := json.Unmarshal(scanner.Bytes(), &line); err != nil {
|
||||||
|
t.Fatalf("trace line decode: %v", err)
|
||||||
|
}
|
||||||
|
if len(line.Violations) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
linesWithViolations++
|
||||||
|
if line.Step != 1 {
|
||||||
|
t.Errorf("step %d unexpectedly emitted violations %v (should be onset-only at step 1)",
|
||||||
|
line.Step, line.Violations)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
t.Fatalf("scan trace: %v", err)
|
||||||
|
}
|
||||||
|
if linesWithViolations != 1 {
|
||||||
|
t.Errorf("expected exactly 1 trace line with violations, got %d", linesWithViolations)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunner_ThrowingPredicateIsLoggedNotPanic(t *testing.T) {
|
func TestRunner_ThrowingPredicateIsLoggedNotPanic(t *testing.T) {
|
||||||
const throwingSpec = `
|
const throwingSpec = `
|
||||||
globalThis.properties = {
|
globalThis.properties = {
|
||||||
|
|||||||
Reference in new issue
Block a user