mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
feat: LTL operators, sampling, and default generators (#17)
* feat(ltl): add Now/Next/Eventually/Implies/Or/And/Not formulas Replace the fold-with-latch evaluator with a residual-formula reducer. Each Observe() instantiates a fresh obligation from the root (stripping an outer Always), reduces each pending obligation against current state, latches Violated on first failure, and surfaces Pending verdicts for deferred obligations. Existing Always/Pure/Thunk tests continue to pass. * feat(ltl): support relative duration for eventually().within() * feat(proto): add Swipe, PressKey, RecentLogs RPCs * feat(verifier,runner): formula handles, new action kinds, rich state - verifier: add formula-spec registry; bindNow/bindNext/bindEventually with chainable .implies/.or/.and/.not and .within(n,unit) on eventually; bindFrom for uniform sampling. bindAlways keeps accepting plain predicates. - verifier: store lastTree, lastAction, step time, logs, exceptions on the Verifier; SnapshotInput replaces the (snapshots, tree) pair. stateObject now produces state.lastAction/time/logs/exceptions matching the TS State type. - verifier: make taps/swipes/waitOnce/pressKey built-in generators actually fire; taps picks a clickable, enabled element from the last hierarchy. - agent: add exceptions field to Message wire format. - driver: add Swipe/PressKey/RecentLogs to Driver interface; wire maestro client and mock driver. LogEntry exposed for runner consumption. - runner: apply Swipe/PressKey/Wait actions; collect logcat and exceptions; pass lastAction and step time into PushSnapshot. * feat(spec-api): LTL operators, new actions, richer State - ltl.ts exports now/next/eventually; always overload accepts a Formula - types.ts: Formula gains implies/or/and/not; EventuallyFormula adds .within; State gains lastAction/time/logs/exceptions; Swipe/PressKey/Wait action types - actions.ts: Swipe/PressKey/Wait/from constructors; waitOnce + pressKey default generators - tests exercise the chaining, sampling, and new actions through a recorded fake runtime * feat(sidecar): add swipe, pressKey, recentLogs RPC handlers * feat(sdk-android): capture uncaught exceptions Install a default uncaught handler on Uatu.start, chained with any existing handler so Android's crash reporter still runs. Expose Uatu.reportError for callers to forward caught throwables. A bounded circular buffer (default 50) drains into each STATE message's new exceptions field. Protocol.kt serializes/deserializes the field, matching the Go wire format added to internal/agent/protocol.go. * feat(spec-api): add @uatu/spec/defaults/properties bundle * feat(sample-app): exercise new LTL operators + defaults spec.ts now imports eventually/next/now/from from @uatu/spec and noUncaughtExceptions from @uatu/spec/defaults/properties. It declares three properties that exercise the new surface: - accountCountNonNegative: plain always() safety - addAccountAdvances: always(now(x).implies(next(y))) - eventuallyLoggedIn: eventually(p).within(30, "seconds") - noUncaughtExceptions: imported default The weighted actions root uses from() for random phone/name sampling and entries for taps/swipes/waitOnce/pressKey built-ins. SampleApplication gains a debug hook gated on the system property uatu.inject_error so the e2e run can synthesize an Uatu.reportError and verify noUncaughtExceptions violates. cmd/uatu/test_run.go adds a subpath alias so specs importing "@uatu/spec/defaults/properties" resolve against the in-tree source when running from the uatu checkout. The spec-integration tests swap the old click-counter fixtures for the new login hierarchy. * feat(trace): record swipe/key/wait details + exceptions trace.Step gains an Exceptions array so the trace captures the class/message/stackTrace for each SDK-reported throwable in a step. trace.Action gains FromX/FromY/ToX/ToY/Key/DurationMillis so the full payload of Swipe/PressKey/Wait actions is visible in trace.jsonl. sample-app's debug error hook now gates on ApplicationInfo.DEBUGGABLE instead of a system property (adb setprop fails on non-rooted emulators).
This commit is contained in:
40 files changed
+2980
-262
No files matched your search
+253
-15
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"time"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
|
||||
@@ -12,15 +13,23 @@ import (
|
||||
)
|
||||
|
||||
type Verifier struct {
|
||||
runtime *goja.Runtime
|
||||
extractors []*extractorState
|
||||
formulas []*formulaState
|
||||
runtime *goja.Runtime
|
||||
extractors []*extractorState
|
||||
formulas []*formulaState
|
||||
formulaSpecs []formulaSpec
|
||||
|
||||
properties map[string]int // property name -> formula index
|
||||
properties map[string]int // property name -> formula-spec index
|
||||
actionGenerator goja.Value
|
||||
|
||||
evaluators map[string]*ltl.Evaluator
|
||||
|
||||
lastTree *hierarchy.Tree
|
||||
lastAction *Action
|
||||
lastLogs []LogEntry
|
||||
lastExceptions []Exception
|
||||
stepTime time.Time
|
||||
runStart time.Time
|
||||
|
||||
rng *rand.Rand
|
||||
}
|
||||
|
||||
@@ -62,13 +71,16 @@ func (v *Verifier) Load(source string) error {
|
||||
if handle == nil {
|
||||
return fmt.Errorf("property %q is not an object", name)
|
||||
}
|
||||
indexValue := handle.Get("__uatuIndex")
|
||||
if indexValue == nil {
|
||||
specIndex, ok := v.extractSpecIndex(handle)
|
||||
if !ok {
|
||||
return fmt.Errorf("property %q was not produced by always()", name)
|
||||
}
|
||||
index := int(indexValue.ToInteger())
|
||||
v.properties[name] = index
|
||||
v.evaluators[name] = ltl.NewEvaluator(ltl.Always(ltl.Thunk(v.formulaThunk(index))))
|
||||
formula, err := v.buildFormula(specIndex)
|
||||
if err != nil {
|
||||
return fmt.Errorf("property %q: %w", name, err)
|
||||
}
|
||||
v.properties[name] = specIndex
|
||||
v.evaluators[name] = ltl.NewEvaluator(formula)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,11 +91,125 @@ func (v *Verifier) Load(source string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildFormula walks the formula-spec registry and produces a Go ltl.Formula
|
||||
// tree rooted at the given spec index. Specs built at the top level are
|
||||
// always wrapped in Always unless the top-level spec is already an Always.
|
||||
func (v *Verifier) buildFormula(rootIndex int) (ltl.Formula, error) {
|
||||
inner, err := v.buildFormulaNode(rootIndex)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, ok := inner.(ltl.AlwaysFormula); ok {
|
||||
return inner, nil
|
||||
}
|
||||
return ltl.Always(inner), nil
|
||||
}
|
||||
|
||||
func (v *Verifier) buildFormulaNode(index int) (ltl.Formula, error) {
|
||||
if index < 0 || index >= len(v.formulaSpecs) {
|
||||
return nil, fmt.Errorf("formula spec index %d out of range", index)
|
||||
}
|
||||
spec := v.formulaSpecs[index]
|
||||
switch spec.kind {
|
||||
case specKindPure:
|
||||
return ltl.Pure(spec.pureValue), nil
|
||||
case specKindThunk:
|
||||
return ltl.Thunk(v.formulaThunk(spec.predicateIndex)), nil
|
||||
case specKindNow:
|
||||
child, err := v.buildFormulaNode(spec.childA)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ltl.Now(child), nil
|
||||
case specKindNext:
|
||||
child, err := v.buildFormulaNode(spec.childA)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ltl.Next(child), nil
|
||||
case specKindEventually:
|
||||
child, err := v.buildFormulaNode(spec.childA)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
formula := ltl.EventuallyFormula{Inner: child}
|
||||
if spec.hasStepBound {
|
||||
formula.StepBound = spec.stepBound
|
||||
formula.HasStepBound = true
|
||||
}
|
||||
if spec.duration > 0 {
|
||||
formula.Duration = spec.duration
|
||||
}
|
||||
return formula, nil
|
||||
case specKindImplies:
|
||||
left, err := v.buildFormulaNode(spec.childA)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
right, err := v.buildFormulaNode(spec.childB)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ltl.Implies(left, right), nil
|
||||
case specKindOr:
|
||||
left, err := v.buildFormulaNode(spec.childA)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
right, err := v.buildFormulaNode(spec.childB)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ltl.Or(left, right), nil
|
||||
case specKindAnd:
|
||||
left, err := v.buildFormulaNode(spec.childA)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
right, err := v.buildFormulaNode(spec.childB)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ltl.And(left, right), nil
|
||||
case specKindNot:
|
||||
child, err := v.buildFormulaNode(spec.childA)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ltl.Not(child), nil
|
||||
case specKindAlways:
|
||||
child, err := v.buildFormulaNode(spec.childA)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ltl.Always(child), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown formula spec kind %d", spec.kind)
|
||||
}
|
||||
}
|
||||
|
||||
// PushSnapshot updates the JS-side state and refreshes every extractor's
|
||||
// current/previous values in registration order. Passing a nil tree is
|
||||
// allowed and yields an empty ax scope.
|
||||
func (v *Verifier) PushSnapshot(snapshots Snapshots, tree *hierarchy.Tree) error {
|
||||
state, err := stateObject(v.runtime, snapshots, tree)
|
||||
func (v *Verifier) PushSnapshot(input SnapshotInput) error {
|
||||
v.lastTree = input.Tree
|
||||
v.lastAction = input.LastAction
|
||||
v.lastLogs = input.Logs
|
||||
v.lastExceptions = input.Exceptions
|
||||
v.stepTime = input.StepTime
|
||||
if v.runStart.IsZero() {
|
||||
v.runStart = input.RunStart
|
||||
}
|
||||
|
||||
state, err := stateObject(v.runtime, stateInput{
|
||||
snapshots: input.Snapshots,
|
||||
tree: input.Tree,
|
||||
lastAction: input.LastAction,
|
||||
stepTime: input.StepTime,
|
||||
runStart: v.runStart,
|
||||
logs: input.Logs,
|
||||
exceptions: input.Exceptions,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("build state: %w", err)
|
||||
}
|
||||
@@ -102,12 +228,47 @@ func (v *Verifier) PushSnapshot(snapshots Snapshots, tree *hierarchy.Tree) error
|
||||
return nil
|
||||
}
|
||||
|
||||
// SnapshotInput bundles everything a step feeds into the verifier. Fields
|
||||
// other than Snapshots are optional; callers that only have snapshots can
|
||||
// populate Snapshots alone and leave the rest zero.
|
||||
type SnapshotInput struct {
|
||||
Snapshots Snapshots
|
||||
Tree *hierarchy.Tree
|
||||
LastAction *Action
|
||||
StepTime time.Time
|
||||
RunStart time.Time
|
||||
Logs []LogEntry
|
||||
Exceptions []Exception
|
||||
}
|
||||
|
||||
// LogEntry mirrors a logcat line captured between steps.
|
||||
type LogEntry struct {
|
||||
UnixMillis int64
|
||||
Level string
|
||||
Tag string
|
||||
Message string
|
||||
}
|
||||
|
||||
// Exception mirrors an SDK-captured uncaught throwable.
|
||||
type Exception struct {
|
||||
Class string
|
||||
Message string
|
||||
StackTrace string
|
||||
UnixMillis int64
|
||||
}
|
||||
|
||||
// EvaluateProperties returns each registered property's running verdict
|
||||
// after the most recent PushSnapshot.
|
||||
// after the most recent PushSnapshot. The step time passed in PushSnapshot is
|
||||
// forwarded to each evaluator so deadline-bound operators see the snapshot's
|
||||
// wall clock rather than time.Now().
|
||||
func (v *Verifier) EvaluateProperties() map[string]ltl.Verdict {
|
||||
verdicts := map[string]ltl.Verdict{}
|
||||
stepTime := v.stepTime
|
||||
if stepTime.IsZero() {
|
||||
stepTime = time.Now()
|
||||
}
|
||||
for name, evaluator := range v.evaluators {
|
||||
verdicts[name] = evaluator.Observe()
|
||||
verdicts[name] = evaluator.ObserveAt(stepTime)
|
||||
}
|
||||
return verdicts
|
||||
}
|
||||
@@ -178,13 +339,90 @@ func (v *Verifier) resolveGenerator(generator goja.Value) (Action, error) {
|
||||
return Action{}, err
|
||||
}
|
||||
return v.resolveGenerator(picked)
|
||||
case internalKindBuiltinTaps, internalKindBuiltinSwipes:
|
||||
return Action{}, ErrNoAction
|
||||
case internalKindBuiltinTaps:
|
||||
return v.generateRandomTap()
|
||||
case internalKindBuiltinSwipes:
|
||||
return v.generateRandomSwipe()
|
||||
case internalKindBuiltinWaitOnce:
|
||||
return Action{Kind: ActionKindWait, DurationMillis: 500}, nil
|
||||
case internalKindBuiltinPressKey:
|
||||
return v.generateRandomPressKey()
|
||||
default:
|
||||
return Action{}, fmt.Errorf("unknown generator kind %q", kindValue.String())
|
||||
}
|
||||
}
|
||||
|
||||
// generateRandomTap picks a visible, tappable element from the last
|
||||
// hierarchy snapshot and returns a Tap action targeting its center.
|
||||
func (v *Verifier) generateRandomTap() (Action, error) {
|
||||
if v.lastTree == nil {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
candidates := make([]*hierarchy.Element, 0, len(v.lastTree.Elements))
|
||||
for _, element := range v.lastTree.Elements {
|
||||
if !element.Clickable || !element.Enabled {
|
||||
continue
|
||||
}
|
||||
if element.Bounds.Right-element.Bounds.Left <= 0 || element.Bounds.Bottom-element.Bounds.Top <= 0 {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, element)
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
picked := candidates[v.rng.IntN(len(candidates))]
|
||||
x, y := picked.Bounds.Center()
|
||||
return Action{Kind: ActionKindTap, X: x, Y: y}, nil
|
||||
}
|
||||
|
||||
// generateRandomSwipe emits a swipe over a random enabled element or the
|
||||
// whole screen, in a random direction. Returns ErrNoAction only when we have
|
||||
// no tree to size a gesture off of.
|
||||
func (v *Verifier) generateRandomSwipe() (Action, error) {
|
||||
if v.lastTree == nil || len(v.lastTree.Elements) == 0 {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
element := v.lastTree.Elements[v.rng.IntN(len(v.lastTree.Elements))]
|
||||
cx, cy := element.Bounds.Center()
|
||||
if cx <= 0 || cy <= 0 {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
// Pick a direction: 0=up 1=down 2=left 3=right; magnitude 200-600 px.
|
||||
magnitude := 200 + v.rng.IntN(401)
|
||||
toX, toY := cx, cy
|
||||
switch v.rng.IntN(4) {
|
||||
case 0:
|
||||
toY = cy - magnitude
|
||||
case 1:
|
||||
toY = cy + magnitude
|
||||
case 2:
|
||||
toX = cx - magnitude
|
||||
case 3:
|
||||
toX = cx + magnitude
|
||||
}
|
||||
if toX < 0 {
|
||||
toX = 0
|
||||
}
|
||||
if toY < 0 {
|
||||
toY = 0
|
||||
}
|
||||
return Action{
|
||||
Kind: ActionKindSwipe,
|
||||
FromX: cx,
|
||||
FromY: cy,
|
||||
ToX: toX,
|
||||
ToY: toY,
|
||||
DurationMillis: 250,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (v *Verifier) generateRandomPressKey() (Action, error) {
|
||||
// Keep exploration gentle: only "back" for now. Home/menu would navigate
|
||||
// away from the app under test.
|
||||
return Action{Kind: ActionKindPressKey, Key: "back"}, nil
|
||||
}
|
||||
|
||||
func (v *Verifier) pickFromResult(result goja.Value) (Action, error) {
|
||||
if result == nil || goja.IsUndefined(result) || goja.IsNull(result) {
|
||||
return Action{}, ErrNoAction
|
||||
|
||||
Reference in new issue
Block a user