mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 11:07:10 +00:00
* feat(ltl): add Now/Next/Eventually/Implies/Or/And/Not formulas Replace the fold-with-latch evaluator with a residual-formula reducer. Each Observe() instantiates a fresh obligation from the root (stripping an outer Always), reduces each pending obligation against current state, latches Violated on first failure, and surfaces Pending verdicts for deferred obligations. Existing Always/Pure/Thunk tests continue to pass. * feat(ltl): support relative duration for eventually().within() * feat(proto): add Swipe, PressKey, RecentLogs RPCs * feat(verifier,runner): formula handles, new action kinds, rich state - verifier: add formula-spec registry; bindNow/bindNext/bindEventually with chainable .implies/.or/.and/.not and .within(n,unit) on eventually; bindFrom for uniform sampling. bindAlways keeps accepting plain predicates. - verifier: store lastTree, lastAction, step time, logs, exceptions on the Verifier; SnapshotInput replaces the (snapshots, tree) pair. stateObject now produces state.lastAction/time/logs/exceptions matching the TS State type. - verifier: make taps/swipes/waitOnce/pressKey built-in generators actually fire; taps picks a clickable, enabled element from the last hierarchy. - agent: add exceptions field to Message wire format. - driver: add Swipe/PressKey/RecentLogs to Driver interface; wire maestro client and mock driver. LogEntry exposed for runner consumption. - runner: apply Swipe/PressKey/Wait actions; collect logcat and exceptions; pass lastAction and step time into PushSnapshot. * feat(spec-api): LTL operators, new actions, richer State - ltl.ts exports now/next/eventually; always overload accepts a Formula - types.ts: Formula gains implies/or/and/not; EventuallyFormula adds .within; State gains lastAction/time/logs/exceptions; Swipe/PressKey/Wait action types - actions.ts: Swipe/PressKey/Wait/from constructors; waitOnce + pressKey default generators - tests exercise the chaining, sampling, and new actions through a recorded fake runtime * feat(sidecar): add swipe, pressKey, recentLogs RPC handlers * feat(sdk-android): capture uncaught exceptions Install a default uncaught handler on Uatu.start, chained with any existing handler so Android's crash reporter still runs. Expose Uatu.reportError for callers to forward caught throwables. A bounded circular buffer (default 50) drains into each STATE message's new exceptions field. Protocol.kt serializes/deserializes the field, matching the Go wire format added to internal/agent/protocol.go. * feat(spec-api): add @uatu/spec/defaults/properties bundle * feat(sample-app): exercise new LTL operators + defaults spec.ts now imports eventually/next/now/from from @uatu/spec and noUncaughtExceptions from @uatu/spec/defaults/properties. It declares three properties that exercise the new surface: - accountCountNonNegative: plain always() safety - addAccountAdvances: always(now(x).implies(next(y))) - eventuallyLoggedIn: eventually(p).within(30, "seconds") - noUncaughtExceptions: imported default The weighted actions root uses from() for random phone/name sampling and entries for taps/swipes/waitOnce/pressKey built-ins. SampleApplication gains a debug hook gated on the system property uatu.inject_error so the e2e run can synthesize an Uatu.reportError and verify noUncaughtExceptions violates. cmd/uatu/test_run.go adds a subpath alias so specs importing "@uatu/spec/defaults/properties" resolve against the in-tree source when running from the uatu checkout. The spec-integration tests swap the old click-counter fixtures for the new login hierarchy. * feat(trace): record swipe/key/wait details + exceptions trace.Step gains an Exceptions array so the trace captures the class/message/stackTrace for each SDK-reported throwable in a step. trace.Action gains FromX/FromY/ToX/ToY/Key/DurationMillis so the full payload of Swipe/PressKey/Wait actions is visible in trace.jsonl. sample-app's debug error hook now gates on ApplicationInfo.DEBUGGABLE instead of a system property (adb setprop fails on non-rooted emulators).
486 lines
13 KiB
Go
486 lines
13 KiB
Go
package verifier
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"math/rand/v2"
|
|
"time"
|
|
|
|
"github.com/dop251/goja"
|
|
|
|
"github.com/priyanshujain/uatu/internal/hierarchy"
|
|
"github.com/priyanshujain/uatu/internal/ltl"
|
|
)
|
|
|
|
type Verifier struct {
|
|
runtime *goja.Runtime
|
|
extractors []*extractorState
|
|
formulas []*formulaState
|
|
formulaSpecs []formulaSpec
|
|
|
|
properties map[string]int // property name -> formula-spec index
|
|
actionGenerator goja.Value
|
|
|
|
evaluators map[string]*ltl.Evaluator
|
|
|
|
lastTree *hierarchy.Tree
|
|
lastAction *Action
|
|
lastLogs []LogEntry
|
|
lastExceptions []Exception
|
|
stepTime time.Time
|
|
runStart time.Time
|
|
|
|
rng *rand.Rand
|
|
}
|
|
|
|
type Option func(*Verifier)
|
|
|
|
func WithRand(rng *rand.Rand) Option {
|
|
return func(v *Verifier) { v.rng = rng }
|
|
}
|
|
|
|
func New(options ...Option) (*Verifier, error) {
|
|
verifier := &Verifier{
|
|
runtime: goja.New(),
|
|
properties: map[string]int{},
|
|
evaluators: map[string]*ltl.Evaluator{},
|
|
rng: rand.New(rand.NewPCG(0, 0)),
|
|
}
|
|
for _, option := range options {
|
|
option(verifier)
|
|
}
|
|
if err := verifier.installRuntimeBindings(); err != nil {
|
|
return nil, fmt.Errorf("install bindings: %w", err)
|
|
}
|
|
return verifier, nil
|
|
}
|
|
|
|
// Load executes the bundled spec source. The spec is expected to assign its
|
|
// property formulas to globalThis.properties and its root action generator
|
|
// to globalThis.actions.
|
|
func (v *Verifier) Load(source string) error {
|
|
if _, err := v.runtime.RunString(source); err != nil {
|
|
return fmt.Errorf("run spec: %w", err)
|
|
}
|
|
|
|
propertiesValue := v.runtime.GlobalObject().Get("properties")
|
|
if propertiesValue != nil && !goja.IsUndefined(propertiesValue) && !goja.IsNull(propertiesValue) {
|
|
propertiesObject := propertiesValue.ToObject(v.runtime)
|
|
for _, name := range propertiesObject.Keys() {
|
|
handle := propertiesObject.Get(name).ToObject(v.runtime)
|
|
if handle == nil {
|
|
return fmt.Errorf("property %q is not an object", name)
|
|
}
|
|
specIndex, ok := v.extractSpecIndex(handle)
|
|
if !ok {
|
|
return fmt.Errorf("property %q was not produced by always()", name)
|
|
}
|
|
formula, err := v.buildFormula(specIndex)
|
|
if err != nil {
|
|
return fmt.Errorf("property %q: %w", name, err)
|
|
}
|
|
v.properties[name] = specIndex
|
|
v.evaluators[name] = ltl.NewEvaluator(formula)
|
|
}
|
|
}
|
|
|
|
if actionsValue := v.runtime.GlobalObject().Get("actions"); actionsValue != nil && !goja.IsUndefined(actionsValue) && !goja.IsNull(actionsValue) {
|
|
v.actionGenerator = actionsValue
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// buildFormula walks the formula-spec registry and produces a Go ltl.Formula
|
|
// tree rooted at the given spec index. Specs built at the top level are
|
|
// always wrapped in Always unless the top-level spec is already an Always.
|
|
func (v *Verifier) buildFormula(rootIndex int) (ltl.Formula, error) {
|
|
inner, err := v.buildFormulaNode(rootIndex)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, ok := inner.(ltl.AlwaysFormula); ok {
|
|
return inner, nil
|
|
}
|
|
return ltl.Always(inner), nil
|
|
}
|
|
|
|
func (v *Verifier) buildFormulaNode(index int) (ltl.Formula, error) {
|
|
if index < 0 || index >= len(v.formulaSpecs) {
|
|
return nil, fmt.Errorf("formula spec index %d out of range", index)
|
|
}
|
|
spec := v.formulaSpecs[index]
|
|
switch spec.kind {
|
|
case specKindPure:
|
|
return ltl.Pure(spec.pureValue), nil
|
|
case specKindThunk:
|
|
return ltl.Thunk(v.formulaThunk(spec.predicateIndex)), nil
|
|
case specKindNow:
|
|
child, err := v.buildFormulaNode(spec.childA)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return ltl.Now(child), nil
|
|
case specKindNext:
|
|
child, err := v.buildFormulaNode(spec.childA)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return ltl.Next(child), nil
|
|
case specKindEventually:
|
|
child, err := v.buildFormulaNode(spec.childA)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
formula := ltl.EventuallyFormula{Inner: child}
|
|
if spec.hasStepBound {
|
|
formula.StepBound = spec.stepBound
|
|
formula.HasStepBound = true
|
|
}
|
|
if spec.duration > 0 {
|
|
formula.Duration = spec.duration
|
|
}
|
|
return formula, nil
|
|
case specKindImplies:
|
|
left, err := v.buildFormulaNode(spec.childA)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
right, err := v.buildFormulaNode(spec.childB)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return ltl.Implies(left, right), nil
|
|
case specKindOr:
|
|
left, err := v.buildFormulaNode(spec.childA)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
right, err := v.buildFormulaNode(spec.childB)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return ltl.Or(left, right), nil
|
|
case specKindAnd:
|
|
left, err := v.buildFormulaNode(spec.childA)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
right, err := v.buildFormulaNode(spec.childB)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return ltl.And(left, right), nil
|
|
case specKindNot:
|
|
child, err := v.buildFormulaNode(spec.childA)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return ltl.Not(child), nil
|
|
case specKindAlways:
|
|
child, err := v.buildFormulaNode(spec.childA)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return ltl.Always(child), nil
|
|
default:
|
|
return nil, fmt.Errorf("unknown formula spec kind %d", spec.kind)
|
|
}
|
|
}
|
|
|
|
// PushSnapshot updates the JS-side state and refreshes every extractor's
|
|
// current/previous values in registration order. Passing a nil tree is
|
|
// allowed and yields an empty ax scope.
|
|
func (v *Verifier) PushSnapshot(input SnapshotInput) error {
|
|
v.lastTree = input.Tree
|
|
v.lastAction = input.LastAction
|
|
v.lastLogs = input.Logs
|
|
v.lastExceptions = input.Exceptions
|
|
v.stepTime = input.StepTime
|
|
if v.runStart.IsZero() {
|
|
v.runStart = input.RunStart
|
|
}
|
|
|
|
state, err := stateObject(v.runtime, stateInput{
|
|
snapshots: input.Snapshots,
|
|
tree: input.Tree,
|
|
lastAction: input.LastAction,
|
|
stepTime: input.StepTime,
|
|
runStart: v.runStart,
|
|
logs: input.Logs,
|
|
exceptions: input.Exceptions,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("build state: %w", err)
|
|
}
|
|
if err := v.runtime.GlobalObject().Set("state", state); err != nil {
|
|
return fmt.Errorf("set state: %w", err)
|
|
}
|
|
for index, extractor := range v.extractors {
|
|
previous := extractor.handle.Get("current")
|
|
_ = extractor.handle.Set("previous", previous)
|
|
newValue, err := extractor.getter(goja.Undefined(), state)
|
|
if err != nil {
|
|
return fmt.Errorf("extractor %d: %w", index, err)
|
|
}
|
|
_ = extractor.handle.Set("current", newValue)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SnapshotInput bundles everything a step feeds into the verifier. Fields
|
|
// other than Snapshots are optional; callers that only have snapshots can
|
|
// populate Snapshots alone and leave the rest zero.
|
|
type SnapshotInput struct {
|
|
Snapshots Snapshots
|
|
Tree *hierarchy.Tree
|
|
LastAction *Action
|
|
StepTime time.Time
|
|
RunStart time.Time
|
|
Logs []LogEntry
|
|
Exceptions []Exception
|
|
}
|
|
|
|
// LogEntry mirrors a logcat line captured between steps.
|
|
type LogEntry struct {
|
|
UnixMillis int64
|
|
Level string
|
|
Tag string
|
|
Message string
|
|
}
|
|
|
|
// Exception mirrors an SDK-captured uncaught throwable.
|
|
type Exception struct {
|
|
Class string
|
|
Message string
|
|
StackTrace string
|
|
UnixMillis int64
|
|
}
|
|
|
|
// EvaluateProperties returns each registered property's running verdict
|
|
// after the most recent PushSnapshot. The step time passed in PushSnapshot is
|
|
// forwarded to each evaluator so deadline-bound operators see the snapshot's
|
|
// wall clock rather than time.Now().
|
|
func (v *Verifier) EvaluateProperties() map[string]ltl.Verdict {
|
|
verdicts := map[string]ltl.Verdict{}
|
|
stepTime := v.stepTime
|
|
if stepTime.IsZero() {
|
|
stepTime = time.Now()
|
|
}
|
|
for name, evaluator := range v.evaluators {
|
|
verdicts[name] = evaluator.ObserveAt(stepTime)
|
|
}
|
|
return verdicts
|
|
}
|
|
|
|
// NextAction resolves the root action generator into a single Action.
|
|
// Returns ErrNoAction when no branch of the generator produces one after a
|
|
// small number of retries. Retrying avoids wedging when most branches of a
|
|
// weighted generator produce no action on the current screen (e.g. a gated
|
|
// login-phone generator when the app is already past login).
|
|
func (v *Verifier) NextAction() (Action, error) {
|
|
if v.actionGenerator == nil {
|
|
return Action{}, ErrNoAction
|
|
}
|
|
const maxRetries = 16
|
|
for range maxRetries {
|
|
action, err := v.resolveGenerator(v.actionGenerator)
|
|
if err == nil {
|
|
return action, nil
|
|
}
|
|
if !errors.Is(err, ErrNoAction) {
|
|
return Action{}, err
|
|
}
|
|
}
|
|
return Action{}, ErrNoAction
|
|
}
|
|
|
|
var ErrNoAction = errors.New("verifier: no action available")
|
|
|
|
func (v *Verifier) formulaThunk(index int) func() bool {
|
|
return func() bool {
|
|
formula := v.formulas[index]
|
|
result, err := formula.predicate(goja.Undefined())
|
|
if err != nil {
|
|
panic(fmt.Errorf("predicate panic: %w", err))
|
|
}
|
|
return result.ToBoolean()
|
|
}
|
|
}
|
|
|
|
func (v *Verifier) resolveGenerator(generator goja.Value) (Action, error) {
|
|
object := generator.ToObject(v.runtime)
|
|
if object == nil {
|
|
return Action{}, fmt.Errorf("generator is not an object")
|
|
}
|
|
kindValue := object.Get(tagInternalKind)
|
|
if kindValue == nil {
|
|
return Action{}, fmt.Errorf("generator missing internal kind tag")
|
|
}
|
|
switch kindValue.String() {
|
|
case internalKindActions:
|
|
generateValue := object.Get("generate")
|
|
generate, ok := goja.AssertFunction(generateValue)
|
|
if !ok {
|
|
return Action{}, fmt.Errorf("actions handle missing generate function")
|
|
}
|
|
result, err := generate(goja.Undefined())
|
|
if err != nil {
|
|
return Action{}, fmt.Errorf("generate: %w", err)
|
|
}
|
|
return v.pickFromResult(result)
|
|
case internalKindWeighted:
|
|
entries := object.Get("entries").ToObject(v.runtime)
|
|
if entries == nil {
|
|
return Action{}, fmt.Errorf("weighted handle missing entries")
|
|
}
|
|
picked, err := v.pickWeighted(entries)
|
|
if err != nil {
|
|
return Action{}, err
|
|
}
|
|
return v.resolveGenerator(picked)
|
|
case internalKindBuiltinTaps:
|
|
return v.generateRandomTap()
|
|
case internalKindBuiltinSwipes:
|
|
return v.generateRandomSwipe()
|
|
case internalKindBuiltinWaitOnce:
|
|
return Action{Kind: ActionKindWait, DurationMillis: 500}, nil
|
|
case internalKindBuiltinPressKey:
|
|
return v.generateRandomPressKey()
|
|
default:
|
|
return Action{}, fmt.Errorf("unknown generator kind %q", kindValue.String())
|
|
}
|
|
}
|
|
|
|
// generateRandomTap picks a visible, tappable element from the last
|
|
// hierarchy snapshot and returns a Tap action targeting its center.
|
|
func (v *Verifier) generateRandomTap() (Action, error) {
|
|
if v.lastTree == nil {
|
|
return Action{}, ErrNoAction
|
|
}
|
|
candidates := make([]*hierarchy.Element, 0, len(v.lastTree.Elements))
|
|
for _, element := range v.lastTree.Elements {
|
|
if !element.Clickable || !element.Enabled {
|
|
continue
|
|
}
|
|
if element.Bounds.Right-element.Bounds.Left <= 0 || element.Bounds.Bottom-element.Bounds.Top <= 0 {
|
|
continue
|
|
}
|
|
candidates = append(candidates, element)
|
|
}
|
|
if len(candidates) == 0 {
|
|
return Action{}, ErrNoAction
|
|
}
|
|
picked := candidates[v.rng.IntN(len(candidates))]
|
|
x, y := picked.Bounds.Center()
|
|
return Action{Kind: ActionKindTap, X: x, Y: y}, nil
|
|
}
|
|
|
|
// generateRandomSwipe emits a swipe over a random enabled element or the
|
|
// whole screen, in a random direction. Returns ErrNoAction only when we have
|
|
// no tree to size a gesture off of.
|
|
func (v *Verifier) generateRandomSwipe() (Action, error) {
|
|
if v.lastTree == nil || len(v.lastTree.Elements) == 0 {
|
|
return Action{}, ErrNoAction
|
|
}
|
|
element := v.lastTree.Elements[v.rng.IntN(len(v.lastTree.Elements))]
|
|
cx, cy := element.Bounds.Center()
|
|
if cx <= 0 || cy <= 0 {
|
|
return Action{}, ErrNoAction
|
|
}
|
|
// Pick a direction: 0=up 1=down 2=left 3=right; magnitude 200-600 px.
|
|
magnitude := 200 + v.rng.IntN(401)
|
|
toX, toY := cx, cy
|
|
switch v.rng.IntN(4) {
|
|
case 0:
|
|
toY = cy - magnitude
|
|
case 1:
|
|
toY = cy + magnitude
|
|
case 2:
|
|
toX = cx - magnitude
|
|
case 3:
|
|
toX = cx + magnitude
|
|
}
|
|
if toX < 0 {
|
|
toX = 0
|
|
}
|
|
if toY < 0 {
|
|
toY = 0
|
|
}
|
|
return Action{
|
|
Kind: ActionKindSwipe,
|
|
FromX: cx,
|
|
FromY: cy,
|
|
ToX: toX,
|
|
ToY: toY,
|
|
DurationMillis: 250,
|
|
}, nil
|
|
}
|
|
|
|
func (v *Verifier) generateRandomPressKey() (Action, error) {
|
|
// Keep exploration gentle: only "back" for now. Home/menu would navigate
|
|
// away from the app under test.
|
|
return Action{Kind: ActionKindPressKey, Key: "back"}, nil
|
|
}
|
|
|
|
func (v *Verifier) pickFromResult(result goja.Value) (Action, error) {
|
|
if result == nil || goja.IsUndefined(result) || goja.IsNull(result) {
|
|
return Action{}, ErrNoAction
|
|
}
|
|
object := result.ToObject(v.runtime)
|
|
if object == nil {
|
|
return Action{}, ErrNoAction
|
|
}
|
|
lengthValue := object.Get("length")
|
|
if lengthValue == nil {
|
|
return jsValueToAction(v.runtime, result)
|
|
}
|
|
length := int(lengthValue.ToInteger())
|
|
if length == 0 {
|
|
return Action{}, ErrNoAction
|
|
}
|
|
pick := v.rng.IntN(length)
|
|
return jsValueToAction(v.runtime, object.Get(fmt.Sprintf("%d", pick)))
|
|
}
|
|
|
|
func (v *Verifier) pickWeighted(entries *goja.Object) (goja.Value, error) {
|
|
lengthValue := entries.Get("length")
|
|
if lengthValue == nil {
|
|
return nil, fmt.Errorf("weighted entries missing length")
|
|
}
|
|
length := int(lengthValue.ToInteger())
|
|
if length == 0 {
|
|
return nil, ErrNoAction
|
|
}
|
|
|
|
weights := make([]float64, length)
|
|
generators := make([]goja.Value, length)
|
|
totalWeight := 0.0
|
|
for index := range length {
|
|
entry := entries.Get(fmt.Sprintf("%d", index)).ToObject(v.runtime)
|
|
if entry == nil {
|
|
return nil, fmt.Errorf("weighted entry %d not an array", index)
|
|
}
|
|
weight := entry.Get("0").ToFloat()
|
|
generator := entry.Get("1")
|
|
if weight < 0 {
|
|
weight = 0
|
|
}
|
|
weights[index] = weight
|
|
generators[index] = generator
|
|
totalWeight += weight
|
|
}
|
|
if totalWeight == 0 {
|
|
return nil, ErrNoAction
|
|
}
|
|
pick := v.rng.Float64() * totalWeight
|
|
cumulative := 0.0
|
|
for index := range length {
|
|
cumulative += weights[index]
|
|
if pick < cumulative {
|
|
return generators[index], nil
|
|
}
|
|
}
|
|
return generators[length-1], nil
|
|
}
|