mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 11:07:10 +00:00
UX refactor (#52)
* feat(ltl): bound fields on AlwaysFormula and named thunks Add StepBound/Duration/Deadline to AlwaysFormula as the dual of bounded Eventually, give ThunkFormula a Name for stable identity, add ThunkNamed, and surface both in describe() and MarshalJSON. * feat(ltl): negation normal form pass nnf/pushNot rewrite a formula so every Not wraps only a Thunk or Error leaf, dualizing Always<->Eventually and preserving bounds. * feat(ltl): NNF in NewEvaluator, bounded-always, Finalize, collapse Apply nnf on construction, reduce bounded Always symmetric to bounded Eventually (vacuous holds once the window closes), add Finalize to resolve undischarged liveness obligations to Violated at run end, and collapse structurally-identical pending obligations. * test(ltl): property-based NNF laws Lock double-negation identity, Always/Eventually duality with bound preservation, leaf pushdown, and not(always true) reaching Violated. * test(ltl): Finalize, bounded eventually, latch, collapse Property tests for monotonic violation latch and eventually-within violating iff n consecutive false, plus Finalize and collapse cases. * feat(inspect): within clause on always residual node A negated bounded eventually serializes as a bounded always; render its bound instead of dropping it. * feat(ltl): witness violations and (bool,error) predicate thunks * test(ltl): migrate thunk call sites to (bool,error) * feat(ltl): flag thrown-predicate witnesses with IsError * refactor(verifier): replace predicate err side-channel with violation witness * test(verifier): witness API for thrown predicates * feat(trace): witnesses map and skipped-verification marker on Step * feat(runner): thread violation witnesses, finalize, skip marker into trace * test(ltl): lock violation witness reason, IsError, and step * test(verifier): finalize surfaces unmet eventually with witness * fix(ltl): eliminate implies and bounded-always false-negatives Rewrite a -> b to (not a) or b in NNF so a pending temporal antecedent can no longer defer the whole implication and drop a consequent that was false at the current step. Carry a pending inner past a bounded-Always window close instead of dropping it to holds, so a deferred obligation is resolved by a later step or Finalize. * test(ltl): lock implies and bounded-always false-negative regressions * fix(web-runtime): seed PRNG for reproducible runs and align weighted pick * feat(testrun): inject seed into web bundle via SANDERLING_SEED define * test: cover web-runtime seeded PRNG, weighted pick, and seed define wiring * test(spec): add Go math/rand/v2 PCG oracle and golden fixture * feat(spec): bit-exact PCG port of Go math/rand/v2 * test(spec): assert pcg.ts matches the PCG golden fixture * feat(spec): shared input corpus and press-key pools * feat(spec): action-tree types and Host interface * feat(spec): verb support matrix and warn-once helper * feat(spec): deterministic shared action picker * test(spec): verb matrix and warn-once semantics * test(spec): picker draw-order and determinism * refactor(spec): actions.ts returns pure GeneratorNode data trees * refactor(spec): wire from() sampling through the picker rng * feat(spec): shared runtime-entry installs next-action over pick.ts * feat(spec): export LongPress/Scroll/longPresses/scrolls factories * test(spec): assert data-tree shapes for action factories * test(spec): runtime-entry serializeAction wire-contract round-trip * refactor(spec): bridge data-tree nodes to the legacy goja picker tags * fix(spec): web runtime walks the spec's globalThis.actions data tree * test(spec): tolerate legacy bridge fields on builtin nodes * refactor(spec): installRuntime accepts a lazy root resolver The web bundle imports the runtime before the spec, so the action root on globalThis.actions only exists after the spec evaluates. Accept a function form so the goja and web hosts resolve the root per tick. * refactor(spec): web-runtime becomes the WEB Host, delegates to shared picker Delete the duplicate picker (resolveGenerator/pickWeighted/randomTap/ randomInput/randomSwipe/randomPressKey/pickFromArray, the mulberry32 PRNG, and the snake_case serializeAction) plus the __sanderling__ action factory binds. web-runtime now implements Host (platform/seedHi/seedLo from the injected 64-bit seed via BigInt, queryCandidates over the live DOM with a per-tick cache, reportUnsupported) and calls installRuntime so both engines run pick.ts over the same Pcg. Swipe/longPress/scroll follow the verbs.ts matrix instead of silently returning null. Keeps the DOM helpers (selector translation, queryElement, elementHandle, buildState, sanitize, extractors) and the global locking. Net -214 lines (741 -> 527). * test(spec): cover the WEB Host surface and seed precision Replace the deleted-picker tests with Host coverage: platform()==web, seedHi() parsing a 64-bit seed without Number precision loss, seedLo()==0, reportUnsupported warning, the installed next-action/extractor globals, and queryCandidates verb routing + per-tick caching over a querySelectorAll stub. * refactor(spec): picker emits native selector + scroll endpoints, setup precedence * feat(spec): goja runtime entry wires the shared picker over the Go host * feat(bundler): optional RuntimeFile prepends a runtime-entry import via stdin * feat(testrun): bundle the goja runtime entry so the verifier runs the shared picker * refactor(spec): drop the legacy goja bridge fields from action factories * feat(spec): serialize selector-only string targets for the runner to re-resolve * refactor(verifier): one DecodeAction reads the unified flat wire contract * refactor(verifier): goja host + shared picker replace the duplicate Go picker * refactor(runner): decode V8 actions via the unified DecodeAction; wire goja runtime * test(verifier): author specs through the shared picker path * test(runner): bundle authored specs with the goja runtime entry * feat(verifier): collect unsupported verbs for the run report * refactor(runner): collapse WebDriver forks behind ActionSource/ExtractorSource * feat(testrun): surface unsupported verbs in run report * test(verifier): cross-runtime goja/node parity gate on the shared picker * test(verifier): unsupported verbs collected deduped in first-seen order * test(runner): summary reports no unsupported verbs on a clean run * test(spec): golden-fixture cross-runtime parity gate for the node picker Replace the env-driven parity harness with a shared scenario module and a committed golden the node picker asserts independently. The goja side asserts the same golden, so neither runtime invokes the other at test time. * test(verifier): assert goja picker against the same cross-runtime golden Drop the node-subprocess coupling: the goja side now installs a stub __sanderlingHost__ with the fixed candidate list and asserts the committed golden, matching pkg/spec/test/parity.test.ts. * refactor(spec): rename pressKey generator export to pressKeys * refactor(spec): update barrel re-exports for pressKeys * test(spec): update pressKeys generator export name * docs(spec): rename pressKey generator to pressKeys * refactor(spec): extract samplerRng into shared sampler-rng module * feat(spec): add fluent seeded value generators (strings/integers/emails/edgeCaseText) * test(spec): cover fluent value generators determinism and chaining * refactor(bundler): inject globalThis trailer from spec named exports * refactor(bundler): reuse registration trailer in web bundler * test(bundler): cover named-export globalThis registration * feat(spec): add named() to Extracted handle type * feat(web-runtime): named() and cross-extractor read guard * feat(verifier): named() and cross-extractor read guard in goja * test(verifier): cross-extractor read guard and named() * test(web-runtime): export runtime and extractors for tests * test(web-runtime): named() and cross-extractor read guard * refactor(folio): drop manual globalThis trailer (bundler injects it) * refactor(folio): seed txn amounts via integers().between(1,500) * refactor(folio-web): drop manual globalThis trailer (bundler injects it) * fix(folio-web): seed card/txn-type selection via from().generate() for reproducible runs * refactor(folio-web): weight valid generators against edgeCaseText for names/amounts * refactor(folio-web): name extractors so violation witnesses are readable * fix(web-runtime): propagate extractor getter throws and unpoison locked global Stop swallowing getter errors in evaluateExtractors so the cross-extractor read guard aborts loudly, matching goja's PushSnapshot. Make the __sanderling__ lock configurable (still non-writable) so a shared test process can reinstall a fake. * test(spec): install fake runtime via defineProperty to survive locked global * test(web-runtime): assert uncaught cross-extractor read aborts evaluateExtractors * feat(runner): add MaxSteps bound to Options * test(runner): MaxSteps stops after exactly N steps * test(driverpb): drop proto getter round-trip tautology * test(sidecar): drop stub-mode placeholder tautology tests * test(mock): drop default-field-value assertion test * test(ltl): drop Verdict.String tautology tests * refactor(runner): extract RenderSummary for snapshot testing * test(runner): golden snapshots for trace stream and violation summary * feat(web-runtime): capture uncaught errors into state.exceptions * test(integration): add throwing and counter web fixtures * test(integration): add specs for the web fixtures * test(integration): drive web fixtures through the real pipeline in headless Chrome * chore(make): add test-browser target for the Chrome-driven suite * ci: run the Chrome-driven browser suite in a separate job * refactor(test): relocate browser suite to test/browser * refactor(permissions): delete dead internal/permissions package * refactor(test): rename package to browser_test * refactor(sidecarassets): rename internal/sidecar to internal/sidecarassets * chore(make): point test-browser at test/browser * docs(decisions): record internal/permissions deletion * refactor(doctor): use sidecarassets package * refactor(testrun): use sidecarassets package * fix(test): resolve testdata relative to browser_test.go * refactor(verifier): remove dead __sanderlingIndex compat alias * refactor(bundler): use encoding/json for JS string literals * docs(action-space): use vendor-neutral native driver wording * refactor(hierarchy): scrub backend tool name from comments * refactor(driver): scrub backend tool name from comments * refactor(driver): add DoubleTap and DoubleTapSelector to DeviceDriver * refactor(sidecar): implement DoubleTap with the sub-100ms inter-tap gap * refactor(chrome): implement DoubleTap as two taps with the gap * refactor(mock): record DoubleTap and DoubleTapSelector actions * refactor(runner): delegate double-tap to driver, drop gesture timing * test(runner): assert double-tap delegates to driver DoubleTap * docs(cmd): add package docs to CLI and developer tools * docs(driver): add package docs to driver interface and chrome backend * docs(driver): add package docs to mock and sidecar backends * docs(platform): add package docs to android and ios device prep * docs: add package docs to bundler and inspect * docs(ltl): add package doc to temporal logic evaluator * docs: add package docs to runner and testrun pipeline * docs: add package docs to trace and verifier * docs(sidecarassets): add package doc for embedded JAR loader * fix(chrome): add disable-dev-shm-usage so Chrome starts in CI * test(chrome): gate real-Chrome driver tests behind the browser tag * chore(make): run chrome driver tests in the browser job * fix(web-runtime): guard global error listeners for non-browser hosts The module registered window error/unhandledrejection listeners at top level, which threw under Node (the spec-api test runner) where globalThis.addEventListener is absent. Register only when the API exists; the real browser run is unaffected. * ci(browser): re-enable unprivileged user namespaces for headless Chrome ubuntu-latest moved to 24.04, whose AppArmor restriction on unprivileged user namespaces stops headless Chrome from opening its DevTools socket even with --no-sandbox, surfacing as the driver's 'websocket url timeout'. Relax the sysctl for the job and add a direct launch check so a future breakage shows Chrome's own stderr rather than an opaque driver timeout. * ci(browser): pin stable Chrome for the driver tests setup-chrome's default latest pulled a dev Chromium (150) whose remote debugging socket never came up under chromedp, while plain --dump-dom worked. Pin the stable channel, which the driver is tested against. * feat(defaults): add scroll and rebalance action weights Use relative-integer weights (taps/typing co-primary 100, scrolls 50, swipes 25, doubleTaps 10); the picker normalizes by their total. Adds scrolls to defaultActions as a first-class reveal behavior. * feat(defaults): trim scroll action weight wiring * fix(build): point sidecar jar ignore and embed paths at sidecarassets * test(defaults): drop stale longPresses re-export assertion longPresses is opt-in vocabulary, no longer re-exported from defaults/actions.ts since e0d3b20; its builtin resolution is already covered by api.test.ts. Trim the defaults test to scrolls, which is an actual default export. * fix(chrome): raise DevTools websocket read timeout to 60s Chrome cold-start on a loaded CI runner can exceed chromedp's 20s default for reading the DevTools websocket URL, flaking the browser tests with "websocket url timeout reached". Give launch more headroom.
This commit is contained in:
98 files changed
+6898
-2227
No files matched your search
+222
-429
@@ -1,3 +1,4 @@
|
||||
// Package verifier runs the spec's extractors and property formulas against each observed step.
|
||||
package verifier
|
||||
|
||||
import (
|
||||
@@ -6,9 +7,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"math/rand/v2"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
@@ -23,14 +22,17 @@ type Verifier struct {
|
||||
formulas []*formulaState
|
||||
formulaSpecs []formulaSpec
|
||||
|
||||
properties map[string]int // property name -> formula-spec index
|
||||
actionGenerator goja.Value
|
||||
setupGenerator goja.Value
|
||||
properties map[string]int // property name -> formula-spec index
|
||||
|
||||
// nextActionFn is the bundle-installed __sanderlingNextAction__, which runs
|
||||
// the shared picker (pick.ts) over the shared Pcg.
|
||||
nextActionFn goja.Callable
|
||||
|
||||
evaluators map[string]*ltl.Evaluator
|
||||
|
||||
priorVerdicts map[string]ltl.Verdict
|
||||
newlyViolated []string
|
||||
witnesses map[string]Witness
|
||||
|
||||
lastTree *hierarchy.Tree
|
||||
lastAction *Action
|
||||
@@ -40,14 +42,41 @@ type Verifier struct {
|
||||
runStart time.Time
|
||||
|
||||
appPackage string
|
||||
platform string
|
||||
seed uint64
|
||||
|
||||
rng *rand.Rand
|
||||
// unsupported collects verbs the picker requested but the platform cannot
|
||||
// dispatch (reportUnsupported host callback), deduped and in first-seen
|
||||
// order, so the runner can surface them in the run report.
|
||||
unsupported []string
|
||||
unsupportedSeen map[string]bool
|
||||
|
||||
// extracting is true only while an extractor getter is running. The handle's
|
||||
// current/previous accessors consult it so a getter that reaches into
|
||||
// another extractor's handle throws instead of reading a stale value.
|
||||
extracting bool
|
||||
}
|
||||
|
||||
// UnsupportedVerbs returns the verbs the picker requested that this platform
|
||||
// cannot dispatch, deduped and in first-seen order.
|
||||
func (v *Verifier) UnsupportedVerbs() []string {
|
||||
return v.unsupported
|
||||
}
|
||||
|
||||
type Option func(*Verifier)
|
||||
|
||||
func WithRand(rng *rand.Rand) Option {
|
||||
return func(v *Verifier) { v.rng = rng }
|
||||
// WithSeed sets the 64-bit seed the JS picker constructs its Pcg from
|
||||
// (new Pcg(seed, 0), matching the web bundle's SANDERLING_SEED). The verifier
|
||||
// exposes it to the bundle via the __sanderlingHost__.seedHi/seedLo binds.
|
||||
func WithSeed(seed uint64) Option {
|
||||
return func(v *Verifier) { v.seed = seed }
|
||||
}
|
||||
|
||||
// WithPlatform names the platform the host reports to the picker
|
||||
// ("android"/"ios"/"web"); it drives the verb-support matrix and the press-key
|
||||
// pool. Empty defaults to "android".
|
||||
func WithPlatform(platform string) Option {
|
||||
return func(v *Verifier) { v.platform = platform }
|
||||
}
|
||||
|
||||
// WithAppPackage scopes random-action target selection to the app under test.
|
||||
@@ -60,15 +89,20 @@ func WithAppPackage(appPackage string) Option {
|
||||
|
||||
func New(options ...Option) (*Verifier, error) {
|
||||
verifier := &Verifier{
|
||||
runtime: goja.New(),
|
||||
properties: map[string]int{},
|
||||
evaluators: map[string]*ltl.Evaluator{},
|
||||
priorVerdicts: map[string]ltl.Verdict{},
|
||||
rng: rand.New(rand.NewPCG(0, 0)),
|
||||
runtime: goja.New(),
|
||||
properties: map[string]int{},
|
||||
evaluators: map[string]*ltl.Evaluator{},
|
||||
priorVerdicts: map[string]ltl.Verdict{},
|
||||
witnesses: map[string]Witness{},
|
||||
platform: "android",
|
||||
unsupportedSeen: map[string]bool{},
|
||||
}
|
||||
for _, option := range options {
|
||||
option(verifier)
|
||||
}
|
||||
if verifier.platform == "" {
|
||||
verifier.platform = "android"
|
||||
}
|
||||
if err := verifier.installRuntimeBindings(); err != nil {
|
||||
return nil, fmt.Errorf("install bindings: %w", err)
|
||||
}
|
||||
@@ -105,12 +139,14 @@ func (v *Verifier) Load(source string) error {
|
||||
}
|
||||
}
|
||||
|
||||
if actionsValue := v.runtime.GlobalObject().Get("actions"); actionsValue != nil && !goja.IsUndefined(actionsValue) && !goja.IsNull(actionsValue) {
|
||||
v.actionGenerator = actionsValue
|
||||
}
|
||||
|
||||
if setupValue := v.runtime.GlobalObject().Get("setup"); setupValue != nil && !goja.IsUndefined(setupValue) && !goja.IsNull(setupValue) {
|
||||
v.setupGenerator = setupValue
|
||||
// The bundle's goja runtime entry installs __sanderlingNextAction__ once the
|
||||
// spec assigned globalThis.actions. Capture it; a spec bundled without the
|
||||
// runtime entry (raw-JS unit fixtures) leaves it nil and NextAction reports
|
||||
// ErrNoAction.
|
||||
if fn := v.runtime.GlobalObject().Get("__sanderlingNextAction__"); fn != nil {
|
||||
if callable, ok := goja.AssertFunction(fn); ok {
|
||||
v.nextActionFn = callable
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -139,7 +175,8 @@ func (v *Verifier) buildFormulaNode(index int) (ltl.Formula, error) {
|
||||
case specKindPure:
|
||||
return ltl.Pure(spec.pureValue), nil
|
||||
case specKindThunk:
|
||||
return ltl.Thunk(v.formulaThunk(spec.predicateIndex)), nil
|
||||
name := fmt.Sprintf("p%d", spec.predicateIndex)
|
||||
return ltl.ThunkNamed(name, v.formulaThunk(spec.predicateIndex)), nil
|
||||
case specKindNow:
|
||||
child, err := v.buildFormulaNode(spec.childA)
|
||||
if err != nil {
|
||||
@@ -244,21 +281,28 @@ func (v *Verifier) PushSnapshot(input SnapshotInput) error {
|
||||
// Extractor previous/current advance exactly once per PushSnapshot.
|
||||
// Predicate thunks read these slots but never trigger advancement, so
|
||||
// invoking a thunk multiple times between snapshots is value-stable.
|
||||
// refreshPredicateErrors relies on this to safely re-call predicates.
|
||||
for index, extractor := range v.extractors {
|
||||
previous := extractor.handle.Get("current")
|
||||
_ = extractor.handle.Set("previous", previous)
|
||||
newValue, err := extractor.getter(goja.Undefined(), state)
|
||||
extractor.previousValue = extractor.currentValue
|
||||
newValue, err := v.runExtractor(extractor, state)
|
||||
if err != nil {
|
||||
return fmt.Errorf("extractor %d: %w", index, err)
|
||||
}
|
||||
_ = extractor.handle.Set("current", newValue)
|
||||
extractor.currentValue = newValue
|
||||
extractor.prev = extractor.curr
|
||||
extractor.curr = encodeExtractorValue(newValue)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// runExtractor invokes an extractor's getter with the extracting flag set, so a
|
||||
// getter that reads another extractor's current/previous throws. The flag is
|
||||
// cleared even if the getter panics.
|
||||
func (v *Verifier) runExtractor(extractor *extractorState, state goja.Value) (goja.Value, error) {
|
||||
v.extracting = true
|
||||
defer func() { v.extracting = false }()
|
||||
return extractor.getter(goja.Undefined(), state)
|
||||
}
|
||||
|
||||
// encodeExtractorValue produces a stable JSON encoding of an extractor's
|
||||
// current value for diff comparison. goja values that don't survive Export
|
||||
// (e.g. wrapped host functions) yield nil; callers treat nil as "unknown" and
|
||||
@@ -328,7 +372,7 @@ func (v *Verifier) OverrideExtractorValues(overrides map[int]json.RawMessage) (s
|
||||
if conversionErr != nil {
|
||||
return skipped, fmt.Errorf("extractor override %d: %w", index, conversionErr)
|
||||
}
|
||||
_ = v.extractors[index].handle.Set("current", value)
|
||||
v.extractors[index].currentValue = value
|
||||
}
|
||||
return skipped, nil
|
||||
}
|
||||
@@ -362,12 +406,12 @@ func (v *Verifier) EvaluateProperties() map[string]ltl.Verdict {
|
||||
for name, evaluator := range v.evaluators {
|
||||
verdicts[name] = evaluator.ObserveAt(stepTime)
|
||||
}
|
||||
v.refreshPredicateErrors()
|
||||
|
||||
var onset []string
|
||||
for name, verdict := range verdicts {
|
||||
if verdict == ltl.VerdictViolated && v.priorVerdicts[name] != ltl.VerdictViolated {
|
||||
onset = append(onset, name)
|
||||
v.captureWitness(name)
|
||||
}
|
||||
}
|
||||
sort.Strings(onset)
|
||||
@@ -380,6 +424,89 @@ func (v *Verifier) EvaluateProperties() map[string]ltl.Verdict {
|
||||
return verdicts
|
||||
}
|
||||
|
||||
// Witness is the verifier-level record of a property violation: the LTL reason
|
||||
// (a predicate's thrown-error text, "predicate false", or a liveness failure),
|
||||
// the step it fired at, and a snapshot of every extractor's current value at
|
||||
// that step. The snapshot lets a reader see the state that produced the
|
||||
// violation without replaying the run.
|
||||
type Witness struct {
|
||||
Property string
|
||||
Reason string
|
||||
Step int
|
||||
IsError bool
|
||||
Extractors map[string]json.RawMessage
|
||||
}
|
||||
|
||||
// captureWitness records the witness for a property that just transitioned to
|
||||
// violated, snapshotting the current extractor values so the cause is visible
|
||||
// after the run.
|
||||
func (v *Verifier) captureWitness(name string) {
|
||||
evaluator, ok := v.evaluators[name]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
violation := evaluator.Violation()
|
||||
if violation == nil {
|
||||
return
|
||||
}
|
||||
v.witnesses[name] = Witness{
|
||||
Property: name,
|
||||
Reason: violation.Reason,
|
||||
Step: violation.Step,
|
||||
IsError: violation.IsError,
|
||||
Extractors: v.extractorSnapshot(),
|
||||
}
|
||||
}
|
||||
|
||||
// extractorSnapshot encodes every named extractor's current value as JSON. A
|
||||
// nil value (extractor never advanced or its value did not survive Export)
|
||||
// is recorded as JSON null.
|
||||
func (v *Verifier) extractorSnapshot() map[string]json.RawMessage {
|
||||
if len(v.extractors) == 0 {
|
||||
return nil
|
||||
}
|
||||
snapshot := make(map[string]json.RawMessage, len(v.extractors))
|
||||
for _, extractor := range v.extractors {
|
||||
value := extractor.curr
|
||||
if value == nil {
|
||||
value = []byte("null")
|
||||
}
|
||||
snapshot[extractor.name] = append(json.RawMessage(nil), value...)
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
// Witness returns the captured violation witness for a property, or nil if the
|
||||
// property has not violated. Callers consult this after EvaluateProperties (or
|
||||
// Finalize) reports a violation to surface the cause and the state at onset.
|
||||
func (v *Verifier) Witness(name string) *Witness {
|
||||
witness, ok := v.witnesses[name]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return &witness
|
||||
}
|
||||
|
||||
// Finalize drives each evaluator to its terminal verdict and returns the names
|
||||
// of properties that violate only at run end (a liveness obligation that never
|
||||
// discharged), capturing a witness for each. Properties already violated
|
||||
// mid-run are not re-reported here.
|
||||
func (v *Verifier) Finalize() []string {
|
||||
var ended []string
|
||||
for name, evaluator := range v.evaluators {
|
||||
if v.priorVerdicts[name] == ltl.VerdictViolated {
|
||||
continue
|
||||
}
|
||||
if evaluator.Finalize() == ltl.VerdictViolated {
|
||||
ended = append(ended, name)
|
||||
v.captureWitness(name)
|
||||
v.priorVerdicts[name] = ltl.VerdictViolated
|
||||
}
|
||||
}
|
||||
sort.Strings(ended)
|
||||
return ended
|
||||
}
|
||||
|
||||
// NewlyViolatedProperties returns the names of properties whose verdict
|
||||
// transitioned from non-Violated to Violated on the most recent
|
||||
// EvaluateProperties call, sorted lexicographically. Returns nil if no
|
||||
@@ -396,14 +523,14 @@ func (v *Verifier) NewlyViolatedProperties() []string {
|
||||
}
|
||||
|
||||
// Residuals returns the residual formula for each registered property after
|
||||
// the most recent EvaluateProperties call. Properties that errored during
|
||||
// predicate evaluation surface as ErrorFormula so the inspect UI can render
|
||||
// "predicate threw" inline.
|
||||
// the most recent EvaluateProperties call. Properties whose violation was
|
||||
// caused by a thrown predicate surface as ErrorFormula, sourced from the
|
||||
// captured witness, so the inspect UI can render "predicate threw" inline.
|
||||
func (v *Verifier) Residuals() map[string]ltl.Formula {
|
||||
residuals := map[string]ltl.Formula{}
|
||||
for name, evaluator := range v.evaluators {
|
||||
if predicateErr := v.PredicateError(name); predicateErr != nil {
|
||||
residuals[name] = ltl.ErrorFormula{Message: predicateErr.Error()}
|
||||
if witness, ok := v.witnesses[name]; ok && witness.IsError {
|
||||
residuals[name] = ltl.ErrorFormula{Message: witness.Reason}
|
||||
continue
|
||||
}
|
||||
residuals[name] = evaluator.Residual()
|
||||
@@ -411,174 +538,42 @@ func (v *Verifier) Residuals() map[string]ltl.Formula {
|
||||
return residuals
|
||||
}
|
||||
|
||||
// NextAction resolves an action for the current step. The setup generator,
|
||||
// when registered, runs first; if it yields an action, that wins. When setup
|
||||
// returns ErrNoAction (all branches empty) the call falls through to the
|
||||
// root action generator with the existing retry semantics. Setup is consulted
|
||||
// every step, so state regression (e.g. a logout under fuzz) automatically
|
||||
// re-engages the precondition.
|
||||
// NextAction resolves an action for the current step by invoking the bundled
|
||||
// __sanderlingNextAction__(), which runs the SHARED picker (pick.ts) over the
|
||||
// shared Pcg. Setup-generator precedence and the 16-attempt retry both live in
|
||||
// runtime-entry.ts now, so this is a thin call-and-decode. A null result (the
|
||||
// generator declined to act) reports ErrNoAction.
|
||||
func (v *Verifier) NextAction() (Action, error) {
|
||||
if v.setupGenerator != nil {
|
||||
action, err := v.resolveGenerator(v.setupGenerator)
|
||||
if err == nil {
|
||||
return action, nil
|
||||
}
|
||||
if !errors.Is(err, ErrNoAction) {
|
||||
return Action{}, err
|
||||
}
|
||||
}
|
||||
if v.actionGenerator == nil {
|
||||
if v.nextActionFn == nil {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
const maxRetries = 16
|
||||
for range maxRetries {
|
||||
action, err := v.resolveGenerator(v.actionGenerator)
|
||||
if err == nil {
|
||||
return action, nil
|
||||
}
|
||||
if !errors.Is(err, ErrNoAction) {
|
||||
return Action{}, err
|
||||
}
|
||||
value, err := v.nextActionFn(goja.Undefined())
|
||||
if err != nil {
|
||||
return Action{}, fmt.Errorf("next action: %w", err)
|
||||
}
|
||||
return Action{}, ErrNoAction
|
||||
if value == nil || goja.IsNull(value) || goja.IsUndefined(value) {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
raw, err := json.Marshal(value.Export())
|
||||
if err != nil {
|
||||
return Action{}, fmt.Errorf("marshal action: %w", err)
|
||||
}
|
||||
return DecodeAction(raw)
|
||||
}
|
||||
|
||||
var ErrNoAction = errors.New("verifier: no action available")
|
||||
|
||||
func (v *Verifier) formulaThunk(index int) func() bool {
|
||||
return func() bool {
|
||||
func (v *Verifier) formulaThunk(index int) func() (bool, error) {
|
||||
return func() (bool, error) {
|
||||
formula := v.formulas[index]
|
||||
result, err := formula.predicate(goja.Undefined())
|
||||
if err != nil {
|
||||
formula.err = err
|
||||
return false
|
||||
return false, err
|
||||
}
|
||||
formula.err = nil
|
||||
return result.ToBoolean()
|
||||
return result.ToBoolean(), nil
|
||||
}
|
||||
}
|
||||
|
||||
// refreshPredicateErrors re-invokes every registered predicate so that
|
||||
// formula.err reflects the current step rather than a latched first-step
|
||||
// throw. EvaluateProperties short-circuits once a property has latched to
|
||||
// violated, so without this refresh the runner's per-step "predicate error"
|
||||
// log freezes on whatever the predicate threw at step 1. The refreshed errors
|
||||
// have no effect on verdicts.
|
||||
//
|
||||
// Invariant: predicates may be re-invoked here outside the LTL gate that
|
||||
// would normally skip them (e.g. an `implies` consequent whose antecedent is
|
||||
// false). They must therefore be side-effect-free reads of extractor state;
|
||||
// any spec that asserts internal preconditions inside a predicate could
|
||||
// surface a spurious error in the inspect UI without affecting verdicts.
|
||||
func (v *Verifier) refreshPredicateErrors() {
|
||||
for _, formula := range v.formulas {
|
||||
result, err := formula.predicate(goja.Undefined())
|
||||
if err != nil {
|
||||
formula.err = err
|
||||
continue
|
||||
}
|
||||
_ = result
|
||||
formula.err = nil
|
||||
}
|
||||
}
|
||||
|
||||
// PredicateError returns the first goja error raised by any thunk in the
|
||||
// named property's formula tree, or nil if none fired. Callers typically
|
||||
// consult this after EvaluateProperties reports a violation to distinguish
|
||||
// a genuine predicate-false verdict from a malformed spec.
|
||||
func (v *Verifier) PredicateError(name string) error {
|
||||
rootIndex, ok := v.properties[name]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return v.firstThunkError(rootIndex)
|
||||
}
|
||||
|
||||
func (v *Verifier) firstThunkError(index int) error {
|
||||
if index < 0 || index >= len(v.formulaSpecs) {
|
||||
return nil
|
||||
}
|
||||
spec := v.formulaSpecs[index]
|
||||
switch spec.kind {
|
||||
case specKindThunk:
|
||||
return v.formulas[spec.predicateIndex].err
|
||||
case specKindImplies, specKindOr, specKindAnd:
|
||||
if err := v.firstThunkError(spec.childA); err != nil {
|
||||
return err
|
||||
}
|
||||
return v.firstThunkError(spec.childB)
|
||||
case specKindNow, specKindNext, specKindEventually, specKindNot, specKindAlways:
|
||||
return v.firstThunkError(spec.childA)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *Verifier) resolveGenerator(generator goja.Value) (Action, error) {
|
||||
object := generator.ToObject(v.runtime)
|
||||
if object == nil {
|
||||
return Action{}, fmt.Errorf("generator is not an object")
|
||||
}
|
||||
kindValue := object.Get(tagInternalKind)
|
||||
if kindValue == nil {
|
||||
return Action{}, fmt.Errorf("generator missing internal kind tag")
|
||||
}
|
||||
switch kindValue.String() {
|
||||
case internalKindActions:
|
||||
generateValue := object.Get("generate")
|
||||
generate, ok := goja.AssertFunction(generateValue)
|
||||
if !ok {
|
||||
return Action{}, fmt.Errorf("actions handle missing generate function")
|
||||
}
|
||||
result, err := generate(goja.Undefined())
|
||||
if err != nil {
|
||||
return Action{}, fmt.Errorf("generate: %w", err)
|
||||
}
|
||||
return v.pickFromResult(result)
|
||||
case internalKindWeighted:
|
||||
entries := object.Get("entries").ToObject(v.runtime)
|
||||
if entries == nil {
|
||||
return Action{}, fmt.Errorf("weighted handle missing entries")
|
||||
}
|
||||
picked, err := v.pickWeighted(entries)
|
||||
if err != nil {
|
||||
return Action{}, err
|
||||
}
|
||||
return v.resolveGenerator(picked)
|
||||
case internalKindBuiltinTaps:
|
||||
return v.generateRandomTap()
|
||||
case internalKindBuiltinDoubleTaps:
|
||||
return v.generateRandomDoubleTap()
|
||||
case internalKindBuiltinTyping:
|
||||
return v.generateRandomInput()
|
||||
case internalKindBuiltinSwipes:
|
||||
return v.generateRandomSwipe()
|
||||
case internalKindBuiltinWaitOnce:
|
||||
return Action{Kind: ActionKindWait, DurationMillis: 500}, nil
|
||||
case internalKindBuiltinPressKey:
|
||||
return v.generateRandomPressKey()
|
||||
case internalKindBuiltinLongPresses:
|
||||
return v.generateRandomLongPress()
|
||||
case internalKindBuiltinScrolls:
|
||||
return v.generateRandomScroll()
|
||||
default:
|
||||
return Action{}, fmt.Errorf("unknown generator kind %q", kindValue.String())
|
||||
}
|
||||
}
|
||||
|
||||
// generateRandomTap picks a visible, tappable element from the last
|
||||
// hierarchy snapshot and returns a Tap action targeting its center.
|
||||
func (v *Verifier) generateRandomTap() (Action, error) {
|
||||
return v.generateRandomTapKind(ActionKindTap)
|
||||
}
|
||||
|
||||
// generateRandomDoubleTap is the DoubleTap counterpart of generateRandomTap.
|
||||
// Real user gestures include double-tap (image zoom, like-to-favorite,
|
||||
// play/pause); a fuzzer that never emits one cannot exercise either those
|
||||
// features or the sub-100ms race windows that single-step Tap cadence misses.
|
||||
func (v *Verifier) generateRandomDoubleTap() (Action, error) {
|
||||
return v.generateRandomTapKind(ActionKindDoubleTap)
|
||||
}
|
||||
|
||||
// inScope reports whether an element belongs to the app under test. Nodes from
|
||||
// another package (the soft keyboard, system UI, permission dialogs) are out of
|
||||
// scope. An unset app package or an element with no package falls through to in
|
||||
@@ -590,31 +585,6 @@ func (v *Verifier) inScope(element *hierarchy.Element) bool {
|
||||
return element.Package == v.appPackage
|
||||
}
|
||||
|
||||
func (v *Verifier) generateRandomTapKind(kind ActionKind) (Action, error) {
|
||||
if v.lastTree == nil {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
candidates := make([]*hierarchy.Element, 0, len(v.lastTree.Elements))
|
||||
for _, element := range v.lastTree.Elements {
|
||||
if !element.Clickable || !element.Enabled {
|
||||
continue
|
||||
}
|
||||
if !v.inScope(element) {
|
||||
continue
|
||||
}
|
||||
if element.Bounds.Right-element.Bounds.Left <= 0 || element.Bounds.Bottom-element.Bounds.Top <= 0 {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, element)
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
picked := candidates[v.rng.IntN(len(candidates))]
|
||||
x, y := picked.Bounds.Center()
|
||||
return Action{Kind: kind, On: selectorForElement(v.lastTree, picked), X: x, Y: y}, nil
|
||||
}
|
||||
|
||||
// selectorForElement builds a canonical "key:value" selector that resolves
|
||||
// back to the given element via hierarchy.Tree.Find. Prefers resource-id (the
|
||||
// testTag carrier on Android / accessibilityIdentifier on iOS), falling back
|
||||
@@ -656,237 +626,60 @@ func selectorForElement(tree *hierarchy.Tree, element *hierarchy.Element) string
|
||||
return ""
|
||||
}
|
||||
|
||||
// inputCorpus is the edge-case string pool the typing builtin draws from to
|
||||
// stress field parsing: empty, whitespace, overflow length, unicode, numeric
|
||||
// boundaries, and common injection payloads.
|
||||
var inputCorpus = []string{
|
||||
"",
|
||||
"a",
|
||||
strings.Repeat("a", 4096),
|
||||
"🙂🔥💸",
|
||||
" ",
|
||||
"\t\n",
|
||||
"-1",
|
||||
"999999999999999999999",
|
||||
"0.0000001",
|
||||
"1e10",
|
||||
"'; DROP TABLE--",
|
||||
"<script>alert(1)</script>",
|
||||
"../../etc/passwd",
|
||||
"%s%n",
|
||||
"NaN",
|
||||
}
|
||||
|
||||
// generateRandomInput picks a visible, editable, enabled element from the last
|
||||
// hierarchy snapshot and types a random edge-case value into it. The runner
|
||||
// taps the target coordinates to focus before typing, so this works on both
|
||||
// native and web with no driver-side dispatch change.
|
||||
func (v *Verifier) generateRandomInput() (Action, error) {
|
||||
// candidatesForVerb enumerates the host-side targets a builtin verb may draw
|
||||
// from, in v.lastTree.Elements ORDER (the order is part of the picker's parity
|
||||
// contract). The filters are LIFTED from the old Go picker:
|
||||
//
|
||||
// taps/doubleTaps/longPresses: clickable + enabled + positive bounds
|
||||
// typing: editable + enabled + positive bounds
|
||||
// scrolls: scrollable attribute + positive bounds
|
||||
// swipes: any in-scope element
|
||||
//
|
||||
// Every candidate carries the resolving selector so the runner can re-route by
|
||||
// id/text. Out-of-scope nodes (the soft keyboard, system UI) are always dropped.
|
||||
func (v *Verifier) candidatesForVerb(verb string) []candidate {
|
||||
if v.lastTree == nil {
|
||||
return Action{}, ErrNoAction
|
||||
return nil
|
||||
}
|
||||
candidates := make([]*hierarchy.Element, 0, len(v.lastTree.Elements))
|
||||
var result []candidate
|
||||
for _, element := range v.lastTree.Elements {
|
||||
if !element.Editable || !element.Enabled {
|
||||
continue
|
||||
}
|
||||
if !v.inScope(element) {
|
||||
continue
|
||||
}
|
||||
if element.Bounds.Right-element.Bounds.Left <= 0 || element.Bounds.Bottom-element.Bounds.Top <= 0 {
|
||||
if !verbAccepts(verb, element) {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, element)
|
||||
x, y := element.Bounds.Center()
|
||||
result = append(result, candidate{
|
||||
x: x,
|
||||
y: y,
|
||||
width: element.Bounds.Width(),
|
||||
height: element.Bounds.Height(),
|
||||
selector: selectorForElement(v.lastTree, element),
|
||||
})
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
picked := candidates[v.rng.IntN(len(candidates))]
|
||||
x, y := picked.Bounds.Center()
|
||||
value := inputCorpus[v.rng.IntN(len(inputCorpus))]
|
||||
return Action{Kind: ActionKindInputText, X: x, Y: y, Text: value}, nil
|
||||
return result
|
||||
}
|
||||
|
||||
// generateRandomSwipe emits a swipe over a random enabled element or the
|
||||
// whole screen, in a random direction. Returns ErrNoAction only when we have
|
||||
// no tree to size a gesture off of.
|
||||
func (v *Verifier) generateRandomSwipe() (Action, error) {
|
||||
if v.lastTree == nil || len(v.lastTree.Elements) == 0 {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
candidates := make([]*hierarchy.Element, 0, len(v.lastTree.Elements))
|
||||
for _, element := range v.lastTree.Elements {
|
||||
if v.inScope(element) {
|
||||
candidates = append(candidates, element)
|
||||
}
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
element := candidates[v.rng.IntN(len(candidates))]
|
||||
cx, cy := element.Bounds.Center()
|
||||
if cx <= 0 || cy <= 0 {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
// Pick a direction: 0=up 1=down 2=left 3=right; magnitude 200-600 px.
|
||||
magnitude := 200 + v.rng.IntN(401)
|
||||
toX, toY := cx, cy
|
||||
switch v.rng.IntN(4) {
|
||||
case 0:
|
||||
toY = cy - magnitude
|
||||
case 1:
|
||||
toY = cy + magnitude
|
||||
case 2:
|
||||
toX = cx - magnitude
|
||||
case 3:
|
||||
toX = cx + magnitude
|
||||
}
|
||||
if toX < 0 {
|
||||
toX = 0
|
||||
}
|
||||
if toY < 0 {
|
||||
toY = 0
|
||||
}
|
||||
return Action{
|
||||
Kind: ActionKindSwipe,
|
||||
FromX: cx,
|
||||
FromY: cy,
|
||||
ToX: toX,
|
||||
ToY: toY,
|
||||
DurationMillis: 250,
|
||||
}, nil
|
||||
type candidate struct {
|
||||
x, y int
|
||||
width, height int
|
||||
selector string
|
||||
}
|
||||
|
||||
// generateRandomLongPress mirrors generateRandomTap: it picks a visible,
|
||||
// clickable, enabled, in-scope element and targets its center. Real users
|
||||
// long-press (context menus, reorder handles, multi-select), so a fuzzer that
|
||||
// never emits one cannot reach those affordances.
|
||||
func (v *Verifier) generateRandomLongPress() (Action, error) {
|
||||
return v.generateRandomTapKind(ActionKindLongPress)
|
||||
}
|
||||
|
||||
// generateRandomScroll picks a scrollable, in-scope container and emits a swipe
|
||||
// across it in a random direction.
|
||||
func (v *Verifier) generateRandomScroll() (Action, error) {
|
||||
if v.lastTree == nil {
|
||||
return Action{}, ErrNoAction
|
||||
// verbAccepts applies the per-verb element filter.
|
||||
func verbAccepts(verb string, element *hierarchy.Element) bool {
|
||||
positiveBounds := element.Bounds.Width() > 0 && element.Bounds.Height() > 0
|
||||
switch verb {
|
||||
case "taps", "doubleTaps", "longPresses":
|
||||
return element.Clickable && element.Enabled && positiveBounds
|
||||
case "typing":
|
||||
return element.Editable && element.Enabled && positiveBounds
|
||||
case "scrolls":
|
||||
return element.Attributes["scrollable"] == "true" && positiveBounds
|
||||
case "swipes":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
candidates := make([]*hierarchy.Element, 0, len(v.lastTree.Elements))
|
||||
for _, element := range v.lastTree.Elements {
|
||||
if element.Attributes["scrollable"] != "true" {
|
||||
continue
|
||||
}
|
||||
if !v.inScope(element) {
|
||||
continue
|
||||
}
|
||||
if element.Bounds.Width() <= 0 || element.Bounds.Height() <= 0 {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, element)
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
picked := candidates[v.rng.IntN(len(candidates))]
|
||||
cx, cy := picked.Bounds.Center()
|
||||
width := picked.Bounds.Width()
|
||||
height := picked.Bounds.Height()
|
||||
directions := []string{"up", "down", "left", "right"}
|
||||
dir := directions[v.rng.IntN(len(directions))]
|
||||
toX, toY := cx, cy
|
||||
// Scroll direction names the content motion; the gesture swipes the
|
||||
// opposite way. Revealing content below ("down") means dragging the
|
||||
// finger up, so toY decreases, and likewise for the other directions.
|
||||
switch dir {
|
||||
case "down":
|
||||
toY = cy - (4*height)/10
|
||||
case "up":
|
||||
toY = cy + (4*height)/10
|
||||
case "left":
|
||||
toX = cx + (4*width)/10
|
||||
case "right":
|
||||
toX = cx - (4*width)/10
|
||||
}
|
||||
if toX < 0 {
|
||||
toX = 0
|
||||
}
|
||||
if toY < 0 {
|
||||
toY = 0
|
||||
}
|
||||
return Action{
|
||||
Kind: ActionKindScroll,
|
||||
Direction: dir,
|
||||
FromX: cx,
|
||||
FromY: cy,
|
||||
ToX: toX,
|
||||
ToY: toY,
|
||||
DurationMillis: 300,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (v *Verifier) generateRandomPressKey() (Action, error) {
|
||||
// Keep exploration gentle: only "back" for now. Home/menu would navigate
|
||||
// away from the app under test.
|
||||
return Action{Kind: ActionKindPressKey, Key: "back"}, nil
|
||||
}
|
||||
|
||||
func (v *Verifier) pickFromResult(result goja.Value) (Action, error) {
|
||||
if result == nil || goja.IsUndefined(result) || goja.IsNull(result) {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
object := result.ToObject(v.runtime)
|
||||
if object == nil {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
lengthValue := object.Get("length")
|
||||
if lengthValue == nil {
|
||||
return jsValueToAction(v.runtime, result)
|
||||
}
|
||||
length := int(lengthValue.ToInteger())
|
||||
if length == 0 {
|
||||
return Action{}, ErrNoAction
|
||||
}
|
||||
pick := v.rng.IntN(length)
|
||||
return jsValueToAction(v.runtime, object.Get(fmt.Sprintf("%d", pick)))
|
||||
}
|
||||
|
||||
func (v *Verifier) pickWeighted(entries *goja.Object) (goja.Value, error) {
|
||||
lengthValue := entries.Get("length")
|
||||
if lengthValue == nil {
|
||||
return nil, fmt.Errorf("weighted entries missing length")
|
||||
}
|
||||
length := int(lengthValue.ToInteger())
|
||||
if length == 0 {
|
||||
return nil, ErrNoAction
|
||||
}
|
||||
|
||||
weights := make([]float64, length)
|
||||
generators := make([]goja.Value, length)
|
||||
totalWeight := 0.0
|
||||
for index := range length {
|
||||
entry := entries.Get(fmt.Sprintf("%d", index)).ToObject(v.runtime)
|
||||
if entry == nil {
|
||||
return nil, fmt.Errorf("weighted entry %d not an array", index)
|
||||
}
|
||||
weight := entry.Get("0").ToFloat()
|
||||
generator := entry.Get("1")
|
||||
if weight < 0 {
|
||||
weight = 0
|
||||
}
|
||||
weights[index] = weight
|
||||
generators[index] = generator
|
||||
totalWeight += weight
|
||||
}
|
||||
if totalWeight == 0 {
|
||||
return nil, ErrNoAction
|
||||
}
|
||||
pick := v.rng.Float64() * totalWeight
|
||||
cumulative := 0.0
|
||||
for index := range length {
|
||||
cumulative += weights[index]
|
||||
if pick < cumulative {
|
||||
return generators[index], nil
|
||||
}
|
||||
}
|
||||
return generators[length-1], nil
|
||||
}
|
||||
Reference in new issue
Block a user