diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af14d23..0a6dfc5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,3 +83,40 @@ jobs: - name: Run tests run: make test + + browser: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + # Pin stable: the action's default (latest) pulls a dev Chromium whose + # remote-debugging socket is flaky under the driver, even though the + # browser otherwise launches headless. + - name: Set up Chrome + uses: browser-actions/setup-chrome@v1 + with: + chrome-version: stable + + # Ubuntu 24.04 (current ubuntu-latest) restricts unprivileged user + # namespaces via AppArmor, which stops headless Chrome from starting even + # with --no-sandbox: the process launches but never opens its DevTools + # socket. Re-enable them so the driver's Chrome can come up. + - name: Allow Chrome under unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + # Fail here with Chrome's own stderr if the browser can't launch, instead + # of letting the driver report an opaque DevTools timeout downstream. + - name: Verify headless Chrome starts + run: | + chrome --version + chrome --headless --no-sandbox --disable-gpu --disable-dev-shm-usage \ + --dump-dom 'data:text/html,ok' + + - name: Drive web fixtures through headless Chrome + run: make test-browser diff --git a/.gitignore b/.gitignore index 3f9bd4f..128829e 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,7 @@ node_modules/ # Sidecar fat JAR. Build artifact copied in by `make sanderling` before # `go build -tags withsidecar`. Never commit: it's ~130 MB. -internal/sidecar/assets/sidecar-all.jar +internal/sidecarassets/assets/sidecar-all.jar # spec-api compiled output pkg/spec/dist/ diff --git a/Makefile b/Makefile index 69b345f..e7733a1 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ BUF := buf GO_PACKAGES := ./... SIDECAR_JAR := sidecar/build/libs/sidecar-all.jar -SIDECAR_EMBED := internal/sidecar/assets/sidecar-all.jar +SIDECAR_EMBED := internal/sidecarassets/assets/sidecar-all.jar SIDECAR_SRC := $(shell find sidecar/src -type f \( -name '*.kt' -o -name '*.kts' \) 2>/dev/null) build.gradle.kts settings.gradle.kts SANDERLING_BIN := bin/sanderling @@ -22,7 +22,7 @@ DOCS_TEMPLATE := docs/_template/page.html INSPECT_DIST := internal/inspect/dist WEB_DIST := inspect-ui/dist -.PHONY: bootstrap proto sidecar sanderling install test test-go test-kotlin test-spec-api web-typecheck web-build web-dev inspect-dev docs clean release-cli release-npm-dry +.PHONY: bootstrap proto sidecar sanderling install test test-go test-browser test-kotlin test-spec-api web-typecheck web-build web-dev inspect-dev docs clean release-cli release-npm-dry bootstrap: $(GO) mod download @@ -74,6 +74,11 @@ test: test-go test-spec-api web-typecheck test-go: $(GO) test $(GO_PACKAGES) +# Drives small web fixtures and the chrome driver through real headless Chrome. +# Kept out of `test` because it needs a Chrome binary on PATH. +test-browser: + $(GO) test -tags browser ./test/browser/... ./internal/driver/chrome/... + test-kotlin: ANDROID_HOME=$(ANDROID_HOME) $(GRADLE) :sidecar:test diff --git a/cmd/internal-tools/bundle-check/main.go b/cmd/internal-tools/bundle-check/main.go index b8821c1..be9fb43 100644 --- a/cmd/internal-tools/bundle-check/main.go +++ b/cmd/internal-tools/bundle-check/main.go @@ -1,3 +1,4 @@ +// Command bundle-check is a developer tool that bundles a spec file to confirm it compiles. package main import ( diff --git a/cmd/internal-tools/hier-check/main.go b/cmd/internal-tools/hier-check/main.go index 26089c0..03ddec5 100644 --- a/cmd/internal-tools/hier-check/main.go +++ b/cmd/internal-tools/hier-check/main.go @@ -1,3 +1,4 @@ +// Command hier-check is a developer tool that parses a hierarchy dump and reports selector matches. package main import ( diff --git a/cmd/sanderling/doctor.go b/cmd/sanderling/doctor.go index cfd579d..dafe612 100644 --- a/cmd/sanderling/doctor.go +++ b/cmd/sanderling/doctor.go @@ -14,7 +14,7 @@ import ( "github.com/chromedp/chromedp" - "github.com/priyanshujain/sanderling/internal/sidecar" + "github.com/priyanshujain/sanderling/internal/sidecarassets" ) type doctorCheck struct { @@ -99,10 +99,10 @@ func checkChromiumLaunch(ctx context.Context) error { } func checkSidecarJAR(_ context.Context) error { - if sidecar.IsPlaceholder() { + if sidecarassets.IsPlaceholder() { return fmt.Errorf("placeholder JAR embedded; run `make sidecar && make sanderling` to embed the real fat JAR") } - if sidecar.EmbeddedSize() == 0 { + if sidecarassets.EmbeddedSize() == 0 { return fmt.Errorf("embedded JAR is empty") } return nil diff --git a/cmd/sanderling/main.go b/cmd/sanderling/main.go index 557ddc1..302d283 100644 --- a/cmd/sanderling/main.go +++ b/cmd/sanderling/main.go @@ -1,3 +1,4 @@ +// Command sanderling is the CLI entry point for the property-based UI fuzzer. package main import ( diff --git a/docs/development/action-space.md b/docs/development/action-space.md index 79968be..65bc3f9 100644 --- a/docs/development/action-space.md +++ b/docs/development/action-space.md @@ -1,6 +1,6 @@ # Action Space -Inventory of user interaction primitives Sanderling currently supports, compared against the full set available through the Maestro driver layer. +Inventory of user interaction primitives Sanderling currently supports, compared against the full set available through the native driver layer. ## Current Actions @@ -15,9 +15,9 @@ Inventory of user interaction primitives Sanderling currently supports, compared | `PressKey` | `key` | back, home, enter, tab, up, down, left, right | | `Wait` | `durationMillis` | Sleep for N ms | -## Gaps vs Maestro Driver +## Gaps vs Native Driver -The Maestro driver layer (both `AndroidDriver` and `IOSDriver`) already implements these actions. They are not yet surfaced in Sanderling's action model. +The native driver layer (both `AndroidDriver` and `IOSDriver`) already implements these actions. They are not yet surfaced in Sanderling's action model. ### User Interaction Gaps @@ -41,7 +41,7 @@ These are less about user gesture modeling and more about test environment setup ### Out of Scope -These Maestro commands don't fit Sanderling's interaction model: +These native driver commands don't fit Sanderling's interaction model: - `LaunchApp`, `StopApp`, `KillApp`, `ClearState` - app lifecycle, handled outside the action stream - `AssertVisible`, `AssertWithAI` - Sanderling has its own property and assertion model diff --git a/docs/development/decisions.md b/docs/development/decisions.md index 454b9de..5f9c3bb 100644 --- a/docs/development/decisions.md +++ b/docs/development/decisions.md @@ -30,9 +30,9 @@ The `driver.go` file defines the `DeviceDriver` interface. Concrete implementati `bindings.go` currently holds shared types (`Action`, `ActionKind`, `LogEntry`, `Exception`) alongside JavaScript runtime wiring. The types half moves to `types.go` so the two concerns are separately navigable. -### `internal/permissions/` stays as-is +### `internal/permissions/` is deleted -Android-only package but there is no iOS equivalent yet. Revisit if iOS gets similar permission setup. +Dead code with zero importers. Removed. Reintroduce a permission helper only when a platform actually needs one. --- diff --git a/docs/manual/spec-language.md b/docs/manual/spec-language.md index fd0b202..3c157ea 100644 --- a/docs/manual/spec-language.md +++ b/docs/manual/spec-language.md @@ -191,7 +191,7 @@ On web, `"back"` maps to Backspace and `"home"` is not supported. All other keys | `taps` | Random tap on a clickable element | | `swipes` | Random swipe gesture | | `waitOnce` | Idles one step | -| `pressKey` | Presses a random supported key | +| `pressKeys` | Presses a random supported key | ### `actions(generator)` diff --git a/docs/manual/writing-specs.md b/docs/manual/writing-specs.md index 9603a8a..3b3bdab 100644 --- a/docs/manual/writing-specs.md +++ b/docs/manual/writing-specs.md @@ -126,7 +126,7 @@ Action generators return a list of actions to perform. The runner samples one fr - `taps` - autonomous random taps on clickable elements. - `swipes` - autonomous random swipe gestures. - `waitOnce` - idles one step. -- `pressKey` - presses a random supported key. +- `pressKeys` - presses a random supported key. **Action constructors:** diff --git a/examples/folio-web/sanderling/spec.ts b/examples/folio-web/sanderling/spec.ts index b201a49..842a743 100644 --- a/examples/folio-web/sanderling/spec.ts +++ b/examples/folio-web/sanderling/spec.ts @@ -3,9 +3,11 @@ import { Tap, actions, always, + edgeCaseText, eventually, extract, from, + integers, next, now, taps, @@ -15,11 +17,11 @@ import { import { noUncaughtExceptions } from "@sanderling/spec/defaults/properties"; // Page-presence checks via stable element ids. -const onLoginPage = extract((s) => !!s.ax.find({ id: "email" })); -const onHomePage = extract((s) => !!s.ax.find({ id: "add-account" })); -const onAddAccountPage = extract((s) => !!s.ax.find({ id: "account-name" })); -const onLedgerPage = extract((s) => !!s.ax.find({ id: "ledger" })); -const onAddTxnPage = extract((s) => !!s.ax.find({ id: "txn-amount" })); +const onLoginPage = extract((s) => !!s.ax.find({ id: "email" })).named("onLoginPage"); +const onHomePage = extract((s) => !!s.ax.find({ id: "add-account" })).named("onHomePage"); +const onAddAccountPage = extract((s) => !!s.ax.find({ id: "account-name" })).named("onAddAccountPage"); +const onLedgerPage = extract((s) => !!s.ax.find({ id: "ledger" })).named("onLedgerPage"); +const onAddTxnPage = extract((s) => !!s.ax.find({ id: "txn-amount" })).named("onAddTxnPage"); // Auth state: true on any authenticated page, false only on login page. const loggedIn = extract((s) => { @@ -32,7 +34,7 @@ const loggedIn = extract((s) => { s.ax.find({ id: "txn-amount" }) || s.ax.find({ id: "add-txn" }) ); -}); +}).named("loggedIn"); // Read raw cents off explicit data-cents attributes; no aria-label parsing. function readCents(value: string | undefined): number { @@ -44,7 +46,7 @@ function readCents(value: string | undefined): number { const totalBalance = extract((s) => { const el = s.ax.find({ id: "total-balance" }); return readCents(el?.attrs?.["data-cents"]); -}); +}).named("totalBalance"); // Account cards expose `data-account-id` + `data-balance` so the spec reads // structured data without parsing aria-label. @@ -54,33 +56,33 @@ const accountCards = extract((s) => { id: el.attrs?.["data-account-id"] ?? "", balance: readCents(el.attrs?.["data-balance"]), })); -}); +}).named("accountCards"); const ledgerTxnCount = extract((s) => { const el = s.ax.find({ id: "ledger" }); return readCents(el?.attrs?.["data-txn-count"]); -}); +}).named("ledgerTxnCount"); const ledgerBalance = extract((s) => { const el = s.ax.find({ id: "ledger-balance" }); return readCents(el?.attrs?.["data-cents"]); -}); +}).named("ledgerBalance"); // UI element handles. -const emailField = extract((s) => s.ax.find({ id: "email" })); -const passwordField = extract((s) => s.ax.find({ id: "password" })); -const loginSubmit = extract((s) => s.ax.find({ id: "login-submit" })); -const logoutButton = extract((s) => s.ax.find({ id: "logout" })); -const addAccountButton = extract((s) => s.ax.find({ id: "add-account" })); -const accountNameField = extract((s) => s.ax.find({ id: "account-name" })); -const addAccountSubmit = extract((s) => s.ax.find({ id: "add-account-submit" })); -const addTxnButton = extract((s) => s.ax.find({ id: "add-txn" })); -const txnAmountField = extract((s) => s.ax.find({ id: "txn-amount" })); -const txnNoteField = extract((s) => s.ax.find({ id: "txn-note" })); -const txnCreditButton = extract((s) => s.ax.find({ id: "txn-credit" })); -const txnDebitButton = extract((s) => s.ax.find({ id: "txn-debit" })); -const txnSubmit = extract((s) => s.ax.find({ id: "txn-submit" })); -const backButton = extract((s) => s.ax.find({ id: "back" })); +const emailField = extract((s) => s.ax.find({ id: "email" })).named("emailField"); +const passwordField = extract((s) => s.ax.find({ id: "password" })).named("passwordField"); +const loginSubmit = extract((s) => s.ax.find({ id: "login-submit" })).named("loginSubmit"); +const logoutButton = extract((s) => s.ax.find({ id: "logout" })).named("logoutButton"); +const addAccountButton = extract((s) => s.ax.find({ id: "add-account" })).named("addAccountButton"); +const accountNameField = extract((s) => s.ax.find({ id: "account-name" })).named("accountNameField"); +const addAccountSubmit = extract((s) => s.ax.find({ id: "add-account-submit" })).named("addAccountSubmit"); +const addTxnButton = extract((s) => s.ax.find({ id: "add-txn" })).named("addTxnButton"); +const txnAmountField = extract((s) => s.ax.find({ id: "txn-amount" })).named("txnAmountField"); +const txnNoteField = extract((s) => s.ax.find({ id: "txn-note" })).named("txnNoteField"); +const txnCreditButton = extract((s) => s.ax.find({ id: "txn-credit" })).named("txnCreditButton"); +const txnDebitButton = extract((s) => s.ax.find({ id: "txn-debit" })).named("txnDebitButton"); +const txnSubmit = extract((s) => s.ax.find({ id: "txn-submit" })).named("txnSubmit"); +const backButton = extract((s) => s.ax.find({ id: "back" })).named("backButton"); // -- Properties -- @@ -175,7 +177,9 @@ const openAddAccount = actions(() => { return btn ? [Tap({ on: btn })] : []; }); -const accountNameSampler = from([ +// Readable enumeration keeps the demo legible; repeats over a run still +// exercise duplicate-name handling. +const accountNames = from([ "Checking", "Savings", "Travel", @@ -183,17 +187,19 @@ const accountNameSampler = from([ "Emergency Fund", "Investments", "Groceries", - " ", - "Checking", - "A".repeat(41), "Petty Cash", ]); -const typeAccountName = actions(() => { - if (!onAddAccountPage.current) return []; - const field = accountNameField.current; - return field ? [InputText({ into: field, text: accountNameSampler.generate() })] : []; -}); +function typeAccountNameWith(sampler: { generate(): string }) { + return actions(() => { + if (!onAddAccountPage.current) return []; + const field = accountNameField.current; + return field ? [InputText({ into: field, text: sampler.generate() })] : []; + }); +} + +const typeAccountName = typeAccountNameWith(accountNames); +const typeAccountNameEdge = typeAccountNameWith(edgeCaseText()); const submitAddAccount = actions(() => { if (!onAddAccountPage.current) return []; @@ -205,8 +211,7 @@ const openAccount = actions(() => { if (!onHomePage.current) return []; const cards = accountCards.current; if (cards.length === 0) return []; - const card = cards[Math.floor(Math.random() * cards.length)]; - return [Tap({ on: card.element })]; + return [Tap({ on: from(cards).generate().element })]; }); const openAddTxn = actions(() => { @@ -215,25 +220,20 @@ const openAddTxn = actions(() => { return btn ? [Tap({ on: btn })] : []; }); -const amountSampler = from([ - "12.34", - "100", - "0.01", - "999.99", - "5.5", - "42", - "0", - "", - "1e4", - "0.001", - "-5", -]); +// Valid happy-path amounts keep the balance properties exercised; the edge +// branch (weighted in actionsRoot) stresses parsing with the adversarial corpus. +const validAmounts = integers().between(1, 99999); -const typeAmount = actions(() => { - if (!onAddTxnPage.current) return []; - const field = txnAmountField.current; - return field ? [InputText({ into: field, text: amountSampler.generate() })] : []; -}); +function typeAmountWith(sampler: { generate(): string }) { + return actions(() => { + if (!onAddTxnPage.current) return []; + const field = txnAmountField.current; + return field ? [InputText({ into: field, text: sampler.generate() })] : []; + }); +} + +const typeAmount = typeAmountWith({ generate: () => String(validAmounts.generate()) }); +const typeAmountEdge = typeAmountWith(edgeCaseText()); const noteSampler = from([ "Coffee", @@ -252,10 +252,9 @@ const typeNote = actions(() => { const toggleTxnType = actions(() => { if (!onAddTxnPage.current) return []; - const credit = txnCreditButton.current; - const debit = txnDebitButton.current; - const target = Math.random() < 0.5 ? credit : debit; - return target ? [Tap({ on: target })] : []; + const targets = [txnCreditButton.current, txnDebitButton.current].filter(Boolean); + if (targets.length === 0) return []; + return [Tap({ on: from(targets).generate() })]; }); const submitTxn = actions(() => { @@ -279,11 +278,13 @@ export const actionsRoot = weighted( [30, loginHelper], [2, adversarialLogin], [14, openAddAccount], - [18, typeAccountName], + [14, typeAccountName], + [4, typeAccountNameEdge], [14, submitAddAccount], [14, openAccount], [12, openAddTxn], - [18, typeAmount], + [14, typeAmount], + [4, typeAmountEdge], [8, typeNote], [6, toggleTxnType], [16, submitTxn], @@ -292,6 +293,3 @@ export const actionsRoot = weighted( [4, taps], [2, waitOnce], ); - -(globalThis as { actions?: unknown; properties?: unknown }).actions = actionsRoot; -(globalThis as { properties?: unknown }).properties = properties; diff --git a/examples/folio/sanderling/spec.ts b/examples/folio/sanderling/spec.ts index a323b6c..1e2f7a3 100644 --- a/examples/folio/sanderling/spec.ts +++ b/examples/folio/sanderling/spec.ts @@ -5,6 +5,7 @@ import { always, extract, from, + integers, next, weighted, whenRoute, @@ -146,7 +147,7 @@ const addAccount = whenRoute(route, ["home", "add-account"], () => { return opts; }); -const amounts = from(["10", "50", "25", "100", "5"]); +const amounts = integers().between(1, 500); const addTxn = whenRoute(route, ["home", "ledger", "add-transaction"], () => { if (route.current === "home") { @@ -161,7 +162,7 @@ const addTxn = whenRoute(route, ["home", "ledger", "add-transaction"], () => { const field = txnAmountField.current; const submit = txnSubmit.current; const opts = []; - if (field) opts.push(InputText({ into: field, text: amounts.generate() })); + if (field) opts.push(InputText({ into: field, text: String(amounts.generate()) })); if (submit) opts.push(Tap({ on: submit })); return opts; }); @@ -181,7 +182,3 @@ export const actionsRoot = weighted( [30, addTxn], [20, defaultActions], ); - -(globalThis as { actions?: unknown; properties?: unknown; setup?: unknown }).actions = actionsRoot; -(globalThis as { properties?: unknown }).properties = properties; -(globalThis as { setup?: unknown }).setup = setup; diff --git a/inspect-ui/src/types.ts b/inspect-ui/src/types.ts index bab0ade..773d18b 100644 --- a/inspect-ui/src/types.ts +++ b/inspect-ui/src/types.ts @@ -103,7 +103,8 @@ export interface Exception { export type ResidualNode = | { op: "true" } | { op: "false" } - | { op: "always" | "now" | "next" | "not"; arg: ResidualNode } + | { op: "now" | "next" | "not"; arg: ResidualNode } + | { op: "always"; arg: ResidualNode; within?: { amount: number; unit: string } } | { op: "eventually"; arg: ResidualNode; within?: { amount: number; unit: string } } | { op: "and" | "or" | "implies"; left: ResidualNode; right: ResidualNode } | { op: "predicate"; name?: string } diff --git a/internal/_oracle/main.go b/internal/_oracle/main.go new file mode 100644 index 0000000..dafd728 --- /dev/null +++ b/internal/_oracle/main.go @@ -0,0 +1,122 @@ +//go:build ignore + +// Oracle generator for the bit-exact PCG port. +// +// Emits the golden fixture consumed by pkg/spec/test/pcg.test.ts. The fixture +// pins the exact draw sequences produced by Go's math/rand/v2 over a PCG source +// so the TypeScript port (pkg/spec/src/pcg.ts) can be asserted bit-for-bit. +// +// Run from the spec package to regenerate: +// +// go run ./internal/_oracle > pkg/spec/test/fixtures/pcg-golden.json +package main + +import ( + "encoding/json" + "math/rand/v2" + "os" +) + +// seed carries the PCG seed pair. The values are emitted as decimal strings so +// JavaScript parses them into BigInt without the float precision loss it would +// suffer on values above 2^53. +type seed struct { + Hi string `json:"hi"` + Lo string `json:"lo"` +} + +type caseEntry struct { + Seed seed `json:"seed"` + Uint64 []string `json:"uint64"` + Float64 []float64 `json:"float64"` + IntN map[string][]int `json:"intN"` +} + +const drawCount = 40 + +var seeds = [][2]uint64{ + {1718000000000000000, 0}, + {42, 0}, + {0, 0}, + {1, 2}, + {18446744073709551615, 18446744073709551615}, +} + +var intNValues = []int{2, 3, 7, 15, 1000} + +func main() { + cases := make([]caseEntry, 0, len(seeds)) + + for _, pair := range seeds { + hi, lo := pair[0], pair[1] + entry := caseEntry{ + Seed: seed{Hi: formatUint64(hi), Lo: formatUint64(lo)}, + IntN: map[string][]int{}, + } + + r := rand.New(rand.NewPCG(hi, lo)) + for i := 0; i < drawCount; i++ { + entry.Uint64 = append(entry.Uint64, formatUint64(r.Uint64())) + } + + r = rand.New(rand.NewPCG(hi, lo)) + for i := 0; i < drawCount; i++ { + entry.Float64 = append(entry.Float64, r.Float64()) + } + + for _, n := range intNValues { + r = rand.New(rand.NewPCG(hi, lo)) + draws := make([]int, 0, drawCount) + for i := 0; i < drawCount; i++ { + draws = append(draws, r.IntN(n)) + } + entry.IntN[itoa(n)] = draws + } + + cases = append(cases, entry) + } + + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err := encoder.Encode(cases); err != nil { + panic(err) + } +} + +// formatUint64 renders a uint64 as a decimal string so JavaScript can parse it +// into a BigInt without precision loss. +func formatUint64(v uint64) string { + if v == 0 { + return "0" + } + var buf [20]byte + i := len(buf) + for v > 0 { + i-- + buf[i] = byte('0' + v%10) + v /= 10 + } + return string(buf[i:]) +} + +func itoa(n int) string { + if n == 0 { + return "0" + } + neg := n < 0 + if neg { + n = -n + } + var buf [20]byte + i := len(buf) + for n > 0 { + i-- + buf[i] = byte('0' + n%10) + n /= 10 + } + if neg { + i-- + buf[i] = '-' + } + return string(buf[i:]) +} diff --git a/internal/android/android.go b/internal/android/android.go index 0c7aa26..784e2f7 100644 --- a/internal/android/android.go +++ b/internal/android/android.go @@ -1,3 +1,4 @@ +// Package android boots and prepares an Android device or emulator for testing via adb. package android import ( diff --git a/internal/bundler/bundler.go b/internal/bundler/bundler.go index b26dca6..043c125 100644 --- a/internal/bundler/bundler.go +++ b/internal/bundler/bundler.go @@ -1,3 +1,4 @@ +// Package bundler compiles a TypeScript spec into a single JavaScript bundle via esbuild. package bundler import ( @@ -6,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "path/filepath" "strings" esbuild "github.com/evanw/esbuild/pkg/api" @@ -13,9 +15,14 @@ import ( type Options struct { EntryFile string - Defines map[string]string - Aliases map[string]string - Sourcemap bool + // RuntimeFile, when set, is imported BEFORE the spec via a stdin entry so + // the bundle installs __sanderlingNextAction__ / __sanderlingExtractors__ + // (the goja runtime entry wires the shared picker). Empty bundles the spec + // alone, as the bundle-check tool and unit fixtures do. + RuntimeFile string + Defines map[string]string + Aliases map[string]string + Sourcemap bool } type Result struct { @@ -45,18 +52,36 @@ func Bundle(options Options) (Result, error) { sourcemap = esbuild.SourceMapInline } - output := esbuild.Build(esbuild.BuildOptions{ - EntryPoints: []string{options.EntryFile}, - Bundle: true, - Format: esbuild.FormatIIFE, - Target: esbuild.ES2020, - Platform: esbuild.PlatformNeutral, - Define: defines, - Alias: options.Aliases, - Sourcemap: sourcemap, - Write: false, - LogLevel: esbuild.LogLevelSilent, - }) + buildOptions := esbuild.BuildOptions{ + Bundle: true, + Format: esbuild.FormatIIFE, + Target: esbuild.ES2020, + Platform: esbuild.PlatformNeutral, + Define: defines, + Alias: options.Aliases, + Sourcemap: sourcemap, + Write: false, + LogLevel: esbuild.LogLevelSilent, + } + if options.RuntimeFile == "" { + buildOptions.EntryPoints = []string{options.EntryFile} + } else { + runtimeAbs, err := filepath.Abs(options.RuntimeFile) + if err != nil { + return Result{}, fmt.Errorf("runtime path: %w", err) + } + specAbs, err := filepath.Abs(options.EntryFile) + if err != nil { + return Result{}, fmt.Errorf("entry path: %w", err) + } + buildOptions.Stdin = &esbuild.StdinOptions{ + Contents: fmt.Sprintf("import %q;\n%s", runtimeAbs, registrationEntry(specAbs)), + ResolveDir: filepath.Dir(specAbs), + Loader: esbuild.LoaderTS, + } + } + + output := esbuild.Build(buildOptions) if len(output.Errors) > 0 { var messages []string @@ -76,3 +101,16 @@ func Bundle(options Options) (Result, error) { SHA256: hex.EncodeToString(sum[:]), }, nil } + +// registrationEntry imports the spec as a namespace and copies its named +// exports onto globalThis so authors write plain `export const properties / +// actionsRoot / setup` without hand-assigning globalThis. The guards let web +// specs omit setup. esbuild keeps the exports because the trailer references +// them; output stays an IIFE. +func registrationEntry(specAbs string) string { + return fmt.Sprintf(`import * as __spec from %q; +if (__spec.actionsRoot !== undefined) globalThis.actions = __spec.actionsRoot; +if (__spec.properties !== undefined) globalThis.properties = __spec.properties; +if (__spec.setup !== undefined) globalThis.setup = __spec.setup; +`, specAbs) +} diff --git a/internal/bundler/bundler_test.go b/internal/bundler/bundler_test.go index 47eebf6..41e9cea 100644 --- a/internal/bundler/bundler_test.go +++ b/internal/bundler/bundler_test.go @@ -146,6 +146,59 @@ func TestBundle_ImportResolution(t *testing.T) { } } +func TestBundle_RegistersNamedExportsOnGlobalThis(t *testing.T) { + directory := t.TempDir() + runtimePath := filepath.Join(directory, "runtime.ts") + specPath := filepath.Join(directory, "spec.ts") + if err := os.WriteFile(runtimePath, []byte(`export {};`), 0o600); err != nil { + t.Fatal(err) + } + spec := ` + export const properties = "PROPS_MARKER"; + export const actionsRoot = "ACTIONS_MARKER"; + export const setup = "SETUP_MARKER"; + ` + if err := os.WriteFile(specPath, []byte(spec), 0o600); err != nil { + t.Fatal(err) + } + result, err := Bundle(Options{EntryFile: specPath, RuntimeFile: runtimePath}) + if err != nil { + t.Fatal(err) + } + body := string(result.JavaScript) + for _, marker := range []string{"PROPS_MARKER", "ACTIONS_MARKER", "SETUP_MARKER"} { + if !strings.Contains(body, marker) { + t.Errorf("named export %q not registered in bundle:\n%s", marker, body) + } + } + if !strings.Contains(body, "globalThis.actions") { + t.Errorf("trailer should assign globalThis.actions:\n%s", body) + } +} + +func TestBundle_RegistersWithoutSetupExport(t *testing.T) { + directory := t.TempDir() + runtimePath := filepath.Join(directory, "runtime.ts") + specPath := filepath.Join(directory, "spec.ts") + if err := os.WriteFile(runtimePath, []byte(`export {};`), 0o600); err != nil { + t.Fatal(err) + } + spec := ` + export const properties = "PROPS_MARKER"; + export const actionsRoot = "ACTIONS_MARKER"; + ` + if err := os.WriteFile(specPath, []byte(spec), 0o600); err != nil { + t.Fatal(err) + } + result, err := Bundle(Options{EntryFile: specPath, RuntimeFile: runtimePath}) + if err != nil { + t.Fatalf("spec without setup export should still bundle: %v", err) + } + if !strings.Contains(string(result.JavaScript), "PROPS_MARKER") { + t.Errorf("properties export not registered") + } +} + func TestBundle_ReportsSyntaxErrors(t *testing.T) { entry := writeFixture(t, "broken.ts", `const x = ;`) _, err := Bundle(Options{EntryFile: entry}) diff --git a/internal/bundler/web_bundle.go b/internal/bundler/web_bundle.go index af92d3c..0a35cb4 100644 --- a/internal/bundler/web_bundle.go +++ b/internal/bundler/web_bundle.go @@ -3,6 +3,7 @@ package bundler import ( "crypto/sha256" "encoding/hex" + "encoding/json" "errors" "fmt" "path/filepath" @@ -47,12 +48,14 @@ func BundleWeb(options WebOptions) (Result, error) { defines := map[string]string{} for key, value := range options.Defines { - defines["process.env."+key] = quoteJSString(value) + encoded, err := json.Marshal(value) + if err != nil { + return Result{}, fmt.Errorf("encode define %q: %w", key, err) + } + defines["process.env."+key] = string(encoded) } - stdinContents := fmt.Sprintf(`import %q; -import %q; -`, runtimeAbs, specAbs) + stdinContents := fmt.Sprintf("import %q;\n%s", runtimeAbs, registrationEntry(specAbs)) output := esbuild.Build(esbuild.BuildOptions{ Stdin: &esbuild.StdinOptions{ @@ -88,26 +91,3 @@ import %q; SHA256: hex.EncodeToString(sum[:]), }, nil } - -func quoteJSString(value string) string { - var builder strings.Builder - builder.WriteByte('"') - for index := 0; index < len(value); index++ { - c := value[index] - switch c { - case '"', '\\': - builder.WriteByte('\\') - builder.WriteByte(c) - case '\n': - builder.WriteString("\\n") - case '\r': - builder.WriteString("\\r") - case '\t': - builder.WriteString("\\t") - default: - builder.WriteByte(c) - } - } - builder.WriteByte('"') - return builder.String() -} diff --git a/internal/bundler/web_bundle_test.go b/internal/bundler/web_bundle_test.go index 0863101..2f5f515 100644 --- a/internal/bundler/web_bundle_test.go +++ b/internal/bundler/web_bundle_test.go @@ -34,8 +34,8 @@ export {}; const fakeSpec = ` const handle = (globalThis as { __sanderling__: { extract: (g: () => unknown) => unknown } }).__sanderling__.extract(() => 42); -(globalThis as { actions?: unknown }).actions = handle; -export {}; +export const actionsRoot = handle; +export const properties = "WEB_PROPS_MARKER"; ` func TestBundleWeb_RegistersExpectedGlobals(t *testing.T) { @@ -61,6 +61,8 @@ func TestBundleWeb_RegistersExpectedGlobals(t *testing.T) { "__sanderlingExtractors__", "__sanderlingNextAction__", "__sanderling__", + "WEB_PROPS_MARKER", + "globalThis.actions", } { if !strings.Contains(source, expected) { t.Errorf("bundle missing %q\nsource head:\n%s", expected, head(source, 500)) @@ -122,6 +124,38 @@ func TestBundleWeb_IsDeterministic(t *testing.T) { } } +// TestBundleWeb_InjectsSeedDefine asserts that a SANDERLING_SEED define is +// inlined into the bundle so --seed reaches the web runtime PRNG. esbuild +// replaces process.env.SANDERLING_SEED with the quoted literal at build time. +func TestBundleWeb_InjectsSeedDefine(t *testing.T) { + directory := t.TempDir() + runtimePath := filepath.Join(directory, "web-runtime.ts") + specPath := filepath.Join(directory, "spec.ts") + runtime := fakeRuntime + "\n(globalThis as Record).__seed = process.env.SANDERLING_SEED;\n" + if err := os.WriteFile(runtimePath, []byte(runtime), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(specPath, []byte(fakeSpec), 0o644); err != nil { + t.Fatal(err) + } + + result, err := BundleWeb(WebOptions{ + EntryFile: specPath, + WebRuntimeFile: runtimePath, + Defines: map[string]string{"SANDERLING_SEED": "8675309"}, + }) + if err != nil { + t.Fatalf("BundleWeb: %v", err) + } + source := string(result.JavaScript) + if !strings.Contains(source, "8675309") { + t.Errorf("seed literal not inlined into bundle\nsource head:\n%s", head(source, 500)) + } + if strings.Contains(source, "process.env.SANDERLING_SEED") { + t.Errorf("define left unsubstituted in bundle") + } +} + func head(text string, max int) string { if len(text) <= max { return text diff --git a/internal/driver/chrome/driver.go b/internal/driver/chrome/driver.go index 2b736be..4dc98eb 100644 --- a/internal/driver/chrome/driver.go +++ b/internal/driver/chrome/driver.go @@ -1,3 +1,4 @@ +// Package chrome implements the device driver for web targets by driving Chrome over the DevTools protocol. package chrome import ( @@ -37,6 +38,13 @@ func New() *Driver { chromedp.Flag("headless", true), chromedp.Flag("disable-gpu", true), chromedp.NoSandbox, + // CI runners give Chrome a tiny /dev/shm; without this the browser + // process hangs on startup and never reports its DevTools socket. + chromedp.Flag("disable-dev-shm-usage", true), + // Cold-starting Chrome on a loaded CI runner can take longer than the + // 20s default to print its DevTools websocket URL; give it more room + // so launch does not flake with "websocket url timeout reached". + chromedp.WSURLReadTimeout(60*time.Second), )..., ) tabCtx, tabCancel := chromedp.NewContext(allocCtx) @@ -142,6 +150,33 @@ func (d *Driver) TapSelector(_ context.Context, selector string) error { return chromedp.Run(d.tabCtx, chromedp.Click(target, chromedp.NodeVisible)) } +// doubleTapGap is the inter-tap delay for DoubleTap: short enough to land both +// events inside a sub-100 ms race window. The browser has no single double-tap +// primitive, so the gesture is two taps with this gap. +const doubleTapGap = 50 * time.Millisecond + +func (d *Driver) DoubleTap(ctx context.Context, x, y int) error { + return webDoubleTap(ctx, func() error { return d.Tap(ctx, x, y) }) +} + +func (d *Driver) DoubleTapSelector(ctx context.Context, selector string) error { + return webDoubleTap(ctx, func() error { return d.TapSelector(ctx, selector) }) +} + +func webDoubleTap(ctx context.Context, tap func() error) error { + if err := tap(); err != nil { + return err + } + timer := time.NewTimer(doubleTapGap) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } + return tap() +} + func (d *Driver) InputText(_ context.Context, text string) error { return chromedp.Run(d.tabCtx, chromedp.ActionFunc(func(ctx context.Context) error { diff --git a/internal/driver/chrome/driver_test.go b/internal/driver/chrome/driver_test.go index 1997544..399f663 100644 --- a/internal/driver/chrome/driver_test.go +++ b/internal/driver/chrome/driver_test.go @@ -1,3 +1,5 @@ +//go:build browser + package chrome import ( diff --git a/internal/driver/chrome/translate.go b/internal/driver/chrome/translate.go index 7fa6f81..561e09f 100644 --- a/internal/driver/chrome/translate.go +++ b/internal/driver/chrome/translate.go @@ -17,7 +17,7 @@ var attrNamePattern = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9_-]*$`) // "descPrefix:bar") into a CSS selector or XPath expression usable from the // chrome driver's TapSelector fallback path. The boolean return is true when // the result is XPath rather than CSS. Unknown prefixes pass through to a CSS -// attribute match by the same name so a Maestro-side addition (e.g. a future +// attribute match by the same name so a sidecar-side addition (e.g. a future // "role:") works without a Sanderling release. func TranslateStringSelector(selector string) (string, bool, error) { if selector == "" { diff --git a/internal/driver/driver.go b/internal/driver/driver.go index fb61368..6eb62f9 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -1,3 +1,4 @@ +// Package driver defines the platform-agnostic device automation interface and shared types. package driver import ( @@ -16,6 +17,8 @@ type DeviceDriver interface { Tap(ctx context.Context, x, y int) error TapSelector(ctx context.Context, selector string) error + DoubleTap(ctx context.Context, x, y int) error + DoubleTapSelector(ctx context.Context, selector string) error InputText(ctx context.Context, text string) error Swipe(ctx context.Context, fromX, fromY, toX, toY int, duration time.Duration) error PressKey(ctx context.Context, key string) error diff --git a/internal/driver/mock/mock.go b/internal/driver/mock/mock.go index 731148e..a6f0f13 100644 --- a/internal/driver/mock/mock.go +++ b/internal/driver/mock/mock.go @@ -1,3 +1,4 @@ +// Package mock provides an in-memory device driver that records actions for tests. package mock import ( @@ -11,37 +12,39 @@ import ( type ActionKind string const ( - ActionLaunch ActionKind = "launch" - ActionTerminate ActionKind = "terminate" - ActionTap ActionKind = "tap" - ActionTapSelector ActionKind = "tap_selector" - ActionInputText ActionKind = "input_text" - ActionSwipe ActionKind = "swipe" - ActionPressKey ActionKind = "press_key" - ActionLongPress ActionKind = "long_press" - ActionHierarchy ActionKind = "hierarchy" - ActionScreenshot ActionKind = "screenshot" - ActionSnapshot ActionKind = "snapshot" - ActionRecentLogs ActionKind = "recent_logs" - ActionWaitForIdle ActionKind = "wait_for_idle" - ActionHealth ActionKind = "health" - ActionMetrics ActionKind = "metrics" + ActionLaunch ActionKind = "launch" + ActionTerminate ActionKind = "terminate" + ActionTap ActionKind = "tap" + ActionTapSelector ActionKind = "tap_selector" + ActionDoubleTap ActionKind = "double_tap" + ActionDoubleTapSelector ActionKind = "double_tap_selector" + ActionInputText ActionKind = "input_text" + ActionSwipe ActionKind = "swipe" + ActionPressKey ActionKind = "press_key" + ActionLongPress ActionKind = "long_press" + ActionHierarchy ActionKind = "hierarchy" + ActionScreenshot ActionKind = "screenshot" + ActionSnapshot ActionKind = "snapshot" + ActionRecentLogs ActionKind = "recent_logs" + ActionWaitForIdle ActionKind = "wait_for_idle" + ActionHealth ActionKind = "health" + ActionMetrics ActionKind = "metrics" ) type Action struct { - Kind ActionKind - BundleID string - ClearState bool - X, Y int + Kind ActionKind + BundleID string + ClearState bool + X, Y int FromX, FromY int - ToX, ToY int - Duration time.Duration - Selector string - Text string - Key string - LogLevel string - LogSince time.Time - Idle time.Duration + ToX, ToY int + Duration time.Duration + Selector string + Text string + Key string + LogLevel string + LogSince time.Time + Idle time.Duration } // Driver is an in-memory Driver implementation for unit tests. @@ -183,6 +186,22 @@ func (d *Driver) TapSelector(ctx context.Context, selector string) error { return nil } +func (d *Driver) DoubleTap(ctx context.Context, x, y int) error { + if err := d.failure(ActionDoubleTap); err != nil { + return err + } + d.record(Action{Kind: ActionDoubleTap, X: x, Y: y}) + return nil +} + +func (d *Driver) DoubleTapSelector(ctx context.Context, selector string) error { + if err := d.failure(ActionDoubleTapSelector); err != nil { + return err + } + d.record(Action{Kind: ActionDoubleTapSelector, Selector: selector}) + return nil +} + func (d *Driver) InputText(ctx context.Context, text string) error { if err := d.failure(ActionInputText); err != nil { return err diff --git a/internal/driver/mock/mock_test.go b/internal/driver/mock/mock_test.go index d56da5c..488403d 100644 --- a/internal/driver/mock/mock_test.go +++ b/internal/driver/mock/mock_test.go @@ -9,22 +9,6 @@ import ( "github.com/priyanshujain/sanderling/internal/driver" ) -func TestNew_Defaults(t *testing.T) { - mock := New() - if !mock.HealthInfo.Ready { - t.Errorf("default HealthInfo should be ready") - } - if mock.HealthInfo.Platform != "android" { - t.Errorf("default platform: got %q", mock.HealthInfo.Platform) - } - if mock.HierarchyJSON == "" { - t.Errorf("default hierarchy should be a non-empty JSON") - } - if len(mock.Actions()) != 0 { - t.Errorf("fresh mock should have zero recorded actions") - } -} - func TestRecordsAllActionsInOrder(t *testing.T) { mock := New() ctx := context.Background() diff --git a/internal/driver/sidecar/client.go b/internal/driver/sidecar/client.go index f0b4e86..7374a47 100644 --- a/internal/driver/sidecar/client.go +++ b/internal/driver/sidecar/client.go @@ -1,3 +1,4 @@ +// Package sidecar implements the device driver by talking to the native sidecar over gRPC. package sidecar import ( @@ -103,6 +104,34 @@ func (c *Client) TapSelector(ctx context.Context, selector string) error { return err } +// doubleTapGap is the inter-tap delay for DoubleTap: short enough to land both +// events inside a sub-100 ms race window, long enough for the sidecar to +// serialize two MotionEvent streams. The sidecar exposes no native double-tap +// RPC, so the gesture is two Taps with this gap. +const doubleTapGap = 50 * time.Millisecond + +func (c *Client) DoubleTap(ctx context.Context, x, y int) error { + return doubleTap(ctx, func() error { return c.Tap(ctx, x, y) }) +} + +func (c *Client) DoubleTapSelector(ctx context.Context, selector string) error { + return doubleTap(ctx, func() error { return c.TapSelector(ctx, selector) }) +} + +func doubleTap(ctx context.Context, tap func() error) error { + if err := tap(); err != nil { + return err + } + timer := time.NewTimer(doubleTapGap) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } + return tap() +} + func (c *Client) InputText(ctx context.Context, text string) error { _, err := c.stub.InputText(ctx, &driverpb.Text{Value: text}) return err diff --git a/internal/hierarchy/hierarchy.go b/internal/hierarchy/hierarchy.go index 65b142f..ea87bcf 100644 --- a/internal/hierarchy/hierarchy.go +++ b/internal/hierarchy/hierarchy.go @@ -1,4 +1,4 @@ -// Package hierarchy parses the TreeNode JSON produced by the Maestro sidecar +// Package hierarchy parses the TreeNode JSON produced by the native sidecar // and resolves selectors against it. // // Selector grammar (v2.0): @@ -83,7 +83,7 @@ type Tree struct { Elements []*Element `json:"elements"` } -// treeNodeJSON mirrors the Maestro TreeNode JSON structure. +// treeNodeJSON mirrors the sidecar TreeNode JSON structure. type treeNodeJSON struct { Attributes map[string]string `json:"attributes"` Children []treeNodeJSON `json:"children"` @@ -110,7 +110,7 @@ type AttrFilter struct { // in the TreeNode attributes map. Both directions are listed so cross-platform // matching works regardless of which name the caller uses. var attributeAliases = map[string][]string{ - // Android XML legacy name; web driver uses content-desc; Maestro normalises to accessibilityText + // Android XML legacy name; web driver uses content-desc; the sidecar normalises to accessibilityText "content-desc": {"accessibilityText"}, // iOS AXElement / UIKit names "label": {"accessibilityText"}, @@ -164,7 +164,7 @@ func matchSelector(element *Element, sel Selector) bool { return true } -// Parse parses a Maestro TreeNode JSON hierarchy. +// Parse parses a sidecar TreeNode JSON hierarchy. func Parse(text string) (*Tree, error) { text = strings.TrimSpace(text) if text == "" { @@ -550,7 +550,7 @@ func match(element *Element, kind, value string) bool { } } -// boundsPattern matches "[l,t,r,b]" (4-value Android/Maestro format). +// boundsPattern matches "[l,t,r,b]" (4-value Android/sidecar format). var boundsPattern = regexp.MustCompile(`^\[(-?\d+),(-?\d+),(-?\d+),(-?\d+)\]$`) // boundsPatternTwo matches "[x1,y1][x2,y2]" (iOS XCUITest format). diff --git a/internal/hierarchy/hierarchy_test.go b/internal/hierarchy/hierarchy_test.go index e59b1e8..e430841 100644 --- a/internal/hierarchy/hierarchy_test.go +++ b/internal/hierarchy/hierarchy_test.go @@ -2,7 +2,7 @@ package hierarchy import "testing" -// sampleDump is a Maestro TreeNode JSON equivalent of the old XML fixture. +// sampleDump is a sidecar TreeNode JSON equivalent of the old XML fixture. const sampleDump = `{ "attributes": {"class": "android.widget.LinearLayout", "package": "app", "bounds": "[0,0,1080,2340]"}, "children": [ diff --git a/internal/inspect/assets.go b/internal/inspect/assets.go index 75a28ea..5442d2b 100644 --- a/internal/inspect/assets.go +++ b/internal/inspect/assets.go @@ -1,3 +1,4 @@ +// Package inspect serves the embedded web UI for browsing recorded runs. package inspect import ( diff --git a/internal/ios/ios.go b/internal/ios/ios.go index c195b20..f91ed13 100644 --- a/internal/ios/ios.go +++ b/internal/ios/ios.go @@ -1,3 +1,4 @@ +// Package ios boots and prepares an iOS simulator for testing via simctl. package ios import ( diff --git a/internal/ltl/evaluator.go b/internal/ltl/evaluator.go index 783df8c..01589b0 100644 --- a/internal/ltl/evaluator.go +++ b/internal/ltl/evaluator.go @@ -1,3 +1,4 @@ +// Package ltl evaluates linear temporal logic formulas incrementally over observed steps. package ltl import ( @@ -31,13 +32,27 @@ func (v Verdict) String() string { // violated) or carries them forward as residuals. Once a single obligation // violates, the overall verdict latches to Violated. type Evaluator struct { - root Formula - pending []Formula - violated bool + root Formula + pending []Formula + violated bool + steps int + violation *Violation +} + +// Violation is the witness for a latched verdict: the failing sub-formula, a +// human-readable reason, and the observation step it fired at. A thrown +// predicate carries the goja error text as its reason and sets IsError; a plain +// false carries "predicate false"; Finalize fills it for liveness obligations +// that never discharged. +type Violation struct { + Formula Formula + Reason string + Step int + IsError bool } func NewEvaluator(formula Formula) *Evaluator { - return &Evaluator{root: formula} + return &Evaluator{root: nnf(formula)} } // Observe evaluates the formula against the current state and returns the @@ -52,6 +67,7 @@ func (e *Evaluator) ObserveAt(now time.Time) Verdict { if e.violated { return VerdictViolated } + e.steps++ fresh := rootObligation(e.root) obligations := append(e.pending, fresh) @@ -65,18 +81,136 @@ func (e *Evaluator) ObserveAt(now time.Time) Verdict { case statusViolated: e.violated = true e.pending = nil + e.violation = result.witness + if e.violation != nil { + e.violation.Step = e.steps + } return VerdictViolated case statusPending: e.pending = append(e.pending, result.formula) } } + e.pending = collapse(e.pending) + if len(e.pending) > 0 { return VerdictPending } return VerdictHolds } +// collapse removes structurally-identical obligations, keeping the first +// occurrence in order. Distinct predicates never merge because ThunkFormula's +// name participates in its describe() key, so deduping cannot hide a violation. +func collapse(obligations []Formula) []Formula { + if len(obligations) < 2 { + return obligations + } + seen := make(map[string]struct{}, len(obligations)) + result := obligations[:0] + for _, obligation := range obligations { + key := obligation.describe() + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + result = append(result, obligation) + } + return result +} + +// Finalize reports the terminal verdict for the run. Pending obligations that +// can never be discharged by a future step (an unbounded eventually that never +// fired, a strong next with no successor) resolve to Violated; safety +// obligations that were never breached resolve to Holds. +func (e *Evaluator) Finalize() Verdict { + if e.violated { + return VerdictViolated + } + for _, obligation := range e.pending { + if finalize(obligation) == statusViolated { + e.violated = true + e.pending = nil + e.violation = &Violation{ + Formula: obligation, + Reason: finalizeReason(obligation), + Step: e.steps, + } + return VerdictViolated + } + } + return VerdictHolds +} + +// Violation returns the witness for a latched violation, or nil if the +// evaluator has not violated. The witness is set by ObserveAt at the step a +// reduction first violated, or by Finalize for a liveness obligation that +// never discharged. +func (e *Evaluator) Violation() *Violation { + return e.violation +} + +// finalizeReason describes why an undischarged obligation resolves to violated +// at run end. +func finalizeReason(formula Formula) string { + switch formula.(type) { + case EventuallyFormula: + return "eventually never satisfied" + case NextFormula: + return "next obligation unmet at run end" + case ThunkFormula: + return "obligation unmet at run end" + default: + return "liveness obligation unmet at run end" + } +} + +// finalize collapses a pending obligation to its terminal status assuming no +// further steps will occur. +func finalize(formula Formula) residualStatus { + switch concrete := formula.(type) { + case PureFormula: + if concrete.Value { + return statusHolds + } + return statusViolated + case ThunkFormula: + return statusViolated + case EventuallyFormula: + return statusViolated + case NextFormula: + return statusViolated + case AlwaysFormula: + return statusHolds + case NowFormula: + return finalize(concrete.Inner) + case NotFormula: + switch finalize(concrete.Inner) { + case statusViolated: + return statusHolds + default: + return statusViolated + } + case AndFormula: + if finalize(concrete.Left) == statusViolated || finalize(concrete.Right) == statusViolated { + return statusViolated + } + return statusHolds + case OrFormula: + if finalize(concrete.Left) == statusHolds || finalize(concrete.Right) == statusHolds { + return statusHolds + } + return statusViolated + case ImpliesFormula: + if finalize(concrete.Antecedent) == statusViolated { + return statusHolds + } + return finalize(concrete.Consequent) + default: + return statusHolds + } +} + // Residual returns a single Formula describing what the evaluator still has // to prove after the most recent ObserveAt. PureFormula{true} means the // property holds for the run so far; PureFormula{false} means it has latched @@ -119,10 +253,46 @@ const ( type reduceResult struct { status residualStatus formula Formula + witness *Violation } -func holds() reduceResult { return reduceResult{status: statusHolds} } +func holds() reduceResult { return reduceResult{status: statusHolds} } + +// violated reports a violation without an attached witness. Used where the +// failing sub-formula is recovered from a child result whose own witness is +// carried up by violatedFrom. func violated() reduceResult { return reduceResult{status: statusViolated} } + +// violatedWith reports a violation that originates at the given sub-formula +// with the given reason. The reason distinguishes a thrown predicate from a +// plain false so callers (and the inspect UI) can render the cause. +func violatedWith(formula Formula, reason string) reduceResult { + return reduceResult{ + status: statusViolated, + witness: &Violation{Formula: formula, Reason: reason}, + } +} + +// violatedByError reports a violation caused by a predicate that threw. The +// witness keeps the error text as its reason and flags IsError so callers can +// render it as a thrown-predicate error rather than a plain false. +func violatedByError(formula Formula, reason string) reduceResult { + return reduceResult{ + status: statusViolated, + witness: &Violation{Formula: formula, Reason: reason, IsError: true}, + } +} + +// violatedFrom propagates a child violation, preferring the child's witness so +// the deepest failing leaf survives. When the child carried no witness the +// fallback formula and reason describe this level instead. +func violatedFrom(child reduceResult, fallback Formula, reason string) reduceResult { + if child.witness != nil { + return reduceResult{status: statusViolated, witness: child.witness} + } + return violatedWith(fallback, reason) +} + func pending(f Formula) reduceResult { return reduceResult{status: statusPending, formula: f} } @@ -133,13 +303,17 @@ func reduce(formula Formula, now time.Time) reduceResult { if concrete.Value { return holds() } - return violated() + return violatedWith(concrete, "pure false") case ThunkFormula: - if concrete.Func() { + result, err := concrete.Func() + if err != nil { + return violatedByError(concrete, err.Error()) + } + if result { return holds() } - return violated() + return violatedWith(concrete, "predicate false") case NowFormula: return reduce(concrete.Inner, now) @@ -162,10 +336,10 @@ func reduce(formula Formula, now time.Time) reduceResult { return holds() } if concrete.HasStepBound && concrete.StepBound <= 1 { - return violated() + return violatedFrom(innerResult, concrete, "eventually bound exhausted") } if concrete.HasDeadline && !now.Before(concrete.Deadline) { - return violated() + return violatedFrom(innerResult, concrete, "eventually deadline reached") } next := concrete if concrete.HasStepBound { @@ -174,18 +348,14 @@ func reduce(formula Formula, now time.Time) reduceResult { return pending(next) case ImpliesFormula: - antecedent := reduce(concrete.Antecedent, now) - switch antecedent.status { - case statusHolds: - return reduce(concrete.Consequent, now) - case statusViolated: - return holds() - case statusPending: - return pending(ImpliesFormula{ - Antecedent: antecedent.formula, - Consequent: concrete.Consequent, - }) - } + // NewEvaluator runs nnf, which rewrites a -> b to (not a) or b, so this + // case is unreachable from a normal evaluator. A directly-constructed + // formula reduced here is evaluated through the same equivalence so a + // pending antecedent cannot drop the consequent. + return reduce(OrFormula{ + Left: pushNot(concrete.Antecedent), + Right: nnf(concrete.Consequent), + }, now) case OrFormula: left := reduce(concrete.Left, now) @@ -194,7 +364,7 @@ func reduce(formula Formula, now time.Time) reduceResult { return holds() } if left.status == statusViolated && right.status == statusViolated { - return violated() + return violatedFrom(left, concrete, "both disjuncts violated") } if left.status == statusViolated { return pending(right.formula) @@ -207,8 +377,11 @@ func reduce(formula Formula, now time.Time) reduceResult { case AndFormula: left := reduce(concrete.Left, now) right := reduce(concrete.Right, now) - if left.status == statusViolated || right.status == statusViolated { - return violated() + if left.status == statusViolated { + return violatedFrom(left, concrete, "conjunct violated") + } + if right.status == statusViolated { + return violatedFrom(right, concrete, "conjunct violated") } if left.status == statusHolds && right.status == statusHolds { return holds() @@ -225,7 +398,7 @@ func reduce(formula Formula, now time.Time) reduceResult { inner := reduce(concrete.Inner, now) switch inner.status { case statusHolds: - return violated() + return violatedWith(concrete, "negated formula held") case statusViolated: return holds() case statusPending: @@ -233,11 +406,38 @@ func reduce(formula Formula, now time.Time) reduceResult { } case AlwaysFormula: + // First-reduction deadline resolution mirrors EventuallyFormula so a + // relative duration becomes a stable absolute deadline. + if !concrete.HasDeadline && concrete.Duration > 0 { + concrete.Deadline = now.Add(concrete.Duration) + concrete.HasDeadline = true + } innerResult := reduce(concrete.Inner, now) if innerResult.status == statusViolated { - return violated() + return violatedFrom(innerResult, concrete, "always inner violated") + } + // A bounded Always is the dual of a bounded Eventually: once the window + // closes without a breach it is vacuously satisfied. A pending inner at + // the closing step is a deferred obligation (a strong next, or an inner + // liveness that has not discharged); it must be carried so a later step + // or Finalize resolves it, never dropped to holds. + if concrete.HasStepBound && concrete.StepBound <= 1 { + if innerResult.status == statusHolds { + return holds() + } + return pending(innerResult.formula) + } + if concrete.HasDeadline && !now.Before(concrete.Deadline) { + if innerResult.status == statusHolds { + return holds() + } + return pending(innerResult.formula) + } + next := concrete + next.Inner = concrete.Inner + if concrete.HasStepBound { + next.StepBound = concrete.StepBound - 1 } - next := AlwaysFormula{Inner: concrete.Inner} if innerResult.status == statusHolds { return pending(next) } diff --git a/internal/ltl/evaluator_test.go b/internal/ltl/evaluator_test.go index f93ecce..6d0628e 100644 --- a/internal/ltl/evaluator_test.go +++ b/internal/ltl/evaluator_test.go @@ -36,10 +36,10 @@ func TestPure_FalseImmediatelyViolates(t *testing.T) { func TestThunk_TransitionFromHoldToViolate(t *testing.T) { values := []bool{true, true, false, true, true} step := 0 - evaluator := NewEvaluator(Always(Thunk(func() bool { + evaluator := NewEvaluator(Always(Thunk(func() (bool, error) { current := values[step] step++ - return current + return current, nil }))) wantSequence := []Verdict{ @@ -59,7 +59,7 @@ func TestThunk_TransitionFromHoldToViolate(t *testing.T) { func TestEvaluator_StickinessAfterViolation(t *testing.T) { state := true - evaluator := NewEvaluator(Always(Thunk(func() bool { return state }))) + evaluator := NewEvaluator(Always(Thunk(func() (bool, error) { return state, nil }))) if got := evaluator.Observe(); got != VerdictHolds { t.Fatalf("step 1: got %v, want holds", got) @@ -83,7 +83,7 @@ func TestEvaluator_TopLevelPureCountedAtEachStep(t *testing.T) { func TestEvaluator_TopLevelThunkRespectsObservation(t *testing.T) { state := true - evaluator := NewEvaluator(Thunk(func() bool { return state })) + evaluator := NewEvaluator(Thunk(func() (bool, error) { return state, nil })) if got := evaluator.Observe(); got != VerdictHolds { t.Errorf("expected holds, got %v", got) } @@ -93,21 +93,12 @@ func TestEvaluator_TopLevelThunkRespectsObservation(t *testing.T) { } } -func TestVerdict_String(t *testing.T) { - if VerdictHolds.String() != "holds" { - t.Errorf("VerdictHolds.String() = %q", VerdictHolds.String()) - } - if VerdictViolated.String() != "violated" { - t.Errorf("VerdictViolated.String() = %q", VerdictViolated.String()) - } -} - func TestDescribe(t *testing.T) { formula := Always(Pure(true)) if got := Describe(formula); !strings.Contains(got, "Always") || !strings.Contains(got, "Pure(true)") { t.Errorf("Describe wrong: %q", got) } - thunk := Always(Thunk(func() bool { return true })) + thunk := Always(Thunk(func() (bool, error) { return true, nil })) if got := Describe(thunk); !strings.Contains(got, "Thunk") { t.Errorf("Describe(thunk) wrong: %q", got) } diff --git a/internal/ltl/false_negative_test.go b/internal/ltl/false_negative_test.go new file mode 100644 index 0000000..2a2b81c --- /dev/null +++ b/internal/ltl/false_negative_test.go @@ -0,0 +1,92 @@ +package ltl + +import ( + "testing" + "time" +) + +// thunkSeq returns a predicate that yields the given boolean per observation, +// repeating the last value once the sequence is exhausted. +func thunkSeq(values ...bool) func() (bool, error) { + step := 0 + return func() (bool, error) { + value := values[len(values)-1] + if step < len(values) { + value = values[step] + } + step++ + return value, nil + } +} + +func runAndFinalize(formula Formula, steps int) (Verdict, *Evaluator) { + evaluator := NewEvaluator(formula) + var last Verdict + for index := range steps { + last = evaluator.ObserveAt(time.Unix(int64(index), 0)) + if last == VerdictViolated { + return last, evaluator + } + } + return evaluator.Finalize(), evaluator +} + +// Implies with a temporal antecedent must still evaluate the consequent at the +// current step. Always(p) holds over the observed run, so a false consequent +// Not(q) at the last step makes the implication violated. The pre-fix engine +// deferred the whole implication and reported Holds. +func TestImplies_TemporalAntecedent_ConsequentFalseViolates(t *testing.T) { + p := Thunk(thunkSeq(true, true, true)) + q := Thunk(thunkSeq(false, false, true)) + formula := Implies(Always(p), Not(q)) + if verdict, _ := runAndFinalize(formula, 3); verdict != VerdictViolated { + t.Fatalf("Implies(Always(p),Not(q)) p=TTT q=FFT: got %v, want violated", verdict) + } +} + +// Single-step witness of the same class: antecedent holds, consequent false. +func TestImplies_TemporalAntecedent_SingleStepViolates(t *testing.T) { + formula := Implies(Always(Thunk(thunkSeq(true))), Not(Thunk(thunkSeq(true)))) + if verdict, _ := runAndFinalize(formula, 1); verdict != VerdictViolated { + t.Fatalf("Implies(Always(true),Not(true)): got %v, want violated", verdict) + } +} + +// A consequent inside an Or must not mask the violation either. +func TestImplies_TemporalAntecedent_OrConsequentViolates(t *testing.T) { + formula := Implies(Always(Thunk(thunkSeq(true))), Or(Not(Thunk(thunkSeq(true))), Pure(false))) + if verdict, _ := runAndFinalize(formula, 1); verdict != VerdictViolated { + t.Fatalf("Implies(Always(true),Or(Not(true),false)): got %v, want violated", verdict) + } +} + +// Control: a false antecedent makes the implication vacuously hold. +func TestImplies_TemporalAntecedent_FalseAntecedentHolds(t *testing.T) { + p := Thunk(thunkSeq(true, true, false)) + q := Thunk(thunkSeq(false, false, true)) + formula := Implies(Always(p), Not(q)) + if verdict, _ := runAndFinalize(formula, 3); verdict != VerdictHolds { + t.Fatalf("Implies(Always(p),Not(q)) p=TTF q=FFT: got %v, want holds", verdict) + } +} + +// A bounded Always whose inner is a still-pending deferred Next obligation must +// carry that obligation past the window, not drop it to holds. The pre-fix +// window-close branch returned holds() unconditionally and lost the violation. +func TestBoundedAlways_PendingInnerCarried(t *testing.T) { + inner := AlwaysFormula{Inner: Next(Thunk(thunkSeq(false))), StepBound: 1, HasStepBound: true} + formula := Always(inner) + if verdict, _ := runAndFinalize(formula, 3); verdict != VerdictViolated { + t.Fatalf("Always(boundedAlways(Next(false),1)): got %v, want violated", verdict) + } +} + +// A bounded Always whose inner genuinely holds each step stays satisfied: the +// fix must not turn a satisfied bounded window into a false positive. +func TestBoundedAlways_HoldingInnerStillHolds(t *testing.T) { + inner := AlwaysFormula{Inner: Not(Thunk(thunkSeq(false))), StepBound: 1, HasStepBound: true} + formula := Always(inner) + if verdict, _ := runAndFinalize(formula, 3); verdict != VerdictHolds { + t.Fatalf("Always(boundedAlways(Not(false),1)): got %v, want holds", verdict) + } +} diff --git a/internal/ltl/finalize_test.go b/internal/ltl/finalize_test.go new file mode 100644 index 0000000..d16a6b5 --- /dev/null +++ b/internal/ltl/finalize_test.go @@ -0,0 +1,161 @@ +package ltl + +import ( + "testing" + "testing/quick" + "time" +) + +func TestFinalize_UnboundedEventuallyUnmetIsViolated(t *testing.T) { + evaluator := NewEvaluator(Eventually(ThunkNamed("p", func() (bool, error) { return false, nil }))) + for index := range 3 { + if got := evaluator.ObserveAt(time.Unix(int64(index), 0)); got != VerdictPending { + t.Fatalf("step %d: got %v, want pending", index, got) + } + } + if got := evaluator.Finalize(); got != VerdictViolated { + t.Errorf("Finalize = %v, want violated", got) + } +} + +func TestFinalize_FinalStepNextIsViolated(t *testing.T) { + evaluator := NewEvaluator(Next(ThunkNamed("p", func() (bool, error) { return true, nil }))) + if got := evaluator.Observe(); got != VerdictPending { + t.Fatalf("step 1: got %v, want pending", got) + } + if got := evaluator.Finalize(); got != VerdictViolated { + t.Errorf("Finalize = %v, want violated", got) + } +} + +func TestFinalize_HoldingRunStaysHolds(t *testing.T) { + evaluator := NewEvaluator(Always(Pure(true))) + evaluator.Observe() + if got := evaluator.Finalize(); got != VerdictHolds { + t.Errorf("Finalize = %v, want holds", got) + } +} + +func TestFinalize_AlreadyViolatedStaysViolated(t *testing.T) { + evaluator := NewEvaluator(Always(Pure(false))) + if got := evaluator.Observe(); got != VerdictViolated { + t.Fatalf("expected violated, got %v", got) + } + if got := evaluator.Finalize(); got != VerdictViolated { + t.Errorf("Finalize = %v, want violated", got) + } +} + +func TestFinalize_BoundedAlwaysVacuouslyHolds(t *testing.T) { + // A bounded Always whose window never closed (still pending) is safe. + evaluator := NewEvaluator(EventuallyWithinSteps(Pure(false), 5)) + evaluator.Observe() + // The negated form of this is a bounded Always; build it directly. + bounded := NewEvaluator(Always(Not(EventuallyWithinSteps(ThunkNamed("p", func() (bool, error) { return false, nil }), 5)))) + bounded.Observe() + if got := bounded.Finalize(); got == VerdictViolated { + t.Errorf("bounded always should not finalize to violated, got %v", got) + } +} + +// TestEventuallyWithin_ViolatesIffNConsecutiveFalse locks the bounded +// eventually contract: with a step bound of n and an inner that is false for +// the first n observations, the verdict violates exactly at step n, and with at +// least one true observation inside the window it holds. +func TestEventuallyWithin_ViolatesIffNConsecutiveFalse(t *testing.T) { + law := func(boundSeed uint8, trueAtSeed uint8) bool { + bound := int(boundSeed%5) + 1 + // trueAt < 0 means inner is never true. + trueAt := int(trueAtSeed)%(bound+2) - 1 + step := 0 + inner := ThunkNamed("p", func() (bool, error) { + current := trueAt >= 0 && step == trueAt + return current, nil + }) + evaluator := NewEvaluator(EventuallyWithinSteps(inner, bound)) + + satisfiedInWindow := trueAt >= 0 && trueAt < bound + var final Verdict = VerdictPending + for index := range bound { + step = index + final = evaluator.ObserveAt(time.Unix(int64(index), 0)) + if final == VerdictHolds || final == VerdictViolated { + break + } + } + + if satisfiedInWindow { + return final == VerdictHolds + } + return final == VerdictViolated + } + if err := quick.Check(law, nil); err != nil { + t.Error(err) + } +} + +// TestViolationLatchIsMonotonic locks: once an evaluator reports Violated, every +// subsequent observation (and Finalize) stays Violated regardless of inputs. +func TestViolationLatchIsMonotonic(t *testing.T) { + law := func(seed uint64) bool { + values := make([]bool, 8) + for index := range values { + values[index] = (seed>>uint(index))&1 == 1 + } + step := 0 + evaluator := NewEvaluator(Always(ThunkNamed("p", func() (bool, error) { + current := values[step%len(values)] + step++ + return current, nil + }))) + seenViolated := false + for index := range 16 { + got := evaluator.ObserveAt(time.Unix(int64(index), 0)) + if got == VerdictViolated { + seenViolated = true + } else if seenViolated { + return false + } + } + if seenViolated && evaluator.Finalize() != VerdictViolated { + return false + } + return true + } + if err := quick.Check(law, nil); err != nil { + t.Error(err) + } +} + +func TestCollapse_IdenticalObligationsMerge(t *testing.T) { + merged := collapse([]Formula{ + Next(Pure(true)), + Next(Pure(true)), + Next(Pure(true)), + }) + if len(merged) != 1 { + t.Errorf("expected 1 obligation after collapse, got %d", len(merged)) + } +} + +func TestCollapse_DistinctPredicatesDoNotMerge(t *testing.T) { + merged := collapse([]Formula{ + Eventually(ThunkNamed("p3", func() (bool, error) { return false, nil })), + Eventually(ThunkNamed("p4", func() (bool, error) { return false, nil })), + }) + if len(merged) != 2 { + t.Errorf("distinct predicates must not merge, got %d", len(merged)) + } +} + +func TestCollapse_NamedThunkLeakBoundsPendingSet(t *testing.T) { + // Always(Eventually(sameThunk)): each step spawns an identical obligation. + // Without collapse the pending set grows unboundedly. + evaluator := NewEvaluator(Always(Eventually(ThunkNamed("p", func() (bool, error) { return false, nil })))) + for index := range 20 { + evaluator.ObserveAt(time.Unix(int64(index), 0)) + } + if len(evaluator.pending) > 2 { + t.Errorf("pending set leaked to %d obligations", len(evaluator.pending)) + } +} diff --git a/internal/ltl/formula.go b/internal/ltl/formula.go index 11f99ec..b60828c 100644 --- a/internal/ltl/formula.go +++ b/internal/ltl/formula.go @@ -13,9 +13,9 @@ type Formula interface { describe() string } -// PredicateLabel lets a ThunkFormula carry a human-readable name for the -// closure it wraps. Verifier wires this in when the spec gives the predicate -// a property name; otherwise it stays empty and serializes without a name. +// PredicateLabel lets a ThunkFormula expose the identity of the closure it +// wraps. ThunkFormula satisfies it through its Name field; an empty name +// serializes without a name. type PredicateLabel interface { PredicateName() string } @@ -33,18 +33,35 @@ func (e ErrorFormula) describe() string { return fmt.Sprintf("Error(%q)", e.Message) } +// AlwaysFormula obliges its inner formula to hold at every step. A bounded +// Always (the dual of a bounded Eventually) holds for the steps inside its +// window and is vacuously satisfied once the window closes. An unbounded +// Always carries no bound fields and is checked at every observed step. type AlwaysFormula struct { - Inner Formula + Inner Formula + StepBound int + HasStepBound bool + Duration time.Duration + Deadline time.Time + HasDeadline bool } type PureFormula struct { Value bool } +// ThunkFormula wraps an opaque predicate closure. Func returns the predicate's +// boolean result and a non-nil error when the predicate threw; a thrown +// predicate is a witnessed violation distinct from a plain false. Name carries +// the predicate's identity so two distinct predicates produce distinct +// describe() keys and are never merged during obligation collapse. type ThunkFormula struct { - Func func() bool + Func func() (bool, error) + Name string } +func (t ThunkFormula) PredicateName() string { return t.Name } + // NowFormula marks its inner formula for evaluation at the current step only. // Primarily used so that now(...).implies(...) parses unambiguously. type NowFormula struct { @@ -96,7 +113,11 @@ func Always(inner Formula) Formula { return AlwaysFormula{Inner: inner} } func Pure(value bool) Formula { return PureFormula{Value: value} } -func Thunk(function func() bool) Formula { return ThunkFormula{Func: function} } +func Thunk(function func() (bool, error)) Formula { return ThunkFormula{Func: function} } + +func ThunkNamed(name string, function func() (bool, error)) Formula { + return ThunkFormula{Func: function, Name: name} +} func Now(inner Formula) Formula { return NowFormula{Inner: inner} } @@ -137,11 +158,27 @@ func (OrFormula) isFormula() {} func (AndFormula) isFormula() {} func (NotFormula) isFormula() {} -func (a AlwaysFormula) describe() string { return "Always(" + a.Inner.describe() + ")" } -func (p PureFormula) describe() string { return fmt.Sprintf("Pure(%t)", p.Value) } -func (ThunkFormula) describe() string { return "Thunk(...)" } -func (n NowFormula) describe() string { return "Now(" + n.Inner.describe() + ")" } -func (n NextFormula) describe() string { return "Next(" + n.Inner.describe() + ")" } +func (a AlwaysFormula) describe() string { + parts := []string{a.Inner.describe()} + if a.HasStepBound { + parts = append(parts, fmt.Sprintf("steps=%d", a.StepBound)) + } + if a.HasDeadline { + parts = append(parts, "deadline="+a.Deadline.Format(time.RFC3339Nano)) + } else if a.Duration > 0 { + parts = append(parts, "within="+a.Duration.String()) + } + return "Always(" + strings.Join(parts, ", ") + ")" +} +func (p PureFormula) describe() string { return fmt.Sprintf("Pure(%t)", p.Value) } +func (t ThunkFormula) describe() string { + if t.Name != "" { + return "Thunk(" + t.Name + ")" + } + return "Thunk(...)" +} +func (n NowFormula) describe() string { return "Now(" + n.Inner.describe() + ")" } +func (n NextFormula) describe() string { return "Next(" + n.Inner.describe() + ")" } func (e EventuallyFormula) describe() string { parts := []string{e.Inner.describe()} if e.HasStepBound { @@ -176,10 +213,20 @@ type withinNode struct { } func (a AlwaysFormula) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - Op string `json:"op"` - Arg Formula `json:"arg"` - }{"always", a.Inner}) + payload := struct { + Op string `json:"op"` + Arg Formula `json:"arg"` + Within *withinNode `json:"within,omitempty"` + }{Op: "always", Arg: a.Inner} + switch { + case a.HasStepBound: + payload.Within = &withinNode{Amount: int64(a.StepBound), Unit: "steps"} + case a.Duration > 0: + payload.Within = &withinNode{Amount: a.Duration.Milliseconds(), Unit: "milliseconds"} + case a.HasDeadline: + payload.Within = &withinNode{Amount: a.Deadline.UnixMilli(), Unit: "deadline"} + } + return json.Marshal(payload) } func (n NowFormula) MarshalJSON() ([]byte, error) { diff --git a/internal/ltl/formula_test.go b/internal/ltl/formula_test.go index 44cabc0..7716de5 100644 --- a/internal/ltl/formula_test.go +++ b/internal/ltl/formula_test.go @@ -50,7 +50,7 @@ func TestAlways_Now_ViolatesImmediately(t *testing.T) { func TestAlways_Next_PendingThenViolated(t *testing.T) { y := true - evaluator := NewEvaluator(Always(Next(Thunk(func() bool { return y })))) + evaluator := NewEvaluator(Always(Next(Thunk(func() (bool, error) { return y, nil })))) if got := evaluator.Observe(); got != VerdictPending { t.Errorf("step 1: got %v, want pending", got) @@ -62,7 +62,7 @@ func TestAlways_Next_PendingThenViolated(t *testing.T) { } func TestAlways_Next_StaysPendingWhileInnerHolds(t *testing.T) { - evaluator := NewEvaluator(Always(Next(Thunk(func() bool { return true })))) + evaluator := NewEvaluator(Always(Next(Thunk(func() (bool, error) { return true, nil })))) for index := range 3 { if got := evaluator.ObserveAt(time.Unix(int64(index), 0)); got != VerdictPending { t.Errorf("step %d: got %v, want pending", index+1, got) @@ -76,8 +76,8 @@ func TestAlways_NowImpliesEventuallyWithin_ViolatesWhenYLate(t *testing.T) { xValues := []bool{true, false, false, false, false} yValues := []bool{false, false, false, true, true} step := 0 - predX := Thunk(func() bool { return xValues[step] }) - predY := Thunk(func() bool { return yValues[step] }) + predX := Thunk(func() (bool, error) { return xValues[step], nil }) + predY := Thunk(func() (bool, error) { return yValues[step], nil }) formula := Always(Implies(Now(predX), EventuallyWithinSteps(predY, 3))) evaluator := NewEvaluator(formula) @@ -107,8 +107,8 @@ func TestAlways_NowImpliesEventuallyWithin_HoldsWhenYInBound(t *testing.T) { xValues := []bool{true, false, false} yValues := []bool{false, false, true} step := 0 - predX := Thunk(func() bool { return xValues[step] }) - predY := Thunk(func() bool { return yValues[step] }) + predX := Thunk(func() (bool, error) { return xValues[step], nil }) + predY := Thunk(func() (bool, error) { return yValues[step], nil }) formula := Always(Implies(Now(predX), EventuallyWithinSteps(predY, 3))) evaluator := NewEvaluator(formula) @@ -187,12 +187,6 @@ func TestNot_InvertsPure(t *testing.T) { } } -func TestVerdict_StringPending(t *testing.T) { - if got := VerdictPending.String(); got != "pending" { - t.Errorf("VerdictPending.String() = %q", got) - } -} - func TestMarshalJSON_AlwaysImpliesEventually(t *testing.T) { formula := Always(Implies(Now(Pure(true)), EventuallyWithinSteps(Pure(false), 3))) body, err := json.Marshal(formula) @@ -231,7 +225,7 @@ func TestMarshalJSON_NextAndThunkAndError(t *testing.T) { if string(body) != `{"op":"next","arg":{"op":"true"}}` { t.Errorf("next marshal wrong: %s", body) } - body, _ = json.Marshal(Thunk(func() bool { return true })) + body, _ = json.Marshal(Thunk(func() (bool, error) { return true, nil })) if string(body) != `{"op":"predicate"}` { t.Errorf("thunk marshal wrong: %s", body) } diff --git a/internal/ltl/nnf.go b/internal/ltl/nnf.go new file mode 100644 index 0000000..51d07ca --- /dev/null +++ b/internal/ltl/nnf.go @@ -0,0 +1,87 @@ +package ltl + +// nnf rewrites a formula into negation normal form: every NotFormula is pushed +// down until it wraps only an opaque leaf (a ThunkFormula or ErrorFormula). +// Temporal operators are dualized along the way (Always <-> Eventually) so the +// evaluator never has to reduce a negated temporal obligation, which it cannot +// do soundly across steps. +func nnf(formula Formula) Formula { + switch concrete := formula.(type) { + case NotFormula: + return pushNot(concrete.Inner) + case AlwaysFormula: + next := concrete + next.Inner = nnf(concrete.Inner) + return next + case EventuallyFormula: + next := concrete + next.Inner = nnf(concrete.Inner) + return next + case NextFormula: + return NextFormula{Inner: nnf(concrete.Inner)} + case NowFormula: + return NowFormula{Inner: nnf(concrete.Inner)} + case AndFormula: + return AndFormula{Left: nnf(concrete.Left), Right: nnf(concrete.Right)} + case OrFormula: + return OrFormula{Left: nnf(concrete.Left), Right: nnf(concrete.Right)} + case ImpliesFormula: + // a -> b is rewritten to (not a) or b so the consequent is always + // reduced live each step. Keeping it as ImpliesFormula let a pending + // (temporal) antecedent defer the whole implication and silently drop a + // consequent that was false at the current step. + return OrFormula{ + Left: pushNot(concrete.Antecedent), + Right: nnf(concrete.Consequent), + } + default: + return formula + } +} + +// pushNot returns the negation normal form of NOT f. +func pushNot(formula Formula) Formula { + switch concrete := formula.(type) { + case PureFormula: + return PureFormula{Value: !concrete.Value} + case ThunkFormula: + return NotFormula{Inner: concrete} + case ErrorFormula: + return NotFormula{Inner: concrete} + case NotFormula: + return nnf(concrete.Inner) + case AndFormula: + return OrFormula{Left: pushNot(concrete.Left), Right: pushNot(concrete.Right)} + case OrFormula: + return AndFormula{Left: pushNot(concrete.Left), Right: pushNot(concrete.Right)} + case ImpliesFormula: + return AndFormula{ + Left: nnf(concrete.Antecedent), + Right: pushNot(concrete.Consequent), + } + case NowFormula: + return NowFormula{Inner: pushNot(concrete.Inner)} + case NextFormula: + return NextFormula{Inner: pushNot(concrete.Inner)} + case AlwaysFormula: + return EventuallyFormula{ + Inner: pushNot(concrete.Inner), + StepBound: concrete.StepBound, + HasStepBound: concrete.HasStepBound, + Duration: concrete.Duration, + Deadline: concrete.Deadline, + HasDeadline: concrete.HasDeadline, + } + case EventuallyFormula: + return AlwaysFormula{ + Inner: pushNot(concrete.Inner), + StepBound: concrete.StepBound, + HasStepBound: concrete.HasStepBound, + Duration: concrete.Duration, + Deadline: concrete.Deadline, + HasDeadline: concrete.HasDeadline, + } + default: + return NotFormula{Inner: formula} + } +} diff --git a/internal/ltl/nnf_test.go b/internal/ltl/nnf_test.go new file mode 100644 index 0000000..4ab4ac2 --- /dev/null +++ b/internal/ltl/nnf_test.go @@ -0,0 +1,97 @@ +package ltl + +import ( + "testing" + "testing/quick" + "time" +) + +// leaf builds a small set of representative atomic formulas indexed by a seed. +func leafFor(seed uint8) Formula { + switch seed % 3 { + case 0: + return Pure(true) + case 1: + return Pure(false) + default: + return ThunkNamed("p", func() (bool, error) { return true, nil }) + } +} + +func TestNNF_DoubleNegationIsIdentity(t *testing.T) { + law := func(seed uint8) bool { + leaf := leafFor(seed) + doubled := nnf(Not(Not(leaf))) + direct := nnf(leaf) + return doubled.describe() == direct.describe() + } + if err := quick.Check(law, nil); err != nil { + t.Error(err) + } +} + +func TestNNF_NotAlwaysIsEventuallyNot(t *testing.T) { + law := func(seed uint8) bool { + leaf := leafFor(seed) + negated := nnf(Not(Always(leaf))) + expected := nnf(Eventually(Not(leaf))) + return negated.describe() == expected.describe() + } + if err := quick.Check(law, nil); err != nil { + t.Error(err) + } +} + +func TestNNF_NotEventuallyIsAlwaysNot(t *testing.T) { + law := func(seed uint8) bool { + leaf := leafFor(seed) + negated := nnf(Not(Eventually(leaf))) + expected := nnf(Always(Not(leaf))) + return negated.describe() == expected.describe() + } + if err := quick.Check(law, nil); err != nil { + t.Error(err) + } +} + +func TestNNF_BoundedEventuallyDualKeepsBound(t *testing.T) { + negated := nnf(Not(EventuallyWithinSteps(Pure(true), 4))) + always, ok := negated.(AlwaysFormula) + if !ok { + t.Fatalf("expected AlwaysFormula, got %T", negated) + } + if !always.HasStepBound || always.StepBound != 4 { + t.Errorf("bound not preserved: %+v", always) + } +} + +func TestNNF_PushesNotToThunkLeaf(t *testing.T) { + formula := nnf(Always(Not(Always(ThunkNamed("p", func() (bool, error) { return true, nil }))))) + always, ok := formula.(AlwaysFormula) + if !ok { + t.Fatalf("expected AlwaysFormula, got %T", formula) + } + eventually, ok := always.Inner.(EventuallyFormula) + if !ok { + t.Fatalf("expected inner EventuallyFormula, got %T", always.Inner) + } + not, ok := eventually.Inner.(NotFormula) + if !ok { + t.Fatalf("expected NotFormula leaf, got %T", eventually.Inner) + } + if _, ok := not.Inner.(ThunkFormula); !ok { + t.Errorf("expected Not to wrap a Thunk, got %T", not.Inner) + } +} + +func TestNNF_NotAlwaysTrueViaEvaluatorReportsViolated(t *testing.T) { + evaluator := NewEvaluator(Always(Not(Always(ThunkNamed("p", func() (bool, error) { return true, nil }))))) + for index := range 4 { + if got := evaluator.ObserveAt(time.Unix(int64(index), 0)); got == VerdictViolated { + t.Fatalf("step %d latched violated prematurely", index) + } + } + if got := evaluator.Finalize(); got != VerdictViolated { + t.Errorf("Finalize = %v, want violated", got) + } +} diff --git a/internal/ltl/witness_test.go b/internal/ltl/witness_test.go new file mode 100644 index 0000000..6e4c580 --- /dev/null +++ b/internal/ltl/witness_test.go @@ -0,0 +1,80 @@ +package ltl + +import ( + "errors" + "testing" + "time" +) + +func TestViolation_PredicateFalseCarriesReasonAndStep(t *testing.T) { + values := []bool{true, false} + step := 0 + evaluator := NewEvaluator(Always(ThunkNamed("p", func() (bool, error) { + current := values[step] + step++ + return current, nil + }))) + if got := evaluator.ObserveAt(time.Unix(0, 0)); got != VerdictHolds { + t.Fatalf("step 1: got %v, want holds", got) + } + if got := evaluator.ObserveAt(time.Unix(1, 0)); got != VerdictViolated { + t.Fatalf("step 2: got %v, want violated", got) + } + witness := evaluator.Violation() + if witness == nil { + t.Fatal("Violation = nil, want non-nil") + } + if witness.Reason != "predicate false" { + t.Errorf("Reason = %q, want %q", witness.Reason, "predicate false") + } + if witness.Step != 2 { + t.Errorf("Step = %d, want 2", witness.Step) + } + if witness.IsError { + t.Errorf("IsError = true, want false for a plain false") + } +} + +func TestViolation_ThrownPredicateSetsIsError(t *testing.T) { + evaluator := NewEvaluator(Always(ThunkNamed("p", func() (bool, error) { + return false, errors.New("boom") + }))) + if got := evaluator.Observe(); got != VerdictViolated { + t.Fatalf("got %v, want violated", got) + } + witness := evaluator.Violation() + if witness == nil { + t.Fatal("Violation = nil, want non-nil") + } + if !witness.IsError { + t.Errorf("IsError = false, want true for a thrown predicate") + } + if witness.Reason != "boom" { + t.Errorf("Reason = %q, want %q", witness.Reason, "boom") + } +} + +func TestViolation_FinalizeFillsWitness(t *testing.T) { + evaluator := NewEvaluator(Eventually(ThunkNamed("p", func() (bool, error) { + return false, nil + }))) + evaluator.ObserveAt(time.Unix(0, 0)) + if got := evaluator.Finalize(); got != VerdictViolated { + t.Fatalf("Finalize = %v, want violated", got) + } + witness := evaluator.Violation() + if witness == nil { + t.Fatal("Violation = nil after Finalize, want non-nil") + } + if witness.Reason != "eventually never satisfied" { + t.Errorf("Reason = %q, want %q", witness.Reason, "eventually never satisfied") + } +} + +func TestViolation_NilBeforeViolation(t *testing.T) { + evaluator := NewEvaluator(Always(Pure(true))) + evaluator.Observe() + if got := evaluator.Violation(); got != nil { + t.Errorf("Violation = %+v, want nil for a holding run", got) + } +} diff --git a/internal/permissions/permissions.go b/internal/permissions/permissions.go deleted file mode 100644 index dc7c69d..0000000 --- a/internal/permissions/permissions.go +++ /dev/null @@ -1,125 +0,0 @@ -package permissions - -import ( - "context" - "fmt" - "os/exec" - "regexp" - "strings" -) - -// Inspector returns the list of uses-permission entries declared by the APK. -type Inspector func(ctx context.Context, apkPath string) ([]string, error) - -// Granter grants a single permission to the given package on the connected -// device. Implementations typically wrap `adb shell pm grant`. -type Granter func(ctx context.Context, packageName, permission string) error - -// GrantDangerous filters declared permissions to the dangerous set and grants -// each one via the supplied granter. Errors from individual grants are -// collected as warnings rather than aborting — Android refuses to grant -// non-runtime permissions and we prefer to soldier on. -func GrantDangerous( - ctx context.Context, - apkPath, packageName string, - inspector Inspector, - granter Granter, -) (granted []string, warnings []string, err error) { - declared, err := inspector(ctx, apkPath) - if err != nil { - return nil, nil, fmt.Errorf("inspect permissions: %w", err) - } - for _, permission := range declared { - if !IsDangerous(permission) { - continue - } - if err := granter(ctx, packageName, permission); err != nil { - warnings = append(warnings, fmt.Sprintf("%s: %v", permission, err)) - continue - } - granted = append(granted, permission) - } - return granted, warnings, nil -} - -var aaptPermissionPattern = regexp.MustCompile(`uses-permission:\s+name='([^']+)'`) - -// AaptInspector shells out to `aapt dump permissions` to list permissions. -func AaptInspector(ctx context.Context, apkPath string) ([]string, error) { - output, err := exec.CommandContext(ctx, "aapt", "dump", "permissions", apkPath).Output() - if err != nil { - return nil, fmt.Errorf("aapt dump permissions: %w", err) - } - var permissions []string - for _, match := range aaptPermissionPattern.FindAllStringSubmatch(string(output), -1) { - permissions = append(permissions, match[1]) - } - return permissions, nil -} - -// AdbGranter returns a Granter that runs `adb shell pm grant` against the -// supplied device serial (empty = default device). -func AdbGranter(adbPath, deviceSerial string) Granter { - return func(ctx context.Context, packageName, permission string) error { - arguments := []string{"shell", "pm", "grant", packageName, permission} - if deviceSerial != "" { - arguments = append([]string{"-s", deviceSerial}, arguments...) - } - command := exec.CommandContext(ctx, adbPath, arguments...) - output, err := command.CombinedOutput() - if err != nil { - return fmt.Errorf("adb pm grant %s: %w (%s)", permission, err, strings.TrimSpace(string(output))) - } - return nil - } -} - -// dangerousPermissions captures Android's PROTECTION_DANGEROUS set as of -// API 34. New runtime permissions added in later releases should be appended -// here when needed. -var dangerousPermissions = map[string]bool{ - "android.permission.READ_CALENDAR": true, - "android.permission.WRITE_CALENDAR": true, - "android.permission.CAMERA": true, - "android.permission.READ_CONTACTS": true, - "android.permission.WRITE_CONTACTS": true, - "android.permission.GET_ACCOUNTS": true, - "android.permission.ACCESS_FINE_LOCATION": true, - "android.permission.ACCESS_COARSE_LOCATION": true, - "android.permission.ACCESS_BACKGROUND_LOCATION": true, - "android.permission.RECORD_AUDIO": true, - "android.permission.READ_PHONE_STATE": true, - "android.permission.READ_PHONE_NUMBERS": true, - "android.permission.CALL_PHONE": true, - "android.permission.ANSWER_PHONE_CALLS": true, - "android.permission.READ_CALL_LOG": true, - "android.permission.WRITE_CALL_LOG": true, - "android.permission.ADD_VOICEMAIL": true, - "android.permission.USE_SIP": true, - "android.permission.PROCESS_OUTGOING_CALLS": true, - "android.permission.BODY_SENSORS": true, - "android.permission.SEND_SMS": true, - "android.permission.RECEIVE_SMS": true, - "android.permission.READ_SMS": true, - "android.permission.RECEIVE_WAP_PUSH": true, - "android.permission.RECEIVE_MMS": true, - "android.permission.READ_EXTERNAL_STORAGE": true, - "android.permission.WRITE_EXTERNAL_STORAGE": true, - "android.permission.ACCESS_MEDIA_LOCATION": true, - "android.permission.ACTIVITY_RECOGNITION": true, - "android.permission.POST_NOTIFICATIONS": true, - "android.permission.NEARBY_WIFI_DEVICES": true, - "android.permission.READ_MEDIA_IMAGES": true, - "android.permission.READ_MEDIA_VIDEO": true, - "android.permission.READ_MEDIA_AUDIO": true, - "android.permission.READ_MEDIA_VISUAL_USER_SELECTED": true, - "android.permission.BLUETOOTH_CONNECT": true, - "android.permission.BLUETOOTH_ADVERTISE": true, - "android.permission.BLUETOOTH_SCAN": true, - "android.permission.UWB_RANGING": true, - "android.permission.BODY_SENSORS_BACKGROUND": true, -} - -func IsDangerous(permission string) bool { - return dangerousPermissions[permission] -} diff --git a/internal/permissions/permissions_test.go b/internal/permissions/permissions_test.go deleted file mode 100644 index 907895b..0000000 --- a/internal/permissions/permissions_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package permissions - -import ( - "context" - "errors" - "slices" - "testing" -) - -func TestGrantDangerous_FiltersAndCallsGranter(t *testing.T) { - declared := []string{ - "android.permission.INTERNET", // not dangerous - "android.permission.CAMERA", // dangerous - "android.permission.WAKE_LOCK", // not dangerous - "android.permission.ACCESS_FINE_LOCATION", // dangerous - "android.permission.READ_EXTERNAL_STORAGE", // dangerous - "android.permission.SYSTEM_ALERT_WINDOW", // not dangerous - } - inspector := func(_ context.Context, _ string) ([]string, error) { return declared, nil } - var requested []string - granter := func(_ context.Context, packageName, permission string) error { - if packageName != "com.example" { - t.Errorf("granter received wrong package: %q", packageName) - } - requested = append(requested, permission) - return nil - } - - granted, warnings, err := GrantDangerous(context.Background(), "/path/to/apk", "com.example", inspector, granter) - if err != nil { - t.Fatal(err) - } - want := []string{ - "android.permission.CAMERA", - "android.permission.ACCESS_FINE_LOCATION", - "android.permission.READ_EXTERNAL_STORAGE", - } - if !slices.Equal(granted, want) { - t.Errorf("granted permissions: got %v, want %v", granted, want) - } - if !slices.Equal(requested, want) { - t.Errorf("granter calls: got %v, want %v", requested, want) - } - if len(warnings) != 0 { - t.Errorf("expected no warnings, got %v", warnings) - } -} - -func TestGrantDangerous_CollectsGranterFailuresAsWarnings(t *testing.T) { - declared := []string{"android.permission.CAMERA", "android.permission.RECORD_AUDIO"} - inspector := func(_ context.Context, _ string) ([]string, error) { return declared, nil } - granter := func(_ context.Context, _, permission string) error { - if permission == "android.permission.CAMERA" { - return errors.New("device denied") - } - return nil - } - - granted, warnings, err := GrantDangerous(context.Background(), "/x", "com.example", inspector, granter) - if err != nil { - t.Fatal(err) - } - if !slices.Equal(granted, []string{"android.permission.RECORD_AUDIO"}) { - t.Errorf("granted: %v", granted) - } - if len(warnings) != 1 || warnings[0] != "android.permission.CAMERA: device denied" { - t.Errorf("warnings: %v", warnings) - } -} - -func TestGrantDangerous_InspectorErrorBubbles(t *testing.T) { - inspector := func(_ context.Context, _ string) ([]string, error) { - return nil, errors.New("aapt missing") - } - _, _, err := GrantDangerous(context.Background(), "/x", "com.example", inspector, nil) - if err == nil || err.Error() == "" { - t.Errorf("expected wrapped inspector error, got %v", err) - } -} - -func TestIsDangerous_KnownPermissions(t *testing.T) { - dangerous := []string{ - "android.permission.CAMERA", - "android.permission.RECORD_AUDIO", - "android.permission.POST_NOTIFICATIONS", - } - for _, permission := range dangerous { - if !IsDangerous(permission) { - t.Errorf("expected %q to be dangerous", permission) - } - } - - normal := []string{ - "android.permission.INTERNET", - "android.permission.WAKE_LOCK", - "android.permission.SYSTEM_ALERT_WINDOW", - } - for _, permission := range normal { - if IsDangerous(permission) { - t.Errorf("expected %q not to be dangerous", permission) - } - } -} - -func TestAaptInspector_ParsesFixtureOutput(t *testing.T) { - // Direct test of the regex; avoids spawning aapt. - output := ` -package: com.example -sdkVersion:'24' -uses-permission: name='android.permission.INTERNET' -uses-permission: name='android.permission.CAMERA' -uses-permission: name='android.permission.WAKE_LOCK' -` - matches := aaptPermissionPattern.FindAllStringSubmatch(output, -1) - got := make([]string, 0, len(matches)) - for _, match := range matches { - got = append(got, match[1]) - } - want := []string{ - "android.permission.INTERNET", - "android.permission.CAMERA", - "android.permission.WAKE_LOCK", - } - if !slices.Equal(got, want) { - t.Errorf("got %v, want %v", got, want) - } -} diff --git a/internal/runner/golden_test.go b/internal/runner/golden_test.go new file mode 100644 index 0000000..156858c --- /dev/null +++ b/internal/runner/golden_test.go @@ -0,0 +1,143 @@ +package runner + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/priyanshujain/sanderling/internal/verifier" +) + +// mustSeededVerifier builds a verifier with a fixed seed so the JS picker's PRNG +// is reproducible across runs, making the trace byte-stable. +func mustSeededVerifier(t *testing.T, spec string, seed uint64) *verifier.Verifier { + t.Helper() + instance, err := verifier.New(verifier.WithSeed(seed)) + if err != nil { + t.Fatal(err) + } + if err := instance.Load(bundleSpec(t, spec)); err != nil { + t.Fatal(err) + } + return instance +} + +// goldenSeed and goldenSteps fix the run so the mock-driven trace and summary +// are byte-reproducible. Set UPDATE_GOLDEN=1 to rewrite the committed goldens; +// the checked-in files are the asserted source of truth. +const ( + goldenSeed = 0x5eed + goldenSteps = 4 +) + +// canonicalizeTrace rewrites the sole non-deterministic field, "timestamp" +// (wall-clock at step start), to a fixed zero value so equality compares only +// the run's observable shape. Every other field is a pure function of the seed, +// the spec, and the programmed mock hierarchy. +func canonicalizeTrace(t *testing.T, raw []byte) []byte { + t.Helper() + var out bytes.Buffer + scanner := bufio.NewScanner(bytes.NewReader(raw)) + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + for scanner.Scan() { + var line map[string]json.RawMessage + if err := json.Unmarshal(scanner.Bytes(), &line); err != nil { + t.Fatalf("decode trace line: %v", err) + } + line["timestamp"] = json.RawMessage(`"0001-01-01T00:00:00Z"`) + encoded, err := json.Marshal(line) + if err != nil { + t.Fatalf("encode trace line: %v", err) + } + out.Write(encoded) + out.WriteByte('\n') + } + if err := scanner.Err(); err != nil { + t.Fatalf("scan trace: %v", err) + } + return out.Bytes() +} + +func assertGolden(t *testing.T, name string, got []byte) { + t.Helper() + path := filepath.Join("testdata", name) + if os.Getenv("UPDATE_GOLDEN") == "1" { + if err := os.WriteFile(path, got, 0o644); err != nil { + t.Fatalf("write golden %s: %v", name, err) + } + return + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read golden %s (run with UPDATE_GOLDEN=1 to create): %v", name, err) + } + if !bytes.Equal(want, got) { + t.Errorf("golden %s mismatch\n--- want ---\n%s\n--- got ---\n%s", name, want, got) + } +} + +// TestGolden_TraceStreamIsReproducible drives a fixed-seed, fixed-step run +// against the mock driver and snapshots the canonicalized trace.jsonl stream. +// A drift here means the run is no longer deterministic or the trace shape +// changed; both must be reviewed, not papered over. +func TestGolden_TraceStreamIsReproducible(t *testing.T) { + state := newHarnessWithSpec(t, fixtureSpec) + state.verifier = mustSeededVerifier(t, fixtureSpec, goldenSeed) + state.mock.HierarchyJSON = `{"attributes":{"resource-id":"HomeScreen"},"children":[{"attributes":{"resource-id":"next","bounds":"[40,80,240,160]"},"children":[],"clickable":true,"enabled":true}]}` + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, err := Run(ctx, Options{ + Duration: time.Hour, + IdleTimeout: 10 * time.Millisecond, + MaxSteps: goldenSteps, + Driver: state.mock, + Verifier: state.verifier, + TraceWriter: state.writer, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + raw, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl")) + if err != nil { + t.Fatal(err) + } + assertGolden(t, "trace.jsonl", canonicalizeTrace(t, raw)) +} + +// TestGolden_ViolationSummaryIsReproducible drives a known violation and +// snapshots the rendered summary string produced by RenderSummary (the same +// function the CLI prints). It also asserts the violated property is named, so +// a regression that drops the property from the report cannot pass silently. +func TestGolden_ViolationSummaryIsReproducible(t *testing.T) { + state := newHarnessWithSpec(t, violationSpec) + state.verifier = mustSeededVerifier(t, violationSpec, goldenSeed) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + summary, err := Run(ctx, Options{ + Duration: time.Hour, + IdleTimeout: 10 * time.Millisecond, + MaxSteps: goldenSteps, + Driver: state.mock, + Verifier: state.verifier, + TraceWriter: state.writer, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + + var rendered bytes.Buffer + RenderSummary(&rendered, summary, "android") + + if !strings.Contains(rendered.String(), "balanceNonNegative") { + t.Errorf("summary must name the violated property balanceNonNegative, got:\n%s", rendered.String()) + } + assertGolden(t, "violation-summary.txt", rendered.Bytes()) +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index b92baad..25f8cc8 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -1,3 +1,4 @@ +// Package runner drives the observe-decide-act loop that steps a spec against a device. package runner import ( @@ -5,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "log/slog" "strings" "time" @@ -20,15 +22,15 @@ import ( "github.com/priyanshujain/sanderling/internal/verifier" ) -// doubleTapGap is the inter-tap delay for ActionKindDoubleTap: short enough to -// land both events inside a sub-100 ms race window, long enough for adb -// `input tap` to serialize two MotionEvent streams. -const doubleTapGap = 50 * time.Millisecond - type Options struct { Duration time.Duration IdleTimeout time.Duration + // MaxSteps caps the run at a fixed number of steps for reproducible + // bounded runs. 0 means unbounded (the duration deadline governs); a + // positive value stops the loop once that many steps have run. + MaxSteps int + BundleID string Driver driver.DeviceDriver Verifier *verifier.Verifier @@ -41,6 +43,10 @@ type Summary struct { EndTime time.Time Steps int Violations []ViolationRecord + // UnsupportedVerbs lists verbs the picker requested that the platform + // could not dispatch, deduped, so the report can flag a spec exercising + // gestures this target does not support. + UnsupportedVerbs []string } type ViolationRecord struct { @@ -65,6 +71,11 @@ func Run(ctx context.Context, options Options) (Summary, error) { // before the deadline is set so the settle time does not eat the run. waitForForeground(ctx, options, logger) + // Pick the action and extractor sources once from the driver's + // capabilities so the step loop runs one uniform path with no per-step + // driver type assertion. + actionSource, extractorSource := pickSources(options) + summary := Summary{StartTime: time.Now()} deadline := summary.StartTime.Add(options.Duration) stepIndex := 0 @@ -74,6 +85,9 @@ func Run(ctx context.Context, options Options) (Summary, error) { if err := ctx.Err(); err != nil { break } + if options.MaxSteps > 0 && stepIndex >= options.MaxSteps { + break + } stepIndex++ stepStart := time.Now() @@ -116,17 +130,15 @@ func Run(ctx context.Context, options Options) (Summary, error) { return nil }) var v8Overrides map[int]json.RawMessage - if web, ok := options.Driver.(driver.WebDriver); ok { - g.Go(func() error { - overrides, err := web.EvaluateExtractors(gctx) - if err != nil { - logger.Warn("v8 extractor evaluation failed", "step", si, "err", err) - return nil - } - v8Overrides = overrides + g.Go(func() error { + overrides, err := extractorSource.ExtractorOverrides(gctx) + if err != nil { + logger.Warn("v8 extractor evaluation failed", "step", si, "err", err) return nil - }) - } + } + v8Overrides = overrides + return nil + }) // All goroutines write to local variables and return nil, so the Wait // error is always nil; ignored intentionally. _ = g.Wait() @@ -166,6 +178,8 @@ func Run(ctx context.Context, options Options) (Summary, error) { // progressing. var violations []string var extractorChanges map[string]trace.ExtractorChange + var witnesses map[string]trace.Witness + skippedVerification := false if !transitional { if err := options.Verifier.PushSnapshot(verifier.SnapshotInput{ Tree: tree, @@ -186,25 +200,16 @@ func Run(ctx context.Context, options Options) (Summary, error) { } options.Verifier.EvaluateProperties() violations = options.Verifier.NewlyViolatedProperties() - for _, name := range violations { - if predicateErr := options.Verifier.PredicateError(name); predicateErr != nil { - logger.Warn("predicate error", "step", stepIndex, "property", name, "err", predicateErr) - } - } + witnesses = collectWitnesses(options.Verifier, violations, logger, stepIndex) extractorChanges = encodeExtractorChanges(options.Verifier.ChangedExtractors()) } else { + skippedVerification = true logger.Warn("transitional tree after retry budget; skipping verifier", "step", stepIndex, "screen", screen, "nodes", treeSize) } logger.Info("step", "index", stepIndex, "screen", screen, "nodes", treeSize) - var nextAction verifier.Action - var nextErr error - if web, ok := options.Driver.(driver.WebDriver); ok { - nextAction, nextErr = nextActionFromV8(ctx, web) - } else { - nextAction, nextErr = options.Verifier.NextAction() - } + nextAction, nextErr := actionSource.NextAction(ctx) var traceAction *trace.Action if nextErr == nil { traceAction = traceActionFor(nextAction, tree) @@ -240,16 +245,18 @@ func Run(ctx context.Context, options Options) (Summary, error) { } step := trace.Step{ - Index: stepIndex, - Timestamp: stepStart, - Screen: screen, - NextAction: traceAction, - Violations: violations, - Hierarchy: tree, - Residuals: residuals, - Metrics: metrics, - ExtractorChanges: extractorChanges, - Transitional: transitional, + Index: stepIndex, + Timestamp: stepStart, + Screen: screen, + NextAction: traceAction, + Violations: violations, + Hierarchy: tree, + Residuals: residuals, + Metrics: metrics, + ExtractorChanges: extractorChanges, + Transitional: transitional, + SkippedVerification: skippedVerification, + Witnesses: witnesses, } if err := options.TraceWriter.WriteStep(step); err != nil { return summary, fmt.Errorf("step %d trace: %w", stepIndex, err) @@ -276,10 +283,51 @@ func Run(ctx context.Context, options Options) (Summary, error) { } } + // Finalize each evaluator once the loop ends so liveness obligations that + // never discharged (an unbounded eventually that never fired, a strong + // next with no successor) are reported as violations rather than silently + // left pending. Properties already violated mid-run are not re-reported. + if ended := options.Verifier.Finalize(); len(ended) > 0 { + witnesses := collectWitnesses(options.Verifier, ended, logger, stepIndex) + summary.Violations = append(summary.Violations, ViolationRecord{ + StepIndex: stepIndex, + Properties: ended, + }) + finalStep := trace.Step{ + Index: stepIndex, + Timestamp: time.Now(), + Violations: ended, + Witnesses: witnesses, + } + if err := options.TraceWriter.WriteStep(finalStep); err != nil { + return summary, fmt.Errorf("finalize trace: %w", err) + } + } + + summary.UnsupportedVerbs = options.Verifier.UnsupportedVerbs() summary.EndTime = time.Now() return summary, nil } +// RenderSummary writes the human-facing run summary: step count, each violation +// record, and any unsupported verbs. The wall-clock duration is excluded so the +// output is deterministic and snapshot-testable; the CLI prints it separately. +func RenderSummary(w io.Writer, summary Summary, platform string) { + fmt.Fprintf(w, "\nrun complete: %d steps\n", summary.Steps) + if len(summary.Violations) == 0 { + fmt.Fprintln(w, "no violations.") + } else { + fmt.Fprintf(w, "%d violation record(s):\n", len(summary.Violations)) + for _, violation := range summary.Violations { + fmt.Fprintf(w, " step %d: %v\n", violation.StepIndex, violation.Properties) + } + } + if len(summary.UnsupportedVerbs) > 0 { + fmt.Fprintf(w, "unsupported on %s: %s\n", + platform, strings.Join(summary.UnsupportedVerbs, ", ")) + } +} + func validate(options Options) error { if options.Driver == nil { return errors.New("runner: Driver is required") @@ -417,26 +465,13 @@ func applyAction(ctx context.Context, drv driver.DeviceDriver, action verifier.A return drv.Tap(ctx, x, y) case verifier.ActionKindDoubleTap: x, y, ok := resolveCoordinates(action, tree) - tap := func() error { - if !ok { - if action.On == "" { - return nil - } - return drv.TapSelector(ctx, action.On) + if !ok { + if action.On == "" { + return nil } - return drv.Tap(ctx, x, y) + return drv.DoubleTapSelector(ctx, action.On) } - if err := tap(); err != nil { - return err - } - timer := time.NewTimer(doubleTapGap) - defer timer.Stop() - select { - case <-ctx.Done(): - return ctx.Err() - case <-timer.C: - } - return tap() + return drv.DoubleTap(ctx, x, y) case verifier.ActionKindLongPress: x, y, ok := resolveCoordinates(action, tree) if !ok { @@ -748,59 +783,31 @@ func captureMetrics(ctx context.Context, options Options, logger *slog.Logger, s } } -// nextActionFromV8 invokes the V8-side action generator and decodes the -// resulting JSON into a verifier.Action. ErrNoAction is returned when the -// generator declined to act this tick. -func nextActionFromV8(ctx context.Context, web driver.WebDriver) (verifier.Action, error) { - raw, err := web.NextActionFromV8(ctx) - if err != nil { - return verifier.Action{}, fmt.Errorf("v8 next action: %w", err) +// collectWitnesses gathers the violation witness for each newly-violated +// property, logs its cause, and returns them keyed by property name for the +// trace. Properties without a captured witness are skipped. +func collectWitnesses(verifierInstance *verifier.Verifier, properties []string, logger *slog.Logger, stepIndex int) map[string]trace.Witness { + if len(properties) == 0 { + return nil } - if len(raw) == 0 || string(raw) == "null" { - return verifier.Action{}, verifier.ErrNoAction + witnesses := map[string]trace.Witness{} + for _, name := range properties { + witness := verifierInstance.Witness(name) + if witness == nil { + continue + } + logger.Warn("property violated", + "step", stepIndex, "property", name, "reason", witness.Reason, "error", witness.IsError) + witnesses[name] = trace.Witness{ + Reason: witness.Reason, + IsError: witness.IsError, + Extractors: witness.Extractors, + } } - var decoded struct { - Kind string `json:"kind"` - X int `json:"x"` - Y int `json:"y"` - FromX int `json:"from_x"` - FromY int `json:"from_y"` - ToX int `json:"to_x"` - ToY int `json:"to_y"` - Key string `json:"key"` - Text string `json:"text"` - DurationMillis int `json:"duration_millis"` - } - if err := json.Unmarshal(raw, &decoded); err != nil { - return verifier.Action{}, fmt.Errorf("decode v8 action: %w", err) - } - switch decoded.Kind { - case "Tap": - return verifier.Action{Kind: verifier.ActionKindTap, X: decoded.X, Y: decoded.Y}, nil - case "DoubleTap": - return verifier.Action{Kind: verifier.ActionKindDoubleTap, X: decoded.X, Y: decoded.Y}, nil - case "InputText": - return verifier.Action{ - Kind: verifier.ActionKindInputText, - X: decoded.X, Y: decoded.Y, - Text: decoded.Text, - }, nil - case "Swipe": - return verifier.Action{ - Kind: verifier.ActionKindSwipe, - FromX: decoded.FromX, - FromY: decoded.FromY, - ToX: decoded.ToX, - ToY: decoded.ToY, - DurationMillis: decoded.DurationMillis, - }, nil - case "PressKey": - return verifier.Action{Kind: verifier.ActionKindPressKey, Key: decoded.Key}, nil - case "Wait": - return verifier.Action{Kind: verifier.ActionKindWait, DurationMillis: decoded.DurationMillis}, nil - default: - return verifier.Action{}, verifier.ErrNoAction + if len(witnesses) == 0 { + return nil } + return witnesses } func encodeExtractorChanges(changes map[string]verifier.ExtractorChange) map[string]trace.ExtractorChange { diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index e904f38..7c37521 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -18,6 +18,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/priyanshujain/sanderling/internal/bundler" "github.com/priyanshujain/sanderling/internal/driver" mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock" "github.com/priyanshujain/sanderling/internal/hierarchy" @@ -26,18 +27,20 @@ import ( ) const fixtureSpec = ` -const balance = __sanderling__.extract(state => state.snapshots.balance ?? 0); +import { actions, always, extract, Tap } from "@sanderling/spec"; +const balance = extract(state => state.snapshots.balance ?? 0); globalThis.properties = { - balanceNonNegative: __sanderling__.always(() => balance.current >= 0), + balanceNonNegative: always(() => balance.current >= 0), }; -globalThis.actions = __sanderling__.actions(() => [__sanderling__.tap({ on: "id:next" })]); +globalThis.actions = actions(() => [Tap({ on: "id:next" })]); ` const violationSpec = ` +import { actions, always } from "@sanderling/spec"; globalThis.properties = { - balanceNonNegative: __sanderling__.always(() => false), + balanceNonNegative: always(() => false), }; -globalThis.actions = __sanderling__.actions(() => []); +globalThis.actions = actions(() => []); ` type harness struct { @@ -50,6 +53,34 @@ func newHarness(t *testing.T) *harness { return newHarnessWithSpec(t, fixtureSpec) } +// bundleSpec compiles an authored TS spec with the goja runtime entry so the +// loaded bundle installs __sanderlingNextAction__ (the shared picker). +func bundleSpec(t *testing.T, specSource string) string { + t.Helper() + dir := t.TempDir() + specPath := filepath.Join(dir, "spec.ts") + if err := os.WriteFile(specPath, []byte(specSource), 0o600); err != nil { + t.Fatal(err) + } + apiPath, err := filepath.Abs("../../pkg/spec/src/index.ts") + if err != nil { + t.Fatal(err) + } + runtimePath, err := filepath.Abs("../../pkg/spec/src/goja-runtime.ts") + if err != nil { + t.Fatal(err) + } + bundle, err := bundler.Bundle(bundler.Options{ + EntryFile: specPath, + RuntimeFile: runtimePath, + Aliases: map[string]string{"@sanderling/spec": apiPath}, + }) + if err != nil { + t.Fatal(err) + } + return string(bundle.JavaScript) +} + func newHarnessWithSpec(t *testing.T, spec string) *harness { t.Helper() directory := t.TempDir() @@ -61,7 +92,7 @@ func newHarnessWithSpec(t *testing.T, spec string) *harness { if err != nil { t.Fatal(err) } - if err := verifierInstance.Load(spec); err != nil { + if err := verifierInstance.Load(bundleSpec(t, spec)); err != nil { t.Fatal(err) } state := &harness{ @@ -94,6 +125,11 @@ func TestRunner_HappyPathStepsAndTraces(t *testing.T) { if len(summary.Violations) != 0 { t.Errorf("no violations expected, got %v", summary.Violations) } + // Every builtin verb is supported on every platform, so a clean run must + // report no unsupported verbs (the runner still wires the field through). + if len(summary.UnsupportedVerbs) != 0 { + t.Errorf("expected no unsupported verbs, got %v", summary.UnsupportedVerbs) + } actions := state.mock.Actions() if !containsAction(actions, mockdriver.ActionTapSelector, "id:next") { @@ -101,6 +137,29 @@ func TestRunner_HappyPathStepsAndTraces(t *testing.T) { } } +func TestRunner_MaxStepsStopsAfterExactlyNSteps(t *testing.T) { + state := newHarness(t) + + const maxSteps = 3 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + // A long duration ensures MaxSteps, not the deadline, ends the run. + summary, err := Run(ctx, Options{ + Duration: time.Hour, + IdleTimeout: 10 * time.Millisecond, + MaxSteps: maxSteps, + Driver: state.mock, + Verifier: state.verifier, + TraceWriter: state.writer, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + if summary.Steps != maxSteps { + t.Errorf("expected exactly %d steps, got %d", maxSteps, summary.Steps) + } +} + func TestRunner_ViolationSurfacesInSummary(t *testing.T) { state := newHarnessWithSpec(t, violationSpec) @@ -196,10 +255,11 @@ func TestRunner_ViolationSurfacesOnlyOnOnsetStep(t *testing.T) { func TestRunner_ThrowingPredicateIsLoggedNotPanic(t *testing.T) { const throwingSpec = ` +import { actions, always, Tap } from "@sanderling/spec"; globalThis.properties = { - broken: __sanderling__.always(() => { throw new Error("bad predicate"); }), + broken: always(() => { throw new Error("bad predicate"); }), }; -globalThis.actions = __sanderling__.actions(() => [__sanderling__.tap({ on: "id:next" })]); +globalThis.actions = actions(() => [Tap({ on: "id:next" })]); ` state := newHarnessWithSpec(t, throwingSpec) @@ -370,30 +430,25 @@ func TestApplyAction_V8InputTextAtOriginStillTaps(t *testing.T) { } } -func TestApplyAction_DoubleTapDispatchesTwoTapsAtCoordinates(t *testing.T) { +func TestApplyAction_DoubleTapDispatchesDoubleTapAtCoordinates(t *testing.T) { driverMock := mockdriver.New() action := verifier.Action{Kind: verifier.ActionKindDoubleTap, X: 100, Y: 200} - start := time.Now() if err := applyAction(context.Background(), driverMock, action, nil); err != nil { t.Fatalf("apply action: %v", err) } - elapsed := time.Since(start) - if elapsed < 40*time.Millisecond { - t.Errorf("expected >= 40ms gap between taps, elapsed %v", elapsed) - } taps := 0 for _, a := range driverMock.Actions() { - if a.Kind == mockdriver.ActionTap && a.X == 100 && a.Y == 200 { + if a.Kind == mockdriver.ActionDoubleTap && a.X == 100 && a.Y == 200 { taps++ } } - if taps != 2 { - t.Errorf("expected 2 Tap calls at (100,200), got %d in %v", taps, driverMock.Actions()) + if taps != 1 { + t.Errorf("expected 1 DoubleTap call at (100,200), got %d in %v", taps, driverMock.Actions()) } } -func TestApplyAction_DoubleTapDispatchesTwoSelectorTaps(t *testing.T) { +func TestApplyAction_DoubleTapDispatchesDoubleTapSelector(t *testing.T) { driverMock := mockdriver.New() action := verifier.Action{Kind: verifier.ActionKindDoubleTap, On: "id:save"} @@ -402,12 +457,12 @@ func TestApplyAction_DoubleTapDispatchesTwoSelectorTaps(t *testing.T) { } taps := 0 for _, a := range driverMock.Actions() { - if a.Kind == mockdriver.ActionTapSelector && a.Selector == "id:save" { + if a.Kind == mockdriver.ActionDoubleTapSelector && a.Selector == "id:save" { taps++ } } - if taps != 2 { - t.Errorf("expected 2 TapSelector calls with id:save, got %d in %v", taps, driverMock.Actions()) + if taps != 1 { + t.Errorf("expected 1 DoubleTapSelector call with id:save, got %d in %v", taps, driverMock.Actions()) } } @@ -962,7 +1017,8 @@ func TestIsTransientApplyError_Classification(t *testing.T) { // after a Wait action - the action already provides settling time. func TestRunner_WaitActionSkipsIdle(t *testing.T) { const waitSpec = ` -globalThis.actions = __sanderling__.actions(() => [__sanderling__.wait({ durationMillis: 5 })]); +import { actions, Wait } from "@sanderling/spec"; +globalThis.actions = actions(() => [Wait({ durationMillis: 5 })]); ` state := newHarnessWithSpec(t, waitSpec) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) diff --git a/internal/runner/source.go b/internal/runner/source.go new file mode 100644 index 0000000..f2a7367 --- /dev/null +++ b/internal/runner/source.go @@ -0,0 +1,72 @@ +package runner + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/priyanshujain/sanderling/internal/driver" + "github.com/priyanshujain/sanderling/internal/verifier" +) + +// ActionSource resolves the next action for a step. Both runtimes (the goja +// picker and the web/V8 picker) implement it so the runner loop has one path +// and no per-step driver type assertion. NextAction returns verifier.ErrNoAction +// when the picker declined to act this tick. +type ActionSource interface { + NextAction(ctx context.Context) (verifier.Action, error) +} + +// ExtractorSource yields per-step extractor overrides the runner applies after +// PushSnapshot. The mobile path has none (returns nil); the web path returns the +// values its extractors computed in V8 against the real DOM. +type ExtractorSource interface { + ExtractorOverrides(ctx context.Context) (map[int]json.RawMessage, error) +} + +// gojaSource drives both action selection and (trivially) extractor overrides +// for the mobile path, where the goja-bundled picker runs in-process and no V8 +// extractor values exist. +type gojaSource struct { + verifier *verifier.Verifier +} + +func (s gojaSource) NextAction(context.Context) (verifier.Action, error) { + return s.verifier.NextAction() +} + +func (gojaSource) ExtractorOverrides(context.Context) (map[int]json.RawMessage, error) { + return nil, nil +} + +// webSource adapts the chrome driver's V8 picker: it evaluates the bundled +// __sanderlingNextAction__ / __sanderlingExtractors__ globals and decodes their +// JSON into the unified action and override shapes. +type webSource struct { + web driver.WebDriver +} + +func (s webSource) NextAction(ctx context.Context) (verifier.Action, error) { + raw, err := s.web.NextActionFromV8(ctx) + if err != nil { + return verifier.Action{}, fmt.Errorf("v8 next action: %w", err) + } + // Both engines emit the unified flat camelCase wire contract; one decoder + // reads it. A null payload means the generator declined to act this tick. + return verifier.DecodeAction(raw) +} + +func (s webSource) ExtractorOverrides(ctx context.Context) (map[int]json.RawMessage, error) { + return s.web.EvaluateExtractors(ctx) +} + +// pickSources selects the runtime's action and extractor sources ONCE at setup +// from the driver's capabilities, so the step loop never type-asserts. +func pickSources(options Options) (ActionSource, ExtractorSource) { + if web, ok := options.Driver.(driver.WebDriver); ok { + source := webSource{web: web} + return source, source + } + source := gojaSource{verifier: options.Verifier} + return source, source +} diff --git a/internal/runner/testdata/trace.jsonl b/internal/runner/testdata/trace.jsonl new file mode 100644 index 0000000..365792e --- /dev/null +++ b/internal/runner/testdata/trace.jsonl @@ -0,0 +1,4 @@ +{"extractor_changes":{"extractor_0":{"prev":null,"curr":0}},"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":1,"timestamp":"0001-01-01T00:00:00Z"} +{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":2,"timestamp":"0001-01-01T00:00:00Z"} +{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":3,"timestamp":"0001-01-01T00:00:00Z"} +{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":4,"timestamp":"0001-01-01T00:00:00Z"} diff --git a/internal/runner/testdata/violation-summary.txt b/internal/runner/testdata/violation-summary.txt new file mode 100644 index 0000000..f042d62 --- /dev/null +++ b/internal/runner/testdata/violation-summary.txt @@ -0,0 +1,4 @@ + +run complete: 4 steps +1 violation record(s): + step 1: [balanceNonNegative] diff --git a/internal/sidecar/embed_test.go b/internal/sidecar/embed_test.go deleted file mode 100644 index 8519865..0000000 --- a/internal/sidecar/embed_test.go +++ /dev/null @@ -1,20 +0,0 @@ -//go:build !withsidecar - -package sidecar - -import "testing" - -func TestStub_IsPlaceholder(t *testing.T) { - if !IsPlaceholder() { - t.Errorf("default build should be a placeholder") - } - if EmbeddedSize() != 0 { - t.Errorf("stub should have empty embedded JAR, got size %d", EmbeddedSize()) - } -} - -func TestStub_ExtractFails(t *testing.T) { - if _, err := Extract(t.TempDir()); err == nil { - t.Fatal("expected Extract to fail in stub mode") - } -} diff --git a/internal/sidecar/embed.go b/internal/sidecarassets/embed.go similarity index 92% rename from internal/sidecar/embed.go rename to internal/sidecarassets/embed.go index 001a3a6..c61976a 100644 --- a/internal/sidecar/embed.go +++ b/internal/sidecarassets/embed.go @@ -1,4 +1,5 @@ -package sidecar +// Package sidecarassets embeds the native sidecar JAR and extracts it to disk at runtime. +package sidecarassets import ( "crypto/sha256" diff --git a/internal/sidecar/embed_real.go b/internal/sidecarassets/embed_real.go similarity index 92% rename from internal/sidecar/embed_real.go rename to internal/sidecarassets/embed_real.go index 3e43ee7..e7edf0f 100644 --- a/internal/sidecar/embed_real.go +++ b/internal/sidecarassets/embed_real.go @@ -1,6 +1,6 @@ //go:build withsidecar -package sidecar +package sidecarassets import _ "embed" diff --git a/internal/sidecar/embed_stub.go b/internal/sidecarassets/embed_stub.go similarity index 92% rename from internal/sidecar/embed_stub.go rename to internal/sidecarassets/embed_stub.go index a2c9f9e..24e064c 100644 --- a/internal/sidecar/embed_stub.go +++ b/internal/sidecarassets/embed_stub.go @@ -1,6 +1,6 @@ //go:build !withsidecar -package sidecar +package sidecarassets var embeddedJAR []byte diff --git a/internal/sidecar/embed_withsidecar_test.go b/internal/sidecarassets/embed_withsidecar_test.go similarity index 98% rename from internal/sidecar/embed_withsidecar_test.go rename to internal/sidecarassets/embed_withsidecar_test.go index a2d1337..cd82946 100644 --- a/internal/sidecar/embed_withsidecar_test.go +++ b/internal/sidecarassets/embed_withsidecar_test.go @@ -1,6 +1,6 @@ //go:build withsidecar -package sidecar +package sidecarassets import ( "crypto/sha256" diff --git a/internal/testrun/driver.go b/internal/testrun/driver.go index 33505d8..d58f41c 100644 --- a/internal/testrun/driver.go +++ b/internal/testrun/driver.go @@ -14,7 +14,7 @@ import ( "github.com/priyanshujain/sanderling/internal/driver/chrome" driverSidecar "github.com/priyanshujain/sanderling/internal/driver/sidecar" "github.com/priyanshujain/sanderling/internal/ios" - "github.com/priyanshujain/sanderling/internal/sidecar" + "github.com/priyanshujain/sanderling/internal/sidecarassets" ) // buildDriver creates the appropriate DeviceDriver for the platform and returns @@ -30,11 +30,11 @@ func buildDriver(ctx context.Context, options Options, stdout io.Writer) (driver } sidecarDirectory := os.TempDir() + "/sanderling-sidecar" - jarPath, err := sidecar.Extract(sidecarDirectory) + jarPath, err := sidecarassets.Extract(sidecarDirectory) if err != nil { return nil, nil, fmt.Errorf("extract sidecar: %w", err) } - fmt.Fprintf(stdout, "sidecar JAR: %s (size=%d)\n", jarPath, sidecar.EmbeddedSize()) + fmt.Fprintf(stdout, "sidecar JAR: %s (size=%d)\n", jarPath, sidecarassets.EmbeddedSize()) sidecarPort, err := pickFreePort() if err != nil { diff --git a/internal/testrun/testrun.go b/internal/testrun/testrun.go index d77e948..6d58441 100644 --- a/internal/testrun/testrun.go +++ b/internal/testrun/testrun.go @@ -1,12 +1,13 @@ +// Package testrun wires together the device, bundler, runner, and verifier into a single test pipeline. package testrun import ( "context" "fmt" "io" - "math/rand/v2" "os" "path/filepath" + "strconv" "time" "github.com/priyanshujain/sanderling/internal/android" @@ -53,14 +54,21 @@ func Execute(ctx context.Context, options Options, stdout io.Writer) error { aliases["@sanderling/spec/defaults"] = filepath.Join(base, "defaults/index.ts") aliases["@sanderling/spec/defaults/properties"] = filepath.Join(base, "defaults/properties.ts") } + seed := resolveSeed(options.Seed) defines := map[string]string{ "SANDERLING_TEST_PHONE": os.Getenv("SANDERLING_TEST_PHONE"), "SANDERLING_TEST_OTP": os.Getenv("SANDERLING_TEST_OTP"), + "SANDERLING_SEED": strconv.FormatInt(seed, 10), + } + gojaRuntimePath := resolveGojaRuntimePath(specAPIPath, options.Spec) + if gojaRuntimePath == "" { + return fmt.Errorf("goja-runtime.ts not found near %s; checkout pkg/spec or set @sanderling/spec alias", options.Spec) } bundle, err := bundler.Bundle(bundler.Options{ - EntryFile: options.Spec, - Defines: defines, - Aliases: aliases, + EntryFile: options.Spec, + RuntimeFile: gojaRuntimePath, + Defines: defines, + Aliases: aliases, }) if err != nil { return fmt.Errorf("bundle spec: %w", err) @@ -101,12 +109,9 @@ func Execute(ctx context.Context, options Options, stdout io.Writer) error { } } - seed := options.Seed - if seed == 0 { - seed = time.Now().UnixNano() - } verifierInstance, err := verifier.New( - verifier.WithRand(rand.New(rand.NewPCG(uint64(seed), 0))), + verifier.WithSeed(uint64(seed)), + verifier.WithPlatform(options.Platform), verifier.WithAppPackage(options.BundleID), ) if err != nil { @@ -161,25 +166,38 @@ func Execute(ctx context.Context, options Options, stdout io.Writer) error { return fmt.Errorf("runner: %w", err) } - fmt.Fprintf(stdout, "\nrun complete: %d steps in %s\n", summary.Steps, summary.EndTime.Sub(summary.StartTime).Round(time.Millisecond)) - if len(summary.Violations) == 0 { - fmt.Fprintln(stdout, "no violations.") - } else { - fmt.Fprintf(stdout, "%d violation record(s):\n", len(summary.Violations)) - for _, violation := range summary.Violations { - fmt.Fprintf(stdout, " step %d: %v\n", violation.StepIndex, violation.Properties) - } - } + fmt.Fprintf(stdout, "\nelapsed: %s\n", summary.EndTime.Sub(summary.StartTime).Round(time.Millisecond)) + runner.RenderSummary(stdout, summary, options.Platform) return nil } +// resolveSeed returns the configured seed, or a time-derived one when unset. +// The same value seeds both the goja PRNG and the web bundle's SANDERLING_SEED +// define, so a single run is reproducible across both runtimes. +func resolveSeed(configured int64) int64 { + if configured != 0 { + return configured + } + return time.Now().UnixNano() +} + // resolveWebRuntimePath returns the path to pkg/spec/src/web-runtime.ts. -// Tries the spec-API checkout first (so monorepo development works without -// publishing the package), then falls back to a sibling of the resolved -// @sanderling/spec entry, and finally to a node_modules path. func resolveWebRuntimePath(specAPIPath, userSpecPath string) string { + return resolveRuntimeSibling(specAPIPath, userSpecPath, "web-runtime.ts") +} + +// resolveGojaRuntimePath returns the path to pkg/spec/src/goja-runtime.ts, the +// native verifier's runtime entry that installs __sanderlingNextAction__. +func resolveGojaRuntimePath(specAPIPath, userSpecPath string) string { + return resolveRuntimeSibling(specAPIPath, userSpecPath, "goja-runtime.ts") +} + +// resolveRuntimeSibling finds a runtime-entry file that sits beside the spec-API +// index.ts. Tries the spec-API checkout first (so monorepo development works +// without publishing the package), then falls back to a node_modules path. +func resolveRuntimeSibling(specAPIPath, userSpecPath, filename string) string { if specAPIPath != "" { - candidate := filepath.Join(filepath.Dir(specAPIPath), "web-runtime.ts") + candidate := filepath.Join(filepath.Dir(specAPIPath), filename) if _, err := os.Stat(candidate); err == nil { return candidate } @@ -187,7 +205,7 @@ func resolveWebRuntimePath(specAPIPath, userSpecPath string) string { if absoluteSpec, err := filepath.Abs(userSpecPath); err == nil { directory := filepath.Dir(absoluteSpec) for { - candidate := filepath.Join(directory, "node_modules", "@sanderling", "spec", "src", "web-runtime.ts") + candidate := filepath.Join(directory, "node_modules", "@sanderling", "spec", "src", filename) if _, err := os.Stat(candidate); err == nil { return candidate } diff --git a/internal/testrun/testrun_test.go b/internal/testrun/testrun_test.go index 9611150..5cd63bf 100644 --- a/internal/testrun/testrun_test.go +++ b/internal/testrun/testrun_test.go @@ -3,9 +3,31 @@ package testrun import ( "os" "path/filepath" + "strconv" "testing" ) +func TestResolveSeed_UsesConfiguredWhenNonZero(t *testing.T) { + if got := resolveSeed(42); got != 42 { + t.Fatalf("got %d, want 42", got) + } +} + +func TestResolveSeed_DerivesWhenZero(t *testing.T) { + if got := resolveSeed(0); got == 0 { + t.Fatal("expected a non-zero time-derived seed") + } +} + +// TestSeedDefineFormatsAsDecimal guards the contract that Execute uses to put +// the seed into the web bundle: strconv.FormatInt base 10, which web-runtime +// folds to a 32-bit PRNG seed. +func TestSeedDefineFormatsAsDecimal(t *testing.T) { + if got := strconv.FormatInt(resolveSeed(8675309), 10); got != "8675309" { + t.Fatalf("got %q, want 8675309", got) + } +} + func TestResolveSpecAPIPath_FindsUpwardSibling(t *testing.T) { root := t.TempDir() apiPath := filepath.Join(root, "pkg", "spec", "src", "index.ts") diff --git a/internal/trace/writer.go b/internal/trace/writer.go index da7e28d..ffd6e7b 100644 --- a/internal/trace/writer.go +++ b/internal/trace/writer.go @@ -1,3 +1,4 @@ +// Package trace records each run's steps, snapshots, and violations to disk for later inspection. package trace import ( @@ -13,23 +14,38 @@ import ( ) type Step struct { - Index int `json:"step"` - Timestamp time.Time `json:"timestamp"` - Screen string `json:"screen,omitempty"` - Snapshots map[string]json.RawMessage `json:"snapshots,omitempty"` + Index int `json:"step"` + Timestamp time.Time `json:"timestamp"` + Screen string `json:"screen,omitempty"` + Snapshots map[string]json.RawMessage `json:"snapshots,omitempty"` // NextAction is the action chosen for the next iteration based on observing this step. - NextAction *Action `json:"next_action,omitempty"` - Exceptions []Exception `json:"exceptions,omitempty"` - Violations []string `json:"violations,omitempty"` - Hierarchy *hierarchy.Tree `json:"hierarchy,omitempty"` - Residuals map[string]json.RawMessage `json:"residuals,omitempty"` - Metrics *Metrics `json:"metrics,omitempty"` - ExtractorChanges map[string]ExtractorChange `json:"extractor_changes,omitempty"` + NextAction *Action `json:"next_action,omitempty"` + Exceptions []Exception `json:"exceptions,omitempty"` + Violations []string `json:"violations,omitempty"` + Hierarchy *hierarchy.Tree `json:"hierarchy,omitempty"` + Residuals map[string]json.RawMessage `json:"residuals,omitempty"` + Metrics *Metrics `json:"metrics,omitempty"` + ExtractorChanges map[string]ExtractorChange `json:"extractor_changes,omitempty"` // Transitional marks a step whose hierarchy still showed a NavHost // cross-fade (multiple route-level *Screen ids) after the runner's // retry budget. The verifier is skipped for these steps so transient // state does not poison the previous/current extractor advance. Transitional bool `json:"transitional,omitempty"` + // SkippedVerification is set true exactly when the verifier was skipped + // for this step, so downstream tooling can tell a deliberately-skipped + // step from one that was verified and came back clean. + SkippedVerification bool `json:"skipped_verification,omitempty"` + // Witnesses records the violation witness for each property that newly + // violated at this step: the cause and the extractor values at onset. + Witnesses map[string]Witness `json:"witnesses,omitempty"` +} + +// Witness is the trace-side record of a property violation: why it fired and a +// snapshot of every extractor's value at the violating step. +type Witness struct { + Reason string `json:"reason,omitempty"` + IsError bool `json:"is_error,omitempty"` + Extractors map[string]json.RawMessage `json:"extractors,omitempty"` } // ExtractorChange records the prev/curr JSON values of an extractor whose @@ -83,14 +99,14 @@ type Exception struct { } type Meta struct { - Seed int64 `json:"seed"` - SpecPath string `json:"spec_path"` - BundleSHA256 string `json:"bundle_sha256"` - Platform string `json:"platform"` - BundleID string `json:"bundle_id"` - StartedAt time.Time `json:"started_at"` - EndedAt *time.Time `json:"ended_at,omitempty"` - SanderlingVersion string `json:"sanderling_version"` + Seed int64 `json:"seed"` + SpecPath string `json:"spec_path"` + BundleSHA256 string `json:"bundle_sha256"` + Platform string `json:"platform"` + BundleID string `json:"bundle_id"` + StartedAt time.Time `json:"started_at"` + EndedAt *time.Time `json:"ended_at,omitempty"` + SanderlingVersion string `json:"sanderling_version"` } type Writer struct { diff --git a/internal/verifier/ax_integration_test.go b/internal/verifier/ax_integration_test.go index fa1b7e9..e634657 100644 --- a/internal/verifier/ax_integration_test.go +++ b/internal/verifier/ax_integration_test.go @@ -61,7 +61,7 @@ func TestStateAxObjectSelectorTestTagAlias(t *testing.T) { // TestStateAxFindWorks verifies that a Parse+PushSnapshot+extract round trip // actually lets the spec resolve selectors through state.ax.find. -// Reads /tmp/live-dump.json (Maestro TreeNode JSON format); skipped if absent. +// Reads /tmp/live-dump.json (sidecar TreeNode JSON format); skipped if absent. func TestStateAxFindWorks(t *testing.T) { jsonText, err := os.ReadFile("/tmp/live-dump.json") if err != nil { diff --git a/internal/verifier/bindings.go b/internal/verifier/bindings.go index eaae32a..a43a824 100644 --- a/internal/verifier/bindings.go +++ b/internal/verifier/bindings.go @@ -2,6 +2,7 @@ package verifier import ( "fmt" + "math/big" "time" "github.com/dop251/goja" @@ -9,8 +10,10 @@ import ( type extractorState struct { getter goja.Callable - handle *goja.Object name string + // currentValue/previousValue back the handle's current/previous accessors. + currentValue goja.Value + previousValue goja.Value // prev/curr cache the JSON-encoded extractor values from the prior and // current PushSnapshot, used by ChangedExtractors to surface per-step // diffs in the trace. @@ -20,11 +23,6 @@ type extractorState struct { type formulaState struct { predicate goja.Callable - // err holds the goja error from this thunk's most recent invocation, or - // nil if the latest call succeeded. The thunk returns false on error so - // the LTL evaluator marks the property violated; PredicateError surfaces - // the underlying cause for the current step. - err error } type specKind int @@ -62,20 +60,7 @@ type formulaSpec struct { const ( tagFormula = "__sanderlingFormula" tagFormulaSpecIndex = "__sanderlingFormulaSpec" - tagActionGenerator = "__sanderlingActionGenerator" - tagInternalKind = "__sanderlingKind" tagSelector = "__sanderlingSelector" - - internalKindActions = "actions" - internalKindWeighted = "weighted" - internalKindBuiltinTaps = "taps" - internalKindBuiltinDoubleTaps = "doubleTaps" - internalKindBuiltinTyping = "typing" - internalKindBuiltinSwipes = "swipes" - internalKindBuiltinWaitOnce = "waitOnce" - internalKindBuiltinPressKey = "pressKey" - internalKindBuiltinLongPresses = "longPresses" - internalKindBuiltinScrolls = "scrolls" ) // installRuntimeBindings exposes globalThis.__sanderling__ to the loaded spec. @@ -97,65 +82,75 @@ func (v *Verifier) installRuntimeBindings() error { if err := sanderling.Set("eventually", v.bindEventually); err != nil { return err } - if err := sanderling.Set("actions", v.bindActions); err != nil { - return err - } - if err := sanderling.Set("weighted", v.bindWeighted); err != nil { - return err - } - if err := sanderling.Set("from", v.bindFrom); err != nil { - return err - } - if err := sanderling.Set("tap", v.bindTap); err != nil { - return err - } - if err := sanderling.Set("doubleTap", v.bindDoubleTap); err != nil { - return err - } - if err := sanderling.Set("longPress", v.bindLongPress); err != nil { - return err - } - if err := sanderling.Set("scroll", v.bindScroll); err != nil { - return err - } - if err := sanderling.Set("inputText", v.bindInputText); err != nil { - return err - } - if err := sanderling.Set("swipe", v.bindSwipe); err != nil { - return err - } - if err := sanderling.Set("pressKey", v.bindPressKey); err != nil { - return err - } - if err := sanderling.Set("wait", v.bindWait); err != nil { - return err - } - if err := sanderling.Set("taps", v.builtinGenerator(internalKindBuiltinTaps)); err != nil { - return err - } - if err := sanderling.Set("doubleTaps", v.builtinGenerator(internalKindBuiltinDoubleTaps)); err != nil { - return err - } - if err := sanderling.Set("typing", v.builtinGenerator(internalKindBuiltinTyping)); err != nil { - return err - } - if err := sanderling.Set("swipes", v.builtinGenerator(internalKindBuiltinSwipes)); err != nil { - return err - } - if err := sanderling.Set("waitOnce", v.builtinGenerator(internalKindBuiltinWaitOnce)); err != nil { - return err - } - if err := sanderling.Set("pressKeys", v.builtinGenerator(internalKindBuiltinPressKey)); err != nil { - return err - } - if err := sanderling.Set("longPresses", v.builtinGenerator(internalKindBuiltinLongPresses)); err != nil { - return err - } - if err := sanderling.Set("scrolls", v.builtinGenerator(internalKindBuiltinScrolls)); err != nil { - return err - } - return v.runtime.GlobalObject().Set("__sanderling__", sanderling) + if err := v.runtime.GlobalObject().Set("__sanderling__", sanderling); err != nil { + return err + } + return v.installHost() +} + +// installHost exposes globalThis.__sanderlingHost__ for the goja runtime entry. +// The shared picker (pick.ts) draws against it: platform() drives the verb +// matrix and press-key pool; seedHi/seedLo construct its Pcg; queryCandidates +// enumerates targets over the hierarchy tree; reportUnsupported records the +// verb for the run report. +func (v *Verifier) installHost() error { + host := v.runtime.NewObject() + if err := host.Set("platform", func(goja.FunctionCall) goja.Value { + return v.runtime.ToValue(v.platform) + }); err != nil { + return err + } + if err := host.Set("seedHi", func(goja.FunctionCall) goja.Value { + return v.runtime.ToValue(new(big.Int).SetUint64(v.seed)) + }); err != nil { + return err + } + if err := host.Set("seedLo", func(goja.FunctionCall) goja.Value { + return v.runtime.ToValue(big.NewInt(0)) + }); err != nil { + return err + } + if err := host.Set("queryCandidates", v.bindQueryCandidates); err != nil { + return err + } + if err := host.Set("reportUnsupported", func(call goja.FunctionCall) goja.Value { + v.recordUnsupported(call.Argument(0).String()) + return goja.Undefined() + }); err != nil { + return err + } + return v.runtime.GlobalObject().Set("__sanderlingHost__", host) +} + +// recordUnsupported notes a verb the picker requested that this platform +// cannot dispatch, deduped and in first-seen order. +func (v *Verifier) recordUnsupported(verb string) { + if verb == "" || v.unsupportedSeen[verb] { + return + } + v.unsupportedSeen[verb] = true + v.unsupported = append(v.unsupported, verb) +} + +// bindQueryCandidates returns the host-enumerated targets for a verb as an +// array of {x, y, selector, width, height}, in tree order. +func (v *Verifier) bindQueryCandidates(call goja.FunctionCall) goja.Value { + verb := call.Argument(0).String() + candidates := v.candidatesForVerb(verb) + array := v.runtime.NewArray() + for index, candidate := range candidates { + item := v.runtime.NewObject() + _ = item.Set("x", candidate.x) + _ = item.Set("y", candidate.y) + _ = item.Set("width", candidate.width) + _ = item.Set("height", candidate.height) + if candidate.selector != "" { + _ = item.Set("selector", candidate.selector) + } + _ = array.Set(fmt.Sprintf("%d", index), item) + } + return array } func (v *Verifier) bindExtract(call goja.FunctionCall) goja.Value { @@ -177,14 +172,43 @@ func (v *Verifier) bindExtract(call goja.FunctionCall) goja.Value { name = fmt.Sprintf("extractor_%d", len(v.extractors)) } - handle := v.runtime.NewObject() - _ = handle.Set("current", goja.Undefined()) - _ = handle.Set("previous", goja.Undefined()) + state := &extractorState{ + getter: getter, + name: name, + currentValue: goja.Undefined(), + previousValue: goja.Undefined(), + } - v.extractors = append(v.extractors, &extractorState{getter: getter, handle: handle, name: name}) + handle := v.runtime.NewObject() + _ = handle.DefineAccessorProperty("current", v.runtime.ToValue(func(goja.FunctionCall) goja.Value { + v.checkNotExtracting("current") + return state.currentValue + }), nil, goja.FLAG_FALSE, goja.FLAG_TRUE) + _ = handle.DefineAccessorProperty("previous", v.runtime.ToValue(func(goja.FunctionCall) goja.Value { + v.checkNotExtracting("previous") + return state.previousValue + }), nil, goja.FLAG_FALSE, goja.FLAG_TRUE) + _ = handle.Set("named", func(call goja.FunctionCall) goja.Value { + state.name = call.Argument(0).String() + return handle + }) + + v.extractors = append(v.extractors, state) return handle } +// checkNotExtracting panics with a JS error when an extractor getter tries to +// read another extractor handle's current/previous. The message is identical to +// the web runtime's so authors see one diagnostic across engines. +func (v *Verifier) checkNotExtracting(slot string) { + if v.extracting { + panic(v.runtime.NewGoError(fmt.Errorf( + "reading .%s of an extractor inside another extractor is not allowed; extractor getters may read only from the state argument", + slot, + ))) + } +} + // bindAlways accepts either a predicate function (legacy shape) or a formula // handle (new shape). Both produce a formula handle tagged with // __sanderlingFormulaSpec. @@ -273,9 +297,6 @@ func (v *Verifier) formulaHandle(kind specKind, index int) *goja.Object { handle := v.runtime.NewObject() _ = handle.Set(tagFormula, true) _ = handle.Set(tagFormulaSpecIndex, index) - // Keep __sanderlingIndex as an alias so older property shapes that read it keep - // working during backward-compat transitions. - _ = handle.Set("__sanderlingIndex", index) _ = handle.Set("implies", v.binaryChain(index, specKindImplies)) _ = handle.Set("or", v.binaryChain(index, specKindOr)) @@ -356,164 +377,3 @@ func (v *Verifier) extractSpecIndex(value goja.Value) (int, bool) { } return int(indexValue.ToInteger()), true } - -func (v *Verifier) bindActions(call goja.FunctionCall) goja.Value { - if len(call.Arguments) != 1 { - panic(v.runtime.NewTypeError("actions requires a single generator argument")) - } - if _, ok := goja.AssertFunction(call.Arguments[0]); !ok { - panic(v.runtime.NewTypeError("actions argument must be a function")) - } - handle := v.runtime.NewObject() - _ = handle.Set(tagActionGenerator, true) - _ = handle.Set(tagInternalKind, internalKindActions) - _ = handle.Set("generate", call.Arguments[0]) - return handle -} - -func (v *Verifier) bindWeighted(call goja.FunctionCall) goja.Value { - entries := v.runtime.NewArray() - for index, argument := range call.Arguments { - object := argument.ToObject(v.runtime) - if object == nil { - panic(v.runtime.NewTypeError(fmt.Sprintf("weighted entry %d must be a [number, generator] tuple", index))) - } - if err := entries.Set(fmt.Sprintf("%d", index), object); err != nil { - panic(v.runtime.NewGoError(err)) - } - } - handle := v.runtime.NewObject() - _ = handle.Set(tagActionGenerator, true) - _ = handle.Set(tagInternalKind, internalKindWeighted) - _ = handle.Set("entries", entries) - return handle -} - -// bindFrom returns a `{ generate }` that picks uniformly at random from the -// provided items using the verifier's seeded rng. -func (v *Verifier) bindFrom(call goja.FunctionCall) goja.Value { - if len(call.Arguments) != 1 { - panic(v.runtime.NewTypeError("from requires an array argument")) - } - itemsValue := call.Arguments[0] - itemsObject := itemsValue.ToObject(v.runtime) - if itemsObject == nil { - panic(v.runtime.NewTypeError("from argument must be an array")) - } - lengthValue := itemsObject.Get("length") - if lengthValue == nil { - panic(v.runtime.NewTypeError("from argument must be array-like")) - } - length := int(lengthValue.ToInteger()) - - handle := v.runtime.NewObject() - _ = handle.Set("generate", func(goja.FunctionCall) goja.Value { - if length == 0 { - return goja.Undefined() - } - index := v.rng.IntN(length) - return itemsObject.Get(fmt.Sprintf("%d", index)) - }) - return handle -} - -func (v *Verifier) bindTap(call goja.FunctionCall) goja.Value { - parameters := call.Argument(0).ToObject(v.runtime) - if parameters == nil { - panic(v.runtime.NewTypeError("Tap requires {on}")) - } - handle := v.runtime.NewObject() - _ = handle.Set("kind", "Tap") - _ = handle.Set("on", parameters.Get("on")) - return handle -} - -func (v *Verifier) bindDoubleTap(call goja.FunctionCall) goja.Value { - parameters := call.Argument(0).ToObject(v.runtime) - if parameters == nil { - panic(v.runtime.NewTypeError("DoubleTap requires {on}")) - } - handle := v.runtime.NewObject() - _ = handle.Set("kind", "DoubleTap") - _ = handle.Set("on", parameters.Get("on")) - return handle -} - -func (v *Verifier) bindLongPress(call goja.FunctionCall) goja.Value { - parameters := call.Argument(0).ToObject(v.runtime) - if parameters == nil { - panic(v.runtime.NewTypeError("LongPress requires {on}")) - } - handle := v.runtime.NewObject() - _ = handle.Set("kind", "LongPress") - _ = handle.Set("on", parameters.Get("on")) - return handle -} - -func (v *Verifier) bindScroll(call goja.FunctionCall) goja.Value { - parameters := call.Argument(0).ToObject(v.runtime) - if parameters == nil { - panic(v.runtime.NewTypeError("Scroll requires {direction}")) - } - handle := v.runtime.NewObject() - _ = handle.Set("kind", "Scroll") - _ = handle.Set("direction", parameters.Get("direction")) - _ = handle.Set("in", parameters.Get("in")) - return handle -} - -func (v *Verifier) bindInputText(call goja.FunctionCall) goja.Value { - parameters := call.Argument(0).ToObject(v.runtime) - if parameters == nil { - panic(v.runtime.NewTypeError("InputText requires {into, text}")) - } - handle := v.runtime.NewObject() - _ = handle.Set("kind", "InputText") - _ = handle.Set("into", parameters.Get("into")) - _ = handle.Set("text", parameters.Get("text")) - return handle -} - -func (v *Verifier) bindSwipe(call goja.FunctionCall) goja.Value { - parameters := call.Argument(0).ToObject(v.runtime) - if parameters == nil { - panic(v.runtime.NewTypeError("Swipe requires {from, to}")) - } - handle := v.runtime.NewObject() - _ = handle.Set("kind", "Swipe") - _ = handle.Set("from", parameters.Get("from")) - _ = handle.Set("to", parameters.Get("to")) - if duration := parameters.Get("durationMillis"); duration != nil && !goja.IsUndefined(duration) { - _ = handle.Set("durationMillis", duration) - } - return handle -} - -func (v *Verifier) bindPressKey(call goja.FunctionCall) goja.Value { - parameters := call.Argument(0).ToObject(v.runtime) - if parameters == nil { - panic(v.runtime.NewTypeError("PressKey requires {key}")) - } - handle := v.runtime.NewObject() - _ = handle.Set("kind", "PressKey") - _ = handle.Set("key", parameters.Get("key")) - return handle -} - -func (v *Verifier) bindWait(call goja.FunctionCall) goja.Value { - parameters := call.Argument(0).ToObject(v.runtime) - if parameters == nil { - panic(v.runtime.NewTypeError("Wait requires {durationMillis}")) - } - handle := v.runtime.NewObject() - _ = handle.Set("kind", "Wait") - _ = handle.Set("durationMillis", parameters.Get("durationMillis")) - return handle -} - -func (v *Verifier) builtinGenerator(kind string) *goja.Object { - handle := v.runtime.NewObject() - _ = handle.Set(tagActionGenerator, true) - _ = handle.Set(tagInternalKind, kind) - return handle -} diff --git a/internal/verifier/extractor_guard_test.go b/internal/verifier/extractor_guard_test.go new file mode 100644 index 0000000..1fcf5d2 --- /dev/null +++ b/internal/verifier/extractor_guard_test.go @@ -0,0 +1,34 @@ +package verifier + +import ( + "strings" + "testing" +) + +func TestPushSnapshot_CrossExtractorReadIsGuarded(t *testing.T) { + verifier := newVerifier(t) + mustLoad(t, verifier, ` +const a = __sanderling__.extract(state => 1); +const b = __sanderling__.extract(() => a.current); +globalThis.a = a; +globalThis.b = b; +`) + err := verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}}) + if err == nil { + t.Fatal("expected guard error, got nil") + } + if !strings.Contains(err.Error(), "inside another extractor is not allowed") { + t.Errorf("guard error wrong: %v", err) + } +} + +func TestExtract_NamedSetsExtractorName(t *testing.T) { + verifier := newVerifier(t) + mustLoad(t, verifier, ` +const route = __sanderling__.extract(state => "home").named("route"); +globalThis.route = route; +`) + if got := verifier.extractors[0].name; got != "route" { + t.Errorf("named() did not set name: got %q, want %q", got, "route") + } +} diff --git a/internal/verifier/marshal.go b/internal/verifier/marshal.go index defee8f..ff40d9e 100644 --- a/internal/verifier/marshal.go +++ b/internal/verifier/marshal.go @@ -266,10 +266,10 @@ func selectorPathFromJS(runtime *goja.Runtime, arg goja.Value) ([]hierarchy.Sele } // selectorStringFromJS returns a string representation of the selector argument -// for tagging returned element objects (used by selectorOf to reconstruct the -// selector when the element is passed back as an action target). Output -// follows the canonical hierarchy selector grammar: "k:v" pairs space-joined -// per object, chains joined by " > ". +// for tagging returned ax element objects (state.ax.find/findAll), so a spec +// reading an element's selector sees the canonical form. Output follows the +// hierarchy selector grammar: "k:v" pairs space-joined per object, chains +// joined by " > ". func selectorStringFromJS(runtime *goja.Runtime, arg goja.Value) string { if arg == nil || goja.IsUndefined(arg) || goja.IsNull(arg) { return "" @@ -414,132 +414,68 @@ func jsonToJSValue(runtime *goja.Runtime, raw json.RawMessage) (goja.Value, erro return runtime.ToValue(generic), nil } -// jsValueToAction converts a JS-side action object into a Go Action. -func jsValueToAction(runtime *goja.Runtime, value goja.Value) (Action, error) { - if value == nil || goja.IsNull(value) || goja.IsUndefined(value) { - return Action{}, fmt.Errorf("nil action") +// wireAction is the unified flat wire contract both runtime entries emit +// (runtime-entry.ts serializeAction). ONE decoder reads it on both the goja +// (native) and the runner (web) sides, so a field rename cannot silently turn +// an action into a no-op on one path only. +type wireAction struct { + Kind string `json:"kind"` + X int `json:"x"` + Y int `json:"y"` + Selector string `json:"selector"` + Text string `json:"text"` + Key string `json:"key"` + Direction string `json:"direction"` + FromX int `json:"fromX"` + FromY int `json:"fromY"` + ToX int `json:"toX"` + ToY int `json:"toY"` + DurationMillis int `json:"durationMillis"` +} + +// DecodeAction turns one serialized action (the flat camelCase wire contract) +// into a Go Action. An empty payload or JSON "null" reports ErrNoAction. +func DecodeAction(raw json.RawMessage) (Action, error) { + if len(raw) == 0 || string(raw) == "null" { + return Action{}, ErrNoAction } - object := value.ToObject(runtime) - kindValue := object.Get("kind") - if kindValue == nil { - return Action{}, fmt.Errorf("action missing kind") + var wire wireAction + if err := json.Unmarshal(raw, &wire); err != nil { + return Action{}, fmt.Errorf("decode action: %w", err) } - kind := kindValue.String() - switch kind { + switch wire.Kind { case "Tap": - on := object.Get("on") - x, y := coordinatesOf(runtime, on) - return Action{Kind: ActionKindTap, On: selectorOf(runtime, on), X: x, Y: y}, nil + return Action{Kind: ActionKindTap, On: wire.Selector, X: wire.X, Y: wire.Y}, nil case "DoubleTap": - on := object.Get("on") - x, y := coordinatesOf(runtime, on) - return Action{Kind: ActionKindDoubleTap, On: selectorOf(runtime, on), X: x, Y: y}, nil + return Action{Kind: ActionKindDoubleTap, On: wire.Selector, X: wire.X, Y: wire.Y}, nil + case "LongPress": + return Action{Kind: ActionKindLongPress, On: wire.Selector, X: wire.X, Y: wire.Y}, nil case "InputText": - into := object.Get("into") - text := object.Get("text") - x, y := coordinatesOf(runtime, into) - return Action{Kind: ActionKindInputText, On: selectorOf(runtime, into), Text: stringOf(text), X: x, Y: y}, nil + return Action{Kind: ActionKindInputText, On: wire.Selector, Text: wire.Text, X: wire.X, Y: wire.Y}, nil case "Swipe": - from := object.Get("from") - to := object.Get("to") - fromX, fromY := coordinatesOf(runtime, from) - toX, toY := coordinatesOf(runtime, to) - if fromX == 0 && fromY == 0 { - fromX, fromY = pointCoordinates(runtime, from) - } - if toX == 0 && toY == 0 { - toX, toY = pointCoordinates(runtime, to) - } return Action{ Kind: ActionKindSwipe, - FromX: fromX, - FromY: fromY, - ToX: toX, - ToY: toY, - DurationMillis: intField(object, "durationMillis"), + FromX: wire.FromX, + FromY: wire.FromY, + ToX: wire.ToX, + ToY: wire.ToY, + DurationMillis: wire.DurationMillis, }, nil - case "LongPress": - on := object.Get("on") - x, y := coordinatesOf(runtime, on) - return Action{Kind: ActionKindLongPress, On: selectorOf(runtime, on), X: x, Y: y}, nil case "Scroll": - in := object.Get("in") - x, y := coordinatesOf(runtime, in) return Action{ - Kind: ActionKindScroll, - Direction: stringOf(object.Get("direction")), - On: selectorOf(runtime, in), - X: x, - Y: y, + Kind: ActionKindScroll, + Direction: wire.Direction, + FromX: wire.FromX, + FromY: wire.FromY, + ToX: wire.ToX, + ToY: wire.ToY, + DurationMillis: wire.DurationMillis, }, nil case "PressKey": - return Action{Kind: ActionKindPressKey, Key: stringOf(object.Get("key"))}, nil + return Action{Kind: ActionKindPressKey, Key: wire.Key}, nil case "Wait": - return Action{Kind: ActionKindWait, DurationMillis: intField(object, "durationMillis")}, nil + return Action{Kind: ActionKindWait, DurationMillis: wire.DurationMillis}, nil default: - return Action{}, fmt.Errorf("unknown action kind %q", kind) + return Action{}, fmt.Errorf("unknown action kind %q", wire.Kind) } } - -// pointCoordinates reads a plain {x, y} literal (not an AX element), which is -// how Swipe endpoints are commonly expressed in specs. -func pointCoordinates(runtime *goja.Runtime, value goja.Value) (int, int) { - if value == nil || goja.IsNull(value) || goja.IsUndefined(value) { - return 0, 0 - } - object := value.ToObject(runtime) - if object == nil { - return 0, 0 - } - x := object.Get("x") - y := object.Get("y") - if x == nil || y == nil { - return 0, 0 - } - return int(x.ToInteger()), int(y.ToInteger()) -} - -func intField(object *goja.Object, name string) int { - value := object.Get(name) - if value == nil || goja.IsUndefined(value) || goja.IsNull(value) { - return 0 - } - return int(value.ToInteger()) -} - -func selectorOf(runtime *goja.Runtime, value goja.Value) string { - if value == nil || goja.IsNull(value) || goja.IsUndefined(value) { - return "" - } - object := value.ToObject(runtime) - if object == nil { - return value.String() - } - if tag := object.Get(tagSelector); tag != nil && !goja.IsUndefined(tag) { - return tag.String() - } - return value.String() -} - -func coordinatesOf(runtime *goja.Runtime, value goja.Value) (int, int) { - if value == nil || goja.IsNull(value) || goja.IsUndefined(value) { - return 0, 0 - } - object := value.ToObject(runtime) - if object == nil { - return 0, 0 - } - xValue := object.Get("x") - yValue := object.Get("y") - if xValue == nil || yValue == nil || goja.IsUndefined(xValue) || goja.IsUndefined(yValue) { - return 0, 0 - } - return int(xValue.ToInteger()), int(yValue.ToInteger()) -} - -func stringOf(value goja.Value) string { - if value == nil || goja.IsNull(value) || goja.IsUndefined(value) { - return "" - } - return value.String() -} diff --git a/internal/verifier/parity_test.go b/internal/verifier/parity_test.go new file mode 100644 index 0000000..e1463f5 --- /dev/null +++ b/internal/verifier/parity_test.go @@ -0,0 +1,103 @@ +package verifier + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// TestCrossRuntimeParity is the W2 acceptance gate: for a FIXED seed and a FIXED +// candidate state, the goja-bundled picker must emit the SAME action stream as +// the node/web pick.ts. Both engines run the SHARED pick.ts over the SHARED Pcg, +// so each asserts the SAME committed golden (pkg/spec/test/fixtures/ +// parity-golden.json); the node side does so in pkg/spec/test/parity.test.ts. +// Matching one golden on both sides proves they match each other without either +// invoking the other. +// +// The candidate ORDER and the per-tick PCG draw order are the parity contract. +// A stub __sanderlingHost__ feeds the SAME three candidates (in the SAME order) +// for every verb, so the hierarchy filter is out of the picture and the only +// variable left is the JS engine. The weighted root mixes a tap branch with a +// typing branch, exercising weighted selection, a builtin, and the input corpus +// within the 20-tick window; reordering candidates or adding/dropping a draw on +// either side shifts the stream and fails the golden. +func TestCrossRuntimeParity(t *testing.T) { + const seed uint64 = 0x9e3779b97f4a7c15 + golden := loadParityGolden(t) + + verifier := newVerifier(t, WithSeed(seed)) + installStubHost(t, verifier) + loadActionSpec(t, verifier, ` + import { taps, typing, weighted } from "@sanderling/spec"; + globalThis.actions = weighted([3, taps], [1, typing]); + `) + + if len(golden) == 0 { + t.Fatal("golden stream is empty") + } + for step, want := range golden { + got, err := verifier.NextAction() + if err != nil { + t.Fatalf("goja next action at step %d: %v", step, err) + } + if got != want { + t.Fatalf("step %d diverged from golden:\n goja=%+v\n want=%+v", step, got, want) + } + } +} + +// installStubHost replaces the verifier's hierarchy-backed __sanderlingHost__ +// with one returning a FIXED candidate list for every verb, keeping the seed the +// verifier was constructed with. It must run BEFORE Load, because the bundled +// goja runtime entry captures the host when the spec evaluates. +func installStubHost(t *testing.T, verifier *Verifier) { + t.Helper() + const stub = ` + const candidates = [ + { x: 50, y: 60, selector: "id:alpha", width: 100, height: 40 }, + { x: 150, y: 160, selector: "id:beta", width: 120, height: 48 }, + { x: 250, y: 260, selector: "id:gamma", width: 80, height: 32 }, + ]; + const seedHi = globalThis.__sanderlingHost__.seedHi; + const seedLo = globalThis.__sanderlingHost__.seedLo; + globalThis.__sanderlingHost__ = { + platform: () => "android", + queryCandidates: () => candidates, + reportUnsupported: () => {}, + seedHi, + seedLo, + }; + ` + if _, err := verifier.runtime.RunString(stub); err != nil { + t.Fatalf("install stub host: %v", err) + } +} + +// loadParityGolden decodes the shared golden stream with the SAME DecodeAction +// the runner uses, so the goja comparison is apples-to-apples with the wire the +// node picker emits. +func loadParityGolden(t *testing.T) []Action { + t.Helper() + path, err := filepath.Abs("../../pkg/spec/test/fixtures/parity-golden.json") + if err != nil { + t.Fatal(err) + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read golden: %v", err) + } + var raw []json.RawMessage + if err := json.Unmarshal(body, &raw); err != nil { + t.Fatalf("decode golden: %v", err) + } + stream := make([]Action, len(raw)) + for i, message := range raw { + action, err := DecodeAction(message) + if err != nil { + t.Fatalf("decode golden action %d: %v", i, err) + } + stream[i] = action + } + return stream +} diff --git a/internal/verifier/scope_test.go b/internal/verifier/scope_test.go index 265d5c9..aa469d8 100644 --- a/internal/verifier/scope_test.go +++ b/internal/verifier/scope_test.go @@ -33,7 +33,10 @@ func pushTree(t *testing.T, v *Verifier, treeJSON string) { // the result lands on its center regardless of seed. func TestTaps_ExcludeOffAppPackage(t *testing.T) { verifier := newVerifier(t, WithAppPackage("com.folio")) - mustLoad(t, verifier, `globalThis.actions = __sanderling__.taps;`) + loadActionSpec(t, verifier, ` + import { taps } from "@sanderling/spec"; + globalThis.actions = taps; + `) pushTree(t, verifier, scopedTreeJSON) action, err := verifier.NextAction() @@ -56,7 +59,10 @@ func TestTyping_ExcludeOffAppPackage(t *testing.T) { ] }` verifier := newVerifier(t, WithAppPackage("com.folio")) - mustLoad(t, verifier, `globalThis.actions = __sanderling__.typing;`) + loadActionSpec(t, verifier, ` + import { typing } from "@sanderling/spec"; + globalThis.actions = typing; + `) pushTree(t, verifier, treeJSON) action, err := verifier.NextAction() @@ -72,7 +78,10 @@ func TestTyping_ExcludeOffAppPackage(t *testing.T) { // exploration never scrolls the keyboard's emoji list instead of the app. func TestSwipes_ExcludeOffAppPackage(t *testing.T) { verifier := newVerifier(t, WithAppPackage("com.folio")) - mustLoad(t, verifier, `globalThis.actions = __sanderling__.swipes;`) + loadActionSpec(t, verifier, ` + import { swipes } from "@sanderling/spec"; + globalThis.actions = swipes; + `) pushTree(t, verifier, scopedTreeJSON) // Both the root and SubmitButton (com.folio) are valid anchors; only the @@ -103,7 +112,10 @@ func TestTaps_AllOffAppYieldsErrNoAction(t *testing.T) { ] }` verifier := newVerifier(t, WithAppPackage("com.folio")) - mustLoad(t, verifier, `globalThis.actions = __sanderling__.taps;`) + loadActionSpec(t, verifier, ` + import { taps } from "@sanderling/spec"; + globalThis.actions = taps; + `) pushTree(t, verifier, treeJSON) if _, err := verifier.NextAction(); !errors.Is(err, ErrNoAction) { @@ -121,7 +133,10 @@ func TestTaps_UnsetAppPackageKeepsAllNodes(t *testing.T) { ] }` verifier := newVerifier(t) - mustLoad(t, verifier, `globalThis.actions = __sanderling__.taps;`) + loadActionSpec(t, verifier, ` + import { taps } from "@sanderling/spec"; + globalThis.actions = taps; + `) pushTree(t, verifier, treeJSON) action, err := verifier.NextAction() @@ -137,7 +152,10 @@ func TestTaps_UnsetAppPackageKeepsAllNodes(t *testing.T) { // mirrors taps: it yields a LongPress on the only clickable in-app node. func TestLongPresses_TargetsClickableElement(t *testing.T) { verifier := newVerifier(t, WithAppPackage("com.folio")) - mustLoad(t, verifier, `globalThis.actions = __sanderling__.longPresses;`) + loadActionSpec(t, verifier, ` + import { longPresses } from "@sanderling/spec"; + globalThis.actions = longPresses; + `) pushTree(t, verifier, scopedTreeJSON) action, err := verifier.NextAction() @@ -163,7 +181,10 @@ func TestScrolls_TargetsScrollableContainer(t *testing.T) { ] }` verifier := newVerifier(t, WithAppPackage("com.folio")) - mustLoad(t, verifier, `globalThis.actions = __sanderling__.scrolls;`) + loadActionSpec(t, verifier, ` + import { scrolls } from "@sanderling/spec"; + globalThis.actions = scrolls; + `) pushTree(t, verifier, treeJSON) for range 50 { @@ -190,8 +211,8 @@ func TestScrolls_TargetsScrollableContainer(t *testing.T) { default: t.Fatalf("unexpected direction %q", action.Direction) } - if action.DurationMillis != 300 { - t.Fatalf("durationMillis = %d, want 300", action.DurationMillis) + if action.DurationMillis != 250 { + t.Fatalf("durationMillis = %d, want 250", action.DurationMillis) } } } @@ -200,7 +221,10 @@ func TestScrolls_TargetsScrollableContainer(t *testing.T) { // no scrollable container is present. func TestScrolls_NoScrollableYieldsErrNoAction(t *testing.T) { verifier := newVerifier(t, WithAppPackage("com.folio")) - mustLoad(t, verifier, `globalThis.actions = __sanderling__.scrolls;`) + loadActionSpec(t, verifier, ` + import { scrolls } from "@sanderling/spec"; + globalThis.actions = scrolls; + `) pushTree(t, verifier, scopedTreeJSON) if _, err := verifier.NextAction(); !errors.Is(err, ErrNoAction) { @@ -219,7 +243,10 @@ func TestTaps_EmptyPackageNodeStaysInScope(t *testing.T) { ] }` verifier := newVerifier(t, WithAppPackage("com.folio")) - mustLoad(t, verifier, `globalThis.actions = __sanderling__.taps;`) + loadActionSpec(t, verifier, ` + import { taps } from "@sanderling/spec"; + globalThis.actions = taps; + `) pushTree(t, verifier, treeJSON) action, err := verifier.NextAction() diff --git a/internal/verifier/spec_integration_test.go b/internal/verifier/spec_integration_test.go index 75f6530..b5dcf54 100644 --- a/internal/verifier/spec_integration_test.go +++ b/internal/verifier/spec_integration_test.go @@ -23,12 +23,17 @@ func bundleIntegrationSpec(t *testing.T) string { if err != nil { t.Fatal(err) } + runtimePath, err := filepath.Abs("../../pkg/spec/src/goja-runtime.ts") + if err != nil { + t.Fatal(err) + } defaultsPath, err := filepath.Abs("../../pkg/spec/src/defaults/properties.ts") if err != nil { t.Fatal(err) } bundle, err := bundler.Bundle(bundler.Options{ - EntryFile: specPath, + EntryFile: specPath, + RuntimeFile: runtimePath, Aliases: map[string]string{ "@sanderling/spec": apiPath, "@sanderling/spec/defaults/properties": defaultsPath, diff --git a/internal/verifier/verifier_test.go b/internal/verifier/verifier_test.go index fa2b0a4..2923ea0 100644 --- a/internal/verifier/verifier_test.go +++ b/internal/verifier/verifier_test.go @@ -3,13 +3,15 @@ package verifier import ( "encoding/json" "errors" - "math/rand/v2" + "os" + "path/filepath" "slices" "strings" "testing" "github.com/dop251/goja" + "github.com/priyanshujain/sanderling/internal/bundler" "github.com/priyanshujain/sanderling/internal/hierarchy" "github.com/priyanshujain/sanderling/internal/ltl" ) @@ -30,6 +32,43 @@ func mustLoad(t *testing.T, verifier *Verifier, source string) { } } +// bundleActionSpec bundles an inline TS spec authored against @sanderling/spec +// together with the goja runtime entry, so loading it installs +// __sanderlingNextAction__ (the shared picker). Action targets must be resolved +// ax elements (carrying x/y) or builtins; raw selector strings no longer +// resolve to coordinates in the unified contract. +func bundleActionSpec(t *testing.T, specSource string) string { + t.Helper() + dir := t.TempDir() + specPath := filepath.Join(dir, "spec.ts") + if err := os.WriteFile(specPath, []byte(specSource), 0o600); err != nil { + t.Fatal(err) + } + apiPath, err := filepath.Abs("../../pkg/spec/src/index.ts") + if err != nil { + t.Fatal(err) + } + runtimePath, err := filepath.Abs("../../pkg/spec/src/goja-runtime.ts") + if err != nil { + t.Fatal(err) + } + bundle, err := bundler.Bundle(bundler.Options{ + EntryFile: specPath, + RuntimeFile: runtimePath, + Aliases: map[string]string{"@sanderling/spec": apiPath}, + }) + if err != nil { + t.Fatal(err) + } + return string(bundle.JavaScript) +} + +// loadActionSpec bundles and loads an inline authored spec into the verifier. +func loadActionSpec(t *testing.T, verifier *Verifier, specSource string) { + t.Helper() + mustLoad(t, verifier, bundleActionSpec(t, specSource)) +} + const helloSpec = ` const screen = __sanderling__.extract(state => state.snapshots.screen ?? ""); const balance = __sanderling__.extract(state => state.snapshots["ledger.balance"] ?? 0); @@ -40,10 +79,6 @@ globalThis.balance = balance; globalThis.properties = { balanceNonNegative: __sanderling__.always(() => balance.current >= 0), }; - -globalThis.actions = __sanderling__.actions(() => [ - __sanderling__.tap({ on: "id:home_button" }), -]); ` func TestLoad_ExposesRuntimeBindings(t *testing.T) { @@ -231,9 +266,25 @@ globalThis.properties = { } func TestNextAction_FromActionsGenerator(t *testing.T) { + const treeJSON = `{ + "attributes": {"resource-id": "root", "bounds": "[0,0,100,100]"}, + "children": [ + {"attributes": {"resource-id": "home_button", "bounds": "[0,40,100,80]"}, "clickable": true, "enabled": true, "children": []} + ] + }` verifier := newVerifier(t) - mustLoad(t, verifier, helloSpec) - _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}}) + loadActionSpec(t, verifier, ` + import { actions, Tap } from "@sanderling/spec"; + globalThis.actions = actions(() => { + const home = state.ax.find("id:home_button"); + return home ? [Tap({ on: home })] : []; + }); + `) + tree, err := hierarchy.Parse(treeJSON) + if err != nil { + t.Fatal(err) + } + _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}, Tree: tree}) action, err := verifier.NextAction() if err != nil { @@ -248,16 +299,31 @@ func TestNextAction_FromActionsGenerator(t *testing.T) { } func TestNextAction_WeightedSelectsByWeight(t *testing.T) { - verifier := newVerifier(t, WithRand(rand.New(rand.NewPCG(42, 0)))) - mustLoad(t, verifier, ` - const tapHome = __sanderling__.actions(() => [__sanderling__.tap({ on: "id:home" })]); - const tapAway = __sanderling__.actions(() => [__sanderling__.tap({ on: "id:away" })]); - globalThis.actions = __sanderling__.weighted( - [1, tapHome], - [99, tapAway], - ); + const treeJSON = `{ + "attributes": {"resource-id": "root", "bounds": "[0,0,100,100]"}, + "children": [ + {"attributes": {"resource-id": "home", "bounds": "[0,0,100,40]"}, "clickable": true, "enabled": true, "children": []}, + {"attributes": {"resource-id": "away", "bounds": "[0,40,100,80]"}, "clickable": true, "enabled": true, "children": []} + ] + }` + verifier := newVerifier(t, WithSeed(42)) + loadActionSpec(t, verifier, ` + import { actions, weighted, Tap } from "@sanderling/spec"; + const tapHome = actions(() => { + const home = state.ax.find("id:home"); + return home ? [Tap({ on: home })] : []; + }); + const tapAway = actions(() => { + const away = state.ax.find("id:away"); + return away ? [Tap({ on: away })] : []; + }); + globalThis.actions = weighted([1, tapHome], [99, tapAway]); `) - _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}}) + tree, err := hierarchy.Parse(treeJSON) + if err != nil { + t.Fatal(err) + } + _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}, Tree: tree}) awayCount := 0 homeCount := 0 @@ -280,8 +346,9 @@ func TestNextAction_WeightedSelectsByWeight(t *testing.T) { func TestNextAction_EmptyGeneratorReturnsErrNoAction(t *testing.T) { verifier := newVerifier(t) - mustLoad(t, verifier, ` - globalThis.actions = __sanderling__.actions(() => []); + loadActionSpec(t, verifier, ` + import { actions } from "@sanderling/spec"; + globalThis.actions = actions(() => []); `) _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}}) @@ -291,13 +358,34 @@ func TestNextAction_EmptyGeneratorReturnsErrNoAction(t *testing.T) { } } +// setupTree carries the three targets the setup-precedence tests resolve. +const setupTree = `{ + "attributes": {"resource-id": "root", "bounds": "[0,0,100,120]"}, + "children": [ + {"attributes": {"resource-id": "setup", "bounds": "[0,0,100,40]"}, "clickable": true, "enabled": true, "children": []}, + {"attributes": {"resource-id": "main", "bounds": "[0,40,100,80]"}, "clickable": true, "enabled": true, "children": []}, + {"attributes": {"resource-id": "login", "bounds": "[0,80,100,120]"}, "clickable": true, "enabled": true, "children": []} + ] +}` + func TestNextAction_SetupTakesPrecedenceWhenYielding(t *testing.T) { verifier := newVerifier(t) - mustLoad(t, verifier, ` - globalThis.setup = __sanderling__.actions(() => [__sanderling__.tap({ on: "id:setup" })]); - globalThis.actions = __sanderling__.actions(() => [__sanderling__.tap({ on: "id:main" })]); + loadActionSpec(t, verifier, ` + import { actions, Tap } from "@sanderling/spec"; + globalThis.setup = actions(() => { + const target = state.ax.find("id:setup"); + return target ? [Tap({ on: target })] : []; + }); + globalThis.actions = actions(() => { + const target = state.ax.find("id:main"); + return target ? [Tap({ on: target })] : []; + }); `) - _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}}) + tree, err := hierarchy.Parse(setupTree) + if err != nil { + t.Fatal(err) + } + _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}, Tree: tree}) action, err := verifier.NextAction() if err != nil { @@ -310,11 +398,19 @@ func TestNextAction_SetupTakesPrecedenceWhenYielding(t *testing.T) { func TestNextAction_FallsThroughToActionsWhenSetupEmpty(t *testing.T) { verifier := newVerifier(t) - mustLoad(t, verifier, ` - globalThis.setup = __sanderling__.actions(() => []); - globalThis.actions = __sanderling__.actions(() => [__sanderling__.tap({ on: "id:main" })]); + loadActionSpec(t, verifier, ` + import { actions, Tap } from "@sanderling/spec"; + globalThis.setup = actions(() => []); + globalThis.actions = actions(() => { + const target = state.ax.find("id:main"); + return target ? [Tap({ on: target })] : []; + }); `) - _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}}) + tree, err := hierarchy.Parse(setupTree) + if err != nil { + t.Fatal(err) + } + _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}, Tree: tree}) action, err := verifier.NextAction() if err != nil { @@ -327,21 +423,30 @@ func TestNextAction_FallsThroughToActionsWhenSetupEmpty(t *testing.T) { func TestNextAction_SetupReengagesAfterRegression(t *testing.T) { verifier := newVerifier(t) - mustLoad(t, verifier, ` - globalThis.loggedIn = __sanderling__.extract(state => state.snapshots["loggedIn"] === true); - globalThis.setup = __sanderling__.actions(() => { + loadActionSpec(t, verifier, ` + import { actions, extract, Tap } from "@sanderling/spec"; + const loggedIn = extract(state => state.snapshots["loggedIn"] === true); + globalThis.setup = actions(() => { if (loggedIn.current) return []; - return [__sanderling__.tap({ on: "id:login" })]; + const target = state.ax.find("id:login"); + return target ? [Tap({ on: target })] : []; + }); + globalThis.actions = actions(() => { + const target = state.ax.find("id:main"); + return target ? [Tap({ on: target })] : []; }); - globalThis.actions = __sanderling__.actions(() => [__sanderling__.tap({ on: "id:main" })]); `) + tree, err := hierarchy.Parse(setupTree) + if err != nil { + t.Fatal(err) + } push := func(loggedIn bool) { raw := json.RawMessage(`false`) if loggedIn { raw = json.RawMessage(`true`) } - if err := verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{"loggedIn": raw}}); err != nil { + if err := verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{"loggedIn": raw}, Tree: tree}); err != nil { t.Fatal(err) } } @@ -376,14 +481,19 @@ func TestNextAction_SetupReengagesAfterRegression(t *testing.T) { func TestNextAction_NoSetupRegistered(t *testing.T) { verifier := newVerifier(t) - mustLoad(t, verifier, ` - globalThis.actions = __sanderling__.actions(() => [__sanderling__.tap({ on: "id:main" })]); + loadActionSpec(t, verifier, ` + import { actions, Tap } from "@sanderling/spec"; + globalThis.actions = actions(() => { + const target = state.ax.find("id:main"); + return target ? [Tap({ on: target })] : []; + }); `) - _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}}) - - if verifier.setupGenerator != nil { - t.Errorf("setupGenerator should be nil when spec does not export setup") + tree, err := hierarchy.Parse(setupTree) + if err != nil { + t.Fatal(err) } + _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}, Tree: tree}) + action, err := verifier.NextAction() if err != nil { t.Fatal(err) @@ -403,7 +513,10 @@ func TestDoubleTapsBuiltin_TargetsClickable(t *testing.T) { ] }` verifier := newVerifier(t) - mustLoad(t, verifier, `globalThis.actions = __sanderling__.doubleTaps;`) + loadActionSpec(t, verifier, ` + import { doubleTaps } from "@sanderling/spec"; + globalThis.actions = doubleTaps; + `) tree, err := hierarchy.Parse(treeJSON) if err != nil { t.Fatal(err) @@ -442,13 +555,25 @@ func TestDoubleTapsBuiltin_TargetsClickable(t *testing.T) { } func TestDoubleTap_RoundTrip(t *testing.T) { + const treeJSON = `{ + "attributes": {"resource-id": "root", "bounds": "[0,0,100,100]"}, + "children": [ + {"attributes": {"resource-id": "save", "bounds": "[0,0,100,40]"}, "clickable": true, "enabled": true, "children": []} + ] + }` verifier := newVerifier(t) - mustLoad(t, verifier, ` - globalThis.actions = __sanderling__.actions(() => [ - __sanderling__.doubleTap({ on: "id:save" }), - ]); + loadActionSpec(t, verifier, ` + import { actions, DoubleTap } from "@sanderling/spec"; + globalThis.actions = actions(() => { + const save = state.ax.find("id:save"); + return save ? [DoubleTap({ on: save })] : []; + }); `) - _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}}) + tree, err := hierarchy.Parse(treeJSON) + if err != nil { + t.Fatal(err) + } + _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}, Tree: tree}) action, err := verifier.NextAction() if err != nil { @@ -463,13 +588,25 @@ func TestDoubleTap_RoundTrip(t *testing.T) { } func TestInputText_RoundTrip(t *testing.T) { + const treeJSON = `{ + "attributes": {"resource-id": "root", "bounds": "[0,0,100,100]"}, + "children": [ + {"attributes": {"resource-id": "phone", "bounds": "[0,0,100,40]"}, "editable": true, "enabled": true, "children": []} + ] + }` verifier := newVerifier(t) - mustLoad(t, verifier, ` - globalThis.actions = __sanderling__.actions(() => [ - __sanderling__.inputText({ into: "id:phone", text: "+919876543210" }), - ]); + loadActionSpec(t, verifier, ` + import { actions, InputText } from "@sanderling/spec"; + globalThis.actions = actions(() => { + const phone = state.ax.find("id:phone"); + return phone ? [InputText({ into: phone, text: "+919876543210" })] : []; + }); `) - _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}}) + tree, err := hierarchy.Parse(treeJSON) + if err != nil { + t.Fatal(err) + } + _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}, Tree: tree}) action, err := verifier.NextAction() if err != nil { @@ -495,7 +632,10 @@ func TestTypingBuiltin_TargetsEditableField(t *testing.T) { ] }` verifier := newVerifier(t) - mustLoad(t, verifier, `globalThis.actions = __sanderling__.typing;`) + loadActionSpec(t, verifier, ` + import { typing } from "@sanderling/spec"; + globalThis.actions = typing; + `) tree, err := hierarchy.Parse(treeJSON) if err != nil { t.Fatal(err) @@ -513,11 +653,21 @@ func TestTypingBuiltin_TargetsEditableField(t *testing.T) { if action.X != 50 || action.Y != 20 { t.Errorf("coords = (%d,%d), want (50,20) at EmailField center", action.X, action.Y) } - if !slices.Contains(inputCorpus, action.Text) { - t.Errorf("text %q not drawn from inputCorpus", action.Text) + // The typing builtin draws from corpus.ts INPUT_CORPUS; the single editable + // field means any draw lands here, so the text must be a corpus member. + if !slices.Contains(testInputCorpus, action.Text) { + t.Errorf("text %q not drawn from the input corpus", action.Text) } } +// testInputCorpus mirrors pkg/spec/src/corpus.ts INPUT_CORPUS so the typing +// builtin's emitted text can be asserted to come from the shared pool. +var testInputCorpus = []string{ + "", "a", strings.Repeat("a", 4096), "🙂🔥💸", " ", "\t\n", "-1", + "999999999999999999999", "0.0000001", "1e10", "'; DROP TABLE--", + "", "../../etc/passwd", "%s%n", "NaN", +} + // TestTypingBuiltin_NoEditableYieldsErrNoAction verifies the typing generator // declines (ErrNoAction) when no editable element is present, so a weighted // layer falls through to another generator. @@ -529,7 +679,10 @@ func TestTypingBuiltin_NoEditableYieldsErrNoAction(t *testing.T) { ] }` verifier := newVerifier(t) - mustLoad(t, verifier, `globalThis.actions = __sanderling__.typing;`) + loadActionSpec(t, verifier, ` + import { typing } from "@sanderling/spec"; + globalThis.actions = typing; + `) tree, err := hierarchy.Parse(treeJSON) if err != nil { t.Fatal(err) @@ -582,12 +735,50 @@ globalThis.properties = { t.Errorf("verdict: got %v, want %v", got, ltl.VerdictViolated) } - predicateErr := verifier.PredicateError("broken") - if predicateErr == nil { - t.Fatal("PredicateError: got nil, want non-nil") + witness := verifier.Witness("broken") + if witness == nil { + t.Fatal("Witness: got nil, want non-nil") } - if !strings.Contains(predicateErr.Error(), "bad predicate") { - t.Errorf("PredicateError message: got %q, want to contain %q", predicateErr.Error(), "bad predicate") + if !witness.IsError { + t.Errorf("Witness.IsError = false, want true for a thrown predicate") + } + if !strings.Contains(witness.Reason, "bad predicate") { + t.Errorf("Witness.Reason: got %q, want to contain %q", witness.Reason, "bad predicate") + } +} + +// An unbounded eventually that never fires stays pending during the run and is +// only reported by Finalize at run end. The witness records why it failed. +func TestFinalize_UnmetEventuallyReportedWithWitness(t *testing.T) { + const spec = ` +globalThis.flag = __sanderling__.extract(state => state.snapshots["flag"] ?? false, "flag"); +globalThis.properties = { + flagEventuallyTrue: __sanderling__.always(__sanderling__.eventually(() => flag.current === true)), +}; +` + verifier := newVerifier(t) + mustLoad(t, verifier, spec) + + for step := 0; step < 3; step++ { + if err := verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{"flag": json.RawMessage(`false`)}}); err != nil { + t.Fatal(err) + } + verdicts := verifier.EvaluateProperties() + if got := verdicts["flagEventuallyTrue"]; got != ltl.VerdictPending { + t.Fatalf("step %d: verdict = %v, want pending", step, got) + } + } + + ended := verifier.Finalize() + if !slices.Contains(ended, "flagEventuallyTrue") { + t.Fatalf("Finalize ended = %v, want to contain flagEventuallyTrue", ended) + } + witness := verifier.Witness("flagEventuallyTrue") + if witness == nil { + t.Fatal("Witness = nil after Finalize, want non-nil") + } + if !strings.Contains(witness.Reason, "eventually") { + t.Errorf("Witness.Reason = %q, want to mention eventually", witness.Reason) } } @@ -702,15 +893,30 @@ func TestSelector_BooleanValue(t *testing.T) { // dynamic array picks the same element under the same seed across runs. The // folio spec relies on this to replace Math.random() in account-card taps. func TestFrom_SeededReplayIsDeterministic(t *testing.T) { + const treeJSON = `{ + "attributes": {"resource-id": "root", "bounds": "[0,0,100,200]"}, + "children": [ + {"attributes": {"resource-id": "card_a", "bounds": "[0,0,100,40]"}, "clickable": true, "enabled": true, "children": []}, + {"attributes": {"resource-id": "card_b", "bounds": "[0,40,100,80]"}, "clickable": true, "enabled": true, "children": []}, + {"attributes": {"resource-id": "card_c", "bounds": "[0,80,100,120]"}, "clickable": true, "enabled": true, "children": []}, + {"attributes": {"resource-id": "card_d", "bounds": "[0,120,100,160]"}, "clickable": true, "enabled": true, "children": []} + ] + }` + tree, err := hierarchy.Parse(treeJSON) + if err != nil { + t.Fatal(err) + } pickedSequence := func(seed uint64) []string { - verifier := newVerifier(t, WithRand(rand.New(rand.NewPCG(seed, 0)))) - mustLoad(t, verifier, ` - globalThis.actions = __sanderling__.actions(() => { - const cards = ["card_a", "card_b", "card_c", "card_d"]; - return [__sanderling__.tap({ on: __sanderling__.from(cards).generate() })]; + verifier := newVerifier(t, WithSeed(seed)) + loadActionSpec(t, verifier, ` + import { actions, from, Tap } from "@sanderling/spec"; + const cards = ["id:card_a", "id:card_b", "id:card_c", "id:card_d"]; + globalThis.actions = actions(() => { + const target = state.ax.find(from(cards).generate()); + return target ? [Tap({ on: target })] : []; }); `) - _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}}) + _ = verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{}, Tree: tree}) var picks []string for range 20 { action, err := verifier.NextAction() @@ -741,13 +947,14 @@ func TestFrom_SeededReplayIsDeterministic(t *testing.T) { } } -// PredicateError must reflect the most recent step's predicate result, not a -// latched first-step error. The runner logs PredicateError once per step; if it -// stays pinned to step 1 forever, downstream debugging looks frozen even though -// the underlying state is changing. -func TestPredicateError_ReflectsCurrentStepNotFirstStep(t *testing.T) { +// A thrown predicate is a witnessed violation at the step it first throws, and +// the property latches violated thereafter (sticky always semantics). The +// witness captures the onset step's error text (count=1) and the extractor +// snapshot at that step; it does not keep updating to later counts because the +// verdict has latched. +func TestThrowingPredicate_WitnessCapturesOnsetStep(t *testing.T) { const spec = ` -globalThis.counter = __sanderling__.extract(state => state.snapshots["count"]); +globalThis.counter = __sanderling__.extract(state => state.snapshots["count"], "counter"); globalThis.properties = { reportsCounter: __sanderling__.always(() => { throw new Error("count=" + counter.current); }), }; @@ -760,16 +967,24 @@ globalThis.properties = { if err := verifier.PushSnapshot(SnapshotInput{Snapshots: Snapshots{"count": raw}}); err != nil { t.Fatal(err) } - _ = verifier.EvaluateProperties() + verdicts := verifier.EvaluateProperties() + if got := verdicts["reportsCounter"]; got != ltl.VerdictViolated { + t.Fatalf("step %d: verdict = %v, want violated", step, got) + } + } - got := verifier.PredicateError("reportsCounter") - if got == nil { - t.Fatalf("step %d: PredicateError = nil, want non-nil", step) - } - want := "count=" + string(rune('0'+step)) - if !strings.Contains(got.Error(), want) { - t.Errorf("step %d: PredicateError = %q, want to contain %q", step, got.Error(), want) - } + witness := verifier.Witness("reportsCounter") + if witness == nil { + t.Fatal("Witness = nil, want non-nil") + } + if witness.Step != 1 { + t.Errorf("Witness.Step = %d, want 1 (onset)", witness.Step) + } + if !strings.Contains(witness.Reason, "count=1") { + t.Errorf("Witness.Reason = %q, want to contain %q", witness.Reason, "count=1") + } + if got := string(witness.Extractors["counter"]); got != `"1"` { + t.Errorf("Witness.Extractors[counter] = %s, want %q", got, `"1"`) } } @@ -845,8 +1060,6 @@ globalThis.card = card; globalThis.properties = { hasTestTag: __sanderling__.always(() => typeof card.current.attrs.testTag === "string"), }; - -globalThis.actions = __sanderling__.actions(() => []); ` // TestSelectorStringFromJS_CanonicalGrammar guarantees the selector tag stamped @@ -1019,3 +1232,27 @@ func TestOverrideExtractorValues_PropagatesNestedObjectFields(t *testing.T) { t.Errorf("scalar field missing: card.current.balance = %d, want 12345", got) } } + +// TestUnsupportedVerbs_CollectedDedupedInOrder drives the real host binding the +// shared picker invokes (__sanderlingHost__.reportUnsupported) and asserts the +// verifier collects each verb once, in first-seen order, for the run report. +func TestUnsupportedVerbs_CollectedDedupedInOrder(t *testing.T) { + verifier := newVerifier(t) + report, ok := goja.AssertFunction( + verifier.runtime.GlobalObject().Get("__sanderlingHost__"). + ToObject(verifier.runtime).Get("reportUnsupported"), + ) + if !ok { + t.Fatal("reportUnsupported host binding missing") + } + for _, verb := range []string{"scrolls", "swipes", "scrolls", "longPresses"} { + if _, err := report(goja.Undefined(), verifier.runtime.ToValue(verb)); err != nil { + t.Fatalf("reportUnsupported(%q): %v", verb, err) + } + } + got := verifier.UnsupportedVerbs() + want := []string{"scrolls", "swipes", "longPresses"} + if !slices.Equal(got, want) { + t.Errorf("UnsupportedVerbs = %v, want %v", got, want) + } +} diff --git a/internal/verifier/worker.go b/internal/verifier/worker.go index 0d68f55..20c7465 100644 --- a/internal/verifier/worker.go +++ b/internal/verifier/worker.go @@ -1,3 +1,4 @@ +// Package verifier runs the spec's extractors and property formulas against each observed step. package verifier import ( @@ -6,9 +7,7 @@ import ( "errors" "fmt" "maps" - "math/rand/v2" "sort" - "strings" "time" "github.com/dop251/goja" @@ -23,14 +22,17 @@ type Verifier struct { formulas []*formulaState formulaSpecs []formulaSpec - properties map[string]int // property name -> formula-spec index - actionGenerator goja.Value - setupGenerator goja.Value + properties map[string]int // property name -> formula-spec index + + // nextActionFn is the bundle-installed __sanderlingNextAction__, which runs + // the shared picker (pick.ts) over the shared Pcg. + nextActionFn goja.Callable evaluators map[string]*ltl.Evaluator priorVerdicts map[string]ltl.Verdict newlyViolated []string + witnesses map[string]Witness lastTree *hierarchy.Tree lastAction *Action @@ -40,14 +42,41 @@ type Verifier struct { runStart time.Time appPackage string + platform string + seed uint64 - rng *rand.Rand + // unsupported collects verbs the picker requested but the platform cannot + // dispatch (reportUnsupported host callback), deduped and in first-seen + // order, so the runner can surface them in the run report. + unsupported []string + unsupportedSeen map[string]bool + + // extracting is true only while an extractor getter is running. The handle's + // current/previous accessors consult it so a getter that reaches into + // another extractor's handle throws instead of reading a stale value. + extracting bool +} + +// UnsupportedVerbs returns the verbs the picker requested that this platform +// cannot dispatch, deduped and in first-seen order. +func (v *Verifier) UnsupportedVerbs() []string { + return v.unsupported } type Option func(*Verifier) -func WithRand(rng *rand.Rand) Option { - return func(v *Verifier) { v.rng = rng } +// WithSeed sets the 64-bit seed the JS picker constructs its Pcg from +// (new Pcg(seed, 0), matching the web bundle's SANDERLING_SEED). The verifier +// exposes it to the bundle via the __sanderlingHost__.seedHi/seedLo binds. +func WithSeed(seed uint64) Option { + return func(v *Verifier) { v.seed = seed } +} + +// WithPlatform names the platform the host reports to the picker +// ("android"/"ios"/"web"); it drives the verb-support matrix and the press-key +// pool. Empty defaults to "android". +func WithPlatform(platform string) Option { + return func(v *Verifier) { v.platform = platform } } // WithAppPackage scopes random-action target selection to the app under test. @@ -60,15 +89,20 @@ func WithAppPackage(appPackage string) Option { func New(options ...Option) (*Verifier, error) { verifier := &Verifier{ - runtime: goja.New(), - properties: map[string]int{}, - evaluators: map[string]*ltl.Evaluator{}, - priorVerdicts: map[string]ltl.Verdict{}, - rng: rand.New(rand.NewPCG(0, 0)), + runtime: goja.New(), + properties: map[string]int{}, + evaluators: map[string]*ltl.Evaluator{}, + priorVerdicts: map[string]ltl.Verdict{}, + witnesses: map[string]Witness{}, + platform: "android", + unsupportedSeen: map[string]bool{}, } for _, option := range options { option(verifier) } + if verifier.platform == "" { + verifier.platform = "android" + } if err := verifier.installRuntimeBindings(); err != nil { return nil, fmt.Errorf("install bindings: %w", err) } @@ -105,12 +139,14 @@ func (v *Verifier) Load(source string) error { } } - if actionsValue := v.runtime.GlobalObject().Get("actions"); actionsValue != nil && !goja.IsUndefined(actionsValue) && !goja.IsNull(actionsValue) { - v.actionGenerator = actionsValue - } - - if setupValue := v.runtime.GlobalObject().Get("setup"); setupValue != nil && !goja.IsUndefined(setupValue) && !goja.IsNull(setupValue) { - v.setupGenerator = setupValue + // The bundle's goja runtime entry installs __sanderlingNextAction__ once the + // spec assigned globalThis.actions. Capture it; a spec bundled without the + // runtime entry (raw-JS unit fixtures) leaves it nil and NextAction reports + // ErrNoAction. + if fn := v.runtime.GlobalObject().Get("__sanderlingNextAction__"); fn != nil { + if callable, ok := goja.AssertFunction(fn); ok { + v.nextActionFn = callable + } } return nil @@ -139,7 +175,8 @@ func (v *Verifier) buildFormulaNode(index int) (ltl.Formula, error) { case specKindPure: return ltl.Pure(spec.pureValue), nil case specKindThunk: - return ltl.Thunk(v.formulaThunk(spec.predicateIndex)), nil + name := fmt.Sprintf("p%d", spec.predicateIndex) + return ltl.ThunkNamed(name, v.formulaThunk(spec.predicateIndex)), nil case specKindNow: child, err := v.buildFormulaNode(spec.childA) if err != nil { @@ -244,21 +281,28 @@ func (v *Verifier) PushSnapshot(input SnapshotInput) error { // Extractor previous/current advance exactly once per PushSnapshot. // Predicate thunks read these slots but never trigger advancement, so // invoking a thunk multiple times between snapshots is value-stable. - // refreshPredicateErrors relies on this to safely re-call predicates. for index, extractor := range v.extractors { - previous := extractor.handle.Get("current") - _ = extractor.handle.Set("previous", previous) - newValue, err := extractor.getter(goja.Undefined(), state) + extractor.previousValue = extractor.currentValue + newValue, err := v.runExtractor(extractor, state) if err != nil { return fmt.Errorf("extractor %d: %w", index, err) } - _ = extractor.handle.Set("current", newValue) + extractor.currentValue = newValue extractor.prev = extractor.curr extractor.curr = encodeExtractorValue(newValue) } return nil } +// runExtractor invokes an extractor's getter with the extracting flag set, so a +// getter that reads another extractor's current/previous throws. The flag is +// cleared even if the getter panics. +func (v *Verifier) runExtractor(extractor *extractorState, state goja.Value) (goja.Value, error) { + v.extracting = true + defer func() { v.extracting = false }() + return extractor.getter(goja.Undefined(), state) +} + // encodeExtractorValue produces a stable JSON encoding of an extractor's // current value for diff comparison. goja values that don't survive Export // (e.g. wrapped host functions) yield nil; callers treat nil as "unknown" and @@ -328,7 +372,7 @@ func (v *Verifier) OverrideExtractorValues(overrides map[int]json.RawMessage) (s if conversionErr != nil { return skipped, fmt.Errorf("extractor override %d: %w", index, conversionErr) } - _ = v.extractors[index].handle.Set("current", value) + v.extractors[index].currentValue = value } return skipped, nil } @@ -362,12 +406,12 @@ func (v *Verifier) EvaluateProperties() map[string]ltl.Verdict { for name, evaluator := range v.evaluators { verdicts[name] = evaluator.ObserveAt(stepTime) } - v.refreshPredicateErrors() var onset []string for name, verdict := range verdicts { if verdict == ltl.VerdictViolated && v.priorVerdicts[name] != ltl.VerdictViolated { onset = append(onset, name) + v.captureWitness(name) } } sort.Strings(onset) @@ -380,6 +424,89 @@ func (v *Verifier) EvaluateProperties() map[string]ltl.Verdict { return verdicts } +// Witness is the verifier-level record of a property violation: the LTL reason +// (a predicate's thrown-error text, "predicate false", or a liveness failure), +// the step it fired at, and a snapshot of every extractor's current value at +// that step. The snapshot lets a reader see the state that produced the +// violation without replaying the run. +type Witness struct { + Property string + Reason string + Step int + IsError bool + Extractors map[string]json.RawMessage +} + +// captureWitness records the witness for a property that just transitioned to +// violated, snapshotting the current extractor values so the cause is visible +// after the run. +func (v *Verifier) captureWitness(name string) { + evaluator, ok := v.evaluators[name] + if !ok { + return + } + violation := evaluator.Violation() + if violation == nil { + return + } + v.witnesses[name] = Witness{ + Property: name, + Reason: violation.Reason, + Step: violation.Step, + IsError: violation.IsError, + Extractors: v.extractorSnapshot(), + } +} + +// extractorSnapshot encodes every named extractor's current value as JSON. A +// nil value (extractor never advanced or its value did not survive Export) +// is recorded as JSON null. +func (v *Verifier) extractorSnapshot() map[string]json.RawMessage { + if len(v.extractors) == 0 { + return nil + } + snapshot := make(map[string]json.RawMessage, len(v.extractors)) + for _, extractor := range v.extractors { + value := extractor.curr + if value == nil { + value = []byte("null") + } + snapshot[extractor.name] = append(json.RawMessage(nil), value...) + } + return snapshot +} + +// Witness returns the captured violation witness for a property, or nil if the +// property has not violated. Callers consult this after EvaluateProperties (or +// Finalize) reports a violation to surface the cause and the state at onset. +func (v *Verifier) Witness(name string) *Witness { + witness, ok := v.witnesses[name] + if !ok { + return nil + } + return &witness +} + +// Finalize drives each evaluator to its terminal verdict and returns the names +// of properties that violate only at run end (a liveness obligation that never +// discharged), capturing a witness for each. Properties already violated +// mid-run are not re-reported here. +func (v *Verifier) Finalize() []string { + var ended []string + for name, evaluator := range v.evaluators { + if v.priorVerdicts[name] == ltl.VerdictViolated { + continue + } + if evaluator.Finalize() == ltl.VerdictViolated { + ended = append(ended, name) + v.captureWitness(name) + v.priorVerdicts[name] = ltl.VerdictViolated + } + } + sort.Strings(ended) + return ended +} + // NewlyViolatedProperties returns the names of properties whose verdict // transitioned from non-Violated to Violated on the most recent // EvaluateProperties call, sorted lexicographically. Returns nil if no @@ -396,14 +523,14 @@ func (v *Verifier) NewlyViolatedProperties() []string { } // Residuals returns the residual formula for each registered property after -// the most recent EvaluateProperties call. Properties that errored during -// predicate evaluation surface as ErrorFormula so the inspect UI can render -// "predicate threw" inline. +// the most recent EvaluateProperties call. Properties whose violation was +// caused by a thrown predicate surface as ErrorFormula, sourced from the +// captured witness, so the inspect UI can render "predicate threw" inline. func (v *Verifier) Residuals() map[string]ltl.Formula { residuals := map[string]ltl.Formula{} for name, evaluator := range v.evaluators { - if predicateErr := v.PredicateError(name); predicateErr != nil { - residuals[name] = ltl.ErrorFormula{Message: predicateErr.Error()} + if witness, ok := v.witnesses[name]; ok && witness.IsError { + residuals[name] = ltl.ErrorFormula{Message: witness.Reason} continue } residuals[name] = evaluator.Residual() @@ -411,174 +538,42 @@ func (v *Verifier) Residuals() map[string]ltl.Formula { return residuals } -// NextAction resolves an action for the current step. The setup generator, -// when registered, runs first; if it yields an action, that wins. When setup -// returns ErrNoAction (all branches empty) the call falls through to the -// root action generator with the existing retry semantics. Setup is consulted -// every step, so state regression (e.g. a logout under fuzz) automatically -// re-engages the precondition. +// NextAction resolves an action for the current step by invoking the bundled +// __sanderlingNextAction__(), which runs the SHARED picker (pick.ts) over the +// shared Pcg. Setup-generator precedence and the 16-attempt retry both live in +// runtime-entry.ts now, so this is a thin call-and-decode. A null result (the +// generator declined to act) reports ErrNoAction. func (v *Verifier) NextAction() (Action, error) { - if v.setupGenerator != nil { - action, err := v.resolveGenerator(v.setupGenerator) - if err == nil { - return action, nil - } - if !errors.Is(err, ErrNoAction) { - return Action{}, err - } - } - if v.actionGenerator == nil { + if v.nextActionFn == nil { return Action{}, ErrNoAction } - const maxRetries = 16 - for range maxRetries { - action, err := v.resolveGenerator(v.actionGenerator) - if err == nil { - return action, nil - } - if !errors.Is(err, ErrNoAction) { - return Action{}, err - } + value, err := v.nextActionFn(goja.Undefined()) + if err != nil { + return Action{}, fmt.Errorf("next action: %w", err) } - return Action{}, ErrNoAction + if value == nil || goja.IsNull(value) || goja.IsUndefined(value) { + return Action{}, ErrNoAction + } + raw, err := json.Marshal(value.Export()) + if err != nil { + return Action{}, fmt.Errorf("marshal action: %w", err) + } + return DecodeAction(raw) } var ErrNoAction = errors.New("verifier: no action available") -func (v *Verifier) formulaThunk(index int) func() bool { - return func() bool { +func (v *Verifier) formulaThunk(index int) func() (bool, error) { + return func() (bool, error) { formula := v.formulas[index] result, err := formula.predicate(goja.Undefined()) if err != nil { - formula.err = err - return false + return false, err } - formula.err = nil - return result.ToBoolean() + return result.ToBoolean(), nil } } -// refreshPredicateErrors re-invokes every registered predicate so that -// formula.err reflects the current step rather than a latched first-step -// throw. EvaluateProperties short-circuits once a property has latched to -// violated, so without this refresh the runner's per-step "predicate error" -// log freezes on whatever the predicate threw at step 1. The refreshed errors -// have no effect on verdicts. -// -// Invariant: predicates may be re-invoked here outside the LTL gate that -// would normally skip them (e.g. an `implies` consequent whose antecedent is -// false). They must therefore be side-effect-free reads of extractor state; -// any spec that asserts internal preconditions inside a predicate could -// surface a spurious error in the inspect UI without affecting verdicts. -func (v *Verifier) refreshPredicateErrors() { - for _, formula := range v.formulas { - result, err := formula.predicate(goja.Undefined()) - if err != nil { - formula.err = err - continue - } - _ = result - formula.err = nil - } -} - -// PredicateError returns the first goja error raised by any thunk in the -// named property's formula tree, or nil if none fired. Callers typically -// consult this after EvaluateProperties reports a violation to distinguish -// a genuine predicate-false verdict from a malformed spec. -func (v *Verifier) PredicateError(name string) error { - rootIndex, ok := v.properties[name] - if !ok { - return nil - } - return v.firstThunkError(rootIndex) -} - -func (v *Verifier) firstThunkError(index int) error { - if index < 0 || index >= len(v.formulaSpecs) { - return nil - } - spec := v.formulaSpecs[index] - switch spec.kind { - case specKindThunk: - return v.formulas[spec.predicateIndex].err - case specKindImplies, specKindOr, specKindAnd: - if err := v.firstThunkError(spec.childA); err != nil { - return err - } - return v.firstThunkError(spec.childB) - case specKindNow, specKindNext, specKindEventually, specKindNot, specKindAlways: - return v.firstThunkError(spec.childA) - } - return nil -} - -func (v *Verifier) resolveGenerator(generator goja.Value) (Action, error) { - object := generator.ToObject(v.runtime) - if object == nil { - return Action{}, fmt.Errorf("generator is not an object") - } - kindValue := object.Get(tagInternalKind) - if kindValue == nil { - return Action{}, fmt.Errorf("generator missing internal kind tag") - } - switch kindValue.String() { - case internalKindActions: - generateValue := object.Get("generate") - generate, ok := goja.AssertFunction(generateValue) - if !ok { - return Action{}, fmt.Errorf("actions handle missing generate function") - } - result, err := generate(goja.Undefined()) - if err != nil { - return Action{}, fmt.Errorf("generate: %w", err) - } - return v.pickFromResult(result) - case internalKindWeighted: - entries := object.Get("entries").ToObject(v.runtime) - if entries == nil { - return Action{}, fmt.Errorf("weighted handle missing entries") - } - picked, err := v.pickWeighted(entries) - if err != nil { - return Action{}, err - } - return v.resolveGenerator(picked) - case internalKindBuiltinTaps: - return v.generateRandomTap() - case internalKindBuiltinDoubleTaps: - return v.generateRandomDoubleTap() - case internalKindBuiltinTyping: - return v.generateRandomInput() - case internalKindBuiltinSwipes: - return v.generateRandomSwipe() - case internalKindBuiltinWaitOnce: - return Action{Kind: ActionKindWait, DurationMillis: 500}, nil - case internalKindBuiltinPressKey: - return v.generateRandomPressKey() - case internalKindBuiltinLongPresses: - return v.generateRandomLongPress() - case internalKindBuiltinScrolls: - return v.generateRandomScroll() - default: - return Action{}, fmt.Errorf("unknown generator kind %q", kindValue.String()) - } -} - -// generateRandomTap picks a visible, tappable element from the last -// hierarchy snapshot and returns a Tap action targeting its center. -func (v *Verifier) generateRandomTap() (Action, error) { - return v.generateRandomTapKind(ActionKindTap) -} - -// generateRandomDoubleTap is the DoubleTap counterpart of generateRandomTap. -// Real user gestures include double-tap (image zoom, like-to-favorite, -// play/pause); a fuzzer that never emits one cannot exercise either those -// features or the sub-100ms race windows that single-step Tap cadence misses. -func (v *Verifier) generateRandomDoubleTap() (Action, error) { - return v.generateRandomTapKind(ActionKindDoubleTap) -} - // inScope reports whether an element belongs to the app under test. Nodes from // another package (the soft keyboard, system UI, permission dialogs) are out of // scope. An unset app package or an element with no package falls through to in @@ -590,31 +585,6 @@ func (v *Verifier) inScope(element *hierarchy.Element) bool { return element.Package == v.appPackage } -func (v *Verifier) generateRandomTapKind(kind ActionKind) (Action, error) { - if v.lastTree == nil { - return Action{}, ErrNoAction - } - candidates := make([]*hierarchy.Element, 0, len(v.lastTree.Elements)) - for _, element := range v.lastTree.Elements { - if !element.Clickable || !element.Enabled { - continue - } - if !v.inScope(element) { - continue - } - if element.Bounds.Right-element.Bounds.Left <= 0 || element.Bounds.Bottom-element.Bounds.Top <= 0 { - continue - } - candidates = append(candidates, element) - } - if len(candidates) == 0 { - return Action{}, ErrNoAction - } - picked := candidates[v.rng.IntN(len(candidates))] - x, y := picked.Bounds.Center() - return Action{Kind: kind, On: selectorForElement(v.lastTree, picked), X: x, Y: y}, nil -} - // selectorForElement builds a canonical "key:value" selector that resolves // back to the given element via hierarchy.Tree.Find. Prefers resource-id (the // testTag carrier on Android / accessibilityIdentifier on iOS), falling back @@ -656,237 +626,60 @@ func selectorForElement(tree *hierarchy.Tree, element *hierarchy.Element) string return "" } -// inputCorpus is the edge-case string pool the typing builtin draws from to -// stress field parsing: empty, whitespace, overflow length, unicode, numeric -// boundaries, and common injection payloads. -var inputCorpus = []string{ - "", - "a", - strings.Repeat("a", 4096), - "🙂🔥💸", - " ", - "\t\n", - "-1", - "999999999999999999999", - "0.0000001", - "1e10", - "'; DROP TABLE--", - "", - "../../etc/passwd", - "%s%n", - "NaN", -} - -// generateRandomInput picks a visible, editable, enabled element from the last -// hierarchy snapshot and types a random edge-case value into it. The runner -// taps the target coordinates to focus before typing, so this works on both -// native and web with no driver-side dispatch change. -func (v *Verifier) generateRandomInput() (Action, error) { +// candidatesForVerb enumerates the host-side targets a builtin verb may draw +// from, in v.lastTree.Elements ORDER (the order is part of the picker's parity +// contract). The filters are LIFTED from the old Go picker: +// +// taps/doubleTaps/longPresses: clickable + enabled + positive bounds +// typing: editable + enabled + positive bounds +// scrolls: scrollable attribute + positive bounds +// swipes: any in-scope element +// +// Every candidate carries the resolving selector so the runner can re-route by +// id/text. Out-of-scope nodes (the soft keyboard, system UI) are always dropped. +func (v *Verifier) candidatesForVerb(verb string) []candidate { if v.lastTree == nil { - return Action{}, ErrNoAction + return nil } - candidates := make([]*hierarchy.Element, 0, len(v.lastTree.Elements)) + var result []candidate for _, element := range v.lastTree.Elements { - if !element.Editable || !element.Enabled { - continue - } if !v.inScope(element) { continue } - if element.Bounds.Right-element.Bounds.Left <= 0 || element.Bounds.Bottom-element.Bounds.Top <= 0 { + if !verbAccepts(verb, element) { continue } - candidates = append(candidates, element) + x, y := element.Bounds.Center() + result = append(result, candidate{ + x: x, + y: y, + width: element.Bounds.Width(), + height: element.Bounds.Height(), + selector: selectorForElement(v.lastTree, element), + }) } - if len(candidates) == 0 { - return Action{}, ErrNoAction - } - picked := candidates[v.rng.IntN(len(candidates))] - x, y := picked.Bounds.Center() - value := inputCorpus[v.rng.IntN(len(inputCorpus))] - return Action{Kind: ActionKindInputText, X: x, Y: y, Text: value}, nil + return result } -// generateRandomSwipe emits a swipe over a random enabled element or the -// whole screen, in a random direction. Returns ErrNoAction only when we have -// no tree to size a gesture off of. -func (v *Verifier) generateRandomSwipe() (Action, error) { - if v.lastTree == nil || len(v.lastTree.Elements) == 0 { - return Action{}, ErrNoAction - } - candidates := make([]*hierarchy.Element, 0, len(v.lastTree.Elements)) - for _, element := range v.lastTree.Elements { - if v.inScope(element) { - candidates = append(candidates, element) - } - } - if len(candidates) == 0 { - return Action{}, ErrNoAction - } - element := candidates[v.rng.IntN(len(candidates))] - cx, cy := element.Bounds.Center() - if cx <= 0 || cy <= 0 { - return Action{}, ErrNoAction - } - // Pick a direction: 0=up 1=down 2=left 3=right; magnitude 200-600 px. - magnitude := 200 + v.rng.IntN(401) - toX, toY := cx, cy - switch v.rng.IntN(4) { - case 0: - toY = cy - magnitude - case 1: - toY = cy + magnitude - case 2: - toX = cx - magnitude - case 3: - toX = cx + magnitude - } - if toX < 0 { - toX = 0 - } - if toY < 0 { - toY = 0 - } - return Action{ - Kind: ActionKindSwipe, - FromX: cx, - FromY: cy, - ToX: toX, - ToY: toY, - DurationMillis: 250, - }, nil +type candidate struct { + x, y int + width, height int + selector string } -// generateRandomLongPress mirrors generateRandomTap: it picks a visible, -// clickable, enabled, in-scope element and targets its center. Real users -// long-press (context menus, reorder handles, multi-select), so a fuzzer that -// never emits one cannot reach those affordances. -func (v *Verifier) generateRandomLongPress() (Action, error) { - return v.generateRandomTapKind(ActionKindLongPress) -} - -// generateRandomScroll picks a scrollable, in-scope container and emits a swipe -// across it in a random direction. -func (v *Verifier) generateRandomScroll() (Action, error) { - if v.lastTree == nil { - return Action{}, ErrNoAction +// verbAccepts applies the per-verb element filter. +func verbAccepts(verb string, element *hierarchy.Element) bool { + positiveBounds := element.Bounds.Width() > 0 && element.Bounds.Height() > 0 + switch verb { + case "taps", "doubleTaps", "longPresses": + return element.Clickable && element.Enabled && positiveBounds + case "typing": + return element.Editable && element.Enabled && positiveBounds + case "scrolls": + return element.Attributes["scrollable"] == "true" && positiveBounds + case "swipes": + return true + default: + return false } - candidates := make([]*hierarchy.Element, 0, len(v.lastTree.Elements)) - for _, element := range v.lastTree.Elements { - if element.Attributes["scrollable"] != "true" { - continue - } - if !v.inScope(element) { - continue - } - if element.Bounds.Width() <= 0 || element.Bounds.Height() <= 0 { - continue - } - candidates = append(candidates, element) - } - if len(candidates) == 0 { - return Action{}, ErrNoAction - } - picked := candidates[v.rng.IntN(len(candidates))] - cx, cy := picked.Bounds.Center() - width := picked.Bounds.Width() - height := picked.Bounds.Height() - directions := []string{"up", "down", "left", "right"} - dir := directions[v.rng.IntN(len(directions))] - toX, toY := cx, cy - // Scroll direction names the content motion; the gesture swipes the - // opposite way. Revealing content below ("down") means dragging the - // finger up, so toY decreases, and likewise for the other directions. - switch dir { - case "down": - toY = cy - (4*height)/10 - case "up": - toY = cy + (4*height)/10 - case "left": - toX = cx + (4*width)/10 - case "right": - toX = cx - (4*width)/10 - } - if toX < 0 { - toX = 0 - } - if toY < 0 { - toY = 0 - } - return Action{ - Kind: ActionKindScroll, - Direction: dir, - FromX: cx, - FromY: cy, - ToX: toX, - ToY: toY, - DurationMillis: 300, - }, nil -} - -func (v *Verifier) generateRandomPressKey() (Action, error) { - // Keep exploration gentle: only "back" for now. Home/menu would navigate - // away from the app under test. - return Action{Kind: ActionKindPressKey, Key: "back"}, nil -} - -func (v *Verifier) pickFromResult(result goja.Value) (Action, error) { - if result == nil || goja.IsUndefined(result) || goja.IsNull(result) { - return Action{}, ErrNoAction - } - object := result.ToObject(v.runtime) - if object == nil { - return Action{}, ErrNoAction - } - lengthValue := object.Get("length") - if lengthValue == nil { - return jsValueToAction(v.runtime, result) - } - length := int(lengthValue.ToInteger()) - if length == 0 { - return Action{}, ErrNoAction - } - pick := v.rng.IntN(length) - return jsValueToAction(v.runtime, object.Get(fmt.Sprintf("%d", pick))) -} - -func (v *Verifier) pickWeighted(entries *goja.Object) (goja.Value, error) { - lengthValue := entries.Get("length") - if lengthValue == nil { - return nil, fmt.Errorf("weighted entries missing length") - } - length := int(lengthValue.ToInteger()) - if length == 0 { - return nil, ErrNoAction - } - - weights := make([]float64, length) - generators := make([]goja.Value, length) - totalWeight := 0.0 - for index := range length { - entry := entries.Get(fmt.Sprintf("%d", index)).ToObject(v.runtime) - if entry == nil { - return nil, fmt.Errorf("weighted entry %d not an array", index) - } - weight := entry.Get("0").ToFloat() - generator := entry.Get("1") - if weight < 0 { - weight = 0 - } - weights[index] = weight - generators[index] = generator - totalWeight += weight - } - if totalWeight == 0 { - return nil, ErrNoAction - } - pick := v.rng.Float64() * totalWeight - cumulative := 0.0 - for index := range length { - cumulative += weights[index] - if pick < cumulative { - return generators[index], nil - } - } - return generators[length-1], nil } diff --git a/pkg/spec/src/action-tree.ts b/pkg/spec/src/action-tree.ts new file mode 100644 index 0000000..910b149 --- /dev/null +++ b/pkg/spec/src/action-tree.ts @@ -0,0 +1,90 @@ +// Shared action-generator tree and host interface for W2 approach B. +// +// Both engines (the goja verifier and the V8 web runtime) walk ONE tree with +// ONE picker (pick.ts), drawing through the shared PCG (pcg.ts). This module +// only declares the data shapes; pick.ts holds the traversal and the parity +// contract for draw order. +// +// Element references never cross the V8/host boundary: builtin generators emit +// an ActionDescriptor whose target is already a resolved Point (plus an +// optional selector on native), so the host serializer never has to chase a +// live element handle. + +import type { + AccessibilityElement, + AttrSelector, + Direction, + Point, + SelectorPath, +} from "./types.ts"; + +// Target is any target shape an action may carry before serialization. Author +// specs supply a string/selector/element; builtin generators resolve to a +// {x, y} Point (so no element handle crosses the V8/host boundary). +export type Target = string | AttrSelector | SelectorPath | AccessibilityElement | Point; + +// BuiltinVerb names a leaf generator backed by host-enumerated candidates +// rather than author-supplied actions. +export type BuiltinVerb = + | "taps" + | "doubleTaps" + | "longPresses" + | "scrolls" + | "typing" + | "swipes" + | "waitOnce" + | "pressKeys"; + +// ActionDescriptor is the camelCase author shape an action takes before the +// engine serializes it to the wire format. Mirrors the factory return shapes +// in actions.ts (Tap/DoubleTap/LongPress {on}; Scroll {direction; in?}; +// InputText {into; text}; Swipe {from; to; durationMillis?}; PressKey {key}; +// Wait {durationMillis}). +export type ActionDescriptor = + | { kind: "Tap"; on: Target } + | { kind: "DoubleTap"; on: Target } + | { kind: "LongPress"; on: Target } + // Scroll carries the author target (in) for spec-written scrolls; the builtin + // generator additionally pre-computes the swipe from/to so the wire contract + // carries a real gesture rather than a zero-length one. + | { kind: "Scroll"; direction: Direction; in?: Target; from?: Target; to?: Target } + | { kind: "InputText"; into: Target; text: string } + | { kind: "Swipe"; from: Target; to: Target; durationMillis?: number } + | { kind: "PressKey"; key: string } + | { kind: "Wait"; durationMillis: number }; + +// GeneratorNode is a node in the action-generator tree. +// weighted: probabilistic choice over child nodes, scanned ascending. +// actions: author callback returning a list to pick uniformly from. +// builtin: host-backed leaf identified by a verb. +export type GeneratorNode = + | { kind: "weighted"; branches: ReadonlyArray } + | { kind: "actions"; generate: () => ActionDescriptor[] } + | { kind: "builtin"; verb: BuiltinVerb }; + +// Candidate is one host-enumerated target for a builtin verb. The host +// resolves geometry (and a native selector) so no element handle crosses into +// the picker. width/height let swipe/scroll size a gesture off the element. +export interface Candidate { + x: number; + y: number; + selector?: string; + width?: number; + height?: number; +} + +// Host is the platform backing the picker draws against. In this foundation +// workflow only the interface is defined and exercised against a stub; the +// goja and DOM implementations land in the rewire workflow. +export interface Host { + platform(): "android" | "ios" | "web"; + // queryCandidates returns the host-enumerated targets for a verb, in a + // deterministic order. The picker indexes into this list with the PCG, so + // the order is part of the parity contract. + queryCandidates(verb: BuiltinVerb): Candidate[]; + // reportUnsupported is invoked at most once per verb@platform (see verbs.ts) + // when a verb has no support on this platform. + reportUnsupported(verb: BuiltinVerb): void; + seedHi(): bigint; + seedLo(): bigint; +} diff --git a/pkg/spec/src/actions.ts b/pkg/spec/src/actions.ts index 7ced33e..5e11445 100644 --- a/pkg/spec/src/actions.ts +++ b/pkg/spec/src/actions.ts @@ -1,7 +1,12 @@ +// Author-facing action factories. Each returns plain GeneratorNode / +// ActionDescriptor DATA (see action-tree.ts); the shared picker (pick.ts) walks +// the tree and the runtime entry (runtime-entry.ts) serializes the result. No +// factory forwards to globalThis.__sanderling__ anymore: the same data tree +// drives both the goja verifier and the V8 web runtime. + import type { AccessibilityElement, Action, - ActionGenerator, Direction, DoubleTapAction, InputTextAction, @@ -16,16 +21,24 @@ import type { WaitAction, WeightedEntry, } from "./types.ts"; +import type { ActionDescriptor, BuiltinVerb, GeneratorNode } from "./action-tree.ts"; +import { getSamplerRng } from "./sampler-rng.ts"; -export function actions(generator: () => Action[]): ActionGenerator { - return globalThis.__sanderling__.actions(generator); +export { setSamplerRng } from "./sampler-rng.ts"; + +function builtinNode(verb: BuiltinVerb): GeneratorNode { + return { kind: "builtin", verb }; +} + +export function actions(generator: () => Action[]): GeneratorNode { + return { kind: "actions", generate: generator as () => ActionDescriptor[] }; } export function whenRoute( routeExtractor: { readonly current: string | null }, routes: string | readonly string[], body: () => Action[], -): ActionGenerator { +): GeneratorNode { const allowed = typeof routes === "string" ? [routes] : routes; return actions(() => { const current = routeExtractor.current; @@ -34,38 +47,45 @@ export function whenRoute( }); } -export function weighted(...entries: WeightedEntry[]): ActionGenerator { - return globalThis.__sanderling__.weighted(...entries); +export function weighted(...entries: WeightedEntry[]): GeneratorNode { + return { kind: "weighted", branches: entries }; } export function from(items: readonly T[]): Sampler { - return globalThis.__sanderling__.from(items); + return { + generate(): T { + if (items.length <= 1) return items[0] as T; + const rng = getSamplerRng(); + const index = rng ? rng.intN(items.length) : 0; + return items[index] as T; + }, + }; } export function Tap(parameters: { on: string | AccessibilityElement }): TapAction { - return globalThis.__sanderling__.tap(parameters); + return { kind: "Tap", on: parameters.on }; } export function DoubleTap(parameters: { on: string | AccessibilityElement }): DoubleTapAction { - return globalThis.__sanderling__.doubleTap(parameters); + return { kind: "DoubleTap", on: parameters.on }; } export function LongPress(parameters: { on: string | AccessibilityElement }): LongPressAction { - return globalThis.__sanderling__.longPress(parameters); + return { kind: "LongPress", on: parameters.on }; } export function Scroll(parameters: { direction: Direction; in?: string | AccessibilityElement; }): ScrollAction { - return globalThis.__sanderling__.scroll(parameters); + return { kind: "Scroll", direction: parameters.direction, in: parameters.in }; } export function InputText(parameters: { into: string | AccessibilityElement; text: string; }): InputTextAction { - return globalThis.__sanderling__.inputText(parameters); + return { kind: "InputText", into: parameters.into, text: parameters.text }; } export function Swipe(parameters: { @@ -73,44 +93,27 @@ export function Swipe(parameters: { to: Point | AccessibilityElement; durationMillis?: number; }): SwipeAction { - return globalThis.__sanderling__.swipe(parameters); + return { + kind: "Swipe", + from: parameters.from, + to: parameters.to, + durationMillis: parameters.durationMillis, + }; } export function PressKey(parameters: { key: Key }): PressKeyAction { - return globalThis.__sanderling__.pressKey(parameters); + return { kind: "PressKey", key: parameters.key }; } export function Wait(parameters: { durationMillis: number }): WaitAction { - return globalThis.__sanderling__.wait(parameters); + return { kind: "Wait", durationMillis: parameters.durationMillis }; } -function builtinGenerator( - name: - | "taps" - | "doubleTaps" - | "longPresses" - | "scrolls" - | "typing" - | "swipes" - | "waitOnce" - | "pressKeys", -): ActionGenerator { - return new Proxy({} as ActionGenerator, { - get(_target, property) { - const runtime = globalThis.__sanderling__[name] as unknown as Record< - string | symbol, - unknown - >; - return runtime[property]; - }, - }); -} - -export const taps: ActionGenerator = builtinGenerator("taps"); -export const doubleTaps: ActionGenerator = builtinGenerator("doubleTaps"); -export const longPresses: ActionGenerator = builtinGenerator("longPresses"); -export const scrolls: ActionGenerator = builtinGenerator("scrolls"); -export const typing: ActionGenerator = builtinGenerator("typing"); -export const swipes: ActionGenerator = builtinGenerator("swipes"); -export const waitOnce: ActionGenerator = builtinGenerator("waitOnce"); -export const pressKey: ActionGenerator = builtinGenerator("pressKeys"); +export const taps: GeneratorNode = builtinNode("taps"); +export const doubleTaps: GeneratorNode = builtinNode("doubleTaps"); +export const longPresses: GeneratorNode = builtinNode("longPresses"); +export const scrolls: GeneratorNode = builtinNode("scrolls"); +export const typing: GeneratorNode = builtinNode("typing"); +export const swipes: GeneratorNode = builtinNode("swipes"); +export const waitOnce: GeneratorNode = builtinNode("waitOnce"); +export const pressKeys: GeneratorNode = builtinNode("pressKeys"); diff --git a/pkg/spec/src/corpus.ts b/pkg/spec/src/corpus.ts new file mode 100644 index 0000000..a4b5941 --- /dev/null +++ b/pkg/spec/src/corpus.ts @@ -0,0 +1,47 @@ +// Single source of truth for the data both action-generator engines draw from: +// the goja verifier (internal/verifier/worker.go) and the V8 web runtime +// (web-runtime.ts). The entries and their ORDER are load-bearing: both engines +// index into these arrays with the shared PCG, so any reordering shifts every +// downstream pick for a given seed and breaks cross-engine reproducibility. + +// INPUT_CORPUS is the edge-case string pool the typing builtin draws from to +// stress field parsing: empty, single char, overflow length, unicode, blank / +// whitespace, numeric boundaries, and common injection payloads. It must stay +// byte-for-byte identical to worker.go inputCorpus. +export const INPUT_CORPUS: readonly string[] = [ + "", + "a", + "a".repeat(4096), + "🙂🔥💸", + " ", + "\t\n", + "-1", + "999999999999999999999", + "0.0000001", + "1e10", + "'; DROP TABLE--", + "", + "../../etc/passwd", + "%s%n", + "NaN", +]; + +// NATIVE_PRESS_KEYS is the key pool the native (goja) press-key builtin may +// emit. Exploration stays gentle: only "back" is in play today, because +// "home"/"menu" navigate away from the app under test. Kept as an array so the +// matrix can declare native support over the full set without the picker +// hardcoding a single value. +export const NATIVE_PRESS_KEYS: readonly string[] = ["back"]; + +// WEB_PRESS_KEYS is the key pool the web press-key builtin draws from. Only +// keys with meaningful in-page semantics are included; "back"/"home" would not +// navigate a browser tab the way they navigate a native app. +export const WEB_PRESS_KEYS: readonly string[] = [ + "enter", + "tab", + "escape", + "up", + "down", + "left", + "right", +]; diff --git a/pkg/spec/src/defaults/actions.ts b/pkg/spec/src/defaults/actions.ts index cd7fb15..853cad9 100644 --- a/pkg/spec/src/defaults/actions.ts +++ b/pkg/spec/src/defaults/actions.ts @@ -1,24 +1,12 @@ -import { doubleTaps, swipes, taps, typing, weighted } from "../actions.ts"; +import { doubleTaps, scrolls, swipes, taps, typing, weighted } from "../actions.ts"; import type { ActionGenerator } from "../types.ts"; -export { doubleTaps, pressKey, swipes, taps, typing, waitOnce } from "../actions.ts"; -// longPresses and scrolls are opt-in generators: not part of defaultActions, -// but authors can include them in their own weighted() set. -export { longPresses, scrolls } from "../actions.ts"; +export { doubleTaps, scrolls, swipes, taps, typing } from "../actions.ts"; -// A broad exploration generator: tap things, type edge-case values into fields, -// and swipe. Layer it under targeted depth flows so the fuzzer wanders the whole -// app while still driving the paths an author wrote. It deliberately omits the -// hardware back key: backing out past the app's root screen exits the app under -// test, and exploration must stay within the app. -// -// doubleTaps fires the same target twice ~50ms apart inside one step. Real users -// double-tap (image zoom, like-to-favorite, play/pause); without this the fuzzer -// can never produce sub-100ms event spacing, so race-window bugs (debounce gaps, -// in-flight guards, init races) stay unreachable. export const defaultActions: ActionGenerator = weighted( - [45, taps], - [15, doubleTaps], - [25, typing], - [15, swipes], + [100, taps], + [100, typing], + [50, scrolls], + [25, swipes], + [10, doubleTaps], ); diff --git a/pkg/spec/src/goja-runtime.ts b/pkg/spec/src/goja-runtime.ts new file mode 100644 index 0000000..b99b8d8 --- /dev/null +++ b/pkg/spec/src/goja-runtime.ts @@ -0,0 +1,23 @@ +// Goja-side runtime entry for the native verifier (internal/verifier). +// +// Go installs globalThis.__sanderlingHost__ (platform/seed/queryCandidates/ +// reportUnsupported, implemented over the hierarchy tree in bindings.go) before +// the spec evaluates. This module bundles AFTER the spec, reads that host, and +// wires the shared picker via installRuntime so the goja verifier and the V8 web +// runtime run the SAME pick.ts over the SAME Pcg. +// +// Extractors are driven by Go (bindExtract + PushSnapshot advance current/ +// previous against the Go-built `state`), so the extractor callback is a no-op +// here; Go never invokes __sanderlingExtractors__ on native. + +import { installRuntime } from "./runtime-entry.ts"; +import type { GeneratorNode, Host } from "./action-tree.ts"; + +const host = (globalThis as { __sanderlingHost__?: Host }).__sanderlingHost__; +if (!host) throw new Error("goja runtime: __sanderlingHost__ not installed"); + +installRuntime( + host, + () => (globalThis as { actions?: GeneratorNode }).actions ?? null, + () => ({}), +); diff --git a/pkg/spec/src/index.ts b/pkg/spec/src/index.ts index 969cf2d..809f4bf 100644 --- a/pkg/spec/src/index.ts +++ b/pkg/spec/src/index.ts @@ -33,14 +33,18 @@ export { always, eventually, next, now } from "./ltl.ts"; export { DoubleTap, InputText, + LongPress, PressKey, + Scroll, Swipe, Tap, Wait, actions, doubleTaps, from, - pressKey, + longPresses, + pressKeys, + scrolls, swipes, taps, typing, @@ -48,3 +52,4 @@ export { weighted, whenRoute, } from "./actions.ts"; +export { edgeCaseText, emails, integers, strings } from "./values.ts"; diff --git a/pkg/spec/src/pcg.ts b/pkg/spec/src/pcg.ts new file mode 100644 index 0000000..d2427ec --- /dev/null +++ b/pkg/spec/src/pcg.ts @@ -0,0 +1,96 @@ +// Bit-exact port of Go's math/rand/v2 PCG source wrapped in rand.Rand. +// +// This MUST match `rand.New(rand.NewPCG(hi, lo))` draw-for-draw, because every +// random decision the action picker makes is shared between the goja verifier +// and the V8 web runtime. Divergence here breaks reproducibility across engines. +// +// The golden fixture in test/fixtures/pcg-golden.json is generated by +// internal/_oracle/main.go and pins the exact sequences asserted in pcg.test.ts. + +const MASK64 = (1n << 64n) - 1n; + +const MUL_HI = 2549297995355413924n; +const MUL_LO = 4865540595714422341n; +const INC_HI = 6364136223846793005n; +const INC_LO = 1442695040888963407n; +const CHEAP_MUL = 0xda942042e4dd58b5n; + +const POW53 = 1n << 53n; + +// mul64 returns the 128-bit product of two uint64 values split into high and +// low 64-bit halves, matching Go's math/bits.Mul64. +function mul64(a: bigint, b: bigint): { hi: bigint; lo: bigint } { + const product = a * b; + return { hi: (product >> 64n) & MASK64, lo: product & MASK64 }; +} + +export class Pcg { + private hi: bigint; + private lo: bigint; + + constructor(hi: bigint, lo: bigint) { + this.hi = hi & MASK64; + this.lo = lo & MASK64; + } + + // next advances the 128-bit LCG state and returns the new (hi, lo) pair. + private next(): { hi: bigint; lo: bigint } { + // 128-bit multiply of state by the 128-bit multiplier, keeping 128 bits. + let { hi, lo } = mul64(this.lo, MUL_LO); + hi = (hi + this.hi * MUL_LO + this.lo * MUL_HI) & MASK64; + + // 128-bit add of the increment with carry propagation. + const loSum = lo + INC_LO; + const carry = loSum >> 64n; + lo = loSum & MASK64; + hi = (hi + INC_HI + carry) & MASK64; + + this.lo = lo; + this.hi = hi; + return { hi, lo }; + } + + // uint64 mirrors PCG.Uint64: the DXSM output permutation over next(). + uint64(): bigint { + let { hi, lo } = this.next(); + hi ^= hi >> 32n; + hi = (hi * CHEAP_MUL) & MASK64; + hi ^= hi >> 48n; + hi = (hi * (lo | 1n)) & MASK64; + return hi & MASK64; + } + + // float64 mirrors rand.Rand.Float64: float64(Uint64()<<11>>11) / (1<<53), + // which keeps the low 53 bits and divides by 2^53. + float64(): number { + const bits = this.uint64() & (POW53 - 1n); + return Number(bits) / Number(POW53); + } + + // intN mirrors rand.Rand.IntN via uint64n (Lemire with a rejection threshold). + // The host arch is 64-bit, so the 32-bit fast path never applies. + intN(n: number | bigint): number { + const bound = typeof n === "bigint" ? n : BigInt(n); + if (bound <= 0n) { + throw new Error("invalid argument to intN"); + } + return Number(this.uint64n(bound)); + } + + private uint64n(n: bigint): bigint { + if ((n & (n - 1n)) === 0n) { + // n is a power of two: mask the low bits. + return this.uint64() & (n - 1n); + } + + let { hi, lo } = mul64(this.uint64(), n); + if (lo < n) { + // Go computes thresh := -n % n in uint64 math, where -n wraps to 2^64 - n. + const threshold = ((1n << 64n) - n) % n; + while (lo < threshold) { + ({ hi, lo } = mul64(this.uint64(), n)); + } + } + return hi; + } +} diff --git a/pkg/spec/src/pick.ts b/pkg/spec/src/pick.ts new file mode 100644 index 0000000..97c7b4f --- /dev/null +++ b/pkg/spec/src/pick.ts @@ -0,0 +1,248 @@ +// The shared deterministic action picker for W2 approach B. +// +// walk() traverses a GeneratorNode tree, and nextAction() wraps it with the +// 16-attempt retry that matches worker.go NextAction. Both engines (the goja +// verifier and the V8 web runtime) run THIS code, drawing through the shared +// Pcg, so a given seed yields an identical action stream on every platform. +// +// PARITY CONTRACT - draw order. +// Every random decision goes through the Pcg in a FIXED, pinned order. Changing +// this order shifts the stream for a seed and breaks cross-engine +// reproducibility, so treat it as load-bearing: +// +// weighted node: ONE float64() draw, then an ASCENDING cumulative scan over +// max(0, weight). (matches worker.go pickWeighted.) +// actions node: the generator runs FIRST (any from(...).generate() inside +// draws intN(itemCount) for >1 items, nothing otherwise), +// THEN if the returned list has >1 entry, ONE intN(len) draw; +// a 0- or 1-element list draws nothing. (pickFromResult.) +// builtin node, per verb, in this exact sequence: +// taps/doubleTaps/longPresses: intN(candidateCount) [1 draw] +// typing: intN(candidateCount), intN(corpusLength) +// swipes: intN(candidateCount), +// 200 + intN(401) magnitude, intN(4) direction +// scrolls: intN(candidateCount), intN(4) direction +// pressKeys: intN(keyCount) +// waitOnce: no draw +// +// Builtin targets are resolved to a {x, y} Point by the host BEFORE the picker +// sees them, so no element handle crosses into this module. + +import type { Pcg } from "./pcg.ts"; +import type { + ActionDescriptor, + BuiltinVerb, + GeneratorNode, + Host, +} from "./action-tree.ts"; +import type { Direction, Point } from "./types.ts"; +import { INPUT_CORPUS, NATIVE_PRESS_KEYS, WEB_PRESS_KEYS } from "./corpus.ts"; +import { setSamplerRng } from "./sampler-rng.ts"; +import { supports, warnUnsupportedOnce } from "./verbs.ts"; + +// SWIPE_MIN_MAGNITUDE / SWIPE_MAGNITUDE_SPAN reproduce worker.go's +// `200 + rng.IntN(401)` swipe distance in pixels (200..600 inclusive). +const SWIPE_MIN_MAGNITUDE = 200; +const SWIPE_MAGNITUDE_SPAN = 401; +const SWIPE_DURATION_MILLIS = 250; + +const DIRECTIONS: readonly Direction[] = ["up", "down", "left", "right"]; + +const MAX_RETRIES = 16; + +// nextAction resolves an action for the current step, retrying walk() up to 16 +// times when it yields null (matches worker.go NextAction). Returns null when +// every attempt comes up empty. +export function nextAction( + root: GeneratorNode, + rng: Pcg, + host: Host, +): ActionDescriptor | null { + for (let attempt = 0; attempt < MAX_RETRIES; attempt++) { + const action = walk(root, rng, host); + if (action !== null) return action; + } + return null; +} + +// walk resolves a single node to an ActionDescriptor or null. +export function walk( + node: GeneratorNode, + rng: Pcg, + host: Host, +): ActionDescriptor | null { + switch (node.kind) { + case "weighted": + return walkWeighted(node.branches, rng, host); + case "actions": { + // Expose the picker's rng to from(...).generate() calls inside the + // generator so author sampling shares this single deterministic stream. + setSamplerRng(rng); + try { + return walkActions(node.generate(), rng); + } finally { + setSamplerRng(null); + } + } + case "builtin": + return walkBuiltin(node.verb, rng, host); + } +} + +function walkWeighted( + branches: ReadonlyArray, + rng: Pcg, + host: Host, +): ActionDescriptor | null { + let total = 0; + for (const [weight] of branches) total += Math.max(0, weight); + if (total <= 0) return null; + const draw = rng.float64() * total; + let cumulative = 0; + for (const [weight, child] of branches) { + cumulative += Math.max(0, weight); + if (draw < cumulative) return walk(child, rng, host); + } + // Floating-point slack: draw can equal total. Fall to the last branch, + // matching worker.go's trailing `return generators[length-1]`. + const last = branches[branches.length - 1]; + return last ? walk(last[1], rng, host) : null; +} + +function walkActions( + generated: ActionDescriptor[], + rng: Pcg, +): ActionDescriptor | null { + if (generated.length === 0) return null; + if (generated.length === 1) return generated[0] ?? null; + return generated[rng.intN(generated.length)] ?? null; +} + +function walkBuiltin( + verb: BuiltinVerb, + rng: Pcg, + host: Host, +): ActionDescriptor | null { + if (!supports(verb, host.platform())) { + warnUnsupportedOnce(host, verb); + return null; + } + if (verb === "waitOnce") { + return { kind: "Wait", durationMillis: 500 }; + } + if (verb === "pressKeys") { + return walkPressKey(rng, host); + } + + const candidates = host.queryCandidates(verb); + if (candidates.length === 0) return null; + const picked = candidates[rng.intN(candidates.length)]; + if (!picked) return null; + const point: Point = { x: picked.x, y: picked.y }; + + switch (verb) { + case "taps": + return tapDescriptor("Tap", point, picked.selector); + case "doubleTaps": + return tapDescriptor("DoubleTap", point, picked.selector); + case "longPresses": + return tapDescriptor("LongPress", point, picked.selector); + case "typing": { + const text = INPUT_CORPUS[rng.intN(INPUT_CORPUS.length)] ?? ""; + return { kind: "InputText", into: withSelector(point, picked.selector), text }; + } + case "swipes": + return buildSwipe(point, rng); + case "scrolls": { + const direction = DIRECTIONS[rng.intN(DIRECTIONS.length)] ?? "down"; + return buildScroll(point, direction, picked, rng); + } + } +} + +// withSelector attaches a native selector to a resolved Point so the runner can +// re-resolve the target by id/text. The web host omits it (point-only). +function withSelector(point: Point, selector?: string): Point { + if (selector === undefined) return point; + return { ...point, selector } as Point; +} + +function tapDescriptor( + kind: "Tap" | "DoubleTap" | "LongPress", + point: Point, + selector?: string, +): ActionDescriptor { + return { kind, on: withSelector(point, selector) } as ActionDescriptor; +} + +// buildScroll lowers a scroll to a swipe over the container, matching +// worker.go's geometry: the gesture drags opposite the named content motion, +// magnitude 40% of the container extent. Missing width/height (web root) yields +// a zero-length endpoint, which the runner re-derives from container bounds. +function buildScroll( + from: Point, + direction: Direction, + candidate: { width?: number; height?: number }, + _rng: Pcg, +): ActionDescriptor { + const width = candidate.width ?? 0; + const height = candidate.height ?? 0; + let toX = from.x; + let toY = from.y; + switch (direction) { + case "down": + toY = from.y - Math.trunc((4 * height) / 10); + break; + case "up": + toY = from.y + Math.trunc((4 * height) / 10); + break; + case "left": + toX = from.x + Math.trunc((4 * width) / 10); + break; + case "right": + toX = from.x - Math.trunc((4 * width) / 10); + break; + } + return { + kind: "Scroll", + direction, + in: from, + from, + to: { x: Math.max(0, toX), y: Math.max(0, toY) }, + } as ActionDescriptor; +} + +function walkPressKey(rng: Pcg, host: Host): ActionDescriptor | null { + const keys = host.platform() === "web" ? WEB_PRESS_KEYS : NATIVE_PRESS_KEYS; + if (keys.length === 0) return null; + const key = keys[rng.intN(keys.length)] ?? keys[0]; + if (key === undefined) return null; + return { kind: "PressKey", key }; +} + +function buildSwipe(from: Point, rng: Pcg): ActionDescriptor { + const magnitude = SWIPE_MIN_MAGNITUDE + rng.intN(SWIPE_MAGNITUDE_SPAN); + let toX = from.x; + let toY = from.y; + switch (rng.intN(4)) { + case 0: + toY = from.y - magnitude; + break; + case 1: + toY = from.y + magnitude; + break; + case 2: + toX = from.x - magnitude; + break; + case 3: + toX = from.x + magnitude; + break; + } + return { + kind: "Swipe", + from, + to: { x: Math.max(0, toX), y: Math.max(0, toY) }, + durationMillis: SWIPE_DURATION_MILLIS, + }; +} + diff --git a/pkg/spec/src/runtime-entry.ts b/pkg/spec/src/runtime-entry.ts new file mode 100644 index 0000000..bc5fc83 --- /dev/null +++ b/pkg/spec/src/runtime-entry.ts @@ -0,0 +1,155 @@ +// The single next-action entry shared by both engines (goja verifier and V8 web +// runtime). installRuntime wires the spec's root generator, the shared Pcg, and +// a Host into the globals the host invokes each tick: +// __sanderlingNextAction__() -> one serialized action (unified wire contract) +// __sanderlingExtractors__() -> the engine's extractor snapshot +// +// Both engines call THIS picker over the SAME Pcg, so a given seed yields an +// identical action stream by construction. + +import { Pcg } from "./pcg.ts"; +import { nextAction, walk } from "./pick.ts"; +import type { ActionDescriptor, GeneratorNode, Host } from "./action-tree.ts"; +import type { Point } from "./types.ts"; + +// SerializedAction is the flat, camelCase wire shape JS emits and Go decodes +// (ONE decoder on each side). Builtin targets are already resolved to a Point, +// and serializeAction collapses author targets that carry {x, y}; a target that +// does not resolve to coordinates drops the action (returns null). +export type SerializedAction = + | { kind: "Tap" | "DoubleTap" | "LongPress"; x: number; y: number; selector?: string } + | { kind: "InputText"; x: number; y: number; text: string; selector?: string } + | { kind: "Swipe"; fromX: number; fromY: number; toX: number; toY: number; durationMillis: number } + | { + kind: "Scroll"; + direction: string; + fromX: number; + fromY: number; + toX: number; + toY: number; + durationMillis: number; + } + | { kind: "PressKey"; key: string } + | { kind: "Wait"; durationMillis: number }; + +const DEFAULT_SWIPE_DURATION = 250; + +// pointOf resolves a target to {x, y, selector?}. Builtins and resolved ax +// elements carry numeric x/y; a bare selector string carries no geometry, so it +// serializes with (0, 0) and the selector, leaving the native runner to +// re-resolve coordinates by id/text. A target with neither shape (null, an +// unrecognized object) returns undefined so the action is dropped. +function pointOf(target: unknown): (Point & { selector?: string }) | undefined { + if (typeof target === "string") { + return target.length > 0 ? { x: 0, y: 0, selector: target } : undefined; + } + if (!target || typeof target !== "object") return undefined; + const obj = target as Record; + if (typeof obj.x === "number" && typeof obj.y === "number") { + const point: Point & { selector?: string } = { x: obj.x, y: obj.y }; + // The picker's builtin candidates carry `selector`; a goja ax element + // carries it under the runtime tag. Either lets the runner re-resolve. + const selector = obj.selector ?? obj.__sanderlingSelector; + if (typeof selector === "string" && selector.length > 0) point.selector = selector; + return point; + } + return undefined; +} + +export function serializeAction(action: ActionDescriptor | null): SerializedAction | null { + if (!action) return null; + switch (action.kind) { + case "Tap": + case "DoubleTap": + case "LongPress": { + const point = pointOf(action.on); + if (!point) return null; + const out: SerializedAction = { kind: action.kind, x: point.x, y: point.y }; + if (point.selector) out.selector = point.selector; + return out; + } + case "InputText": { + const point = pointOf(action.into); + if (!point) return null; + const out: SerializedAction = { kind: "InputText", x: point.x, y: point.y, text: action.text }; + if (point.selector) out.selector = point.selector; + return out; + } + case "Swipe": { + const from = pointOf(action.from); + const to = pointOf(action.to); + if (!from || !to) return null; + return { + kind: "Swipe", + fromX: from.x, + fromY: from.y, + toX: to.x, + toY: to.y, + durationMillis: action.durationMillis ?? DEFAULT_SWIPE_DURATION, + }; + } + case "Scroll": { + const from = pointOf(action.from) ?? pointOf(action.in); + if (!from) return null; + // The builtin generator pre-computes `to`; an author Scroll without it + // collapses to a zero-length gesture the runner re-derives from bounds. + const to = pointOf(action.to) ?? from; + return { + kind: "Scroll", + direction: action.direction, + fromX: from.x, + fromY: from.y, + toX: to.x, + toY: to.y, + durationMillis: DEFAULT_SWIPE_DURATION, + }; + } + case "PressKey": + return { kind: "PressKey", key: action.key }; + case "Wait": + return { kind: "Wait", durationMillis: action.durationMillis }; + } +} + +// installRuntime defines the next-action and extractor globals for one engine. +// root is the generator the spec assigned; pass a function when the spec runs +// AFTER this call (the web bundle imports the runtime before the spec, so the +// root only exists on globalThis.actions once the spec has evaluated). +// evaluateExtractors is the engine's snapshot of the spec's extract() handles. +// +// Setup precedence: when the spec assigned globalThis.setup, it is walked ONCE +// per tick first; if it yields an action that wins, otherwise the call falls +// through to the action root's 16-attempt retry. This matches the native +// verifier's prior NextAction precedence and applies on both engines. +export function installRuntime( + host: Host, + root: GeneratorNode | null | (() => GeneratorNode | null), + evaluateExtractors: () => Record, +): void { + const rng = new Pcg(host.seedHi(), host.seedLo()); + const resolveRoot = typeof root === "function" ? root : () => root; + const resolveSetup = () => + (globalThis as { setup?: GeneratorNode }).setup ?? null; + defineLockedGlobal("__sanderlingExtractors__", () => evaluateExtractors()); + defineLockedGlobal("__sanderlingNextAction__", () => { + // resolveRoot runs first: on web it also resets the per-tick candidate + // cache, which setup's walk below must see fresh. + const current = resolveRoot(); + const setup = resolveSetup(); + if (setup) { + const setupAction = walk(setup, rng, host); + if (setupAction) return serializeAction(setupAction); + } + if (!current) return null; + return serializeAction(nextAction(current, rng, host)); + }); +} + +function defineLockedGlobal(name: string, value: unknown): void { + Object.defineProperty(globalThis, name, { + value, + writable: false, + configurable: false, + enumerable: false, + }); +} diff --git a/pkg/spec/src/sampler-rng.ts b/pkg/spec/src/sampler-rng.ts new file mode 100644 index 0000000..0924dbe --- /dev/null +++ b/pkg/spec/src/sampler-rng.ts @@ -0,0 +1,17 @@ +// samplerRng is the picker's Pcg while it evaluates an `actions` node's +// generator (set by pick.ts walkActions). Author sampling, from(...).generate() +// and the values.ts generators, draws from it so every sample shares the single +// deterministic stream. It is null outside a walk; eager spec-time generate() +// calls then fall back to a fixed deterministic default. + +import type { Pcg } from "./pcg.ts"; + +let samplerRng: Pcg | null = null; + +export function setSamplerRng(rng: Pcg | null): void { + samplerRng = rng; +} + +export function getSamplerRng(): Pcg | null { + return samplerRng; +} diff --git a/pkg/spec/src/types.ts b/pkg/spec/src/types.ts index 384be42..62d7ed2 100644 --- a/pkg/spec/src/types.ts +++ b/pkg/spec/src/types.ts @@ -125,6 +125,7 @@ export interface WebState extends State { export interface Extracted { readonly current: T; readonly previous: T | undefined; + named(name: string): Extracted; } export interface Point { @@ -175,10 +176,10 @@ export type Key = | "left" | "right"; -export interface ActionGenerator { - readonly __sanderlingActionGenerator: true; - generate(): Action[]; -} +// ActionGenerator is a node in the action-generator tree (see action-tree.ts). +// Author specs treat it as an opaque handle they compose with weighted(); the +// shared picker walks the underlying GeneratorNode. +export type ActionGenerator = GeneratorNode; export interface Formula { readonly __sanderlingFormula: true; @@ -196,45 +197,21 @@ export interface Sampler { generate(): T; } +// SanderlingRuntime is the host-bound surface that stays on +// globalThis.__sanderling__: extract plus the LTL formula constructors. Action +// factories no longer live here; they return plain data trees (see actions.ts). export interface SanderlingRuntime { extract: (getter: (state: State) => T, name?: string) => Extracted; always: (predicateOrFormula: (() => boolean) | Formula) => Formula; now: (predicate: () => boolean) => Formula; next: (predicate: () => boolean) => Formula; eventually: (predicate: () => boolean) => EventuallyFormula; - actions: (generator: () => Action[]) => ActionGenerator; - weighted: (...entries: WeightedEntry[]) => ActionGenerator; - from: (items: readonly T[]) => Sampler; - tap: (parameters: { on: string | AccessibilityElement }) => TapAction; - doubleTap: (parameters: { on: string | AccessibilityElement }) => DoubleTapAction; - longPress: (parameters: { on: string | AccessibilityElement }) => LongPressAction; - scroll: (parameters: { - direction: Direction; - in?: string | AccessibilityElement; - }) => ScrollAction; - inputText: (parameters: { - into: string | AccessibilityElement; - text: string; - }) => InputTextAction; - swipe: (parameters: { - from: Point | AccessibilityElement; - to: Point | AccessibilityElement; - durationMillis?: number; - }) => SwipeAction; - pressKey: (parameters: { key: Key }) => PressKeyAction; - wait: (parameters: { durationMillis: number }) => WaitAction; - taps: ActionGenerator; - doubleTaps: ActionGenerator; - longPresses: ActionGenerator; - scrolls: ActionGenerator; - typing: ActionGenerator; - swipes: ActionGenerator; - waitOnce: ActionGenerator; - pressKeys: ActionGenerator; } export type WeightedEntry = readonly [number, ActionGenerator]; +import type { GeneratorNode } from "./action-tree.ts"; + declare global { // eslint-disable-next-line no-var var __sanderling__: SanderlingRuntime; diff --git a/pkg/spec/src/values.ts b/pkg/spec/src/values.ts new file mode 100644 index 0000000..818c386 --- /dev/null +++ b/pkg/spec/src/values.ts @@ -0,0 +1,104 @@ +// Author-facing, fluent, seeded value generators. Each builder is itself a +// Sampler (it has .generate()), so `integers().between(0, 9)` is usable +// anywhere a Sampler is expected and chains read left to right. +// +// Every draw goes through the shared samplerRng (sampler-rng.ts) that from() +// uses, so generators are deterministic AND identical across the goja verifier +// and the V8 web runtime. Outside an actions() walk samplerRng is null and each +// generator returns a fixed deterministic default (mirroring from()'s index 0), +// so module-load-time calls never throw and never use an unseeded source. + +import type { Pcg } from "./pcg.ts"; +import type { Sampler } from "./types.ts"; +import { getSamplerRng } from "./sampler-rng.ts"; +import { INPUT_CORPUS } from "./corpus.ts"; + +const ALPHA = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; +const DIGITS = "0123456789"; +const ALPHANUMERIC = ALPHA + DIGITS; + +function draw(rng: Pcg | null, bound: number): number { + return rng && bound > 1 ? rng.intN(bound) : 0; +} + +class StringBuilder implements Sampler { + private minLength = 1; + private maxLength = 16; + private charset = ALPHANUMERIC; + + length(min: number, max: number): this { + this.minLength = min; + this.maxLength = max; + return this; + } + + alpha(): this { + this.charset = ALPHA; + return this; + } + + generate(): string { + const rng = getSamplerRng(); + const span = this.maxLength - this.minLength + 1; + const count = this.minLength + draw(rng, span); + let result = ""; + for (let i = 0; i < count; i++) { + result += this.charset[draw(rng, this.charset.length)]; + } + return result; + } +} + +class IntegerBuilder implements Sampler { + private minValue = 0; + private maxValue = 2 ** 31 - 1; + + between(min: number, max: number): this { + this.minValue = min; + this.maxValue = max; + return this; + } + + generate(): number { + const rng = getSamplerRng(); + const span = this.maxValue - this.minValue + 1; + return this.minValue + draw(rng, span); + } +} + +class EmailBuilder implements Sampler { + private host = "example.com"; + + domain(domain: string): this { + this.host = domain; + return this; + } + + generate(): string { + const local = new StringBuilder().length(3, 8).alpha().generate(); + return `${local}@${this.host}`; + } +} + +class EdgeCaseTextBuilder implements Sampler { + generate(): string { + const rng = getSamplerRng(); + return INPUT_CORPUS[draw(rng, INPUT_CORPUS.length)] ?? ""; + } +} + +export function strings(): StringBuilder { + return new StringBuilder(); +} + +export function integers(): IntegerBuilder { + return new IntegerBuilder(); +} + +export function emails(): EmailBuilder { + return new EmailBuilder(); +} + +export function edgeCaseText(): EdgeCaseTextBuilder { + return new EdgeCaseTextBuilder(); +} diff --git a/pkg/spec/src/verbs.ts b/pkg/spec/src/verbs.ts new file mode 100644 index 0000000..49b87fe --- /dev/null +++ b/pkg/spec/src/verbs.ts @@ -0,0 +1,58 @@ +// Verb support matrix and a warn-once helper, shared by both engines. +// +// The matrix is the single declared decision about which builtin verbs each +// platform supports. The picker (pick.ts) consults it before drawing so an +// unsupported verb is surfaced once (via host.reportUnsupported) rather than +// silently producing a null action every tick. + +import type { BuiltinVerb, Host } from "./action-tree.ts"; + +export type Platform = "android" | "ios" | "web"; + +// VERB_SUPPORT declares, per verb, the platforms that can execute it. +// Tap/DoubleTap/InputText/Wait: native + web. +// PressKey: native (full key set incl. back/home) + web (enter/tab/escape/arrows). +// Swipe/LongPress/Scroll: native yes; web yes (the chrome driver supports +// pointer drags / long-press / wheel), declared here rather than silently +// no-op as the older web-runtime did. +const VERB_SUPPORT: Record> = { + taps: ["android", "ios", "web"], + doubleTaps: ["android", "ios", "web"], + typing: ["android", "ios", "web"], + waitOnce: ["android", "ios", "web"], + pressKeys: ["android", "ios", "web"], + swipes: ["android", "ios", "web"], + longPresses: ["android", "ios", "web"], + scrolls: ["android", "ios", "web"], +}; + +// supports reports whether a verb is executable on a platform. +export function supports(verb: BuiltinVerb, platform: Platform): boolean { + return VERB_SUPPORT[verb].includes(platform); +} + +// A warn-once registry keyed by `${verb}@${platform}`. Module-level so the +// "warn at most once" guarantee holds across every picker walk in a run. +const warnedKeys = new Set(); + +function warnKey(verb: BuiltinVerb, platform: Platform): string { + return `${verb}@${platform}`; +} + +// warnUnsupportedOnce calls host.reportUnsupported(verb) at most once per +// verb@platform for the lifetime of the process. Returns true on the first +// (reporting) call for a key, false on subsequent suppressed calls. The picker +// uses the boolean only for testability; behavior is the single report. +export function warnUnsupportedOnce(host: Host, verb: BuiltinVerb): boolean { + const key = warnKey(verb, host.platform()); + if (warnedKeys.has(key)) return false; + warnedKeys.add(key); + host.reportUnsupported(verb); + return true; +} + +// resetWarnings clears the warn-once registry. Test-only: production never +// resets, so a verb is reported once per process. +export function resetWarnings(): void { + warnedKeys.clear(); +} diff --git a/pkg/spec/src/web-runtime.ts b/pkg/spec/src/web-runtime.ts index 9e06dbc..a6edb79 100644 --- a/pkg/spec/src/web-runtime.ts +++ b/pkg/spec/src/web-runtime.ts @@ -2,36 +2,75 @@ // V8-side runtime for `sanderling test --platform web`. // -// The user spec is bundled with this file as the first import so that -// globalThis.__sanderling__ is installed before the spec evaluates. The host -// invokes window.__sanderlingExtractors__() and window.__sanderlingNextAction__() -// over CDP each tick. LTL predicates are intentionally stubbed: properties -// run host-side in goja, which loads its own bundle of the same spec. +// This file is the WEB Host. It installs globalThis.__sanderling__ (extract + +// LTL formula binds) before the spec evaluates, implements the Host interface +// (platform/seed/queryCandidates/reportUnsupported) over the live DOM, and then +// delegates ALL action generation to the shared picker via installRuntime +// (runtime-entry.ts -> pick.ts). The goja verifier runs the SAME picker over the +// SAME Pcg, so a given seed yields an identical action stream by construction. // -// Element references never cross V8/host. Action targets that reference an -// AccessibilityElement collapse to `{x, y}` from getBoundingClientRect() -// before serialization. +// The host invokes window.__sanderlingExtractors__() and +// window.__sanderlingNextAction__() over CDP each tick. LTL predicates are +// stubbed: properties run host-side in goja, which loads its own bundle. +// +// Element references never cross V8/host. queryCandidates resolves each element +// to a {x, y} Point via getBoundingClientRect before the picker sees it. + +import { installRuntime } from "./runtime-entry.ts"; +import type { BuiltinVerb, Candidate, Host } from "./action-tree.ts"; interface Handle { - current: unknown; - previous: unknown; + readonly current: unknown; + readonly previous: unknown; + named(name: string): Handle; } interface ExtractorEntry { getter: (state: unknown) => unknown; handle: Handle; name: string; -} - -interface ActionGeneratorHandle { - __sanderlingActionGenerator: true; - __sanderlingKind: string; - generate?: () => unknown; - entries?: readonly [number, ActionGeneratorHandle][]; + currentValue: unknown; + previousValue: unknown; } const extractors: ExtractorEntry[] = []; -let actionsRoot: ActionGeneratorHandle | null = null; + +// extracting is true only while an extractor getter is running. The current/ +// previous accessors consult it so a getter that reaches into another +// extractor's handle throws instead of reading a stale cross-extractor value. +let extracting = false; + +function checkNotExtracting(slot: "current" | "previous"): void { + if (extracting) { + throw new Error( + `reading .${slot} of an extractor inside another extractor is not allowed; extractor getters may read only from the state argument`, + ); + } +} + +// SANDERLING_SEED is the host-computed 64-bit seed, injected as a decimal +// string via the bundle define. We parse it into a BigInt without ever going +// through a JS Number (which loses precision above 2^53), matching the goja +// side's rand.NewPCG(seed, 0): hi = seed, lo = 0. +function injectedSeed(): string | undefined { + try { + return process.env.SANDERLING_SEED; + } catch { + return undefined; + } +} + +function seedBigInt(): bigint { + const raw = injectedSeed(); + if (!raw) return 0n; + try { + return BigInt(raw); + } catch { + return 0n; + } +} + +const SEED_HI = seedBigInt(); function noopFormula(): unknown { const formula: Record = { __sanderlingFormula: true }; @@ -277,6 +316,38 @@ function buildAx(): unknown { }; } +// Uncaught errors and unhandled rejections are buffered here as they fire, so +// the default noUncaughtExceptions property can observe them. Without this the +// web state.exceptions would always be empty and a page that throws would +// silently pass. +interface CapturedException { + class: string; + message: string; + stackTrace: string; + unixMillis: number; +} + +const capturedExceptions: CapturedException[] = []; + +function recordException(error: unknown): void { + const asError = error instanceof Error ? error : undefined; + capturedExceptions.push({ + class: asError?.name ?? "Error", + message: asError?.message ?? String(error), + stackTrace: asError?.stack ?? "", + unixMillis: Date.now(), + }); +} + +if (typeof globalThis.addEventListener === "function") { + globalThis.addEventListener("error", (event: ErrorEvent) => { + recordException(event.error ?? event.message); + }); + globalThis.addEventListener("unhandledrejection", (event: PromiseRejectionEvent) => { + recordException(event.reason); + }); +} + function buildState(): unknown { return { snapshots: {}, @@ -286,91 +357,42 @@ function buildState(): unknown { lastAction: null, time: 0, logs: [], - exceptions: [], + exceptions: capturedExceptions.slice(), }; } const runtime = { extract(getter: (state: unknown) => T, name?: string): Handle { - const handle: Handle = { current: undefined, previous: undefined }; const resolvedName = name && name.length > 0 ? name : `extractor_${extractors.length}`; - extractors.push({ + const entry: ExtractorEntry = { getter: getter as (s: unknown) => unknown, - handle, + handle: undefined as unknown as Handle, name: resolvedName, - }); + currentValue: undefined, + previousValue: undefined, + }; + const handle: Handle = { + get current() { + checkNotExtracting("current"); + return entry.currentValue; + }, + get previous() { + checkNotExtracting("previous"); + return entry.previousValue; + }, + named(name: string): Handle { + entry.name = name; + return handle; + }, + }; + entry.handle = handle; + extractors.push(entry); return handle; }, always: noopFormula, now: noopFormula, next: noopFormula, eventually: noopFormula, - actions(generator: () => unknown): ActionGeneratorHandle { - const handle: ActionGeneratorHandle = { - __sanderlingActionGenerator: true, - __sanderlingKind: "actions", - generate: generator, - }; - if (!actionsRoot) actionsRoot = handle; - return handle; - }, - weighted(...entries: [number, ActionGeneratorHandle][]): ActionGeneratorHandle { - const handle: ActionGeneratorHandle = { - __sanderlingActionGenerator: true, - __sanderlingKind: "weighted", - entries, - }; - actionsRoot = handle; - return handle; - }, - from(items: readonly T[]): { generate: () => T | undefined } { - return { - generate(): T | undefined { - if (items.length === 0) return undefined; - return items[Math.floor(Math.random() * items.length)]; - }, - }; - }, - tap(p: { on: unknown }): unknown { - return { kind: "Tap", on: p.on }; - }, - doubleTap(p: { on: unknown }): unknown { - return { kind: "DoubleTap", on: p.on }; - }, - longPress(): unknown { - // Why: web has no long-press gesture for v1; the factory returns null so LongPress() calls no-op. - return null; - }, - scroll(): unknown { - // Why: web has no native scroll gesture for v1; the factory returns null so Scroll() calls no-op. - return null; - }, - inputText(p: { into: unknown; text: string }): unknown { - return { kind: "InputText", into: p.into, text: p.text }; - }, - swipe(_p: { from: unknown; to: unknown; durationMillis?: number }): unknown { - // Why: web has no swipe gesture; the factory returns null so Swipe() calls in specs no-op. - return null; - }, - pressKey(p: { key: string }): unknown { - if (!WEB_PRESS_KEYS.includes(p.key)) { - throw new Error( - `pressKey: unsupported key ${JSON.stringify(p.key)} (allowed: ${WEB_PRESS_KEYS.join(", ")})`, - ); - } - return { kind: "PressKey", key: p.key }; - }, - wait(p: { durationMillis: number }): unknown { - return { kind: "Wait", durationMillis: p.durationMillis }; - }, - taps: { __sanderlingActionGenerator: true, __sanderlingKind: "taps" } as ActionGeneratorHandle, - doubleTaps: { __sanderlingActionGenerator: true, __sanderlingKind: "doubleTaps" } as ActionGeneratorHandle, - longPresses: { __sanderlingActionGenerator: true, __sanderlingKind: "longPresses" } as ActionGeneratorHandle, - scrolls: { __sanderlingActionGenerator: true, __sanderlingKind: "scrolls" } as ActionGeneratorHandle, - typing: { __sanderlingActionGenerator: true, __sanderlingKind: "typing" } as ActionGeneratorHandle, - swipes: { __sanderlingActionGenerator: true, __sanderlingKind: "swipes" } as ActionGeneratorHandle, - waitOnce: { __sanderlingActionGenerator: true, __sanderlingKind: "waitOnce" } as ActionGeneratorHandle, - pressKeys: { __sanderlingActionGenerator: true, __sanderlingKind: "pressKey" } as ActionGeneratorHandle, }; // Lock the runtime globals so a misbehaving (or malicious) page script can't @@ -378,11 +400,16 @@ const runtime = { // the host invoking the extractor/next-action callbacks. defineLockedGlobal("__sanderling__", runtime); +// writable:false stops a page script from shadowing the runtime via plain +// assignment (the realistic in-page threat). configurable:true is required so +// unit tests sharing one process can reinstall a fake via defineProperty; a +// non-configurable lock would poison globalThis.__sanderling__ for every later +// test in the run. function defineLockedGlobal(name: string, value: unknown): void { Object.defineProperty(globalThis, name, { value, writable: false, - configurable: false, + configurable: true, enumerable: false, }); } @@ -393,14 +420,19 @@ function evaluateExtractors(): Record { for (let i = 0; i < extractors.length; i++) { const entry = extractors[i]; if (!entry) continue; - entry.handle.previous = entry.handle.current; + entry.previousValue = entry.currentValue; + // Let getter throws propagate, matching the goja side, where a getter + // error aborts PushSnapshot rather than yielding undefined. Swallowing + // here would silence the cross-extractor read guard (and every other + // author error) on web only, breaking cross-engine parity. let value: unknown; + extracting = true; try { value = entry.getter(state); - } catch { - value = undefined; + } finally { + extracting = false; } - entry.handle.current = value; + entry.currentValue = value; result[i] = sanitize(value); } return result; @@ -435,96 +467,21 @@ function sanitizeAt(value: unknown, depth: number, seen: WeakSet): unkno return out; } -function pickWeighted(handle: ActionGeneratorHandle): ActionGeneratorHandle | null { - const entries = handle.entries ?? []; - if (entries.length === 0) return null; - let total = 0; - for (const [weight] of entries) total += Math.max(0, weight); - if (total <= 0) return null; - let pick = Math.random() * total; - for (const [weight, generator] of entries) { - pick -= Math.max(0, weight); - if (pick <= 0) return generator; - } - return entries[entries.length - 1]?.[1] ?? null; -} - -function resolveGenerator(handle: ActionGeneratorHandle): unknown { - switch (handle.__sanderlingKind) { - case "actions": { - const generated = handle.generate?.(); - return pickFromArray(generated); - } - case "weighted": { - const inner = pickWeighted(handle); - if (!inner) return null; - return resolveGenerator(inner); - } - case "taps": - return randomTap("Tap"); - case "doubleTaps": - return randomTap("DoubleTap"); - case "typing": - return randomInput(); - case "swipes": - return randomSwipe(); - case "longPresses": - case "scrolls": - // Why: web has no long-press or native scroll gesture for v1; both no-op. - return null; - case "waitOnce": - return { kind: "Wait", durationMillis: 500 }; - case "pressKey": - return randomPressKey(); - default: - return null; - } -} - -// Per-tick cache so the 16-attempt retry loop in __sanderlingNextAction__ -// doesn't re-walk the DOM and re-flush layout on every iteration. -let randomTapCandidates: HTMLElement[] | null = null; - -function randomTap(kind: "Tap" | "DoubleTap"): unknown { - if (!randomTapCandidates) { - randomTapCandidates = Array.from( - document.querySelectorAll( - 'a, button, input, select, textarea, [role="button"], [onclick]', - ), - ).filter((element) => { - if ((element as HTMLButtonElement).disabled) return false; - const rect = element.getBoundingClientRect(); - return rect.width > 0 && rect.height > 0; - }); - } - const candidates = randomTapCandidates; - if (candidates.length === 0) return null; - const picked = candidates[Math.floor(Math.random() * candidates.length)]; - if (!picked) return null; - const rect = picked.getBoundingClientRect(); - return { - kind, - on: { - x: Math.round(rect.left + rect.width / 2), - y: Math.round(rect.top + rect.height / 2), - }, - }; -} - -// Per-tick cache mirroring randomTapCandidates, reset each NextAction tick. -let randomInputCandidates: HTMLElement[] | null = null; +// Per-verb DOM selector sets. The tappable set backs taps/doubleTaps/longPresses +// (every tappable element is also a valid long-press target); the editable set +// backs typing; swipes/scrolls target the scrollable element. +const TAPPABLE_SELECTOR = 'a, button, input, select, textarea, [role="button"], [onclick]'; +const EDITABLE_SELECTOR = "input, textarea, [contenteditable]"; const NON_TEXT_INPUT_TYPES = [ "button", "submit", "checkbox", "radio", "range", "color", "file", "image", "reset", ]; -// Edge-case input pool mirroring the goja-side inputCorpus: empty, whitespace, -// overflow length, unicode, numeric boundaries, and common injection payloads. -const WEB_INPUT_CORPUS = [ - "", "a", "a".repeat(4096), "🙂🔥💸", " ", "\t\n", "-1", - "999999999999999999999", "0.0000001", "1e10", "'; DROP TABLE--", - "", "../../etc/passwd", "%s%n", "NaN", -]; +function isVisible(element: HTMLElement): boolean { + if ((element as HTMLButtonElement).disabled) return false; + const rect = element.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0; +} function isEditableElement(element: HTMLElement): boolean { if (element.isContentEditable) return true; @@ -537,141 +494,112 @@ function isEditableElement(element: HTMLElement): boolean { return false; } -function randomInput(): unknown { - if (!randomInputCandidates) { - randomInputCandidates = Array.from( - document.querySelectorAll("input, textarea, [contenteditable]"), - ).filter((element) => { - if (!isEditableElement(element)) return false; - if ((element as HTMLInputElement).disabled) return false; - const rect = element.getBoundingClientRect(); - return rect.width > 0 && rect.height > 0; - }); - } - const candidates = randomInputCandidates; - if (candidates.length === 0) return null; - const picked = candidates[Math.floor(Math.random() * candidates.length)]; - if (!picked) return null; - const rect = picked.getBoundingClientRect(); - const text = WEB_INPUT_CORPUS[Math.floor(Math.random() * WEB_INPUT_CORPUS.length)]; +function pointOf(element: Element): Candidate { + const rect = element.getBoundingClientRect(); return { - kind: "InputText", - into: { - x: Math.round(rect.left + rect.width / 2), - y: Math.round(rect.top + rect.height / 2), - }, - text, + x: Math.round(rect.left + rect.width / 2), + y: Math.round(rect.top + rect.height / 2), + width: Math.round(rect.width), + height: Math.round(rect.height), }; } -function randomSwipe(): unknown { - // Why: web has no swipe gesture; pointer drags into empty divs are noise. - return null; +function tappableCandidates(): Candidate[] { + return Array.from(document.querySelectorAll(TAPPABLE_SELECTOR)) + .filter(isVisible) + .map(pointOf); } -const WEB_PRESS_KEYS = ["enter", "tab", "escape", "up", "down", "left", "right"]; - -function randomPressKey(): unknown { - // Why: only emit keys with meaningful browser semantics; "back"/"home" don't navigate. - const key = WEB_PRESS_KEYS[Math.floor(Math.random() * WEB_PRESS_KEYS.length)]; - return { kind: "PressKey", key }; +function editableCandidates(): Candidate[] { + return Array.from(document.querySelectorAll(EDITABLE_SELECTOR)) + .filter((element) => isEditableElement(element) && isVisible(element)) + .map(pointOf); } -function pickFromArray(value: unknown): unknown { - if (!value) return null; - if (!Array.isArray(value)) return value; - if (value.length === 0) return null; - return value[Math.floor(Math.random() * value.length)]; -} - -function serializeAction(action: unknown): unknown { - if (!action || typeof action !== "object") return null; - const obj = action as Record; - switch (obj.kind) { - case "Tap": { - const point = pointOf(obj.on); - if (!point) { - console.warn("[sanderling] Tap target did not resolve to coordinates"); - return null; - } - return { kind: "Tap", x: point.x, y: point.y }; +// Scrollable candidates back swipe/scroll: elements that overflow their box, +// plus the scrolling root so a page-level scroll always has a target. +function scrollableCandidates(): Candidate[] { + const root = document.scrollingElement ?? document.documentElement; + const candidates: Candidate[] = root ? [pointOf(root)] : []; + for (const element of Array.from(document.querySelectorAll("*"))) { + if (element === root) continue; + if (element.scrollHeight <= element.clientHeight && element.scrollWidth <= element.clientWidth) { + continue; } - case "DoubleTap": { - const point = pointOf(obj.on); - if (!point) { - console.warn("[sanderling] DoubleTap target did not resolve to coordinates"); - return null; - } - return { kind: "DoubleTap", x: point.x, y: point.y }; - } - case "InputText": { - const point = pointOf(obj.into); - if (!point) { - console.warn("[sanderling] InputText target did not resolve to coordinates"); - return null; - } - return { - kind: "InputText", - x: point.x, - y: point.y, - text: obj.text ?? "", - }; - } - case "Swipe": { - const from = pointOf(obj.from); - const to = pointOf(obj.to); - if (!from || !to) { - console.warn("[sanderling] Swipe endpoints did not resolve to coordinates"); - return null; - } - return { - kind: "Swipe", - from_x: from.x, - from_y: from.y, - to_x: to.x, - to_y: to.y, - duration_millis: obj.durationMillis ?? 250, - }; - } - case "LongPress": - case "Scroll": - // Why: web has no long-press or native scroll gesture for v1; drop the action. - return null; - case "PressKey": - return { kind: "PressKey", key: obj.key }; - case "Wait": - return { kind: "Wait", duration_millis: obj.durationMillis ?? 0 }; - default: - return null; + if (!isVisible(element)) continue; + candidates.push(pointOf(element)); } + return candidates; } -function pointOf(value: unknown): { x: number; y: number } | undefined { - if (!value || typeof value !== "object") return undefined; - const obj = value as Record; - if (typeof obj.x === "number" && typeof obj.y === "number") { - return { x: obj.x, y: obj.y }; - } - return undefined; +// Per-tick candidate cache: the picker's 16-attempt retry re-queries the same +// verb, so we avoid re-walking the DOM and re-flushing layout within one tick. +// installRuntime resets it before each __sanderlingNextAction__ invocation. +const candidateCache = new Map(); + +function resetCandidateCache(): void { + candidateCache.clear(); } -defineLockedGlobal("__sanderlingExtractors__", function (): Record { - return evaluateExtractors(); -}); +const host: Host = { + platform: () => "web", + seedHi: () => SEED_HI, + // lo = 0 matches the goja side's rand.NewPCG(seed, 0). + seedLo: () => 0n, + queryCandidates(verb: BuiltinVerb): Candidate[] { + const cached = candidateCache.get(verb); + if (cached) return cached; + let candidates: Candidate[]; + switch (verb) { + case "taps": + case "doubleTaps": + case "longPresses": + candidates = tappableCandidates(); + break; + case "typing": + candidates = editableCandidates(); + break; + case "swipes": + case "scrolls": + candidates = scrollableCandidates(); + break; + default: + candidates = []; + } + candidateCache.set(verb, candidates); + return candidates; + }, + reportUnsupported(verb: BuiltinVerb): void { + console.warn(`[sanderling] verb ${verb} is unsupported on web`); + }, +}; -defineLockedGlobal("__sanderlingNextAction__", function (): unknown { - if (!actionsRoot) return null; - // Reset per-tick caches so each invocation gets a fresh DOM scan. - randomTapCandidates = null; - randomInputCandidates = null; - // Match the goja runtime: retry up to 16 times when a weighted entry's - // generator returns []. Otherwise on routes where most generators are - // gated to other pages, ~80% of ticks would emit no action. - for (let attempt = 0; attempt < 16; attempt++) { - const action = serializeAction(resolveGenerator(actionsRoot)); - if (action !== null) return action; - } - return null; -}); +// The spec assigns its action root to globalThis.actions, and it runs AFTER +// this module (the web bundle imports the runtime first). Resolve the root +// lazily so installRuntime captures it once the spec has evaluated. The root +// resolver runs once per __sanderlingNextAction__ tick (before the retry loop), +// so it is also where we reset the per-tick candidate cache. +installRuntime( + host, + () => { + resetCandidateCache(); + return (globalThis as { actions?: import("./action-tree.ts").GeneratorNode }).actions ?? null; + }, + evaluateExtractors, +); + +// Test-only exports. The IIFE bundle the host installs has no export surface, +// so these are stripped from production output; they only exist for unit tests. +export const __testing__ = { + host, + seedBigInt, + tappableCandidates, + editableCandidates, + scrollableCandidates, + resetCandidateCache, + runtime, + extractors, + evaluateExtractors, +}; export {}; diff --git a/pkg/spec/test/api.test.ts b/pkg/spec/test/api.test.ts index 565683e..fde96a3 100644 --- a/pkg/spec/test/api.test.ts +++ b/pkg/spec/test/api.test.ts @@ -4,38 +4,46 @@ import { test } from "node:test"; import { DoubleTap, InputText, + LongPress, PressKey, + Scroll, Swipe, Tap, Wait, actions, always, + doubleTaps, eventually, extract, from, keyedBy, + longPresses, next, now, - pressKey, + pressKeys, + scrolls, swipes, taps, + typing, waitOnce, weighted, whenRoute, } from "../src/index.ts"; +import { setSamplerRng } from "../src/actions.ts"; +import { Pcg } from "../src/pcg.ts"; +import type { GeneratorNode } from "../src/action-tree.ts"; import type { AccessibilityElement, - Action, - ActionGenerator, EventuallyFormula, Extracted, Formula, - Sampler, State, SanderlingRuntime, - WeightedEntry, } from "../src/types.ts"; +// extract() and the LTL constructors stay host-bound on __sanderling__; the +// action factories now return plain data and need no runtime. This fake records +// the host-bound calls so the forwarding tests can assert them. interface RecordedRuntime extends SanderlingRuntime { extracts: Array<(state: State) => unknown>; extractNames: Array; @@ -48,13 +56,10 @@ interface RecordedRuntime extends SanderlingRuntime { orCalls: number; andCalls: number; notCalls: number; - actionGenerators: Array<() => Action[]>; - weightedCalls: WeightedEntry[][]; - fromCalls: unknown[][]; } function makeChainableFormula(record: RecordedRuntime): Formula { - const formula: Formula = { + return { __sanderlingFormula: true, implies(other: Formula): Formula { record.impliesCalls++; @@ -76,7 +81,6 @@ function makeChainableFormula(record: RecordedRuntime): Formula { return makeChainableFormula(record); }, }; - return formula; } function makeChainableEventually(record: RecordedRuntime): EventuallyFormula { @@ -103,11 +107,8 @@ function installFakeRuntime(): RecordedRuntime { orCalls: 0, andCalls: 0, notCalls: 0, - actionGenerators: [] as Array<() => Action[]>, - weightedCalls: [] as WeightedEntry[][], - fromCalls: [] as unknown[][], }; - const runtime = { + const runtime: SanderlingRuntime = { extract: (getter: (state: State) => T, name?: string): Extracted => { calls.extracts.push(getter as (state: State) => unknown); calls.extractNames.push(name); @@ -129,42 +130,17 @@ function installFakeRuntime(): RecordedRuntime { calls.eventuallyPredicates.push(predicate); return makeChainableEventually(recorded); }, - actions: (generator: () => Action[]): ActionGenerator => { - calls.actionGenerators.push(generator); - return { __sanderlingActionGenerator: true, generate: generator }; - }, - weighted: (...entries: WeightedEntry[]): ActionGenerator => { - calls.weightedCalls.push(entries); - return { __sanderlingActionGenerator: true, generate: () => [] }; - }, - from: (items: readonly T[]): Sampler => { - calls.fromCalls.push(items as unknown[]); - return { generate: () => items[0] as T }; - }, - tap: ({ on }) => ({ kind: "Tap", on }), - doubleTap: ({ on }) => ({ kind: "DoubleTap", on }), - longPress: ({ on }) => ({ kind: "LongPress", on }), - scroll: ({ direction, in: container }) => ({ kind: "Scroll", direction, in: container }), - inputText: ({ into, text }) => ({ kind: "InputText", into, text }), - swipe: ({ from: fromPoint, to, durationMillis }) => ({ - kind: "Swipe", - from: fromPoint, - to, - durationMillis, - }), - pressKey: ({ key }) => ({ kind: "PressKey", key }), - wait: ({ durationMillis }) => ({ kind: "Wait", durationMillis }), - taps: { __sanderlingActionGenerator: true, generate: () => [] }, - doubleTaps: { __sanderlingActionGenerator: true, generate: () => [] }, - longPresses: { __sanderlingActionGenerator: true, generate: () => [] }, - scrolls: { __sanderlingActionGenerator: true, generate: () => [] }, - typing: { __sanderlingActionGenerator: true, generate: () => [] }, - swipes: { __sanderlingActionGenerator: true, generate: () => [] }, - waitOnce: { __sanderlingActionGenerator: true, generate: () => [] }, - pressKeys: { __sanderlingActionGenerator: true, generate: () => [] }, - } satisfies SanderlingRuntime; + }; const recorded = Object.assign(runtime, calls) as RecordedRuntime; - globalThis.__sanderling__ = recorded; + // web-runtime.ts locks __sanderling__ as non-writable (configurable) when it + // is imported earlier in the same bun-test process, so a plain assignment + // would throw. defineProperty installs (and reinstalls) the fake regardless. + Object.defineProperty(globalThis, "__sanderling__", { + value: recorded, + writable: false, + configurable: true, + enumerable: false, + }); return recorded; } @@ -202,14 +178,6 @@ test("always wraps a predicate into a formula via the runtime", () => { assert.equal(formula.__sanderlingFormula, true); }); -test("always accepts a formula handle", () => { - const runtime = installFakeRuntime(); - const inner = now(() => true); - const wrapped = always(inner); - assert.equal(runtime.alwaysArgs.at(-1), inner); - assert.equal(wrapped.__sanderlingFormula, true); -}); - test("now/next/eventually forward predicates", () => { const runtime = installFakeRuntime(); const p1 = () => true; @@ -240,20 +208,30 @@ test("formula chaining exposes implies/or/and/not", () => { assert.equal(runtime.notCalls, 1); }); -test("Tap returns a TapAction with the supplied selector", () => { - installFakeRuntime(); - const action = Tap({ on: "id:login_continue" }); - assert.deepEqual(action, { kind: "Tap", on: "id:login_continue" }); +test("Tap returns a TapAction descriptor", () => { + assert.deepEqual(Tap({ on: "id:login_continue" }), { + kind: "Tap", + on: "id:login_continue", + }); }); -test("DoubleTap returns a DoubleTapAction with the supplied selector", () => { - installFakeRuntime(); - const action = DoubleTap({ on: "id:save" }); - assert.deepEqual(action, { kind: "DoubleTap", on: "id:save" }); +test("DoubleTap returns a DoubleTapAction descriptor", () => { + assert.deepEqual(DoubleTap({ on: "id:save" }), { kind: "DoubleTap", on: "id:save" }); +}); + +test("LongPress returns a LongPressAction descriptor", () => { + assert.deepEqual(LongPress({ on: "id:row" }), { kind: "LongPress", on: "id:row" }); +}); + +test("Scroll returns a ScrollAction descriptor", () => { + assert.deepEqual(Scroll({ direction: "down", in: "id:list" }), { + kind: "Scroll", + direction: "down", + in: "id:list", + }); }); test("Tap accepts an AccessibilityElement", () => { - installFakeRuntime(); const element: AccessibilityElement = { id: "login_continue", find: () => undefined, @@ -264,63 +242,92 @@ test("Tap accepts an AccessibilityElement", () => { assert.equal(action.on, element); }); -test("InputText returns an InputTextAction", () => { - installFakeRuntime(); - const action = InputText({ into: "id:phone", text: "+1234567890" }); - assert.deepEqual(action, { kind: "InputText", into: "id:phone", text: "+1234567890" }); -}); - -test("Swipe returns a SwipeAction with the supplied endpoints", () => { - installFakeRuntime(); - const action = Swipe({ from: { x: 10, y: 20 }, to: { x: 30, y: 40 }, durationMillis: 400 }); - assert.deepEqual(action, { - kind: "Swipe", - from: { x: 10, y: 20 }, - to: { x: 30, y: 40 }, - durationMillis: 400, +test("InputText returns an InputTextAction descriptor", () => { + assert.deepEqual(InputText({ into: "id:phone", text: "+1234567890" }), { + kind: "InputText", + into: "id:phone", + text: "+1234567890", }); }); -test("PressKey returns a PressKeyAction", () => { - installFakeRuntime(); - const action = PressKey({ key: "back" }); - assert.deepEqual(action, { kind: "PressKey", key: "back" }); +test("Swipe returns a SwipeAction descriptor", () => { + assert.deepEqual( + Swipe({ from: { x: 10, y: 20 }, to: { x: 30, y: 40 }, durationMillis: 400 }), + { kind: "Swipe", from: { x: 10, y: 20 }, to: { x: 30, y: 40 }, durationMillis: 400 }, + ); }); -test("Wait returns a WaitAction", () => { - installFakeRuntime(); - const action = Wait({ durationMillis: 500 }); - assert.deepEqual(action, { kind: "Wait", durationMillis: 500 }); +test("PressKey returns a PressKeyAction descriptor", () => { + assert.deepEqual(PressKey({ key: "back" }), { kind: "PressKey", key: "back" }); }); -test("actions wraps a generator into the runtime's ActionGenerator", () => { - const runtime = installFakeRuntime(); +test("Wait returns a WaitAction descriptor", () => { + assert.deepEqual(Wait({ durationMillis: 500 }), { kind: "Wait", durationMillis: 500 }); +}); + +test("actions returns an actions node carrying the generator", () => { const generator = () => [Tap({ on: "id:x" })]; - const wrapped = actions(generator); - assert.equal(runtime.actionGenerators[0], generator); - assert.equal(wrapped.__sanderlingActionGenerator, true); + const node = actions(generator); + assert.equal(node.kind, "actions"); + assert.equal((node as { generate: unknown }).generate, generator); }); -test("weighted forwards weighted entries to the runtime", () => { - const runtime = installFakeRuntime(); - const entries: WeightedEntry[] = [ +test("weighted returns a weighted node carrying the branches", () => { + const node = weighted([80, taps], [20, swipes]); + assert.equal(node.kind, "weighted"); + assert.deepEqual((node as { branches: unknown }).branches, [ [80, taps], [20, swipes], + ]); +}); + +test("builtin factories return builtin nodes carrying their verb", () => { + const cases: Array<[GeneratorNode, string]> = [ + [taps, "taps"], + [doubleTaps, "doubleTaps"], + [longPresses, "longPresses"], + [scrolls, "scrolls"], + [typing, "typing"], + [swipes, "swipes"], + [waitOnce, "waitOnce"], + [pressKeys, "pressKeys"], ]; - weighted(...entries); - assert.deepEqual(runtime.weightedCalls[0], entries); + for (const [node, verb] of cases) { + assert.equal(node.kind, "builtin"); + assert.equal((node as { verb: string }).verb, verb); + } }); -test("from forwards items to the runtime", () => { - const runtime = installFakeRuntime(); - const sampler = from(["a", "b", "c"]); - assert.deepEqual(runtime.fromCalls[0], ["a", "b", "c"]); - assert.equal(sampler.generate(), "a"); +test("from with a single item returns it without drawing", () => { + const sampler = from(["only"]); + setSamplerRng(new Pcg(42n, 0n)); + try { + assert.equal(sampler.generate(), "only"); + } finally { + setSamplerRng(null); + } }); -function elementWithChildren( - cells: Record, -): AccessibilityElement { +test("from draws intN(len) from the active picker rng", () => { + const items = ["a", "b", "c"]; + const sampler = from(items); + const rng = new Pcg(42n, 0n); + const oracle = new Pcg(42n, 0n); + setSamplerRng(rng); + try { + const index = oracle.intN(items.length); + assert.equal(sampler.generate(), items[index]); + } finally { + setSamplerRng(null); + } +}); + +test("from falls back to the first item outside a picker walk", () => { + setSamplerRng(null); + assert.equal(from(["a", "b", "c"]).generate(), "a"); +}); + +function elementWithChildren(cells: Record): AccessibilityElement { return { find: selector => { if (typeof selector === "string" || Array.isArray(selector)) return undefined; @@ -346,58 +353,42 @@ test("keyedBy joins testTag-resolved texts with a stable delimiter", () => { }); test("keyedBy returns empty string for an undefined element", () => { - installFakeRuntime(); assert.equal(keyedBy(undefined, ["TxnDate"]), ""); }); test("keyedBy substitutes empty strings for missing children", () => { - installFakeRuntime(); const row = elementWithChildren({ TxnDate: "2026-04-26" }); - assert.equal( - keyedBy(row, ["TxnDate", "TxnNote", "TxnAmount"]), - "2026-04-26\x1f\x1f", - ); + assert.equal(keyedBy(row, ["TxnDate", "TxnNote", "TxnAmount"]), "2026-04-26\x1f\x1f"); }); -test("whenRoute returns [] when current route does not match", () => { - installFakeRuntime(); +test("whenRoute body is skipped when the current route does not match", () => { const route = { current: "home" as string | null }; let bodyCalled = false; - const generator = whenRoute(route, "ledger", () => { + const node = whenRoute(route, "ledger", () => { bodyCalled = true; return [Tap({ on: "id:x" })]; }); - assert.deepEqual(generator.generate(), []); + assert.equal(node.kind, "actions"); + assert.deepEqual((node as { generate: () => unknown }).generate(), []); assert.equal(bodyCalled, false); }); -test("whenRoute calls body when current route matches", () => { - installFakeRuntime(); +test("whenRoute runs the body when the current route matches", () => { const route = { current: "ledger" as string | null }; - const generator = whenRoute(route, "ledger", () => [Tap({ on: "id:x" })]); - const result = generator.generate(); + const node = whenRoute(route, "ledger", () => [Tap({ on: "id:x" })]); + const result = (node as { generate: () => Array<{ kind: string }> }).generate(); assert.equal(result.length, 1); assert.equal(result[0]?.kind, "Tap"); }); test("whenRoute accepts an array of allowed routes", () => { - installFakeRuntime(); const route = { current: "add-account" as string | null }; - const generator = whenRoute(route, ["home", "add-account"], () => [Tap({ on: "id:x" })]); - assert.equal(generator.generate().length, 1); + const node = whenRoute(route, ["home", "add-account"], () => [Tap({ on: "id:x" })]); + assert.equal((node as { generate: () => unknown[] }).generate().length, 1); }); -test("whenRoute returns [] for null route", () => { - installFakeRuntime(); +test("whenRoute body is skipped for a null route", () => { const route = { current: null as string | null }; - const generator = whenRoute(route, ["home"], () => [Tap({ on: "id:x" })]); - assert.deepEqual(generator.generate(), []); -}); - -test("default generators proxy through to the runtime", () => { - installFakeRuntime(); - assert.equal(taps.__sanderlingActionGenerator, true); - assert.equal(swipes.__sanderlingActionGenerator, true); - assert.equal(waitOnce.__sanderlingActionGenerator, true); - assert.equal(pressKey.__sanderlingActionGenerator, true); + const node = whenRoute(route, ["home"], () => [Tap({ on: "id:x" })]); + assert.deepEqual((node as { generate: () => unknown }).generate(), []); }); diff --git a/pkg/spec/test/defaults.test.ts b/pkg/spec/test/defaults.test.ts index 9c471b6..50c533d 100644 --- a/pkg/spec/test/defaults.test.ts +++ b/pkg/spec/test/defaults.test.ts @@ -2,99 +2,29 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import type { - Action, - ActionGenerator, EventuallyFormula, Extracted, Formula, - Sampler, State, SanderlingRuntime, - WeightedEntry, } from "../src/types.ts"; -interface RecordedRuntime extends SanderlingRuntime { - currentState: State; - extractors: Array<(state: State) => unknown>; - alwaysArgs: Array<(() => boolean) | Formula>; - lastPredicate: (() => boolean) | undefined; -} - -function installRuntime(initialState: State): RecordedRuntime { - const extractors: Array<(state: State) => unknown> = []; - const extracted: Array<{ value: unknown }> = []; - const alwaysArgs: Array<(() => boolean) | Formula> = []; - let lastPredicate: (() => boolean) | undefined; - - const runtime = { - extract: (getter: (state: State) => T): Extracted => { - extractors.push(getter as (state: State) => unknown); - const slot = { value: getter(state.currentState) }; - extracted.push(slot); - return { - get current(): T { - return slot.value as T; - }, - previous: undefined, - }; - }, - always: (predicateOrFormula: (() => boolean) | Formula): Formula => { - alwaysArgs.push(predicateOrFormula); - if (typeof predicateOrFormula === "function") { - lastPredicate = predicateOrFormula; - } - return { __sanderlingFormula: true } as Formula; - }, +// The defaults bundle still calls extract() and the LTL constructors, which +// stay host-bound on __sanderling__. Action factories now return plain data +// trees, so the fake runtime only needs the extract + formula surface. +function installRuntime(initialState: State): void { + const state = { current: initialState }; + const runtime: SanderlingRuntime = { + extract: (getter: (s: State) => T): Extracted => ({ + current: getter(state.current), + previous: undefined, + }), + always: () => ({ __sanderlingFormula: true } as Formula), now: () => ({ __sanderlingFormula: true } as Formula), next: () => ({ __sanderlingFormula: true } as Formula), eventually: () => ({ __sanderlingFormula: true } as EventuallyFormula), - actions: (generator: () => Action[]): ActionGenerator => ({ - __sanderlingActionGenerator: true, - generate: generator, - }), - weighted: (..._entries: WeightedEntry[]): ActionGenerator => ({ - __sanderlingActionGenerator: true, - generate: () => [], - }), - from: (_items: readonly T[]): Sampler => ({ generate: () => _items[0] as T }), - tap: ({ on }) => ({ kind: "Tap", on }), - doubleTap: ({ on }) => ({ kind: "DoubleTap", on }), - longPress: ({ on }) => ({ kind: "LongPress", on }), - scroll: ({ direction, in: container }) => ({ kind: "Scroll", direction, in: container }), - inputText: ({ into, text }) => ({ kind: "InputText", into, text }), - swipe: (p) => ({ kind: "Swipe", from: p.from, to: p.to, durationMillis: p.durationMillis }), - pressKey: ({ key }) => ({ kind: "PressKey", key }), - wait: ({ durationMillis }) => ({ kind: "Wait", durationMillis }), - taps: { __sanderlingActionGenerator: true, generate: () => [] } as ActionGenerator, - doubleTaps: { __sanderlingActionGenerator: true, generate: () => [] } as ActionGenerator, - longPresses: { __sanderlingActionGenerator: true, generate: () => [] } as ActionGenerator, - scrolls: { __sanderlingActionGenerator: true, generate: () => [] } as ActionGenerator, - typing: { __sanderlingActionGenerator: true, generate: () => [] } as ActionGenerator, - swipes: { __sanderlingActionGenerator: true, generate: () => [] } as ActionGenerator, - waitOnce: { __sanderlingActionGenerator: true, generate: () => [] } as ActionGenerator, - pressKeys: { __sanderlingActionGenerator: true, generate: () => [] } as ActionGenerator, - } satisfies SanderlingRuntime; - - const state = { currentState: initialState }; - const recorded = Object.assign(runtime, { - currentState: initialState, - extractors, - alwaysArgs, - get lastPredicate() { - return lastPredicate; - }, - }) as unknown as RecordedRuntime; - globalThis.__sanderling__ = recorded; - // Re-bind state ref so subsequent extract() calls read the up-to-date state. - Object.defineProperty(recorded, "currentState", { - get() { - return state.currentState; - }, - set(next: State) { - state.currentState = next; - }, - }); - return recorded; + }; + globalThis.__sanderling__ = runtime; } const emptyState: State = { @@ -116,29 +46,29 @@ test("defaults bundle exports formulas tagged as LTL properties", async () => { assert.equal(defaults.noLogcatErrors.__sanderlingFormula, true); }); -test("defaults bundle exports defaultActions as a valid ActionGenerator", async () => { +test("defaults bundle exports defaultActions as a weighted node", async () => { installRuntime(emptyState); const defaults = await import("../src/defaults/actions.ts"); - assert.equal(defaults.defaultActions.__sanderlingActionGenerator, true); - assert.equal(typeof defaults.defaultActions.generate, "function"); + assert.equal(defaults.defaultActions.kind, "weighted"); }); -test("typing resolves as a valid ActionGenerator", async () => { +test("typing resolves as a builtin node", async () => { installRuntime(emptyState); const { typing } = await import("../src/defaults/actions.ts"); - assert.equal(typing.__sanderlingActionGenerator, true); + assert.equal(typing.kind, "builtin"); + assert.equal((typing as { verb: string }).verb, "typing"); }); -test("longPresses and scrolls resolve as valid ActionGenerators", async () => { +test("scrolls resolves as a builtin node", async () => { installRuntime(emptyState); - const { longPresses, scrolls } = await import("../src/defaults/actions.ts"); - assert.equal(longPresses.__sanderlingActionGenerator, true); - assert.equal(scrolls.__sanderlingActionGenerator, true); + const { scrolls } = await import("../src/defaults/actions.ts"); + assert.equal(scrolls.kind, "builtin"); + assert.equal((scrolls as { verb: string }).verb, "scrolls"); }); test("defaults barrel re-exports both properties and actions", async () => { installRuntime(emptyState); const barrel = await import("../src/defaults/index.ts"); - assert.equal(barrel.defaultActions.__sanderlingActionGenerator, true); + assert.equal(barrel.defaultActions.kind, "weighted"); assert.equal(barrel.noUncaughtExceptions.__sanderlingFormula, true); }); diff --git a/pkg/spec/test/fixtures/parity-golden.json b/pkg/spec/test/fixtures/parity-golden.json new file mode 100644 index 0000000..b9865d4 --- /dev/null +++ b/pkg/spec/test/fixtures/parity-golden.json @@ -0,0 +1,127 @@ +[ + { + "kind": "Tap", + "x": 250, + "y": 260, + "selector": "id:gamma" + }, + { + "kind": "Tap", + "x": 150, + "y": 160, + "selector": "id:beta" + }, + { + "kind": "Tap", + "x": 50, + "y": 60, + "selector": "id:alpha" + }, + { + "kind": "InputText", + "x": 150, + "y": 160, + "text": "a", + "selector": "id:beta" + }, + { + "kind": "InputText", + "x": 50, + "y": 60, + "text": "\t\n", + "selector": "id:alpha" + }, + { + "kind": "Tap", + "x": 250, + "y": 260, + "selector": "id:gamma" + }, + { + "kind": "Tap", + "x": 150, + "y": 160, + "selector": "id:beta" + }, + { + "kind": "Tap", + "x": 250, + "y": 260, + "selector": "id:gamma" + }, + { + "kind": "Tap", + "x": 150, + "y": 160, + "selector": "id:beta" + }, + { + "kind": "Tap", + "x": 250, + "y": 260, + "selector": "id:gamma" + }, + { + "kind": "Tap", + "x": 250, + "y": 260, + "selector": "id:gamma" + }, + { + "kind": "Tap", + "x": 150, + "y": 160, + "selector": "id:beta" + }, + { + "kind": "Tap", + "x": 250, + "y": 260, + "selector": "id:gamma" + }, + { + "kind": "Tap", + "x": 50, + "y": 60, + "selector": "id:alpha" + }, + { + "kind": "InputText", + "x": 150, + "y": 160, + "text": "999999999999999999999", + "selector": "id:beta" + }, + { + "kind": "InputText", + "x": 150, + "y": 160, + "text": "-1", + "selector": "id:beta" + }, + { + "kind": "InputText", + "x": 250, + "y": 260, + "text": "-1", + "selector": "id:gamma" + }, + { + "kind": "Tap", + "x": 50, + "y": 60, + "selector": "id:alpha" + }, + { + "kind": "Tap", + "x": 250, + "y": 260, + "selector": "id:gamma" + }, + { + "kind": "Tap", + "x": 50, + "y": 60, + "selector": "id:alpha" + } +] diff --git a/pkg/spec/test/fixtures/pcg-golden.json b/pkg/spec/test/fixtures/pcg-golden.json new file mode 100644 index 0000000..ca3aa73 --- /dev/null +++ b/pkg/spec/test/fixtures/pcg-golden.json @@ -0,0 +1,1512 @@ +[ + { + "seed": { + "hi": "1718000000000000000", + "lo": "0" + }, + "uint64": [ + "17348611194921198332", + "10040944959238080856", + "8546612062296762771", + "11911064850236528314", + "7060439982049491705", + "4727496195952149700", + "632791401341060692", + "16695182092268216511", + "10991305199402290007", + "14802327373375977896", + "11383701653022170987", + "2532857567118737717", + "4864384924974856626", + "9678998534922072645", + "1249089104885130196", + "10172862666788984732", + "15454036291116160617", + "11781475151410497333", + "17706524219203127367", + "3288130660272180635", + "16195642289273821815", + "5180832297649062942", + "884936370795154983", + "12713420131221904441", + "1441470040918610617", + "18321196531671865052", + "1284049813417881835", + "12027944458602364836", + "12785102827252194295", + "2391210769595888371", + "7742723220204507445", + "9399304330579474949", + "7009950411738525457", + "233209627215066062", + "2274803216116889585", + "16628428634611566461", + "16602072773870075273", + "13307501757050341346", + "9479879966388146928", + "9350508562274355147" + ], + "float64": [ + 0.08275938712640096, + 0.7688282740021277, + 0.8645494100958778, + 0.39384445359856746, + 0.8663032055372631, + 0.8575125573917366, + 0.253958355366374, + 0.5375559145768632, + 0.28001030586197373, + 0.38846679612270396, + 0.8447680648668955, + 0.20367891112804293, + 0.05521443466004983, + 0.5847028783642335, + 0.6767484051898642, + 0.414635901880263, + 0.7426913789920039, + 0.006497714508646113, + 0.819087424228436, + 0.05583670100934357, + 0.0774968144676923, + 0.1879303571641946, + 0.24767341850060098, + 0.47317514155373097, + 0.0353228735208041, + 0.06142281446656339, + 0.5581667622168686, + 0.3700876852908448, + 0.4315531015571413, + 0.47772531369954974, + 0.6149592370880216, + 0.5324083268276928, + 0.2608348592707178, + 0.8914697698415865, + 0.5538907024326801, + 0.12643334820152996, + 0.2003450052997532, + 0.42948509170150495, + 0.47810093668758036, + 0.1149897791658473 + ], + "intN": { + "1000": [ + 940, + 544, + 463, + 645, + 382, + 256, + 34, + 905, + 595, + 802, + 617, + 137, + 263, + 524, + 67, + 551, + 837, + 638, + 959, + 178, + 877, + 280, + 47, + 689, + 78, + 993, + 69, + 652, + 693, + 129, + 419, + 509, + 380, + 12, + 123, + 901, + 900, + 721, + 513, + 506 + ], + "15": [ + 14, + 8, + 6, + 9, + 5, + 3, + 0, + 13, + 8, + 12, + 9, + 2, + 3, + 7, + 1, + 8, + 12, + 9, + 14, + 2, + 13, + 4, + 0, + 10, + 1, + 14, + 1, + 9, + 10, + 1, + 6, + 7, + 5, + 0, + 1, + 13, + 13, + 10, + 7, + 7 + ], + "2": [ + 0, + 0, + 1, + 0, + 1, + 0, + 0, + 1, + 1, + 0, + 1, + 1, + 0, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 0, + 1, + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 0, + 0, + 1 + ], + "3": [ + 2, + 1, + 1, + 1, + 1, + 0, + 0, + 2, + 1, + 2, + 1, + 0, + 0, + 1, + 0, + 1, + 2, + 1, + 2, + 0, + 2, + 0, + 0, + 2, + 0, + 2, + 0, + 1, + 2, + 0, + 1, + 1, + 1, + 0, + 0, + 2, + 2, + 2, + 1, + 1 + ], + "7": [ + 6, + 3, + 3, + 4, + 2, + 1, + 0, + 6, + 4, + 5, + 4, + 0, + 1, + 3, + 0, + 3, + 5, + 4, + 6, + 1, + 6, + 1, + 0, + 4, + 0, + 6, + 0, + 4, + 4, + 0, + 2, + 3, + 2, + 0, + 0, + 6, + 6, + 5, + 3, + 3 + ] + } + }, + { + "seed": { + "hi": "42", + "lo": "0" + }, + "uint64": [ + "15818923871262189983", + "17722066050711855076", + "2514175219165311687", + "1632697813927551345", + "3420513755306226561", + "17826860435361894773", + "17132589681425434452", + "6708534997569091786", + "11943912673905232833", + "17729159255747996613", + "9861653340906094072", + "11253915131898209768", + "4166815047917170816", + "4522597549831867061", + "12690056282563164493", + "11005962917559305066", + "9253944782480308990", + "4100344308106584178", + "9087748972365011641", + "11140840327820824429", + "11082442715008555486", + "13655427885467798722", + "10053642218006668", + "4790098033610881453", + "16524165261873775022", + "11534003919357636538", + "11447319655187096435", + "11423782560467231634", + "6162267594286981940", + "10979719413435778507", + "9396635032880542089", + "16183675812321746121", + "11884351765493195295", + "7609191463528820950", + "12761729699876744058", + "9754348074297120360", + "4407452443053730173", + "5870077580694772103", + "1339966861964530071", + "12700187420381780895" + ], + "float64": [ + 0.25335066677989804, + 0.5445773428118597, + 0.1295216259328179, + 0.2658705277527088, + 0.7533127188031444, + 0.17909121180177456, + 0.09955357738929616, + 0.7970015804874262, + 0.04068546818086116, + 0.33208129776300466, + 0.8634599945543746, + 0.4355696610847959, + 0.6094005552217965, + 0.1091930936657578, + 0.8792668690746638, + 0.9073439245004071, + 0.3942566119474973, + 0.22966597506366626, + 0.9428151133241364, + 0.8817445619156224, + 0.39830712918302047, + 0.057044955476520665, + 0.11617850717745304, + 0.8077127126487561, + 0.55087364434444, + 0.5316717387645362, + 0.9077851432822243, + 0.29464269420451417, + 0.14913670788801037, + 0.9937296720219647, + 0.23605675033427176, + 0.7489509908836222, + 0.4280962795174367, + 0.7899561590889117, + 0.8366146845850004, + 0.9501822292720137, + 0.32551822186478285, + 0.7095286423271298, + 0.7662062387740204, + 0.004049116262525021 + ], + "intN": { + "1000": [ + 857, + 960, + 136, + 88, + 185, + 966, + 928, + 363, + 647, + 961, + 534, + 610, + 225, + 245, + 687, + 596, + 501, + 222, + 492, + 603, + 600, + 740, + 0, + 259, + 895, + 625, + 620, + 619, + 334, + 595, + 509, + 877, + 644, + 412, + 691, + 528, + 238, + 318, + 72, + 688 + ], + "15": [ + 12, + 14, + 2, + 1, + 2, + 14, + 13, + 5, + 9, + 14, + 8, + 9, + 3, + 3, + 10, + 8, + 7, + 3, + 7, + 9, + 9, + 11, + 0, + 3, + 13, + 9, + 9, + 9, + 5, + 8, + 7, + 13, + 9, + 6, + 10, + 7, + 3, + 4, + 1, + 10 + ], + "2": [ + 1, + 0, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 1, + 0, + 0, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 1, + 1 + ], + "3": [ + 2, + 2, + 0, + 0, + 0, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 0, + 0, + 2, + 1, + 1, + 0, + 1, + 1, + 1, + 2, + 0, + 0, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 0, + 0, + 0, + 2 + ], + "7": [ + 6, + 6, + 0, + 0, + 1, + 6, + 6, + 2, + 4, + 6, + 3, + 4, + 1, + 1, + 4, + 4, + 3, + 1, + 3, + 4, + 4, + 5, + 0, + 1, + 6, + 4, + 4, + 4, + 2, + 4, + 3, + 6, + 4, + 2, + 4, + 3, + 1, + 2, + 0, + 4 + ] + } + }, + { + "seed": { + "hi": "0", + "lo": "0" + }, + "uint64": [ + "4107282207882862730", + "12464933722704884221", + "6335497120186221275", + "8128828598453185623", + "7371889741448783355", + "1418823048812169170", + "13446722647129416609", + "8878188716898717410", + "13708822528903384745", + "16582119225744604457", + "6951088302835677065", + "4580399714873389198", + "4662281982647059055", + "427459239689272317", + "13773885170329372921", + "9068706895795695628", + "10649950287751359566", + "15304463719903092841", + "2992830529688691246", + "2656865971666195087", + "5315926386399609871", + "5962616490126614837", + "14669413009525085053", + "15137080102080446231", + "5390616415851564044", + "9584934278684878893", + "16289716273760509342", + "11298720576883687201", + "17155501937552697760", + "3945808130719801762", + "514692979842194301", + "12255250582248869772", + "14704735570187722263", + "3175139522179632522", + "3522950458692549131", + "16148541362668152345", + "14248069443876374819", + "16432995152846430396", + "8267525728330725005", + "8988896498726235416" + ], + "float64": [ + 0.9999275824802834, + 0.8856419373528862, + 0.38147752771154886, + 0.4812673234167829, + 0.44417259544314847, + 0.5210016660132573, + 0.8861088591612437, + 0.6769530468231688, + 0.9850412088603281, + 0.98505615011337, + 0.7258280010771482, + 0.5265225438938772, + 0.6172804429789503, + 0.45750899917936716, + 0.20844546426658817, + 0.8287199289311515, + 0.3820020574870655, + 0.1368001376786313, + 0.2709362861488154, + 0.9713775086898567, + 0.18638713932534923, + 0.9834114348205164, + 0.6320080910560218, + 0.5534854925026335, + 0.47864618007440773, + 0.14147257148693415, + 0.5218071740027204, + 0.40997332623007676, + 0.6433250072489507, + 0.0727037478273369, + 0.14239968337356823, + 0.6061368963550975, + 0.5536001046938875, + 0.5113003921145929, + 0.12607138542245033, + 0.8482434945883056, + 0.8534753049700935, + 0.42895822437005604, + 0.8797420273637769, + 0.9679858858320669 + ], + "intN": { + "1000": [ + 222, + 675, + 343, + 440, + 399, + 76, + 728, + 481, + 743, + 898, + 376, + 248, + 252, + 23, + 746, + 491, + 577, + 829, + 162, + 144, + 288, + 323, + 795, + 820, + 292, + 519, + 883, + 612, + 930, + 213, + 27, + 664, + 797, + 172, + 190, + 875, + 772, + 890, + 448, + 487 + ], + "15": [ + 3, + 10, + 5, + 6, + 5, + 1, + 10, + 7, + 11, + 13, + 5, + 3, + 3, + 0, + 11, + 7, + 8, + 12, + 2, + 2, + 4, + 4, + 11, + 12, + 4, + 7, + 13, + 9, + 13, + 3, + 0, + 9, + 11, + 2, + 2, + 13, + 11, + 13, + 6, + 7 + ], + "2": [ + 0, + 1, + 1, + 1, + 1, + 0, + 1, + 0, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + 0, + 1, + 0, + 0, + 1, + 0, + 1, + 0, + 1, + 1, + 1, + 0, + 1, + 0 + ], + "3": [ + 0, + 2, + 1, + 1, + 1, + 0, + 2, + 1, + 2, + 2, + 1, + 0, + 0, + 0, + 2, + 1, + 1, + 2, + 0, + 0, + 0, + 0, + 2, + 2, + 0, + 1, + 2, + 1, + 2, + 0, + 0, + 1, + 2, + 0, + 0, + 2, + 2, + 2, + 1, + 1 + ], + "7": [ + 1, + 4, + 2, + 3, + 2, + 0, + 5, + 3, + 5, + 6, + 2, + 1, + 1, + 0, + 5, + 3, + 4, + 5, + 1, + 1, + 2, + 2, + 5, + 5, + 2, + 3, + 6, + 4, + 6, + 1, + 0, + 4, + 5, + 1, + 1, + 6, + 5, + 6, + 3, + 3 + ] + } + }, + { + "seed": { + "hi": "1", + "lo": "2" + }, + "uint64": [ + "14192431797130687760", + "11371241257079532652", + "14470142590855381128", + "14694613213362438554", + "4321634407747778896", + "760102831717374652", + "9221744211007427193", + "8289669384274456462", + "2449715415482412441", + "3389241988064777392", + "12210202232702069999", + "8204908297817606218", + "17358349022401942459", + "2240328155279531677", + "7311121042813227358", + "14454429957748299131", + "13481244625344276711", + "9381769212557126946", + "1350674201389090105", + "6093522341581845358", + "897456374646289561", + "11814882063598695543", + "3824056318896229933", + "398870565032109638", + "6158782141916471494", + "13037077871211764336", + "13617661536098221011", + "15181698440823082209", + "17681833180131704924", + "8499734460532927466", + "10917103977888096435", + "12419507493515068174", + "9261752443204402422", + "137166597355566241", + "11349393104106307790", + "17472900827608112440", + "12896061982357793967", + "1873294050595685917", + "6999251097555031736", + "16665206720432262420" + ], + "float64": [ + 0.6764556596678251, + 0.4613862177205994, + 0.5085473976760264, + 0.4297927436037299, + 0.797802349388613, + 0.3883664855410056, + 0.8192750264193612, + 0.3381816951746133, + 0.9730458047755973, + 0.281449117585586, + 0.6047654075331631, + 0.9278107175107462, + 0.16387541502137226, + 0.7263900707339023, + 0.6974917552729882, + 0.7640946923790318, + 0.7188183661358182, + 0.5856191500346635, + 0.9549597149363428, + 0.5168804691962643, + 0.6376730728931249, + 0.7154100238054885, + 0.5555372701914557, + 0.2835285143893962, + 0.7621737605905075, + 0.40640264504220447, + 0.8641378928622266, + 0.5071161917625454, + 0.07783141632718182, + 0.6600901283539973, + 0.04201984782505119, + 0.8423173805095716, + 0.26108108238250227, + 0.22855145936377508, + 0.035753970079918096, + 0.8816805802479513, + 0.7504939806762162, + 0.9774186864653448, + 0.0729723638493942, + 0.209621116179195 + ], + "intN": { + "1000": [ + 769, + 616, + 784, + 796, + 234, + 41, + 499, + 449, + 132, + 183, + 661, + 444, + 940, + 121, + 396, + 783, + 730, + 508, + 73, + 330, + 48, + 640, + 207, + 21, + 333, + 706, + 738, + 823, + 958, + 460, + 591, + 673, + 502, + 7, + 615, + 947, + 699, + 101, + 379, + 903 + ], + "15": [ + 11, + 9, + 11, + 11, + 3, + 0, + 7, + 6, + 1, + 2, + 9, + 6, + 14, + 1, + 5, + 11, + 10, + 7, + 1, + 4, + 0, + 9, + 3, + 0, + 5, + 10, + 11, + 12, + 14, + 6, + 8, + 10, + 7, + 0, + 9, + 14, + 10, + 1, + 5, + 13 + ], + "2": [ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 1, + 0, + 1, + 1, + 0, + 1, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 1, + 0, + 0, + 1, + 0, + 0, + 1, + 1, + 0, + 0 + ], + "3": [ + 2, + 1, + 2, + 2, + 0, + 0, + 1, + 1, + 0, + 0, + 1, + 1, + 2, + 0, + 1, + 2, + 2, + 1, + 0, + 0, + 0, + 1, + 0, + 0, + 1, + 2, + 2, + 2, + 2, + 1, + 1, + 2, + 1, + 0, + 1, + 2, + 2, + 0, + 1, + 2 + ], + "7": [ + 5, + 4, + 5, + 5, + 1, + 0, + 3, + 3, + 0, + 1, + 4, + 3, + 6, + 0, + 2, + 5, + 5, + 3, + 0, + 2, + 0, + 4, + 1, + 0, + 2, + 4, + 5, + 5, + 6, + 3, + 4, + 4, + 3, + 0, + 4, + 6, + 4, + 0, + 2, + 6 + ] + } + }, + { + "seed": { + "hi": "18446744073709551615", + "lo": "18446744073709551615" + }, + "uint64": [ + "12052289606655298394", + "17828373617109055061", + "4446257040818138278", + "2950454318280042937", + "13448990765710037875", + "9047611384071292806", + "6815444741608200402", + "16851917226913739706", + "14573110404792781661", + "6345753343264978949", + "11952216291580125278", + "14785772924590964546", + "6669801752484504849", + "17628458195611004436", + "16845938295979325689", + "5421602792290398873", + "18177540326229036216", + "15200904872266137472", + "1228197805364656477", + "9618357061219300901", + "14441611119321898527", + "12397752575825322936", + "15064326886444374105", + "12578040094139254627", + "13980117273335737392", + "3124103884428408787", + "4966061901406379704", + "2941679919340807695", + "15339986315800900532", + "5123378954928494146", + "4585411320310324729", + "1556952170602577307", + "4695655734189378079", + "11831436482741387985", + "13864755018169153859", + "7567464165522311526", + "10652649354298538703", + "17321832326605694883", + "13684450285717229986", + "13981911772325785534" + ], + "float64": [ + 0.07294207591835833, + 0.34708813341576195, + 0.6336940118011583, + 0.5662317259223563, + 0.1379206062398296, + 0.48664764566295626, + 0.666367519387473, + 0.9386514396952557, + 0.9402711805381433, + 0.5201470284844169, + 0.9625722212158612, + 0.550553776012727, + 0.49674752935166755, + 0.15201774093788112, + 0.2748567610922513, + 0.9188250372840968, + 0.11238012317553281, + 0.6394584326589978, + 0.35734823099282276, + 0.8521468431624198, + 0.34091773538506576, + 0.4270363287115435, + 0.4762559810340107, + 0.44297171717719797, + 0.1048083817198684, + 0.8452063813308491, + 0.34362646551468057, + 0.5920777529638047, + 0.08060052369875459, + 0.8093279641586337, + 0.08292251853612165, + 0.8564148043096113, + 0.32251118101914134, + 0.5533197529567478, + 0.2970251958575222, + 0.15729545886669594, + 0.6816586400544454, + 0.10970776939869997, + 0.2791786541570682, + 0.3040377913616721 + ], + "intN": { + "1000": [ + 653, + 966, + 241, + 159, + 729, + 490, + 369, + 913, + 790, + 344, + 647, + 801, + 361, + 955, + 913, + 293, + 985, + 824, + 66, + 521, + 782, + 672, + 816, + 681, + 757, + 169, + 269, + 159, + 831, + 277, + 248, + 84, + 254, + 641, + 751, + 410, + 577, + 939, + 741, + 757 + ], + "15": [ + 9, + 14, + 3, + 2, + 10, + 7, + 5, + 13, + 11, + 5, + 9, + 12, + 5, + 14, + 13, + 4, + 14, + 12, + 0, + 7, + 11, + 10, + 12, + 10, + 11, + 2, + 4, + 2, + 12, + 4, + 3, + 1, + 3, + 9, + 11, + 6, + 8, + 14, + 11, + 11 + ], + "2": [ + 0, + 1, + 0, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 1, + 0, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 0, + 1, + 1, + 0, + 1, + 0, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + 1, + 0, + 0 + ], + "3": [ + 1, + 2, + 0, + 0, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 2, + 2, + 0, + 2, + 2, + 0, + 1, + 2, + 2, + 2, + 2, + 2, + 0, + 0, + 0, + 2, + 0, + 0, + 0, + 0, + 1, + 2, + 1, + 1, + 2, + 2, + 2 + ], + "7": [ + 4, + 6, + 1, + 1, + 5, + 3, + 2, + 6, + 5, + 2, + 4, + 5, + 2, + 6, + 6, + 2, + 6, + 5, + 0, + 3, + 5, + 4, + 5, + 4, + 5, + 1, + 1, + 1, + 5, + 1, + 1, + 0, + 1, + 4, + 5, + 2, + 4, + 6, + 5, + 5 + ] + } + } +] diff --git a/pkg/spec/test/parity-harness.ts b/pkg/spec/test/parity-harness.ts new file mode 100644 index 0000000..fba8f96 --- /dev/null +++ b/pkg/spec/test/parity-harness.ts @@ -0,0 +1,52 @@ +// Shared cross-runtime parity scenario: the FIXED seed, FIXED candidate list, +// and FIXED action root that both the node test (parity.test.ts) and the goja +// test (internal/verifier/parity_test.go) drive. Each side runs the SAME +// pick.ts over the SAME Pcg and asserts the SAME committed golden +// (fixtures/parity-golden.json); matching one golden on both sides proves the +// two engines agree without either invoking the other. +// +// The candidate ORDER and the per-tick PCG draw order are the parity contract. +// The weighted root mixes a tap branch (1 candidate draw) with a typing branch +// (1 candidate draw + 1 corpus draw), so a tick exercises weighted selection, a +// builtin, and the input corpus together; reordering candidates or adding or +// dropping a draw on either side shifts the stream and fails the golden. + +import { Pcg } from "../src/pcg.ts"; +import { nextAction } from "../src/pick.ts"; +import { serializeAction, type SerializedAction } from "../src/runtime-entry.ts"; +import { taps, typing, weighted } from "../src/actions.ts"; +import type { BuiltinVerb, Candidate, GeneratorNode, Host } from "../src/action-tree.ts"; + +export const PARITY_SEED_HI = 0x9e3779b97f4a7c15n; +export const PARITY_STEPS = 20; + +export const PARITY_CANDIDATES: Candidate[] = [ + { x: 50, y: 60, selector: "id:alpha", width: 100, height: 40 }, + { x: 150, y: 160, selector: "id:beta", width: 120, height: 48 }, + { x: 250, y: 260, selector: "id:gamma", width: 80, height: 32 }, +]; + +// A 3:1 weighted split over taps and typing. The stub host returns the same +// candidate list for every verb so the only variables are the draw order and +// the JS engine's number/bigint behavior. +export const PARITY_ROOT: GeneratorNode = weighted([3, taps], [1, typing]); + +const HOST: Host = { + platform: () => "android", + queryCandidates: (_verb: BuiltinVerb) => PARITY_CANDIDATES, + reportUnsupported: () => {}, + seedHi: () => PARITY_SEED_HI, + seedLo: () => 0n, +}; + +// runParity emits the parity scenario's action stream from a fresh Pcg. It +// drives pick.ts directly (not installRuntime, whose globals are locked once) +// so the caller can run it repeatedly to assert determinism. +export function runParity(): (SerializedAction | null)[] { + const rng = new Pcg(PARITY_SEED_HI, 0n); + const stream: (SerializedAction | null)[] = []; + for (let i = 0; i < PARITY_STEPS; i++) { + stream.push(serializeAction(nextAction(PARITY_ROOT, rng, HOST))); + } + return stream; +} diff --git a/pkg/spec/test/parity.test.ts b/pkg/spec/test/parity.test.ts new file mode 100644 index 0000000..cc39620 --- /dev/null +++ b/pkg/spec/test/parity.test.ts @@ -0,0 +1,24 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { test } from "node:test"; + +import { PARITY_STEPS, runParity } from "./parity-harness.ts"; + +const goldenPath = fileURLToPath(new URL("./fixtures/parity-golden.json", import.meta.url)); +const golden = JSON.parse(readFileSync(goldenPath, "utf8")); + +// The node side of the W2 acceptance gate: the shared picker's stream for the +// fixed parity scenario must match the committed golden byte-for-byte. The goja +// test asserts the SAME golden, so a match on both sides proves cross-runtime +// parity. A drift here means pick.ts, the corpus, or the Pcg changed. +test("node picker matches the cross-runtime golden", () => { + const stream = runParity(); + assert.equal(stream.length, PARITY_STEPS); + assert.deepEqual(stream, golden); +}); + +// Determinism: a second run from the same seed yields the identical stream. +test("node picker is deterministic across repeated runs", () => { + assert.deepEqual(runParity(), runParity()); +}); diff --git a/pkg/spec/test/pcg.test.ts b/pkg/spec/test/pcg.test.ts new file mode 100644 index 0000000..c3e8cd3 --- /dev/null +++ b/pkg/spec/test/pcg.test.ts @@ -0,0 +1,58 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { Pcg } from "../src/pcg.ts"; + +const here = dirname(fileURLToPath(import.meta.url)); +const fixturePath = join(here, "fixtures", "pcg-golden.json"); + +interface GoldenCase { + seed: { hi: string; lo: string }; + uint64: string[]; + float64: number[]; + intN: Record; +} + +const cases: GoldenCase[] = JSON.parse(readFileSync(fixturePath, "utf8")); + +function seedLabel(c: GoldenCase): string { + return `seed(${c.seed.hi}, ${c.seed.lo})`; +} + +test("golden fixture has the expected shape", () => { + assert.ok(cases.length >= 5, "expected several seed cases"); + for (const c of cases) { + assert.equal(c.uint64.length, 40); + assert.equal(c.float64.length, 40); + for (const draws of Object.values(c.intN)) { + assert.equal(draws.length, 40); + } + } +}); + +for (const c of cases) { + test(`${seedLabel(c)} uint64 sequence is bit-exact`, () => { + const pcg = new Pcg(BigInt(c.seed.hi), BigInt(c.seed.lo)); + c.uint64.forEach((want, i) => { + assert.equal(pcg.uint64(), BigInt(want), `uint64 draw ${i}`); + }); + }); + + test(`${seedLabel(c)} float64 sequence is bit-exact`, () => { + const pcg = new Pcg(BigInt(c.seed.hi), BigInt(c.seed.lo)); + c.float64.forEach((want, i) => { + assert.equal(pcg.float64(), want, `float64 draw ${i}`); + }); + }); + + for (const [n, expected] of Object.entries(c.intN)) { + test(`${seedLabel(c)} intN(${n}) sequence is bit-exact`, () => { + const pcg = new Pcg(BigInt(c.seed.hi), BigInt(c.seed.lo)); + expected.forEach((want, i) => { + assert.equal(pcg.intN(Number(n)), want, `intN(${n}) draw ${i}`); + }); + }); + } +} diff --git a/pkg/spec/test/pick.test.ts b/pkg/spec/test/pick.test.ts new file mode 100644 index 0000000..dd39683 --- /dev/null +++ b/pkg/spec/test/pick.test.ts @@ -0,0 +1,335 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { Pcg } from "../src/pcg.ts"; +import { nextAction, walk } from "../src/pick.ts"; +import { INPUT_CORPUS, NATIVE_PRESS_KEYS, WEB_PRESS_KEYS } from "../src/corpus.ts"; +import { resetWarnings } from "../src/verbs.ts"; +import type { + ActionDescriptor, + BuiltinVerb, + Candidate, + GeneratorNode, + Host, +} from "../src/action-tree.ts"; + +type Platform = "android" | "ios" | "web"; + +// stubHost returns a fixed candidate list for every verb and records each +// reportUnsupported call so warn-once semantics are observable. +function stubHost( + platform: Platform, + candidates: Candidate[], +): Host & { unsupported: BuiltinVerb[] } { + const unsupported: BuiltinVerb[] = []; + return { + unsupported, + platform: () => platform, + queryCandidates: () => candidates, + reportUnsupported: (verb) => { + unsupported.push(verb); + }, + seedHi: () => 0n, + seedLo: () => 0n, + }; +} + +const POINTS: Candidate[] = [ + { x: 10, y: 20, selector: "id:a", width: 100, height: 200 }, + { x: 30, y: 40, selector: "id:b", width: 100, height: 200 }, + { x: 50, y: 60, selector: "id:c", width: 100, height: 200 }, +]; + +function builtin(verb: BuiltinVerb): GeneratorNode { + return { kind: "builtin", verb }; +} + +test("taps draws one candidate index and targets its point", () => { + resetWarnings(); + const host = stubHost("android", POINTS); + const rng = new Pcg(42n, 0n); + const oracle = new Pcg(42n, 0n); + const expectedIndex = oracle.intN(POINTS.length); + + const action = walk(builtin("taps"), rng, host); + assert.deepEqual(action, { + kind: "Tap", + on: { + x: POINTS[expectedIndex]!.x, + y: POINTS[expectedIndex]!.y, + selector: POINTS[expectedIndex]!.selector, + }, + }); +}); + +test("typing draws candidate index then corpus index, in that order", () => { + resetWarnings(); + const host = stubHost("android", POINTS); + const rng = new Pcg(42n, 0n); + const oracle = new Pcg(42n, 0n); + const candidateIndex = oracle.intN(POINTS.length); + const corpusIndex = oracle.intN(INPUT_CORPUS.length); + + const action = walk(builtin("typing"), rng, host) as ActionDescriptor & { + kind: "InputText"; + }; + assert.equal(action.kind, "InputText"); + assert.deepEqual(action.into, { + x: POINTS[candidateIndex]!.x, + y: POINTS[candidateIndex]!.y, + selector: POINTS[candidateIndex]!.selector, + }); + assert.equal(action.text, INPUT_CORPUS[corpusIndex]); +}); + +test("swipes draw candidate, magnitude (200+intN(401)), then direction", () => { + resetWarnings(); + const host = stubHost("android", POINTS); + const rng = new Pcg(7n, 0n); + const oracle = new Pcg(7n, 0n); + const candidateIndex = oracle.intN(POINTS.length); + const magnitude = 200 + oracle.intN(401); + const direction = oracle.intN(4); + + const from = { x: POINTS[candidateIndex]!.x, y: POINTS[candidateIndex]!.y }; + const expectedTo = { x: from.x, y: from.y }; + switch (direction) { + case 0: + expectedTo.y = Math.max(0, from.y - magnitude); + break; + case 1: + expectedTo.y = Math.max(0, from.y + magnitude); + break; + case 2: + expectedTo.x = Math.max(0, from.x - magnitude); + break; + case 3: + expectedTo.x = Math.max(0, from.x + magnitude); + break; + } + + const action = walk(builtin("swipes"), rng, host) as ActionDescriptor & { + kind: "Swipe"; + }; + assert.equal(action.kind, "Swipe"); + assert.deepEqual(action.from, from); + assert.deepEqual(action.to, expectedTo); + assert.equal(action.durationMillis, 250); +}); + +test("scrolls draw candidate index then direction", () => { + resetWarnings(); + const host = stubHost("android", POINTS); + const rng = new Pcg(99n, 0n); + const oracle = new Pcg(99n, 0n); + const candidateIndex = oracle.intN(POINTS.length); + const directionIndex = oracle.intN(4); + const directions = ["up", "down", "left", "right"] as const; + + const action = walk(builtin("scrolls"), rng, host) as ActionDescriptor & { + kind: "Scroll"; + }; + assert.equal(action.kind, "Scroll"); + assert.equal(action.direction, directions[directionIndex]); + assert.deepEqual(action.in, { + x: POINTS[candidateIndex]!.x, + y: POINTS[candidateIndex]!.y, + }); +}); + +test("doubleTaps and longPresses draw exactly one candidate index", () => { + resetWarnings(); + for (const verb of ["doubleTaps", "longPresses"] as const) { + const host = stubHost("android", POINTS); + const rng = new Pcg(123n, 0n); + const oracle = new Pcg(123n, 0n); + const index = oracle.intN(POINTS.length); + const action = walk(builtin(verb), rng, host); + const kind = verb === "doubleTaps" ? "DoubleTap" : "LongPress"; + assert.deepEqual(action, { + kind, + on: { + x: POINTS[index]!.x, + y: POINTS[index]!.y, + selector: POINTS[index]!.selector, + }, + }); + } +}); + +test("waitOnce emits a 500ms wait and draws nothing", () => { + resetWarnings(); + const host = stubHost("android", POINTS); + const rng = new Pcg(42n, 0n); + const before = rng.float64(); + const after = rng.float64(); + + const fresh = new Pcg(42n, 0n); + fresh.float64(); + const action = walk(builtin("waitOnce"), fresh, host); + const afterWait = fresh.float64(); + + assert.deepEqual(action, { kind: "Wait", durationMillis: 500 }); + // The wait consumed no draw, so the stream is identical with or without it. + assert.equal(afterWait, after); + assert.notEqual(before, after); +}); + +test("pressKeys on native draws from NATIVE_PRESS_KEYS", () => { + resetWarnings(); + const host = stubHost("android", POINTS); + const rng = new Pcg(42n, 0n); + const oracle = new Pcg(42n, 0n); + const index = oracle.intN(NATIVE_PRESS_KEYS.length); + const action = walk(builtin("pressKeys"), rng, host); + assert.deepEqual(action, { kind: "PressKey", key: NATIVE_PRESS_KEYS[index] }); +}); + +test("pressKeys on web draws from WEB_PRESS_KEYS", () => { + resetWarnings(); + const host = stubHost("web", POINTS); + const rng = new Pcg(2024n, 0n); + const oracle = new Pcg(2024n, 0n); + const index = oracle.intN(WEB_PRESS_KEYS.length); + const action = walk(builtin("pressKeys"), rng, host); + assert.deepEqual(action, { kind: "PressKey", key: WEB_PRESS_KEYS[index] }); +}); + +test("empty candidate list yields null without drawing", () => { + resetWarnings(); + const host = stubHost("android", []); + const rng = new Pcg(42n, 0n); + assert.equal(walk(builtin("taps"), rng, host), null); + // No draw consumed: next float64 matches a fresh stream's first draw. + assert.equal(rng.float64(), new Pcg(42n, 0n).float64()); +}); + +test("weighted: one float64 draw, ascending cumulative scan", () => { + resetWarnings(); + const host = stubHost("android", POINTS); + const tap: GeneratorNode = builtin("taps"); + const wait: GeneratorNode = builtin("waitOnce"); + // total = 3; draw = float64()*3 decides the branch by ascending cumulative. + const node: GeneratorNode = { + kind: "weighted", + branches: [ + [1, tap], + [2, wait], + ], + }; + + const rng = new Pcg(42n, 0n); + const oracle = new Pcg(42n, 0n); + const draw = oracle.float64() * 3; + const expectFirst = draw < 1; // cumulative after branch 0 is 1. + + const action = walk(node, rng, host); + if (expectFirst) { + assert.equal((action as ActionDescriptor).kind, "Tap"); + } else { + assert.deepEqual(action, { kind: "Wait", durationMillis: 500 }); + } +}); + +test("weighted: a zero-weight branch is never selected", () => { + resetWarnings(); + const host = stubHost("android", POINTS); + const poison: GeneratorNode = { + kind: "actions", + generate: () => { + throw new Error("zero-weight branch must never be walked"); + }, + }; + const node: GeneratorNode = { + kind: "weighted", + branches: [ + [0, poison], + [1, builtin("waitOnce")], + ], + }; + // Drive many seeds: the zero-weight branch must never be entered. + for (let seed = 1; seed <= 200; seed++) { + const action = walk(node, new Pcg(BigInt(seed), 0n), host); + assert.deepEqual(action, { kind: "Wait", durationMillis: 500 }); + } +}); + +test("weighted: all-zero (or negative) weights return null", () => { + resetWarnings(); + const host = stubHost("android", POINTS); + const node: GeneratorNode = { + kind: "weighted", + branches: [ + [0, builtin("waitOnce")], + [-5, builtin("taps")], + ], + }; + assert.equal(walk(node, new Pcg(42n, 0n), host), null); +}); + +test("actions: single-element list draws nothing", () => { + resetWarnings(); + const only: ActionDescriptor = { kind: "Wait", durationMillis: 10 }; + const node: GeneratorNode = { kind: "actions", generate: () => [only] }; + const rng = new Pcg(42n, 0n); + const action = walk(node, rng, stubHost("android", POINTS)); + assert.deepEqual(action, only); + // No draw consumed for a singleton list. + assert.equal(rng.float64(), new Pcg(42n, 0n).float64()); +}); + +test("actions: multi-element list draws one intN(len) index", () => { + resetWarnings(); + const list: ActionDescriptor[] = [ + { kind: "Wait", durationMillis: 1 }, + { kind: "Wait", durationMillis: 2 }, + { kind: "Wait", durationMillis: 3 }, + ]; + const node: GeneratorNode = { kind: "actions", generate: () => [...list] }; + const rng = new Pcg(42n, 0n); + const oracle = new Pcg(42n, 0n); + const index = oracle.intN(list.length); + assert.deepEqual(walk(node, rng, stubHost("android", POINTS)), list[index]); +}); + +test("actions: empty list yields null", () => { + resetWarnings(); + const node: GeneratorNode = { kind: "actions", generate: () => [] }; + assert.equal(walk(node, new Pcg(42n, 0n), stubHost("android", POINTS)), null); +}); + +test("corpus reproducibility: same seed yields the same typed text", () => { + resetWarnings(); + const host = stubHost("android", POINTS); + const first = walk(builtin("typing"), new Pcg(2024n, 0n), host) as ActionDescriptor & { + kind: "InputText"; + }; + const second = walk(builtin("typing"), new Pcg(2024n, 0n), host) as ActionDescriptor & { + kind: "InputText"; + }; + assert.equal(first.text, second.text); + assert.ok(INPUT_CORPUS.includes(first.text)); +}); + +test("nextAction retries up to 16 times then returns null", () => { + resetWarnings(); + // An empty weighted tree always returns null; nextAction must give up, not loop. + const empty: GeneratorNode = { kind: "weighted", branches: [] }; + assert.equal(nextAction(empty, new Pcg(42n, 0n), stubHost("android", POINTS)), null); +}); + +test("nextAction returns the first non-null walk result", () => { + resetWarnings(); + let calls = 0; + // First two walks yield null (empty list), third yields an action. nextAction + // should return on the third attempt. + const node: GeneratorNode = { + kind: "actions", + generate: () => { + calls++; + return calls >= 3 ? [{ kind: "Wait", durationMillis: 5 }] : []; + }, + }; + const action = nextAction(node, new Pcg(42n, 0n), stubHost("android", POINTS)); + assert.deepEqual(action, { kind: "Wait", durationMillis: 5 }); + assert.equal(calls, 3); +}); diff --git a/pkg/spec/test/runtime-entry.test.ts b/pkg/spec/test/runtime-entry.test.ts new file mode 100644 index 0000000..d80ca2c --- /dev/null +++ b/pkg/spec/test/runtime-entry.test.ts @@ -0,0 +1,181 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { serializeAction } from "../src/runtime-entry.ts"; +import type { ActionDescriptor } from "../src/action-tree.ts"; + +// decodeWire mirrors the Go decodeAction reading the unified flat contract: a +// field rename (fromX vs from_x) would make these assertions fail here, the +// same way it would silently no-op web actions in production. +const POINT = { x: 12, y: 34 }; + +test("Tap serializes to the flat point contract", () => { + assert.deepEqual(serializeAction({ kind: "Tap", on: POINT }), { + kind: "Tap", + x: 12, + y: 34, + }); +}); + +test("Tap carries a resolved selector when present", () => { + const target = { x: 1, y: 2, selector: "id:save" }; + assert.deepEqual(serializeAction({ kind: "Tap", on: target }), { + kind: "Tap", + x: 1, + y: 2, + selector: "id:save", + }); +}); + +test("DoubleTap and LongPress share the point contract", () => { + assert.deepEqual(serializeAction({ kind: "DoubleTap", on: POINT }), { + kind: "DoubleTap", + x: 12, + y: 34, + }); + assert.deepEqual(serializeAction({ kind: "LongPress", on: POINT }), { + kind: "LongPress", + x: 12, + y: 34, + }); +}); + +test("InputText carries x, y and text", () => { + assert.deepEqual(serializeAction({ kind: "InputText", into: POINT, text: "hi" }), { + kind: "InputText", + x: 12, + y: 34, + text: "hi", + }); +}); + +test("Swipe emits camelCase fromX/fromY/toX/toY/durationMillis", () => { + const action: ActionDescriptor = { + kind: "Swipe", + from: { x: 5, y: 6 }, + to: { x: 7, y: 8 }, + durationMillis: 300, + }; + assert.deepEqual(serializeAction(action), { + kind: "Swipe", + fromX: 5, + fromY: 6, + toX: 7, + toY: 8, + durationMillis: 300, + }); +}); + +test("Swipe defaults durationMillis to 250", () => { + const action = serializeAction({ + kind: "Swipe", + from: { x: 0, y: 0 }, + to: { x: 0, y: 1 }, + }) as { durationMillis: number }; + assert.equal(action.durationMillis, 250); +}); + +test("Scroll emits direction plus from/to point and duration", () => { + assert.deepEqual(serializeAction({ kind: "Scroll", direction: "down", in: POINT }), { + kind: "Scroll", + direction: "down", + fromX: 12, + fromY: 34, + toX: 12, + toY: 34, + durationMillis: 250, + }); +}); + +test("PressKey and Wait pass through their fields", () => { + assert.deepEqual(serializeAction({ kind: "PressKey", key: "enter" }), { + kind: "PressKey", + key: "enter", + }); + assert.deepEqual(serializeAction({ kind: "Wait", durationMillis: 500 }), { + kind: "Wait", + durationMillis: 500, + }); +}); + +test("builtin Scroll carries the pre-computed to endpoint", () => { + assert.deepEqual( + serializeAction({ + kind: "Scroll", + direction: "down", + in: { x: 100, y: 200 }, + from: { x: 100, y: 200 }, + to: { x: 100, y: 120 }, + }), + { + kind: "Scroll", + direction: "down", + fromX: 100, + fromY: 200, + toX: 100, + toY: 120, + durationMillis: 250, + }, + ); +}); + +test("a string target serializes selector-only for the runner to re-resolve", () => { + assert.deepEqual(serializeAction({ kind: "Tap", on: "id:save" }), { + kind: "Tap", + x: 0, + y: 0, + selector: "id:save", + }); +}); + +test("an empty target or null action is dropped", () => { + assert.equal(serializeAction({ kind: "Tap", on: "" }), null); + assert.equal(serializeAction(null), null); +}); + +// Wire round-trip: serialize -> JSON -> parse asserts every field the Go +// decodeAction reads survives by its exact camelCase name. A rename (fromX vs +// from_x) would silently turn web/native actions into no-ops; this catches it. +test("serialized actions JSON round-trip with the decoder's field names", () => { + const cases: ActionDescriptor[] = [ + { kind: "Tap", on: { x: 1, y: 2, selector: "id:a" } as never }, + { kind: "InputText", into: { x: 3, y: 4 } as never, text: "x" }, + { kind: "Swipe", from: { x: 5, y: 6 }, to: { x: 7, y: 8 }, durationMillis: 9 }, + { kind: "Scroll", direction: "up", in: { x: 1, y: 1 } as never }, + { kind: "PressKey", key: "enter" }, + { kind: "Wait", durationMillis: 10 }, + ]; + for (const descriptor of cases) { + const wire = serializeAction(descriptor); + assert.ok(wire, `expected ${descriptor.kind} to serialize`); + const decoded = JSON.parse(JSON.stringify(wire)) as Record; + assert.equal(decoded.kind, wire!.kind); + switch (wire!.kind) { + case "Tap": + assert.equal(typeof decoded.x, "number"); + assert.equal(typeof decoded.y, "number"); + assert.equal(decoded.selector, "id:a"); + break; + case "InputText": + assert.equal(typeof decoded.x, "number"); + assert.equal(decoded.text, "x"); + break; + case "Swipe": + assert.equal(decoded.fromX, 5); + assert.equal(decoded.toY, 8); + assert.equal(decoded.durationMillis, 9); + break; + case "Scroll": + assert.equal(decoded.direction, "up"); + assert.equal(typeof decoded.fromX, "number"); + assert.equal(typeof decoded.toX, "number"); + break; + case "PressKey": + assert.equal(decoded.key, "enter"); + break; + case "Wait": + assert.equal(decoded.durationMillis, 10); + break; + } + } +}); diff --git a/pkg/spec/test/values.test.ts b/pkg/spec/test/values.test.ts new file mode 100644 index 0000000..153c620 --- /dev/null +++ b/pkg/spec/test/values.test.ts @@ -0,0 +1,75 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { Pcg } from "../src/pcg.ts"; +import { setSamplerRng } from "../src/sampler-rng.ts"; +import { edgeCaseText, emails, integers, strings } from "../src/values.ts"; +import { INPUT_CORPUS } from "../src/corpus.ts"; +import type { Sampler } from "../src/types.ts"; + +function withRng(seed: bigint, body: () => T): T { + setSamplerRng(new Pcg(seed, 0n)); + try { + return body(); + } finally { + setSamplerRng(null); + } +} + +test("same seed yields an identical draw sequence", () => { + const drawAll = () => [ + strings().generate(), + integers().between(0, 999).generate(), + emails().domain("folio.app").generate(), + edgeCaseText().generate(), + ]; + assert.deepEqual(withRng(7n, drawAll), withRng(7n, drawAll)); +}); + +test("a different seed yields a different sequence", () => { + const drawAll = () => [strings().generate(), integers().generate()]; + assert.notDeepEqual(withRng(1n, drawAll), withRng(2n, drawAll)); +}); + +test("integers().between constrains the output range", () => { + withRng(42n, () => { + for (let i = 0; i < 200; i++) { + const value = integers().between(5, 9).generate(); + assert.ok(Number.isInteger(value)); + assert.ok(value >= 5 && value <= 9, `out of range: ${value}`); + } + }); +}); + +test("strings().length(min, max).alpha() constrains length and charset", () => { + withRng(99n, () => { + for (let i = 0; i < 200; i++) { + const value = strings().length(3, 6).alpha().generate(); + assert.ok(value.length >= 3 && value.length <= 6, `bad length: ${value}`); + assert.match(value, /^[A-Za-z]*$/); + } + }); +}); + +test("emails().domain sets the host and edgeCaseText draws from the corpus", () => { + withRng(3n, () => { + const email = emails().domain("folio.app").generate(); + assert.match(email, /^[A-Za-z]+@folio\.app$/); + assert.ok(INPUT_CORPUS.includes(edgeCaseText().generate())); + }); +}); + +test("a builder is itself a Sampler", () => { + withRng(11n, () => { + const sampler: Sampler = integers().between(0, 9); + const value = sampler.generate(); + assert.ok(value >= 0 && value <= 9); + }); +}); + +test("outside a walk generators return a fixed deterministic default", () => { + setSamplerRng(null); + assert.equal(integers().between(10, 20).generate(), 10); + assert.equal(strings().length(4, 8).alpha().generate(), "aaaa"); + assert.equal(emails().domain("folio.app").generate(), "aaa@folio.app"); + assert.equal(edgeCaseText().generate(), INPUT_CORPUS[0]); +}); diff --git a/pkg/spec/test/verbs.test.ts b/pkg/spec/test/verbs.test.ts new file mode 100644 index 0000000..9187a6c --- /dev/null +++ b/pkg/spec/test/verbs.test.ts @@ -0,0 +1,72 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { supports, warnUnsupportedOnce, resetWarnings } from "../src/verbs.ts"; +import type { BuiltinVerb, Host } from "../src/action-tree.ts"; + +const ALL_VERBS: BuiltinVerb[] = [ + "taps", + "doubleTaps", + "longPresses", + "scrolls", + "typing", + "swipes", + "waitOnce", + "pressKeys", +]; + +function countingHost(platform: "android" | "ios" | "web"): Host & { calls: BuiltinVerb[] } { + const calls: BuiltinVerb[] = []; + return { + calls, + platform: () => platform, + queryCandidates: () => [], + reportUnsupported: (verb) => { + calls.push(verb); + }, + seedHi: () => 0n, + seedLo: () => 0n, + }; +} + +test("every verb is supported on native platforms", () => { + for (const verb of ALL_VERBS) { + assert.equal(supports(verb, "android"), true, `android ${verb}`); + assert.equal(supports(verb, "ios"), true, `ios ${verb}`); + } +}); + +test("web supports the full builtin verb set (declared, not silently no-op)", () => { + for (const verb of ALL_VERBS) { + assert.equal(supports(verb, "web"), true, `web ${verb}`); + } +}); + +test("warn-once reports a verb exactly once per platform", () => { + resetWarnings(); + const host = countingHost("web"); + assert.equal(warnUnsupportedOnce(host, "swipes"), true); + assert.equal(warnUnsupportedOnce(host, "swipes"), false); + assert.equal(warnUnsupportedOnce(host, "swipes"), false); + assert.deepEqual(host.calls, ["swipes"]); +}); + +test("warn-once keys by verb@platform independently", () => { + resetWarnings(); + const web = countingHost("web"); + const android = countingHost("android"); + warnUnsupportedOnce(web, "scrolls"); + warnUnsupportedOnce(android, "scrolls"); + warnUnsupportedOnce(web, "scrolls"); + // Same verb on two platforms: one report each, suppressed on repeat. + assert.deepEqual(web.calls, ["scrolls"]); + assert.deepEqual(android.calls, ["scrolls"]); +}); + +test("resetWarnings re-arms reporting", () => { + resetWarnings(); + const host = countingHost("web"); + warnUnsupportedOnce(host, "longPresses"); + resetWarnings(); + warnUnsupportedOnce(host, "longPresses"); + assert.deepEqual(host.calls, ["longPresses", "longPresses"]); +}); diff --git a/pkg/spec/test/web-runtime.test.ts b/pkg/spec/test/web-runtime.test.ts new file mode 100644 index 0000000..2ee1e3b --- /dev/null +++ b/pkg/spec/test/web-runtime.test.ts @@ -0,0 +1,185 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +// The web runtime is the WEB Host: it parses the injected seed, reports its +// platform, and delegates action generation to the shared picker. These tests +// guard the Host surface and the seed-precision contract. DOM candidate +// enumeration is exercised against a minimal querySelectorAll stub. + +// A 64-bit seed that loses precision as a JS Number must survive as a BigInt. +process.env.SANDERLING_SEED = "9007199254740993"; + +const { __testing__ } = await import("../src/web-runtime.ts"); +const { host } = __testing__; + +test("platform is web", () => { + assert.equal(host.platform(), "web"); +}); + +test("seedHi parses the injected 64-bit seed without Number precision loss", () => { + assert.equal(host.seedHi(), 9007199254740993n); +}); + +test("seedLo is 0 to match goja rand.NewPCG(seed, 0)", () => { + assert.equal(host.seedLo(), 0n); +}); + +test("reportUnsupported warns once via console.warn", () => { + const original = console.warn; + const messages: string[] = []; + console.warn = (m: string) => messages.push(m); + try { + host.reportUnsupported("swipes"); + } finally { + console.warn = original; + } + assert.equal(messages.length, 1); + assert.match(messages[0]!, /swipes/); +}); + +// installRuntime locked the next-action and extractor globals at import. +test("installRuntime defined the host-invoked globals", () => { + const g = globalThis as Record; + assert.equal(typeof g.__sanderlingNextAction__, "function"); + assert.equal(typeof g.__sanderlingExtractors__, "function"); + assert.equal(typeof g.__sanderling__, "object"); +}); + +// A button and a text input, each with a deterministic bounding box, exercise +// the per-verb selector routing without a full DOM. +function fakeElement(tag: string, rect: { x: number; y: number; w: number; h: number }) { + return { + tagName: tag.toUpperCase(), + disabled: false, + isContentEditable: false, + type: tag === "input" ? "text" : "", + scrollHeight: 0, + clientHeight: 0, + scrollWidth: 0, + clientWidth: 0, + getBoundingClientRect: () => ({ + left: rect.x, + top: rect.y, + width: rect.w, + height: rect.h, + right: rect.x + rect.w, + bottom: rect.y + rect.h, + }), + }; +} + +function withFakeDocument(map: Record, run: () => void) { + const g = globalThis as Record; + const original = g.document; + g.document = { + querySelectorAll: (selector: string) => map[selector] ?? [], + scrollingElement: null, + documentElement: null, + }; + try { + run(); + } finally { + g.document = original; + } +} + +test("queryCandidates routes taps to the tappable selector set", () => { + const button = fakeElement("button", { x: 10, y: 20, w: 40, h: 8 }); + withFakeDocument( + { 'a, button, input, select, textarea, [role="button"], [onclick]': [button] }, + () => { + __testing__.resetCandidateCache(); + const candidates = host.queryCandidates("taps"); + assert.equal(candidates.length, 1); + assert.deepEqual({ x: candidates[0]!.x, y: candidates[0]!.y }, { x: 30, y: 24 }); + }, + ); +}); + +test("queryCandidates routes typing to editable inputs only", () => { + const input = fakeElement("input", { x: 0, y: 0, w: 100, h: 20 }); + withFakeDocument({ "input, textarea, [contenteditable]": [input] }, () => { + __testing__.resetCandidateCache(); + const candidates = host.queryCandidates("typing"); + assert.equal(candidates.length, 1); + assert.deepEqual({ x: candidates[0]!.x, y: candidates[0]!.y }, { x: 50, y: 10 }); + }); +}); + +test("queryCandidates caches within a tick until reset", () => { + const first = fakeElement("button", { x: 0, y: 0, w: 10, h: 10 }); + withFakeDocument( + { 'a, button, input, select, textarea, [role="button"], [onclick]': [first] }, + () => { + __testing__.resetCandidateCache(); + const a = host.queryCandidates("taps"); + const b = host.queryCandidates("taps"); + assert.equal(a, b); + }, + ); +}); + +// evaluateExtractors builds State, which references document and window. +const emptyDocument = { + querySelector: () => null, + querySelectorAll: () => [], +}; + +function withState(run: () => void) { + const g = globalThis as Record; + const originalDocument = g.document; + const originalWindow = g.window; + g.document = emptyDocument; + g.window = {}; + try { + run(); + } finally { + g.document = originalDocument; + g.window = originalWindow; + } +} + +test("named() sets the extractor's display name", () => { + const handle = __testing__.runtime.extract(() => "home").named("route"); + const entry = __testing__.extractors.find((e) => e.handle === handle); + assert.equal(entry?.name, "route"); +}); + +test("reading another extractor's current inside a getter throws", () => { + const first = __testing__.runtime.extract(() => 1); + let caught: Error | undefined; + __testing__.runtime.extract(() => { + try { + return first.current; + } catch (error) { + caught = error as Error; + return undefined; + } + }); + withState(() => __testing__.evaluateExtractors()); + assert.match( + caught?.message ?? "", + /inside another extractor is not allowed/, + ); +}); + +// An uncaught cross-extractor read must abort evaluateExtractors loudly, exactly +// like goja's PushSnapshot. If the getter throw were swallowed, web would +// silently yield undefined and the guard would be a no-op for real authors. +test("an uncaught cross-extractor read aborts evaluateExtractors", () => { + __testing__.extractors.length = 0; + const first = __testing__.runtime.extract(() => 1); + __testing__.runtime.extract(() => first.previous); + let thrown: Error | undefined; + withState(() => { + try { + __testing__.evaluateExtractors(); + } catch (error) { + thrown = error as Error; + } + }); + assert.match( + thrown?.message ?? "", + /inside another extractor is not allowed/, + ); +}); diff --git a/proto/driverpb/driver_test.go b/proto/driverpb/driver_test.go index 294e61f..860500b 100644 --- a/proto/driverpb/driver_test.go +++ b/proto/driverpb/driver_test.go @@ -44,25 +44,3 @@ func TestDriverServiceDescriptor(t *testing.T) { } } } - -func TestMessageFields(t *testing.T) { - lr := &LaunchRequest{BundleId: "com.example", ClearState: true} - if lr.GetBundleId() != "com.example" || !lr.GetClearState() { - t.Fatalf("LaunchRequest field round-trip failed: %+v", lr) - } - - tap := &Point{X: 10, Y: 20} - if tap.GetX() != 10 || tap.GetY() != 20 { - t.Fatalf("Point field round-trip failed: %+v", tap) - } - - img := &Image{Png: []byte{1, 2, 3}, Width: 100, Height: 200} - if len(img.GetPng()) != 3 || img.GetWidth() != 100 || img.GetHeight() != 200 { - t.Fatalf("Image field round-trip failed: %+v", img) - } - - hs := &HealthStatus{Ready: true, Version: "0.0.1", Platform: "android"} - if !hs.GetReady() || hs.GetVersion() != "0.0.1" || hs.GetPlatform() != "android" { - t.Fatalf("HealthStatus field round-trip failed: %+v", hs) - } -} diff --git a/test/browser/browser_test.go b/test/browser/browser_test.go new file mode 100644 index 0000000..5fba473 --- /dev/null +++ b/test/browser/browser_test.go @@ -0,0 +1,182 @@ +//go:build browser + +// Package browser_test drives small self-contained web fixtures through the real +// bundle -> run -> verify pipeline against headless Chrome, proving the web path +// surfaces a known violation by property name and leaves a correct page clean. +// +// The suite is gated behind the `browser` build tag so the default +// `go test ./...` (which has no Chrome dependency) never runs it. Invoke it with +// `go test -tags browser ./test/browser/...` or `make test-browser`. +package browser_test + +import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" + "runtime" + "slices" + "strconv" + "testing" + "time" + + "github.com/priyanshujain/sanderling/internal/bundler" + "github.com/priyanshujain/sanderling/internal/driver" + "github.com/priyanshujain/sanderling/internal/driver/chrome" + chromerunner "github.com/priyanshujain/sanderling/internal/runner" + "github.com/priyanshujain/sanderling/internal/trace" + "github.com/priyanshujain/sanderling/internal/verifier" +) + +const ( + fixtureSeed = 1 + fixtureMaxSteps = 12 +) + +// TestBrowserUncaughtExceptionSurfaces drives a page whose button throws an +// uncaught Error on the third click and asserts the default noUncaughtExceptions +// property fires within the step budget. +func TestBrowserUncaughtExceptionSurfaces(t *testing.T) { + violations := runFixture(t, "throwing") + if !slices.Contains(violations, "noUncaughtExceptions") { + t.Fatalf("noUncaughtExceptions did not fire; violations=%v", violations) + } +} + +// TestBrowserCounterInvariantHolds drives a correct counter page whose buttons +// move the value by exactly one and asserts the changesByAtMostOne property +// never fires. +func TestBrowserCounterInvariantHolds(t *testing.T) { + violations := runFixture(t, "counter") + if slices.Contains(violations, "changesByAtMostOne") { + t.Fatalf("changesByAtMostOne fired on a correct page; violations=%v", violations) + } +} + +// runFixture serves the named testdata case over an in-process file server, +// drives it through headless Chrome with a fixed seed and a bounded step count, +// and returns every property name that was ever reported violated. +func runFixture(t *testing.T, name string) []string { + t.Helper() + + server := httptest.NewServer(http.FileServer(http.Dir(testdataDir(t)))) + t.Cleanup(server.Close) + url := server.URL + "/" + name + "/" + + specPath := filepath.Join(testdataDir(t), name, "spec.ts") + gojaBundle, webBundle := bundleSpec(t, specPath) + + driverInstance := chrome.New() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = driverInstance.Terminate(ctx) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + if err := driverInstance.Launch(ctx, url, false, nil); err != nil { + t.Fatalf("launch %s: %v", url, err) + } + web, ok := any(driverInstance).(driver.WebDriver) + if !ok { + t.Fatal("chrome driver does not satisfy WebDriver") + } + if err := web.InstallBundle(ctx, webBundle); err != nil { + t.Fatalf("install web bundle: %v", err) + } + + verifierInstance, err := verifier.New( + verifier.WithSeed(fixtureSeed), + verifier.WithPlatform("web"), + ) + if err != nil { + t.Fatalf("verifier: %v", err) + } + if err := verifierInstance.Load(string(gojaBundle)); err != nil { + t.Fatalf("load spec: %v", err) + } + + traceWriter, err := trace.NewWriter(t.TempDir()) + if err != nil { + t.Fatalf("trace writer: %v", err) + } + t.Cleanup(func() { _ = traceWriter.Close() }) + + summary, err := chromerunner.Run(ctx, chromerunner.Options{ + Duration: 90 * time.Second, + IdleTimeout: time.Second, + MaxSteps: fixtureMaxSteps, + Driver: driverInstance, + Verifier: verifierInstance, + TraceWriter: traceWriter, + }) + if err != nil { + t.Fatalf("run: %v", err) + } + + var violated []string + for _, record := range summary.Violations { + violated = append(violated, record.Properties...) + } + return violated +} + +// bundleSpec compiles the fixture spec for both runtimes: the goja bundle the +// host verifier loads and the web bundle injected into the page. Aliases and +// runtime-entry paths are resolved relative to the repo's pkg/spec checkout. +func bundleSpec(t *testing.T, specPath string) (goja, web []byte) { + t.Helper() + specSrc := specSrcDir(t) + aliases := map[string]string{ + "@sanderling/spec": filepath.Join(specSrc, "index.ts"), + "@sanderling/spec/defaults": filepath.Join(specSrc, "defaults/index.ts"), + "@sanderling/spec/defaults/properties": filepath.Join(specSrc, "defaults/properties.ts"), + } + defines := map[string]string{"SANDERLING_SEED": strconv.Itoa(fixtureSeed)} + + gojaResult, err := bundler.Bundle(bundler.Options{ + EntryFile: specPath, + RuntimeFile: filepath.Join(specSrc, "goja-runtime.ts"), + Defines: defines, + Aliases: aliases, + }) + if err != nil { + t.Fatalf("bundle goja spec: %v", err) + } + webResult, err := bundler.BundleWeb(bundler.WebOptions{ + EntryFile: specPath, + WebRuntimeFile: filepath.Join(specSrc, "web-runtime.ts"), + Defines: defines, + Aliases: aliases, + }) + if err != nil { + t.Fatalf("bundle web spec: %v", err) + } + return gojaResult.JavaScript, webResult.JavaScript +} + +// repoRoot walks up from this test file to the module root (the directory +// holding go.mod), so paths resolve regardless of the working directory. +func repoRoot(t *testing.T) string { + t.Helper() + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("cannot resolve caller path") + } + return filepath.Clean(filepath.Join(filepath.Dir(thisFile), "..", "..")) +} + +func testdataDir(t *testing.T) string { + t.Helper() + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("cannot resolve caller path") + } + return filepath.Join(filepath.Dir(thisFile), "testdata") +} + +func specSrcDir(t *testing.T) string { + return filepath.Join(repoRoot(t), "pkg", "spec", "src") +} diff --git a/test/browser/testdata/counter/index.html b/test/browser/testdata/counter/index.html new file mode 100644 index 0000000..400f7b1 --- /dev/null +++ b/test/browser/testdata/counter/index.html @@ -0,0 +1,27 @@ + + + + + counter + + + +
0
+ + + + diff --git a/test/browser/testdata/counter/spec.ts b/test/browser/testdata/counter/spec.ts new file mode 100644 index 0000000..64cdeb4 --- /dev/null +++ b/test/browser/testdata/counter/spec.ts @@ -0,0 +1,18 @@ +import { always, extract, taps } from "@sanderling/spec"; + +const counter = extract((s) => { + const el = s.ax.find({ id: "counter" }); + return el ? parseInt(el.text, 10) || 0 : 0; +}).named("counter"); + +// The + and - buttons each move the counter by exactly 1, so no step ever +// changes it by more than 1. A bug that double-applied a click would break this. +const changesByAtMostOne = always(() => { + const previous = counter.previous; + if (previous === undefined) return true; + return Math.abs(counter.current - previous) <= 1; +}); + +export const properties = { changesByAtMostOne }; + +export const actionsRoot = taps; diff --git a/test/browser/testdata/throwing/index.html b/test/browser/testdata/throwing/index.html new file mode 100644 index 0000000..4da9eba --- /dev/null +++ b/test/browser/testdata/throwing/index.html @@ -0,0 +1,21 @@ + + + + + throwing + + + +
0
+ + + diff --git a/test/browser/testdata/throwing/spec.ts b/test/browser/testdata/throwing/spec.ts new file mode 100644 index 0000000..4803b81 --- /dev/null +++ b/test/browser/testdata/throwing/spec.ts @@ -0,0 +1,6 @@ +import { taps } from "@sanderling/spec"; +import { noUncaughtExceptions } from "@sanderling/spec/defaults/properties"; + +export const properties = { noUncaughtExceptions }; + +export const actionsRoot = taps;