From 53245fc589da09d804177bccbf5c926134aa17c7 Mon Sep 17 00:00:00 2001 From: PJ Date: Sat, 15 Aug 2026 13:33:49 +0530 Subject: [PATCH 1/8] fix(verifier): report whether the last action was confirmed applied Both hosts get applied: true when the runner saw the dispatch succeed and applied: null when it could not, so an unconfirmed action stops arriving at the spec as no action at all. --- internal/verifier/marshal.go | 16 ++++++++++++++-- internal/verifier/types.go | 5 +++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/internal/verifier/marshal.go b/internal/verifier/marshal.go index f9604b5..212f4a3 100644 --- a/internal/verifier/marshal.go +++ b/internal/verifier/marshal.go @@ -344,7 +344,19 @@ func lastActionFields(action *Action) []actionField { point := func(x, y int) []actionField { return []actionField{{key: "x", value: x}, {key: "y", value: y}} } - fields := []actionField{{key: "kind", value: string(action.Kind)}} + // An action whose apply call failed is not an action that did not happen: + // the dispatch may have landed before the error. That is unknown, and + // unknown is null here for the same reason every other absence in the spec + // surface is, so a property decides for itself instead of being handed a + // "nothing happened" the runner cannot vouch for. + var applied any + if action.Applied { + applied = true + } + fields := []actionField{ + {key: "kind", value: string(action.Kind)}, + {key: "applied", value: applied}, + } if action.On != "" { fields = append(fields, actionField{key: "on", value: action.On}) } @@ -397,7 +409,7 @@ func objectFromFields(runtime *goja.Runtime, fields []actionField) *goja.Object // has no Go-side state object to read: the runner pushes this JSON into the // page before each extractor evaluation. A nil action encodes as JSON null, // the same value the goja host reports on the first step of a run and after a -// step whose action was never applied. +// step whose action was never dispatched. func EncodeLastAction(action *Action) json.RawMessage { if action == nil { return json.RawMessage("null") diff --git a/internal/verifier/types.go b/internal/verifier/types.go index 5b35eb7..125703b 100644 --- a/internal/verifier/types.go +++ b/internal/verifier/types.go @@ -33,6 +33,11 @@ type Action struct { // Direction is the scroll direction for ActionKindScroll: one of "up", // "down", "left", "right". Empty for every other kind. Direction string + // Applied is meaningful only on the action a step reports to the spec as + // state.lastAction: true when the runner saw the dispatch succeed, false + // when the apply call failed and nothing can say whether the action + // reached the app. The spec is told which of the two it is. + Applied bool } // LogEntry mirrors a logcat line captured between steps. From 6e52fa0837af87a499d4202771aa9b0507dfc25c Mon Sep 17 00:00:00 2001 From: PJ Date: Sat, 15 Aug 2026 13:33:55 +0530 Subject: [PATCH 2/8] fix(runner): an apply error leaves the action's fate unknown, not undone A deadline that fires after the tap was dispatched leaves the effect committed. Reporting nil made the spec see an effect with no action to cause it, which is how the counting property convicts a healthy app. --- internal/runner/runner.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index d730556..1ef8d5e 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -298,11 +298,18 @@ func Run(ctx context.Context, options Options) (Summary, error) { logger.Warn("apply error; marking step transitional", "step", stepIndex, "err", err) transitional = true applySkipped = true - lastAction = nil + // The error says the call failed, not that the gesture never + // reached the app: a deadline that fires after dispatch leaves + // the effect committed. Reporting no action here would let a + // property convict the app for an effect with no cause, so the + // action is reported with its fate unknown instead. + unconfirmed := nextAction + lastAction = &unconfirmed } else { consecutiveApplyFailures = 0 - actionCopy := nextAction - lastAction = &actionCopy + applied := nextAction + applied.Applied = true + lastAction = &applied } } else { lastAction = nil From 5b08e09d6f241ed7936563ccc226d9f777050c9b Mon Sep 17 00:00:00 2001 From: PJ Date: Sat, 15 Aug 2026 13:41:45 +0530 Subject: [PATCH 3/8] feat(spec): give state.lastAction an applied field Three states, not two: no action is a null lastAction, applied: true is an action the runner confirmed, applied: null is one it dispatched and never learned the fate of. --- pkg/spec/src/index.ts | 1 + pkg/spec/src/types.ts | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/spec/src/index.ts b/pkg/spec/src/index.ts index 4ac8c31..4f75729 100644 --- a/pkg/spec/src/index.ts +++ b/pkg/spec/src/index.ts @@ -12,6 +12,7 @@ export type { InputTextAction, Key, KnownAttrSelectors, + LastAction, LogEntry, Point, PressKeyAction, diff --git a/pkg/spec/src/types.ts b/pkg/spec/src/types.ts index f748d83..c28fd05 100644 --- a/pkg/spec/src/types.ts +++ b/pkg/spec/src/types.ts @@ -102,10 +102,20 @@ export interface ExceptionRecord { unixMillis?: number; } +/** + * The previous step's action as the runner reports it. `applied` is true when + * the runner saw the dispatch succeed and null when the apply call failed with + * the gesture possibly already delivered: an RPC deadline can fire after the + * tap landed. Null is unknown, not "it did not happen" (`state.lastAction` is + * itself null for that), so a property attributing an effect to this action + * has to decline unless `applied` is true. + */ +export type LastAction = Action & { applied: true | null }; + export interface State { snapshots: Snapshots; ax: AccessibilityTree; - lastAction: Action | null; + lastAction: LastAction | null; time: number; logs: readonly LogEntry[]; exceptions: readonly ExceptionRecord[]; From a5066dac27fe7c2e473a4e2d8baa71e00d9e5473 Mon Sep 17 00:00:00 2001 From: PJ Date: Sat, 15 Aug 2026 13:41:52 +0530 Subject: [PATCH 4/8] fix(folio): do not attribute an effect to an unconfirmed action submitChangesBalanceByTypedAmount and createdAccountHasNonZeroBalance both convict by pinning an effect on the last action, so both decline unless the runner saw it applied. The fixtures now say which fate they mean. --- examples/folio/sanderling/predicates.ts | 60 +++++++++++--- pkg/spec/test/folio-new-account.test.ts | 27 +++++- .../folio-submit-balance-predicate.test.ts | 82 +++++++++++-------- pkg/spec/test/folio-transition-frame.test.ts | 10 ++- 4 files changed, 128 insertions(+), 51 deletions(-) diff --git a/examples/folio/sanderling/predicates.ts b/examples/folio/sanderling/predicates.ts index e7ea34b..ab93c60 100644 --- a/examples/folio/sanderling/predicates.ts +++ b/examples/folio/sanderling/predicates.ts @@ -123,10 +123,22 @@ export function readHomeCards(args: { return { value: reading, carrier: reading, fresh: true }; } -function isTapOn( - lastAction: { kind?: string; on?: string | object } | null, - target: string, -): boolean { +// state.lastAction as the two hosts build it (internal/verifier/marshal.go +// lastActionFields), read defensively: every field is what a Go struct decided +// to emit, not something this file can trust a compile-time shape for. +// +// `applied` is true when the runner saw the action's dispatch succeed and null +// when the apply call failed with the gesture possibly already delivered: an +// RPC deadline can fire after the tap landed, and nothing can find out +// afterwards. That is unknown, not "it did not happen", which is what +// `lastAction === null` says. +export interface ObservedAction { + kind?: string; + on?: string | object; + applied?: true | null; +} + +function isTapOn(lastAction: ObservedAction | null, target: string): boolean { if (lastAction == null) return false; if (lastAction.kind !== "Tap" && lastAction.kind !== "DoubleTap") return false; const on = lastAction.on; @@ -138,19 +150,27 @@ function isTapOn( // Repository.createTransaction: AddTransactionViewModel.submit() is the only // caller, AddTransactionEvent.Submit is the only thing that runs it, and the // TxnSubmit button's onClick is the only thing that sends that event. -export function isTxnSubmitTap( - lastAction: { kind?: string; on?: string | object } | null, -): boolean { +export function isTxnSubmitTap(lastAction: ObservedAction | null): boolean { return isTapOn(lastAction, "TxnSubmit"); } // Likewise the only action that creates an account. -export function isAddAccountSubmitTap( - lastAction: { kind?: string; on?: string | object } | null, -): boolean { +export function isAddAccountSubmitTap(lastAction: ObservedAction | null): boolean { return isTapOn(lastAction, "AddAccountSubmit"); } +// Did the runner see this action's dispatch succeed? `applied` is null when the +// apply call failed with the gesture possibly already delivered (an RPC +// deadline can fire after the tap landed), and the runner has no way to find +// out afterwards. Such an action may have caused anything the next reading +// shows, so it counts toward how many submits a window COULD hold, but it never +// licenses attributing an effect to it: a property that demands the effect of +// an action that may never have run convicts the app of the runner's own +// uncertainty. +export function confirmedApplied(lastAction: ObservedAction | null): boolean { + return lastAction != null && lastAction.applied === true; +} + // Counts the submit actions inside the window the balance property compares // over: from the last Home total we read to this step, inclusive of this step's // action. @@ -164,9 +184,15 @@ export function isAddAccountSubmitTap( // // The reset lands on `fresh`, the same event that advances the carrier, so the // count always describes exactly the interval the two compared totals span. +// +// A submit whose dispatch the runner could not confirm counts here, because +// this number is an upper bound on the submits the window holds and the tap may +// well have landed. Leaving it out is what convicted a healthy app: +// committedTransactionsExceedSubmits saw a transaction rise of one against a +// window of zero and called it a double submit. export function countSubmitsInWindow(args: { previousCount: number; - lastAction: { kind?: string; on?: string | object } | null; + lastAction: ObservedAction | null; fresh: boolean; }): { reported: number; next: number } { const { previousCount, lastAction, fresh } = args; @@ -343,7 +369,7 @@ export function homeTxnCountsOf(cards: readonly CardReading[]): Record testTag:AddAccountSubmit" }; -const idle = { kind: "Tap", on: "testTag:HomeScreen > testTag:AccountCard" }; +const created = { + kind: "Tap", + on: "testTag:AddAccountScreen > testTag:AddAccountSubmit", + applied: true as const, +}; +const idle = { kind: "Tap", on: "testTag:HomeScreen > testTag:AccountCard", applied: true as const }; const account = (name: string, balance: number | null) => ({ name, balance }); @@ -27,7 +31,7 @@ test("a double-tapped create is judged the same way", () => { assert.equal( createdAccountHasNonZeroBalance({ route: "home", - lastAction: { kind: "DoubleTap", on: "id:AddAccountSubmit" }, + lastAction: { kind: "DoubleTap", on: "id:AddAccountSubmit", applied: true }, typedName: "Travel", before: [account("Checking", 0)], after: [account("Checking", 0), account("Travel", 5000)], @@ -233,3 +237,20 @@ test("a card that was already there is not a card that was just created", () => false, ); }); + +// The apply call failed with the gesture possibly already delivered, so nobody +// knows whether that account was created. The card carrying the typed name may +// be an older one that scrolled into view, and attributing it to a creation +// that may never have happened is a conviction built on a guess. +test("a create the runner could not confirm attributes nothing", () => { + assert.equal( + createdAccountHasNonZeroBalance({ + route: "home", + lastAction: { ...created, applied: null }, + typedName: "Travel", + before: [account("Checking", 0)], + after: [account("Checking", 0), account("Travel", 5000)], + }), + false, + ); +}); diff --git a/pkg/spec/test/folio-submit-balance-predicate.test.ts b/pkg/spec/test/folio-submit-balance-predicate.test.ts index 57a94ce..672fb5b 100644 --- a/pkg/spec/test/folio-submit-balance-predicate.test.ts +++ b/pkg/spec/test/folio-submit-balance-predicate.test.ts @@ -12,7 +12,7 @@ test("single submit: delta matches typed amount", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: 1000, @@ -26,7 +26,7 @@ test("double submit: delta is twice the typed amount, fires", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: 1000, @@ -40,7 +40,7 @@ test("DoubleTap kind also caught when delta exceeds typed amount", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "DoubleTap", on: submitOn }, + lastAction: { kind: "DoubleTap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: 0, @@ -54,7 +54,7 @@ test("wrong action kind: vacuous true even with mismatch", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "InputText", on: submitOn }, + lastAction: { kind: "InputText", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: 1000, @@ -68,7 +68,7 @@ test("wrong target: vacuous true even with mismatch", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: "testTag:LoginScreen > testTag:LoginSubmit" }, + lastAction: { kind: "Tap", on: "testTag:LoginScreen > testTag:LoginSubmit", applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: 1000, @@ -96,7 +96,7 @@ test("zero typedAmount: vacuous true", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 0, prevTotalBalance: 1000, @@ -110,7 +110,7 @@ test("selector as object: coerced safely and TxnSubmit detected", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: { testTag: "TxnSubmit" } }, + lastAction: { kind: "Tap", on: { testTag: "TxnSubmit" }, applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: 0, @@ -124,7 +124,7 @@ test("selector as object without TxnSubmit: vacuous true", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: { testTag: "LoginSubmit" } }, + lastAction: { kind: "Tap", on: { testTag: "LoginSubmit" }, applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: 0, @@ -138,7 +138,7 @@ test("raw whole-dollar input: single submit clears", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: parseTypedAmount("50"), prevTotalBalance: 5000, @@ -152,7 +152,7 @@ test("raw whole-dollar input: double submit fires", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: parseTypedAmount("50"), prevTotalBalance: 5000, @@ -166,7 +166,7 @@ test("decimal input from empty prior balance clears", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: parseTypedAmount("5.50"), prevTotalBalance: 0, @@ -180,7 +180,7 @@ test("DoubleTap kind with raw whole-dollar input fires", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "DoubleTap", on: submitOn }, + lastAction: { kind: "DoubleTap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: parseTypedAmount("100"), prevTotalBalance: 0, @@ -194,7 +194,7 @@ test("route gate: ledger landing with stale carrier is skipped", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "ledger", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 5000, prevTotalBalance: 0, @@ -208,7 +208,7 @@ test("route gate: add-transaction landing with double-submit delta is skipped", assert.equal( submitChangesBalanceByTypedAmount({ route: "add-transaction", - lastAction: { kind: "DoubleTap", on: submitOn }, + lastAction: { kind: "DoubleTap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 5000, prevTotalBalance: 0, @@ -222,7 +222,7 @@ test("route gate: null route is skipped", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: null, - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 5000, prevTotalBalance: 0, @@ -236,7 +236,7 @@ test("route gate: home landing with matching delta passes", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 5000, prevTotalBalance: 0, @@ -250,7 +250,7 @@ test("route gate: home landing with double-insert delta fires", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 5000, prevTotalBalance: 0, @@ -275,7 +275,7 @@ test("above 2^53 a healthy single submit is not reported", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 1600, prevTotalBalance: HUGE_BALANCE, @@ -289,7 +289,7 @@ test("above 2^53 a double-submit delta is not reported either", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 1600, prevTotalBalance: HUGE_BALANCE, @@ -303,7 +303,7 @@ test("an unreadable previous balance above 2^53 is not evidence", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 1600, prevTotalBalance: HUGE_BALANCE, @@ -320,7 +320,7 @@ test("typed amount above 2^53 is not evidence", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 1e23, prevTotalBalance: 0, @@ -336,7 +336,7 @@ test("boundary: a double submit landing exactly on MAX_SAFE_INTEGER still fires" assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 4503599627370495, prevTotalBalance: 0, @@ -350,7 +350,7 @@ test("boundary: a single submit landing exactly on MAX_SAFE_INTEGER passes", () assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 9007199254740991, prevTotalBalance: 0, @@ -364,7 +364,7 @@ test("boundary: one cent past MAX_SAFE_INTEGER stops being evidence", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 4503599627370496, prevTotalBalance: 0, @@ -381,7 +381,7 @@ test("a large but exact difference between safe balances still fires", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: -9007199254740991, @@ -397,7 +397,7 @@ test("21-digit typed amount with an unmoved balance is not a violation", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: parseTypedAmount("999999999999999999999"), prevTotalBalance: 220900, @@ -417,7 +417,7 @@ test("freshness: two submits in the window is vacuous, not a conviction", () => assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "DoubleTap", on: submitOn }, + lastAction: { kind: "DoubleTap", on: submitOn, applied: true }, submitsInWindow: 2, typedAmount: 19600, prevTotalBalance: 0, @@ -431,7 +431,7 @@ test("freshness: two submits cannot convict even on a clean 2x delta", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 2, typedAmount: 500, prevTotalBalance: 1000, @@ -448,7 +448,7 @@ test("freshness boundary: exactly one submit is the window that convicts", () => assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "DoubleTap", on: submitOn }, + lastAction: { kind: "DoubleTap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 19600, prevTotalBalance: 0, @@ -462,7 +462,7 @@ test("freshness boundary: one submit with a healthy 1x delta still passes", () = assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 19600, prevTotalBalance: 0, @@ -476,7 +476,7 @@ test("freshness boundary: three submits is vacuous", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 3, typedAmount: 500, prevTotalBalance: 0, @@ -493,7 +493,7 @@ test("freshness boundary: a window with no submit in it is vacuous", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 0, typedAmount: 500, prevTotalBalance: 1000, @@ -502,3 +502,21 @@ test("freshness boundary: a window with no submit in it is vacuous", () => { true, ); }); + +// applied: null is the runner saying it dispatched the tap and never learned +// whether it landed. A submit that committed nothing leaves the balance where +// it was, so demanding the typed amount of movement for it convicts an app that +// did exactly what it should have. +test("a submit the runner could not confirm demands no balance move", () => { + assert.equal( + submitChangesBalanceByTypedAmount({ + route: "home", + lastAction: { kind: "Tap", on: submitOn, applied: null }, + submitsInWindow: 1, + typedAmount: 500, + prevTotalBalance: 1000, + currTotalBalance: 1000, + }), + true, + ); +}); diff --git a/pkg/spec/test/folio-transition-frame.test.ts b/pkg/spec/test/folio-transition-frame.test.ts index f785642..64bcbf3 100644 --- a/pkg/spec/test/folio-transition-frame.test.ts +++ b/pkg/spec/test/folio-transition-frame.test.ts @@ -94,7 +94,7 @@ test("the measured android transition chain no longer convicts at delta 0", () = const step = ( tags: string[], totalText: string | undefined, - lastAction: { kind: string; on: string } | null, + lastAction: { kind: string; on: string; applied: true } | null, ) => { const route = routeOfFrame(SCREENS, frame(...tags)); const reading = readHomeTotalBalance({ route, totalText, previousCarrier: carrier }); @@ -104,8 +104,12 @@ test("the measured android transition chain no longer convicts at delta 0", () = return { route, total: reading.value, submits: window.reported }; }; - const back = { kind: "DoubleTap", on: "id:BackButton" }; - const phantomSubmit = { kind: "Tap", on: "testTag:AddTransactionScreen > testTag:TxnSubmit" }; + const back = { kind: "DoubleTap", on: "id:BackButton", applied: true as const }; + const phantomSubmit = { + kind: "Tap", + on: "testTag:AddTransactionScreen > testTag:TxnSubmit", + applied: true as const, + }; const transition = step(["AddTransactionScreen", "HomeScreen"], "$86,911.00", back); assert.equal(transition.route, null); From 46059c111bf59b070c7000825d38e740f5080c0e Mon Sep 17 00:00:00 2001 From: PJ Date: Sat, 15 Aug 2026 13:41:59 +0530 Subject: [PATCH 5/8] test(folio): an unconfirmed submit belongs in the window The count is an upper bound on the submits a window holds, so the tap that may have landed counts and committedTransactionsExceedSubmits has nothing to convict on. --- pkg/spec/test/folio-submit-window.test.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pkg/spec/test/folio-submit-window.test.ts b/pkg/spec/test/folio-submit-window.test.ts index da422ed..6e19281 100644 --- a/pkg/spec/test/folio-submit-window.test.ts +++ b/pkg/spec/test/folio-submit-window.test.ts @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { + committedTransactionsExceedSubmits, countSubmitsInWindow, isTxnSubmitTap, readHomeTotalBalance, @@ -149,3 +150,25 @@ test("an unreadable Home does not close the window", () => { assert.equal(trace[2]?.total, null); assert.equal(trace[3]?.submits, 2); }); + +// The window is an upper bound on the submits it holds, so a submit whose +// dispatch the runner could not confirm belongs in it: the tap may well have +// landed, and a bound that leaves it out is one the transaction it committed +// exceeds. That is the false conviction, a rise of one against a window of +// zero, on the property carrying most of the detection on android. +test("a submit the runner could not confirm still counts toward the window", () => { + const window = countSubmitsInWindow({ + previousCount: 0, + lastAction: { kind: "Tap", on: submitOn, applied: null }, + fresh: true, + }); + assert.equal(window.reported, 1); + assert.equal( + committedTransactionsExceedSubmits({ + countsBefore: { Travel: 3 }, + countsAfter: { Travel: 4 }, + submitsInWindow: window.reported, + }), + false, + ); +}); From 7833c5fa5377edc5280454e2560b4a08a56f216e Mon Sep 17 00:00:00 2001 From: PJ Date: Sat, 15 Aug 2026 13:42:05 +0530 Subject: [PATCH 6/8] test(runner): a tap that lands under a failed apply is not a double submit Drives the real folio counting predicates through the runner against a device that commits the tap and then times out. The double-submit case is the control: without it a green proves only that the property never fired. --- internal/runner/uncertain_last_action_test.go | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 internal/runner/uncertain_last_action_test.go diff --git a/internal/runner/uncertain_last_action_test.go b/internal/runner/uncertain_last_action_test.go new file mode 100644 index 0000000..f687e99 --- /dev/null +++ b/internal/runner/uncertain_last_action_test.go @@ -0,0 +1,150 @@ +package runner + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/priyanshujain/sanderling/internal/driver" + mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock" +) + +// An apply error is not proof that nothing landed. An RPC deadline that fires +// after the tap was dispatched leaves the transaction committed, and a runner +// that reports "no action" for it hands +// submitCommitsOneTransactionPerAction a rise of one transaction against a +// window of zero submits: a conviction manufactured out of the runner's own +// uncertainty, on the property carrying most of the detection on android. +// +// The spec below is the real folio predicate pair, imported from the example, +// so what this asserts is the verdict the shipped property reaches. +const uncertainApplySpecTemplate = ` +import { actions, always, extract, next, Tap } from "@sanderling/spec"; +import { + committedTransactionsExceedSubmits, + countSubmitsInWindow, +} from "%s"; + +let submits = 0; +const submitsInWindow = extract("submitsInWindow", state => { + const window = countSubmitsInWindow({ + previousCount: submits, + lastAction: state.lastAction, + fresh: true, + }); + submits = window.next; + return window.reported; +}); + +const counts = extract("counts", state => { + const text = state.ax.find("id:TxnCount")?.text; + return text ? { Travel: parseInt(text, 10) } : null; +}); + +globalThis.properties = { + submitCommitsOneTransactionPerAction: always( + next(() => + !committedTransactionsExceedSubmits({ + countsBefore: counts.previous ?? null, + countsAfter: counts.current, + submitsInWindow: submitsInWindow.current, + }), + ), + ), +}; +globalThis.actions = actions(() => [Tap({ on: "id:TxnSubmit" })]); +` + +const homeWithTxnCount = `{"attributes":{"resource-id":"HomeScreen"},"children":[ + {"attributes":{"resource-id":"TxnCount","text":"%d"},"children":[]}, + {"attributes":{"resource-id":"TxnSubmit","bounds":"[40,80,240,160]"},"children":[],"clickable":true,"enabled":true} +]}` + +// dispatchThenFailDriver is the device condition the runner cannot see through: +// the tap reaches the app and commits, then the call the runner is waiting on +// times out. Every later hierarchy read shows the committed transactions. +type dispatchThenFailDriver struct { + *mockdriver.Driver + commitsPerTap int64 + committed atomic.Int64 +} + +func (d *dispatchThenFailDriver) Tap(context.Context, int, int) error { + return d.dispatchThenFail() +} + +func (d *dispatchThenFailDriver) TapSelector(context.Context, string) error { + return d.dispatchThenFail() +} + +func (d *dispatchThenFailDriver) dispatchThenFail() error { + d.committed.Add(d.commitsPerTap) + return errors.New("rpc error: code = DeadlineExceeded desc = context deadline exceeded") +} + +func (d *dispatchThenFailDriver) Snapshot(context.Context) (string, driver.Image, error) { + return fmt.Sprintf(homeWithTxnCount, d.committed.Load()), driver.Image{}, nil +} + +func TestRunner_ApplyErrorAfterDispatchDoesNotConvictTheSubmitCountingProperty(t *testing.T) { + predicates, err := filepath.Abs("../../examples/folio/sanderling/predicates.ts") + if err != nil { + t.Fatal(err) + } + spec := fmt.Sprintf(uncertainApplySpecTemplate, predicates) + + run := func(t *testing.T, commitsPerTap int64) []ViolationRecord { + t.Helper() + state := newHarnessWithSpec(t, spec) + device := &dispatchThenFailDriver{Driver: state.mock, commitsPerTap: commitsPerTap} + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + summary, err := Run(ctx, Options{ + Duration: time.Hour, + IdleTimeout: 20 * time.Millisecond, + MaxSteps: 2, + Driver: device, + Verifier: state.verifier, + TraceWriter: state.writer, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + if summary.Steps != 2 { + t.Fatalf("steps = %d, want 2; the run never reached the step that judges the pair", summary.Steps) + } + if got := device.committed.Load(); got != commitsPerTap*2 { + t.Fatalf("the device committed %d transaction(s), want %d; the taps never reached it", + got, commitsPerTap*2) + } + return summary.Violations + } + + t.Run("one transaction per tap is not a double submit", func(t *testing.T) { + if violations := run(t, 1); len(violations) != 0 { + t.Errorf("the counting property convicted a healthy app: %v\n"+ + "one transaction rose against a submit the runner dispatched but "+ + "could not confirm, and the spec was told no action happened", + violations) + } + }) + + // The control. Without it a green above proves nothing: a property that + // never sees a comparable pair is silently vacuous and reports the same + // empty violation list. + t.Run("two transactions per tap still convicts", func(t *testing.T) { + violations := run(t, 2) + if len(violations) == 0 { + t.Fatal("the counting property missed a double submit; the harness never " + + "put the property in a position to fire, so the case above proves nothing") + } + if violations[0].Properties[0] != "submitCommitsOneTransactionPerAction" { + t.Errorf("violated %v, want submitCommitsOneTransactionPerAction", violations[0].Properties) + } + }) +} From 157cf63520623223e1d69b36fc7b0703f7158948 Mon Sep 17 00:00:00 2001 From: PJ Date: Sat, 15 Aug 2026 13:42:12 +0530 Subject: [PATCH 7/8] test(verifier): pin the three lastAction states on both hosts The web page is handed the same applied field the goja object exposes, so a property cannot read one thing on native and another on web. --- internal/runner/web_last_action_test.go | 43 ++++++++++++++++++++++++- internal/verifier/marshal_test.go | 40 +++++++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/internal/runner/web_last_action_test.go b/internal/runner/web_last_action_test.go index 22ea458..e3a7ecc 100644 --- a/internal/runner/web_last_action_test.go +++ b/internal/runner/web_last_action_test.go @@ -3,6 +3,7 @@ package runner import ( "context" "encoding/json" + "errors" "testing" "time" @@ -71,7 +72,47 @@ func TestRunner_WebInstallsLastActionInThePage(t *testing.T) { // Every later step carries what the runner actually applied. The shape is // the goja host's (internal/verifier/marshal.go lastActionFields), pinned // against it by TestLastAction_WebJSONMatchesTheGojaObject. - const want = `{"kind":"Tap","on":"id:TxnSubmit"}` + const want = `{"kind":"Tap","applied":true,"on":"id:TxnSubmit"}` + if web.installed[1] != want { + t.Errorf("step 2 installed %s, want %s", web.installed[1], want) + } +} + +// failingTapWebDriver dispatches the tap and then fails the call, the shape an +// RPC deadline takes: the page has the click, the runner has an error. +type failingTapWebDriver struct { + *tappingWebDriver +} + +func (d *failingTapWebDriver) Tap(context.Context, int, int) error { + return errors.New("rpc error: code = DeadlineExceeded desc = context deadline exceeded") +} + +// The web leg of the same three states the goja host reports. "applied":null is +// not "no action": a property gated on the last action still sees the tap and +// decides for itself, which it cannot do if the page is handed a bare null. +func TestRunner_WebInstallsAnUnconfirmedActionWithItsFateUnknown(t *testing.T) { + state := newHarnessWithSpec(t, lastActionSpec) + web := &failingTapWebDriver{tappingWebDriver: &tappingWebDriver{Driver: state.mock}} + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := Run(ctx, Options{ + Duration: time.Hour, + IdleTimeout: 20 * time.Millisecond, + MaxSteps: 2, + Driver: web, + Verifier: state.verifier, + TraceWriter: state.writer, + }); err != nil { + t.Fatalf("Run: %v", err) + } + + if len(web.installed) < 2 { + t.Fatalf("the page was handed lastAction %d time(s); the web path never installed it", + len(web.installed)) + } + const want = `{"kind":"Tap","applied":null,"on":"id:TxnSubmit"}` if web.installed[1] != want { t.Errorf("step 2 installed %s, want %s", web.installed[1], want) } diff --git a/internal/verifier/marshal_test.go b/internal/verifier/marshal_test.go index 1b91d1f..b76e480 100644 --- a/internal/verifier/marshal_test.go +++ b/internal/verifier/marshal_test.go @@ -168,6 +168,7 @@ func TestLastAction_WebJSONMatchesTheGojaObject(t *testing.T) { }{ {"nil", nil}, {"Tap", &Action{Kind: ActionKindTap, On: "id:TxnSubmit", X: 12, Y: 34}}, + {"TapApplied", &Action{Kind: ActionKindTap, On: "id:TxnSubmit", Applied: true}}, {"TapWithoutSelector", &Action{Kind: ActionKindTap, X: 12, Y: 34}}, {"DoubleTap", &Action{Kind: ActionKindDoubleTap, On: `desc:say "hi" `}}, {"InputText", &Action{Kind: ActionKindInputText, On: "id:field", Text: "50"}}, @@ -193,3 +194,42 @@ func TestLastAction_WebJSONMatchesTheGojaObject(t *testing.T) { }) } } + +// A spec has to be able to tell three things apart: no action ran, an action +// ran, and an action was dispatched whose fate the runner cannot vouch for. +// The third used to be reported as the first, which is how a property that +// reasons "an effect landed with no action to cause it" convicts an app over +// an RPC deadline. +func TestLastAction_SeparatesNoActionFromAnActionOfUnknownFate(t *testing.T) { + verifier := newVerifier(t) + mustLoad(t, verifier, ` + globalThis.fate = __sanderling__.extract(state => + state.lastAction === null ? "no action" + : state.lastAction.applied === true ? "applied" + : state.lastAction.applied === null ? "unknown" + : "unreadable"); + `) + + for _, testCase := range []struct { + name string + action *Action + want string + }{ + {"nothing ran", nil, "no action"}, + {"dispatch confirmed", &Action{Kind: ActionKindTap, On: "id:TxnSubmit", Applied: true}, "applied"}, + {"dispatch unconfirmed", &Action{Kind: ActionKindTap, On: "id:TxnSubmit"}, "unknown"}, + } { + t.Run(testCase.name, func(t *testing.T) { + if err := verifier.PushSnapshot(SnapshotInput{ + Snapshots: Snapshots{}, + LastAction: testCase.action, + }); err != nil { + t.Fatal(err) + } + handle := verifier.runtime.GlobalObject().Get("fate").ToObject(verifier.runtime) + if got := handle.Get("current").String(); got != testCase.want { + t.Errorf("the spec read %q, want %q", got, testCase.want) + } + }) + } +} From a8e37a39eb55a7d30960f862a6595fbea38694a8 Mon Sep 17 00:00:00 2001 From: PJ Date: Sat, 15 Aug 2026 13:42:18 +0530 Subject: [PATCH 8/8] docs(spec-language): document the three lastAction states --- docs/manual/spec-language.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/manual/spec-language.md b/docs/manual/spec-language.md index 5b52379..2c1532c 100644 --- a/docs/manual/spec-language.md +++ b/docs/manual/spec-language.md @@ -29,7 +29,7 @@ Every extractor callback receives a `State`: interface State { ax: AccessibilityTree; snapshots: Record; - lastAction: Action | null; + lastAction: (Action & { applied: true | null }) | null; logs: readonly LogEntry[]; exceptions: readonly ExceptionRecord[]; time: number; // ms since run start @@ -40,11 +40,13 @@ interface State { |---|---| | `ax` | Live UI hierarchy for this step | | `snapshots` | Key-value data pushed by the app SDK (empty if SDK not integrated) | -| `lastAction` | The action dispatched in the previous step, or `null` on the first step | +| `lastAction` | The action dispatched in the previous step, or `null` on the first step and on any step that dispatched nothing | | `logs` | Log entries collected since the previous step | | `exceptions` | Uncaught exceptions or `Sanderling.reportError()` calls since the previous step | | `time` | Milliseconds elapsed since the run started | +`lastAction.applied` is `true` when the runner saw the dispatch succeed and `null` when the apply call failed with the action possibly already delivered: an RPC deadline can fire after the tap reached the app, and nothing can find out afterwards. So there are three states, not two. `state.lastAction === null` means no action ran; `applied === null` means one ran whose fate is unknown. A property that attributes an effect to the action ("this submit must move the balance by the typed amount") has to decline unless `applied` is `true`, or a timeout convicts a healthy app. A property that counts what the app COULD have done should include it: an unconfirmed submit belongs in an upper bound on how many submits a window holds. + ## Selectors Selectors are passed to `ax.find()`, `ax.findAll()`, and element-scoped `.find()` / `.findAll()`.