diff --git a/docs/manual/spec-language.md b/docs/manual/spec-language.md index 5b52379..2c1532c 100644 --- a/docs/manual/spec-language.md +++ b/docs/manual/spec-language.md @@ -29,7 +29,7 @@ Every extractor callback receives a `State`: interface State { ax: AccessibilityTree; snapshots: Record; - lastAction: Action | null; + lastAction: (Action & { applied: true | null }) | null; logs: readonly LogEntry[]; exceptions: readonly ExceptionRecord[]; time: number; // ms since run start @@ -40,11 +40,13 @@ interface State { |---|---| | `ax` | Live UI hierarchy for this step | | `snapshots` | Key-value data pushed by the app SDK (empty if SDK not integrated) | -| `lastAction` | The action dispatched in the previous step, or `null` on the first step | +| `lastAction` | The action dispatched in the previous step, or `null` on the first step and on any step that dispatched nothing | | `logs` | Log entries collected since the previous step | | `exceptions` | Uncaught exceptions or `Sanderling.reportError()` calls since the previous step | | `time` | Milliseconds elapsed since the run started | +`lastAction.applied` is `true` when the runner saw the dispatch succeed and `null` when the apply call failed with the action possibly already delivered: an RPC deadline can fire after the tap reached the app, and nothing can find out afterwards. So there are three states, not two. `state.lastAction === null` means no action ran; `applied === null` means one ran whose fate is unknown. A property that attributes an effect to the action ("this submit must move the balance by the typed amount") has to decline unless `applied` is `true`, or a timeout convicts a healthy app. A property that counts what the app COULD have done should include it: an unconfirmed submit belongs in an upper bound on how many submits a window holds. + ## Selectors Selectors are passed to `ax.find()`, `ax.findAll()`, and element-scoped `.find()` / `.findAll()`. diff --git a/examples/folio/sanderling/predicates.ts b/examples/folio/sanderling/predicates.ts index e7ea34b..ab93c60 100644 --- a/examples/folio/sanderling/predicates.ts +++ b/examples/folio/sanderling/predicates.ts @@ -123,10 +123,22 @@ export function readHomeCards(args: { return { value: reading, carrier: reading, fresh: true }; } -function isTapOn( - lastAction: { kind?: string; on?: string | object } | null, - target: string, -): boolean { +// state.lastAction as the two hosts build it (internal/verifier/marshal.go +// lastActionFields), read defensively: every field is what a Go struct decided +// to emit, not something this file can trust a compile-time shape for. +// +// `applied` is true when the runner saw the action's dispatch succeed and null +// when the apply call failed with the gesture possibly already delivered: an +// RPC deadline can fire after the tap landed, and nothing can find out +// afterwards. That is unknown, not "it did not happen", which is what +// `lastAction === null` says. +export interface ObservedAction { + kind?: string; + on?: string | object; + applied?: true | null; +} + +function isTapOn(lastAction: ObservedAction | null, target: string): boolean { if (lastAction == null) return false; if (lastAction.kind !== "Tap" && lastAction.kind !== "DoubleTap") return false; const on = lastAction.on; @@ -138,19 +150,27 @@ function isTapOn( // Repository.createTransaction: AddTransactionViewModel.submit() is the only // caller, AddTransactionEvent.Submit is the only thing that runs it, and the // TxnSubmit button's onClick is the only thing that sends that event. -export function isTxnSubmitTap( - lastAction: { kind?: string; on?: string | object } | null, -): boolean { +export function isTxnSubmitTap(lastAction: ObservedAction | null): boolean { return isTapOn(lastAction, "TxnSubmit"); } // Likewise the only action that creates an account. -export function isAddAccountSubmitTap( - lastAction: { kind?: string; on?: string | object } | null, -): boolean { +export function isAddAccountSubmitTap(lastAction: ObservedAction | null): boolean { return isTapOn(lastAction, "AddAccountSubmit"); } +// Did the runner see this action's dispatch succeed? `applied` is null when the +// apply call failed with the gesture possibly already delivered (an RPC +// deadline can fire after the tap landed), and the runner has no way to find +// out afterwards. Such an action may have caused anything the next reading +// shows, so it counts toward how many submits a window COULD hold, but it never +// licenses attributing an effect to it: a property that demands the effect of +// an action that may never have run convicts the app of the runner's own +// uncertainty. +export function confirmedApplied(lastAction: ObservedAction | null): boolean { + return lastAction != null && lastAction.applied === true; +} + // Counts the submit actions inside the window the balance property compares // over: from the last Home total we read to this step, inclusive of this step's // action. @@ -164,9 +184,15 @@ export function isAddAccountSubmitTap( // // The reset lands on `fresh`, the same event that advances the carrier, so the // count always describes exactly the interval the two compared totals span. +// +// A submit whose dispatch the runner could not confirm counts here, because +// this number is an upper bound on the submits the window holds and the tap may +// well have landed. Leaving it out is what convicted a healthy app: +// committedTransactionsExceedSubmits saw a transaction rise of one against a +// window of zero and called it a double submit. export function countSubmitsInWindow(args: { previousCount: number; - lastAction: { kind?: string; on?: string | object } | null; + lastAction: ObservedAction | null; fresh: boolean; }): { reported: number; next: number } { const { previousCount, lastAction, fresh } = args; @@ -343,7 +369,7 @@ export function homeTxnCountsOf(cards: readonly CardReading[]): Record { + const window = countSubmitsInWindow({ + previousCount: submits, + lastAction: state.lastAction, + fresh: true, + }); + submits = window.next; + return window.reported; +}); + +const counts = extract("counts", state => { + const text = state.ax.find("id:TxnCount")?.text; + return text ? { Travel: parseInt(text, 10) } : null; +}); + +globalThis.properties = { + submitCommitsOneTransactionPerAction: always( + next(() => + !committedTransactionsExceedSubmits({ + countsBefore: counts.previous ?? null, + countsAfter: counts.current, + submitsInWindow: submitsInWindow.current, + }), + ), + ), +}; +globalThis.actions = actions(() => [Tap({ on: "id:TxnSubmit" })]); +` + +const homeWithTxnCount = `{"attributes":{"resource-id":"HomeScreen"},"children":[ + {"attributes":{"resource-id":"TxnCount","text":"%d"},"children":[]}, + {"attributes":{"resource-id":"TxnSubmit","bounds":"[40,80,240,160]"},"children":[],"clickable":true,"enabled":true} +]}` + +// dispatchThenFailDriver is the device condition the runner cannot see through: +// the tap reaches the app and commits, then the call the runner is waiting on +// times out. Every later hierarchy read shows the committed transactions. +type dispatchThenFailDriver struct { + *mockdriver.Driver + commitsPerTap int64 + committed atomic.Int64 +} + +func (d *dispatchThenFailDriver) Tap(context.Context, int, int) error { + return d.dispatchThenFail() +} + +func (d *dispatchThenFailDriver) TapSelector(context.Context, string) error { + return d.dispatchThenFail() +} + +func (d *dispatchThenFailDriver) dispatchThenFail() error { + d.committed.Add(d.commitsPerTap) + return errors.New("rpc error: code = DeadlineExceeded desc = context deadline exceeded") +} + +func (d *dispatchThenFailDriver) Snapshot(context.Context) (string, driver.Image, error) { + return fmt.Sprintf(homeWithTxnCount, d.committed.Load()), driver.Image{}, nil +} + +func TestRunner_ApplyErrorAfterDispatchDoesNotConvictTheSubmitCountingProperty(t *testing.T) { + predicates, err := filepath.Abs("../../examples/folio/sanderling/predicates.ts") + if err != nil { + t.Fatal(err) + } + spec := fmt.Sprintf(uncertainApplySpecTemplate, predicates) + + run := func(t *testing.T, commitsPerTap int64) []ViolationRecord { + t.Helper() + state := newHarnessWithSpec(t, spec) + device := &dispatchThenFailDriver{Driver: state.mock, commitsPerTap: commitsPerTap} + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + summary, err := Run(ctx, Options{ + Duration: time.Hour, + IdleTimeout: 20 * time.Millisecond, + MaxSteps: 2, + Driver: device, + Verifier: state.verifier, + TraceWriter: state.writer, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + if summary.Steps != 2 { + t.Fatalf("steps = %d, want 2; the run never reached the step that judges the pair", summary.Steps) + } + if got := device.committed.Load(); got != commitsPerTap*2 { + t.Fatalf("the device committed %d transaction(s), want %d; the taps never reached it", + got, commitsPerTap*2) + } + return summary.Violations + } + + t.Run("one transaction per tap is not a double submit", func(t *testing.T) { + if violations := run(t, 1); len(violations) != 0 { + t.Errorf("the counting property convicted a healthy app: %v\n"+ + "one transaction rose against a submit the runner dispatched but "+ + "could not confirm, and the spec was told no action happened", + violations) + } + }) + + // The control. Without it a green above proves nothing: a property that + // never sees a comparable pair is silently vacuous and reports the same + // empty violation list. + t.Run("two transactions per tap still convicts", func(t *testing.T) { + violations := run(t, 2) + if len(violations) == 0 { + t.Fatal("the counting property missed a double submit; the harness never " + + "put the property in a position to fire, so the case above proves nothing") + } + if violations[0].Properties[0] != "submitCommitsOneTransactionPerAction" { + t.Errorf("violated %v, want submitCommitsOneTransactionPerAction", violations[0].Properties) + } + }) +} diff --git a/internal/runner/web_last_action_test.go b/internal/runner/web_last_action_test.go index 22ea458..e3a7ecc 100644 --- a/internal/runner/web_last_action_test.go +++ b/internal/runner/web_last_action_test.go @@ -3,6 +3,7 @@ package runner import ( "context" "encoding/json" + "errors" "testing" "time" @@ -71,7 +72,47 @@ func TestRunner_WebInstallsLastActionInThePage(t *testing.T) { // Every later step carries what the runner actually applied. The shape is // the goja host's (internal/verifier/marshal.go lastActionFields), pinned // against it by TestLastAction_WebJSONMatchesTheGojaObject. - const want = `{"kind":"Tap","on":"id:TxnSubmit"}` + const want = `{"kind":"Tap","applied":true,"on":"id:TxnSubmit"}` + if web.installed[1] != want { + t.Errorf("step 2 installed %s, want %s", web.installed[1], want) + } +} + +// failingTapWebDriver dispatches the tap and then fails the call, the shape an +// RPC deadline takes: the page has the click, the runner has an error. +type failingTapWebDriver struct { + *tappingWebDriver +} + +func (d *failingTapWebDriver) Tap(context.Context, int, int) error { + return errors.New("rpc error: code = DeadlineExceeded desc = context deadline exceeded") +} + +// The web leg of the same three states the goja host reports. "applied":null is +// not "no action": a property gated on the last action still sees the tap and +// decides for itself, which it cannot do if the page is handed a bare null. +func TestRunner_WebInstallsAnUnconfirmedActionWithItsFateUnknown(t *testing.T) { + state := newHarnessWithSpec(t, lastActionSpec) + web := &failingTapWebDriver{tappingWebDriver: &tappingWebDriver{Driver: state.mock}} + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := Run(ctx, Options{ + Duration: time.Hour, + IdleTimeout: 20 * time.Millisecond, + MaxSteps: 2, + Driver: web, + Verifier: state.verifier, + TraceWriter: state.writer, + }); err != nil { + t.Fatalf("Run: %v", err) + } + + if len(web.installed) < 2 { + t.Fatalf("the page was handed lastAction %d time(s); the web path never installed it", + len(web.installed)) + } + const want = `{"kind":"Tap","applied":null,"on":"id:TxnSubmit"}` if web.installed[1] != want { t.Errorf("step 2 installed %s, want %s", web.installed[1], want) } diff --git a/internal/verifier/marshal.go b/internal/verifier/marshal.go index c5b311a..d738e84 100644 --- a/internal/verifier/marshal.go +++ b/internal/verifier/marshal.go @@ -344,7 +344,19 @@ func lastActionFields(action *Action) []actionField { point := func(x, y int) []actionField { return []actionField{{key: "x", value: x}, {key: "y", value: y}} } - fields := []actionField{{key: "kind", value: string(action.Kind)}} + // An action whose apply call failed is not an action that did not happen: + // the dispatch may have landed before the error. That is unknown, and + // unknown is null here for the same reason every other absence in the spec + // surface is, so a property decides for itself instead of being handed a + // "nothing happened" the runner cannot vouch for. + var applied any + if action.Applied { + applied = true + } + fields := []actionField{ + {key: "kind", value: string(action.Kind)}, + {key: "applied", value: applied}, + } if action.On != "" { fields = append(fields, actionField{key: "on", value: action.On}) } @@ -397,7 +409,7 @@ func objectFromFields(runtime *goja.Runtime, fields []actionField) *goja.Object // has no Go-side state object to read: the runner pushes this JSON into the // page before each extractor evaluation. A nil action encodes as JSON null, // the same value the goja host reports on the first step of a run and after a -// step whose action was never applied. +// step whose action was never dispatched. func EncodeLastAction(action *Action) json.RawMessage { if action == nil { return json.RawMessage("null") diff --git a/internal/verifier/marshal_test.go b/internal/verifier/marshal_test.go index 1b91d1f..b76e480 100644 --- a/internal/verifier/marshal_test.go +++ b/internal/verifier/marshal_test.go @@ -168,6 +168,7 @@ func TestLastAction_WebJSONMatchesTheGojaObject(t *testing.T) { }{ {"nil", nil}, {"Tap", &Action{Kind: ActionKindTap, On: "id:TxnSubmit", X: 12, Y: 34}}, + {"TapApplied", &Action{Kind: ActionKindTap, On: "id:TxnSubmit", Applied: true}}, {"TapWithoutSelector", &Action{Kind: ActionKindTap, X: 12, Y: 34}}, {"DoubleTap", &Action{Kind: ActionKindDoubleTap, On: `desc:say "hi" `}}, {"InputText", &Action{Kind: ActionKindInputText, On: "id:field", Text: "50"}}, @@ -193,3 +194,42 @@ func TestLastAction_WebJSONMatchesTheGojaObject(t *testing.T) { }) } } + +// A spec has to be able to tell three things apart: no action ran, an action +// ran, and an action was dispatched whose fate the runner cannot vouch for. +// The third used to be reported as the first, which is how a property that +// reasons "an effect landed with no action to cause it" convicts an app over +// an RPC deadline. +func TestLastAction_SeparatesNoActionFromAnActionOfUnknownFate(t *testing.T) { + verifier := newVerifier(t) + mustLoad(t, verifier, ` + globalThis.fate = __sanderling__.extract(state => + state.lastAction === null ? "no action" + : state.lastAction.applied === true ? "applied" + : state.lastAction.applied === null ? "unknown" + : "unreadable"); + `) + + for _, testCase := range []struct { + name string + action *Action + want string + }{ + {"nothing ran", nil, "no action"}, + {"dispatch confirmed", &Action{Kind: ActionKindTap, On: "id:TxnSubmit", Applied: true}, "applied"}, + {"dispatch unconfirmed", &Action{Kind: ActionKindTap, On: "id:TxnSubmit"}, "unknown"}, + } { + t.Run(testCase.name, func(t *testing.T) { + if err := verifier.PushSnapshot(SnapshotInput{ + Snapshots: Snapshots{}, + LastAction: testCase.action, + }); err != nil { + t.Fatal(err) + } + handle := verifier.runtime.GlobalObject().Get("fate").ToObject(verifier.runtime) + if got := handle.Get("current").String(); got != testCase.want { + t.Errorf("the spec read %q, want %q", got, testCase.want) + } + }) + } +} diff --git a/internal/verifier/types.go b/internal/verifier/types.go index 5b35eb7..125703b 100644 --- a/internal/verifier/types.go +++ b/internal/verifier/types.go @@ -33,6 +33,11 @@ type Action struct { // Direction is the scroll direction for ActionKindScroll: one of "up", // "down", "left", "right". Empty for every other kind. Direction string + // Applied is meaningful only on the action a step reports to the spec as + // state.lastAction: true when the runner saw the dispatch succeed, false + // when the apply call failed and nothing can say whether the action + // reached the app. The spec is told which of the two it is. + Applied bool } // LogEntry mirrors a logcat line captured between steps. diff --git a/pkg/spec/src/index.ts b/pkg/spec/src/index.ts index 724e297..b80170f 100644 --- a/pkg/spec/src/index.ts +++ b/pkg/spec/src/index.ts @@ -13,6 +13,7 @@ export type { InputTextAction, Key, KnownAttrSelectors, + LastAction, LogEntry, LongPressAction, Point, diff --git a/pkg/spec/src/types.ts b/pkg/spec/src/types.ts index f748d83..c28fd05 100644 --- a/pkg/spec/src/types.ts +++ b/pkg/spec/src/types.ts @@ -102,10 +102,20 @@ export interface ExceptionRecord { unixMillis?: number; } +/** + * The previous step's action as the runner reports it. `applied` is true when + * the runner saw the dispatch succeed and null when the apply call failed with + * the gesture possibly already delivered: an RPC deadline can fire after the + * tap landed. Null is unknown, not "it did not happen" (`state.lastAction` is + * itself null for that), so a property attributing an effect to this action + * has to decline unless `applied` is true. + */ +export type LastAction = Action & { applied: true | null }; + export interface State { snapshots: Snapshots; ax: AccessibilityTree; - lastAction: Action | null; + lastAction: LastAction | null; time: number; logs: readonly LogEntry[]; exceptions: readonly ExceptionRecord[]; diff --git a/pkg/spec/test/folio-new-account.test.ts b/pkg/spec/test/folio-new-account.test.ts index def23bb..d5ee3ab 100644 --- a/pkg/spec/test/folio-new-account.test.ts +++ b/pkg/spec/test/folio-new-account.test.ts @@ -3,8 +3,12 @@ import { test } from "node:test"; import { createdAccountHasNonZeroBalance } from "../../../examples/folio/sanderling/predicates.ts"; -const created = { kind: "Tap", on: "testTag:AddAccountScreen > testTag:AddAccountSubmit" }; -const idle = { kind: "Tap", on: "testTag:HomeScreen > testTag:AccountCard" }; +const created = { + kind: "Tap", + on: "testTag:AddAccountScreen > testTag:AddAccountSubmit", + applied: true as const, +}; +const idle = { kind: "Tap", on: "testTag:HomeScreen > testTag:AccountCard", applied: true as const }; const account = (name: string, balance: number | null) => ({ name, balance }); @@ -27,7 +31,7 @@ test("a double-tapped create is judged the same way", () => { assert.equal( createdAccountHasNonZeroBalance({ route: "home", - lastAction: { kind: "DoubleTap", on: "id:AddAccountSubmit" }, + lastAction: { kind: "DoubleTap", on: "id:AddAccountSubmit", applied: true }, typedName: "Travel", before: [account("Checking", 0)], after: [account("Checking", 0), account("Travel", 5000)], @@ -233,3 +237,20 @@ test("a card that was already there is not a card that was just created", () => false, ); }); + +// The apply call failed with the gesture possibly already delivered, so nobody +// knows whether that account was created. The card carrying the typed name may +// be an older one that scrolled into view, and attributing it to a creation +// that may never have happened is a conviction built on a guess. +test("a create the runner could not confirm attributes nothing", () => { + assert.equal( + createdAccountHasNonZeroBalance({ + route: "home", + lastAction: { ...created, applied: null }, + typedName: "Travel", + before: [account("Checking", 0)], + after: [account("Checking", 0), account("Travel", 5000)], + }), + false, + ); +}); diff --git a/pkg/spec/test/folio-submit-balance-predicate.test.ts b/pkg/spec/test/folio-submit-balance-predicate.test.ts index 57a94ce..672fb5b 100644 --- a/pkg/spec/test/folio-submit-balance-predicate.test.ts +++ b/pkg/spec/test/folio-submit-balance-predicate.test.ts @@ -12,7 +12,7 @@ test("single submit: delta matches typed amount", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: 1000, @@ -26,7 +26,7 @@ test("double submit: delta is twice the typed amount, fires", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: 1000, @@ -40,7 +40,7 @@ test("DoubleTap kind also caught when delta exceeds typed amount", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "DoubleTap", on: submitOn }, + lastAction: { kind: "DoubleTap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: 0, @@ -54,7 +54,7 @@ test("wrong action kind: vacuous true even with mismatch", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "InputText", on: submitOn }, + lastAction: { kind: "InputText", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: 1000, @@ -68,7 +68,7 @@ test("wrong target: vacuous true even with mismatch", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: "testTag:LoginScreen > testTag:LoginSubmit" }, + lastAction: { kind: "Tap", on: "testTag:LoginScreen > testTag:LoginSubmit", applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: 1000, @@ -96,7 +96,7 @@ test("zero typedAmount: vacuous true", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 0, prevTotalBalance: 1000, @@ -110,7 +110,7 @@ test("selector as object: coerced safely and TxnSubmit detected", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: { testTag: "TxnSubmit" } }, + lastAction: { kind: "Tap", on: { testTag: "TxnSubmit" }, applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: 0, @@ -124,7 +124,7 @@ test("selector as object without TxnSubmit: vacuous true", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: { testTag: "LoginSubmit" } }, + lastAction: { kind: "Tap", on: { testTag: "LoginSubmit" }, applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: 0, @@ -138,7 +138,7 @@ test("raw whole-dollar input: single submit clears", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: parseTypedAmount("50"), prevTotalBalance: 5000, @@ -152,7 +152,7 @@ test("raw whole-dollar input: double submit fires", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: parseTypedAmount("50"), prevTotalBalance: 5000, @@ -166,7 +166,7 @@ test("decimal input from empty prior balance clears", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: parseTypedAmount("5.50"), prevTotalBalance: 0, @@ -180,7 +180,7 @@ test("DoubleTap kind with raw whole-dollar input fires", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "DoubleTap", on: submitOn }, + lastAction: { kind: "DoubleTap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: parseTypedAmount("100"), prevTotalBalance: 0, @@ -194,7 +194,7 @@ test("route gate: ledger landing with stale carrier is skipped", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "ledger", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 5000, prevTotalBalance: 0, @@ -208,7 +208,7 @@ test("route gate: add-transaction landing with double-submit delta is skipped", assert.equal( submitChangesBalanceByTypedAmount({ route: "add-transaction", - lastAction: { kind: "DoubleTap", on: submitOn }, + lastAction: { kind: "DoubleTap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 5000, prevTotalBalance: 0, @@ -222,7 +222,7 @@ test("route gate: null route is skipped", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: null, - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 5000, prevTotalBalance: 0, @@ -236,7 +236,7 @@ test("route gate: home landing with matching delta passes", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 5000, prevTotalBalance: 0, @@ -250,7 +250,7 @@ test("route gate: home landing with double-insert delta fires", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 5000, prevTotalBalance: 0, @@ -275,7 +275,7 @@ test("above 2^53 a healthy single submit is not reported", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 1600, prevTotalBalance: HUGE_BALANCE, @@ -289,7 +289,7 @@ test("above 2^53 a double-submit delta is not reported either", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 1600, prevTotalBalance: HUGE_BALANCE, @@ -303,7 +303,7 @@ test("an unreadable previous balance above 2^53 is not evidence", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 1600, prevTotalBalance: HUGE_BALANCE, @@ -320,7 +320,7 @@ test("typed amount above 2^53 is not evidence", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 1e23, prevTotalBalance: 0, @@ -336,7 +336,7 @@ test("boundary: a double submit landing exactly on MAX_SAFE_INTEGER still fires" assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 4503599627370495, prevTotalBalance: 0, @@ -350,7 +350,7 @@ test("boundary: a single submit landing exactly on MAX_SAFE_INTEGER passes", () assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 9007199254740991, prevTotalBalance: 0, @@ -364,7 +364,7 @@ test("boundary: one cent past MAX_SAFE_INTEGER stops being evidence", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 4503599627370496, prevTotalBalance: 0, @@ -381,7 +381,7 @@ test("a large but exact difference between safe balances still fires", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 500, prevTotalBalance: -9007199254740991, @@ -397,7 +397,7 @@ test("21-digit typed amount with an unmoved balance is not a violation", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: parseTypedAmount("999999999999999999999"), prevTotalBalance: 220900, @@ -417,7 +417,7 @@ test("freshness: two submits in the window is vacuous, not a conviction", () => assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "DoubleTap", on: submitOn }, + lastAction: { kind: "DoubleTap", on: submitOn, applied: true }, submitsInWindow: 2, typedAmount: 19600, prevTotalBalance: 0, @@ -431,7 +431,7 @@ test("freshness: two submits cannot convict even on a clean 2x delta", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 2, typedAmount: 500, prevTotalBalance: 1000, @@ -448,7 +448,7 @@ test("freshness boundary: exactly one submit is the window that convicts", () => assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "DoubleTap", on: submitOn }, + lastAction: { kind: "DoubleTap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 19600, prevTotalBalance: 0, @@ -462,7 +462,7 @@ test("freshness boundary: one submit with a healthy 1x delta still passes", () = assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 1, typedAmount: 19600, prevTotalBalance: 0, @@ -476,7 +476,7 @@ test("freshness boundary: three submits is vacuous", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 3, typedAmount: 500, prevTotalBalance: 0, @@ -493,7 +493,7 @@ test("freshness boundary: a window with no submit in it is vacuous", () => { assert.equal( submitChangesBalanceByTypedAmount({ route: "home", - lastAction: { kind: "Tap", on: submitOn }, + lastAction: { kind: "Tap", on: submitOn, applied: true }, submitsInWindow: 0, typedAmount: 500, prevTotalBalance: 1000, @@ -502,3 +502,21 @@ test("freshness boundary: a window with no submit in it is vacuous", () => { true, ); }); + +// applied: null is the runner saying it dispatched the tap and never learned +// whether it landed. A submit that committed nothing leaves the balance where +// it was, so demanding the typed amount of movement for it convicts an app that +// did exactly what it should have. +test("a submit the runner could not confirm demands no balance move", () => { + assert.equal( + submitChangesBalanceByTypedAmount({ + route: "home", + lastAction: { kind: "Tap", on: submitOn, applied: null }, + submitsInWindow: 1, + typedAmount: 500, + prevTotalBalance: 1000, + currTotalBalance: 1000, + }), + true, + ); +}); diff --git a/pkg/spec/test/folio-submit-window.test.ts b/pkg/spec/test/folio-submit-window.test.ts index da422ed..6e19281 100644 --- a/pkg/spec/test/folio-submit-window.test.ts +++ b/pkg/spec/test/folio-submit-window.test.ts @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { + committedTransactionsExceedSubmits, countSubmitsInWindow, isTxnSubmitTap, readHomeTotalBalance, @@ -149,3 +150,25 @@ test("an unreadable Home does not close the window", () => { assert.equal(trace[2]?.total, null); assert.equal(trace[3]?.submits, 2); }); + +// The window is an upper bound on the submits it holds, so a submit whose +// dispatch the runner could not confirm belongs in it: the tap may well have +// landed, and a bound that leaves it out is one the transaction it committed +// exceeds. That is the false conviction, a rise of one against a window of +// zero, on the property carrying most of the detection on android. +test("a submit the runner could not confirm still counts toward the window", () => { + const window = countSubmitsInWindow({ + previousCount: 0, + lastAction: { kind: "Tap", on: submitOn, applied: null }, + fresh: true, + }); + assert.equal(window.reported, 1); + assert.equal( + committedTransactionsExceedSubmits({ + countsBefore: { Travel: 3 }, + countsAfter: { Travel: 4 }, + submitsInWindow: window.reported, + }), + false, + ); +}); diff --git a/pkg/spec/test/folio-transition-frame.test.ts b/pkg/spec/test/folio-transition-frame.test.ts index f785642..64bcbf3 100644 --- a/pkg/spec/test/folio-transition-frame.test.ts +++ b/pkg/spec/test/folio-transition-frame.test.ts @@ -94,7 +94,7 @@ test("the measured android transition chain no longer convicts at delta 0", () = const step = ( tags: string[], totalText: string | undefined, - lastAction: { kind: string; on: string } | null, + lastAction: { kind: string; on: string; applied: true } | null, ) => { const route = routeOfFrame(SCREENS, frame(...tags)); const reading = readHomeTotalBalance({ route, totalText, previousCarrier: carrier }); @@ -104,8 +104,12 @@ test("the measured android transition chain no longer convicts at delta 0", () = return { route, total: reading.value, submits: window.reported }; }; - const back = { kind: "DoubleTap", on: "id:BackButton" }; - const phantomSubmit = { kind: "Tap", on: "testTag:AddTransactionScreen > testTag:TxnSubmit" }; + const back = { kind: "DoubleTap", on: "id:BackButton", applied: true as const }; + const phantomSubmit = { + kind: "Tap", + on: "testTag:AddTransactionScreen > testTag:TxnSubmit", + applied: true as const, + }; const transition = step(["AddTransactionScreen", "HomeScreen"], "$86,911.00", back); assert.equal(transition.route, null);