Merge branch 'runner-uncertain-last-action' into pr-73-followups

docs/manual/spec-language.md

@@ -29,7 +29,7 @@ Every extractor callback receives a `State`:
 interface State {
   ax: AccessibilityTree;
   snapshots: Record<string, unknown>;
-  lastAction: Action | null;
+  lastAction: (Action & { applied: true | null }) | null;
   logs: readonly LogEntry[];
   exceptions: readonly ExceptionRecord[];
   time: number;   // ms since run start
@@ -40,11 +40,13 @@ interface State {
 |---|---|
 | `ax` | Live UI hierarchy for this step |
 | `snapshots` | Key-value data pushed by the app SDK (empty if SDK not integrated) |
-| `lastAction` | The action dispatched in the previous step, or `null` on the first step |
+| `lastAction` | The action dispatched in the previous step, or `null` on the first step and on any step that dispatched nothing |
 | `logs` | Log entries collected since the previous step |
 | `exceptions` | Uncaught exceptions or `Sanderling.reportError()` calls since the previous step |
 | `time` | Milliseconds elapsed since the run started |
 
+`lastAction.applied` is `true` when the runner saw the dispatch succeed and `null` when the apply call failed with the action possibly already delivered: an RPC deadline can fire after the tap reached the app, and nothing can find out afterwards. So there are three states, not two. `state.lastAction === null` means no action ran; `applied === null` means one ran whose fate is unknown. A property that attributes an effect to the action ("this submit must move the balance by the typed amount") has to decline unless `applied` is `true`, or a timeout convicts a healthy app. A property that counts what the app COULD have done should include it: an unconfirmed submit belongs in an upper bound on how many submits a window holds.
+
 ## Selectors
 
 Selectors are passed to `ax.find()`, `ax.findAll()`, and element-scoped `.find()` / `.findAll()`.

examples/folio/sanderling/predicates.ts

@@ -123,10 +123,22 @@ export function readHomeCards<T>(args: {
   return { value: reading, carrier: reading, fresh: true };
 }
 
-function isTapOn(
-  lastAction: { kind?: string; on?: string | object } | null,
-  target: string,
-): boolean {
+// state.lastAction as the two hosts build it (internal/verifier/marshal.go
+// lastActionFields), read defensively: every field is what a Go struct decided
+// to emit, not something this file can trust a compile-time shape for.
+//
+// `applied` is true when the runner saw the action's dispatch succeed and null
+// when the apply call failed with the gesture possibly already delivered: an
+// RPC deadline can fire after the tap landed, and nothing can find out
+// afterwards. That is unknown, not "it did not happen", which is what
+// `lastAction === null` says.
+export interface ObservedAction {
+  kind?: string;
+  on?: string | object;
+  applied?: true | null;
+}
+
+function isTapOn(lastAction: ObservedAction | null, target: string): boolean {
   if (lastAction == null) return false;
   if (lastAction.kind !== "Tap" && lastAction.kind !== "DoubleTap") return false;
   const on = lastAction.on;
@@ -138,19 +150,27 @@ function isTapOn(
 // Repository.createTransaction: AddTransactionViewModel.submit() is the only
 // caller, AddTransactionEvent.Submit is the only thing that runs it, and the
 // TxnSubmit button's onClick is the only thing that sends that event.
-export function isTxnSubmitTap(
-  lastAction: { kind?: string; on?: string | object } | null,
-): boolean {
+export function isTxnSubmitTap(lastAction: ObservedAction | null): boolean {
   return isTapOn(lastAction, "TxnSubmit");
 }
 
 // Likewise the only action that creates an account.
-export function isAddAccountSubmitTap(
-  lastAction: { kind?: string; on?: string | object } | null,
-): boolean {
+export function isAddAccountSubmitTap(lastAction: ObservedAction | null): boolean {
   return isTapOn(lastAction, "AddAccountSubmit");
 }
 
+// Did the runner see this action's dispatch succeed? `applied` is null when the
+// apply call failed with the gesture possibly already delivered (an RPC
+// deadline can fire after the tap landed), and the runner has no way to find
+// out afterwards. Such an action may have caused anything the next reading
+// shows, so it counts toward how many submits a window COULD hold, but it never
+// licenses attributing an effect to it: a property that demands the effect of
+// an action that may never have run convicts the app of the runner's own
+// uncertainty.
+export function confirmedApplied(lastAction: ObservedAction | null): boolean {
+  return lastAction != null && lastAction.applied === true;
+}
+
 // Counts the submit actions inside the window the balance property compares
 // over: from the last Home total we read to this step, inclusive of this step's
 // action.
@@ -164,9 +184,15 @@ export function isAddAccountSubmitTap(
 //
 // The reset lands on `fresh`, the same event that advances the carrier, so the
 // count always describes exactly the interval the two compared totals span.
+//
+// A submit whose dispatch the runner could not confirm counts here, because
+// this number is an upper bound on the submits the window holds and the tap may
+// well have landed. Leaving it out is what convicted a healthy app:
+// committedTransactionsExceedSubmits saw a transaction rise of one against a
+// window of zero and called it a double submit.
 export function countSubmitsInWindow(args: {
   previousCount: number;
-  lastAction: { kind?: string; on?: string | object } | null;
+  lastAction: ObservedAction | null;
   fresh: boolean;
 }): { reported: number; next: number } {
   const { previousCount, lastAction, fresh } = args;
@@ -343,7 +369,7 @@ export function homeTxnCountsOf(cards: readonly CardReading[]): Record<string, T
 // same as fine, and both come back false here.
 export function createdAccountHasNonZeroBalance(args: {
   route: string | null;
-  lastAction: { kind?: string; on?: string | object } | null;
+  lastAction: ObservedAction | null;
   typedName: string | undefined;
   before: Account[] | null;
   after: Account[] | null;
@@ -351,6 +377,10 @@ export function createdAccountHasNonZeroBalance(args: {
   const { route, lastAction, before, after } = args;
   if (route !== "home") return false;
   if (!isAddAccountSubmitTap(lastAction)) return false;
+  // A creation nobody can confirm happened is not a creation to attribute a
+  // card to: the card that turned up may be an older account of the same name
+  // scrolling into view.
+  if (!confirmedApplied(lastAction)) return false;
   if (before === null || after === null) return false;
   const typed = (args.typedName ?? "").trim();
   if (typed === "") return false;
@@ -465,7 +495,7 @@ export function parseTypedAmount(text: string | undefined | null): number {
 // the bug: the double-tap is a single action.
 export function submitChangesBalanceByTypedAmount(args: {
   route: string | null;
-  lastAction: { kind?: string; on?: string | object } | null;
+  lastAction: ObservedAction | null;
   submitsInWindow: number;
   typedAmount: number;
   prevTotalBalance: number | null;
@@ -476,6 +506,10 @@ export function submitChangesBalanceByTypedAmount(args: {
 
   if (route !== "home") return true;
   if (!isTxnSubmitTap(lastAction)) return true;
+  // The whole rule is that the delta belongs to THIS submit. A submit the
+  // runner could not confirm may have committed nothing, and a balance that
+  // did not move is then exactly what a healthy app looks like.
+  if (!confirmedApplied(lastAction)) return true;
   if (submitsInWindow !== 1) return true;
   if (typedAmount === 0) return true;
   // An unknown total on either side is not evidence of anything. Comparing one

internal/runner/runner.go

@@ -298,11 +298,18 @@ func Run(ctx context.Context, options Options) (Summary, error) {
 				logger.Warn("apply error; marking step transitional", "step", stepIndex, "err", err)
 				transitional = true
 				applySkipped = true
-				lastAction = nil
+				// The error says the call failed, not that the gesture never
+				// reached the app: a deadline that fires after dispatch leaves
+				// the effect committed. Reporting no action here would let a
+				// property convict the app for an effect with no cause, so the
+				// action is reported with its fate unknown instead.
+				unconfirmed := nextAction
+				lastAction = &unconfirmed
 			} else {
 				consecutiveApplyFailures = 0
-				actionCopy := nextAction
-				lastAction = &actionCopy
+				applied := nextAction
+				applied.Applied = true
+				lastAction = &applied
 			}
 		} else {
 			lastAction = nil

internal/runner/uncertain_last_action_test.go

@@ -0,0 +1,150 @@
+package runner
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/priyanshujain/sanderling/internal/driver"
+	mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
+)
+
+// An apply error is not proof that nothing landed. An RPC deadline that fires
+// after the tap was dispatched leaves the transaction committed, and a runner
+// that reports "no action" for it hands
+// submitCommitsOneTransactionPerAction a rise of one transaction against a
+// window of zero submits: a conviction manufactured out of the runner's own
+// uncertainty, on the property carrying most of the detection on android.
+//
+// The spec below is the real folio predicate pair, imported from the example,
+// so what this asserts is the verdict the shipped property reaches.
+const uncertainApplySpecTemplate = `
+import { actions, always, extract, next, Tap } from "@sanderling/spec";
+import {
+  committedTransactionsExceedSubmits,
+  countSubmitsInWindow,
+} from "%s";
+
+let submits = 0;
+const submitsInWindow = extract("submitsInWindow", state => {
+  const window = countSubmitsInWindow({
+    previousCount: submits,
+    lastAction: state.lastAction,
+    fresh: true,
+  });
+  submits = window.next;
+  return window.reported;
+});
+
+const counts = extract("counts", state => {
+  const text = state.ax.find("id:TxnCount")?.text;
+  return text ? { Travel: parseInt(text, 10) } : null;
+});
+
+globalThis.properties = {
+  submitCommitsOneTransactionPerAction: always(
+    next(() =>
+      !committedTransactionsExceedSubmits({
+        countsBefore: counts.previous ?? null,
+        countsAfter: counts.current,
+        submitsInWindow: submitsInWindow.current,
+      }),
+    ),
+  ),
+};
+globalThis.actions = actions(() => [Tap({ on: "id:TxnSubmit" })]);
+`
+
+const homeWithTxnCount = `{"attributes":{"resource-id":"HomeScreen"},"children":[
+	{"attributes":{"resource-id":"TxnCount","text":"%d"},"children":[]},
+	{"attributes":{"resource-id":"TxnSubmit","bounds":"[40,80,240,160]"},"children":[],"clickable":true,"enabled":true}
+]}`
+
+// dispatchThenFailDriver is the device condition the runner cannot see through:
+// the tap reaches the app and commits, then the call the runner is waiting on
+// times out. Every later hierarchy read shows the committed transactions.
+type dispatchThenFailDriver struct {
+	*mockdriver.Driver
+	commitsPerTap int64
+	committed     atomic.Int64
+}
+
+func (d *dispatchThenFailDriver) Tap(context.Context, int, int) error {
+	return d.dispatchThenFail()
+}
+
+func (d *dispatchThenFailDriver) TapSelector(context.Context, string) error {
+	return d.dispatchThenFail()
+}
+
+func (d *dispatchThenFailDriver) dispatchThenFail() error {
+	d.committed.Add(d.commitsPerTap)
+	return errors.New("rpc error: code = DeadlineExceeded desc = context deadline exceeded")
+}
+
+func (d *dispatchThenFailDriver) Snapshot(context.Context) (string, driver.Image, error) {
+	return fmt.Sprintf(homeWithTxnCount, d.committed.Load()), driver.Image{}, nil
+}
+
+func TestRunner_ApplyErrorAfterDispatchDoesNotConvictTheSubmitCountingProperty(t *testing.T) {
+	predicates, err := filepath.Abs("../../examples/folio/sanderling/predicates.ts")
+	if err != nil {
+		t.Fatal(err)
+	}
+	spec := fmt.Sprintf(uncertainApplySpecTemplate, predicates)
+
+	run := func(t *testing.T, commitsPerTap int64) []ViolationRecord {
+		t.Helper()
+		state := newHarnessWithSpec(t, spec)
+		device := &dispatchThenFailDriver{Driver: state.mock, commitsPerTap: commitsPerTap}
+
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		summary, err := Run(ctx, Options{
+			Duration:    time.Hour,
+			IdleTimeout: 20 * time.Millisecond,
+			MaxSteps:    2,
+			Driver:      device,
+			Verifier:    state.verifier,
+			TraceWriter: state.writer,
+		})
+		if err != nil {
+			t.Fatalf("Run: %v", err)
+		}
+		if summary.Steps != 2 {
+			t.Fatalf("steps = %d, want 2; the run never reached the step that judges the pair", summary.Steps)
+		}
+		if got := device.committed.Load(); got != commitsPerTap*2 {
+			t.Fatalf("the device committed %d transaction(s), want %d; the taps never reached it",
+				got, commitsPerTap*2)
+		}
+		return summary.Violations
+	}
+
+	t.Run("one transaction per tap is not a double submit", func(t *testing.T) {
+		if violations := run(t, 1); len(violations) != 0 {
+			t.Errorf("the counting property convicted a healthy app: %v\n"+
+				"one transaction rose against a submit the runner dispatched but "+
+				"could not confirm, and the spec was told no action happened",
+				violations)
+		}
+	})
+
+	// The control. Without it a green above proves nothing: a property that
+	// never sees a comparable pair is silently vacuous and reports the same
+	// empty violation list.
+	t.Run("two transactions per tap still convicts", func(t *testing.T) {
+		violations := run(t, 2)
+		if len(violations) == 0 {
+			t.Fatal("the counting property missed a double submit; the harness never " +
+				"put the property in a position to fire, so the case above proves nothing")
+		}
+		if violations[0].Properties[0] != "submitCommitsOneTransactionPerAction" {
+			t.Errorf("violated %v, want submitCommitsOneTransactionPerAction", violations[0].Properties)
+		}
+	})
+}

internal/runner/web_last_action_test.go

@@ -3,6 +3,7 @@ package runner
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"testing"
 	"time"
 
@@ -71,7 +72,47 @@ func TestRunner_WebInstallsLastActionInThePage(t *testing.T) {
 	// Every later step carries what the runner actually applied. The shape is
 	// the goja host's (internal/verifier/marshal.go lastActionFields), pinned
 	// against it by TestLastAction_WebJSONMatchesTheGojaObject.
-	const want = `{"kind":"Tap","on":"id:TxnSubmit"}`
+	const want = `{"kind":"Tap","applied":true,"on":"id:TxnSubmit"}`
+	if web.installed[1] != want {
+		t.Errorf("step 2 installed %s, want %s", web.installed[1], want)
+	}
+}
+
+// failingTapWebDriver dispatches the tap and then fails the call, the shape an
+// RPC deadline takes: the page has the click, the runner has an error.
+type failingTapWebDriver struct {
+	*tappingWebDriver
+}
+
+func (d *failingTapWebDriver) Tap(context.Context, int, int) error {
+	return errors.New("rpc error: code = DeadlineExceeded desc = context deadline exceeded")
+}
+
+// The web leg of the same three states the goja host reports. "applied":null is
+// not "no action": a property gated on the last action still sees the tap and
+// decides for itself, which it cannot do if the page is handed a bare null.
+func TestRunner_WebInstallsAnUnconfirmedActionWithItsFateUnknown(t *testing.T) {
+	state := newHarnessWithSpec(t, lastActionSpec)
+	web := &failingTapWebDriver{tappingWebDriver: &tappingWebDriver{Driver: state.mock}}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if _, err := Run(ctx, Options{
+		Duration:    time.Hour,
+		IdleTimeout: 20 * time.Millisecond,
+		MaxSteps:    2,
+		Driver:      web,
+		Verifier:    state.verifier,
+		TraceWriter: state.writer,
+	}); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	if len(web.installed) < 2 {
+		t.Fatalf("the page was handed lastAction %d time(s); the web path never installed it",
+			len(web.installed))
+	}
+	const want = `{"kind":"Tap","applied":null,"on":"id:TxnSubmit"}`
 	if web.installed[1] != want {
 		t.Errorf("step 2 installed %s, want %s", web.installed[1], want)
 	}

internal/verifier/marshal.go

@@ -344,7 +344,19 @@ func lastActionFields(action *Action) []actionField {
 	point := func(x, y int) []actionField {
 		return []actionField{{key: "x", value: x}, {key: "y", value: y}}
 	}
-	fields := []actionField{{key: "kind", value: string(action.Kind)}}
+	// An action whose apply call failed is not an action that did not happen:
+	// the dispatch may have landed before the error. That is unknown, and
+	// unknown is null here for the same reason every other absence in the spec
+	// surface is, so a property decides for itself instead of being handed a
+	// "nothing happened" the runner cannot vouch for.
+	var applied any
+	if action.Applied {
+		applied = true
+	}
+	fields := []actionField{
+		{key: "kind", value: string(action.Kind)},
+		{key: "applied", value: applied},
+	}
 	if action.On != "" {
 		fields = append(fields, actionField{key: "on", value: action.On})
 	}
@@ -397,7 +409,7 @@ func objectFromFields(runtime *goja.Runtime, fields []actionField) *goja.Object
 // has no Go-side state object to read: the runner pushes this JSON into the
 // page before each extractor evaluation. A nil action encodes as JSON null,
 // the same value the goja host reports on the first step of a run and after a
-// step whose action was never applied.
+// step whose action was never dispatched.
 func EncodeLastAction(action *Action) json.RawMessage {
 	if action == nil {
 		return json.RawMessage("null")

internal/verifier/marshal_test.go

@@ -168,6 +168,7 @@ func TestLastAction_WebJSONMatchesTheGojaObject(t *testing.T) {
 	}{
 		{"nil", nil},
 		{"Tap", &Action{Kind: ActionKindTap, On: "id:TxnSubmit", X: 12, Y: 34}},
+		{"TapApplied", &Action{Kind: ActionKindTap, On: "id:TxnSubmit", Applied: true}},
 		{"TapWithoutSelector", &Action{Kind: ActionKindTap, X: 12, Y: 34}},
 		{"DoubleTap", &Action{Kind: ActionKindDoubleTap, On: `desc:say "hi" <b>`}},
 		{"InputText", &Action{Kind: ActionKindInputText, On: "id:field", Text: "50"}},
@@ -193,3 +194,42 @@ func TestLastAction_WebJSONMatchesTheGojaObject(t *testing.T) {
 		})
 	}
 }
+
+// A spec has to be able to tell three things apart: no action ran, an action
+// ran, and an action was dispatched whose fate the runner cannot vouch for.
+// The third used to be reported as the first, which is how a property that
+// reasons "an effect landed with no action to cause it" convicts an app over
+// an RPC deadline.
+func TestLastAction_SeparatesNoActionFromAnActionOfUnknownFate(t *testing.T) {
+	verifier := newVerifier(t)
+	mustLoad(t, verifier, `
+		globalThis.fate = __sanderling__.extract(state =>
+			state.lastAction === null ? "no action"
+			: state.lastAction.applied === true ? "applied"
+			: state.lastAction.applied === null ? "unknown"
+			: "unreadable");
+	`)
+
+	for _, testCase := range []struct {
+		name   string
+		action *Action
+		want   string
+	}{
+		{"nothing ran", nil, "no action"},
+		{"dispatch confirmed", &Action{Kind: ActionKindTap, On: "id:TxnSubmit", Applied: true}, "applied"},
+		{"dispatch unconfirmed", &Action{Kind: ActionKindTap, On: "id:TxnSubmit"}, "unknown"},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			if err := verifier.PushSnapshot(SnapshotInput{
+				Snapshots:  Snapshots{},
+				LastAction: testCase.action,
+			}); err != nil {
+				t.Fatal(err)
+			}
+			handle := verifier.runtime.GlobalObject().Get("fate").ToObject(verifier.runtime)
+			if got := handle.Get("current").String(); got != testCase.want {
+				t.Errorf("the spec read %q, want %q", got, testCase.want)
+			}
+		})
+	}
+}

internal/verifier/types.go

@@ -33,6 +33,11 @@ type Action struct {
 	// Direction is the scroll direction for ActionKindScroll: one of "up",
 	// "down", "left", "right". Empty for every other kind.
 	Direction string
+	// Applied is meaningful only on the action a step reports to the spec as
+	// state.lastAction: true when the runner saw the dispatch succeed, false
+	// when the apply call failed and nothing can say whether the action
+	// reached the app. The spec is told which of the two it is.
+	Applied bool
 }
 
 // LogEntry mirrors a logcat line captured between steps.

pkg/spec/src/index.ts

@@ -13,6 +13,7 @@ export type {
   InputTextAction,
   Key,
   KnownAttrSelectors,
+  LastAction,
   LogEntry,
   LongPressAction,
   Point,

pkg/spec/src/types.ts

@@ -102,10 +102,20 @@ export interface ExceptionRecord {
   unixMillis?: number;
 }
 
+/**
+ * The previous step's action as the runner reports it. `applied` is true when
+ * the runner saw the dispatch succeed and null when the apply call failed with
+ * the gesture possibly already delivered: an RPC deadline can fire after the
+ * tap landed. Null is unknown, not "it did not happen" (`state.lastAction` is
+ * itself null for that), so a property attributing an effect to this action
+ * has to decline unless `applied` is true.
+ */
+export type LastAction = Action & { applied: true | null };
+
 export interface State {
   snapshots: Snapshots;
   ax: AccessibilityTree;
-  lastAction: Action | null;
+  lastAction: LastAction | null;
   time: number;
   logs: readonly LogEntry[];
   exceptions: readonly ExceptionRecord[];

pkg/spec/test/folio-new-account.test.ts

@@ -3,8 +3,12 @@ import { test } from "node:test";
 
 import { createdAccountHasNonZeroBalance } from "../../../examples/folio/sanderling/predicates.ts";
 
-const created = { kind: "Tap", on: "testTag:AddAccountScreen > testTag:AddAccountSubmit" };
-const idle = { kind: "Tap", on: "testTag:HomeScreen > testTag:AccountCard" };
+const created = {
+  kind: "Tap",
+  on: "testTag:AddAccountScreen > testTag:AddAccountSubmit",
+  applied: true as const,
+};
+const idle = { kind: "Tap", on: "testTag:HomeScreen > testTag:AccountCard", applied: true as const };
 
 const account = (name: string, balance: number | null) => ({ name, balance });
 
@@ -27,7 +31,7 @@ test("a double-tapped create is judged the same way", () => {
   assert.equal(
     createdAccountHasNonZeroBalance({
       route: "home",
-      lastAction: { kind: "DoubleTap", on: "id:AddAccountSubmit" },
+      lastAction: { kind: "DoubleTap", on: "id:AddAccountSubmit", applied: true },
       typedName: "Travel",
       before: [account("Checking", 0)],
       after: [account("Checking", 0), account("Travel", 5000)],
@@ -233,3 +237,20 @@ test("a card that was already there is not a card that was just created", () =>
     false,
   );
 });
+
+// The apply call failed with the gesture possibly already delivered, so nobody
+// knows whether that account was created. The card carrying the typed name may
+// be an older one that scrolled into view, and attributing it to a creation
+// that may never have happened is a conviction built on a guess.
+test("a create the runner could not confirm attributes nothing", () => {
+  assert.equal(
+    createdAccountHasNonZeroBalance({
+      route: "home",
+      lastAction: { ...created, applied: null },
+      typedName: "Travel",
+      before: [account("Checking", 0)],
+      after: [account("Checking", 0), account("Travel", 5000)],
+    }),
+    false,
+  );
+});

pkg/spec/test/folio-submit-balance-predicate.test.ts

@@ -12,7 +12,7 @@ test("single submit: delta matches typed amount", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 500,
       prevTotalBalance: 1000,
@@ -26,7 +26,7 @@ test("double submit: delta is twice the typed amount, fires", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 500,
       prevTotalBalance: 1000,
@@ -40,7 +40,7 @@ test("DoubleTap kind also caught when delta exceeds typed amount", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "DoubleTap", on: submitOn },
+      lastAction: { kind: "DoubleTap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 500,
       prevTotalBalance: 0,
@@ -54,7 +54,7 @@ test("wrong action kind: vacuous true even with mismatch", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "InputText", on: submitOn },
+      lastAction: { kind: "InputText", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 500,
       prevTotalBalance: 1000,
@@ -68,7 +68,7 @@ test("wrong target: vacuous true even with mismatch", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: "testTag:LoginScreen > testTag:LoginSubmit" },
+      lastAction: { kind: "Tap", on: "testTag:LoginScreen > testTag:LoginSubmit", applied: true },
       submitsInWindow: 1,
       typedAmount: 500,
       prevTotalBalance: 1000,
@@ -96,7 +96,7 @@ test("zero typedAmount: vacuous true", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 0,
       prevTotalBalance: 1000,
@@ -110,7 +110,7 @@ test("selector as object: coerced safely and TxnSubmit detected", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: { testTag: "TxnSubmit" } },
+      lastAction: { kind: "Tap", on: { testTag: "TxnSubmit" }, applied: true },
       submitsInWindow: 1,
       typedAmount: 500,
       prevTotalBalance: 0,
@@ -124,7 +124,7 @@ test("selector as object without TxnSubmit: vacuous true", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: { testTag: "LoginSubmit" } },
+      lastAction: { kind: "Tap", on: { testTag: "LoginSubmit" }, applied: true },
       submitsInWindow: 1,
       typedAmount: 500,
       prevTotalBalance: 0,
@@ -138,7 +138,7 @@ test("raw whole-dollar input: single submit clears", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: parseTypedAmount("50"),
       prevTotalBalance: 5000,
@@ -152,7 +152,7 @@ test("raw whole-dollar input: double submit fires", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: parseTypedAmount("50"),
       prevTotalBalance: 5000,
@@ -166,7 +166,7 @@ test("decimal input from empty prior balance clears", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: parseTypedAmount("5.50"),
       prevTotalBalance: 0,
@@ -180,7 +180,7 @@ test("DoubleTap kind with raw whole-dollar input fires", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "DoubleTap", on: submitOn },
+      lastAction: { kind: "DoubleTap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: parseTypedAmount("100"),
       prevTotalBalance: 0,
@@ -194,7 +194,7 @@ test("route gate: ledger landing with stale carrier is skipped", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "ledger",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 5000,
       prevTotalBalance: 0,
@@ -208,7 +208,7 @@ test("route gate: add-transaction landing with double-submit delta is skipped",
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "add-transaction",
-      lastAction: { kind: "DoubleTap", on: submitOn },
+      lastAction: { kind: "DoubleTap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 5000,
       prevTotalBalance: 0,
@@ -222,7 +222,7 @@ test("route gate: null route is skipped", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: null,
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 5000,
       prevTotalBalance: 0,
@@ -236,7 +236,7 @@ test("route gate: home landing with matching delta passes", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 5000,
       prevTotalBalance: 0,
@@ -250,7 +250,7 @@ test("route gate: home landing with double-insert delta fires", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 5000,
       prevTotalBalance: 0,
@@ -275,7 +275,7 @@ test("above 2^53 a healthy single submit is not reported", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 1600,
       prevTotalBalance: HUGE_BALANCE,
@@ -289,7 +289,7 @@ test("above 2^53 a double-submit delta is not reported either", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 1600,
       prevTotalBalance: HUGE_BALANCE,
@@ -303,7 +303,7 @@ test("an unreadable previous balance above 2^53 is not evidence", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 1600,
       prevTotalBalance: HUGE_BALANCE,
@@ -320,7 +320,7 @@ test("typed amount above 2^53 is not evidence", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 1e23,
       prevTotalBalance: 0,
@@ -336,7 +336,7 @@ test("boundary: a double submit landing exactly on MAX_SAFE_INTEGER still fires"
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 4503599627370495,
       prevTotalBalance: 0,
@@ -350,7 +350,7 @@ test("boundary: a single submit landing exactly on MAX_SAFE_INTEGER passes", ()
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 9007199254740991,
       prevTotalBalance: 0,
@@ -364,7 +364,7 @@ test("boundary: one cent past MAX_SAFE_INTEGER stops being evidence", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 4503599627370496,
       prevTotalBalance: 0,
@@ -381,7 +381,7 @@ test("a large but exact difference between safe balances still fires", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 500,
       prevTotalBalance: -9007199254740991,
@@ -397,7 +397,7 @@ test("21-digit typed amount with an unmoved balance is not a violation", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: parseTypedAmount("999999999999999999999"),
       prevTotalBalance: 220900,
@@ -417,7 +417,7 @@ test("freshness: two submits in the window is vacuous, not a conviction", () =>
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "DoubleTap", on: submitOn },
+      lastAction: { kind: "DoubleTap", on: submitOn, applied: true },
       submitsInWindow: 2,
       typedAmount: 19600,
       prevTotalBalance: 0,
@@ -431,7 +431,7 @@ test("freshness: two submits cannot convict even on a clean 2x delta", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 2,
       typedAmount: 500,
       prevTotalBalance: 1000,
@@ -448,7 +448,7 @@ test("freshness boundary: exactly one submit is the window that convicts", () =>
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "DoubleTap", on: submitOn },
+      lastAction: { kind: "DoubleTap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 19600,
       prevTotalBalance: 0,
@@ -462,7 +462,7 @@ test("freshness boundary: one submit with a healthy 1x delta still passes", () =
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 1,
       typedAmount: 19600,
       prevTotalBalance: 0,
@@ -476,7 +476,7 @@ test("freshness boundary: three submits is vacuous", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 3,
       typedAmount: 500,
       prevTotalBalance: 0,
@@ -493,7 +493,7 @@ test("freshness boundary: a window with no submit in it is vacuous", () => {
   assert.equal(
     submitChangesBalanceByTypedAmount({
       route: "home",
-      lastAction: { kind: "Tap", on: submitOn },
+      lastAction: { kind: "Tap", on: submitOn, applied: true },
       submitsInWindow: 0,
       typedAmount: 500,
       prevTotalBalance: 1000,
@@ -502,3 +502,21 @@ test("freshness boundary: a window with no submit in it is vacuous", () => {
     true,
   );
 });
+
+// applied: null is the runner saying it dispatched the tap and never learned
+// whether it landed. A submit that committed nothing leaves the balance where
+// it was, so demanding the typed amount of movement for it convicts an app that
+// did exactly what it should have.
+test("a submit the runner could not confirm demands no balance move", () => {
+  assert.equal(
+    submitChangesBalanceByTypedAmount({
+      route: "home",
+      lastAction: { kind: "Tap", on: submitOn, applied: null },
+      submitsInWindow: 1,
+      typedAmount: 500,
+      prevTotalBalance: 1000,
+      currTotalBalance: 1000,
+    }),
+    true,
+  );
+});

pkg/spec/test/folio-submit-window.test.ts

@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
 import { test } from "node:test";
 
 import {
+  committedTransactionsExceedSubmits,
   countSubmitsInWindow,
   isTxnSubmitTap,
   readHomeTotalBalance,
@@ -149,3 +150,25 @@ test("an unreadable Home does not close the window", () => {
   assert.equal(trace[2]?.total, null);
   assert.equal(trace[3]?.submits, 2);
 });
+
+// The window is an upper bound on the submits it holds, so a submit whose
+// dispatch the runner could not confirm belongs in it: the tap may well have
+// landed, and a bound that leaves it out is one the transaction it committed
+// exceeds. That is the false conviction, a rise of one against a window of
+// zero, on the property carrying most of the detection on android.
+test("a submit the runner could not confirm still counts toward the window", () => {
+  const window = countSubmitsInWindow({
+    previousCount: 0,
+    lastAction: { kind: "Tap", on: submitOn, applied: null },
+    fresh: true,
+  });
+  assert.equal(window.reported, 1);
+  assert.equal(
+    committedTransactionsExceedSubmits({
+      countsBefore: { Travel: 3 },
+      countsAfter: { Travel: 4 },
+      submitsInWindow: window.reported,
+    }),
+    false,
+  );
+});

pkg/spec/test/folio-transition-frame.test.ts

@@ -94,7 +94,7 @@ test("the measured android transition chain no longer convicts at delta 0", () =
   const step = (
     tags: string[],
     totalText: string | undefined,
-    lastAction: { kind: string; on: string } | null,
+    lastAction: { kind: string; on: string; applied: true } | null,
   ) => {
     const route = routeOfFrame(SCREENS, frame(...tags));
     const reading = readHomeTotalBalance({ route, totalText, previousCarrier: carrier });
@@ -104,8 +104,12 @@ test("the measured android transition chain no longer convicts at delta 0", () =
     return { route, total: reading.value, submits: window.reported };
   };
 
-  const back = { kind: "DoubleTap", on: "id:BackButton" };
-  const phantomSubmit = { kind: "Tap", on: "testTag:AddTransactionScreen > testTag:TxnSubmit" };
+  const back = { kind: "DoubleTap", on: "id:BackButton", applied: true as const };
+  const phantomSubmit = {
+    kind: "Tap",
+    on: "testTag:AddTransactionScreen > testTag:TxnSubmit",
+    applied: true as const,
+  };
 
   const transition = step(["AddTransactionScreen", "HomeScreen"], "$86,911.00", back);
   assert.equal(transition.route, null);