From 19a470121fbfd0f6fdb176b86d01fcafc61998b1 Mon Sep 17 00:00:00 2001 From: pjay Date: Fri, 5 Jun 2026 23:42:39 +0530 Subject: [PATCH] fix: iOS spec driving, InputText replace semantics, native DoubleTap (#58) * fix(hierarchy): bounds-containment fallback for scoped and path queries Compose on iOS surfaces a testTag node as an empty leaf sibling of the content it labels instead of as an ancestor, so descendant search under the tagged node finds nothing and every path or scoped query returns null. When structural search yields no match, fall back to nodes whose bounds lie inside the scope node's bounds. * feat(sidecar): derive iOS clickable and editable from element type The XCTest hierarchy mapping dropped the element type, leaving no clickable or editable flags on iOS, so the fuzzer's tap and typing verbs never found a candidate inside the app. Map the raw accessibility tree directly and derive clickable, editable, scrollable, and class from the XCUIElementType raw value. * feat(proto): add EraseText RPC for InputText replace semantics * feat(driver): add EraseText to the device driver surface * fix(runner): erase existing field text before InputText InputText appended on native platforms, so repeated draws grew fields without bound. The folio fuzz run wedged on the add-account screen: each draw concatenated another name until the 40-character validation error became permanent. Replace semantics also makes retried typing idempotent. The web driver already replaced via select-all; native now matches. * feat(sidecar): EraseText backend support on android and ios * fix(folio): saturation-gate account creation in the spec The 2-3 step add-account loop outcompeted the 5-step transaction chain at every weighted re-draw, so runs filled with account creation and rarely exercised the balance properties. Stop offering add-account once three accounts exist; the renormalized weights then favor the transaction flow at every step of its chain. * fix(folio): author spec weights to match testing intent Revert the account saturation gate: it starved newAccountBalanceIsZero once it tripped, and a magic account count is app-state tuning, not intent. Instead weight the generators by what the properties need: the transaction chain leads, account creation stays exercised, and doubleTaps gets explicit weight everywhere because double-submission idempotency is what the spec is testing for. * fix(folio): lower doubleTaps weight to 5 * fix(sidecar): surface visible text on iOS static elements Static text and button strings live in the accessibility label on iOS, so the text attribute came through empty and every balance extractor parsed to zero, silently disarming both folio properties. Non-editable elements now fall back title, value, then label; editable fields keep value-only so an empty field's caption does not read as content. * feat(driver): native DoubleTap RPC for a tight inter-tap gap Composing two Tap round trips from the Go client spread the taps by hundreds of milliseconds on iOS, wide enough for the app to navigate between them, so double-submission races could never reproduce. The sidecar now lands both taps back-to-back next to the device transport. * feat(sidecar): pipeline iOS double-tap requests Queue the second tap at the XCTest runner while the first executes. The runner serializes handlers, so this is the tightest gap the transport allows (~350ms per tap round trip); recorded here with measurements for the iOS double-tap limitation. --- examples/folio/sanderling/spec.ts | 19 +- internal/driver/chrome/driver.go | 17 ++ internal/driver/driver.go | 4 + internal/driver/mock/mock.go | 36 ++- internal/driver/sidecar/client.go | 19 +- internal/hierarchy/hierarchy.go | 181 +++++++------ internal/hierarchy/hierarchy_test.go | 142 +++++++++++ internal/runner/runner.go | 23 ++ internal/runner/runner_test.go | 46 ++++ proto/driverpb/driver.pb.go | 238 +++++++++++------- proto/driverpb/driver.proto | 12 + proto/driverpb/driver_grpc.pb.go | 88 +++++++ proto/driverpb/driver_test.go | 2 + .../dev/sanderling/sidecar/DriverBackend.kt | 153 ++++++++++- .../dev/sanderling/sidecar/DriverService.kt | 15 ++ .../sanderling/sidecar/DriverServiceTest.kt | 37 +++ .../dev/sanderling/sidecar/InputTextTest.kt | 12 + .../sanderling/sidecar/IosHierarchyTest.kt | 126 ++++++++++ .../sanderling/sidecar/SnapshotHandlerTest.kt | 1 + 19 files changed, 981 insertions(+), 190 deletions(-) create mode 100644 sidecar/src/test/kotlin/dev/sanderling/sidecar/IosHierarchyTest.kt diff --git a/examples/folio/sanderling/spec.ts b/examples/folio/sanderling/spec.ts index 1e2f7a3..649238a 100644 --- a/examples/folio/sanderling/spec.ts +++ b/examples/folio/sanderling/spec.ts @@ -10,7 +10,7 @@ import { weighted, whenRoute, } from "@sanderling/spec"; -import { defaultActions } from "@sanderling/spec/defaults"; +import { defaultActions, doubleTaps } from "@sanderling/spec/defaults"; import { computeHomeTotalBalance, parseTypedAmount, @@ -174,11 +174,16 @@ export const properties = { export const setup = login; -// Targeted depth (addAccount / addTxn) drives the deep flows; defaultActions -// adds breadth so the fuzzer wanders the whole app and types edge-case values -// into every field, stressing the balance invariants above. +// Weights declare testing intent. The transaction chain is the focus: it is +// the deepest flow and both balance properties observe it. Account creation +// stays in the mix because newAccountBalanceIsZero needs fresh accounts to +// fire. doubleTaps gets explicit weight on every screen because rapid +// double-submission is a failure mode these forms must be idempotent under. +// defaultActions adds breadth so the fuzzer wanders the whole app and types +// edge-case values into every field. export const actionsRoot = weighted( - [50, addAccount], - [30, addTxn], - [20, defaultActions], + [25, addAccount], + [45, addTxn], + [5, doubleTaps], + [25, defaultActions], ); diff --git a/internal/driver/chrome/driver.go b/internal/driver/chrome/driver.go index 4dc98eb..df8aad1 100644 --- a/internal/driver/chrome/driver.go +++ b/internal/driver/chrome/driver.go @@ -193,6 +193,23 @@ func (d *Driver) InputText(_ context.Context, text string) error { ) } +// EraseText clears the focused field. InputText above already replaces via +// select-all, so the character count is not needed to bound the deletion. +func (d *Driver) EraseText(_ context.Context, _ int) error { + return chromedp.Run(d.tabCtx, + chromedp.ActionFunc(func(ctx context.Context) error { + if err := chromedp.Evaluate(` + (function() { + const el = document.activeElement; + if (el && typeof el.select === 'function') el.select(); + })()`, nil).Do(ctx); err != nil { + return err + } + return input.InsertText("").Do(ctx) + }), + ) +} + func (d *Driver) Swipe(_ context.Context, fromX, fromY, toX, toY int, duration time.Duration) error { millis := max(duration.Milliseconds(), 50) script := fmt.Sprintf(` diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 6eb62f9..e4757d2 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -20,6 +20,10 @@ type DeviceDriver interface { DoubleTap(ctx context.Context, x, y int) error DoubleTapSelector(ctx context.Context, selector string) error InputText(ctx context.Context, text string) error + // EraseText deletes characterCount characters from the focused field. + // The runner calls it before InputText so the verb replaces existing + // content instead of appending to it. + EraseText(ctx context.Context, characterCount int) error Swipe(ctx context.Context, fromX, fromY, toX, toY int, duration time.Duration) error PressKey(ctx context.Context, key string) error LongPress(ctx context.Context, x, y int) error diff --git a/internal/driver/mock/mock.go b/internal/driver/mock/mock.go index a6f0f13..f60c323 100644 --- a/internal/driver/mock/mock.go +++ b/internal/driver/mock/mock.go @@ -19,6 +19,7 @@ const ( ActionDoubleTap ActionKind = "double_tap" ActionDoubleTapSelector ActionKind = "double_tap_selector" ActionInputText ActionKind = "input_text" + ActionEraseText ActionKind = "erase_text" ActionSwipe ActionKind = "swipe" ActionPressKey ActionKind = "press_key" ActionLongPress ActionKind = "long_press" @@ -32,19 +33,20 @@ const ( ) type Action struct { - Kind ActionKind - BundleID string - ClearState bool - X, Y int - FromX, FromY int - ToX, ToY int - Duration time.Duration - Selector string - Text string - Key string - LogLevel string - LogSince time.Time - Idle time.Duration + Kind ActionKind + BundleID string + ClearState bool + X, Y int + FromX, FromY int + ToX, ToY int + Duration time.Duration + Selector string + Text string + CharacterCount int + Key string + LogLevel string + LogSince time.Time + Idle time.Duration } // Driver is an in-memory Driver implementation for unit tests. @@ -210,6 +212,14 @@ func (d *Driver) InputText(ctx context.Context, text string) error { return nil } +func (d *Driver) EraseText(ctx context.Context, characterCount int) error { + if err := d.failure(ActionEraseText); err != nil { + return err + } + d.record(Action{Kind: ActionEraseText, CharacterCount: characterCount}) + return nil +} + func (d *Driver) Swipe(ctx context.Context, fromX, fromY, toX, toY int, duration time.Duration) error { if err := d.failure(ActionSwipe); err != nil { return err diff --git a/internal/driver/sidecar/client.go b/internal/driver/sidecar/client.go index 7374a47..82f68a9 100644 --- a/internal/driver/sidecar/client.go +++ b/internal/driver/sidecar/client.go @@ -104,14 +104,18 @@ func (c *Client) TapSelector(ctx context.Context, selector string) error { return err } -// doubleTapGap is the inter-tap delay for DoubleTap: short enough to land both -// events inside a sub-100 ms race window, long enough for the sidecar to -// serialize two MotionEvent streams. The sidecar exposes no native double-tap -// RPC, so the gesture is two Taps with this gap. +// doubleTapGap is the inter-tap delay for the selector fallback: short enough +// to land both events inside a sub-100 ms race window, long enough for the +// sidecar to serialize two MotionEvent streams. const doubleTapGap = 50 * time.Millisecond +// DoubleTap dispatches the native RPC so the backend can land both taps as +// close together as the platform allows. Composing two Tap round trips from +// here spreads them by hundreds of milliseconds on iOS, wide enough for +// navigation to interleave between the taps. func (c *Client) DoubleTap(ctx context.Context, x, y int) error { - return doubleTap(ctx, func() error { return c.Tap(ctx, x, y) }) + _, err := c.stub.DoubleTap(ctx, &driverpb.Point{X: int32(x), Y: int32(y)}) + return err } func (c *Client) DoubleTapSelector(ctx context.Context, selector string) error { @@ -137,6 +141,11 @@ func (c *Client) InputText(ctx context.Context, text string) error { return err } +func (c *Client) EraseText(ctx context.Context, characterCount int) error { + _, err := c.stub.EraseText(ctx, &driverpb.EraseTextRequest{CharacterCount: int32(characterCount)}) + return err +} + func (c *Client) Swipe(ctx context.Context, fromX, fromY, toX, toY int, duration time.Duration) error { _, err := c.stub.Swipe(ctx, &driverpb.SwipeRequest{ From: &driverpb.Point{X: int32(fromX), Y: int32(fromY)}, diff --git a/internal/hierarchy/hierarchy.go b/internal/hierarchy/hierarchy.go index ea87bcf..9135649 100644 --- a/internal/hierarchy/hierarchy.go +++ b/internal/hierarchy/hierarchy.go @@ -75,6 +75,7 @@ type Element struct { type Node struct { Element Children []*Node `json:"-"` + tree *Tree } // Tree is a flat collection of every node in a hierarchy dump, in pre-order. @@ -180,7 +181,7 @@ func Parse(text string) (*Tree, error) { } func walkNode(node *treeNodeJSON, tree *Tree) *Node { - n := &Node{Element: *elementFromNode(node)} + n := &Node{Element: *elementFromNode(node), tree: tree} tree.Elements = append(tree.Elements, &n.Element) for i := range node.Children { n.Children = append(n.Children, walkNode(&node.Children[i], tree)) @@ -333,69 +334,50 @@ func (t *Tree) FindAllBySelectorPath(path []Selector) []*Node { return t.Root.FindAllBySelectorPath(path) } -// Find returns the first Node in this node's subtree (descendants only) matching -// the string selector. Path queries within the selector are not supported here. +// Find returns the first Node scoped to this node (descendants, with spatial +// fallback) matching the string selector. Path queries within the selector are +// not supported here. func (n *Node) Find(selector string) *Node { - kind, value, ok := parseSelector(selector) - if !ok { - return nil - } - for _, child := range n.Children { - if nodes := searchSubtree(child, kind, value); len(nodes) > 0 { - return nodes[0] - } - } - return nil + return firstNode(n.FindAll(selector)) } -// FindAll returns all Nodes in this node's subtree (descendants only) matching -// the string selector. +// FindAll returns all Nodes scoped to this node (descendants, with spatial +// fallback) matching the string selector. func (n *Node) FindAll(selector string) []*Node { kind, value, ok := parseSelector(selector) if !ok { return nil } - var result []*Node - for _, child := range n.Children { - result = append(result, searchSubtree(child, kind, value)...) - } - return result + return n.scopedNodes(func(element *Element) bool { + return match(element, kind, value) + }) } -// FindBySelector returns the first Node in this node's subtree matching sel (AND semantics). +// FindBySelector returns the first Node scoped to this node matching sel (AND semantics). func (n *Node) FindBySelector(sel Selector) *Node { - for _, child := range n.Children { - if nodes := searchSubtreeBySelector(child, sel); len(nodes) > 0 { - return nodes[0] - } - } - return nil + return firstNode(n.FindAllBySelector(sel)) } -// FindAllBySelector returns all Nodes in this node's subtree matching sel (AND semantics). +// FindAllBySelector returns all Nodes scoped to this node matching sel (AND semantics). func (n *Node) FindAllBySelector(sel Selector) []*Node { - var result []*Node - for _, child := range n.Children { - result = append(result, searchSubtreeBySelector(child, sel)...) - } - return result + return n.scopedNodes(func(element *Element) bool { + return matchSelector(element, sel) + }) } // FindBySelectorPath walks a chain of selectors. The first selector is matched -// against descendants of the receiver; each subsequent selector is matched -// against descendants of the previous match. Returns the deepest match or nil. +// in the receiver's scope; each subsequent selector is matched in the scope of +// the previous match. Returns the deepest match or nil. func (n *Node) FindBySelectorPath(path []Selector) *Node { if len(path) == 0 { return nil } - for _, child := range n.Children { - for _, candidate := range searchSubtreeBySelector(child, path[0]) { - if len(path) == 1 { - return candidate - } - if deeper := candidate.FindBySelectorPath(path[1:]); deeper != nil { - return deeper - } + for _, candidate := range n.FindAllBySelector(path[0]) { + if len(path) == 1 { + return candidate + } + if deeper := candidate.FindBySelectorPath(path[1:]); deeper != nil { + return deeper } } return nil @@ -407,19 +389,86 @@ func (n *Node) FindAllBySelectorPath(path []Selector) []*Node { if len(path) == 0 { return nil } + var result []*Node + for _, candidate := range n.FindAllBySelector(path[0]) { + if len(path) == 1 { + result = append(result, candidate) + continue + } + result = append(result, candidate.FindAllBySelectorPath(path[1:])...) + } + return result +} + +func firstNode(nodes []*Node) *Node { + if len(nodes) == 0 { + return nil + } + return nodes[0] +} + +// scopedNodes returns this node's descendants matching accept, in pre-order. +// When no descendant matches, nodes spatially contained in this node's bounds +// are matched instead. Compose on iOS emits a testTag node as an empty leaf +// sibling of the content it labels rather than as an ancestor, so descendant +// search under the tagged node finds nothing; bounds containment recovers the +// intended scope. +func (n *Node) scopedNodes(accept func(*Element) bool) []*Node { var result []*Node for _, child := range n.Children { - for _, candidate := range searchSubtreeBySelector(child, path[0]) { - if len(path) == 1 { - result = append(result, candidate) - continue - } - result = append(result, candidate.FindAllBySelectorPath(path[1:])...) + collectMatches(child, accept, &result) + } + if len(result) > 0 { + return result + } + for _, candidate := range n.spatialScope() { + if accept(&candidate.Element) { + result = append(result, candidate) } } return result } +func collectMatches(node *Node, accept func(*Element) bool, result *[]*Node) { + if accept(&node.Element) { + *result = append(*result, node) + } + for _, child := range node.Children { + collectMatches(child, accept, result) + } +} + +// spatialScope returns every node in the tree, in pre-order, whose positive +// bounds lie fully inside this node's bounds, excluding the node itself. +func (n *Node) spatialScope() []*Node { + if n.tree == nil || n.tree.Root == nil { + return nil + } + if n.Bounds.Width() <= 0 || n.Bounds.Height() <= 0 { + return nil + } + var result []*Node + var walk func(*Node) + walk = func(candidate *Node) { + if candidate != n && + candidate.Bounds.Width() > 0 && candidate.Bounds.Height() > 0 && + containsBounds(n.Bounds, candidate.Bounds) { + result = append(result, candidate) + } + for _, child := range candidate.Children { + walk(child) + } + } + walk(n.tree.Root) + return result +} + +// containsBounds reports whether outer fully contains inner (inclusive). +func containsBounds(outer, inner Bounds) bool { + return inner.Left >= outer.Left && inner.Top >= outer.Top && + inner.Right <= outer.Right && inner.Bottom <= outer.Bottom +} + func findPathNode(root *Node, segments []string) *Node { if root == nil || len(segments) == 0 { return nil @@ -440,18 +489,12 @@ func findPathNode(root *Node, segments []string) *Node { } func findPathDescendantsNode(root *Node, segments []string) *Node { - kind, value, ok := parseSelector(segments[0]) - if !ok { - return nil - } - for _, child := range root.Children { - for _, node := range searchSubtree(child, kind, value) { - if len(segments) == 1 { - return node - } - if result := findPathDescendantsNode(node, segments[1:]); result != nil { - return result - } + for _, node := range root.FindAll(segments[0]) { + if len(segments) == 1 { + return node + } + if result := findPathDescendantsNode(node, segments[1:]); result != nil { + return result } } return nil @@ -477,19 +520,13 @@ func findPathAllNodes(root *Node, segments []string) []*Node { } func findPathAllDescendantsNodes(root *Node, segments []string) []*Node { - kind, value, ok := parseSelector(segments[0]) - if !ok { - return nil - } var result []*Node - for _, child := range root.Children { - for _, node := range searchSubtree(child, kind, value) { - if len(segments) == 1 { - result = append(result, node) - continue - } - result = append(result, findPathAllDescendantsNodes(node, segments[1:])...) + for _, node := range root.FindAll(segments[0]) { + if len(segments) == 1 { + result = append(result, node) + continue } + result = append(result, findPathAllDescendantsNodes(node, segments[1:])...) } return result } diff --git a/internal/hierarchy/hierarchy_test.go b/internal/hierarchy/hierarchy_test.go index e430841..ebcf8ea 100644 --- a/internal/hierarchy/hierarchy_test.go +++ b/internal/hierarchy/hierarchy_test.go @@ -730,3 +730,145 @@ func TestExplicitPackageAttributeWins(t *testing.T) { t.Errorf("package = %q, want app.folio (explicit attr should win)", got) } } + +// iosFlatDump mirrors the Compose-on-iOS accessibility shape: the testTag node +// surfaces as an empty leaf SIBLING of the container holding the content it +// labels, with equal bounds, instead of as an ancestor. +const iosFlatDump = `{ + "attributes": {"bounds": "[0,0][402,874]"}, + "children": [ + { + "attributes": {"accessibilityText": "Folio", "bounds": "[0,0][402,874]"}, + "children": [ + { + "attributes": {"resource-id": "LoginScreen", "bounds": "[0,62][402,840]"}, + "children": [] + }, + { + "attributes": {"bounds": "[0,62][402,840]"}, + "children": [ + { + "attributes": {"accessibilityText": "EMAIL", "bounds": "[20,106][60,120]"}, + "children": [] + }, + { + "attributes": {"resource-id": "LoginEmail", "accessibilityText": "Email", "bounds": "[34,125][368,173]"}, + "children": [ + {"attributes": {"bounds": "[34,125][368,173]"}, "children": []} + ] + }, + { + "attributes": {"resource-id": "LoginPassword", "accessibilityText": "Password", "bounds": "[34,205][368,253]"}, + "children": [] + }, + { + "attributes": {"text": "Sign in", "bounds": "[20,297][382,345]"}, + "children": [] + }, + { + "attributes": {"resource-id": "LoginSubmit", "accessibilityText": "Sign in", "bounds": "[20,297][382,345]"}, + "children": [ + {"attributes": {"text": "Sign in", "resource-id": "SubmitLabel", "bounds": "[168,312][233,330]"}, "children": []} + ] + } + ] + } + ] + }, + { + "attributes": {"bounds": "[0,0][402,54]"}, + "children": [ + { + "attributes": {"resource-id": "StatusClock", "accessibilityText": "3:24 PM", "bounds": "[55,22][92,42]"}, + "children": [] + } + ] + } + ] +}` + +func TestIOSFlatSelectorPathFallsBackToBounds(t *testing.T) { + tree, err := Parse(iosFlatDump) + if err != nil { + t.Fatalf("Parse: %v", err) + } + path := []Selector{ + {Filters: []AttrFilter{{Attr: "testTag", Value: "LoginScreen"}}}, + {Filters: []AttrFilter{{Attr: "testTag", Value: "LoginEmail"}}}, + } + node := tree.FindBySelectorPath(path) + if node == nil { + t.Fatal("expected LoginEmail via bounds containment under leaf LoginScreen") + } + if node.ResourceID != "LoginEmail" { + t.Fatalf("got %q, want LoginEmail", node.ResourceID) + } +} + +func TestIOSFlatStringPathFallsBackToBounds(t *testing.T) { + tree, _ := Parse(iosFlatDump) + element := tree.Find("id:LoginScreen > id:LoginSubmit") + if element == nil { + t.Fatal("expected LoginSubmit via bounds containment under leaf LoginScreen") + } + if element.ResourceID != "LoginSubmit" { + t.Fatalf("got %q, want LoginSubmit", element.ResourceID) + } +} + +func TestIOSFlatScopedNodeFindFallsBackToBounds(t *testing.T) { + tree, _ := Parse(iosFlatDump) + screen := tree.FindNode("id:LoginScreen") + if screen == nil { + t.Fatal("expected LoginScreen node") + } + node := screen.Find("id:LoginPassword") + if node == nil { + t.Fatal("expected LoginPassword via bounds containment") + } + if node.ResourceID != "LoginPassword" { + t.Fatalf("got %q, want LoginPassword", node.ResourceID) + } +} + +func TestIOSFlatFindAllBySelectorPathReturnsEachField(t *testing.T) { + tree, _ := Parse(iosFlatDump) + path := []Selector{ + {Filters: []AttrFilter{{Attr: "testTag", Value: "LoginScreen"}}}, + {Filters: []AttrFilter{{Attr: "label", Value: "word"}}}, + } + // label "word" substring-matches "Password" only. + nodes := tree.FindAllBySelectorPath(path) + if len(nodes) != 1 { + t.Fatalf("got %d nodes, want 1", len(nodes)) + } + if nodes[0].ResourceID != "LoginPassword" { + t.Fatalf("got %q, want LoginPassword", nodes[0].ResourceID) + } +} + +func TestIOSFlatSpatialScopeExcludesOutsideBounds(t *testing.T) { + tree, _ := Parse(iosFlatDump) + // StatusClock sits in the status bar above LoginScreen's bounds; the + // spatial fallback must not leak it into the screen's scope. + if tree.Find("id:LoginScreen > id:StatusClock") != nil { + t.Fatal("StatusClock is outside LoginScreen bounds, expected nil") + } +} + +func TestIOSFlatStructuralChildStillPreferred(t *testing.T) { + tree, _ := Parse(iosFlatDump) + submit := tree.FindNode("id:LoginSubmit") + if submit == nil { + t.Fatal("expected LoginSubmit node") + } + // "Sign in" exists as the structural child of LoginSubmit and as an + // equal-bounds sibling decoration; the structural child must win. + node := submit.Find("text:Sign in") + if node == nil { + t.Fatal("expected structural child match") + } + if node.ResourceID != "SubmitLabel" { + t.Fatalf("expected the structural child SubmitLabel, got id=%q", node.ResourceID) + } +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 8b4b038..af070bf 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -497,6 +497,15 @@ func applyAction(ctx context.Context, drv driver.DeviceDriver, action verifier.A return err } } + // InputText replaces the field's content: erase what the target + // holds before typing. Appending instead lets repeated draws grow + // the field without bound (e.g. into a max-length validation error + // the fuzzer can never escape) and makes retried typing land twice. + if count := existingTextLength(action, tree); count > 0 { + if err := drv.EraseText(ctx, count); err != nil { + return err + } + } return drv.InputText(ctx, action.Text) case verifier.ActionKindSwipe: duration := time.Duration(action.DurationMillis) * time.Millisecond @@ -547,6 +556,20 @@ func collectLogs(ctx context.Context, drv driver.DeviceDriver, since time.Time) return result } +// existingTextLength returns the character count of the InputText target's +// current text, so the runner can erase it before typing. Zero when the +// target cannot be resolved or holds no text. +func existingTextLength(action verifier.Action, tree *hierarchy.Tree) int { + if action.On == "" || tree == nil { + return 0 + } + element := tree.Find(action.On) + if element == nil { + return 0 + } + return len([]rune(element.Text)) +} + func resolveCoordinates(action verifier.Action, tree *hierarchy.Tree) (int, int, bool) { // When On is empty, X/Y are authoritative (web V8 path emits coordinates // directly from getBoundingClientRect; the runtime nullifies unresolved diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 7c37521..1fe2394 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -370,6 +370,52 @@ func TestRunner_LogsWaitForIdleDriverErrors(t *testing.T) { } } +func TestApplyAction_InputTextErasesExistingTextBeforeTyping(t *testing.T) { + tree, err := hierarchy.Parse(`{"attributes":{"resource-id":"root","bounds":"[0,0,1080,2340]"},"children":[ + {"attributes":{"resource-id":"username","text":"stale-value","bounds":"[10,10,500,100]"},"children":[]} + ]}`) + if err != nil { + t.Fatalf("Parse: %v", err) + } + driverMock := mockdriver.New() + action := verifier.Action{Kind: verifier.ActionKindInputText, On: "id:username", Text: "alice"} + + if err := applyAction(context.Background(), driverMock, action, tree); err != nil { + t.Fatalf("applyAction: %v", err) + } + actions := driverMock.Actions() + if len(actions) != 3 { + t.Fatalf("want tap, erase, input; got %v", actions) + } + if actions[0].Kind != mockdriver.ActionTap { + t.Errorf("first action = %v, want tap", actions[0].Kind) + } + if actions[1].Kind != mockdriver.ActionEraseText || actions[1].CharacterCount != len("stale-value") { + t.Errorf("second action = %+v, want erase_text of %d characters", actions[1], len("stale-value")) + } + if actions[2].Kind != mockdriver.ActionInputText || actions[2].Text != "alice" { + t.Errorf("third action = %+v, want input_text alice", actions[2]) + } +} + +func TestApplyAction_InputTextSkipsEraseWhenTargetEmpty(t *testing.T) { + tree, err := hierarchy.Parse(`{"attributes":{"resource-id":"root","bounds":"[0,0,1080,2340]"},"children":[ + {"attributes":{"resource-id":"username","bounds":"[10,10,500,100]"},"children":[]} + ]}`) + if err != nil { + t.Fatalf("Parse: %v", err) + } + driverMock := mockdriver.New() + action := verifier.Action{Kind: verifier.ActionKindInputText, On: "id:username", Text: "alice"} + + if err := applyAction(context.Background(), driverMock, action, tree); err != nil { + t.Fatalf("applyAction: %v", err) + } + if containsAction(driverMock.Actions(), mockdriver.ActionEraseText, "") { + t.Errorf("empty field must not be erased: %v", driverMock.Actions()) + } +} + func TestApplyAction_InputTextSurfacesFocusTapError(t *testing.T) { t.Run("selector focus tap fails", func(t *testing.T) { driverMock := mockdriver.New() diff --git a/proto/driverpb/driver.pb.go b/proto/driverpb/driver.pb.go index 1fb9719..1ac1ceb 100644 --- a/proto/driverpb/driver.pb.go +++ b/proto/driverpb/driver.pb.go @@ -213,6 +213,50 @@ func (x *Text) GetValue() string { return "" } +type EraseTextRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + CharacterCount int32 `protobuf:"varint,1,opt,name=character_count,json=characterCount,proto3" json:"character_count,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EraseTextRequest) Reset() { + *x = EraseTextRequest{} + mi := &file_driverpb_driver_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EraseTextRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EraseTextRequest) ProtoMessage() {} + +func (x *EraseTextRequest) ProtoReflect() protoreflect.Message { + mi := &file_driverpb_driver_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EraseTextRequest.ProtoReflect.Descriptor instead. +func (*EraseTextRequest) Descriptor() ([]byte, []int) { + return file_driverpb_driver_proto_rawDescGZIP(), []int{4} +} + +func (x *EraseTextRequest) GetCharacterCount() int32 { + if x != nil { + return x.CharacterCount + } + return 0 +} + type Selector struct { state protoimpl.MessageState `protogen:"open.v1"` Value string `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` @@ -222,7 +266,7 @@ type Selector struct { func (x *Selector) Reset() { *x = Selector{} - mi := &file_driverpb_driver_proto_msgTypes[4] + mi := &file_driverpb_driver_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -234,7 +278,7 @@ func (x *Selector) String() string { func (*Selector) ProtoMessage() {} func (x *Selector) ProtoReflect() protoreflect.Message { - mi := &file_driverpb_driver_proto_msgTypes[4] + mi := &file_driverpb_driver_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -247,7 +291,7 @@ func (x *Selector) ProtoReflect() protoreflect.Message { // Deprecated: Use Selector.ProtoReflect.Descriptor instead. func (*Selector) Descriptor() ([]byte, []int) { - return file_driverpb_driver_proto_rawDescGZIP(), []int{4} + return file_driverpb_driver_proto_rawDescGZIP(), []int{5} } func (x *Selector) GetValue() string { @@ -268,7 +312,7 @@ type Image struct { func (x *Image) Reset() { *x = Image{} - mi := &file_driverpb_driver_proto_msgTypes[5] + mi := &file_driverpb_driver_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -280,7 +324,7 @@ func (x *Image) String() string { func (*Image) ProtoMessage() {} func (x *Image) ProtoReflect() protoreflect.Message { - mi := &file_driverpb_driver_proto_msgTypes[5] + mi := &file_driverpb_driver_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -293,7 +337,7 @@ func (x *Image) ProtoReflect() protoreflect.Message { // Deprecated: Use Image.ProtoReflect.Descriptor instead. func (*Image) Descriptor() ([]byte, []int) { - return file_driverpb_driver_proto_rawDescGZIP(), []int{5} + return file_driverpb_driver_proto_rawDescGZIP(), []int{6} } func (x *Image) GetPng() []byte { @@ -326,7 +370,7 @@ type HierarchyJSON struct { func (x *HierarchyJSON) Reset() { *x = HierarchyJSON{} - mi := &file_driverpb_driver_proto_msgTypes[6] + mi := &file_driverpb_driver_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -338,7 +382,7 @@ func (x *HierarchyJSON) String() string { func (*HierarchyJSON) ProtoMessage() {} func (x *HierarchyJSON) ProtoReflect() protoreflect.Message { - mi := &file_driverpb_driver_proto_msgTypes[6] + mi := &file_driverpb_driver_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -351,7 +395,7 @@ func (x *HierarchyJSON) ProtoReflect() protoreflect.Message { // Deprecated: Use HierarchyJSON.ProtoReflect.Descriptor instead. func (*HierarchyJSON) Descriptor() ([]byte, []int) { - return file_driverpb_driver_proto_rawDescGZIP(), []int{6} + return file_driverpb_driver_proto_rawDescGZIP(), []int{7} } func (x *HierarchyJSON) GetJson() string { @@ -371,7 +415,7 @@ type SnapshotResponse struct { func (x *SnapshotResponse) Reset() { *x = SnapshotResponse{} - mi := &file_driverpb_driver_proto_msgTypes[7] + mi := &file_driverpb_driver_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -383,7 +427,7 @@ func (x *SnapshotResponse) String() string { func (*SnapshotResponse) ProtoMessage() {} func (x *SnapshotResponse) ProtoReflect() protoreflect.Message { - mi := &file_driverpb_driver_proto_msgTypes[7] + mi := &file_driverpb_driver_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -396,7 +440,7 @@ func (x *SnapshotResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SnapshotResponse.ProtoReflect.Descriptor instead. func (*SnapshotResponse) Descriptor() ([]byte, []int) { - return file_driverpb_driver_proto_rawDescGZIP(), []int{7} + return file_driverpb_driver_proto_rawDescGZIP(), []int{8} } func (x *SnapshotResponse) GetHierarchy() *HierarchyJSON { @@ -422,7 +466,7 @@ type Duration struct { func (x *Duration) Reset() { *x = Duration{} - mi := &file_driverpb_driver_proto_msgTypes[8] + mi := &file_driverpb_driver_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -434,7 +478,7 @@ func (x *Duration) String() string { func (*Duration) ProtoMessage() {} func (x *Duration) ProtoReflect() protoreflect.Message { - mi := &file_driverpb_driver_proto_msgTypes[8] + mi := &file_driverpb_driver_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -447,7 +491,7 @@ func (x *Duration) ProtoReflect() protoreflect.Message { // Deprecated: Use Duration.ProtoReflect.Descriptor instead. func (*Duration) Descriptor() ([]byte, []int) { - return file_driverpb_driver_proto_rawDescGZIP(), []int{8} + return file_driverpb_driver_proto_rawDescGZIP(), []int{9} } func (x *Duration) GetMillis() int64 { @@ -468,7 +512,7 @@ type HealthStatus struct { func (x *HealthStatus) Reset() { *x = HealthStatus{} - mi := &file_driverpb_driver_proto_msgTypes[9] + mi := &file_driverpb_driver_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -480,7 +524,7 @@ func (x *HealthStatus) String() string { func (*HealthStatus) ProtoMessage() {} func (x *HealthStatus) ProtoReflect() protoreflect.Message { - mi := &file_driverpb_driver_proto_msgTypes[9] + mi := &file_driverpb_driver_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -493,7 +537,7 @@ func (x *HealthStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use HealthStatus.ProtoReflect.Descriptor instead. func (*HealthStatus) Descriptor() ([]byte, []int) { - return file_driverpb_driver_proto_rawDescGZIP(), []int{9} + return file_driverpb_driver_proto_rawDescGZIP(), []int{10} } func (x *HealthStatus) GetReady() bool { @@ -528,7 +572,7 @@ type SwipeRequest struct { func (x *SwipeRequest) Reset() { *x = SwipeRequest{} - mi := &file_driverpb_driver_proto_msgTypes[10] + mi := &file_driverpb_driver_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -540,7 +584,7 @@ func (x *SwipeRequest) String() string { func (*SwipeRequest) ProtoMessage() {} func (x *SwipeRequest) ProtoReflect() protoreflect.Message { - mi := &file_driverpb_driver_proto_msgTypes[10] + mi := &file_driverpb_driver_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -553,7 +597,7 @@ func (x *SwipeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SwipeRequest.ProtoReflect.Descriptor instead. func (*SwipeRequest) Descriptor() ([]byte, []int) { - return file_driverpb_driver_proto_rawDescGZIP(), []int{10} + return file_driverpb_driver_proto_rawDescGZIP(), []int{11} } func (x *SwipeRequest) GetFrom() *Point { @@ -588,7 +632,7 @@ type PressKeyRequest struct { func (x *PressKeyRequest) Reset() { *x = PressKeyRequest{} - mi := &file_driverpb_driver_proto_msgTypes[11] + mi := &file_driverpb_driver_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -600,7 +644,7 @@ func (x *PressKeyRequest) String() string { func (*PressKeyRequest) ProtoMessage() {} func (x *PressKeyRequest) ProtoReflect() protoreflect.Message { - mi := &file_driverpb_driver_proto_msgTypes[11] + mi := &file_driverpb_driver_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -613,7 +657,7 @@ func (x *PressKeyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PressKeyRequest.ProtoReflect.Descriptor instead. func (*PressKeyRequest) Descriptor() ([]byte, []int) { - return file_driverpb_driver_proto_rawDescGZIP(), []int{11} + return file_driverpb_driver_proto_rawDescGZIP(), []int{12} } func (x *PressKeyRequest) GetKey() string { @@ -636,7 +680,7 @@ type RecentLogsRequest struct { func (x *RecentLogsRequest) Reset() { *x = RecentLogsRequest{} - mi := &file_driverpb_driver_proto_msgTypes[12] + mi := &file_driverpb_driver_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -648,7 +692,7 @@ func (x *RecentLogsRequest) String() string { func (*RecentLogsRequest) ProtoMessage() {} func (x *RecentLogsRequest) ProtoReflect() protoreflect.Message { - mi := &file_driverpb_driver_proto_msgTypes[12] + mi := &file_driverpb_driver_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -661,7 +705,7 @@ func (x *RecentLogsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RecentLogsRequest.ProtoReflect.Descriptor instead. func (*RecentLogsRequest) Descriptor() ([]byte, []int) { - return file_driverpb_driver_proto_rawDescGZIP(), []int{12} + return file_driverpb_driver_proto_rawDescGZIP(), []int{13} } func (x *RecentLogsRequest) GetSinceUnixMillis() int64 { @@ -687,7 +731,7 @@ type LogEntries struct { func (x *LogEntries) Reset() { *x = LogEntries{} - mi := &file_driverpb_driver_proto_msgTypes[13] + mi := &file_driverpb_driver_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -699,7 +743,7 @@ func (x *LogEntries) String() string { func (*LogEntries) ProtoMessage() {} func (x *LogEntries) ProtoReflect() protoreflect.Message { - mi := &file_driverpb_driver_proto_msgTypes[13] + mi := &file_driverpb_driver_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -712,7 +756,7 @@ func (x *LogEntries) ProtoReflect() protoreflect.Message { // Deprecated: Use LogEntries.ProtoReflect.Descriptor instead. func (*LogEntries) Descriptor() ([]byte, []int) { - return file_driverpb_driver_proto_rawDescGZIP(), []int{13} + return file_driverpb_driver_proto_rawDescGZIP(), []int{14} } func (x *LogEntries) GetEntries() []*LogEntry { @@ -734,7 +778,7 @@ type LogEntry struct { func (x *LogEntry) Reset() { *x = LogEntry{} - mi := &file_driverpb_driver_proto_msgTypes[14] + mi := &file_driverpb_driver_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -746,7 +790,7 @@ func (x *LogEntry) String() string { func (*LogEntry) ProtoMessage() {} func (x *LogEntry) ProtoReflect() protoreflect.Message { - mi := &file_driverpb_driver_proto_msgTypes[14] + mi := &file_driverpb_driver_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -759,7 +803,7 @@ func (x *LogEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use LogEntry.ProtoReflect.Descriptor instead. func (*LogEntry) Descriptor() ([]byte, []int) { - return file_driverpb_driver_proto_rawDescGZIP(), []int{14} + return file_driverpb_driver_proto_rawDescGZIP(), []int{15} } func (x *LogEntry) GetUnixMillis() int64 { @@ -799,7 +843,7 @@ type MetricsRequest struct { func (x *MetricsRequest) Reset() { *x = MetricsRequest{} - mi := &file_driverpb_driver_proto_msgTypes[15] + mi := &file_driverpb_driver_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -811,7 +855,7 @@ func (x *MetricsRequest) String() string { func (*MetricsRequest) ProtoMessage() {} func (x *MetricsRequest) ProtoReflect() protoreflect.Message { - mi := &file_driverpb_driver_proto_msgTypes[15] + mi := &file_driverpb_driver_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -824,7 +868,7 @@ func (x *MetricsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MetricsRequest.ProtoReflect.Descriptor instead. func (*MetricsRequest) Descriptor() ([]byte, []int) { - return file_driverpb_driver_proto_rawDescGZIP(), []int{15} + return file_driverpb_driver_proto_rawDescGZIP(), []int{16} } func (x *MetricsRequest) GetBundleId() string { @@ -845,7 +889,7 @@ type MetricsResponse struct { func (x *MetricsResponse) Reset() { *x = MetricsResponse{} - mi := &file_driverpb_driver_proto_msgTypes[16] + mi := &file_driverpb_driver_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -857,7 +901,7 @@ func (x *MetricsResponse) String() string { func (*MetricsResponse) ProtoMessage() {} func (x *MetricsResponse) ProtoReflect() protoreflect.Message { - mi := &file_driverpb_driver_proto_msgTypes[16] + mi := &file_driverpb_driver_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -870,7 +914,7 @@ func (x *MetricsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MetricsResponse.ProtoReflect.Descriptor instead. func (*MetricsResponse) Descriptor() ([]byte, []int) { - return file_driverpb_driver_proto_rawDescGZIP(), []int{16} + return file_driverpb_driver_proto_rawDescGZIP(), []int{17} } func (x *MetricsResponse) GetCpuPercent() float64 { @@ -912,7 +956,9 @@ const file_driverpb_driver_proto_rawDesc = "" + "\x01x\x18\x01 \x01(\x05R\x01x\x12\f\n" + "\x01y\x18\x02 \x01(\x05R\x01y\"\x1c\n" + "\x04Text\x12\x14\n" + - "\x05value\x18\x01 \x01(\tR\x05value\" \n" + + "\x05value\x18\x01 \x01(\tR\x05value\";\n" + + "\x10EraseTextRequest\x12'\n" + + "\x0fcharacter_count\x18\x01 \x01(\x05R\x0echaracterCount\" \n" + "\bSelector\x12\x14\n" + "\x05value\x18\x01 \x01(\tR\x05value\"G\n" + "\x05Image\x12\x10\n" + @@ -957,13 +1003,16 @@ const file_driverpb_driver_proto_rawDesc = "" + "cpuPercent\x12\x1d\n" + "\n" + "heap_bytes\x18\x02 \x01(\x03R\theapBytes\x12,\n" + - "\x12total_memory_bytes\x18\x03 \x01(\x03R\x10totalMemoryBytes2\xff\b\n" + + "\x12total_memory_bytes\x18\x03 \x01(\x03R\x10totalMemoryBytes2\x98\n" + + "\n" + "\x06Driver\x12J\n" + "\x06Launch\x12#.sanderling.driver.v1.LaunchRequest\x1a\x1b.sanderling.driver.v1.Empty\x12E\n" + "\tTerminate\x12\x1b.sanderling.driver.v1.Empty\x1a\x1b.sanderling.driver.v1.Empty\x12?\n" + - "\x03Tap\x12\x1b.sanderling.driver.v1.Point\x1a\x1b.sanderling.driver.v1.Empty\x12J\n" + + "\x03Tap\x12\x1b.sanderling.driver.v1.Point\x1a\x1b.sanderling.driver.v1.Empty\x12E\n" + + "\tDoubleTap\x12\x1b.sanderling.driver.v1.Point\x1a\x1b.sanderling.driver.v1.Empty\x12J\n" + "\vTapSelector\x12\x1e.sanderling.driver.v1.Selector\x1a\x1b.sanderling.driver.v1.Empty\x12D\n" + - "\tInputText\x12\x1a.sanderling.driver.v1.Text\x1a\x1b.sanderling.driver.v1.Empty\x12H\n" + + "\tInputText\x12\x1a.sanderling.driver.v1.Text\x1a\x1b.sanderling.driver.v1.Empty\x12P\n" + + "\tEraseText\x12&.sanderling.driver.v1.EraseTextRequest\x1a\x1b.sanderling.driver.v1.Empty\x12H\n" + "\x05Swipe\x12\".sanderling.driver.v1.SwipeRequest\x1a\x1b.sanderling.driver.v1.Empty\x12N\n" + "\bPressKey\x12%.sanderling.driver.v1.PressKeyRequest\x1a\x1b.sanderling.driver.v1.Empty\x12E\n" + "\tLongPress\x12\x1b.sanderling.driver.v1.Point\x1a\x1b.sanderling.driver.v1.Empty\x12F\n" + @@ -990,66 +1039,71 @@ func file_driverpb_driver_proto_rawDescGZIP() []byte { return file_driverpb_driver_proto_rawDescData } -var file_driverpb_driver_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_driverpb_driver_proto_msgTypes = make([]protoimpl.MessageInfo, 19) var file_driverpb_driver_proto_goTypes = []any{ (*Empty)(nil), // 0: sanderling.driver.v1.Empty (*LaunchRequest)(nil), // 1: sanderling.driver.v1.LaunchRequest (*Point)(nil), // 2: sanderling.driver.v1.Point (*Text)(nil), // 3: sanderling.driver.v1.Text - (*Selector)(nil), // 4: sanderling.driver.v1.Selector - (*Image)(nil), // 5: sanderling.driver.v1.Image - (*HierarchyJSON)(nil), // 6: sanderling.driver.v1.HierarchyJSON - (*SnapshotResponse)(nil), // 7: sanderling.driver.v1.SnapshotResponse - (*Duration)(nil), // 8: sanderling.driver.v1.Duration - (*HealthStatus)(nil), // 9: sanderling.driver.v1.HealthStatus - (*SwipeRequest)(nil), // 10: sanderling.driver.v1.SwipeRequest - (*PressKeyRequest)(nil), // 11: sanderling.driver.v1.PressKeyRequest - (*RecentLogsRequest)(nil), // 12: sanderling.driver.v1.RecentLogsRequest - (*LogEntries)(nil), // 13: sanderling.driver.v1.LogEntries - (*LogEntry)(nil), // 14: sanderling.driver.v1.LogEntry - (*MetricsRequest)(nil), // 15: sanderling.driver.v1.MetricsRequest - (*MetricsResponse)(nil), // 16: sanderling.driver.v1.MetricsResponse - nil, // 17: sanderling.driver.v1.LaunchRequest.EnvEntry + (*EraseTextRequest)(nil), // 4: sanderling.driver.v1.EraseTextRequest + (*Selector)(nil), // 5: sanderling.driver.v1.Selector + (*Image)(nil), // 6: sanderling.driver.v1.Image + (*HierarchyJSON)(nil), // 7: sanderling.driver.v1.HierarchyJSON + (*SnapshotResponse)(nil), // 8: sanderling.driver.v1.SnapshotResponse + (*Duration)(nil), // 9: sanderling.driver.v1.Duration + (*HealthStatus)(nil), // 10: sanderling.driver.v1.HealthStatus + (*SwipeRequest)(nil), // 11: sanderling.driver.v1.SwipeRequest + (*PressKeyRequest)(nil), // 12: sanderling.driver.v1.PressKeyRequest + (*RecentLogsRequest)(nil), // 13: sanderling.driver.v1.RecentLogsRequest + (*LogEntries)(nil), // 14: sanderling.driver.v1.LogEntries + (*LogEntry)(nil), // 15: sanderling.driver.v1.LogEntry + (*MetricsRequest)(nil), // 16: sanderling.driver.v1.MetricsRequest + (*MetricsResponse)(nil), // 17: sanderling.driver.v1.MetricsResponse + nil, // 18: sanderling.driver.v1.LaunchRequest.EnvEntry } var file_driverpb_driver_proto_depIdxs = []int32{ - 17, // 0: sanderling.driver.v1.LaunchRequest.env:type_name -> sanderling.driver.v1.LaunchRequest.EnvEntry - 6, // 1: sanderling.driver.v1.SnapshotResponse.hierarchy:type_name -> sanderling.driver.v1.HierarchyJSON - 5, // 2: sanderling.driver.v1.SnapshotResponse.screenshot:type_name -> sanderling.driver.v1.Image + 18, // 0: sanderling.driver.v1.LaunchRequest.env:type_name -> sanderling.driver.v1.LaunchRequest.EnvEntry + 7, // 1: sanderling.driver.v1.SnapshotResponse.hierarchy:type_name -> sanderling.driver.v1.HierarchyJSON + 6, // 2: sanderling.driver.v1.SnapshotResponse.screenshot:type_name -> sanderling.driver.v1.Image 2, // 3: sanderling.driver.v1.SwipeRequest.from:type_name -> sanderling.driver.v1.Point 2, // 4: sanderling.driver.v1.SwipeRequest.to:type_name -> sanderling.driver.v1.Point - 14, // 5: sanderling.driver.v1.LogEntries.entries:type_name -> sanderling.driver.v1.LogEntry + 15, // 5: sanderling.driver.v1.LogEntries.entries:type_name -> sanderling.driver.v1.LogEntry 1, // 6: sanderling.driver.v1.Driver.Launch:input_type -> sanderling.driver.v1.LaunchRequest 0, // 7: sanderling.driver.v1.Driver.Terminate:input_type -> sanderling.driver.v1.Empty 2, // 8: sanderling.driver.v1.Driver.Tap:input_type -> sanderling.driver.v1.Point - 4, // 9: sanderling.driver.v1.Driver.TapSelector:input_type -> sanderling.driver.v1.Selector - 3, // 10: sanderling.driver.v1.Driver.InputText:input_type -> sanderling.driver.v1.Text - 10, // 11: sanderling.driver.v1.Driver.Swipe:input_type -> sanderling.driver.v1.SwipeRequest - 11, // 12: sanderling.driver.v1.Driver.PressKey:input_type -> sanderling.driver.v1.PressKeyRequest - 2, // 13: sanderling.driver.v1.Driver.LongPress:input_type -> sanderling.driver.v1.Point - 0, // 14: sanderling.driver.v1.Driver.Screenshot:input_type -> sanderling.driver.v1.Empty - 0, // 15: sanderling.driver.v1.Driver.Hierarchy:input_type -> sanderling.driver.v1.Empty - 0, // 16: sanderling.driver.v1.Driver.Snapshot:input_type -> sanderling.driver.v1.Empty - 12, // 17: sanderling.driver.v1.Driver.RecentLogs:input_type -> sanderling.driver.v1.RecentLogsRequest - 8, // 18: sanderling.driver.v1.Driver.WaitForIdle:input_type -> sanderling.driver.v1.Duration - 0, // 19: sanderling.driver.v1.Driver.Health:input_type -> sanderling.driver.v1.Empty - 15, // 20: sanderling.driver.v1.Driver.Metrics:input_type -> sanderling.driver.v1.MetricsRequest - 0, // 21: sanderling.driver.v1.Driver.Launch:output_type -> sanderling.driver.v1.Empty - 0, // 22: sanderling.driver.v1.Driver.Terminate:output_type -> sanderling.driver.v1.Empty - 0, // 23: sanderling.driver.v1.Driver.Tap:output_type -> sanderling.driver.v1.Empty - 0, // 24: sanderling.driver.v1.Driver.TapSelector:output_type -> sanderling.driver.v1.Empty - 0, // 25: sanderling.driver.v1.Driver.InputText:output_type -> sanderling.driver.v1.Empty - 0, // 26: sanderling.driver.v1.Driver.Swipe:output_type -> sanderling.driver.v1.Empty - 0, // 27: sanderling.driver.v1.Driver.PressKey:output_type -> sanderling.driver.v1.Empty - 0, // 28: sanderling.driver.v1.Driver.LongPress:output_type -> sanderling.driver.v1.Empty - 5, // 29: sanderling.driver.v1.Driver.Screenshot:output_type -> sanderling.driver.v1.Image - 6, // 30: sanderling.driver.v1.Driver.Hierarchy:output_type -> sanderling.driver.v1.HierarchyJSON - 7, // 31: sanderling.driver.v1.Driver.Snapshot:output_type -> sanderling.driver.v1.SnapshotResponse - 13, // 32: sanderling.driver.v1.Driver.RecentLogs:output_type -> sanderling.driver.v1.LogEntries - 0, // 33: sanderling.driver.v1.Driver.WaitForIdle:output_type -> sanderling.driver.v1.Empty - 9, // 34: sanderling.driver.v1.Driver.Health:output_type -> sanderling.driver.v1.HealthStatus - 16, // 35: sanderling.driver.v1.Driver.Metrics:output_type -> sanderling.driver.v1.MetricsResponse - 21, // [21:36] is the sub-list for method output_type - 6, // [6:21] is the sub-list for method input_type + 2, // 9: sanderling.driver.v1.Driver.DoubleTap:input_type -> sanderling.driver.v1.Point + 5, // 10: sanderling.driver.v1.Driver.TapSelector:input_type -> sanderling.driver.v1.Selector + 3, // 11: sanderling.driver.v1.Driver.InputText:input_type -> sanderling.driver.v1.Text + 4, // 12: sanderling.driver.v1.Driver.EraseText:input_type -> sanderling.driver.v1.EraseTextRequest + 11, // 13: sanderling.driver.v1.Driver.Swipe:input_type -> sanderling.driver.v1.SwipeRequest + 12, // 14: sanderling.driver.v1.Driver.PressKey:input_type -> sanderling.driver.v1.PressKeyRequest + 2, // 15: sanderling.driver.v1.Driver.LongPress:input_type -> sanderling.driver.v1.Point + 0, // 16: sanderling.driver.v1.Driver.Screenshot:input_type -> sanderling.driver.v1.Empty + 0, // 17: sanderling.driver.v1.Driver.Hierarchy:input_type -> sanderling.driver.v1.Empty + 0, // 18: sanderling.driver.v1.Driver.Snapshot:input_type -> sanderling.driver.v1.Empty + 13, // 19: sanderling.driver.v1.Driver.RecentLogs:input_type -> sanderling.driver.v1.RecentLogsRequest + 9, // 20: sanderling.driver.v1.Driver.WaitForIdle:input_type -> sanderling.driver.v1.Duration + 0, // 21: sanderling.driver.v1.Driver.Health:input_type -> sanderling.driver.v1.Empty + 16, // 22: sanderling.driver.v1.Driver.Metrics:input_type -> sanderling.driver.v1.MetricsRequest + 0, // 23: sanderling.driver.v1.Driver.Launch:output_type -> sanderling.driver.v1.Empty + 0, // 24: sanderling.driver.v1.Driver.Terminate:output_type -> sanderling.driver.v1.Empty + 0, // 25: sanderling.driver.v1.Driver.Tap:output_type -> sanderling.driver.v1.Empty + 0, // 26: sanderling.driver.v1.Driver.DoubleTap:output_type -> sanderling.driver.v1.Empty + 0, // 27: sanderling.driver.v1.Driver.TapSelector:output_type -> sanderling.driver.v1.Empty + 0, // 28: sanderling.driver.v1.Driver.InputText:output_type -> sanderling.driver.v1.Empty + 0, // 29: sanderling.driver.v1.Driver.EraseText:output_type -> sanderling.driver.v1.Empty + 0, // 30: sanderling.driver.v1.Driver.Swipe:output_type -> sanderling.driver.v1.Empty + 0, // 31: sanderling.driver.v1.Driver.PressKey:output_type -> sanderling.driver.v1.Empty + 0, // 32: sanderling.driver.v1.Driver.LongPress:output_type -> sanderling.driver.v1.Empty + 6, // 33: sanderling.driver.v1.Driver.Screenshot:output_type -> sanderling.driver.v1.Image + 7, // 34: sanderling.driver.v1.Driver.Hierarchy:output_type -> sanderling.driver.v1.HierarchyJSON + 8, // 35: sanderling.driver.v1.Driver.Snapshot:output_type -> sanderling.driver.v1.SnapshotResponse + 14, // 36: sanderling.driver.v1.Driver.RecentLogs:output_type -> sanderling.driver.v1.LogEntries + 0, // 37: sanderling.driver.v1.Driver.WaitForIdle:output_type -> sanderling.driver.v1.Empty + 10, // 38: sanderling.driver.v1.Driver.Health:output_type -> sanderling.driver.v1.HealthStatus + 17, // 39: sanderling.driver.v1.Driver.Metrics:output_type -> sanderling.driver.v1.MetricsResponse + 23, // [23:40] is the sub-list for method output_type + 6, // [6:23] is the sub-list for method input_type 6, // [6:6] is the sub-list for extension type_name 6, // [6:6] is the sub-list for extension extendee 0, // [0:6] is the sub-list for field type_name @@ -1066,7 +1120,7 @@ func file_driverpb_driver_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_driverpb_driver_proto_rawDesc), len(file_driverpb_driver_proto_rawDesc)), NumEnums: 0, - NumMessages: 18, + NumMessages: 19, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/driverpb/driver.proto b/proto/driverpb/driver.proto index d1a7918..217b9ef 100644 --- a/proto/driverpb/driver.proto +++ b/proto/driverpb/driver.proto @@ -11,8 +11,16 @@ service Driver { rpc Terminate(Empty) returns (Empty); rpc Tap(Point) returns (Empty); + // DoubleTap lands two taps as close together as the platform allows. + // Composing two Tap calls from the client spreads them by hundreds of + // milliseconds on iOS, wide enough for navigation to interleave, which + // defeats double-submission testing. + rpc DoubleTap(Point) returns (Empty); rpc TapSelector(Selector) returns (Empty); rpc InputText(Text) returns (Empty); + // EraseText deletes characters from the focused field so InputText can + // replace existing content instead of appending to it. + rpc EraseText(EraseTextRequest) returns (Empty); rpc Swipe(SwipeRequest) returns (Empty); rpc PressKey(PressKeyRequest) returns (Empty); rpc LongPress(Point) returns (Empty); @@ -47,6 +55,10 @@ message Text { string value = 1; } +message EraseTextRequest { + int32 character_count = 1; +} + message Selector { string value = 1; } diff --git a/proto/driverpb/driver_grpc.pb.go b/proto/driverpb/driver_grpc.pb.go index 09e4aad..a15d19f 100644 --- a/proto/driverpb/driver_grpc.pb.go +++ b/proto/driverpb/driver_grpc.pb.go @@ -22,8 +22,10 @@ const ( Driver_Launch_FullMethodName = "/sanderling.driver.v1.Driver/Launch" Driver_Terminate_FullMethodName = "/sanderling.driver.v1.Driver/Terminate" Driver_Tap_FullMethodName = "/sanderling.driver.v1.Driver/Tap" + Driver_DoubleTap_FullMethodName = "/sanderling.driver.v1.Driver/DoubleTap" Driver_TapSelector_FullMethodName = "/sanderling.driver.v1.Driver/TapSelector" Driver_InputText_FullMethodName = "/sanderling.driver.v1.Driver/InputText" + Driver_EraseText_FullMethodName = "/sanderling.driver.v1.Driver/EraseText" Driver_Swipe_FullMethodName = "/sanderling.driver.v1.Driver/Swipe" Driver_PressKey_FullMethodName = "/sanderling.driver.v1.Driver/PressKey" Driver_LongPress_FullMethodName = "/sanderling.driver.v1.Driver/LongPress" @@ -43,8 +45,16 @@ type DriverClient interface { Launch(ctx context.Context, in *LaunchRequest, opts ...grpc.CallOption) (*Empty, error) Terminate(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) Tap(ctx context.Context, in *Point, opts ...grpc.CallOption) (*Empty, error) + // DoubleTap lands two taps as close together as the platform allows. + // Composing two Tap calls from the client spreads them by hundreds of + // milliseconds on iOS, wide enough for navigation to interleave, which + // defeats double-submission testing. + DoubleTap(ctx context.Context, in *Point, opts ...grpc.CallOption) (*Empty, error) TapSelector(ctx context.Context, in *Selector, opts ...grpc.CallOption) (*Empty, error) InputText(ctx context.Context, in *Text, opts ...grpc.CallOption) (*Empty, error) + // EraseText deletes characters from the focused field so InputText can + // replace existing content instead of appending to it. + EraseText(ctx context.Context, in *EraseTextRequest, opts ...grpc.CallOption) (*Empty, error) Swipe(ctx context.Context, in *SwipeRequest, opts ...grpc.CallOption) (*Empty, error) PressKey(ctx context.Context, in *PressKeyRequest, opts ...grpc.CallOption) (*Empty, error) LongPress(ctx context.Context, in *Point, opts ...grpc.CallOption) (*Empty, error) @@ -98,6 +108,16 @@ func (c *driverClient) Tap(ctx context.Context, in *Point, opts ...grpc.CallOpti return out, nil } +func (c *driverClient) DoubleTap(ctx context.Context, in *Point, opts ...grpc.CallOption) (*Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Empty) + err := c.cc.Invoke(ctx, Driver_DoubleTap_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *driverClient) TapSelector(ctx context.Context, in *Selector, opts ...grpc.CallOption) (*Empty, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Empty) @@ -118,6 +138,16 @@ func (c *driverClient) InputText(ctx context.Context, in *Text, opts ...grpc.Cal return out, nil } +func (c *driverClient) EraseText(ctx context.Context, in *EraseTextRequest, opts ...grpc.CallOption) (*Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Empty) + err := c.cc.Invoke(ctx, Driver_EraseText_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *driverClient) Swipe(ctx context.Context, in *SwipeRequest, opts ...grpc.CallOption) (*Empty, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Empty) @@ -225,8 +255,16 @@ type DriverServer interface { Launch(context.Context, *LaunchRequest) (*Empty, error) Terminate(context.Context, *Empty) (*Empty, error) Tap(context.Context, *Point) (*Empty, error) + // DoubleTap lands two taps as close together as the platform allows. + // Composing two Tap calls from the client spreads them by hundreds of + // milliseconds on iOS, wide enough for navigation to interleave, which + // defeats double-submission testing. + DoubleTap(context.Context, *Point) (*Empty, error) TapSelector(context.Context, *Selector) (*Empty, error) InputText(context.Context, *Text) (*Empty, error) + // EraseText deletes characters from the focused field so InputText can + // replace existing content instead of appending to it. + EraseText(context.Context, *EraseTextRequest) (*Empty, error) Swipe(context.Context, *SwipeRequest) (*Empty, error) PressKey(context.Context, *PressKeyRequest) (*Empty, error) LongPress(context.Context, *Point) (*Empty, error) @@ -259,12 +297,18 @@ func (UnimplementedDriverServer) Terminate(context.Context, *Empty) (*Empty, err func (UnimplementedDriverServer) Tap(context.Context, *Point) (*Empty, error) { return nil, status.Error(codes.Unimplemented, "method Tap not implemented") } +func (UnimplementedDriverServer) DoubleTap(context.Context, *Point) (*Empty, error) { + return nil, status.Error(codes.Unimplemented, "method DoubleTap not implemented") +} func (UnimplementedDriverServer) TapSelector(context.Context, *Selector) (*Empty, error) { return nil, status.Error(codes.Unimplemented, "method TapSelector not implemented") } func (UnimplementedDriverServer) InputText(context.Context, *Text) (*Empty, error) { return nil, status.Error(codes.Unimplemented, "method InputText not implemented") } +func (UnimplementedDriverServer) EraseText(context.Context, *EraseTextRequest) (*Empty, error) { + return nil, status.Error(codes.Unimplemented, "method EraseText not implemented") +} func (UnimplementedDriverServer) Swipe(context.Context, *SwipeRequest) (*Empty, error) { return nil, status.Error(codes.Unimplemented, "method Swipe not implemented") } @@ -370,6 +414,24 @@ func _Driver_Tap_Handler(srv interface{}, ctx context.Context, dec func(interfac return interceptor(ctx, in, info, handler) } +func _Driver_DoubleTap_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Point) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DriverServer).DoubleTap(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Driver_DoubleTap_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DriverServer).DoubleTap(ctx, req.(*Point)) + } + return interceptor(ctx, in, info, handler) +} + func _Driver_TapSelector_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(Selector) if err := dec(in); err != nil { @@ -406,6 +468,24 @@ func _Driver_InputText_Handler(srv interface{}, ctx context.Context, dec func(in return interceptor(ctx, in, info, handler) } +func _Driver_EraseText_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(EraseTextRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DriverServer).EraseText(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Driver_EraseText_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DriverServer).EraseText(ctx, req.(*EraseTextRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Driver_Swipe_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(SwipeRequest) if err := dec(in); err != nil { @@ -605,6 +685,10 @@ var Driver_ServiceDesc = grpc.ServiceDesc{ MethodName: "Tap", Handler: _Driver_Tap_Handler, }, + { + MethodName: "DoubleTap", + Handler: _Driver_DoubleTap_Handler, + }, { MethodName: "TapSelector", Handler: _Driver_TapSelector_Handler, @@ -613,6 +697,10 @@ var Driver_ServiceDesc = grpc.ServiceDesc{ MethodName: "InputText", Handler: _Driver_InputText_Handler, }, + { + MethodName: "EraseText", + Handler: _Driver_EraseText_Handler, + }, { MethodName: "Swipe", Handler: _Driver_Swipe_Handler, diff --git a/proto/driverpb/driver_test.go b/proto/driverpb/driver_test.go index 860500b..902d405 100644 --- a/proto/driverpb/driver_test.go +++ b/proto/driverpb/driver_test.go @@ -16,8 +16,10 @@ func TestDriverServiceDescriptor(t *testing.T) { "Launch": true, "Terminate": true, "Tap": true, + "DoubleTap": true, "TapSelector": true, "InputText": true, + "EraseText": true, "Swipe": true, "PressKey": true, "LongPress": true, diff --git a/sidecar/src/main/kotlin/dev/sanderling/sidecar/DriverBackend.kt b/sidecar/src/main/kotlin/dev/sanderling/sidecar/DriverBackend.kt index a10e58c..e21f5a5 100644 --- a/sidecar/src/main/kotlin/dev/sanderling/sidecar/DriverBackend.kt +++ b/sidecar/src/main/kotlin/dev/sanderling/sidecar/DriverBackend.kt @@ -4,8 +4,18 @@ interface DriverBackend { fun launch(bundleId: String, clearState: Boolean, env: Map = emptyMap()) fun terminate(bundleId: String) fun tap(x: Int, y: Int) + + // doubleTap lands two taps as close together as the platform allows. + // The default composes two taps back-to-back; backends with higher + // per-tap latency override to tighten the gap. + fun doubleTap(x: Int, y: Int) { + tap(x, y) + tap(x, y) + } + fun tapSelector(selector: String) fun inputText(text: String) + fun eraseText(characterCount: Int) fun swipe(fromX: Int, fromY: Int, toX: Int, toY: Int, durationMillis: Long) fun pressKey(key: String) fun longPress(x: Int, y: Int) @@ -402,6 +412,16 @@ class StubDriverBackend( runAdb(listOf("shell", "input", "text", escapeForAdbInputText(text))) } + @Volatile var lastEraseCharacterCount: Int? = null + private set + + override fun eraseText(characterCount: Int) { + lastEraseCharacterCount = characterCount + repeat(characterCount) { + runAdb(listOf("shell", "input", "keyevent", "KEYCODE_DEL")) + } + } + @Volatile var lastSwipe: SwipeRecord? = null private set @Volatile var lastKey: String? = null @@ -531,6 +551,8 @@ class MaestroDriverBackend(private val serial: String?) : DriverBackend { override fun inputText(text: String) = driver.inputText(text) + override fun eraseText(characterCount: Int) = driver.eraseText(characterCount) + override fun swipe(fromX: Int, fromY: Int, toX: Int, toY: Int, durationMillis: Long) = driver.swipe(maestro.Point(fromX, fromY), maestro.Point(toX, toY), maxOf(durationMillis, 250L)) @@ -629,6 +651,7 @@ private fun pngHeight(bytes: ByteArray): Int { class IosDriverBackend(private val udid: String) : DriverBackend { private lateinit var driver: maestro.drivers.IOSDriver + private lateinit var localDevice: ios.LocalIOSDevice private val reconnectLock = java.util.concurrent.locks.ReentrantLock() init { @@ -670,6 +693,7 @@ class IosDriverBackend(private val udid: String) : DriverBackend { deviceController = simctlDevice, insights = maestro.utils.NoopInsights, ) + localDevice = device driver = maestro.drivers.IOSDriver(device, maestro.utils.NoopInsights) driver.open() warmup() @@ -720,6 +744,19 @@ class IosDriverBackend(private val udid: String) : DriverBackend { override fun tap(x: Int, y: Int) = withReconnect { driver.tap(maestro.Point(x, y)) } + // The second tap request is already queued at the XCTest runner while the + // first executes, so the on-device gap collapses to the runner's + // turnaround instead of a full transport round trip. Sequential requests + // leave a gap wide enough for the app to navigate between the taps. + override fun doubleTap(x: Int, y: Int): Unit = withReconnect { + val point = maestro.Point(x, y) + val firstTap = java.util.concurrent.CompletableFuture.runAsync { driver.tap(point) } + Thread.sleep(40) + driver.tap(point) + firstTap.join() + Unit + } + override fun longPress(x: Int, y: Int) = withReconnect { driver.longPress(maestro.Point(x, y)) } override fun tapSelector(selector: String) = withReconnect { @@ -730,6 +767,8 @@ class IosDriverBackend(private val udid: String) : DriverBackend { override fun inputText(text: String) = withReconnect { driver.inputText(text) } + override fun eraseText(characterCount: Int) = withReconnect { driver.eraseText(characterCount) } + override fun swipe(fromX: Int, fromY: Int, toX: Int, toY: Int, durationMillis: Long) = withReconnect { driver.swipe(maestro.Point(fromX, fromY), maestro.Point(toX, toY), maxOf(durationMillis, 250L)) } @@ -750,7 +789,7 @@ class IosDriverBackend(private val udid: String) : DriverBackend { override fun hierarchy(): String = withReconnect { com.fasterxml.jackson.module.kotlin.jacksonObjectMapper() - .writeValueAsString(driver.contentDescriptor(false)) + .writeValueAsString(iosAxElementToTreeNode(localDevice.viewHierarchy(false).axElement)) } override fun recentLogs(sinceUnixMillis: Long, minLevel: String): List = emptyList() @@ -784,3 +823,115 @@ private fun keyCodeToMaestro(adbKeyCode: String): maestro.KeyCode? { else -> null } } + +// XCUIElementType raw values that accept a tap. The XCTest hierarchy carries +// no clickable flag, so tappability is derived from the element type that +// XCUITest itself derives from the accessibility traits. +internal val IOS_CLICKABLE_ELEMENT_TYPES = setOf( + 9, // button + 10, // radioButton + 12, // checkBox + 14, // popUpButton + 15, // comboBox + 16, // menuButton + 17, // toolbarButton + 20, // key + 37, // segmentedControl + 40, // switch + 41, // toggle + 42, // link + 54, // menuItem + 75, // cell + 79, // stepper + 80, // tab +) + +// XCUIElementType raw values that accept text input. +internal val IOS_EDITABLE_ELEMENT_TYPES = setOf( + 45, // searchField + 49, // textField + 50, // secureTextField + 52, // textView +) + +// XCUIElementType raw values that scroll. +internal val IOS_SCROLLABLE_ELEMENT_TYPES = setOf( + 26, // table + 32, // collectionView + 46, // scrollView + 58, // webView +) + +// Checkable element types and the value convention ("1" when on) mirror the +// upstream content-descriptor mapping. +internal val IOS_CHECKABLE_ELEMENT_TYPES = setOf( + 12, // checkBox + 40, // switch + 41, // toggle +) + +private val IOS_ELEMENT_TYPE_NAMES = arrayOf( + "Any", "Other", "Application", "Group", "Window", "Sheet", "Drawer", + "Alert", "Dialog", "Button", "RadioButton", "RadioGroup", "CheckBox", + "DisclosureTriangle", "PopUpButton", "ComboBox", "MenuButton", + "ToolbarButton", "Popover", "Keyboard", "Key", "NavigationBar", "TabBar", + "TabGroup", "Toolbar", "StatusBar", "Table", "TableRow", "TableColumn", + "Outline", "OutlineRow", "Browser", "CollectionView", "Slider", + "PageIndicator", "ProgressIndicator", "ActivityIndicator", + "SegmentedControl", "Picker", "PickerWheel", "Switch", "Toggle", "Link", + "Image", "Icon", "SearchField", "ScrollView", "ScrollBar", "StaticText", + "TextField", "SecureTextField", "DatePicker", "TextView", "Menu", + "MenuItem", "MenuBar", "MenuBarItem", "Map", "WebView", "IncrementArrow", + "DecrementArrow", "Timeline", "RatingIndicator", "ValueIndicator", + "SplitGroup", "Splitter", "RelevanceIndicator", "ColorWell", "HelpTag", + "Matte", "DockItem", "Ruler", "RulerMarker", "Grid", "LevelIndicator", + "Cell", "LayoutArea", "LayoutItem", "Handle", "Stepper", "Tab", + "TouchBar", "StatusItem", +) + +internal fun iosElementTypeName(elementType: Int): String = + IOS_ELEMENT_TYPE_NAMES.getOrElse(elementType) { "Other" } + +// iosAxElementToTreeNode maps the raw XCTest accessibility tree to the +// TreeNode JSON shape the runner parses. The upstream content-descriptor +// mapping drops the element type, leaving no way to tell buttons and text +// fields apart, so fuzz verbs found no tap or typing candidates on iOS. +internal fun iosAxElementToTreeNode(element: hierarchy.AXElement): Map { + val elementType = element.elementType + val title = element.title.orEmpty() + val value = element.value.orEmpty() + val label = element.label.orEmpty() + val editable = elementType in IOS_EDITABLE_ELEMENT_TYPES + val checked = elementType in IOS_CHECKABLE_ELEMENT_TYPES && value == "1" + // text carries the element's visible string, matching what Android + // surfaces in its text attribute. Editable fields expose only their + // typed content: their label is the field caption (e.g. "Email"), and + // leaking it into text would make an empty field look filled. Static + // text and buttons carry the visible string in the label. + val text = if (editable) value else title.ifEmpty { value }.ifEmpty { label } + val attributes = linkedMapOf( + "accessibilityText" to label, + "title" to title, + "value" to value, + "text" to text, + "hintText" to element.placeholderValue.orEmpty(), + "resource-id" to element.identifier.orEmpty(), + "bounds" to element.frame.boundsString, + "class" to iosElementTypeName(elementType), + "enabled" to element.enabled.toString(), + "focused" to element.hasFocus.toString(), + "selected" to element.selected.toString(), + "checked" to checked.toString(), + "scrollable" to (elementType in IOS_SCROLLABLE_ELEMENT_TYPES).toString(), + ) + return linkedMapOf( + "attributes" to attributes, + "children" to element.children.map { iosAxElementToTreeNode(it) }, + "clickable" to (elementType in IOS_CLICKABLE_ELEMENT_TYPES), + "enabled" to element.enabled, + "focused" to element.hasFocus, + "checked" to checked, + "selected" to element.selected, + "editable" to editable, + ) +} diff --git a/sidecar/src/main/kotlin/dev/sanderling/sidecar/DriverService.kt b/sidecar/src/main/kotlin/dev/sanderling/sidecar/DriverService.kt index e2e86b7..3fc703d 100644 --- a/sidecar/src/main/kotlin/dev/sanderling/sidecar/DriverService.kt +++ b/sidecar/src/main/kotlin/dev/sanderling/sidecar/DriverService.kt @@ -4,6 +4,7 @@ import com.google.protobuf.ByteString import dev.sanderling.driver.v1.DriverGrpc import dev.sanderling.driver.v1.Duration import dev.sanderling.driver.v1.Empty +import dev.sanderling.driver.v1.EraseTextRequest import dev.sanderling.driver.v1.HealthStatus import dev.sanderling.driver.v1.HierarchyJSON import dev.sanderling.driver.v1.Image @@ -53,6 +54,13 @@ class DriverService( } } + override fun doubleTap(request: Point, responseObserver: StreamObserver) { + runRpc(responseObserver) { + backend.doubleTap(request.x, request.y) + Empty.getDefaultInstance() + } + } + override fun longPress(request: Point, responseObserver: StreamObserver) { runRpc(responseObserver) { backend.longPress(request.x, request.y) @@ -74,6 +82,13 @@ class DriverService( } } + override fun eraseText(request: EraseTextRequest, responseObserver: StreamObserver) { + runRpc(responseObserver) { + backend.eraseText(request.characterCount) + Empty.getDefaultInstance() + } + } + override fun swipe(request: SwipeRequest, responseObserver: StreamObserver) { runRpc(responseObserver) { val from = request.from diff --git a/sidecar/src/test/kotlin/dev/sanderling/sidecar/DriverServiceTest.kt b/sidecar/src/test/kotlin/dev/sanderling/sidecar/DriverServiceTest.kt index 36e92e4..ab52bab 100644 --- a/sidecar/src/test/kotlin/dev/sanderling/sidecar/DriverServiceTest.kt +++ b/sidecar/src/test/kotlin/dev/sanderling/sidecar/DriverServiceTest.kt @@ -3,6 +3,7 @@ package dev.sanderling.sidecar import dev.sanderling.driver.v1.DriverGrpc import dev.sanderling.driver.v1.Duration import dev.sanderling.driver.v1.Empty +import dev.sanderling.driver.v1.EraseTextRequest import dev.sanderling.driver.v1.LaunchRequest import dev.sanderling.driver.v1.Point import dev.sanderling.driver.v1.PressKeyRequest @@ -82,6 +83,42 @@ class DriverServiceTest { assertEquals("hello world", backend.lastInputText) } + @Test fun doubleTapDefaultComposesTwoTaps() { + // Interface delegation would bind the default doubleTap to the + // delegate, bypassing the tap override, so implement the interface + // directly. + val taps = mutableListOf>() + val backend = object : DriverBackend { + override fun launch(bundleId: String, clearState: Boolean, env: Map) {} + override fun terminate(bundleId: String) {} + override fun tap(x: Int, y: Int) { taps.add(x to y) } + override fun tapSelector(selector: String) {} + override fun inputText(text: String) {} + override fun eraseText(characterCount: Int) {} + override fun swipe(fromX: Int, fromY: Int, toX: Int, toY: Int, durationMillis: Long) {} + override fun pressKey(key: String) {} + override fun longPress(x: Int, y: Int) {} + override fun screenshot(): Triple = Triple(byteArrayOf(), 0, 0) + override fun hierarchy(): String = "{}" + override fun recentLogs(sinceUnixMillis: Long, minLevel: String): List = emptyList() + override fun waitForIdle(durationMillis: Long) {} + override fun healthy(): Boolean = true + override fun metrics(bundleId: String): MetricsSample = MetricsSample(0.0, 0L, 0L) + } + val client = newClient(backend) + + client.doubleTap(Point.newBuilder().setX(120).setY(340).build()) + assertEquals(listOf(120 to 340, 120 to 340), taps) + } + + @Test fun eraseTextForwardsCharacterCount() { + val backend = StubDriverBackend("android") + val client = newClient(backend) + + client.eraseText(EraseTextRequest.newBuilder().setCharacterCount(11).build()) + assertEquals(11, backend.lastEraseCharacterCount) + } + @Test fun screenshotReturnsBackendBytes() { val backend = object : DriverBackend by StubDriverBackend("android") { override fun screenshot(): Triple = Triple(byteArrayOf(1, 2, 3), 1080, 2340) diff --git a/sidecar/src/test/kotlin/dev/sanderling/sidecar/InputTextTest.kt b/sidecar/src/test/kotlin/dev/sanderling/sidecar/InputTextTest.kt index a9b2409..55f928c 100644 --- a/sidecar/src/test/kotlin/dev/sanderling/sidecar/InputTextTest.kt +++ b/sidecar/src/test/kotlin/dev/sanderling/sidecar/InputTextTest.kt @@ -15,6 +15,18 @@ class InputTextTest { assertEquals(listOf(listOf("shell", "input", "text", "Emergency%sFund")), commands) } + @Test fun eraseTextSendsOneDeleteKeyPerCharacter() { + val commands = mutableListOf>() + val backend = StubDriverBackend("android") { commands.add(it) } + + backend.eraseText(3) + + assertEquals( + List(3) { listOf("shell", "input", "keyevent", "KEYCODE_DEL") }, + commands, + ) + } + @Test fun escapeForAdbInputTextSubstitutesSpaces() { assertEquals("hello%sworld", StubDriverBackend.escapeForAdbInputText("hello world")) } diff --git a/sidecar/src/test/kotlin/dev/sanderling/sidecar/IosHierarchyTest.kt b/sidecar/src/test/kotlin/dev/sanderling/sidecar/IosHierarchyTest.kt new file mode 100644 index 0000000..12db274 --- /dev/null +++ b/sidecar/src/test/kotlin/dev/sanderling/sidecar/IosHierarchyTest.kt @@ -0,0 +1,126 @@ +package dev.sanderling.sidecar + +import hierarchy.AXElement +import hierarchy.AXFrame +import org.junit.Test +import kotlin.test.assertEquals + +class IosHierarchyTest { + + private fun element( + elementType: Int, + label: String = "", + identifier: String = "", + title: String = "", + value: String = "", + placeholderValue: String = "", + children: ArrayList = arrayListOf(), + ) = AXElement( + label, + elementType, + identifier, + 0, + 0L, + 0, + false, + 0, + false, + placeholderValue, + value, + AXFrame(10f, 20f, 100f, 50f), + true, + title, + children, + ) + + @Test + fun buttonIsClickableNotEditable() { + val node = iosAxElementToTreeNode(element(elementType = 9, label = "Sign in", identifier = "LoginSubmit")) + assertEquals(true, node["clickable"]) + assertEquals(false, node["editable"]) + @Suppress("UNCHECKED_CAST") + val attributes = node["attributes"] as Map + assertEquals("Button", attributes["class"]) + assertEquals("LoginSubmit", attributes["resource-id"]) + assertEquals("Sign in", attributes["accessibilityText"]) + } + + @Test + fun textFieldIsEditableNotClickable() { + val node = iosAxElementToTreeNode(element(elementType = 49, value = "demo@folio.app")) + assertEquals(false, node["clickable"]) + assertEquals(true, node["editable"]) + @Suppress("UNCHECKED_CAST") + val attributes = node["attributes"] as Map + assertEquals("TextField", attributes["class"]) + assertEquals("demo@folio.app", attributes["text"]) + assertEquals("demo@folio.app", attributes["value"]) + } + + @Test + fun titleWinsOverValueForText() { + val node = iosAxElementToTreeNode(element(elementType = 48, title = "Heading", value = "ignored")) + @Suppress("UNCHECKED_CAST") + val attributes = node["attributes"] as Map + assertEquals("Heading", attributes["text"]) + assertEquals("StaticText", attributes["class"]) + } + + @Test + fun staticTextFallsBackToLabelForText() { + val node = iosAxElementToTreeNode(element(elementType = 48, label = "$1,234.00")) + @Suppress("UNCHECKED_CAST") + val attributes = node["attributes"] as Map + assertEquals("$1,234.00", attributes["text"]) + } + + @Test + fun buttonFallsBackToLabelForText() { + val node = iosAxElementToTreeNode(element(elementType = 9, label = "Sign in")) + @Suppress("UNCHECKED_CAST") + val attributes = node["attributes"] as Map + assertEquals("Sign in", attributes["text"]) + } + + @Test + fun emptyTextFieldDoesNotLeakLabelIntoText() { + val node = iosAxElementToTreeNode(element(elementType = 49, label = "Email")) + @Suppress("UNCHECKED_CAST") + val attributes = node["attributes"] as Map + assertEquals("", attributes["text"]) + assertEquals("Email", attributes["accessibilityText"]) + } + + @Test + fun switchOnIsChecked() { + val node = iosAxElementToTreeNode(element(elementType = 40, value = "1")) + assertEquals(true, node["checked"]) + assertEquals(true, node["clickable"]) + } + + @Test + fun scrollViewIsScrollable() { + val node = iosAxElementToTreeNode(element(elementType = 46)) + @Suppress("UNCHECKED_CAST") + val attributes = node["attributes"] as Map + assertEquals("true", attributes["scrollable"]) + assertEquals("ScrollView", attributes["class"]) + } + + @Test + fun childrenAreMappedRecursively() { + val child = element(elementType = 48, title = "Inner") + val node = iosAxElementToTreeNode(element(elementType = 1, children = arrayListOf(child))) + @Suppress("UNCHECKED_CAST") + val children = node["children"] as List> + assertEquals(1, children.size) + @Suppress("UNCHECKED_CAST") + val childAttributes = children[0]["attributes"] as Map + assertEquals("Inner", childAttributes["text"]) + } + + @Test + fun unknownElementTypeFallsBackToOther() { + assertEquals("Other", iosElementTypeName(999)) + } +} diff --git a/sidecar/src/test/kotlin/dev/sanderling/sidecar/SnapshotHandlerTest.kt b/sidecar/src/test/kotlin/dev/sanderling/sidecar/SnapshotHandlerTest.kt index 417896d..13c6e45 100644 --- a/sidecar/src/test/kotlin/dev/sanderling/sidecar/SnapshotHandlerTest.kt +++ b/sidecar/src/test/kotlin/dev/sanderling/sidecar/SnapshotHandlerTest.kt @@ -60,6 +60,7 @@ class SnapshotHandlerTest { override fun tap(x: Int, y: Int) {} override fun tapSelector(selector: String) {} override fun inputText(text: String) {} + override fun eraseText(characterCount: Int) {} override fun swipe(fromX: Int, fromY: Int, toX: Int, toY: Int, durationMillis: Long) {} override fun pressKey(key: String) {} override fun longPress(x: Int, y: Int) {}