diff --git a/internal/verifier/scope_test.go b/internal/verifier/scope_test.go index aa469d8..8ce0cb1 100644 --- a/internal/verifier/scope_test.go +++ b/internal/verifier/scope_test.go @@ -48,6 +48,36 @@ func TestTaps_ExcludeOffAppPackage(t *testing.T) { } } +// TestTaps_ExcludeKeyboardRegionNoPackageKey proves the region exclusion catches +// a keyboard key that carries no package (Gboard's "Settings" key is a bare +// FrameLayout with a content-desc and no resource-id, so the package filter +// alone misses it). The IME's own elements set the keyboard's top edge; any +// candidate below it is dropped, leaving only the in-app button above it. +func TestTaps_ExcludeKeyboardRegionNoPackageKey(t *testing.T) { + const treeJSON = `{ + "attributes": {"resource-id": "root", "bounds": "[0,0,1080,2400]", "package": "com.folio"}, + "children": [ + {"attributes": {"testTag": "SubmitButton", "bounds": "[100,400,500,500]", "package": "com.folio"}, "clickable": true, "enabled": true, "children": []}, + {"attributes": {"resource-id": "com.google.android.inputmethod.latin:id/keyboard_holder", "bounds": "[0,1503,1080,2268]"}, "children": []}, + {"attributes": {"content-desc": "Settings", "bounds": "[461,1503,618,1635]"}, "clickable": true, "enabled": true, "children": []} + ] + }` + verifier := newVerifier(t, WithAppPackage("com.folio")) + loadActionSpec(t, verifier, ` + import { taps } from "@sanderling/spec"; + globalThis.actions = taps; + `) + pushTree(t, verifier, treeJSON) + + action, err := verifier.NextAction() + if err != nil { + t.Fatal(err) + } + if action.X != 300 || action.Y != 450 { + t.Errorf("coords = (%d,%d), want (300,450) at SubmitButton; a keyboard key leaked into targets", action.X, action.Y) + } +} + // TestTyping_ExcludeOffAppPackage proves keyboard glyph buttons that report as // editable never become typing targets once the app package is set. func TestTyping_ExcludeOffAppPackage(t *testing.T) { diff --git a/internal/verifier/worker.go b/internal/verifier/worker.go index b386bea..f0bccc0 100644 --- a/internal/verifier/worker.go +++ b/internal/verifier/worker.go @@ -7,7 +7,9 @@ import ( "errors" "fmt" "maps" + "math" "sort" + "strings" "time" "github.com/dop251/goja" @@ -648,10 +650,55 @@ func selectorForElement(tree *hierarchy.Tree, element *hierarchy.Element) string // // Every candidate carries the resolving selector so the runner can re-route by // id/text. Out-of-scope nodes (the soft keyboard, system UI) are always dropped. +// keyboardRegionTop returns the Y above which the on-screen keyboard occupies +// the display, or math.MaxInt when no keyboard is up. Taps below this line land +// on the keyboard, not the app, so candidates there are dropped: a tap on a key +// (e.g. Gboard's "Settings") navigates out of the app under test. The IME's +// root view reports inflated full-screen bounds, so only IME elements anchored +// in the lower half of the screen set the line. +func keyboardRegionTop(tree *hierarchy.Tree) int { + if tree == nil { + return math.MaxInt + } + screenBottom := 0 + for _, element := range tree.Elements { + if element.Bounds.Bottom > screenBottom { + screenBottom = element.Bounds.Bottom + } + } + top := math.MaxInt + for _, element := range tree.Elements { + if !isInputMethodElement(element) { + continue + } + if element.Bounds.Top*2 < screenBottom { + continue // a full-screen IME decor view, not the keyboard itself + } + if element.Bounds.Top < top { + top = element.Bounds.Top + } + } + return top +} + +// isInputMethodElement reports whether an element belongs to the soft keyboard, +// identified by its IME resource-id or package. iOS keyboards do not match, so +// this exclusion is effectively Android-only. +func isInputMethodElement(element *hierarchy.Element) bool { + return strings.Contains(element.ResourceID, "inputmethod") || + strings.Contains(element.Package, "inputmethod") +} + func (v *Verifier) candidatesForVerb(verb string) []candidate { if v.lastTree == nil { return nil } + // The keyboard-region exclusion is part of keeping exploration in the app, + // so it is opt-in with app scoping: an unscoped run keeps every node. + keyboardTop := math.MaxInt + if v.appPackage != "" { + keyboardTop = keyboardRegionTop(v.lastTree) + } var result []candidate for _, element := range v.lastTree.Elements { if !v.inScope(element) { @@ -661,6 +708,13 @@ func (v *Verifier) candidatesForVerb(verb string) []candidate { continue } x, y := element.Bounds.Center() + // Drop candidates the on-screen keyboard occludes: a tap there hits a + // key, not the app, and keys like Gboard's "Settings" navigate out of + // the app under test. The keyboard's own elements carry no package, so + // the scope filter alone misses them. + if y >= keyboardTop { + continue + } result = append(result, candidate{ x: x, y: y,