fix(verifier): exclude soft-keyboard region from action candidates

The fuzzer was tapping Gboard's "Settings" key, navigating out of the
app. That key is a bare FrameLayout with a content-desc and no package or
resource-id, so the package-based scope filter missed it. Candidates whose
center falls in the keyboard region (derived from the IME elements' bounds)
are now dropped, so no tap or long-press lands on a key. Opt-in with app
scoping; unscoped runs keep every node.
This commit is contained in:
pj committed 2026-06-10 16:53:12 +05:30
1 parent 02cd0983f5
commit 41955d5635
2 files changed
+84

No files matched your search

+30
View File
@@ -48,6 +48,36 @@ func TestTaps_ExcludeOffAppPackage(t *testing.T) {
}
}
// TestTaps_ExcludeKeyboardRegionNoPackageKey proves the region exclusion catches
// a keyboard key that carries no package (Gboard's "Settings" key is a bare
// FrameLayout with a content-desc and no resource-id, so the package filter
// alone misses it). The IME's own elements set the keyboard's top edge; any
// candidate below it is dropped, leaving only the in-app button above it.
func TestTaps_ExcludeKeyboardRegionNoPackageKey(t *testing.T) {
const treeJSON = `{
"attributes": {"resource-id": "root", "bounds": "[0,0,1080,2400]", "package": "com.folio"},
"children": [
{"attributes": {"testTag": "SubmitButton", "bounds": "[100,400,500,500]", "package": "com.folio"}, "clickable": true, "enabled": true, "children": []},
{"attributes": {"resource-id": "com.google.android.inputmethod.latin:id/keyboard_holder", "bounds": "[0,1503,1080,2268]"}, "children": []},
{"attributes": {"content-desc": "Settings", "bounds": "[461,1503,618,1635]"}, "clickable": true, "enabled": true, "children": []}
]
}`
verifier := newVerifier(t, WithAppPackage("com.folio"))
loadActionSpec(t, verifier, `
import { taps } from "@sanderling/spec";
globalThis.actions = taps;
`)
pushTree(t, verifier, treeJSON)
action, err := verifier.NextAction()
if err != nil {
t.Fatal(err)
}
if action.X != 300 || action.Y != 450 {
t.Errorf("coords = (%d,%d), want (300,450) at SubmitButton; a keyboard key leaked into targets", action.X, action.Y)
}
}
// TestTyping_ExcludeOffAppPackage proves keyboard glyph buttons that report as
// editable never become typing targets once the app package is set.
func TestTyping_ExcludeOffAppPackage(t *testing.T) {
+54
View File
@@ -7,7 +7,9 @@ import (
"errors"
"fmt"
"maps"
"math"
"sort"
"strings"
"time"
"github.com/dop251/goja"
@@ -648,10 +650,55 @@ func selectorForElement(tree *hierarchy.Tree, element *hierarchy.Element) string
//
// Every candidate carries the resolving selector so the runner can re-route by
// id/text. Out-of-scope nodes (the soft keyboard, system UI) are always dropped.
// keyboardRegionTop returns the Y above which the on-screen keyboard occupies
// the display, or math.MaxInt when no keyboard is up. Taps below this line land
// on the keyboard, not the app, so candidates there are dropped: a tap on a key
// (e.g. Gboard's "Settings") navigates out of the app under test. The IME's
// root view reports inflated full-screen bounds, so only IME elements anchored
// in the lower half of the screen set the line.
func keyboardRegionTop(tree *hierarchy.Tree) int {
if tree == nil {
return math.MaxInt
}
screenBottom := 0
for _, element := range tree.Elements {
if element.Bounds.Bottom > screenBottom {
screenBottom = element.Bounds.Bottom
}
}
top := math.MaxInt
for _, element := range tree.Elements {
if !isInputMethodElement(element) {
continue
}
if element.Bounds.Top*2 < screenBottom {
continue // a full-screen IME decor view, not the keyboard itself
}
if element.Bounds.Top < top {
top = element.Bounds.Top
}
}
return top
}
// isInputMethodElement reports whether an element belongs to the soft keyboard,
// identified by its IME resource-id or package. iOS keyboards do not match, so
// this exclusion is effectively Android-only.
func isInputMethodElement(element *hierarchy.Element) bool {
return strings.Contains(element.ResourceID, "inputmethod") ||
strings.Contains(element.Package, "inputmethod")
}
func (v *Verifier) candidatesForVerb(verb string) []candidate {
if v.lastTree == nil {
return nil
}
// The keyboard-region exclusion is part of keeping exploration in the app,
// so it is opt-in with app scoping: an unscoped run keeps every node.
keyboardTop := math.MaxInt
if v.appPackage != "" {
keyboardTop = keyboardRegionTop(v.lastTree)
}
var result []candidate
for _, element := range v.lastTree.Elements {
if !v.inScope(element) {
@@ -661,6 +708,13 @@ func (v *Verifier) candidatesForVerb(verb string) []candidate {
continue
}
x, y := element.Bounds.Center()
// Drop candidates the on-screen keyboard occludes: a tap there hits a
// key, not the app, and keys like Gboard's "Settings" navigate out of
// the app under test. The keyboard's own elements carry no package, so
// the scope filter alone misses them.
if y >= keyboardTop {
continue
}
result = append(result, candidate{
x: x,
y: y,