mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
fix(verifier): exclude soft-keyboard region from action candidates
The fuzzer was tapping Gboard's "Settings" key, navigating out of the app. That key is a bare FrameLayout with a content-desc and no package or resource-id, so the package-based scope filter missed it. Candidates whose center falls in the keyboard region (derived from the IME elements' bounds) are now dropped, so no tap or long-press lands on a key. Opt-in with app scoping; unscoped runs keep every node.
This commit is contained in:
1 parent
02cd0983f5
commit
41955d5635
2 files changed
+84
No files matched your search
@@ -48,6 +48,36 @@ func TestTaps_ExcludeOffAppPackage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestTaps_ExcludeKeyboardRegionNoPackageKey proves the region exclusion catches
|
||||
// a keyboard key that carries no package (Gboard's "Settings" key is a bare
|
||||
// FrameLayout with a content-desc and no resource-id, so the package filter
|
||||
// alone misses it). The IME's own elements set the keyboard's top edge; any
|
||||
// candidate below it is dropped, leaving only the in-app button above it.
|
||||
func TestTaps_ExcludeKeyboardRegionNoPackageKey(t *testing.T) {
|
||||
const treeJSON = `{
|
||||
"attributes": {"resource-id": "root", "bounds": "[0,0,1080,2400]", "package": "com.folio"},
|
||||
"children": [
|
||||
{"attributes": {"testTag": "SubmitButton", "bounds": "[100,400,500,500]", "package": "com.folio"}, "clickable": true, "enabled": true, "children": []},
|
||||
{"attributes": {"resource-id": "com.google.android.inputmethod.latin:id/keyboard_holder", "bounds": "[0,1503,1080,2268]"}, "children": []},
|
||||
{"attributes": {"content-desc": "Settings", "bounds": "[461,1503,618,1635]"}, "clickable": true, "enabled": true, "children": []}
|
||||
]
|
||||
}`
|
||||
verifier := newVerifier(t, WithAppPackage("com.folio"))
|
||||
loadActionSpec(t, verifier, `
|
||||
import { taps } from "@sanderling/spec";
|
||||
globalThis.actions = taps;
|
||||
`)
|
||||
pushTree(t, verifier, treeJSON)
|
||||
|
||||
action, err := verifier.NextAction()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if action.X != 300 || action.Y != 450 {
|
||||
t.Errorf("coords = (%d,%d), want (300,450) at SubmitButton; a keyboard key leaked into targets", action.X, action.Y)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTyping_ExcludeOffAppPackage proves keyboard glyph buttons that report as
|
||||
// editable never become typing targets once the app package is set.
|
||||
func TestTyping_ExcludeOffAppPackage(t *testing.T) {
|
||||
|
||||
@@ -7,7 +7,9 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
@@ -648,10 +650,55 @@ func selectorForElement(tree *hierarchy.Tree, element *hierarchy.Element) string
|
||||
//
|
||||
// Every candidate carries the resolving selector so the runner can re-route by
|
||||
// id/text. Out-of-scope nodes (the soft keyboard, system UI) are always dropped.
|
||||
// keyboardRegionTop returns the Y above which the on-screen keyboard occupies
|
||||
// the display, or math.MaxInt when no keyboard is up. Taps below this line land
|
||||
// on the keyboard, not the app, so candidates there are dropped: a tap on a key
|
||||
// (e.g. Gboard's "Settings") navigates out of the app under test. The IME's
|
||||
// root view reports inflated full-screen bounds, so only IME elements anchored
|
||||
// in the lower half of the screen set the line.
|
||||
func keyboardRegionTop(tree *hierarchy.Tree) int {
|
||||
if tree == nil {
|
||||
return math.MaxInt
|
||||
}
|
||||
screenBottom := 0
|
||||
for _, element := range tree.Elements {
|
||||
if element.Bounds.Bottom > screenBottom {
|
||||
screenBottom = element.Bounds.Bottom
|
||||
}
|
||||
}
|
||||
top := math.MaxInt
|
||||
for _, element := range tree.Elements {
|
||||
if !isInputMethodElement(element) {
|
||||
continue
|
||||
}
|
||||
if element.Bounds.Top*2 < screenBottom {
|
||||
continue // a full-screen IME decor view, not the keyboard itself
|
||||
}
|
||||
if element.Bounds.Top < top {
|
||||
top = element.Bounds.Top
|
||||
}
|
||||
}
|
||||
return top
|
||||
}
|
||||
|
||||
// isInputMethodElement reports whether an element belongs to the soft keyboard,
|
||||
// identified by its IME resource-id or package. iOS keyboards do not match, so
|
||||
// this exclusion is effectively Android-only.
|
||||
func isInputMethodElement(element *hierarchy.Element) bool {
|
||||
return strings.Contains(element.ResourceID, "inputmethod") ||
|
||||
strings.Contains(element.Package, "inputmethod")
|
||||
}
|
||||
|
||||
func (v *Verifier) candidatesForVerb(verb string) []candidate {
|
||||
if v.lastTree == nil {
|
||||
return nil
|
||||
}
|
||||
// The keyboard-region exclusion is part of keeping exploration in the app,
|
||||
// so it is opt-in with app scoping: an unscoped run keeps every node.
|
||||
keyboardTop := math.MaxInt
|
||||
if v.appPackage != "" {
|
||||
keyboardTop = keyboardRegionTop(v.lastTree)
|
||||
}
|
||||
var result []candidate
|
||||
for _, element := range v.lastTree.Elements {
|
||||
if !v.inScope(element) {
|
||||
@@ -661,6 +708,13 @@ func (v *Verifier) candidatesForVerb(verb string) []candidate {
|
||||
continue
|
||||
}
|
||||
x, y := element.Bounds.Center()
|
||||
// Drop candidates the on-screen keyboard occludes: a tap there hits a
|
||||
// key, not the app, and keys like Gboard's "Settings" navigate out of
|
||||
// the app under test. The keyboard's own elements carry no package, so
|
||||
// the scope filter alone misses them.
|
||||
if y >= keyboardTop {
|
||||
continue
|
||||
}
|
||||
result = append(result, candidate{
|
||||
x: x,
|
||||
y: y,
|
||||
|
||||
Reference in new issue
Block a user