mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 12:07:09 +00:00
fix(verifier): exclude soft-keyboard region from action candidates
The fuzzer was tapping Gboard's "Settings" key, navigating out of the app. That key is a bare FrameLayout with a content-desc and no package or resource-id, so the package-based scope filter missed it. Candidates whose center falls in the keyboard region (derived from the IME elements' bounds) are now dropped, so no tap or long-press lands on a key. Opt-in with app scoping; unscoped runs keep every node.
This commit is contained in:
1 parent
02cd0983f5
commit
41955d5635
2 files changed
+84
No files matched your search
@@ -48,6 +48,36 @@ func TestTaps_ExcludeOffAppPackage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestTaps_ExcludeKeyboardRegionNoPackageKey proves the region exclusion catches
|
||||||
|
// a keyboard key that carries no package (Gboard's "Settings" key is a bare
|
||||||
|
// FrameLayout with a content-desc and no resource-id, so the package filter
|
||||||
|
// alone misses it). The IME's own elements set the keyboard's top edge; any
|
||||||
|
// candidate below it is dropped, leaving only the in-app button above it.
|
||||||
|
func TestTaps_ExcludeKeyboardRegionNoPackageKey(t *testing.T) {
|
||||||
|
const treeJSON = `{
|
||||||
|
"attributes": {"resource-id": "root", "bounds": "[0,0,1080,2400]", "package": "com.folio"},
|
||||||
|
"children": [
|
||||||
|
{"attributes": {"testTag": "SubmitButton", "bounds": "[100,400,500,500]", "package": "com.folio"}, "clickable": true, "enabled": true, "children": []},
|
||||||
|
{"attributes": {"resource-id": "com.google.android.inputmethod.latin:id/keyboard_holder", "bounds": "[0,1503,1080,2268]"}, "children": []},
|
||||||
|
{"attributes": {"content-desc": "Settings", "bounds": "[461,1503,618,1635]"}, "clickable": true, "enabled": true, "children": []}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
verifier := newVerifier(t, WithAppPackage("com.folio"))
|
||||||
|
loadActionSpec(t, verifier, `
|
||||||
|
import { taps } from "@sanderling/spec";
|
||||||
|
globalThis.actions = taps;
|
||||||
|
`)
|
||||||
|
pushTree(t, verifier, treeJSON)
|
||||||
|
|
||||||
|
action, err := verifier.NextAction()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if action.X != 300 || action.Y != 450 {
|
||||||
|
t.Errorf("coords = (%d,%d), want (300,450) at SubmitButton; a keyboard key leaked into targets", action.X, action.Y)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestTyping_ExcludeOffAppPackage proves keyboard glyph buttons that report as
|
// TestTyping_ExcludeOffAppPackage proves keyboard glyph buttons that report as
|
||||||
// editable never become typing targets once the app package is set.
|
// editable never become typing targets once the app package is set.
|
||||||
func TestTyping_ExcludeOffAppPackage(t *testing.T) {
|
func TestTyping_ExcludeOffAppPackage(t *testing.T) {
|
||||||
|
|||||||
@@ -7,7 +7,9 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"maps"
|
"maps"
|
||||||
|
"math"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/dop251/goja"
|
"github.com/dop251/goja"
|
||||||
@@ -648,10 +650,55 @@ func selectorForElement(tree *hierarchy.Tree, element *hierarchy.Element) string
|
|||||||
//
|
//
|
||||||
// Every candidate carries the resolving selector so the runner can re-route by
|
// Every candidate carries the resolving selector so the runner can re-route by
|
||||||
// id/text. Out-of-scope nodes (the soft keyboard, system UI) are always dropped.
|
// id/text. Out-of-scope nodes (the soft keyboard, system UI) are always dropped.
|
||||||
|
// keyboardRegionTop returns the Y above which the on-screen keyboard occupies
|
||||||
|
// the display, or math.MaxInt when no keyboard is up. Taps below this line land
|
||||||
|
// on the keyboard, not the app, so candidates there are dropped: a tap on a key
|
||||||
|
// (e.g. Gboard's "Settings") navigates out of the app under test. The IME's
|
||||||
|
// root view reports inflated full-screen bounds, so only IME elements anchored
|
||||||
|
// in the lower half of the screen set the line.
|
||||||
|
func keyboardRegionTop(tree *hierarchy.Tree) int {
|
||||||
|
if tree == nil {
|
||||||
|
return math.MaxInt
|
||||||
|
}
|
||||||
|
screenBottom := 0
|
||||||
|
for _, element := range tree.Elements {
|
||||||
|
if element.Bounds.Bottom > screenBottom {
|
||||||
|
screenBottom = element.Bounds.Bottom
|
||||||
|
}
|
||||||
|
}
|
||||||
|
top := math.MaxInt
|
||||||
|
for _, element := range tree.Elements {
|
||||||
|
if !isInputMethodElement(element) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if element.Bounds.Top*2 < screenBottom {
|
||||||
|
continue // a full-screen IME decor view, not the keyboard itself
|
||||||
|
}
|
||||||
|
if element.Bounds.Top < top {
|
||||||
|
top = element.Bounds.Top
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return top
|
||||||
|
}
|
||||||
|
|
||||||
|
// isInputMethodElement reports whether an element belongs to the soft keyboard,
|
||||||
|
// identified by its IME resource-id or package. iOS keyboards do not match, so
|
||||||
|
// this exclusion is effectively Android-only.
|
||||||
|
func isInputMethodElement(element *hierarchy.Element) bool {
|
||||||
|
return strings.Contains(element.ResourceID, "inputmethod") ||
|
||||||
|
strings.Contains(element.Package, "inputmethod")
|
||||||
|
}
|
||||||
|
|
||||||
func (v *Verifier) candidatesForVerb(verb string) []candidate {
|
func (v *Verifier) candidatesForVerb(verb string) []candidate {
|
||||||
if v.lastTree == nil {
|
if v.lastTree == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
// The keyboard-region exclusion is part of keeping exploration in the app,
|
||||||
|
// so it is opt-in with app scoping: an unscoped run keeps every node.
|
||||||
|
keyboardTop := math.MaxInt
|
||||||
|
if v.appPackage != "" {
|
||||||
|
keyboardTop = keyboardRegionTop(v.lastTree)
|
||||||
|
}
|
||||||
var result []candidate
|
var result []candidate
|
||||||
for _, element := range v.lastTree.Elements {
|
for _, element := range v.lastTree.Elements {
|
||||||
if !v.inScope(element) {
|
if !v.inScope(element) {
|
||||||
@@ -661,6 +708,13 @@ func (v *Verifier) candidatesForVerb(verb string) []candidate {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
x, y := element.Bounds.Center()
|
x, y := element.Bounds.Center()
|
||||||
|
// Drop candidates the on-screen keyboard occludes: a tap there hits a
|
||||||
|
// key, not the app, and keys like Gboard's "Settings" navigate out of
|
||||||
|
// the app under test. The keyboard's own elements carry no package, so
|
||||||
|
// the scope filter alone misses them.
|
||||||
|
if y >= keyboardTop {
|
||||||
|
continue
|
||||||
|
}
|
||||||
result = append(result, candidate{
|
result = append(result, candidate{
|
||||||
x: x,
|
x: x,
|
||||||
y: y,
|
y: y,
|
||||||
|
|||||||
Reference in new issue
Block a user