mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-03 19:47:10 +00:00
refactor(spec): one candidate producer over one target-eligibility rule
Both hosts routed verbs themselves and both policies enumerated their own actions, and all four drifted. Web sent `swipes` to scrollable containers only, so swipe-to-dismiss on a list row was reachable on native and unreachable on web; the model policy folded gestures its own way and could not reach what the seeded picker drew. A host now reports facts about every element and never decides which verb may act on it: targets.ts acceptsTarget owns that for both. pick.ts builtinCandidates is the single enumeration, and the model policy reads it through __sanderlingEnumerateBuiltin__ instead of reimplementing it in Go. Gesture verbs change with it: scrolls stay vertical over scrollable containers, swipes go free-form in all four directions from any element with real bounds. Claude-Session: https://claude.ai/code/session_01Fj4wJUikdABuMQEETwW55J
This commit is contained in:
1 parent
ef04a6de9d
commit
b5f64bf665
23 files changed
+1494
-594
No files matched your search
@@ -91,7 +91,7 @@ func (v *Verifier) installRuntimeBindings() error {
|
||||
|
||||
// installHost exposes globalThis.__sanderlingHost__ for the goja runtime entry.
|
||||
// The shared picker (pick.ts) draws against it: platform() drives the verb
|
||||
// matrix and press-key pool; seedHi/seedLo construct its Pcg; queryCandidates
|
||||
// matrix and press-key pool; seedHi/seedLo construct its Pcg; queryTargets
|
||||
// enumerates targets over the hierarchy tree; reportUnsupported records the
|
||||
// verb for the run report.
|
||||
func (v *Verifier) installHost() error {
|
||||
@@ -111,7 +111,7 @@ func (v *Verifier) installHost() error {
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := host.Set("queryCandidates", v.bindQueryCandidates); err != nil {
|
||||
if err := host.Set("queryTargets", v.bindQueryTargets); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := host.Set("reportUnsupported", func(call goja.FunctionCall) goja.Value {
|
||||
@@ -133,20 +133,25 @@ func (v *Verifier) recordUnsupported(verb string) {
|
||||
v.unsupported = append(v.unsupported, verb)
|
||||
}
|
||||
|
||||
// bindQueryCandidates returns the host-enumerated targets for a verb as an
|
||||
// array of {x, y, selector, width, height}, in tree order.
|
||||
func (v *Verifier) bindQueryCandidates(call goja.FunctionCall) goja.Value {
|
||||
verb := call.Argument(0).String()
|
||||
candidates := v.candidatesForVerb(verb)
|
||||
// bindQueryTargets returns every host-enumerated target as an array of
|
||||
// {x, y, selector, width, height, clickable, enabled, editable, scrollable}, in
|
||||
// tree order. It takes no verb: the shared rule in targets.ts decides which of
|
||||
// these a verb may act on.
|
||||
func (v *Verifier) bindQueryTargets(goja.FunctionCall) goja.Value {
|
||||
targets := v.targets()
|
||||
array := v.runtime.NewArray()
|
||||
for index, candidate := range candidates {
|
||||
for index, target := range targets {
|
||||
item := v.runtime.NewObject()
|
||||
_ = item.Set("x", candidate.x)
|
||||
_ = item.Set("y", candidate.y)
|
||||
_ = item.Set("width", candidate.width)
|
||||
_ = item.Set("height", candidate.height)
|
||||
if candidate.selector != "" {
|
||||
_ = item.Set("selector", candidate.selector)
|
||||
_ = item.Set("x", target.x)
|
||||
_ = item.Set("y", target.y)
|
||||
_ = item.Set("width", target.width)
|
||||
_ = item.Set("height", target.height)
|
||||
_ = item.Set("clickable", target.clickable)
|
||||
_ = item.Set("enabled", target.enabled)
|
||||
_ = item.Set("editable", target.editable)
|
||||
_ = item.Set("scrollable", target.scrollable)
|
||||
if target.selector != "" {
|
||||
_ = item.Set("selector", target.selector)
|
||||
}
|
||||
_ = array.Set(fmt.Sprintf("%d", index), item)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
package verifier
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// hostParityScreen is the canonical screen both hosts are driven over: one row
|
||||
// per fact combination that any verb distinguishes. Each host builds it in its
|
||||
// own model (this file as a hierarchy tree, pkg/spec/test/host-parity.test.ts as
|
||||
// a DOM), enumerates every verb, and asserts the SAME committed golden.
|
||||
var hostParityScreen = []struct {
|
||||
name string
|
||||
clickable bool
|
||||
enabled bool
|
||||
editable bool
|
||||
scrollable bool
|
||||
positiveBounds bool
|
||||
}{
|
||||
{name: "root", enabled: true, scrollable: true, positiveBounds: true},
|
||||
{name: "save", clickable: true, enabled: true, positiveBounds: true},
|
||||
{name: "cancel", clickable: true, positiveBounds: true},
|
||||
{name: "amount", enabled: true, editable: true, positiveBounds: true},
|
||||
{name: "list", enabled: true, scrollable: true, positiveBounds: true},
|
||||
{name: "row", enabled: true, positiveBounds: true},
|
||||
{name: "collapsed", clickable: true, enabled: true},
|
||||
}
|
||||
|
||||
// hostParityTreeJSON is hostParityScreen as the native host sees it. Tree order
|
||||
// is pre-order, so it matches hostParityScreen index for index.
|
||||
const hostParityTreeJSON = `{
|
||||
"attributes": {"resource-id": "root", "scrollable": "true", "bounds": "[0,0,400,800]"},
|
||||
"enabled": true,
|
||||
"children": [
|
||||
{"attributes": {"resource-id": "save", "bounds": "[0,0,200,60]"}, "clickable": true, "enabled": true, "children": []},
|
||||
{"attributes": {"resource-id": "cancel", "bounds": "[200,0,400,60]"}, "clickable": true, "enabled": false, "children": []},
|
||||
{"attributes": {"resource-id": "amount", "bounds": "[0,100,400,160]"}, "editable": true, "enabled": true, "children": []},
|
||||
{"attributes": {"resource-id": "list", "scrollable": "true", "bounds": "[0,200,400,600]"}, "enabled": true, "children": []},
|
||||
{"attributes": {"resource-id": "row", "bounds": "[0,600,400,680]"}, "enabled": true, "children": []},
|
||||
{"attributes": {"resource-id": "collapsed", "bounds": "[0,0,0,0]"}, "clickable": true, "enabled": true, "children": []}
|
||||
]
|
||||
}`
|
||||
|
||||
// TestHostsAgreeOnTargetEligibility is the guard on the claim that one
|
||||
// specification induces one action space on every platform. The native host and
|
||||
// the web host used to route verbs themselves and had drifted: web sent
|
||||
// `swipes` to scrollable containers only, so a swipe on a list row was
|
||||
// reachable on Android and unreachable on web for the same spec.
|
||||
//
|
||||
// Per-verb eligibility now has ONE definition (pkg/spec/src/targets.ts); a host
|
||||
// reports facts and never filters. This test is what notices if a second
|
||||
// definition grows back on either side: both hosts enumerate the same canonical
|
||||
// screen and must name the same targets, verb for verb, in the same order.
|
||||
// pkg/spec/test/host-parity.test.ts asserts the SAME golden from the web host,
|
||||
// so a match on both sides proves the two hosts agree without either invoking
|
||||
// the other.
|
||||
func TestHostsAgreeOnTargetEligibility(t *testing.T) {
|
||||
golden := loadHostParityGolden(t)
|
||||
for _, verb := range policyVerbs {
|
||||
t.Run(verb, func(t *testing.T) {
|
||||
want, ok := golden[verb]
|
||||
if !ok {
|
||||
t.Fatalf("golden has no entry for %s", verb)
|
||||
}
|
||||
verifier := newVerifier(t, WithSeed(0x5eed))
|
||||
loadActionSpec(t, verifier, fmt.Sprintf(
|
||||
"import { %s } from \"@sanderling/spec\";\nglobalThis.actions = %s;", verb, verb))
|
||||
pushTree(t, verifier, hostParityTreeJSON)
|
||||
|
||||
entries, err := verifier.enumerateBuiltin(verb)
|
||||
if err != nil {
|
||||
t.Fatalf("enumerate %s: %v", verb, err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
t.Fatalf("%s enumerated nothing at all", verb)
|
||||
}
|
||||
got := []string{}
|
||||
for _, entry := range entries {
|
||||
if entry.targetIndex < 0 {
|
||||
continue
|
||||
}
|
||||
if entry.targetIndex >= len(hostParityScreen) {
|
||||
t.Fatalf("%s produced target index %d, off the %d-row screen",
|
||||
verb, entry.targetIndex, len(hostParityScreen))
|
||||
}
|
||||
got = append(got, hostParityScreen[entry.targetIndex].name)
|
||||
}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("native host targets for %s\n got=%v\nwant=%v", verb, got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func loadHostParityGolden(t *testing.T) map[string][]string {
|
||||
t.Helper()
|
||||
path, err := filepath.Abs("../../pkg/spec/test/fixtures/host-parity-golden.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read golden: %v", err)
|
||||
}
|
||||
golden := map[string][]string{}
|
||||
if err := json.Unmarshal(body, &golden); err != nil {
|
||||
t.Fatalf("decode golden: %v", err)
|
||||
}
|
||||
return golden
|
||||
}
|
||||
+100
-121
@@ -1,6 +1,7 @@
|
||||
package verifier
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
@@ -130,54 +131,21 @@ type ActionCandidate struct {
|
||||
prob float64
|
||||
}
|
||||
|
||||
// verbActionKind maps a picker verb to the action kind it dispatches.
|
||||
func verbActionKind(verb string) ActionKind {
|
||||
switch verb {
|
||||
case "taps":
|
||||
return ActionKindTap
|
||||
case "doubleTaps":
|
||||
return ActionKindDoubleTap
|
||||
case "longPresses":
|
||||
return ActionKindLongPress
|
||||
case "typing":
|
||||
return ActionKindInputText
|
||||
case "scrolls":
|
||||
return ActionKindScroll
|
||||
case "swipes":
|
||||
return ActionKindSwipe
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// maxLabelRunes caps a visible-text label so joined descendant text stays short
|
||||
// enough to render on one numbered line.
|
||||
const maxLabelRunes = 40
|
||||
|
||||
// gestureDirections are the directional scrolls emitted per scrollable
|
||||
// container. Vertical only: most mobile lists scroll up/down, and keeping the
|
||||
// set tiny is the whole point of folding per-element swipes away.
|
||||
var gestureDirections = []string{"down", "up"}
|
||||
|
||||
// Candidates enumerates every action the spec's weighted actionsRoot yields at
|
||||
// the current step, each tagged with a plainly-worded description and its
|
||||
// effective weight, for the LLM generator to pick one number from. It walks the
|
||||
// SAME tree the seeded picker draws: weighted branches recurse (accumulating the
|
||||
// selection probability), authored actions()/whenRoute leaves are called once
|
||||
// for their concrete actions, and builtin verbs enumerate per applicable
|
||||
// element. Disabled controls are dropped, per-element gestures fold into a few
|
||||
// directional scrolls over scrollable containers, and identical descriptions
|
||||
// dedup (summing weight).
|
||||
// for their concrete actions, and builtin verbs come straight from the picker's
|
||||
// own enumeration. Identical descriptions dedup, summing weight.
|
||||
func (v *Verifier) Candidates() []ActionCandidate {
|
||||
if v.lastTree == nil {
|
||||
return nil
|
||||
}
|
||||
// A cross-fade frame's layout is mid-animation, often in a collapsed
|
||||
// coordinate space, so acting on it taps garbage (e.g. the soft keyboard).
|
||||
// Skip it so the LLM re-observes a settled frame next step.
|
||||
if v.lastTree.Transitional() {
|
||||
return nil
|
||||
}
|
||||
root := v.runtime.GlobalObject().Get("actions")
|
||||
if root == nil || goja.IsUndefined(root) || goja.IsNull(root) {
|
||||
return nil
|
||||
@@ -333,6 +301,18 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, nodeIndex map[*hier
|
||||
Direction: direction,
|
||||
Action: Action{Kind: kind, On: target.selector, Direction: direction},
|
||||
}, true
|
||||
case ActionKindSwipe:
|
||||
from := v.resolveTarget(object.Get("from"), nodeIndex)
|
||||
to := v.resolveTarget(object.Get("to"), nodeIndex)
|
||||
return ActionCandidate{
|
||||
Kind: kind,
|
||||
Action: Action{
|
||||
Kind: kind,
|
||||
FromX: from.x, FromY: from.y,
|
||||
ToX: to.x, ToY: to.y,
|
||||
DurationMillis: intField(object, "durationMillis"),
|
||||
},
|
||||
}, true
|
||||
case ActionKindPressKey:
|
||||
return ActionCandidate{
|
||||
Kind: kind,
|
||||
@@ -410,103 +390,80 @@ func (v *Verifier) findBySelector(selector string) *hierarchy.Element {
|
||||
return v.lastTree.Find(selector)
|
||||
}
|
||||
|
||||
// collectBuiltin enumerates a builtin verb over the current tree: tap-family and
|
||||
// typing emit one candidate per applicable element; scrolls/swipes fold into
|
||||
// directional gestures over scrollable containers.
|
||||
// collectBuiltin turns the picker's own enumeration of a builtin verb into
|
||||
// candidates. The list comes from the bundle's __sanderlingEnumerateBuiltin__
|
||||
// (pick.ts builtinCandidates), which is what the seeded policy draws from, so
|
||||
// the two policies select over one action space and cannot drift apart. Each
|
||||
// entry's action arrives on the wire contract DecodeAction already reads, so a
|
||||
// chosen candidate executes the action the seeded draw would have executed.
|
||||
func (v *Verifier) collectBuiltin(verb string, prob float64, weighted bool, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) {
|
||||
switch verb {
|
||||
case "taps", "doubleTaps", "longPresses":
|
||||
kind := verbActionKind(verb)
|
||||
for _, element := range v.elementsForVerb(verb) {
|
||||
x, y := element.Bounds.Center()
|
||||
*out = append(*out, ActionCandidate{
|
||||
Kind: kind,
|
||||
Label: visibleLabel(element, nodeIndex),
|
||||
Action: Action{Kind: kind, On: selectorForElement(v.lastTree, element), X: x, Y: y},
|
||||
prob: prob,
|
||||
Weighted: weighted,
|
||||
})
|
||||
entries, err := v.enumerateBuiltin(verb)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
targets := v.targets()
|
||||
for _, entry := range entries {
|
||||
candidate := ActionCandidate{
|
||||
Kind: entry.action.Kind,
|
||||
Direction: entry.action.Direction,
|
||||
// Builtin typing enumerates the field, not the value: the seeded
|
||||
// policy draws its text from the corpus and the model writes its own.
|
||||
LLMText: entry.action.Kind == ActionKindInputText,
|
||||
Action: entry.action,
|
||||
prob: prob,
|
||||
Weighted: weighted,
|
||||
}
|
||||
case "typing":
|
||||
for _, element := range v.elementsForVerb(verb) {
|
||||
x, y := element.Bounds.Center()
|
||||
*out = append(*out, ActionCandidate{
|
||||
Kind: ActionKindInputText,
|
||||
Label: visibleLabel(element, nodeIndex),
|
||||
InputType: inputTypeHint(element),
|
||||
LLMText: true,
|
||||
Action: Action{Kind: ActionKindInputText, On: selectorForElement(v.lastTree, element), X: x, Y: y},
|
||||
prob: prob,
|
||||
Weighted: weighted,
|
||||
})
|
||||
if entry.targetIndex >= 0 && entry.targetIndex < len(targets) {
|
||||
element := targets[entry.targetIndex].element
|
||||
candidate.Label = visibleLabel(element, nodeIndex)
|
||||
candidate.InputType = inputTypeHint(element)
|
||||
}
|
||||
case "scrolls", "swipes":
|
||||
v.collectGestures(prob, weighted, out)
|
||||
*out = append(*out, candidate)
|
||||
}
|
||||
}
|
||||
|
||||
// collectGestures emits directional scrolls scoped to each scrollable container,
|
||||
// never per element and never element-labeled. Folding both scrolls and swipes
|
||||
// here is what removes the flood of mislabeled `Swipe "X"` gestures.
|
||||
func (v *Verifier) collectGestures(prob float64, weighted bool, out *[]ActionCandidate) {
|
||||
scope := v.scopedElements()
|
||||
for _, element := range v.lastTree.Elements {
|
||||
if !scope[element] {
|
||||
continue
|
||||
}
|
||||
if element.Attributes["scrollable"] != "true" {
|
||||
continue
|
||||
}
|
||||
if element.Bounds.Width() <= 0 || element.Bounds.Height() <= 0 {
|
||||
continue
|
||||
}
|
||||
selector := selectorForElement(v.lastTree, element)
|
||||
for _, direction := range gestureDirections {
|
||||
action := Action{Kind: ActionKindScroll, On: selector, Direction: direction}
|
||||
if selector == "" {
|
||||
action.FromX, action.FromY, action.ToX, action.ToY = scrollGeometry(element.Bounds, direction)
|
||||
}
|
||||
*out = append(*out, ActionCandidate{
|
||||
Kind: ActionKindScroll,
|
||||
Direction: direction,
|
||||
Action: action,
|
||||
prob: prob,
|
||||
Weighted: weighted,
|
||||
})
|
||||
}
|
||||
}
|
||||
// builtinCandidate is one entry of the shared builtin enumeration: the concrete
|
||||
// action, plus the index of the host candidate it targets (-1 when the verb has
|
||||
// no target, as for a key press or a wait).
|
||||
type builtinCandidate struct {
|
||||
action Action
|
||||
targetIndex int
|
||||
}
|
||||
|
||||
// scrollGeometry lowers a directional scroll to swipe endpoints over the given
|
||||
// container bounds, matching the runner's own derivation, used only when the
|
||||
// container has no resolving selector.
|
||||
func scrollGeometry(bounds hierarchy.Bounds, direction string) (fromX, fromY, toX, toY int) {
|
||||
cx, cy := bounds.Center()
|
||||
fromX, fromY, toX, toY = cx, cy, cx, cy
|
||||
switch direction {
|
||||
case "down":
|
||||
toY = cy - 4*bounds.Height()/10
|
||||
case "up":
|
||||
toY = cy + 4*bounds.Height()/10
|
||||
case "left":
|
||||
toX = cx + 4*bounds.Width()/10
|
||||
case "right":
|
||||
toX = cx - 4*bounds.Width()/10
|
||||
// enumerateBuiltin invokes the bundle's shared enumeration for one verb. A spec
|
||||
// loaded without the runtime entry (a raw-JS unit fixture) has no enumerator, so
|
||||
// the verb contributes nothing rather than falling back to a second enumeration.
|
||||
func (v *Verifier) enumerateBuiltin(verb string) ([]builtinCandidate, error) {
|
||||
if v.enumerateBuiltinFn == nil {
|
||||
return nil, errors.New("verifier: builtin enumeration not available")
|
||||
}
|
||||
return fromX, fromY, max(0, toX), max(0, toY)
|
||||
}
|
||||
|
||||
// elementsForVerb returns the in-scope elements a builtin verb applies to, in
|
||||
// tree order (the seeded picker's enumeration order), reusing verbAccepts.
|
||||
func (v *Verifier) elementsForVerb(verb string) []*hierarchy.Element {
|
||||
scope := v.scopedElements()
|
||||
var elements []*hierarchy.Element
|
||||
for _, element := range v.lastTree.Elements {
|
||||
if scope[element] && verbAccepts(verb, element) {
|
||||
elements = append(elements, element)
|
||||
value, err := v.enumerateBuiltinFn(goja.Undefined(), v.runtime.ToValue(verb))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("enumerate %s: %w", verb, err)
|
||||
}
|
||||
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
|
||||
return nil, nil
|
||||
}
|
||||
raw, err := json.Marshal(value.Export())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal %s enumeration: %w", verb, err)
|
||||
}
|
||||
var wire []struct {
|
||||
Action json.RawMessage `json:"action"`
|
||||
TargetIndex int `json:"targetIndex"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &wire); err != nil {
|
||||
return nil, fmt.Errorf("decode %s enumeration: %w", verb, err)
|
||||
}
|
||||
entries := make([]builtinCandidate, 0, len(wire))
|
||||
for _, item := range wire {
|
||||
action, err := DecodeAction(item.Action)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, builtinCandidate{action: action, targetIndex: item.TargetIndex})
|
||||
}
|
||||
return elements
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// finalizeCandidates renders each candidate's description, dedups identical
|
||||
@@ -555,6 +512,18 @@ func describeCandidate(candidate ActionCandidate) string {
|
||||
return fmt.Sprintf("Type %q into %q", candidate.Action.Text, candidate.Label)
|
||||
case ActionKindScroll:
|
||||
return "Scroll " + candidate.Direction
|
||||
case ActionKindSwipe:
|
||||
// A swipe carries endpoints and no selector, so the coordinates are what
|
||||
// keep two swipes distinct. The label is prepended when the origin
|
||||
// element has one, because "swipe that row" is the interaction a model
|
||||
// reaches for and a bare pair of points does not say which row.
|
||||
where := fmt.Sprintf("from (%d,%d) to (%d,%d)",
|
||||
candidate.Action.FromX, candidate.Action.FromY,
|
||||
candidate.Action.ToX, candidate.Action.ToY)
|
||||
if candidate.Label == "" {
|
||||
return "Swipe " + where
|
||||
}
|
||||
return fmt.Sprintf("Swipe %q %s", candidate.Label, where)
|
||||
case ActionKindPressKey:
|
||||
return "Press " + candidate.Action.Key
|
||||
case ActionKindWait:
|
||||
@@ -682,3 +651,13 @@ func stringField(object *goja.Object, key string) string {
|
||||
}
|
||||
return value.String()
|
||||
}
|
||||
|
||||
// intField reads a numeric property off a goja object, returning 0 when absent,
|
||||
// null, or undefined.
|
||||
func intField(object *goja.Object, key string) int {
|
||||
value := object.Get(key)
|
||||
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
|
||||
return 0
|
||||
}
|
||||
return int(value.ToInteger())
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
// enumTreeJSON exercises every labeling path: a clickable wrapper whose own text
|
||||
// is empty but whose child Text reads "Add credit" (descendant borrowing), an
|
||||
// editable field labeled by its hint, a text-labeled button, a DISABLED button,
|
||||
// and a scrollable list (the only valid gesture origin).
|
||||
// and a scrollable list (the only valid scroll origin).
|
||||
const enumTreeJSON = `{
|
||||
"attributes": {"bounds": "[0,0,1080,2400]"},
|
||||
"children": [
|
||||
@@ -25,10 +25,13 @@ const enumTreeJSON = `{
|
||||
}`
|
||||
|
||||
// enumVerifier loads a spec whose actions root is the given plain-object graph
|
||||
// and stages the given tree, so Candidates walks a controlled action tree.
|
||||
// and stages the given tree, so Candidates walks a controlled action tree. The
|
||||
// spec is bundled with the goja runtime entry because the model arm reads the
|
||||
// picker's own builtin enumeration out of that bundle.
|
||||
func enumVerifier(t *testing.T, actionsJS, treeJSON string) *Verifier {
|
||||
t.Helper()
|
||||
v := newLoadedVerifier(t, "globalThis.actions = "+actionsJS+";")
|
||||
v := newVerifier(t)
|
||||
loadActionSpec(t, v, "globalThis.actions = "+actionsJS+";")
|
||||
tree, err := hierarchy.Parse(treeJSON)
|
||||
if err != nil {
|
||||
t.Fatalf("parse tree: %v", err)
|
||||
@@ -119,26 +122,38 @@ func TestCandidatesLabelsEditableFieldByHintNotTypedValue(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCandidatesFoldsGesturesIntoDirectionalScrolls(t *testing.T) {
|
||||
func TestCandidatesKeepsGestureVerbsDistinct(t *testing.T) {
|
||||
v := enumVerifier(t,
|
||||
"{kind:'weighted', branches:[[1,{kind:'builtin',verb:'scrolls'}],[1,{kind:'builtin',verb:'swipes'}]]}",
|
||||
enumTreeJSON)
|
||||
candidates := v.Candidates()
|
||||
|
||||
// Gestures are directional and scoped to the one scrollable container: no
|
||||
// per-element, element-labeled Swipe entries.
|
||||
for _, candidate := range candidates {
|
||||
if strings.HasPrefix(candidate.Description, "Swipe") {
|
||||
t.Errorf("gesture kept as element-labeled swipe: %q", candidate.Description)
|
||||
}
|
||||
}
|
||||
// `scrolls` folds to one directional pair over the single scrollable
|
||||
// container, which is what keeps the list short.
|
||||
if !hasCandidate(candidates, "Scroll down") || !hasCandidate(candidates, "Scroll up") {
|
||||
t.Errorf("want directional scrolls, got %v", descriptions(candidates))
|
||||
}
|
||||
// scrolls and swipes fold into the SAME directional entries: one each.
|
||||
if got := count(candidates, "Scroll down"); got != 1 {
|
||||
t.Errorf("Scroll down appears %d times, want 1 (folded)", got)
|
||||
t.Errorf("Scroll down appears %d times, want 1 over the one container", got)
|
||||
}
|
||||
// `swipes` is a different verb, not a second name for the scroll: a
|
||||
// free-form drag from any element, named by the control it starts on. That
|
||||
// is what puts swipe-to-dismiss on a row within the model's reach.
|
||||
if !hasCandidatePrefix(candidates, `Swipe "Sign in"`) {
|
||||
t.Errorf("want a swipe naming the non-scrollable row, got %v", descriptions(candidates))
|
||||
}
|
||||
if hasCandidatePrefix(candidates, "Scroll \"") {
|
||||
t.Errorf("scroll candidates must stay container-scoped: %v", descriptions(candidates))
|
||||
}
|
||||
}
|
||||
|
||||
func hasCandidatePrefix(candidates []ActionCandidate, prefix string) bool {
|
||||
for _, candidate := range candidates {
|
||||
if strings.HasPrefix(candidate.Description, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestCandidatesWeightsCombineAcrossPaths(t *testing.T) {
|
||||
@@ -228,6 +243,23 @@ func TestCandidatesCallsAuthoredLeafOnce(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCandidatesSurfaceAuthoredUntargetedActions(t *testing.T) {
|
||||
// A spec that authors a swipe, a key press, or a wait must reach the model
|
||||
// with all three: the seeded picker executes whatever the leaf returns, so a
|
||||
// kind the enumeration drops is an action only one policy can take.
|
||||
actions := `{kind:'actions', generate: () => [
|
||||
{kind:'Swipe', from:{x:10,y:600}, to:{x:10,y:100}, durationMillis: 250},
|
||||
{kind:'PressKey', key:'back'},
|
||||
{kind:'Wait'}
|
||||
]}`
|
||||
candidates := enumVerifier(t, actions, enumTreeJSON).Candidates()
|
||||
for _, want := range []string{"Swipe from (10,600) to (10,100)", "Press back", "Wait"} {
|
||||
if !hasCandidate(candidates, want) {
|
||||
t.Errorf("authored %q missing: %v", want, descriptions(candidates))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCandidatesOffRouteLeafYieldsNothing(t *testing.T) {
|
||||
v := enumVerifier(t, "{kind:'actions', generate: () => []}", enumTreeJSON)
|
||||
if got := v.Candidates(); len(got) != 0 {
|
||||
@@ -248,9 +280,14 @@ func TestCandidatesSkipsCrossFadeFrames(t *testing.T) {
|
||||
]
|
||||
}`
|
||||
v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", crossFade)
|
||||
if got := v.Candidates(); got != nil {
|
||||
if got := v.Candidates(); len(got) != 0 {
|
||||
t.Errorf("cross-fade frame should yield no candidates, got %v", descriptions(got))
|
||||
}
|
||||
// The seeded policy is skipped by the SAME guard, in the shared producer,
|
||||
// so neither arm acts on a mid-animation layout.
|
||||
if got := v.targets(); len(got) != 0 {
|
||||
t.Errorf("cross-fade frame should yield no host targets, got %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCandidatesNilWithoutTreeOrActions(t *testing.T) {
|
||||
@@ -345,5 +382,3 @@ func newLoadedVerifier(t *testing.T, source string) *Verifier {
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
|
||||
@@ -48,22 +48,25 @@ func TestCrossRuntimeParity(t *testing.T) {
|
||||
}
|
||||
|
||||
// installStubHost replaces the verifier's hierarchy-backed __sanderlingHost__
|
||||
// with one returning a FIXED candidate list for every verb, keeping the seed the
|
||||
// verifier was constructed with. It must run BEFORE Load, because the bundled
|
||||
// goja runtime entry captures the host when the spec evaluates.
|
||||
// with one returning a FIXED target list, keeping the seed the verifier was
|
||||
// constructed with. Every fact is set so the shared eligibility rule admits all
|
||||
// three targets for every verb, leaving the draw order as the only variable. It
|
||||
// must run BEFORE Load, because the bundled goja runtime entry captures the host
|
||||
// when the spec evaluates.
|
||||
func installStubHost(t *testing.T, verifier *Verifier) {
|
||||
t.Helper()
|
||||
const stub = `
|
||||
const candidates = [
|
||||
{ x: 50, y: 60, selector: "id:alpha", width: 100, height: 40 },
|
||||
{ x: 150, y: 160, selector: "id:beta", width: 120, height: 48 },
|
||||
{ x: 250, y: 260, selector: "id:gamma", width: 80, height: 32 },
|
||||
const facts = { clickable: true, enabled: true, editable: true, scrollable: true };
|
||||
const targets = [
|
||||
{ x: 50, y: 60, selector: "id:alpha", width: 100, height: 40, ...facts },
|
||||
{ x: 150, y: 160, selector: "id:beta", width: 120, height: 48, ...facts },
|
||||
{ x: 250, y: 260, selector: "id:gamma", width: 80, height: 32, ...facts },
|
||||
];
|
||||
const seedHi = globalThis.__sanderlingHost__.seedHi;
|
||||
const seedLo = globalThis.__sanderlingHost__.seedLo;
|
||||
globalThis.__sanderlingHost__ = {
|
||||
platform: () => "android",
|
||||
queryCandidates: () => candidates,
|
||||
queryTargets: () => targets,
|
||||
reportUnsupported: () => {},
|
||||
seedHi,
|
||||
seedLo,
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
package verifier
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// policyTreeJSON gives every builtin verb something to act on, with every label
|
||||
// distinct so no two candidates render the same way.
|
||||
const policyTreeJSON = `{
|
||||
"attributes": {"bounds": "[0,0,400,800]"},
|
||||
"children": [
|
||||
{"attributes": {"resource-id": "Save", "text": "Save", "bounds": "[0,0,200,60]"}, "clickable": true, "enabled": true, "children": []},
|
||||
{"attributes": {"resource-id": "Cancel", "text": "Cancel", "bounds": "[200,0,400,60]"}, "clickable": true, "enabled": true, "children": []},
|
||||
{"attributes": {"resource-id": "Amount", "class": "EditText", "hintText": "Amount", "bounds": "[0,100,400,160]"}, "enabled": true, "children": []},
|
||||
{"attributes": {"resource-id": "Note", "class": "EditText", "hintText": "Note", "bounds": "[0,200,400,260]"}, "enabled": true, "children": []},
|
||||
{"attributes": {"resource-id": "List", "scrollable": "true", "bounds": "[0,300,400,700]"}, "children": []}
|
||||
]
|
||||
}`
|
||||
|
||||
// policyVerbs is every builtin verb a spec can put in its action tree.
|
||||
var policyVerbs = []string{
|
||||
"taps", "doubleTaps", "longPresses", "typing", "scrolls", "swipes", "pressKeys", "waitOnce",
|
||||
}
|
||||
|
||||
// seededDrawBudget is how many times the seeded picker is driven per verb. The
|
||||
// candidate sets here are a handful of entries wide, so this exhausts them many
|
||||
// times over; a miss would mean the picker cannot reach one of its own
|
||||
// candidates, which is itself the bug worth failing on.
|
||||
const seededDrawBudget = 300
|
||||
|
||||
// TestPoliciesEnumerateTheSameActions is the guard on the claim the paper makes
|
||||
// about the two action policies: they differ in the pick and in nothing else.
|
||||
// For every builtin verb, the actions the seeded picker can draw and the
|
||||
// candidates the model policy is offered must be the same set. Enumeration lives
|
||||
// in one place (pick.ts builtinCandidates) precisely so this cannot drift, and
|
||||
// this test is what notices if a second one ever grows back.
|
||||
func TestPoliciesEnumerateTheSameActions(t *testing.T) {
|
||||
for _, verb := range policyVerbs {
|
||||
t.Run(verb, func(t *testing.T) {
|
||||
seeded := seededReachableActions(t, verb)
|
||||
model := modelOfferedActions(t, verb)
|
||||
if len(model) == 0 {
|
||||
t.Fatalf("%s offered the model no candidates at all", verb)
|
||||
}
|
||||
if !slices.Equal(slices.Sorted(maps.Keys(seeded)), slices.Sorted(maps.Keys(model))) {
|
||||
t.Errorf("action spaces differ for %s\n seeded=%v\n model=%v",
|
||||
verb, slices.Sorted(maps.Keys(seeded)), slices.Sorted(maps.Keys(model)))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestModelCandidateDescriptionsAreUniqueAndNamed checks the rendering half of
|
||||
// the contract: every enumerated action reaches the model as its own distinctly
|
||||
// named line, so the number it picks and the action that executes agree.
|
||||
func TestModelCandidateDescriptionsAreUniqueAndNamed(t *testing.T) {
|
||||
for _, verb := range policyVerbs {
|
||||
t.Run(verb, func(t *testing.T) {
|
||||
verifier := loadVerbSpec(t, verb)
|
||||
seen := map[string]bool{}
|
||||
for _, candidate := range verifier.Candidates() {
|
||||
if candidate.Description == "" {
|
||||
t.Fatalf("%s produced a candidate with no description: %+v", verb, candidate.Action)
|
||||
}
|
||||
if seen[candidate.Description] {
|
||||
t.Errorf("%s rendered %q twice, so the model cannot address both",
|
||||
verb, candidate.Description)
|
||||
}
|
||||
seen[candidate.Description] = true
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestModelIsOfferedTheUntargetedVerbs pins the two verbs the model arm used to
|
||||
// be blind to: with no element to enumerate over, a key press and a wait were
|
||||
// dropped, so the model could never navigate back or let the app settle.
|
||||
func TestModelIsOfferedTheUntargetedVerbs(t *testing.T) {
|
||||
verifier := loadVerbSpec(t, "pressKeys")
|
||||
if !hasCandidate(verifier.Candidates(), "Press back") {
|
||||
t.Errorf("pressKeys missing from the model's candidates: %v",
|
||||
descriptions(verifier.Candidates()))
|
||||
}
|
||||
verifier = loadVerbSpec(t, "waitOnce")
|
||||
if !hasCandidate(verifier.Candidates(), "Wait") {
|
||||
t.Errorf("waitOnce missing from the model's candidates: %v",
|
||||
descriptions(verifier.Candidates()))
|
||||
}
|
||||
}
|
||||
|
||||
// loadVerbSpec builds a verifier whose whole action tree is one builtin verb,
|
||||
// with policyTreeJSON pushed as the current state.
|
||||
func loadVerbSpec(t *testing.T, verb string) *Verifier {
|
||||
t.Helper()
|
||||
verifier := newVerifier(t, WithSeed(0x5eed))
|
||||
loadActionSpec(t, verifier, fmt.Sprintf(
|
||||
"import { %s } from \"@sanderling/spec\";\nglobalThis.actions = %s;", verb, verb))
|
||||
pushTree(t, verifier, policyTreeJSON)
|
||||
return verifier
|
||||
}
|
||||
|
||||
// seededReachableActions drives the seeded picker over its draw budget and
|
||||
// returns every distinct action it produced.
|
||||
func seededReachableActions(t *testing.T, verb string) map[string]Action {
|
||||
t.Helper()
|
||||
verifier := loadVerbSpec(t, verb)
|
||||
reachable := map[string]Action{}
|
||||
for range seededDrawBudget {
|
||||
action, err := verifier.NextAction()
|
||||
if errors.Is(err, ErrNoAction) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("%s next action: %v", verb, err)
|
||||
}
|
||||
reachable[actionIdentity(action)] = action
|
||||
}
|
||||
return reachable
|
||||
}
|
||||
|
||||
// modelOfferedActions returns the actions behind the numbered list the model
|
||||
// policy picks from.
|
||||
func modelOfferedActions(t *testing.T, verb string) map[string]Action {
|
||||
t.Helper()
|
||||
verifier := loadVerbSpec(t, verb)
|
||||
offered := map[string]Action{}
|
||||
for _, candidate := range verifier.Candidates() {
|
||||
offered[actionIdentity(candidate.Action)] = candidate.Action
|
||||
}
|
||||
return offered
|
||||
}
|
||||
|
||||
// actionIdentity keys an action by everything except the values the policy owns
|
||||
// rather than the candidate set: the typed text, which the seeded arm draws from
|
||||
// the edge-case corpus and the model writes itself, and a swipe's drag distance,
|
||||
// which the seeded arm draws and the enumeration lists at a nominal length.
|
||||
// Comparing those would compare policies instead of action spaces. A swipe's
|
||||
// direction is NOT policy-owned, so it survives as the sign of the drag.
|
||||
func actionIdentity(action Action) string {
|
||||
action.Text = ""
|
||||
if action.Kind == ActionKindSwipe {
|
||||
action.ToX = sign(action.ToX - action.FromX)
|
||||
action.ToY = sign(action.ToY - action.FromY)
|
||||
}
|
||||
return fmt.Sprintf("%+v", action)
|
||||
}
|
||||
|
||||
func sign(value int) int {
|
||||
switch {
|
||||
case value > 0:
|
||||
return 1
|
||||
case value < 0:
|
||||
return -1
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
+101
-23
@@ -105,30 +105,108 @@ func TestTyping_ExcludeOffAppPackage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestSwipes_ExcludeOffAppPackage proves swipes anchor on app nodes only, so
|
||||
// exploration never scrolls the keyboard's emoji list instead of the app.
|
||||
func TestSwipes_ExcludeOffAppPackage(t *testing.T) {
|
||||
verifier := newVerifier(t, WithAppPackage("com.folio"))
|
||||
loadActionSpec(t, verifier, `
|
||||
import { swipes } from "@sanderling/spec";
|
||||
globalThis.actions = swipes;
|
||||
`)
|
||||
pushTree(t, verifier, scopedTreeJSON)
|
||||
// gestureTreeJSON gives each gesture verb a legal and an illegal anchor: an app
|
||||
// list holding a plain row, against the soft keyboard's own scrollable strip
|
||||
// holding one of its keys.
|
||||
const gestureTreeJSON = `{
|
||||
"attributes": {"resource-id": "root", "bounds": "[0,0,100,500]", "package": "com.folio"},
|
||||
"children": [
|
||||
{"attributes": {"testTag": "AppList", "scrollable": "true", "bounds": "[0,0,100,300]", "package": "com.folio"}, "children": [
|
||||
{"attributes": {"testTag": "Row", "bounds": "[0,0,100,60]", "package": "com.folio"}, "children": []}
|
||||
]},
|
||||
{"attributes": {"testTag": "EmojiStrip", "scrollable": "true", "bounds": "[0,400,100,440]", "package": "com.google.android.inputmethod.latin"}, "children": [
|
||||
{"attributes": {"testTag": "EmojiKey", "bounds": "[0,400,100,420]", "package": "com.google.android.inputmethod.latin"}, "children": []}
|
||||
]}
|
||||
]
|
||||
}`
|
||||
|
||||
// Both the root and SubmitButton (com.folio) are valid anchors; only the
|
||||
// keyboard key at center (50,420) must be excluded. Draw many times so the
|
||||
// invariant is not satisfied by a lucky seed.
|
||||
for i := range 200 {
|
||||
action, err := verifier.NextAction()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if action.Kind != ActionKindSwipe {
|
||||
t.Fatalf("kind = %v, want Swipe", action.Kind)
|
||||
}
|
||||
if action.FromX == 50 && action.FromY == 420 {
|
||||
t.Fatalf("draw %d anchored on the keyboard key (50,420); off-app node leaked into swipe targets", i)
|
||||
}
|
||||
// TestGestures_ExcludeOffAppPackage proves both gesture verbs anchor on app
|
||||
// nodes only, so exploration never drags the keyboard instead of the app, and
|
||||
// pins what each verb accepts within the app and which way it drags. Scrolls
|
||||
// anchor on the scrollable list alone and stay vertical, because every
|
||||
// scrollable container earns a candidate and scrolling a list means up and down.
|
||||
// Swipes anchor on any app element, the row and the root included, and drag in
|
||||
// all four directions, which is what puts swipe-to-dismiss on a list row inside
|
||||
// the action space.
|
||||
func TestGestures_ExcludeOffAppPackage(t *testing.T) {
|
||||
tests := []struct {
|
||||
verb string
|
||||
kind ActionKind
|
||||
anchors map[[2]int]bool
|
||||
directions map[string]bool
|
||||
}{
|
||||
{
|
||||
"scrolls",
|
||||
ActionKindScroll,
|
||||
map[[2]int]bool{{50, 150}: true},
|
||||
map[string]bool{"down": true, "up": true},
|
||||
},
|
||||
{
|
||||
"swipes",
|
||||
ActionKindSwipe,
|
||||
map[[2]int]bool{{50, 250}: true, {50, 150}: true, {50, 30}: true},
|
||||
map[string]bool{"down": true, "up": true, "left": true, "right": true},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.verb, func(t *testing.T) {
|
||||
verifier := newVerifier(t, WithAppPackage("com.folio"))
|
||||
loadActionSpec(t, verifier, `
|
||||
import { `+test.verb+` } from "@sanderling/spec";
|
||||
globalThis.actions = `+test.verb+`;
|
||||
`)
|
||||
pushTree(t, verifier, gestureTreeJSON)
|
||||
|
||||
// Draw many times so neither the exclusion nor the coverage below is
|
||||
// satisfied by a lucky seed. The keyboard strip (50,420) and its key
|
||||
// (50,410) are the anchors that must never appear.
|
||||
seen := map[[2]int]bool{}
|
||||
seenDirections := map[string]bool{}
|
||||
for i := range 400 {
|
||||
action, err := verifier.NextAction()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if action.Kind != test.kind {
|
||||
t.Fatalf("kind = %v, want %v", action.Kind, test.kind)
|
||||
}
|
||||
anchor := [2]int{action.FromX, action.FromY}
|
||||
if !test.anchors[anchor] {
|
||||
t.Fatalf("draw %d anchored at %v, outside %v", i, anchor, test.anchors)
|
||||
}
|
||||
direction := gestureDirection(action)
|
||||
if !test.directions[direction] {
|
||||
t.Fatalf("draw %d dragged %s, outside %v", i, direction, test.directions)
|
||||
}
|
||||
seen[anchor] = true
|
||||
seenDirections[direction] = true
|
||||
}
|
||||
if len(seen) != len(test.anchors) {
|
||||
t.Errorf("reached anchors %v, want all of %v", seen, test.anchors)
|
||||
}
|
||||
if len(seenDirections) != len(test.directions) {
|
||||
t.Errorf("reached directions %v, want all of %v", seenDirections, test.directions)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// gestureDirection names which way a drawn gesture drags. A scroll carries the
|
||||
// name it was enumerated under; a swipe carries only its endpoints, so the sign
|
||||
// of the drag is what says where the finger went.
|
||||
func gestureDirection(action Action) string {
|
||||
if action.Kind == ActionKindScroll {
|
||||
return action.Direction
|
||||
}
|
||||
switch {
|
||||
case action.ToX > action.FromX:
|
||||
return "right"
|
||||
case action.ToX < action.FromX:
|
||||
return "left"
|
||||
case action.ToY > action.FromY:
|
||||
return "down"
|
||||
default:
|
||||
return "up"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+68
-55
@@ -39,22 +39,30 @@ type Verifier struct {
|
||||
// reimplementing the corpus on the Go side.
|
||||
sampleInputFn goja.Callable
|
||||
|
||||
// enumerateBuiltinFn is the bundle-installed __sanderlingEnumerateBuiltin__,
|
||||
// which lists every action a builtin verb can yield right now. It is the same
|
||||
// enumeration the seeded picker draws from, so the LLM action backend selects
|
||||
// over the picker's action space rather than one of its own.
|
||||
enumerateBuiltinFn goja.Callable
|
||||
|
||||
evaluators map[string]*ltl.Evaluator
|
||||
|
||||
priorVerdicts map[string]ltl.Verdict
|
||||
newlyViolated []string
|
||||
witnesses map[string]Witness
|
||||
|
||||
lastTree *hierarchy.Tree
|
||||
lastScreenshot []byte
|
||||
scopeCache map[*hierarchy.Element]bool
|
||||
scopeCacheTree *hierarchy.Tree
|
||||
lastAction *Action
|
||||
lastLogs []LogEntry
|
||||
lastExceptions []Exception
|
||||
stepTime time.Time
|
||||
stepIndex int
|
||||
runStart time.Time
|
||||
lastTree *hierarchy.Tree
|
||||
lastScreenshot []byte
|
||||
scopeCache map[*hierarchy.Element]bool
|
||||
scopeCacheTree *hierarchy.Tree
|
||||
targetCache []targetElement
|
||||
targetCacheTree *hierarchy.Tree
|
||||
lastAction *Action
|
||||
lastLogs []LogEntry
|
||||
lastExceptions []Exception
|
||||
stepTime time.Time
|
||||
stepIndex int
|
||||
runStart time.Time
|
||||
|
||||
appPackage string
|
||||
platform string
|
||||
@@ -178,6 +186,12 @@ func (v *Verifier) Load(source string) error {
|
||||
}
|
||||
}
|
||||
|
||||
if fn := v.runtime.GlobalObject().Get("__sanderlingEnumerateBuiltin__"); fn != nil {
|
||||
if callable, ok := goja.AssertFunction(fn); ok {
|
||||
v.enumerateBuiltinFn = callable
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -433,7 +447,7 @@ type SnapshotInput struct {
|
||||
// callers may leave it nil.
|
||||
ScreenshotPNG []byte
|
||||
LastAction *Action
|
||||
StepTime time.Time
|
||||
StepTime time.Time
|
||||
// StepIndex is the runner's step number for this snapshot. Evaluators label
|
||||
// observations with it so violation witnesses carry runner step numbers even
|
||||
// when transitional steps were skipped. Zero means unlabeled; evaluators
|
||||
@@ -750,66 +764,65 @@ func selectorForElement(tree *hierarchy.Tree, element *hierarchy.Element) string
|
||||
return ""
|
||||
}
|
||||
|
||||
// candidatesForVerb enumerates the host-side targets a builtin verb may draw
|
||||
// from, in v.lastTree.Elements ORDER (the order is part of the picker's parity
|
||||
// contract). The filters are LIFTED from the old Go picker:
|
||||
// targets enumerates every element this host can offer, in v.lastTree.Elements
|
||||
// ORDER (the order is part of the picker's parity contract). It is the native
|
||||
// half of the single candidate producer: the picker reads it through
|
||||
// __sanderlingHost__.queryTargets, applies the SHARED per-verb eligibility rule
|
||||
// (pkg/spec/src/targets.ts), and expands what survives into concrete actions for
|
||||
// both policies. Which verb may act on which element is decided there, once, so
|
||||
// this host and the web host cannot mean different things by the same verb.
|
||||
//
|
||||
// taps/doubleTaps/longPresses: clickable + enabled + positive bounds
|
||||
// typing: editable + enabled + positive bounds
|
||||
// scrolls: scrollable attribute + positive bounds
|
||||
// swipes: any in-scope element with positive bounds
|
||||
// This host's job is the facts: clickable/enabled/editable come off the
|
||||
// accessibility node, scrollable off its attribute, and the geometry off its
|
||||
// bounds. Every target carries the resolving selector so the runner can re-route
|
||||
// by id/text. Out-of-scope nodes (the soft keyboard, system UI, the launcher)
|
||||
// are dropped by scopedElements.
|
||||
//
|
||||
// Every candidate carries the resolving selector so the runner can re-route by
|
||||
// id/text. Out-of-scope nodes (the soft keyboard, system UI, the launcher) are
|
||||
// dropped by scopedElements.
|
||||
func (v *Verifier) candidatesForVerb(verb string) []candidate {
|
||||
if v.lastTree == nil {
|
||||
// A cross-fade frame yields nothing at all. Its layout is mid-animation, often
|
||||
// in a collapsed coordinate space, so acting on it lands on garbage; skipping it
|
||||
// here rather than in one policy means both policies re-observe a settled frame
|
||||
// instead of one of them acting on the animation.
|
||||
func (v *Verifier) targets() []targetElement {
|
||||
if v.lastTree == nil || v.lastTree.Transitional() {
|
||||
return nil
|
||||
}
|
||||
if v.targetCacheTree == v.lastTree {
|
||||
return v.targetCache
|
||||
}
|
||||
scope := v.scopedElements()
|
||||
var result []candidate
|
||||
result := make([]targetElement, 0, len(v.lastTree.Elements))
|
||||
for _, element := range v.lastTree.Elements {
|
||||
if !scope[element] {
|
||||
continue
|
||||
}
|
||||
if !verbAccepts(verb, element) {
|
||||
continue
|
||||
}
|
||||
x, y := element.Bounds.Center()
|
||||
result = append(result, candidate{
|
||||
x: x,
|
||||
y: y,
|
||||
width: element.Bounds.Width(),
|
||||
height: element.Bounds.Height(),
|
||||
selector: selectorForElement(v.lastTree, element),
|
||||
result = append(result, targetElement{
|
||||
element: element,
|
||||
x: x,
|
||||
y: y,
|
||||
width: element.Bounds.Width(),
|
||||
height: element.Bounds.Height(),
|
||||
selector: selectorForElement(v.lastTree, element),
|
||||
clickable: element.Clickable,
|
||||
enabled: element.Enabled,
|
||||
editable: element.Editable,
|
||||
scrollable: element.Attributes["scrollable"] == "true",
|
||||
})
|
||||
}
|
||||
v.targetCache = result
|
||||
v.targetCacheTree = v.lastTree
|
||||
return result
|
||||
}
|
||||
|
||||
type candidate struct {
|
||||
// targetElement is one host-enumerated element with the facts the shared
|
||||
// eligibility rule reads. The host reports facts; it does not filter by verb.
|
||||
type targetElement struct {
|
||||
element *hierarchy.Element
|
||||
x, y int
|
||||
width, height int
|
||||
selector string
|
||||
}
|
||||
|
||||
// verbAccepts applies the per-verb element filter.
|
||||
func verbAccepts(verb string, element *hierarchy.Element) bool {
|
||||
positiveBounds := element.Bounds.Width() > 0 && element.Bounds.Height() > 0
|
||||
switch verb {
|
||||
case "taps", "doubleTaps", "longPresses":
|
||||
return element.Clickable && element.Enabled && positiveBounds
|
||||
case "typing":
|
||||
return element.Editable && element.Enabled && positiveBounds
|
||||
case "scrolls":
|
||||
return element.Attributes["scrollable"] == "true" && positiveBounds
|
||||
case "swipes":
|
||||
// Any visible element is a valid swipe origin, but it must have real
|
||||
// bounds: a zero-bounds node centers at (0,0), and a downward swipe from
|
||||
// the top-left corner is the system gesture that pulls down the
|
||||
// notification shade, dragging the fuzzer out of the app.
|
||||
return positiveBounds
|
||||
default:
|
||||
return false
|
||||
}
|
||||
clickable bool
|
||||
enabled bool
|
||||
editable bool
|
||||
scrollable bool
|
||||
}
|
||||
@@ -1,24 +1,45 @@
|
||||
package verifier
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/priyanshujain/sanderling/internal/hierarchy"
|
||||
)
|
||||
|
||||
// TestVerbAcceptsSwipeRequiresPositiveBounds locks the fix for the notification
|
||||
// shade: a zero-bounds element centers at (0,0), and a downward swipe from the
|
||||
// top-left corner is the system gesture that pulls the shade over the app. The
|
||||
// swipe verb must reject zero-bounds nodes like every other verb does.
|
||||
func TestVerbAcceptsSwipeRequiresPositiveBounds(t *testing.T) {
|
||||
zeroBounds := &hierarchy.Element{Bounds: hierarchy.Bounds{}}
|
||||
if verbAccepts("swipes", zeroBounds) {
|
||||
t.Error("swipes must reject a zero-bounds element (it centers at (0,0) and pulls the notification shade)")
|
||||
// TestTargetsReportFactsWithoutFiltering pins the native host's half of the
|
||||
// split introduced to stop the two hosts drifting: it reports what an element
|
||||
// IS and never decides which verb may act on it. The verb decision is one shared
|
||||
// rule (pkg/spec/src/targets.ts) both hosts consume, asserted across engines by
|
||||
// TestHostsAgreeOnTargetEligibility.
|
||||
func TestTargetsReportFactsWithoutFiltering(t *testing.T) {
|
||||
tree, err := hierarchy.Parse(hostParityTreeJSON)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
realBounds := &hierarchy.Element{Bounds: hierarchy.Bounds{Left: 100, Top: 400, Right: 980, Bottom: 600}}
|
||||
if !verbAccepts("swipes", realBounds) {
|
||||
t.Error("swipes must accept an element with positive bounds")
|
||||
v := &Verifier{lastTree: tree}
|
||||
targets := v.targets()
|
||||
if len(targets) != len(hostParityScreen) {
|
||||
t.Fatalf("targets() returned %d elements, want all %d in the tree",
|
||||
len(targets), len(hostParityScreen))
|
||||
}
|
||||
for index, target := range targets {
|
||||
want := hostParityScreen[index]
|
||||
got := []bool{
|
||||
target.clickable, target.enabled, target.editable, target.scrollable,
|
||||
}
|
||||
expected := []bool{
|
||||
want.clickable, want.enabled, want.editable, want.scrollable,
|
||||
}
|
||||
if !slices.Equal(got, expected) {
|
||||
t.Errorf("%s clickable/enabled/editable/scrollable = %v, want %v",
|
||||
want.name, got, expected)
|
||||
}
|
||||
positiveBounds := target.width > 0 && target.height > 0
|
||||
if positiveBounds != want.positiveBounds {
|
||||
t.Errorf("%s positive bounds = %v, want %v",
|
||||
want.name, positiveBounds, want.positiveBounds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user