mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
pj
133ce17d72
a navhost mid-transition carries >1 route *Screen in a collapsed coordinate space; acting on it taps garbage (soft keyboard). real runs showed the llm acting on 44% of steps being such frames. skip them so the llm re-observes a settled frame next step.
703 lines
23 KiB
Go
703 lines
23 KiB
Go
package verifier
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"math"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/dop251/goja"
|
|
|
|
"github.com/priyanshujain/sanderling/internal/hierarchy"
|
|
)
|
|
|
|
// LLMConfig is the spec-declared configuration for the LLM action generator,
|
|
// read off globalThis.generator when the spec assigned `generator = llm({...})`.
|
|
// It is orthogonal to globalThis.actions (the weighted tree the LLM picks from);
|
|
// only the picker differs.
|
|
type LLMConfig struct {
|
|
Model string
|
|
// Instructions is optional spec-level guidance appended to the prompt to
|
|
// steer the model toward bug-hunting (empty when unset).
|
|
Instructions string
|
|
}
|
|
|
|
// LLMConfig reports the LLM action-generator config when the spec declared one
|
|
// (globalThis.generator.kind === "llm"). The second return is false for every
|
|
// other spec, so the runner falls back to the seeded picker.
|
|
func (v *Verifier) LLMConfig() (LLMConfig, bool) {
|
|
generator := v.runtime.GlobalObject().Get("generator")
|
|
if generator == nil || goja.IsUndefined(generator) || goja.IsNull(generator) {
|
|
return LLMConfig{}, false
|
|
}
|
|
object := generator.ToObject(v.runtime)
|
|
if object == nil {
|
|
return LLMConfig{}, false
|
|
}
|
|
kind := object.Get("kind")
|
|
if kind == nil || kind.String() != "llm" {
|
|
return LLMConfig{}, false
|
|
}
|
|
config := object.Get("config")
|
|
if config == nil || goja.IsUndefined(config) || goja.IsNull(config) {
|
|
return LLMConfig{}, false
|
|
}
|
|
configObject := config.ToObject(v.runtime)
|
|
if configObject == nil {
|
|
return LLMConfig{}, false
|
|
}
|
|
model := ""
|
|
if value := configObject.Get("model"); value != nil && !goja.IsUndefined(value) {
|
|
model = value.String()
|
|
}
|
|
instructions := ""
|
|
if value := configObject.Get("instructions"); value != nil && !goja.IsUndefined(value) && !goja.IsNull(value) {
|
|
instructions = value.String()
|
|
}
|
|
return LLMConfig{Model: model, Instructions: instructions}, true
|
|
}
|
|
|
|
// Screenshot returns the most recent step's screenshot PNG (set by
|
|
// PushSnapshot), or nil if none was captured.
|
|
func (v *Verifier) Screenshot() []byte {
|
|
return v.lastScreenshot
|
|
}
|
|
|
|
// CurrentScreen returns the screen id of the most recent snapshot's first
|
|
// element, matching the runner's own screen labeling. Empty when no tree is
|
|
// loaded.
|
|
func (v *Verifier) CurrentScreen() string {
|
|
if v.lastTree == nil || len(v.lastTree.Elements) == 0 {
|
|
return ""
|
|
}
|
|
return v.lastTree.Elements[0].Screen
|
|
}
|
|
|
|
// SampleInput draws one InputText value from the shared corpus via the bundled
|
|
// __sanderlingSampleInput__. It errors when the bundle did not install the
|
|
// callable (a raw-JS fixture) so the caller can skip typing rather than send an
|
|
// empty string.
|
|
func (v *Verifier) SampleInput() (string, error) {
|
|
if v.sampleInputFn == nil {
|
|
return "", errors.New("verifier: input sampler not available")
|
|
}
|
|
value, err := v.sampleInputFn(goja.Undefined())
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
|
|
return "", nil
|
|
}
|
|
return value.String(), nil
|
|
}
|
|
|
|
// ActionCandidate is one selectable action the LLM generator may choose from,
|
|
// enumerated by collect-walking the spec's weighted actionsRoot (the same tree
|
|
// the seeded picker draws). Each candidate is a concrete, ready-to-execute
|
|
// action carrying a plainly-worded Description (numbered and echoed for
|
|
// strict-skip) plus its effective Weight so the model sees the spec's testing
|
|
// priorities.
|
|
type ActionCandidate struct {
|
|
// Index is the candidate's 1-based position in the numbered list the model
|
|
// picks a number from.
|
|
Index int
|
|
// Kind is the resulting action kind.
|
|
Kind ActionKind
|
|
// Description is the rendered action shown to the model and echoed back as
|
|
// chosen_action, e.g. `Tap "Add credit"`. Dedup keys on it, so it is unique.
|
|
Description string
|
|
// Label is the visible-text target label (empty for gestures).
|
|
Label string
|
|
// Weight is the effective selection weight as a percentage (1..100),
|
|
// meaningful only when Weighted is true (the tree used `weighted`).
|
|
Weight int
|
|
Weighted bool
|
|
// InputType hints a typing field's expected input (e.g. "number", or the
|
|
// field's hint); empty when unknown or not a typing candidate.
|
|
InputType string
|
|
// Direction is up/down/left/right for gesture (Scroll) candidates, else "".
|
|
Direction string
|
|
// LLMText is true for builtin typing, where the model supplies the value;
|
|
// false for authored InputText, whose sampled Action.Text is replayed as-is.
|
|
LLMText bool
|
|
// Action is the concrete action executed when this candidate is chosen. For
|
|
// builtin typing it carries no text until the model's value is filled in.
|
|
Action Action
|
|
|
|
// prob is the internal accumulated selection probability, summed across
|
|
// dedup, then rounded into Weight. Not exposed in the prompt directly.
|
|
prob float64
|
|
}
|
|
|
|
// transitionalTree reports a NavHost cross-fade: more than one route *Screen
|
|
// alive at once. Mirrors runner.isTransitionalHierarchy so the LLM rejects the
|
|
// same frames the verifier's transitional handling does.
|
|
func transitionalTree(tree *hierarchy.Tree) bool {
|
|
screens := 0
|
|
for _, element := range tree.Elements {
|
|
if strings.HasSuffix(element.ResourceID, "Screen") {
|
|
screens++
|
|
if screens > 1 {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// verbActionKind maps a picker verb to the action kind it dispatches.
|
|
func verbActionKind(verb string) ActionKind {
|
|
switch verb {
|
|
case "taps":
|
|
return ActionKindTap
|
|
case "doubleTaps":
|
|
return ActionKindDoubleTap
|
|
case "longPresses":
|
|
return ActionKindLongPress
|
|
case "typing":
|
|
return ActionKindInputText
|
|
case "scrolls":
|
|
return ActionKindScroll
|
|
case "swipes":
|
|
return ActionKindSwipe
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// maxLabelRunes caps a visible-text label so joined descendant text stays short
|
|
// enough to render on one numbered line.
|
|
const maxLabelRunes = 40
|
|
|
|
// gestureDirections are the directional scrolls emitted per scrollable
|
|
// container. Vertical only: most mobile lists scroll up/down, and keeping the
|
|
// set tiny is the whole point of folding per-element swipes away.
|
|
var gestureDirections = []string{"down", "up"}
|
|
|
|
// Candidates enumerates every action the spec's weighted actionsRoot yields at
|
|
// the current step, each tagged with a plainly-worded description and its
|
|
// effective weight, for the LLM generator to pick one number from. It walks the
|
|
// SAME tree the seeded picker draws: weighted branches recurse (accumulating the
|
|
// selection probability), authored actions()/whenRoute leaves are called once
|
|
// for their concrete actions, and builtin verbs enumerate per applicable
|
|
// element. Disabled controls are dropped, per-element gestures fold into a few
|
|
// directional scrolls over scrollable containers, and identical descriptions
|
|
// dedup (summing weight).
|
|
func (v *Verifier) Candidates() []ActionCandidate {
|
|
if v.lastTree == nil {
|
|
return nil
|
|
}
|
|
// A cross-fade frame carries more than one route *Screen at once (a NavHost
|
|
// mid-transition); its layout is mid-animation, often in a collapsed
|
|
// coordinate space, so acting on it taps garbage (e.g. the soft keyboard).
|
|
// Skip it so the LLM re-observes a settled frame next step. Mirrors the
|
|
// runner's isTransitionalHierarchy.
|
|
if transitionalTree(v.lastTree) {
|
|
return nil
|
|
}
|
|
root := v.runtime.GlobalObject().Get("actions")
|
|
if root == nil || goja.IsUndefined(root) || goja.IsNull(root) {
|
|
return nil
|
|
}
|
|
nodeIndex := buildNodeIndex(v.lastTree)
|
|
var raw []ActionCandidate
|
|
v.collectNode(root, 1.0, false, nodeIndex, &raw)
|
|
return finalizeCandidates(raw)
|
|
}
|
|
|
|
// collectNode dispatches one GeneratorNode of the action tree. prob is the
|
|
// accumulated probability the seeded picker reaches this node; weighted records
|
|
// whether any weighted node lies on the path (so weights are shown only when the
|
|
// spec actually declared them).
|
|
func (v *Verifier) collectNode(node goja.Value, prob float64, weighted bool, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) {
|
|
object := node.ToObject(v.runtime)
|
|
if object == nil {
|
|
return
|
|
}
|
|
kind := object.Get("kind")
|
|
if kind == nil || goja.IsUndefined(kind) {
|
|
return
|
|
}
|
|
switch kind.String() {
|
|
case "weighted":
|
|
v.collectWeighted(object, prob, nodeIndex, out)
|
|
case "actions":
|
|
v.collectActions(object, prob, weighted, nodeIndex, out)
|
|
case "builtin":
|
|
verb := object.Get("verb")
|
|
if verb != nil && !goja.IsUndefined(verb) {
|
|
v.collectBuiltin(verb.String(), prob, weighted, nodeIndex, out)
|
|
}
|
|
case "llm":
|
|
// The llm marker is the generator, not part of the candidate tree.
|
|
}
|
|
}
|
|
|
|
// collectWeighted recurses each branch, splitting the incoming probability by
|
|
// the branch weight over the sibling total (matching the seeded picker's single
|
|
// weighted draw).
|
|
func (v *Verifier) collectWeighted(object *goja.Object, prob float64, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) {
|
|
branches := object.Get("branches")
|
|
if branches == nil {
|
|
return
|
|
}
|
|
array := branches.ToObject(v.runtime)
|
|
if array == nil {
|
|
return
|
|
}
|
|
length := int(array.Get("length").ToInteger())
|
|
weights := make([]float64, length)
|
|
children := make([]goja.Value, length)
|
|
total := 0.0
|
|
for i := range length {
|
|
entry := array.Get(strconv.Itoa(i))
|
|
pair := entry.ToObject(v.runtime)
|
|
if pair == nil {
|
|
continue
|
|
}
|
|
weight := pair.Get("0").ToFloat()
|
|
if weight < 0 || math.IsNaN(weight) {
|
|
weight = 0
|
|
}
|
|
weights[i] = weight
|
|
children[i] = pair.Get("1")
|
|
total += weight
|
|
}
|
|
if total <= 0 {
|
|
return
|
|
}
|
|
for i := range length {
|
|
if children[i] == nil {
|
|
continue
|
|
}
|
|
v.collectNode(children[i], prob*weights[i]/total, true, nodeIndex, out)
|
|
}
|
|
}
|
|
|
|
// collectActions calls an authored leaf's generator once (safe: it reads state
|
|
// and, off-route, returns []), turning each concrete descriptor into a
|
|
// candidate. It runs OUTSIDE the picker's rng scope, so from(...).generate()
|
|
// draws nothing and no seed advances.
|
|
func (v *Verifier) collectActions(object *goja.Object, prob float64, weighted bool, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) {
|
|
generate, ok := goja.AssertFunction(object.Get("generate"))
|
|
if !ok {
|
|
return
|
|
}
|
|
result, err := generate(goja.Undefined())
|
|
if err != nil {
|
|
return
|
|
}
|
|
array := result.ToObject(v.runtime)
|
|
if array == nil {
|
|
return
|
|
}
|
|
length := int(array.Get("length").ToInteger())
|
|
for i := range length {
|
|
candidate, ok := v.candidateFromDescriptor(array.Get(strconv.Itoa(i)), nodeIndex)
|
|
if !ok {
|
|
continue
|
|
}
|
|
candidate.prob = prob
|
|
candidate.Weighted = weighted
|
|
*out = append(*out, candidate)
|
|
}
|
|
}
|
|
|
|
// candidateFromDescriptor lowers one authored ActionDescriptor (as a goja
|
|
// object) into a ready-to-run candidate, resolving the target's coordinates,
|
|
// selector, and visible-text label. Actions on a disabled control are dropped.
|
|
func (v *Verifier) candidateFromDescriptor(value goja.Value, nodeIndex map[*hierarchy.Element]*hierarchy.Node) (ActionCandidate, bool) {
|
|
object := value.ToObject(v.runtime)
|
|
if object == nil {
|
|
return ActionCandidate{}, false
|
|
}
|
|
kindValue := object.Get("kind")
|
|
if kindValue == nil || goja.IsUndefined(kindValue) {
|
|
return ActionCandidate{}, false
|
|
}
|
|
kind := ActionKind(kindValue.String())
|
|
switch kind {
|
|
case ActionKindTap, ActionKindDoubleTap, ActionKindLongPress:
|
|
target := v.resolveTarget(object.Get("on"), nodeIndex)
|
|
if target.disabled {
|
|
return ActionCandidate{}, false
|
|
}
|
|
return ActionCandidate{
|
|
Kind: kind,
|
|
Label: target.label,
|
|
Action: Action{Kind: kind, On: target.selector, X: target.x, Y: target.y},
|
|
}, true
|
|
case ActionKindInputText:
|
|
target := v.resolveTarget(object.Get("into"), nodeIndex)
|
|
if target.disabled {
|
|
return ActionCandidate{}, false
|
|
}
|
|
text := stringField(object, "text")
|
|
return ActionCandidate{
|
|
Kind: kind,
|
|
Label: target.label,
|
|
InputType: target.inputType,
|
|
Action: Action{Kind: kind, On: target.selector, X: target.x, Y: target.y, Text: text},
|
|
}, true
|
|
case ActionKindScroll:
|
|
target := v.resolveTarget(object.Get("in"), nodeIndex)
|
|
direction := stringField(object, "direction")
|
|
if direction == "" {
|
|
direction = "down"
|
|
}
|
|
return ActionCandidate{
|
|
Kind: kind,
|
|
Direction: direction,
|
|
Action: Action{Kind: kind, On: target.selector, Direction: direction},
|
|
}, true
|
|
case ActionKindPressKey:
|
|
return ActionCandidate{
|
|
Kind: kind,
|
|
Action: Action{Kind: kind, Key: stringField(object, "key")},
|
|
}, true
|
|
case ActionKindWait:
|
|
return ActionCandidate{Kind: kind, Action: Action{Kind: kind}}, true
|
|
default:
|
|
return ActionCandidate{}, false
|
|
}
|
|
}
|
|
|
|
// resolvedTarget is the geometry, selector, label, and input hint a target
|
|
// (ax element, selector string, or bare point) resolves to.
|
|
type resolvedTarget struct {
|
|
x, y int
|
|
selector string
|
|
label string
|
|
inputType string
|
|
disabled bool
|
|
}
|
|
|
|
// resolveTarget reads an authored action's target. Ax element handles carry
|
|
// x/y/__sanderlingSelector plus their own text; a bare selector string resolves
|
|
// against the current tree; a point carries geometry only.
|
|
func (v *Verifier) resolveTarget(value goja.Value, nodeIndex map[*hierarchy.Element]*hierarchy.Node) resolvedTarget {
|
|
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
|
|
return resolvedTarget{}
|
|
}
|
|
if selector, ok := value.Export().(string); ok {
|
|
return v.targetFromSelector(selector, nodeIndex)
|
|
}
|
|
object := value.ToObject(v.runtime)
|
|
if object == nil {
|
|
return resolvedTarget{}
|
|
}
|
|
selector := stringField(object, tagSelector)
|
|
if selector == "" {
|
|
selector = stringField(object, "selector")
|
|
}
|
|
target := resolvedTarget{
|
|
x: int(object.Get("x").ToInteger()),
|
|
y: int(object.Get("y").ToInteger()),
|
|
selector: selector,
|
|
}
|
|
if element := v.findBySelector(selector); element != nil {
|
|
target.label = visibleLabel(element, nodeIndex)
|
|
target.inputType = inputTypeHint(element)
|
|
target.disabled = !element.Enabled && hasEnabled(element)
|
|
}
|
|
if target.label == "" {
|
|
target.label = truncateLabel(stringField(object, "text"))
|
|
}
|
|
return target
|
|
}
|
|
|
|
// targetFromSelector resolves a bare selector-string target against the tree.
|
|
func (v *Verifier) targetFromSelector(selector string, nodeIndex map[*hierarchy.Element]*hierarchy.Node) resolvedTarget {
|
|
target := resolvedTarget{selector: selector}
|
|
element := v.findBySelector(selector)
|
|
if element == nil {
|
|
return target
|
|
}
|
|
target.x, target.y = element.Bounds.Center()
|
|
target.label = visibleLabel(element, nodeIndex)
|
|
target.inputType = inputTypeHint(element)
|
|
target.disabled = !element.Enabled && hasEnabled(element)
|
|
return target
|
|
}
|
|
|
|
func (v *Verifier) findBySelector(selector string) *hierarchy.Element {
|
|
if selector == "" || v.lastTree == nil {
|
|
return nil
|
|
}
|
|
return v.lastTree.Find(selector)
|
|
}
|
|
|
|
// collectBuiltin enumerates a builtin verb over the current tree: tap-family and
|
|
// typing emit one candidate per applicable element; scrolls/swipes fold into
|
|
// directional gestures over scrollable containers.
|
|
func (v *Verifier) collectBuiltin(verb string, prob float64, weighted bool, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) {
|
|
switch verb {
|
|
case "taps", "doubleTaps", "longPresses":
|
|
kind := verbActionKind(verb)
|
|
for _, element := range v.elementsForVerb(verb) {
|
|
x, y := element.Bounds.Center()
|
|
*out = append(*out, ActionCandidate{
|
|
Kind: kind,
|
|
Label: visibleLabel(element, nodeIndex),
|
|
Action: Action{Kind: kind, On: selectorForElement(v.lastTree, element), X: x, Y: y},
|
|
prob: prob,
|
|
Weighted: weighted,
|
|
})
|
|
}
|
|
case "typing":
|
|
for _, element := range v.elementsForVerb(verb) {
|
|
x, y := element.Bounds.Center()
|
|
*out = append(*out, ActionCandidate{
|
|
Kind: ActionKindInputText,
|
|
Label: visibleLabel(element, nodeIndex),
|
|
InputType: inputTypeHint(element),
|
|
LLMText: true,
|
|
Action: Action{Kind: ActionKindInputText, On: selectorForElement(v.lastTree, element), X: x, Y: y},
|
|
prob: prob,
|
|
Weighted: weighted,
|
|
})
|
|
}
|
|
case "scrolls", "swipes":
|
|
v.collectGestures(prob, weighted, out)
|
|
}
|
|
}
|
|
|
|
// collectGestures emits directional scrolls scoped to each scrollable container,
|
|
// never per element and never element-labeled. Folding both scrolls and swipes
|
|
// here is what removes the flood of mislabeled `Swipe "X"` gestures.
|
|
func (v *Verifier) collectGestures(prob float64, weighted bool, out *[]ActionCandidate) {
|
|
scope := v.scopedElements()
|
|
for _, element := range v.lastTree.Elements {
|
|
if !scope[element] {
|
|
continue
|
|
}
|
|
if element.Attributes["scrollable"] != "true" {
|
|
continue
|
|
}
|
|
if element.Bounds.Width() <= 0 || element.Bounds.Height() <= 0 {
|
|
continue
|
|
}
|
|
selector := selectorForElement(v.lastTree, element)
|
|
for _, direction := range gestureDirections {
|
|
action := Action{Kind: ActionKindScroll, On: selector, Direction: direction}
|
|
if selector == "" {
|
|
action.FromX, action.FromY, action.ToX, action.ToY = scrollGeometry(element.Bounds, direction)
|
|
}
|
|
*out = append(*out, ActionCandidate{
|
|
Kind: ActionKindScroll,
|
|
Direction: direction,
|
|
Action: action,
|
|
prob: prob,
|
|
Weighted: weighted,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
// scrollGeometry lowers a directional scroll to swipe endpoints over the given
|
|
// container bounds, matching the runner's own derivation, used only when the
|
|
// container has no resolving selector.
|
|
func scrollGeometry(bounds hierarchy.Bounds, direction string) (fromX, fromY, toX, toY int) {
|
|
cx, cy := bounds.Center()
|
|
fromX, fromY, toX, toY = cx, cy, cx, cy
|
|
switch direction {
|
|
case "down":
|
|
toY = cy - 4*bounds.Height()/10
|
|
case "up":
|
|
toY = cy + 4*bounds.Height()/10
|
|
case "left":
|
|
toX = cx + 4*bounds.Width()/10
|
|
case "right":
|
|
toX = cx - 4*bounds.Width()/10
|
|
}
|
|
return fromX, fromY, max(0, toX), max(0, toY)
|
|
}
|
|
|
|
// elementsForVerb returns the in-scope elements a builtin verb applies to, in
|
|
// tree order (the seeded picker's enumeration order), reusing verbAccepts.
|
|
func (v *Verifier) elementsForVerb(verb string) []*hierarchy.Element {
|
|
scope := v.scopedElements()
|
|
var elements []*hierarchy.Element
|
|
for _, element := range v.lastTree.Elements {
|
|
if scope[element] && verbAccepts(verb, element) {
|
|
elements = append(elements, element)
|
|
}
|
|
}
|
|
return elements
|
|
}
|
|
|
|
// finalizeCandidates renders each candidate's description, dedups identical
|
|
// descriptions (summing weight), numbers the survivors 1..N, and rounds the
|
|
// accumulated probability into a percentage Weight.
|
|
func finalizeCandidates(raw []ActionCandidate) []ActionCandidate {
|
|
seen := make(map[string]int, len(raw))
|
|
result := make([]ActionCandidate, 0, len(raw))
|
|
for _, candidate := range raw {
|
|
candidate.Description = describeCandidate(candidate)
|
|
if index, ok := seen[candidate.Description]; ok {
|
|
result[index].prob += candidate.prob
|
|
result[index].Weighted = result[index].Weighted || candidate.Weighted
|
|
continue
|
|
}
|
|
seen[candidate.Description] = len(result)
|
|
result = append(result, candidate)
|
|
}
|
|
for i := range result {
|
|
result[i].Index = i + 1
|
|
if result[i].Weighted {
|
|
result[i].Weight = max(1, int(math.Round(result[i].prob*100)))
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// describeCandidate renders the plain, echo-friendly description shown in the
|
|
// numbered list. It is the dedup key, so it must be stable and unique per
|
|
// distinct action.
|
|
func describeCandidate(candidate ActionCandidate) string {
|
|
switch candidate.Kind {
|
|
case ActionKindTap:
|
|
return fmt.Sprintf("Tap %q", candidate.Label)
|
|
case ActionKindDoubleTap:
|
|
return fmt.Sprintf("Double-tap %q", candidate.Label)
|
|
case ActionKindLongPress:
|
|
return fmt.Sprintf("Long-press %q", candidate.Label)
|
|
case ActionKindInputText:
|
|
if candidate.LLMText {
|
|
if candidate.InputType != "" {
|
|
return fmt.Sprintf("Type into %q (%s)", candidate.Label, candidate.InputType)
|
|
}
|
|
return fmt.Sprintf("Type into %q", candidate.Label)
|
|
}
|
|
return fmt.Sprintf("Type %q into %q", candidate.Action.Text, candidate.Label)
|
|
case ActionKindScroll:
|
|
return "Scroll " + candidate.Direction
|
|
case ActionKindPressKey:
|
|
return "Press " + candidate.Action.Key
|
|
case ActionKindWait:
|
|
return "Wait"
|
|
default:
|
|
return string(candidate.Kind)
|
|
}
|
|
}
|
|
|
|
// buildNodeIndex maps each Element pointer to its Node so descendant text can be
|
|
// borrowed for a control whose own text is empty.
|
|
func buildNodeIndex(tree *hierarchy.Tree) map[*hierarchy.Element]*hierarchy.Node {
|
|
index := make(map[*hierarchy.Element]*hierarchy.Node)
|
|
if tree == nil || tree.Root == nil {
|
|
return index
|
|
}
|
|
var walk func(node *hierarchy.Node)
|
|
walk = func(node *hierarchy.Node) {
|
|
index[&node.Element] = node
|
|
for _, child := range node.Children {
|
|
walk(child)
|
|
}
|
|
}
|
|
walk(tree.Root)
|
|
return index
|
|
}
|
|
|
|
// visibleLabel names a control by what a user would read: its own text, then
|
|
// description, then a field hint, then text borrowed from its descendants (the
|
|
// case that fixes empty-text Compose buttons whose word lives on a child), then
|
|
// its class as a last resort.
|
|
func visibleLabel(element *hierarchy.Element, nodeIndex map[*hierarchy.Element]*hierarchy.Node) string {
|
|
// An editable field's own text is the transient typed value ("1"); its hint
|
|
// names its purpose ("Amount") and stays stable, so prefer the hint there.
|
|
if element.Editable {
|
|
if hint := element.Attributes["hintText"]; hint != "" {
|
|
return truncateLabel(hint)
|
|
}
|
|
}
|
|
if element.Text != "" {
|
|
return truncateLabel(element.Text)
|
|
}
|
|
if element.Description != "" {
|
|
return truncateLabel(element.Description)
|
|
}
|
|
if hint := element.Attributes["hintText"]; hint != "" {
|
|
return truncateLabel(hint)
|
|
}
|
|
if node := nodeIndex[element]; node != nil {
|
|
if text := descendantText(node); text != "" {
|
|
return truncateLabel(text)
|
|
}
|
|
}
|
|
if element.Class != "" {
|
|
return element.Class
|
|
}
|
|
if element.ResourceID != "" {
|
|
return element.ResourceID
|
|
}
|
|
return "control"
|
|
}
|
|
|
|
// descendantText joins the visible text of a node's descendants in tree order,
|
|
// so a clickable wrapper borrows the label of the Text child it contains.
|
|
func descendantText(node *hierarchy.Node) string {
|
|
var parts []string
|
|
var walk func(node *hierarchy.Node)
|
|
walk = func(node *hierarchy.Node) {
|
|
for _, child := range node.Children {
|
|
switch {
|
|
case child.Element.Text != "":
|
|
parts = append(parts, child.Element.Text)
|
|
case child.Element.Description != "":
|
|
parts = append(parts, child.Element.Description)
|
|
}
|
|
walk(child)
|
|
}
|
|
}
|
|
walk(node)
|
|
return strings.Join(parts, " ")
|
|
}
|
|
|
|
// truncateLabel trims and shortens a label to one line's worth of runes.
|
|
func truncateLabel(text string) string {
|
|
text = strings.TrimSpace(strings.ReplaceAll(text, "\n", " "))
|
|
runes := []rune(text)
|
|
if len(runes) <= maxLabelRunes {
|
|
return text
|
|
}
|
|
return strings.TrimSpace(string(runes[:maxLabelRunes])) + "…"
|
|
}
|
|
|
|
// inputTypeHint reports a typing field's expected input as a short word the
|
|
// model can use to synthesize a value, or "" when nothing distinguishes it.
|
|
func inputTypeHint(element *hierarchy.Element) string {
|
|
haystack := strings.ToLower(element.Class + " " +
|
|
element.Attributes["inputType"] + " " + element.Attributes["hintText"])
|
|
switch {
|
|
case strings.Contains(haystack, "number") || strings.Contains(haystack, "amount") || strings.Contains(haystack, "numeric"):
|
|
return "number"
|
|
case strings.Contains(haystack, "email"):
|
|
return "email"
|
|
case strings.Contains(haystack, "password"):
|
|
return "password"
|
|
case strings.Contains(haystack, "phone"):
|
|
return "phone"
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// hasEnabled reports whether the source tree carried an explicit enabled flag
|
|
// for the element, so a missing flag is not mistaken for "disabled".
|
|
func hasEnabled(element *hierarchy.Element) bool {
|
|
_, ok := element.Attributes["enabled"]
|
|
return ok
|
|
}
|
|
|
|
// stringField reads a string property off a goja object, returning "" when
|
|
// absent, null, or undefined.
|
|
func stringField(object *goja.Object, key string) string {
|
|
value := object.Get(key)
|
|
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
|
|
return ""
|
|
}
|
|
return value.String()
|
|
}
|