diff --git a/internal/verifier/llm.go b/internal/verifier/llm.go index 7054ed7..31f3400 100644 --- a/internal/verifier/llm.go +++ b/internal/verifier/llm.go @@ -145,6 +145,12 @@ type ActionCandidate struct { // enough to render on one numbered line. const maxLabelRunes = 40 +// gestureDurationMillis is how long a drag takes when the descriptor did not say. +// It mirrors DEFAULT_SWIPE_DURATION in runtime-entry.ts, which is what the +// seeded policy's action carries by the time it reaches the runner: the two +// policies must hand the driver the same gesture, not two speeds of it. +const gestureDurationMillis = 250 + // The label sources a candidate's target can be named by. This is the // observation channel the model reads, and nothing else: the seeded picker // selects by index and never asks for a label, so the two seeded cells of a @@ -297,8 +303,8 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, labels labelContext kind := ActionKind(kindValue.String()) switch kind { case ActionKindTap, ActionKindDoubleTap, ActionKindLongPress: - target := v.resolveTarget(object.Get("on"), labels) - if target.disabled { + target, ok := v.resolveTarget(object.Get("on"), labels) + if !ok || target.disabled { return ActionCandidate{}, false } return ActionCandidate{ @@ -307,8 +313,8 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, labels labelContext Action: Action{Kind: kind, On: target.selector, X: target.x, Y: target.y}, }, true case ActionKindInputText: - target := v.resolveTarget(object.Get("into"), labels) - if target.disabled { + target, ok := v.resolveTarget(object.Get("into"), labels) + if !ok || target.disabled { return ActionCandidate{}, false } text := stringField(object, "text") @@ -319,26 +325,41 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, labels labelContext Action: Action{Kind: kind, On: target.selector, X: target.x, Y: target.y, Text: text}, }, true case ActionKindScroll: - target := v.resolveTarget(object.Get("in"), labels) + container, _ := v.resolveTarget(object.Get("in"), labels) direction := stringField(object, "direction") if direction == "" { direction = "down" } - return ActionCandidate{ - Kind: kind, - Direction: direction, - Action: Action{Kind: kind, On: target.selector, Direction: direction}, - }, true + action := Action{ + Kind: kind, + On: container.selector, + Direction: direction, + DurationMillis: gestureDurationMillis, + } + // Endpoints only when the descriptor computed the whole gesture (the + // builtin generator does). Anchoring an authored scroll on the + // container's own point instead would hand the runner a drag from a + // point to itself, which it executes as written. + from, hasFrom := v.resolveTarget(object.Get("from"), labels) + to, hasTo := v.resolveTarget(object.Get("to"), labels) + if hasFrom && hasTo { + action.FromX, action.FromY = from.x, from.y + action.ToX, action.ToY = to.x, to.y + } + return ActionCandidate{Kind: kind, Direction: direction, Action: action}, true case ActionKindSwipe: - from := v.resolveTarget(object.Get("from"), labels) - to := v.resolveTarget(object.Get("to"), labels) + from, hasFrom := v.resolveTarget(object.Get("from"), labels) + to, hasTo := v.resolveTarget(object.Get("to"), labels) + if !hasFrom || !hasTo { + return ActionCandidate{}, false + } return ActionCandidate{ Kind: kind, Action: Action{ Kind: kind, FromX: from.x, FromY: from.y, ToX: to.x, ToY: to.y, - DurationMillis: intField(object, "durationMillis"), + DurationMillis: intFieldOr(object, "durationMillis", gestureDurationMillis), }, }, true case ActionKindPressKey: @@ -347,7 +368,10 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, labels labelContext Action: Action{Kind: kind, Key: stringField(object, "key")}, }, true case ActionKindWait: - return ActionCandidate{Kind: kind, Action: Action{Kind: kind}}, true + return ActionCandidate{ + Kind: kind, + Action: Action{Kind: kind, DurationMillis: intField(object, "durationMillis")}, + }, true default: return ActionCandidate{}, false } @@ -366,26 +390,36 @@ type resolvedTarget struct { // resolveTarget reads an authored action's target. Ax element handles carry // x/y/__sanderlingSelector plus their own text and id; a bare selector string // resolves against the current tree; a point carries geometry only. -func (v *Verifier) resolveTarget(value goja.Value, labels labelContext) resolvedTarget { +// +// The second return is false when the value names no target the seeded policy +// could act on either: runtime-entry.ts pointOf accepts a non-empty selector +// string or an object with numeric coordinates, and drops the whole action +// otherwise. Lowering one of those to (0, 0) instead would offer the model an +// action the seeded policy never takes, aimed at the screen corner. +func (v *Verifier) resolveTarget(value goja.Value, labels labelContext) (resolvedTarget, bool) { if value == nil || goja.IsUndefined(value) || goja.IsNull(value) { - return resolvedTarget{} + return resolvedTarget{}, false } if selector, ok := value.Export().(string); ok { - return v.targetFromSelector(selector, labels) + if selector == "" { + return resolvedTarget{}, false + } + return v.targetFromSelector(selector, labels), true } object := value.ToObject(v.runtime) if object == nil { - return resolvedTarget{} + return resolvedTarget{}, false + } + x, hasX := numberField(object, "x") + y, hasY := numberField(object, "y") + if !hasX || !hasY { + return resolvedTarget{}, false } selector := stringField(object, tagSelector) if selector == "" { selector = stringField(object, "selector") } - target := resolvedTarget{ - x: int(object.Get("x").ToInteger()), - y: int(object.Get("y").ToInteger()), - selector: selector, - } + target := resolvedTarget{x: x, y: y, selector: selector} if element := v.findBySelector(selector); element != nil { target.label = labels.label(element) target.inputType = inputTypeHint(element) @@ -394,7 +428,7 @@ func (v *Verifier) resolveTarget(value goja.Value, labels labelContext) resolved if target.label == "" { target.label = truncateLabel(stringField(object, labels.handleField())) } - return target + return target, true } // targetFromSelector resolves a bare selector-string target against the tree. @@ -746,3 +780,33 @@ func intField(object *goja.Object, key string) int { } return int(value.ToInteger()) } + +// intFieldOr reads a numeric property, falling back when the descriptor left it +// out. It mirrors the serializer's `??`, so an explicit zero is kept. +func intFieldOr(object *goja.Object, key string, fallback int) int { + value := object.Get(key) + if value == nil || goja.IsUndefined(value) || goja.IsNull(value) { + return fallback + } + return int(value.ToInteger()) +} + +// numberField reads a property that must actually BE a number, which is what +// tells a target carrying no coordinates apart from one anchored at (0, 0). +func numberField(object *goja.Object, key string) (int, bool) { + value := object.Get(key) + if value == nil { + return 0, false + } + switch number := value.Export().(type) { + case int64: + return int(number), true + case float64: + if math.IsNaN(number) { + return 0, false + } + return int(number), true + default: + return 0, false + } +} diff --git a/internal/verifier/llm_test.go b/internal/verifier/llm_test.go index ce7948e..e4502e3 100644 --- a/internal/verifier/llm_test.go +++ b/internal/verifier/llm_test.go @@ -450,6 +450,111 @@ func TestCandidatesSurfaceAuthoredUntargetedActions(t *testing.T) { } } +// TestCandidatesAuthoredWaitKeepsItsDuration: a Wait that loses its duration is +// a wait of zero, which the runner cannot dispatch at all, so the model would be +// idling on paper while the seeded arm really waits. +func TestCandidatesAuthoredWaitKeepsItsDuration(t *testing.T) { + actions := `{kind:'actions', generate: () => [{kind:'Wait', durationMillis: 500}]}` + candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidate, ok := findCandidate(candidates, "Wait") + if !ok { + t.Fatalf("authored wait missing: %v", descriptions(candidates)) + } + if candidate.Action.DurationMillis != 500 { + t.Errorf("wait duration = %d, want the authored 500", candidate.Action.DurationMillis) + } +} + +// TestCandidatesAuthoredScrollNamesItsContainer: the container is the whole +// point of an authored scroll. Dropped, the runner re-derives the gesture from +// the screen and the scroll lands on whatever else is scrollable. +func TestCandidatesAuthoredScrollNamesItsContainer(t *testing.T) { + actions := `{kind:'actions', generate: () => [{kind:'Scroll', direction:'down', in:'id:List'}]}` + candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidate, ok := findCandidate(candidates, "Scroll down") + if !ok { + t.Fatalf("authored scroll missing: %v", descriptions(candidates)) + } + if candidate.Action.On != "id:List" { + t.Errorf("scroll container = %q, want id:List", candidate.Action.On) + } + if candidate.Action.DurationMillis != gestureDurationMillis { + t.Errorf("scroll duration = %d, want %d", candidate.Action.DurationMillis, gestureDurationMillis) + } +} + +// TestCandidatesAuthoredScrollKeepsPrecomputedEndpoints: a descriptor that +// already carries the gesture is executed as written rather than re-derived. +func TestCandidatesAuthoredScrollKeepsPrecomputedEndpoints(t *testing.T) { + actions := `{kind:'actions', generate: () => [ + {kind:'Scroll', direction:'down', in:'id:List', from:{x:540,y:1400}, to:{x:540,y:920}} + ]}` + candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidate, ok := findCandidate(candidates, "Scroll down") + if !ok { + t.Fatalf("authored scroll missing: %v", descriptions(candidates)) + } + action := candidate.Action + got := [4]int{action.FromX, action.FromY, action.ToX, action.ToY} + if got != [4]int{540, 1400, 540, 920} { + t.Errorf("scroll endpoints = %v, want the descriptor's (540,1400)->(540,920)", got) + } +} + +// TestCandidatesAuthoredSwipeDefaultsItsDuration keeps the gesture default in +// one place: the serializer the seeded arm goes through fills an omitted +// duration, and a candidate that left it at zero would depend on the runner +// happening to pick the same fallback. +func TestCandidatesAuthoredSwipeDefaultsItsDuration(t *testing.T) { + actions := `{kind:'actions', generate: () => [ + {kind:'Swipe', from:{x:10,y:600}, to:{x:10,y:100}}, + {kind:'Swipe', from:{x:20,y:600}, to:{x:20,y:100}, durationMillis: 400} + ]}` + candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + omitted, ok := findCandidate(candidates, "Swipe from (10,600) to (10,100)") + if !ok { + t.Fatalf("authored swipe missing: %v", descriptions(candidates)) + } + if omitted.Action.DurationMillis != gestureDurationMillis { + t.Errorf("omitted duration = %d, want %d", omitted.Action.DurationMillis, gestureDurationMillis) + } + authored, ok := findCandidate(candidates, "Swipe from (20,600) to (20,100)") + if !ok { + t.Fatalf("authored swipe missing: %v", descriptions(candidates)) + } + if authored.Action.DurationMillis != 400 { + t.Errorf("authored duration = %d, want 400", authored.Action.DurationMillis) + } +} + +// TestCandidatesDropTargetsThatResolveToNothing: the seeded picker drops an +// action whose target resolves to neither coordinates nor a selector. Offering +// it to the model instead would put a tap on the screen origin within reach, +// which on Android is the corner that pulls the notification shade down. +func TestCandidatesDropTargetsThatResolveToNothing(t *testing.T) { + actions := `{kind:'actions', generate: () => [ + {kind:'Tap', on: null}, + {kind:'Tap', on: {}}, + {kind:'InputText', into: {}, text:'x'}, + {kind:'Swipe', from:{x:1,y:2}, to:{}} + ]}` + candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + if len(candidates) != 0 { + t.Errorf("targetless actions reached the model: %v", descriptions(candidates)) + } +} + +// TestCandidatesKeepATargetOnTheScreenOrigin is the other side of that rule: a +// point at (0,0) IS a target the seeded picker executes, so the drop must key on +// a target with no coordinates rather than on coordinates that are zero. +func TestCandidatesKeepATargetOnTheScreenOrigin(t *testing.T) { + actions := `{kind:'actions', generate: () => [{kind:'Tap', on: {x: 0, y: 0}}]}` + candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + if len(candidates) != 1 { + t.Fatalf("want the origin tap kept, got %v", descriptions(candidates)) + } +} + func TestCandidatesOffRouteLeafYieldsNothing(t *testing.T) { v := enumVerifier(t, "{kind:'actions', generate: () => []}", enumTreeJSON) if got := v.Candidates(LabelSourceVisibleText); len(got) != 0 {