mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
fix(verifier): lower authored actions the way the seeded arm does
The authored descriptor path had no parity guard and diverged from the wire format on almost every verb. A Wait lost its duration and was skipped as a zero-duration wait. A Scroll lost its endpoints and its 250ms. A target that resolved to nothing became a tap at the origin, a phantom focus tap, or a swipe to (0,0) instead of being dropped. An authored target object with no x property panicked the whole run at candidate enumeration: ToInteger was called on a nil goja.Value. A target on the screen origin is still kept, so the drop rule cannot swallow it. Builtins were never affected. They serialize through the same path the seeded arm uses, which the existing policy parity test covers. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
2cbe03c3fb
commit
73f387f8bd
2 files changed
+193
-24
No files matched your search
+88
-24
@@ -145,6 +145,12 @@ type ActionCandidate struct {
|
||||
// enough to render on one numbered line.
|
||||
const maxLabelRunes = 40
|
||||
|
||||
// gestureDurationMillis is how long a drag takes when the descriptor did not say.
|
||||
// It mirrors DEFAULT_SWIPE_DURATION in runtime-entry.ts, which is what the
|
||||
// seeded policy's action carries by the time it reaches the runner: the two
|
||||
// policies must hand the driver the same gesture, not two speeds of it.
|
||||
const gestureDurationMillis = 250
|
||||
|
||||
// The label sources a candidate's target can be named by. This is the
|
||||
// observation channel the model reads, and nothing else: the seeded picker
|
||||
// selects by index and never asks for a label, so the two seeded cells of a
|
||||
@@ -297,8 +303,8 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, labels labelContext
|
||||
kind := ActionKind(kindValue.String())
|
||||
switch kind {
|
||||
case ActionKindTap, ActionKindDoubleTap, ActionKindLongPress:
|
||||
target := v.resolveTarget(object.Get("on"), labels)
|
||||
if target.disabled {
|
||||
target, ok := v.resolveTarget(object.Get("on"), labels)
|
||||
if !ok || target.disabled {
|
||||
return ActionCandidate{}, false
|
||||
}
|
||||
return ActionCandidate{
|
||||
@@ -307,8 +313,8 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, labels labelContext
|
||||
Action: Action{Kind: kind, On: target.selector, X: target.x, Y: target.y},
|
||||
}, true
|
||||
case ActionKindInputText:
|
||||
target := v.resolveTarget(object.Get("into"), labels)
|
||||
if target.disabled {
|
||||
target, ok := v.resolveTarget(object.Get("into"), labels)
|
||||
if !ok || target.disabled {
|
||||
return ActionCandidate{}, false
|
||||
}
|
||||
text := stringField(object, "text")
|
||||
@@ -319,26 +325,41 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, labels labelContext
|
||||
Action: Action{Kind: kind, On: target.selector, X: target.x, Y: target.y, Text: text},
|
||||
}, true
|
||||
case ActionKindScroll:
|
||||
target := v.resolveTarget(object.Get("in"), labels)
|
||||
container, _ := v.resolveTarget(object.Get("in"), labels)
|
||||
direction := stringField(object, "direction")
|
||||
if direction == "" {
|
||||
direction = "down"
|
||||
}
|
||||
return ActionCandidate{
|
||||
Kind: kind,
|
||||
Direction: direction,
|
||||
Action: Action{Kind: kind, On: target.selector, Direction: direction},
|
||||
}, true
|
||||
action := Action{
|
||||
Kind: kind,
|
||||
On: container.selector,
|
||||
Direction: direction,
|
||||
DurationMillis: gestureDurationMillis,
|
||||
}
|
||||
// Endpoints only when the descriptor computed the whole gesture (the
|
||||
// builtin generator does). Anchoring an authored scroll on the
|
||||
// container's own point instead would hand the runner a drag from a
|
||||
// point to itself, which it executes as written.
|
||||
from, hasFrom := v.resolveTarget(object.Get("from"), labels)
|
||||
to, hasTo := v.resolveTarget(object.Get("to"), labels)
|
||||
if hasFrom && hasTo {
|
||||
action.FromX, action.FromY = from.x, from.y
|
||||
action.ToX, action.ToY = to.x, to.y
|
||||
}
|
||||
return ActionCandidate{Kind: kind, Direction: direction, Action: action}, true
|
||||
case ActionKindSwipe:
|
||||
from := v.resolveTarget(object.Get("from"), labels)
|
||||
to := v.resolveTarget(object.Get("to"), labels)
|
||||
from, hasFrom := v.resolveTarget(object.Get("from"), labels)
|
||||
to, hasTo := v.resolveTarget(object.Get("to"), labels)
|
||||
if !hasFrom || !hasTo {
|
||||
return ActionCandidate{}, false
|
||||
}
|
||||
return ActionCandidate{
|
||||
Kind: kind,
|
||||
Action: Action{
|
||||
Kind: kind,
|
||||
FromX: from.x, FromY: from.y,
|
||||
ToX: to.x, ToY: to.y,
|
||||
DurationMillis: intField(object, "durationMillis"),
|
||||
DurationMillis: intFieldOr(object, "durationMillis", gestureDurationMillis),
|
||||
},
|
||||
}, true
|
||||
case ActionKindPressKey:
|
||||
@@ -347,7 +368,10 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, labels labelContext
|
||||
Action: Action{Kind: kind, Key: stringField(object, "key")},
|
||||
}, true
|
||||
case ActionKindWait:
|
||||
return ActionCandidate{Kind: kind, Action: Action{Kind: kind}}, true
|
||||
return ActionCandidate{
|
||||
Kind: kind,
|
||||
Action: Action{Kind: kind, DurationMillis: intField(object, "durationMillis")},
|
||||
}, true
|
||||
default:
|
||||
return ActionCandidate{}, false
|
||||
}
|
||||
@@ -366,26 +390,36 @@ type resolvedTarget struct {
|
||||
// resolveTarget reads an authored action's target. Ax element handles carry
|
||||
// x/y/__sanderlingSelector plus their own text and id; a bare selector string
|
||||
// resolves against the current tree; a point carries geometry only.
|
||||
func (v *Verifier) resolveTarget(value goja.Value, labels labelContext) resolvedTarget {
|
||||
//
|
||||
// The second return is false when the value names no target the seeded policy
|
||||
// could act on either: runtime-entry.ts pointOf accepts a non-empty selector
|
||||
// string or an object with numeric coordinates, and drops the whole action
|
||||
// otherwise. Lowering one of those to (0, 0) instead would offer the model an
|
||||
// action the seeded policy never takes, aimed at the screen corner.
|
||||
func (v *Verifier) resolveTarget(value goja.Value, labels labelContext) (resolvedTarget, bool) {
|
||||
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
|
||||
return resolvedTarget{}
|
||||
return resolvedTarget{}, false
|
||||
}
|
||||
if selector, ok := value.Export().(string); ok {
|
||||
return v.targetFromSelector(selector, labels)
|
||||
if selector == "" {
|
||||
return resolvedTarget{}, false
|
||||
}
|
||||
return v.targetFromSelector(selector, labels), true
|
||||
}
|
||||
object := value.ToObject(v.runtime)
|
||||
if object == nil {
|
||||
return resolvedTarget{}
|
||||
return resolvedTarget{}, false
|
||||
}
|
||||
x, hasX := numberField(object, "x")
|
||||
y, hasY := numberField(object, "y")
|
||||
if !hasX || !hasY {
|
||||
return resolvedTarget{}, false
|
||||
}
|
||||
selector := stringField(object, tagSelector)
|
||||
if selector == "" {
|
||||
selector = stringField(object, "selector")
|
||||
}
|
||||
target := resolvedTarget{
|
||||
x: int(object.Get("x").ToInteger()),
|
||||
y: int(object.Get("y").ToInteger()),
|
||||
selector: selector,
|
||||
}
|
||||
target := resolvedTarget{x: x, y: y, selector: selector}
|
||||
if element := v.findBySelector(selector); element != nil {
|
||||
target.label = labels.label(element)
|
||||
target.inputType = inputTypeHint(element)
|
||||
@@ -394,7 +428,7 @@ func (v *Verifier) resolveTarget(value goja.Value, labels labelContext) resolved
|
||||
if target.label == "" {
|
||||
target.label = truncateLabel(stringField(object, labels.handleField()))
|
||||
}
|
||||
return target
|
||||
return target, true
|
||||
}
|
||||
|
||||
// targetFromSelector resolves a bare selector-string target against the tree.
|
||||
@@ -746,3 +780,33 @@ func intField(object *goja.Object, key string) int {
|
||||
}
|
||||
return int(value.ToInteger())
|
||||
}
|
||||
|
||||
// intFieldOr reads a numeric property, falling back when the descriptor left it
|
||||
// out. It mirrors the serializer's `??`, so an explicit zero is kept.
|
||||
func intFieldOr(object *goja.Object, key string, fallback int) int {
|
||||
value := object.Get(key)
|
||||
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
|
||||
return fallback
|
||||
}
|
||||
return int(value.ToInteger())
|
||||
}
|
||||
|
||||
// numberField reads a property that must actually BE a number, which is what
|
||||
// tells a target carrying no coordinates apart from one anchored at (0, 0).
|
||||
func numberField(object *goja.Object, key string) (int, bool) {
|
||||
value := object.Get(key)
|
||||
if value == nil {
|
||||
return 0, false
|
||||
}
|
||||
switch number := value.Export().(type) {
|
||||
case int64:
|
||||
return int(number), true
|
||||
case float64:
|
||||
if math.IsNaN(number) {
|
||||
return 0, false
|
||||
}
|
||||
return int(number), true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
@@ -450,6 +450,111 @@ func TestCandidatesSurfaceAuthoredUntargetedActions(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestCandidatesAuthoredWaitKeepsItsDuration: a Wait that loses its duration is
|
||||
// a wait of zero, which the runner cannot dispatch at all, so the model would be
|
||||
// idling on paper while the seeded arm really waits.
|
||||
func TestCandidatesAuthoredWaitKeepsItsDuration(t *testing.T) {
|
||||
actions := `{kind:'actions', generate: () => [{kind:'Wait', durationMillis: 500}]}`
|
||||
candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText)
|
||||
candidate, ok := findCandidate(candidates, "Wait")
|
||||
if !ok {
|
||||
t.Fatalf("authored wait missing: %v", descriptions(candidates))
|
||||
}
|
||||
if candidate.Action.DurationMillis != 500 {
|
||||
t.Errorf("wait duration = %d, want the authored 500", candidate.Action.DurationMillis)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCandidatesAuthoredScrollNamesItsContainer: the container is the whole
|
||||
// point of an authored scroll. Dropped, the runner re-derives the gesture from
|
||||
// the screen and the scroll lands on whatever else is scrollable.
|
||||
func TestCandidatesAuthoredScrollNamesItsContainer(t *testing.T) {
|
||||
actions := `{kind:'actions', generate: () => [{kind:'Scroll', direction:'down', in:'id:List'}]}`
|
||||
candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText)
|
||||
candidate, ok := findCandidate(candidates, "Scroll down")
|
||||
if !ok {
|
||||
t.Fatalf("authored scroll missing: %v", descriptions(candidates))
|
||||
}
|
||||
if candidate.Action.On != "id:List" {
|
||||
t.Errorf("scroll container = %q, want id:List", candidate.Action.On)
|
||||
}
|
||||
if candidate.Action.DurationMillis != gestureDurationMillis {
|
||||
t.Errorf("scroll duration = %d, want %d", candidate.Action.DurationMillis, gestureDurationMillis)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCandidatesAuthoredScrollKeepsPrecomputedEndpoints: a descriptor that
|
||||
// already carries the gesture is executed as written rather than re-derived.
|
||||
func TestCandidatesAuthoredScrollKeepsPrecomputedEndpoints(t *testing.T) {
|
||||
actions := `{kind:'actions', generate: () => [
|
||||
{kind:'Scroll', direction:'down', in:'id:List', from:{x:540,y:1400}, to:{x:540,y:920}}
|
||||
]}`
|
||||
candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText)
|
||||
candidate, ok := findCandidate(candidates, "Scroll down")
|
||||
if !ok {
|
||||
t.Fatalf("authored scroll missing: %v", descriptions(candidates))
|
||||
}
|
||||
action := candidate.Action
|
||||
got := [4]int{action.FromX, action.FromY, action.ToX, action.ToY}
|
||||
if got != [4]int{540, 1400, 540, 920} {
|
||||
t.Errorf("scroll endpoints = %v, want the descriptor's (540,1400)->(540,920)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCandidatesAuthoredSwipeDefaultsItsDuration keeps the gesture default in
|
||||
// one place: the serializer the seeded arm goes through fills an omitted
|
||||
// duration, and a candidate that left it at zero would depend on the runner
|
||||
// happening to pick the same fallback.
|
||||
func TestCandidatesAuthoredSwipeDefaultsItsDuration(t *testing.T) {
|
||||
actions := `{kind:'actions', generate: () => [
|
||||
{kind:'Swipe', from:{x:10,y:600}, to:{x:10,y:100}},
|
||||
{kind:'Swipe', from:{x:20,y:600}, to:{x:20,y:100}, durationMillis: 400}
|
||||
]}`
|
||||
candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText)
|
||||
omitted, ok := findCandidate(candidates, "Swipe from (10,600) to (10,100)")
|
||||
if !ok {
|
||||
t.Fatalf("authored swipe missing: %v", descriptions(candidates))
|
||||
}
|
||||
if omitted.Action.DurationMillis != gestureDurationMillis {
|
||||
t.Errorf("omitted duration = %d, want %d", omitted.Action.DurationMillis, gestureDurationMillis)
|
||||
}
|
||||
authored, ok := findCandidate(candidates, "Swipe from (20,600) to (20,100)")
|
||||
if !ok {
|
||||
t.Fatalf("authored swipe missing: %v", descriptions(candidates))
|
||||
}
|
||||
if authored.Action.DurationMillis != 400 {
|
||||
t.Errorf("authored duration = %d, want 400", authored.Action.DurationMillis)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCandidatesDropTargetsThatResolveToNothing: the seeded picker drops an
|
||||
// action whose target resolves to neither coordinates nor a selector. Offering
|
||||
// it to the model instead would put a tap on the screen origin within reach,
|
||||
// which on Android is the corner that pulls the notification shade down.
|
||||
func TestCandidatesDropTargetsThatResolveToNothing(t *testing.T) {
|
||||
actions := `{kind:'actions', generate: () => [
|
||||
{kind:'Tap', on: null},
|
||||
{kind:'Tap', on: {}},
|
||||
{kind:'InputText', into: {}, text:'x'},
|
||||
{kind:'Swipe', from:{x:1,y:2}, to:{}}
|
||||
]}`
|
||||
candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText)
|
||||
if len(candidates) != 0 {
|
||||
t.Errorf("targetless actions reached the model: %v", descriptions(candidates))
|
||||
}
|
||||
}
|
||||
|
||||
// TestCandidatesKeepATargetOnTheScreenOrigin is the other side of that rule: a
|
||||
// point at (0,0) IS a target the seeded picker executes, so the drop must key on
|
||||
// a target with no coordinates rather than on coordinates that are zero.
|
||||
func TestCandidatesKeepATargetOnTheScreenOrigin(t *testing.T) {
|
||||
actions := `{kind:'actions', generate: () => [{kind:'Tap', on: {x: 0, y: 0}}]}`
|
||||
candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText)
|
||||
if len(candidates) != 1 {
|
||||
t.Fatalf("want the origin tap kept, got %v", descriptions(candidates))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCandidatesOffRouteLeafYieldsNothing(t *testing.T) {
|
||||
v := enumVerifier(t, "{kind:'actions', generate: () => []}", enumTreeJSON)
|
||||
if got := v.Candidates(LabelSourceVisibleText); len(got) != 0 {
|
||||
|
||||
Reference in new issue
Block a user