fix(verifier): name a web handle by the same ladder as a tree element

The handle fallback read only text, which is textContent and therefore always
empty for an input, so the model could not tell the amount field from the note
field. It now mirrors visibleLabel's ladder rather than introducing a second
naming scheme.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
pj committed 2026-08-13 22:55:20 +05:30
1 parent 037a803f0e
commit 78e561ba7e
2 files changed
+133 -9

No files matched your search

+39 -9
View File
@@ -504,7 +504,7 @@ func (v *Verifier) resolveTarget(value goja.Value, labels labelContext) (resolve
target.inputType = inputTypeHint(element)
}
if target.label == "" {
target.label = truncateLabel(stringField(object, labels.handleField()))
target.label = truncateLabel(v.handleLabel(object, labels))
}
return target, true
}
@@ -721,15 +721,35 @@ func (l labelContext) label(element *hierarchy.Element) string {
return visibleLabel(element, l.nodeIndex)
}
// handleField is the ax-element handle field a label falls back to when the
// target's selector no longer resolves against the current tree. Reading the
// handle's text there would leak visible text into an identifier-labelled run,
// which is the one thing that arm must not see.
func (l labelContext) handleField() string {
if l.source == LabelSourceResourceID {
return "id"
// handleLabel names a target from the ax handle alone, for the web tick path
// where the handle was built in V8 and carries no selector to resolve against
// the tree. It walks visibleLabel's rungs over the fields a handle has: an
// editable field's hint names its purpose, its own text is the transient typed
// value. The identifier arm reads the handle's id and nothing a user could
// read, which is the one thing that arm must not see.
func (v *Verifier) handleLabel(object *goja.Object, labels labelContext) string {
if labels.source == LabelSourceResourceID {
return stringField(object, "id")
}
return "text"
hint := v.handleAttribute(object, "hintText")
if hint != "" && boolField(object, "editable") {
return hint
}
if text := stringField(object, "text"); text != "" {
return text
}
if desc := stringField(object, "desc"); desc != "" {
return desc
}
return hint
}
func (v *Verifier) handleAttribute(object *goja.Object, name string) string {
attrs := object.Get("attrs")
if attrs == nil || goja.IsUndefined(attrs) || goja.IsNull(attrs) {
return ""
}
return stringField(attrs.ToObject(v.runtime), name)
}
// resourceIdentifierLabel names a control by the identifier the app assigned it,
@@ -841,6 +861,16 @@ func stringField(object *goja.Object, key string) string {
return value.String()
}
// boolField reads a boolean property off a goja object, returning false when
// absent, null, or undefined.
func boolField(object *goja.Object, key string) bool {
value := object.Get(key)
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
return false
}
return value.ToBoolean()
}
// intField reads a numeric property off a goja object, returning 0 when absent,
// null, or undefined.
func intField(object *goja.Object, key string) int {
+94
View File
@@ -1,6 +1,7 @@
package verifier
import (
"encoding/json"
"strings"
"testing"
@@ -739,3 +740,96 @@ func TestCandidatesAcceptASingleItemAuthoredSampler(t *testing.T) {
t.Errorf("sampled tap missing: %v", descriptions(candidates))
}
}
// webFieldTreeJSON is the add-transaction screen as the chrome driver dumps it:
// two inputs a user tells apart by the <label> bound to each, which the dump
// does not carry. Named from the tree alone they collapse onto their CSS class.
const webFieldTreeJSON = `{
"attributes": {"tag": "html", "bounds": "[0,0,400,800]"},
"children": [
{"attributes": {"resource-id": "txn-amount", "tag": "input", "class": "input amount-input", "bounds": "[0,100,400,160]"}, "editable": true, "enabled": true, "children": []},
{"attributes": {"resource-id": "txn-note", "tag": "input", "class": "input", "bounds": "[0,200,400,260]"}, "editable": true, "enabled": true, "children": []}
]
}`
// The web tick evaluates extractors in V8 and injects the handles here, so an
// authored action's target is a plain object with no selector to resolve
// against the tree. Naming it by the handle's own text names every input "",
// and the model then cannot tell the amount field from the note field.
// pkg/spec/test/web-runtime.test.ts asserts the producing side builds these
// handles with the hint each assertion here reads.
func TestCandidatesNameWebAuthoredFieldsByTheirHint(t *testing.T) {
const amountHandle = `{
"id": "txn-amount", "text": "", "desc": "", "class": "input amount-input",
"clickable": true, "enabled": true, "editable": true, "focused": false,
"x": 200, "y": 130, "bounds": {"left": 0, "top": 100, "right": 400, "bottom": 160},
"attrs": {"tag": "input", "aria-label": "", "id": "txn-amount",
"class": "input amount-input", "placeholder": "0.00", "hintText": "Amount"}
}`
const noteHandle = `{
"id": "txn-note", "text": "", "desc": "", "class": "input",
"clickable": true, "enabled": true, "editable": true, "focused": false,
"x": 200, "y": 230, "bounds": {"left": 0, "top": 200, "right": 400, "bottom": 260},
"attrs": {"tag": "input", "aria-label": "", "id": "txn-note", "class": "input",
"placeholder": "What's this for?", "hintText": "Note (optional)"}
}`
v := newVerifier(t)
loadActionSpec(t, v, `
import { InputText, actions, extract } from "@sanderling/spec";
const amount = extract((s) => s.ax.find({ id: "txn-amount" })).named("amount");
const note = extract((s) => s.ax.find({ id: "txn-note" })).named("note");
globalThis.actions = actions(() => {
if (!amount.current || !note.current) return [];
return [
InputText({ into: amount.current, text: "12.34" }),
InputText({ into: note.current, text: "Coffee" }),
];
});
`)
pushTree(t, v, webFieldTreeJSON)
if _, err := v.OverrideExtractorValues(map[int]json.RawMessage{
0: json.RawMessage(amountHandle),
1: json.RawMessage(noteHandle),
}); err != nil {
t.Fatal(err)
}
candidates := mustCandidates(t, v, LabelSourceVisibleText)
if !hasCandidate(candidates, `Type "12.34" into "Amount"`) {
t.Errorf("amount field unnamed: %v", descriptions(candidates))
}
if !hasCandidate(candidates, `Type "Coffee" into "Note (optional)"`) {
t.Errorf("note field unnamed: %v", descriptions(candidates))
}
}
// The identifier arm must stay blind to anything a user reads, hint included.
func TestCandidatesNameWebAuthoredFieldsByIdentifierOnThatArm(t *testing.T) {
const amountHandle = `{
"id": "txn-amount", "text": "", "editable": true, "enabled": true,
"x": 200, "y": 130,
"attrs": {"tag": "input", "hintText": "Amount"}
}`
v := newVerifier(t)
loadActionSpec(t, v, `
import { InputText, actions, extract } from "@sanderling/spec";
const amount = extract((s) => s.ax.find({ id: "txn-amount" })).named("amount");
globalThis.actions = actions(() =>
amount.current ? [InputText({ into: amount.current, text: "12.34" })] : []);
`)
pushTree(t, v, webFieldTreeJSON)
if _, err := v.OverrideExtractorValues(map[int]json.RawMessage{
0: json.RawMessage(amountHandle),
}); err != nil {
t.Fatal(err)
}
candidates := mustCandidates(t, v, LabelSourceResourceID)
if !hasCandidate(candidates, `Type "12.34" into "txn-amount"`) {
t.Errorf("identifier arm lost the field name: %v", descriptions(candidates))
}
if hasCandidate(candidates, `Type "12.34" into "Amount"`) {
t.Error("identifier arm leaked the hint a user reads")
}
}