mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
fix(chrome): name a web field by its hint, not its CSS class
visibleLabel reads hintText first for an editable element. The dump never emitted it, so an empty web input fell through text, description and descendant text to its class name, and the model was shown an identifier no user can read on exactly the fields a labelling experiment varies. Same ladder as fieldHint in web-runtime.ts, so one field is named one way on both hosts. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
ebed84afc3
commit
9b9c7cf6e2
4 files changed
+103
-2
No files matched your search
@@ -408,6 +408,23 @@ func (d *Driver) Hierarchy(ctx context.Context) (string, error) {
|
|||||||
if (tag === 'input') return !NON_TEXT_INPUT_TYPES.includes((el.type || '').toLowerCase());
|
if (tag === 'input') return !NON_TEXT_INPUT_TYPES.includes((el.type || '').toLowerCase());
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
// An editable field's own text is the transient typed value; its hint names
|
||||||
|
// its purpose, which is the rung visibleLabel (internal/verifier/llm.go) reads
|
||||||
|
// first for such an element. Without it a web field reached the model named by
|
||||||
|
// its CSS class, an identifier no user can read. Same ladder as fieldHint in
|
||||||
|
// pkg/spec/src/web-runtime.ts, so one field is named one way on both hosts.
|
||||||
|
function fieldHint(el) {
|
||||||
|
if (!isEditableElement(el)) return '';
|
||||||
|
const ariaLabel = el.getAttribute('aria-label');
|
||||||
|
if (ariaLabel) return ariaLabel;
|
||||||
|
for (const label of el.labels || []) {
|
||||||
|
const text = (label.textContent || '').trim();
|
||||||
|
if (text) return text;
|
||||||
|
}
|
||||||
|
const placeholder = el.getAttribute('placeholder');
|
||||||
|
if (placeholder) return placeholder;
|
||||||
|
return el.getAttribute('name') || '';
|
||||||
|
}
|
||||||
// Shadow roots are part of the page a user sees, so they are part of the page
|
// Shadow roots are part of the page a user sees, so they are part of the page
|
||||||
// we enumerate. Compose for Web mounts its canvas AND its accessibility tree
|
// we enumerate. Compose for Web mounts its canvas AND its accessibility tree
|
||||||
// inside a shadow root on the mount element, so a light-DOM-only walk reports
|
// inside a shadow root on the mount element, so a light-DOM-only walk reports
|
||||||
@@ -446,6 +463,8 @@ func (d *Driver) Hierarchy(ctx context.Context) (string, error) {
|
|||||||
if (el.className && typeof el.className === 'string' && el.className.trim()) {
|
if (el.className && typeof el.className === 'string' && el.className.trim()) {
|
||||||
attrs['class'] = el.className.trim();
|
attrs['class'] = el.className.trim();
|
||||||
}
|
}
|
||||||
|
const hint = fieldHint(el);
|
||||||
|
if (hint) attrs['hintText'] = hint;
|
||||||
// The goja host reads scrollable off this attribute (internal/verifier
|
// The goja host reads scrollable off this attribute (internal/verifier
|
||||||
// worker.go targets). Without it every web element looks unscrollable there,
|
// worker.go targets). Without it every web element looks unscrollable there,
|
||||||
// so the goja-side enumeration offers no scroll while the V8 picker, which
|
// so the goja-side enumeration offers no scroll while the V8 picker, which
|
||||||
|
|||||||
@@ -306,6 +306,79 @@ func TestHierarchy_ScrollableAttribute(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestHierarchy_HintTextNamesAnEditableField covers the attribute visibleLabel
|
||||||
|
// (internal/verifier/llm.go) reads FIRST for an editable element. Without it a
|
||||||
|
// web field reached the model named by its CSS class, an identifier no user can
|
||||||
|
// read, on exactly the channel the label-source experiment varies. The ladder is
|
||||||
|
// fieldHint's in pkg/spec/src/web-runtime.ts, rung for rung.
|
||||||
|
func TestHierarchy_HintTextNamesAnEditableField(t *testing.T) {
|
||||||
|
const html = `<body>` +
|
||||||
|
`<label id="amount-label" for="amount">Amount</label>` +
|
||||||
|
`<input id="amount" placeholder="0.00" name="amount-field">` +
|
||||||
|
`<input id="search" aria-label="Search" placeholder="Type here" name="q">` +
|
||||||
|
`<label id="note-label" for="note"> </label>` +
|
||||||
|
`<input id="note" placeholder="What's this for?" name="note-field">` +
|
||||||
|
`<input id="reference" name="reference-field">` +
|
||||||
|
`<input id="unnamed">` +
|
||||||
|
`<input id="agree" type="checkbox" placeholder="ignored">` +
|
||||||
|
`<button id="go" placeholder="ignored">go</button>` +
|
||||||
|
`</body>`
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "text/html")
|
||||||
|
_, _ = w.Write([]byte(html))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
d := New()
|
||||||
|
defer d.Terminate(context.Background())
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := d.Launch(ctx, server.URL, false, nil); err != nil {
|
||||||
|
t.Fatalf("Launch: %v", err)
|
||||||
|
}
|
||||||
|
dump, err := d.Hierarchy(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Hierarchy: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
type node struct {
|
||||||
|
Attributes map[string]string `json:"attributes"`
|
||||||
|
Children []node `json:"children"`
|
||||||
|
}
|
||||||
|
var root node
|
||||||
|
if err := json.Unmarshal([]byte(dump), &root); err != nil {
|
||||||
|
t.Fatalf("unmarshal hierarchy: %v", err)
|
||||||
|
}
|
||||||
|
hintByID := map[string]string{}
|
||||||
|
var walk func(n node)
|
||||||
|
walk = func(n node) {
|
||||||
|
if id := n.Attributes["resource-id"]; id != "" {
|
||||||
|
hintByID[id] = n.Attributes["hintText"]
|
||||||
|
}
|
||||||
|
for _, c := range n.Children {
|
||||||
|
walk(c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walk(root)
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
id string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"search", "Search"},
|
||||||
|
{"amount", "Amount"},
|
||||||
|
{"note", "What's this for?"},
|
||||||
|
{"reference", "reference-field"},
|
||||||
|
{"unnamed", ""},
|
||||||
|
{"agree", ""},
|
||||||
|
{"go", ""},
|
||||||
|
} {
|
||||||
|
if hintByID[tc.id] != tc.want {
|
||||||
|
t.Errorf("%q: hintText = %q, want %q", tc.id, hintByID[tc.id], tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestRunCtx_CallerCancelPropagates confirms that cancelling the caller's
|
// TestRunCtx_CallerCancelPropagates confirms that cancelling the caller's
|
||||||
// context cancels the chromedp-bound context returned by runCtx. This is the
|
// context cancels the chromedp-bound context returned by runCtx. This is the
|
||||||
// channel by which step deadlines and Ctrl-C reach in-flight CDP calls.
|
// channel by which step deadlines and Ctrl-C reach in-flight CDP calls.
|
||||||
|
|||||||
+1
-1
@@ -25,7 +25,7 @@
|
|||||||
<div id="shadow-overlay">
|
<div id="shadow-overlay">
|
||||||
<button id="shadow-save">save</button>
|
<button id="shadow-save">save</button>
|
||||||
<button id="shadow-cancel" disabled>cancel</button>
|
<button id="shadow-cancel" disabled>cancel</button>
|
||||||
<input id="shadow-amount" type="text" value="10" />
|
<input id="shadow-amount" type="text" value="10" placeholder="0.00" />
|
||||||
<div id="shadow-plain">plain</div>
|
<div id="shadow-plain">plain</div>
|
||||||
</div>
|
</div>
|
||||||
<div id="shadow-scroller"><div id="shadow-scroller-content"></div></div>`;
|
<div id="shadow-scroller"><div id="shadow-scroller-content"></div></div>`;
|
||||||
|
|||||||
+10
-1
@@ -52,7 +52,16 @@
|
|||||||
<option id="choice-first">first</option>
|
<option id="choice-first">first</option>
|
||||||
</select>
|
</select>
|
||||||
<input id="amount" type="text" value="10" />
|
<input id="amount" type="text" value="10" />
|
||||||
<input id="agree" type="checkbox" />
|
<!-- One field per rung of the hint ladder both producers name an editable
|
||||||
|
field by. A rung read on one side only sends the model a different name
|
||||||
|
for the same field on the two hosts, which is the observation channel
|
||||||
|
the label-source arms vary. -->
|
||||||
|
<label id="hint-label-text" for="hint-label">Amount</label>
|
||||||
|
<input id="hint-label" type="text" placeholder="0.00" name="amount-field" />
|
||||||
|
<input id="hint-aria" type="text" aria-label="Search" placeholder="Type here" name="q" />
|
||||||
|
<input id="hint-placeholder" type="text" placeholder="What's this for?" name="note-field" />
|
||||||
|
<input id="hint-name" type="text" name="reference-field" />
|
||||||
|
<input id="agree" type="checkbox" placeholder="not a hint" />
|
||||||
<textarea id="notes"></textarea>
|
<textarea id="notes"></textarea>
|
||||||
<div id="bio" contenteditable="true">bio</div>
|
<div id="bio" contenteditable="true">bio</div>
|
||||||
<div id="menu" role="button">menu</div>
|
<div id="menu" role="button">menu</div>
|
||||||
|
|||||||
Reference in new issue
Block a user