From 9b9c7cf6e2126894d4d9a704d09d9bd889e401bf Mon Sep 17 00:00:00 2001 From: PJ Date: Sat, 15 Aug 2026 13:45:30 +0530 Subject: [PATCH] fix(chrome): name a web field by its hint, not its CSS class visibleLabel reads hintText first for an editable element. The dump never emitted it, so an empty web input fell through text, description and descendant text to its class name, and the model was shown an identifier no user can read on exactly the fields a labelling experiment varies. Same ladder as fieldHint in web-runtime.ts, so one field is named one way on both hosts. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX --- internal/driver/chrome/driver.go | 19 +++++ internal/driver/chrome/driver_test.go | 73 +++++++++++++++++++ .../chrome/testdata/fact-parity-shadow.html | 2 +- .../driver/chrome/testdata/fact-parity.html | 11 ++- 4 files changed, 103 insertions(+), 2 deletions(-) diff --git a/internal/driver/chrome/driver.go b/internal/driver/chrome/driver.go index fac3554..a220f03 100644 --- a/internal/driver/chrome/driver.go +++ b/internal/driver/chrome/driver.go @@ -408,6 +408,23 @@ func (d *Driver) Hierarchy(ctx context.Context) (string, error) { if (tag === 'input') return !NON_TEXT_INPUT_TYPES.includes((el.type || '').toLowerCase()); return false; } + // An editable field's own text is the transient typed value; its hint names + // its purpose, which is the rung visibleLabel (internal/verifier/llm.go) reads + // first for such an element. Without it a web field reached the model named by + // its CSS class, an identifier no user can read. Same ladder as fieldHint in + // pkg/spec/src/web-runtime.ts, so one field is named one way on both hosts. + function fieldHint(el) { + if (!isEditableElement(el)) return ''; + const ariaLabel = el.getAttribute('aria-label'); + if (ariaLabel) return ariaLabel; + for (const label of el.labels || []) { + const text = (label.textContent || '').trim(); + if (text) return text; + } + const placeholder = el.getAttribute('placeholder'); + if (placeholder) return placeholder; + return el.getAttribute('name') || ''; + } // Shadow roots are part of the page a user sees, so they are part of the page // we enumerate. Compose for Web mounts its canvas AND its accessibility tree // inside a shadow root on the mount element, so a light-DOM-only walk reports @@ -446,6 +463,8 @@ func (d *Driver) Hierarchy(ctx context.Context) (string, error) { if (el.className && typeof el.className === 'string' && el.className.trim()) { attrs['class'] = el.className.trim(); } + const hint = fieldHint(el); + if (hint) attrs['hintText'] = hint; // The goja host reads scrollable off this attribute (internal/verifier // worker.go targets). Without it every web element looks unscrollable there, // so the goja-side enumeration offers no scroll while the V8 picker, which diff --git a/internal/driver/chrome/driver_test.go b/internal/driver/chrome/driver_test.go index 36ef3fb..f63c6e6 100644 --- a/internal/driver/chrome/driver_test.go +++ b/internal/driver/chrome/driver_test.go @@ -306,6 +306,79 @@ func TestHierarchy_ScrollableAttribute(t *testing.T) { } } +// TestHierarchy_HintTextNamesAnEditableField covers the attribute visibleLabel +// (internal/verifier/llm.go) reads FIRST for an editable element. Without it a +// web field reached the model named by its CSS class, an identifier no user can +// read, on exactly the channel the label-source experiment varies. The ladder is +// fieldHint's in pkg/spec/src/web-runtime.ts, rung for rung. +func TestHierarchy_HintTextNamesAnEditableField(t *testing.T) { + const html = `` + + `` + + `` + + `` + + `` + + `` + + `` + + `` + + `` + + `` + + `` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(html)) + })) + defer server.Close() + + d := New() + defer d.Terminate(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := d.Launch(ctx, server.URL, false, nil); err != nil { + t.Fatalf("Launch: %v", err) + } + dump, err := d.Hierarchy(ctx) + if err != nil { + t.Fatalf("Hierarchy: %v", err) + } + + type node struct { + Attributes map[string]string `json:"attributes"` + Children []node `json:"children"` + } + var root node + if err := json.Unmarshal([]byte(dump), &root); err != nil { + t.Fatalf("unmarshal hierarchy: %v", err) + } + hintByID := map[string]string{} + var walk func(n node) + walk = func(n node) { + if id := n.Attributes["resource-id"]; id != "" { + hintByID[id] = n.Attributes["hintText"] + } + for _, c := range n.Children { + walk(c) + } + } + walk(root) + + for _, tc := range []struct { + id string + want string + }{ + {"search", "Search"}, + {"amount", "Amount"}, + {"note", "What's this for?"}, + {"reference", "reference-field"}, + {"unnamed", ""}, + {"agree", ""}, + {"go", ""}, + } { + if hintByID[tc.id] != tc.want { + t.Errorf("%q: hintText = %q, want %q", tc.id, hintByID[tc.id], tc.want) + } + } +} + // TestRunCtx_CallerCancelPropagates confirms that cancelling the caller's // context cancels the chromedp-bound context returned by runCtx. This is the // channel by which step deadlines and Ctrl-C reach in-flight CDP calls. diff --git a/internal/driver/chrome/testdata/fact-parity-shadow.html b/internal/driver/chrome/testdata/fact-parity-shadow.html index 9529c50..f7405e7 100644 --- a/internal/driver/chrome/testdata/fact-parity-shadow.html +++ b/internal/driver/chrome/testdata/fact-parity-shadow.html @@ -25,7 +25,7 @@
- +
plain
`; diff --git a/internal/driver/chrome/testdata/fact-parity.html b/internal/driver/chrome/testdata/fact-parity.html index c378ec9..75b2d1d 100644 --- a/internal/driver/chrome/testdata/fact-parity.html +++ b/internal/driver/chrome/testdata/fact-parity.html @@ -52,7 +52,16 @@ - + + + + + + +
bio