diff --git a/internal/driver/chrome/fact_parity_test.go b/internal/driver/chrome/fact_parity_test.go new file mode 100644 index 0000000..c680e97 --- /dev/null +++ b/internal/driver/chrome/fact_parity_test.go @@ -0,0 +1,343 @@ +//go:build browser + +package chrome + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "runtime" + "slices" + "testing" + "time" + + "github.com/chromedp/chromedp" + + "github.com/priyanshujain/sanderling/internal/bundler" + "github.com/priyanshujain/sanderling/internal/hierarchy" +) + +// One page, two fact producers. +// +// Sanderling enumerates candidate actions from two independent hosts. The goja +// host reads the hierarchy dump this driver builds; the V8 host reads the live +// DOM through collectTargets in pkg/spec/src/web-runtime.ts. Both feed the SAME +// eligibility rule (pkg/spec/src/targets.ts), so the two enumerations agree only +// as far as the two producers agree on the facts. +// +// internal/verifier/host_parity_test.go and pkg/spec/test/host-parity.test.ts +// pin the other half: given identical facts, both hosts select identical +// candidates. They hand-author those facts on both sides, so neither producer is +// on their path, and three producer bugs lived in that gap: the dump emitted no +// `scrollable` attribute at all, it derived `clickable` from el.onclick (which +// React assigns to its root container, making the whole viewport a tap target +// here and nowhere else), and it rooted at document.body while the web runtime +// walks the whole document, hiding `html` and with it every page-level scroll. +// +// This test starts from one real DOM in a real browser and compares what the two +// producers derive from it, element by element. It found a fourth: the dump left +// `editable` unset rather than false, and an unset field sends the parser into +// the native EditText class-name heuristic, which reads a CSS class as an +// Android text widget. + +// elementFacts is one element as a producer reports it: the tag, for readable +// failures, and every fact acceptsTarget consults. +type elementFacts struct { + tag string + clickable bool + enabled bool + editable bool + scrollable bool + positiveBounds bool +} + +// factRow pairs an element's facts with the id both producers key on, kept in +// enumeration order because the order is part of the picker's parity contract. +type factRow struct { + id string + facts elementFacts +} + +func TestHierarchy_DerivesTheSameFactsAsTheWebRuntime(t *testing.T) { + server := httptest.NewServer(http.FileServer(http.Dir("testdata"))) + defer server.Close() + + d := New() + defer d.Terminate(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + if err := d.Launch(ctx, server.URL+"/fact-parity.html", false, nil); err != nil { + t.Fatalf("Launch: %v", err) + } + + dump, err := d.Hierarchy(ctx) + if err != nil { + t.Fatalf("Hierarchy: %v", err) + } + fromDump := factsFromHierarchyDump(t, dump) + fromWebRuntime := factsFromWebRuntime(ctx, t, d) + + requireEveryElementNamed(t, "the hierarchy dump", fromDump) + requireEveryElementNamed(t, "the web runtime", fromWebRuntime) + requireBothPolarities(t, fromWebRuntime) + compareEnumeratedElements(t, fromDump, fromWebRuntime) + compareDerivedFacts(t, fromDump, fromWebRuntime) +} + +// factsFromHierarchyDump reads the dump the way the goja host does: parse it, +// then take exactly the fields internal/verifier/worker.go targets() reads off +// each element. +func factsFromHierarchyDump(t *testing.T, dump string) []factRow { + t.Helper() + tree, err := hierarchy.Parse(dump) + if err != nil { + t.Fatalf("parse hierarchy: %v", err) + } + rows := make([]factRow, 0, len(tree.Elements)) + for _, element := range tree.Elements { + rows = append(rows, factRow{ + id: element.ResourceID, + facts: elementFacts{ + tag: element.Attributes["tag"], + clickable: element.Clickable, + enabled: element.Enabled, + editable: element.Editable, + scrollable: element.Attributes["scrollable"] == "true", + positiveBounds: hasPositiveBounds( + element.Bounds.Width(), + element.Bounds.Height(), + ), + }, + }) + } + return rows +} + +// factsFromWebRuntime installs pkg/spec/test/dom-facts-probe.ts into the page +// and calls it. The probe is bundled through the production web bundler and +// reports what the production collectTargets derives, so the facts come from the +// shipped runtime rather than a copy of it; only the id-carrying readback is +// test-only. +func factsFromWebRuntime( + ctx context.Context, + t *testing.T, + d *Driver, +) []factRow { + t.Helper() + specSource := filepath.Join(repoRootDir(t), "pkg", "spec") + probe, err := bundler.BundleWeb(bundler.WebOptions{ + EntryFile: filepath.Join(specSource, "test", "dom-facts-probe.ts"), + WebRuntimeFile: filepath.Join(specSource, "src", "web-runtime.ts"), + }) + if err != nil { + t.Fatalf("bundle dom facts probe: %v", err) + } + if err := d.InstallBundle(ctx, probe.JavaScript); err != nil { + t.Fatalf("install dom facts probe: %v", err) + } + + var encoded string + script := `JSON.stringify(window.__sanderlingDomFacts__())` + if err := chromedp.Run(d.tabCtx, chromedp.Evaluate(script, &encoded)); err != nil { + t.Fatalf("read web runtime facts: %v", err) + } + var wire []struct { + ID string `json:"id"` + Tag string `json:"tag"` + Clickable bool `json:"clickable"` + Enabled bool `json:"enabled"` + Editable bool `json:"editable"` + Scrollable bool `json:"scrollable"` + Width int `json:"width"` + Height int `json:"height"` + } + if err := json.Unmarshal([]byte(encoded), &wire); err != nil { + t.Fatalf("decode web runtime facts: %v", err) + } + rows := make([]factRow, 0, len(wire)) + for _, item := range wire { + rows = append(rows, factRow{ + id: item.ID, + facts: elementFacts{ + tag: item.Tag, + clickable: item.Clickable, + enabled: item.Enabled, + editable: item.Editable, + scrollable: item.Scrollable, + positiveBounds: hasPositiveBounds(item.Width, item.Height), + }, + }) + } + return rows +} + +// hasPositiveBounds is the positiveBounds fact of pkg/spec/src/targets.ts, +// applied to both producers so the geometry comparison cannot drift from the +// rule it stands in for. +func hasPositiveBounds(width, height int) bool { + return width > 0 && height > 0 +} + +// requireEveryElementNamed fails when a producer enumerates an element the +// fixture gave no id, because such an element cannot be joined and would sit in +// the comparison invisibly. +func requireEveryElementNamed(t *testing.T, producer string, rows []factRow) { + t.Helper() + for index, row := range rows { + if row.id == "" { + t.Fatalf( + "%s enumerated an unnamed <%s> at position %d; every element in "+ + "testdata/fact-parity.html must carry an id to be comparable", + producer, + row.facts.tag, + index, + ) + } + } +} + +// requireBothPolarities keeps the comparison from going vacuous. A fixture, or +// an emulated viewport, that leaves a fact constant would compare false against +// false on every element and pass while proving nothing about that fact. +func requireBothPolarities(t *testing.T, rows []factRow) { + t.Helper() + for _, fact := range []struct { + name string + read func(elementFacts) bool + }{ + {"clickable", func(f elementFacts) bool { return f.clickable }}, + {"enabled", func(f elementFacts) bool { return f.enabled }}, + {"editable", func(f elementFacts) bool { return f.editable }}, + {"scrollable", func(f elementFacts) bool { return f.scrollable }}, + {"positiveBounds", func(f elementFacts) bool { return f.positiveBounds }}, + } { + var sawTrue, sawFalse bool + for _, row := range rows { + if fact.read(row.facts) { + sawTrue = true + } else { + sawFalse = true + } + } + if !sawTrue || !sawFalse { + t.Errorf( + "testdata/fact-parity.html no longer exercises %s both ways (the web "+ + "runtime saw true=%v, false=%v), so comparing it proves nothing", + fact.name, + sawTrue, + sawFalse, + ) + } + } +} + +// compareEnumeratedElements is the check that the two producers walk the same +// document. It is what notices a producer that roots at body and never sees +// `html`, or one that enumerates the head subtree the other drops. +func compareEnumeratedElements( + t *testing.T, + fromDump, fromWebRuntime []factRow, +) { + t.Helper() + dumpIDs := enumeratedIDs(fromDump) + webIDs := enumeratedIDs(fromWebRuntime) + for _, row := range fromWebRuntime { + if !slices.Contains(dumpIDs, row.id) { + t.Errorf( + "the hierarchy dump never enumerated %q (<%s>); the web runtime does, "+ + "so the goja host cannot offer a single action on it", + row.id, + row.facts.tag, + ) + } + } + for _, row := range fromDump { + if !slices.Contains(webIDs, row.id) { + t.Errorf( + "the web runtime never enumerated %q (<%s>); the hierarchy dump does, "+ + "so the V8 host cannot offer a single action on it", + row.id, + row.facts.tag, + ) + } + } + if len(dumpIDs) == len(webIDs) && !slices.Equal(dumpIDs, webIDs) { + t.Errorf( + "the two producers enumerate the same elements in different order\n"+ + " dump=%v\n web=%v", + dumpIDs, + webIDs, + ) + } +} + +// compareDerivedFacts compares the facts fact by fact, over every element both +// producers saw, so a failure names the element and the fact that diverged. +func compareDerivedFacts(t *testing.T, fromDump, fromWebRuntime []factRow) { + t.Helper() + webByID := map[string]elementFacts{} + for _, row := range fromWebRuntime { + webByID[row.id] = row.facts + } + for _, row := range fromDump { + web, ok := webByID[row.id] + if !ok { + continue + } + dump := row.facts + if dump.tag != web.tag { + t.Errorf( + "%q: the hierarchy dump calls it <%s>, the web runtime <%s>; the id "+ + "join is resolving to different elements", + row.id, + dump.tag, + web.tag, + ) + } + for _, fact := range []struct { + name string + dump bool + web bool + }{ + {"clickable", dump.clickable, web.clickable}, + {"enabled", dump.enabled, web.enabled}, + {"editable", dump.editable, web.editable}, + {"scrollable", dump.scrollable, web.scrollable}, + {"positiveBounds", dump.positiveBounds, web.positiveBounds}, + } { + if fact.dump != fact.web { + t.Errorf( + "%q (<%s>): the hierarchy dump derives %s=%v, the web runtime "+ + "derives %s=%v", + row.id, + dump.tag, + fact.name, + fact.dump, + fact.name, + fact.web, + ) + } + } + } +} + +func enumeratedIDs(rows []factRow) []string { + ids := make([]string, 0, len(rows)) + for _, row := range rows { + ids = append(ids, row.id) + } + return ids +} + +func repoRootDir(t *testing.T) string { + t.Helper() + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("cannot resolve caller path") + } + return filepath.Clean( + filepath.Join(filepath.Dir(thisFile), "..", "..", ".."), + ) +} diff --git a/internal/driver/chrome/testdata/fact-parity.html b/internal/driver/chrome/testdata/fact-parity.html new file mode 100644 index 0000000..dba230a --- /dev/null +++ b/internal/driver/chrome/testdata/fact-parity.html @@ -0,0 +1,72 @@ + + + + + fact parity + + + + + + detail + + + + +
bio
+ +
attribute click
+
delegating root
+
plain
+
styled
+ collapsed +
+
+
+
+ + + diff --git a/pkg/spec/test/dom-facts-probe.ts b/pkg/spec/test/dom-facts-probe.ts new file mode 100644 index 0000000..7a3e01b --- /dev/null +++ b/pkg/spec/test/dom-facts-probe.ts @@ -0,0 +1,41 @@ +/// + +// The V8 host's fact producer, read back element by element. +// +// internal/driver/chrome/fact_parity_test.go bundles this probe into a live page +// and compares what it returns against the hierarchy dump the goja host reads +// off the SAME page. Both sides derive their facts here, in production code: +// collectTargets is the shipped walk, and targetElements is the shipped +// enumeration domain, so the id at index i names the element facts[i] describes. +// The dump carries that id as `resource-id`, which is what the Go side joins on. + +import { __testing__ } from "../src/web-runtime.ts"; + +const { collectTargets, targetElements } = __testing__; + +function domFacts(): unknown[] { + const elements = targetElements(); + const facts = collectTargets(); + if (elements.length !== facts.length) { + throw new Error( + `collectTargets reported ${facts.length} targets over ${elements.length} elements`, + ); + } + return elements.map((element, index) => { + const target = facts[index]!; + return { + id: element.id, + tag: element.tagName.toLowerCase(), + clickable: target.clickable, + enabled: target.enabled, + editable: target.editable, + scrollable: target.scrollable, + width: target.width ?? 0, + height: target.height ?? 0, + }; + }); +} + +type FactsGlobal = { __sanderlingDomFacts__: () => unknown[] }; + +(globalThis as unknown as FactsGlobal).__sanderlingDomFacts__ = domFacts;