Files
sanderling/internal/driver/chrome/driver.go
T
pj c8ca350115 fix(chrome): state every boolean flag the dump can state
internal/hierarchy writes the attribute a selector matches on only where
the producer stated the flag, so a state emitted as null is one no
selector can ask about: {clickable: false} and {enabled: false} matched
nothing at all against a web dump while matching on android, which
states every flag both ways. only secure stays three-valued.
2026-08-18 20:43:29 +05:30

1297 lines
50 KiB
Go

// Package chrome implements the device driver for web targets by driving Chrome over the DevTools protocol.
package chrome
import (
"context"
"encoding/json"
"fmt"
"net/url"
"strconv"
"strings"
"sync"
"time"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/input"
"github.com/chromedp/cdproto/network"
"github.com/chromedp/cdproto/page"
"github.com/chromedp/cdproto/runtime"
"github.com/chromedp/cdproto/storage"
"github.com/chromedp/chromedp"
"github.com/chromedp/chromedp/kb"
"github.com/priyanshujain/sanderling/internal/driver"
)
// Driver implements DeviceDriver via chromedp for web platform testing.
type Driver struct {
allocCtx context.Context
allocCancel context.CancelFunc
tabCtx context.Context
tabCancel context.CancelFunc
logsMu sync.Mutex
logs []driver.LogEntry
navigationsMu sync.Mutex
navigations []driver.Navigation
// pickerState is the seeded picker's draw position, held here rather than
// in the page: a navigation replaces the page's runtime, and a runtime that
// starts over restarts the seed's stream at its first draw.
pickerState string
}
// New creates a new ChromeDriver. Call Terminate when done.
func New() *Driver {
allocCtx, allocCancel := chromedp.NewExecAllocator(context.Background(),
append(chromedp.DefaultExecAllocatorOptions[:],
chromedp.Flag("headless", true),
chromedp.Flag("disable-gpu", true),
// Chrome refuses to fall back to the SwiftShader WebGL backend
// without this flag, so with --disable-gpu a canvas app (Compose
// for Web, Flutter web, anything on WebGL) gets a null context and
// paints nothing: black screenshots and an empty accessibility DOM.
chromedp.Flag("enable-unsafe-swiftshader", true),
chromedp.NoSandbox,
// CI runners give Chrome a tiny /dev/shm; without this the browser
// process hangs on startup and never reports its DevTools socket.
chromedp.Flag("disable-dev-shm-usage", true),
// Cold-starting Chrome on a loaded CI runner can take longer than the
// 20s default to print its DevTools websocket URL; give it more room
// so launch does not flake with "websocket url timeout reached".
chromedp.WSURLReadTimeout(60*time.Second),
)...,
)
tabCtx, tabCancel := chromedp.NewContext(allocCtx)
d := &Driver{
allocCtx: allocCtx,
allocCancel: allocCancel,
tabCtx: tabCtx,
tabCancel: tabCancel,
}
chromedp.ListenTarget(tabCtx, func(ev any) {
e, ok := ev.(*runtime.EventConsoleAPICalled)
if !ok {
return
}
var parts []string
for _, arg := range e.Args {
// An object argument, which is what console.error(err) passes,
// carries no value at all: CDP sends a description instead. Reading
// only the value logged those calls with an empty message, so the
// entry named a level and nothing a reader could act on.
if arg.Value == nil {
if arg.Description != "" {
parts = append(parts, arg.Description)
}
continue
}
var s string
if err := json.Unmarshal(arg.Value, &s); err == nil {
parts = append(parts, s)
} else {
parts = append(parts, string(arg.Value))
}
}
d.logsMu.Lock()
d.logs = append(d.logs, driver.LogEntry{
UnixMillis: int64(e.Timestamp.Time().UnixMilli()),
Level: consoleLevel(e.Type),
Tag: "console",
Message: strings.Join(parts, " "),
})
d.logsMu.Unlock()
})
chromedp.ListenTarget(tabCtx, func(ev any) {
e, ok := ev.(*page.EventFrameNavigated)
if !ok || e.Frame == nil || e.Frame.ParentID != "" {
return
}
d.navigationsMu.Lock()
d.navigations = append(d.navigations, driver.Navigation{
URL: e.Frame.URL,
UnixMillis: time.Now().UnixMilli(),
})
d.navigationsMu.Unlock()
})
return d
}
func (d *Driver) Launch(ctx context.Context, bundleID string, clearState bool, _ map[string]string) error {
// Allocate the browser against the driver's own context before anything
// caller-bound runs. chromedp starts Chrome under whichever context first
// calls Run, so allocating under a caller deadline would tie the browser
// process to this one call and kill it the moment Launch returns.
if err := chromedp.Run(d.tabCtx); err != nil {
return err
}
// Everything after allocation goes through runCtx, so a caller deadline or
// a SIGTERM aborts a launch that would otherwise wait forever on a target
// that accepts the connection and never answers.
runCtx, cancel := d.runCtx(ctx)
defer cancel()
if clearState {
if err := d.clearState(runCtx, bundleID); err != nil {
return err
}
}
if err := chromedp.Run(runCtx, chromedp.Navigate(bundleID)); err != nil {
return err
}
// After navigation, read CSS custom properties --frame-w / --frame-h (common
// mobile-frame convention) so screenshots fit the app without grey borders.
// Falls back to the body scroll dimensions if the properties are absent.
var dims [2]int64
if err := chromedp.Run(runCtx, chromedp.Evaluate(`
(function() {
const s = getComputedStyle(document.documentElement);
const pw = parseInt(s.getPropertyValue('--frame-w'), 10);
const ph = parseInt(s.getPropertyValue('--frame-h'), 10);
const w = isNaN(pw) ? document.body.scrollWidth : pw;
const h = isNaN(ph) ? document.body.scrollHeight : ph;
return [w, h];
})()`, &dims)); err == nil && dims[0] > 0 && dims[1] > 0 {
_ = chromedp.Run(runCtx, chromedp.EmulateViewport(dims[0], dims[1]))
}
// The opening navigation is the harness arriving, not the app navigating.
_, _ = d.Navigations(ctx)
return nil
}
// Navigations returns the document-replacing main-frame navigations seen since
// the last call and forgets them. Each one replaced the page's runtime, which
// is what separates "the app reloaded" from "the picker repeated itself".
func (d *Driver) Navigations(context.Context) ([]driver.Navigation, error) {
d.navigationsMu.Lock()
defer d.navigationsMu.Unlock()
drained := d.navigations
d.navigations = nil
return drained, nil
}
// clearState wipes the target's stored data before the application loads.
// Script cannot do it: the tab still sits on about:blank, whose opaque origin
// denies storage access, so `localStorage.clear()` throws SecurityError and
// every web run dies at launch. The Storage domain clears by origin instead,
// which needs no navigation. sessionStorage is per-tab and outside that
// domain's reach; it only survives when a relaunch reuses a tab already on
// the target origin, which is the one case where script can reach it.
func (d *Driver) clearState(runCtx context.Context, bundleID string) error {
if err := chromedp.Run(runCtx, network.ClearBrowserCookies()); err != nil {
return fmt.Errorf("clear cookies: %w", err)
}
origin := securityOrigin(bundleID)
if origin == "" {
return nil
}
clearForOrigin := storage.ClearDataForOrigin(origin, string(storage.TypeAll))
if err := chromedp.Run(runCtx, clearForOrigin); err != nil {
return fmt.Errorf("clear storage for %s: %w", origin, err)
}
script := fmt.Sprintf(
`location.origin === %q && (sessionStorage.clear(), true)`, origin)
return chromedp.Run(runCtx, chromedp.ActionFunc(func(ctx context.Context) error {
_, exception, err := runtime.Evaluate(script).Do(ctx)
if err != nil {
return fmt.Errorf("clear session storage: %w", err)
}
if exception != nil {
return fmt.Errorf("clear session storage: %s", exceptionMessage(exception))
}
return nil
}))
}
// securityOrigin returns the scheme://host[:port] the Storage domain keys data
// by, or "" for a target that has no such origin (data:, file:, about:blank),
// where there is no per-origin storage to clear.
func securityOrigin(bundleID string) string {
parsed, err := url.Parse(bundleID)
if err != nil || parsed.Host == "" {
return ""
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return ""
}
return parsed.Scheme + "://" + parsed.Host
}
// exceptionMessage renders a page exception for an error string. The
// description carries the actual message ("SecurityError: Failed to read the
// 'localStorage' property..."); Text alone is the useless "Uncaught".
func exceptionMessage(exception *runtime.ExceptionDetails) string {
if exception == nil {
return ""
}
if exception.Exception != nil && exception.Exception.Description != "" {
return exception.Exception.Description
}
return exception.Text
}
func (d *Driver) Terminate(_ context.Context) error {
d.tabCancel()
d.allocCancel()
return nil
}
// pointInViewScript scrolls a point the caller took from the hierarchy back
// inside the viewport and reports where to dispatch at, plus whether anything
// is there to receive it.
//
// The emulated viewport is sized once at launch, but getBoundingClientRect goes
// on reporting elements the growing document has pushed below it, so the two
// disagree the moment an app adds content. Input coordinates are
// viewport-relative: a click below the fold is hit-tested to the document root,
// which delivers it to <html> and never to the element the caller named. No
// error is raised on any layer, so the step reads as an action that landed and
// changed nothing.
//
// Only a point outside the viewport is moved, so a gesture that already had a
// reachable target dispatches exactly where it did before.
const pointInViewScript = `
(function(x, y) {
const root = document.scrollingElement || document.documentElement;
let shiftX = 0, shiftY = 0;
if (x < 0 || x >= window.innerWidth) shiftX = Math.round(x - window.innerWidth / 2);
if (y < 0 || y >= window.innerHeight) shiftY = Math.round(y - window.innerHeight / 2);
if (shiftX || shiftY) {
const fromX = root.scrollLeft, fromY = root.scrollTop;
root.scrollLeft = fromX + shiftX;
root.scrollTop = fromY + shiftY;
shiftX = root.scrollLeft - fromX;
shiftY = root.scrollTop - fromY;
}
const atX = x - shiftX, atY = y - shiftY;
return [atX, atY, document.elementFromPoint(atX, atY) ? 1 : 0];
})(%d, %d)`
// pointInView returns the point to dispatch a gesture at for the point the
// caller named, having scrolled it into view. It fails with
// driver.ErrGestureUndelivered when no scroll can put an element under it.
func pointInView(runCtx context.Context, x, y int) (int, int, error) {
var point [3]int
script := fmt.Sprintf(pointInViewScript, x, y)
if err := chromedp.Run(runCtx, chromedp.Evaluate(script, &point)); err != nil {
return 0, 0, err
}
if point[2] == 0 {
return 0, 0, fmt.Errorf(
"%w: (%d,%d)",
driver.ErrGestureUndelivered,
x,
y,
)
}
return point[0], point[1], nil
}
func (d *Driver) Tap(ctx context.Context, x, y int) error {
runCtx, cancel := d.runCtx(ctx)
defer cancel()
atX, atY, err := pointInView(runCtx, x, y)
if err != nil {
return err
}
return chromedp.Run(runCtx,
chromedp.MouseClickXY(float64(atX), float64(atY)),
)
}
// requireSelectorMatch reports driver.ErrSelectorMatchedNothing when the
// selector names no node on the page right now. chromedp.Click waits instead,
// so without this the caller hears a deadline (or nothing at all) for an action
// that had no target.
func requireSelectorMatch(runCtx context.Context, target, selector string) error {
var nodes []*cdp.Node
if err := chromedp.Run(runCtx,
chromedp.Nodes(target, &nodes, chromedp.BySearch, chromedp.AtLeast(0)),
); err != nil {
return err
}
if len(nodes) == 0 {
return fmt.Errorf("%w: %q", driver.ErrSelectorMatchedNothing, selector)
}
return nil
}
func (d *Driver) TapSelector(ctx context.Context, selector string) error {
runCtx, cancel := d.runCtx(ctx)
defer cancel()
target, isXPath, err := TranslateStringSelector(selector)
if err != nil {
// Fall back to passing the string straight through; chromedp will
// reject it loudly if it isn't a valid CSS selector.
target = selector
}
if err := requireSelectorMatch(runCtx, target, selector); err != nil {
return err
}
if isXPath {
return chromedp.Run(runCtx, chromedp.Click(target, chromedp.NodeVisible, chromedp.BySearch))
}
return chromedp.Run(runCtx, chromedp.Click(target, chromedp.NodeVisible))
}
// doubleTapGap is the inter-tap delay for DoubleTap: short enough to land both
// events inside a sub-100 ms race window. The browser has no single double-tap
// primitive, so the gesture is two taps with this gap.
const doubleTapGap = 50 * time.Millisecond
// DoubleTap resolves the point once and dispatches both taps there: resolving
// per tap would scroll the second one away from the element the first hit.
func (d *Driver) DoubleTap(ctx context.Context, x, y int) error {
runCtx, cancel := d.runCtx(ctx)
defer cancel()
atX, atY, err := pointInView(runCtx, x, y)
if err != nil {
return err
}
return webDoubleTap(ctx, func(clickCount int) error {
return chromedp.Run(
runCtx,
chromedp.MouseClickXY(
float64(atX),
float64(atY),
chromedp.ClickCount(clickCount),
),
)
})
}
func (d *Driver) DoubleTapSelector(ctx context.Context, selector string) error {
runCtx, cancel := d.runCtx(ctx)
defer cancel()
target, isXPath, err := TranslateStringSelector(selector)
if err != nil {
target = selector
}
if err := requireSelectorMatch(runCtx, target, selector); err != nil {
return err
}
options := []chromedp.QueryOption{chromedp.NodeVisible}
if isXPath {
options = append(options, chromedp.BySearch)
}
return webDoubleTap(ctx, func(clickCount int) error {
if clickCount < 2 {
return chromedp.Run(runCtx, chromedp.Click(target, options...))
}
return chromedp.Run(runCtx, chromedp.DoubleClick(target, options...))
})
}
// webDoubleTap dispatches the pair a browser reads as one double click. Blink
// raises dblclick off the click count the second event carries, so two taps
// that both say "first click" arrive at a dblclick handler as two ordinary
// clicks and the gesture never happens at all.
func webDoubleTap(ctx context.Context, tap func(clickCount int) error) error {
if err := tap(1); err != nil {
return err
}
timer := time.NewTimer(doubleTapGap)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
}
return tap(2)
}
func (d *Driver) InputText(callerCtx context.Context, text string) error {
runCtx, cancel := d.runCtx(callerCtx)
defer cancel()
return chromedp.Run(runCtx,
chromedp.ActionFunc(func(ctx context.Context) error {
if err := selectFocusedText(ctx); err != nil {
return err
}
return input.InsertText(text).Do(ctx)
}),
)
}
// selectAllScript selects everything in the focused field so the InsertText
// that follows replaces rather than appends.
//
// document.activeElement stops at a shadow boundary: it names the HOST, not the
// focused node inside. Compose for Web focuses a hidden <input> inside the
// shadow root it mounts, so the host answer has no select() and the selection
// never happened - every InputText appended to the last one, and a fuzzer that
// types into the same field twice built up garbage it could never clear.
// Descending activeElement through each shadow root finds the real field.
const selectAllScript = `
(function() {
let el = document.activeElement;
while (el && el.shadowRoot && el.shadowRoot.activeElement) {
el = el.shadowRoot.activeElement;
}
if (el && typeof el.select === 'function') el.select();
})()`
func selectFocusedText(ctx context.Context) error {
return chromedp.Evaluate(selectAllScript, nil).Do(ctx)
}
// ReplacesTextOnInput reports that InputText replaces existing content via
// select-all, so the runner skips its pre-erase.
func (d *Driver) ReplacesTextOnInput() bool {
return true
}
// EraseText clears the focused field. InputText above already replaces via
// select-all, so the character count is not needed to bound the deletion.
func (d *Driver) EraseText(callerCtx context.Context, _ int) error {
runCtx, cancel := d.runCtx(callerCtx)
defer cancel()
return chromedp.Run(runCtx,
chromedp.ActionFunc(func(ctx context.Context) error {
if err := selectFocusedText(ctx); err != nil {
return err
}
return input.InsertText("").Do(ctx)
}),
)
}
// Swipe drags a finger across the page as a trusted touch stream. Events
// synthesized in the page carry isTrusted false: they reach a handler that
// happens to listen, but never enter the input pipeline that scrolls, honours
// touch-action or resolves a gesture.
func (d *Driver) Swipe(ctx context.Context, fromX, fromY, toX, toY int, duration time.Duration) error {
runCtx, cancel := d.runCtx(ctx)
defer cancel()
atX, atY, err := pointInView(runCtx, fromX, fromY)
if err != nil {
return err
}
toX, toY = toX-(fromX-atX), toY-(fromY-atY)
steps := max(int(duration.Milliseconds())/16, 1)
actions := []chromedp.Action{touchAt(input.TouchStart, atX, atY)}
for i := 1; i <= steps; i++ {
actions = append(actions, touchAt(input.TouchMove,
atX+(toX-atX)*i/steps, atY+(toY-atY)*i/steps))
}
actions = append(
actions,
input.DispatchTouchEvent(input.TouchEnd, []*input.TouchPoint{}),
)
return chromedp.Run(runCtx, actions...)
}
func touchAt(kind input.TouchType, x, y int) *input.DispatchTouchEventParams {
return input.DispatchTouchEvent(
kind,
[]*input.TouchPoint{{X: float64(x), Y: float64(y)}},
)
}
// Scroll moves the content under the point with a trusted wheel, which is how a
// browser scrolls.
//
// A finger drag scrolls too, but it ends in a fling whose distance follows the
// release velocity: five identical 240 px drags moved the page 354 to 616 px,
// so two runs of one seed would explore different screens. A wheel delta lands
// exactly, and chains from the element under the point out to its scrollable
// ancestors, which is what scrolling a named container means. The drag stays as
// Swipe, the verb for the gestures only a finger reaches.
func (d *Driver) Scroll(
ctx context.Context,
fromX, fromY, toX, toY int,
_ time.Duration,
) error {
runCtx, cancel := d.runCtx(ctx)
defer cancel()
atX, atY, err := pointInView(runCtx, fromX, fromY)
if err != nil {
return err
}
wheel := input.DispatchMouseEvent(input.MouseWheel, float64(atX), float64(atY)).
WithDeltaX(float64(fromX - toX)).
WithDeltaY(float64(fromY - toY))
// The wheel is applied off the CDP round trip, so without this the next
// read races it: a step could observe the page before its own scroll, and
// the pending scroll then lands during the following one.
return chromedp.Run(runCtx, wheel, chromedp.Evaluate(
`new Promise(done => requestAnimationFrame(() => requestAnimationFrame(done)))`,
nil,
awaitPromise,
))
}
func (d *Driver) PressKey(ctx context.Context, key string) error {
k, ok := keyMap[key]
if !ok {
return fmt.Errorf("unsupported key: %q", key)
}
runCtx, cancel := d.runCtx(ctx)
defer cancel()
return chromedp.Run(runCtx, chromedp.KeyEvent(k))
}
func (d *Driver) LongPress(ctx context.Context, x, y int) error {
runCtx, cancel := d.runCtx(ctx)
defer cancel()
x, y, err := pointInView(runCtx, x, y)
if err != nil {
return err
}
script := fmt.Sprintf(`
(function() {
const el = document.elementFromPoint(%d, %d);
if (!el) return;
el.dispatchEvent(new PointerEvent('pointerdown', {clientX: %d, clientY: %d, bubbles: true}));
setTimeout(function() {
el.dispatchEvent(new PointerEvent('pointerup', {clientX: %d, clientY: %d, bubbles: true}));
}, 600);
})();`,
x, y,
x, y,
x, y,
)
return chromedp.Run(runCtx, chromedp.Evaluate(script, nil))
}
// keyMap covers the keys web specs may emit (enter/tab/escape/arrows).
// "back"/"home" are intentionally absent: backspace/NUL have no navigation
// semantics in a browser, and the V8 action mix already excludes them.
var keyMap = map[string]string{
"enter": kb.Enter,
"tab": kb.Tab,
"escape": kb.Escape,
"up": kb.ArrowUp,
"down": kb.ArrowDown,
"left": kb.ArrowLeft,
"right": kb.ArrowRight,
}
func (d *Driver) Hierarchy(ctx context.Context) (string, error) {
runCtx, cancel := d.runCtx(ctx)
defer cancel()
script := `
(function() {
// Hash first (a HashRouter names the screen there), then the pathname, which
// is where a path-routed SPA keeps it. Reporting '/' for every step of a
// BrowserRouter app made every screen look like the same screen.
const route = window.location.hash.replace(/^#/, '').split('?')[0] ||
window.location.pathname || '/';
// clickable and editable are resolved through the SAME selector sets
// pkg/spec/src/web-runtime.ts uses, so the goja host (which reads this dump)
// and the V8 host (which reads the DOM directly) cannot mean different things
// by one fact on one platform. Testing el.onclick instead made every React
// root a full-viewport tap target here and nowhere else.
const NON_TEXT_INPUT_TYPES =
['button','submit','checkbox','radio','range','color','file','image','reset'];
// The disabled property belongs to real form controls only, so it reads
// undefined on the role-based controls the tappable set now covers, and every
// one of them looked enabled however plainly it was marked otherwise.
// isEnabled in pkg/spec/src/web-runtime.ts answers the same two ways.
function isEnabled(el) {
if (el.disabled) return false;
return el.getAttribute('aria-disabled') !== 'true';
}
function isEditableElement(el) {
if (el.isContentEditable) return true;
const tag = el.tagName.toLowerCase();
if (tag === 'textarea') return true;
if (tag === 'input') return !NON_TEXT_INPUT_TYPES.includes((el.type || '').toLowerCase());
return false;
}
// An editable field's own text is the transient typed value; its hint names
// its purpose, which is the rung visibleLabel (internal/verifier/llm.go) reads
// first for such an element. Without it a web field reached the model named by
// its CSS class, an identifier no user can read. Same ladder as fieldHint in
// pkg/spec/src/web-runtime.ts, so one field is named one way on both hosts.
function fieldHint(el) {
if (!isEditableElement(el)) return '';
const ariaLabel = el.getAttribute('aria-label');
if (ariaLabel) return ariaLabel;
for (const label of el.labels || []) {
const text = (label.textContent || '').trim();
if (text) return text;
}
const placeholder = el.getAttribute('placeholder');
if (placeholder) return placeholder;
return el.getAttribute('name') || '';
}
// Shadow roots are part of the page a user sees, so they are part of the page
// we enumerate. Compose for Web mounts its canvas AND its accessibility tree
// inside a shadow root on the mount element, so a light-DOM-only walk reports
// four nodes for a whole app and offers no action on any of them.
function deepQuery(sel) {
const out = [];
const visit = (root) => {
for (const el of root.querySelectorAll(sel)) out.push(el);
for (const el of root.querySelectorAll('*')) if (el.shadowRoot) visit(el.shadowRoot);
};
visit(document);
return out;
}
const TAPPABLE_ROLES = [
'button', 'link', 'checkbox', 'radio', 'switch', 'tab', 'option',
'menuitem', 'menuitemcheckbox', 'menuitemradio', 'treeitem'];
const clickableSet = new Set(deepQuery(
'a, button, input, select, textarea, ' +
TAPPABLE_ROLES.map(role => '[role="' + role + '"]').join(', ') +
', [onclick]'));
const editableSet = new Set(deepQuery(
'input, textarea, [contenteditable]').filter(isEditableElement));
// Descended once for the whole dump, for the reason selectAllScript above
// descends: document.activeElement names the shadow host, so a Compose for
// Web app reported focus on its mount element and never on the field.
let focusedElement = document.activeElement;
while (focusedElement && focusedElement.shadowRoot && focusedElement.shadowRoot.activeElement) {
focusedElement = focusedElement.shadowRoot.activeElement;
}
// Descending is still not enough on Compose for Web: it takes keystrokes on a
// 1px transparent input pinned to the caret, and that input is a SIBLING of
// the accessibility tree rather than a node in it. DOM focus therefore never
// reaches the semantics element carrying the test tag, so confirmFocus in
// internal/runner/runner.go saw an unnamed element hold focus after every
// focus tap and refused to type. Compose declares the caret's box in these
// custom properties, which the input inherits from the container that
// positions it, so the field being typed into is the innermost editable box
// that caret sits in.
const CARET_ORIGIN_PROPERTY = '--compose-internal-web-backing-input-left';
function fieldBehindTheCaret(caretInput) {
if (!caretInput || caretInput.tagName !== 'INPUT') return null;
if (!getComputedStyle(caretInput).getPropertyValue(CARET_ORIGIN_PROPERTY).trim()) return null;
const caret = caretInput.getBoundingClientRect();
const x = (caret.left + caret.right) / 2;
const y = (caret.top + caret.bottom) / 2;
let field = null;
let fieldArea = Infinity;
for (const candidate of editableSet) {
if (candidate === caretInput) continue;
const box = candidate.getBoundingClientRect();
const area = box.width * box.height;
if (area <= 0 || area >= fieldArea) continue;
if (x < box.left || x > box.right || y < box.top || y > box.bottom) continue;
field = candidate;
fieldArea = area;
}
return field;
}
focusedElement = fieldBehindTheCaret(focusedElement) || focusedElement;
function buildTree(el, isRoot) {
const rect = el.getBoundingClientRect();
// Every attribute the markup wrote, keyed as written, which is what attrs
// means on the native hosts and what rawAttributes in
// pkg/spec/src/web-runtime.ts already gives the page-side handle. Emitting
// only the standard set left a spec's data-* reads (folio-web's data-cents,
// data-account-id, data-balance) undefined on the goja host and absent from
// the trace, so an offline replay of the same step could not see them at
// all. The derived keys below overwrite anything of the same name.
const attrs = {};
for (const attribute of el.attributes || []) {
attrs[attribute.name] = attribute.value;
}
const bounds = '[' + Math.round(rect.left) + ',' + Math.round(rect.top) + ',' +
Math.round(rect.right) + ',' + Math.round(rect.bottom) + ']';
if (rect.width > 0 || rect.height > 0) attrs.bounds = bounds;
const text = (el.textContent || '').trim().slice(0, 200);
if (text) attrs.text = text;
if (el.id) attrs['resource-id'] = el.id;
// The V8 host names a target by data-testid (IDENTITY_KEYS in
// pkg/spec/src/web-runtime.ts) and TapSelector translates the selector into
// a CSS attribute match, so a dump without this attribute leaves the goja
// host unable to resolve a target the other two resolve fine.
const testid = el.getAttribute('data-testid');
if (testid) attrs['data-testid'] = testid;
const label = el.getAttribute('aria-label') || el.getAttribute('alt') || el.getAttribute('title') || '';
if (label) attrs['content-desc'] = label;
const tag = (el.tagName || '').toLowerCase();
if (tag) attrs['tag'] = tag;
if (el.className && typeof el.className === 'string' && el.className.trim()) {
attrs['class'] = el.className.trim();
}
const hint = fieldHint(el);
if (hint) attrs['hintText'] = hint;
// The goja host reads scrollable off this attribute (internal/verifier
// worker.go targets). Without it every web element looks unscrollable there,
// so the goja-side enumeration offers no scroll while the V8 picker, which
// computes the same overflow test in web-runtime.ts, offers plenty.
if (el.scrollHeight > el.clientHeight || el.scrollWidth > el.clientWidth) {
attrs['scrollable'] = 'true';
}
if (isRoot) attrs['sanderling-screen'] = route;
const isClickable = clickableSet.has(el);
const isEditable = editableSet.has(el);
const children = [];
// Shadow content first, then light children: the shadow tree is what the
// host actually renders, and targetElements in web-runtime.ts walks the same
// order, which is the order the two enumerations are compared in.
if (el.shadowRoot) {
for (const child of el.shadowRoot.children) {
children.push(buildTree(child, false));
}
}
for (const child of el.children) {
if (child.tagName === 'HEAD') continue;
children.push(buildTree(child, false));
}
return {
attributes: attrs,
children: children,
// Emitted as plain booleans, never null: internal/hierarchy writes the
// attribute a selector matches on only where the producer stated the
// flag, so a state that arrives as null is one no selector can ask about.
// {clickable: false} and {enabled: false} matched nothing at all here
// while matching on android, which states every flag both ways.
clickable: isClickable,
enabled: isEnabled(el),
focused: focusedElement === el,
// A component keeps what it likes in these two properties, so what is
// emitted is the flag the field declares and not the property's value.
checked: el.checked === true,
selected: el.selected === true,
// Emitted as a plain boolean, never null, on every editable field: a
// consumer deciding what a typed value may be recorded as has to tell
// "not a secure entry" apart from "nobody said", and android says nothing.
secure: isEditable ? el.type === 'password' : null,
// Emitted as a plain boolean, never null: internal/hierarchy falls back to
// the native heuristic when the field is absent, which reads any class
// name containing "EditText" as an Android text widget. On web that is a
// CSS class, so a page styling a div with it made the goja host offer
// typing into a div the web runtime never calls editable.
editable: isEditable,
};
}
// Rooted at documentElement, not body, because collectTargets in
// pkg/spec/src/web-runtime.ts walks querySelectorAll("*") and therefore sees
// html. Page-level scrolling lives on html on a standard page, so a dump
// rooted at body hides it from the goja host and the two enumerations
// disagree on exactly the page scroll. The head subtree is skipped: it is all
// zero-bounds, so it changes no eligible set, and it would otherwise pull
// script and title text into the trace and the replay view.
return buildTree(document.documentElement, true);
})()`
var result any
if err := chromedp.Run(runCtx, chromedp.Evaluate(script, &result)); err != nil {
return "", fmt.Errorf("hierarchy: %w", err)
}
bytes, err := json.Marshal(result)
if err != nil {
return "", fmt.Errorf("hierarchy marshal: %w", err)
}
return string(bytes), nil
}
func (d *Driver) Screenshot(ctx context.Context) (driver.Image, error) {
runCtx, cancel := d.runCtx(ctx)
defer cancel()
var buf []byte
if err := chromedp.Run(runCtx, chromedp.CaptureScreenshot(&buf)); err != nil {
return driver.Image{}, fmt.Errorf("screenshot: %w", err)
}
w, h := pngDimensions(buf)
return driver.Image{PNG: buf, Width: w, Height: h}, nil
}
// Snapshot pairs hierarchy and screenshot back-to-back. The chromedp tab
// is single-threaded so the two CDP round-trips are already serialized:
// pairing them here matches the DeviceDriver contract without extra locking.
func (d *Driver) Snapshot(ctx context.Context) (string, driver.Image, error) {
hierarchy, err := d.Hierarchy(ctx)
if err != nil {
return "", driver.Image{}, err
}
image, err := d.Screenshot(ctx)
if err != nil {
return hierarchy, driver.Image{}, err
}
return hierarchy, image, nil
}
func (d *Driver) RecentLogs(_ context.Context, since time.Time, minLevel string) ([]driver.LogEntry, error) {
sinceMillis := since.UnixMilli()
d.logsMu.Lock()
defer d.logsMu.Unlock()
var result []driver.LogEntry
for _, entry := range d.logs {
if entry.UnixMillis < sinceMillis {
continue
}
if minLevel != "" && !meetsLevel(entry.Level, minLevel) {
continue
}
result = append(result, entry)
}
return result, nil
}
// domQuietPeriod is how long the DOM must stop changing before the page counts
// as settled. Compose for Web syncs its accessibility DOM off the frame loop:
// measured at ~136 ms behind an InputText on the folio wasm build, so waiting
// for frames alone (~16 ms each) returns while the app still reports the old
// text, and the next step types into a field it believes is still empty.
const domQuietPeriod = 150 * time.Millisecond
// transitionSettlePeriod is how much longer the settle waits for a route
// transition to finish once the DOM has gone quiet. A canvas app's cross-fade
// is invisible to a mutation observer: Compose splices the incoming screen's
// accessibility nodes in when the animation STARTS and removes the outgoing
// screen's when it ends, and nothing in between touches the DOM, so the tree
// sits byte-identical (and quiet) with both routes live for the whole
// animation. Settling on quiet alone returns there, and the next step then
// verifies a tree that names the screen the app is leaving: on the folio wasm
// build a submit that landed on Home was recorded as still being on the
// transaction screen, so a property gated on where the action landed read the
// wrong route and went vacuous. The wait is bounded so a page that genuinely
// shows two *Screen ids at rest costs this much per step and no more.
const transitionSettlePeriod = 800 * time.Millisecond
// settleReturnMargin is what WaitForIdle holds back from the caller's timeout,
// so returning late by our own doing surfaces as a settled page rather than a
// context cancellation.
const settleReturnMargin = 100 * time.Millisecond
// settleScanMargin covers the in-page work the two waits do not themselves
// account for: liveScreens() walks the document and every shadow root on each
// 16 ms poll, and the whole script costs one CDP round trip.
const settleScanMargin = 250 * time.Millisecond
// MinIdleTimeout is the shortest timeout WaitForIdle can be handed and still
// spend the waits it is built from: the DOM quiet period, the route-transition
// window that only opens once that quiet period has elapsed, and the second
// quiet period the transition's own closing mutation starts. A caller that
// passes less caps the settle below its own budget, and the step then samples a
// page that is still mid-transition - which is the exact failure the transition
// wait exists to prevent. internal/runner raises a shorter caller timeout to
// this value.
func (d *Driver) MinIdleTimeout() time.Duration {
return 2*domQuietPeriod + transitionSettlePeriod +
settleScanMargin + settleReturnMargin
}
func (d *Driver) WaitForIdle(ctx context.Context, timeout time.Duration) error {
runCtx, cancel := d.runCtx(ctx)
defer cancel()
// Leave the caller's deadline some room: returning late by our own doing
// would surface as a context cancellation instead of a settled page.
budget := max(timeout-settleReturnMargin, domQuietPeriod)
script := fmt.Sprintf(settleScript,
domQuietPeriod.Milliseconds(),
budget.Milliseconds(),
transitionSettlePeriod.Milliseconds(),
)
return chromedp.Run(runCtx,
chromedp.WaitReady("body", chromedp.ByQuery),
chromedp.Evaluate(script, nil, awaitPromise),
)
}
// liveScreensFunction defines liveScreens(), the page-side count of live ids
// ending in "Screen". More than one is a route transition in flight: the same
// rule the tree parser applies (Transitional in internal/hierarchy), so the
// driver and the runner agree on what a settled route looks like. It descends
// shadow roots because a canvas app keeps its whole accessibility tree inside
// one.
const liveScreensFunction = `
const liveScreens = () => {
let count = 0;
const visit = (root) => {
count += root.querySelectorAll('[id$="Screen"]').length;
for (const element of root.querySelectorAll('*')) {
if (element.shadowRoot) visit(element.shadowRoot);
}
};
visit(document);
return count;
};`
// settleScript resolves once the document has gone quiet for %d ms and is not
// mid route transition, or after %d ms whatever happens; the transition wait
// itself gives up after %d ms. Shadow roots get their own observer: a canvas
// app keeps its whole accessibility tree inside one, and mutations there do not
// reach an observer on the document.
//
// The transition window opens when the quiet period ends, not when the script
// starts. Anchored at the start it is already spent by the time the check can
// first run on any page that keeps mutating for longer than the window, so the
// wait resolves immediately with both routes still live - the mid-transition
// return this whole wait exists to prevent. Each mutation reopens it, and the
// budget above bounds the total either way.
const settleScript = `
new Promise(resolve => {
const quietMillis = %d, budgetMillis = %d, transitionMillis = %d;
const observers = [];
let transitionDeadline = 0;
let timer = null;
const finish = () => {
clearTimeout(timer);
for (const observer of observers) observer.disconnect();
resolve();
};
` + liveScreensFunction + `
const quiet = () => {
if (transitionDeadline === 0) transitionDeadline = Date.now() + transitionMillis;
if (liveScreens() > 1 && Date.now() < transitionDeadline) {
timer = setTimeout(quiet, 16);
return;
}
finish();
};
const restart = () => {
clearTimeout(timer);
transitionDeadline = 0;
timer = setTimeout(quiet, quietMillis);
};
const watch = (root) => {
const observer = new MutationObserver(restart);
observer.observe(root, {subtree: true, childList: true, attributes: true, characterData: true});
observers.push(observer);
for (const element of root.querySelectorAll('*')) {
if (element.shadowRoot) watch(element.shadowRoot);
}
};
watch(document);
setTimeout(finish, budgetMillis);
restart();
})`
func awaitPromise(params *runtime.EvaluateParams) *runtime.EvaluateParams {
return params.WithAwaitPromise(true)
}
func (d *Driver) Health(_ context.Context) (driver.Health, error) {
select {
case <-d.tabCtx.Done():
return driver.Health{Ready: false, Version: "chrome", Platform: "web"}, nil
default:
return driver.Health{Ready: true, Version: "chrome", Platform: "web"}, nil
}
}
func (d *Driver) Metrics(ctx context.Context, _ string) (driver.Metrics, error) {
runCtx, cancel := d.runCtx(ctx)
defer cancel()
var result map[string]any
script := `
(function() {
const mem = performance.memory || {};
return {heap: mem.usedJSHeapSize || 0, totalMem: mem.totalJSHeapSize || 0};
})()`
if err := chromedp.Run(runCtx, chromedp.Evaluate(script, &result)); err != nil {
return driver.Metrics{}, nil
}
heap, _ := result["heap"].(float64)
total, _ := result["totalMem"].(float64)
return driver.Metrics{
HeapBytes: int64(heap),
TotalMemoryBytes: int64(total),
}, nil
}
// consoleLevel places a console call on driver.LogEntry's logcat scale. The
// verbs a spec acts on are all named here; the rest are info rather than "E"
// because promoting them would convict an app of an error it never logged.
func consoleLevel(apiType runtime.APIType) string {
switch apiType {
case runtime.APITypeError, runtime.APITypeAssert:
return "E"
case runtime.APITypeWarning:
return "W"
case runtime.APITypeDebug:
return "D"
default:
return "I"
}
}
// meetsLevel keeps an entry whose level the scale cannot rank. Ranking an
// unknown level below every threshold drops it, and a dropped entry is
// indistinguishable from a quiet app: the caller sees silence and reports it as
// health.
func meetsLevel(level, minLevel string) bool {
order := map[string]int{"V": 0, "D": 1, "I": 2, "W": 3, "E": 4, "F": 5}
rank, ranked := order[level]
if !ranked {
return true
}
return rank >= order[minLevel]
}
func pngDimensions(png []byte) (int, int) {
if len(png) < 24 {
return 0, 0
}
w := int(png[16])<<24 | int(png[17])<<16 | int(png[18])<<8 | int(png[19])
h := int(png[20])<<24 | int(png[21])<<16 | int(png[22])<<8 | int(png[23])
return w, h
}
var (
_ driver.DeviceDriver = (*Driver)(nil)
_ driver.WebDriver = (*Driver)(nil)
)
// runCtx returns a chromedp-bound context that is also cancelled when the
// caller's ctx is cancelled. This is how step deadlines and Ctrl-C propagate
// into a CDP round-trip - chromedp.Run only honors the ctx it is given, and
// d.tabCtx alone has no link to the caller.
func (d *Driver) runCtx(ctx context.Context) (context.Context, context.CancelFunc) {
derived, cancel := context.WithCancel(d.tabCtx)
if ctx == nil || ctx.Done() == nil {
return derived, cancel
}
go func() {
select {
case <-ctx.Done():
cancel()
case <-derived.Done():
}
}()
return derived, cancel
}
// InstallBundle registers the source so it runs at every freshly-navigated
// document context, then immediately evaluates it against the current page so
// the very first tick has access to the registered globals.
func (d *Driver) InstallBundle(ctx context.Context, source []byte) error {
runCtx, cancel := d.runCtx(ctx)
defer cancel()
return chromedp.Run(runCtx,
chromedp.ActionFunc(func(ctx context.Context) error {
if _, err := page.AddScriptToEvaluateOnNewDocument(string(source)).Do(ctx); err != nil {
return fmt.Errorf("addScriptToEvaluateOnNewDocument: %w", err)
}
_, exception, err := runtime.Evaluate(string(source)).Do(ctx)
if err != nil {
return fmt.Errorf("evaluate bundle: %w", err)
}
if exception != nil {
return fmt.Errorf("bundle threw: %s", exceptionMessage(exception))
}
return nil
}),
)
}
// EvaluateExtractors invokes the bundle-installed extractor table and returns
// each extractor's JSON-encoded current value keyed by its registration index.
//
// The read waits out a route transition first, bounded by
// transitionSettlePeriod. The hierarchy fetch already re-fetches a transitional
// tree (fetchSyncedState in internal/runner); without the same rule here the
// two halves of one step describe different moments, and the spec's own
// extractors are the half that loses: on the folio wasm build the extractors
// sampled mid cross-fade and reported the route the app was leaving, so a
// property gated on where the action landed skipped the only step that action
// could be judged on.
func (d *Driver) EvaluateExtractors(ctx context.Context) (map[int]json.RawMessage, error) {
script := fmt.Sprintf(extractorScript, transitionSettlePeriod.Milliseconds())
var encoded string
runCtx, cancel := d.runCtx(ctx)
defer cancel()
if err := chromedp.Run(runCtx, chromedp.Evaluate(script, &encoded, awaitPromise)); err != nil {
return nil, fmt.Errorf("evaluate extractors: %w", err)
}
if encoded == "" || encoded == "{}" {
return map[int]json.RawMessage{}, nil
}
stringMap := map[string]json.RawMessage{}
if err := json.Unmarshal([]byte(encoded), &stringMap); err != nil {
return nil, fmt.Errorf("decode extractor map: %w", err)
}
result := make(map[int]json.RawMessage, len(stringMap))
for key, entry := range stringMap {
index, err := strconv.Atoi(key)
if err != nil {
return nil, fmt.Errorf("non-integer extractor key %q", key)
}
reading, err := extractorReading(entry)
if err != nil {
return nil, fmt.Errorf("extractor %d: %w", index, err)
}
result[index] = reading
}
return result, nil
}
// extractorReading unwraps one entry of the page's extractor table. The page
// wraps every reading in a {"value": ...} envelope (evaluateExtractors in
// pkg/spec/src/web-runtime.ts) because JSON has no undefined: an absent `value`
// is the getter returning undefined, and returning it as an empty payload is
// what makes the goja host record undefined too. Reading it as JSON null would
// claim the getter returned null, so `x.current === undefined` would answer one
// thing on native and another on web.
func extractorReading(entry json.RawMessage) (json.RawMessage, error) {
var envelope struct {
Value json.RawMessage `json:"value"`
}
if err := json.Unmarshal(entry, &envelope); err != nil {
return nil, fmt.Errorf(
"reading %s is not a {\"value\"} envelope; the page and the host are "+
"running different bundles: %w", entry, err)
}
return envelope.Value, nil
}
// SetLastAction installs the previous step's action as state.lastAction inside
// the page runtime. The page cannot derive it: only the runner knows which
// action was actually applied. Without this call every web state.lastAction is
// null, so a property gated on what the last action did is vacuously true and
// reports a green run while checking nothing.
//
// The call is deliberately unguarded. A `setter && setter(...)` form evaluates
// to undefined on a page whose runtime does not define the setter, and chromedp
// reports that as success, so "the page cannot accept lastAction" would be
// indistinguishable from "installed". That page is reachable: a run resolving
// its web runtime from an older published @sanderling/spec would silently no-op
// every step. Unguarded, the missing global throws and the run fails loudly.
func (d *Driver) SetLastAction(ctx context.Context, encoded json.RawMessage) error {
payload := strings.TrimSpace(string(encoded))
if payload == "" {
payload = "null"
}
script := fmt.Sprintf(`window.__sanderlingSetLastAction__(%s)`, payload)
runCtx, cancel := d.runCtx(ctx)
defer cancel()
if err := chromedp.Run(runCtx, chromedp.Evaluate(script, nil)); err != nil {
return fmt.Errorf("set last action: %w", err)
}
return nil
}
// SetLogs installs the entries this step's log fetch returned as state.logs
// inside the page runtime. The page cannot derive them: console output reaches
// the driver over CDP and nothing in the page reads it back. Without this call
// every web state.logs is empty, and since the page's reading of an extractor
// replaces the host's, the default noLogcatErrors then reports green on a run
// whose console was full of errors.
//
// Unguarded for the same reason as SetLastAction: on a page with no setter,
// "the page cannot accept logs" has to fail the run rather than be reported as
// a successful install.
func (d *Driver) SetLogs(ctx context.Context, encoded json.RawMessage) error {
payload := strings.TrimSpace(string(encoded))
if payload == "" {
payload = "[]"
}
script := fmt.Sprintf(`window.__sanderlingSetLogs__(%s)`, payload)
runCtx, cancel := d.runCtx(ctx)
defer cancel()
if err := chromedp.Run(runCtx, chromedp.Evaluate(script, nil)); err != nil {
return fmt.Errorf("set logs: %w", err)
}
return nil
}
// extractorScript resolves the extractor table once the page is not mid route
// transition, giving up on that wait after %d ms.
//
// A missing table rejects rather than reporting {}, for the same reason
// SetLastAction no longer guards its call: an empty override map is what a
// spec with no extractors returns, so the guarded form made "this page has no
// sanderling runtime" read as a normal step whose properties then ran on
// goja's dump-derived values instead of the page's.
const extractorScript = `
new Promise((resolve, reject) => {
const deadline = Date.now() + %d;` + liveScreensFunction + `
const read = () => {
if (liveScreens() > 1 && Date.now() < deadline) {
setTimeout(read, 16);
return;
}
if (typeof window.__sanderlingExtractors__ !== "function") {
reject(new Error("__sanderlingExtractors__ is not installed in the page"));
return;
}
resolve(JSON.stringify(window.__sanderlingExtractors__()));
};
read();
})`
// Exceptions returns the uncaught errors and unhandled rejections the page
// runtime has buffered so far. The buffer is cumulative, which is what
// state.exceptions means inside the page (buildState in
// pkg/spec/src/web-runtime.ts), so the host and the page read one list.
func (d *Driver) Exceptions(ctx context.Context) ([]driver.Exception, error) {
const script = `JSON.stringify(window.__sanderlingExceptions__ ? window.__sanderlingExceptions__() : [])`
var encoded string
runCtx, cancel := d.runCtx(ctx)
defer cancel()
if err := chromedp.Run(runCtx, chromedp.Evaluate(script, &encoded)); err != nil {
return nil, fmt.Errorf("evaluate exceptions: %w", err)
}
if encoded == "" || encoded == "[]" {
return nil, nil
}
var captured []struct {
Class string `json:"class"`
Message string `json:"message"`
StackTrace string `json:"stackTrace"`
UnixMillis int64 `json:"unixMillis"`
}
if err := json.Unmarshal([]byte(encoded), &captured); err != nil {
return nil, fmt.Errorf("decode exceptions %s: %w", encoded, err)
}
result := make([]driver.Exception, 0, len(captured))
for _, entry := range captured {
result = append(result, driver.Exception{
Class: entry.Class,
Message: entry.Message,
StackTrace: entry.StackTrace,
UnixMillis: entry.UnixMillis,
})
}
return result, nil
}
// nextActionScript puts the carried draw position back before the picker
// decides and reads the new one out afterwards, in the one evaluation, so no
// navigation can land between the restore and the draw.
const nextActionScript = `((carried) => {
if (!window.__sanderlingNextAction__) return "{}";
if (carried !== "" && window.__sanderlingRestorePickerState__) {
window.__sanderlingRestorePickerState__(carried);
}
const action = window.__sanderlingNextAction__();
const state = window.__sanderlingPickerState__ ? window.__sanderlingPickerState__() : "";
return JSON.stringify({action, state});
})(%s)`
// NextActionFromV8 invokes the bundle-installed action generator and returns
// the resulting Action JSON. Returns an empty json.RawMessage when the
// generator declines to act this tick.
//
// The picker's draw position rides along: it lives here rather than in the
// page, because a page that navigates gets a fresh runtime whose picker would
// otherwise start the seed's stream over at its first draw on every reload.
func (d *Driver) NextActionFromV8(ctx context.Context) (json.RawMessage, error) {
script := fmt.Sprintf(nextActionScript, strconv.Quote(d.pickerState))
var encoded string
runCtx, cancel := d.runCtx(ctx)
defer cancel()
if err := chromedp.Run(runCtx, chromedp.Evaluate(script, &encoded)); err != nil {
return nil, fmt.Errorf("evaluate next action: %w", err)
}
if encoded == "" {
return nil, nil
}
var decoded struct {
Action json.RawMessage `json:"action"`
State string `json:"state"`
}
if err := json.Unmarshal([]byte(encoded), &decoded); err != nil {
return nil, fmt.Errorf("decode next action %s: %w", encoded, err)
}
// An empty state means the page had no runtime to ask, so the position we
// already hold is still the run's position.
if decoded.State != "" {
d.pickerState = decoded.State
}
if len(decoded.Action) == 0 || string(decoded.Action) == "null" {
return nil, nil
}
return decoded.Action, nil
}