mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
refactor(spec): one candidate producer over one target-eligibility rule
Both hosts routed verbs themselves and both policies enumerated their own actions, and all four drifted. Web sent `swipes` to scrollable containers only, so swipe-to-dismiss on a list row was reachable on native and unreachable on web; the model policy folded gestures its own way and could not reach what the seeded picker drew. A host now reports facts about every element and never decides which verb may act on it: targets.ts acceptsTarget owns that for both. pick.ts builtinCandidates is the single enumeration, and the model policy reads it through __sanderlingEnumerateBuiltin__ instead of reimplementing it in Go. Gesture verbs change with it: scrolls stay vertical over scrollable containers, swipes go free-form in all four directions from any element with real bounds. Claude-Session: https://claude.ai/code/session_01Fj4wJUikdABuMQEETwW55J
This commit is contained in:
1 parent
ef04a6de9d
commit
b5f64bf665
23 files changed
+1494
-594
No files matched your search
@@ -65,9 +65,9 @@ export type GeneratorNode =
|
||||
| { kind: "builtin"; verb: BuiltinVerb }
|
||||
| { kind: "llm"; config: { model: string; instructions?: string } };
|
||||
|
||||
// Candidate is one host-enumerated target for a builtin verb. The host
|
||||
// resolves geometry (and a native selector) so no element handle crosses into
|
||||
// the picker. width/height let swipe/scroll size a gesture off the element.
|
||||
// Candidate is the resolved geometry of one target. The host resolves it (and a
|
||||
// native selector) so no element handle crosses into the picker. width/height
|
||||
// let swipe/scroll size a gesture off the element.
|
||||
export interface Candidate {
|
||||
x: number;
|
||||
y: number;
|
||||
@@ -76,15 +76,27 @@ export interface Candidate {
|
||||
height?: number;
|
||||
}
|
||||
|
||||
// TargetElement is one host-enumerated element, offered to EVERY verb, carrying
|
||||
// the facts per-verb eligibility is decided from. The host reports the facts; it
|
||||
// does not apply them. targets.ts acceptsTarget owns that decision for both
|
||||
// hosts, so a verb cannot mean one thing on native and another on web.
|
||||
export interface TargetElement extends Candidate {
|
||||
clickable: boolean;
|
||||
enabled: boolean;
|
||||
editable: boolean;
|
||||
scrollable: boolean;
|
||||
}
|
||||
|
||||
// Host is the platform backing the picker draws against. In this foundation
|
||||
// workflow only the interface is defined and exercised against a stub; the
|
||||
// goja and DOM implementations land in the rewire workflow.
|
||||
export interface Host {
|
||||
platform(): "android" | "ios" | "web";
|
||||
// queryCandidates returns the host-enumerated targets for a verb, in a
|
||||
// deterministic order. The picker indexes into this list with the PCG, so
|
||||
// the order is part of the parity contract.
|
||||
queryCandidates(verb: BuiltinVerb): Candidate[];
|
||||
// queryTargets returns every element the host can offer, in a deterministic
|
||||
// order, with no per-verb filtering: the picker applies acceptsTarget. The
|
||||
// picker indexes into this list with the PCG, so the order is part of the
|
||||
// parity contract.
|
||||
queryTargets(): TargetElement[];
|
||||
// reportUnsupported is invoked at most once per verb@platform (see verbs.ts)
|
||||
// when a verb has no support on this platform.
|
||||
reportUnsupported(verb: BuiltinVerb): void;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Goja-side runtime entry for the native verifier (internal/verifier).
|
||||
//
|
||||
// Go installs globalThis.__sanderlingHost__ (platform/seed/queryCandidates/
|
||||
// Go installs globalThis.__sanderlingHost__ (platform/seed/queryTargets/
|
||||
// reportUnsupported, implemented over the hierarchy tree in bindings.go) before
|
||||
// the spec evaluates. This module bundles AFTER the spec, reads that host, and
|
||||
// wires the shared picker via installRuntime so the goja verifier and the V8 web
|
||||
|
||||
+161
-119
@@ -1,29 +1,44 @@
|
||||
// The shared deterministic action picker for W2 approach B.
|
||||
// The shared deterministic action picker.
|
||||
//
|
||||
// walk() traverses a GeneratorNode tree, and nextAction() wraps it with the
|
||||
// 16-attempt retry that matches worker.go NextAction. Both engines (the goja
|
||||
// verifier and the V8 web runtime) run THIS code, drawing through the shared
|
||||
// Pcg, so a given seed yields an identical action stream on every platform.
|
||||
//
|
||||
// builtinCandidates() is the ONE enumeration of what a builtin verb can do at
|
||||
// the current step. The seeded policy draws a single entry from it below; the
|
||||
// model policy (Go, internal/verifier/llm.go) reads the same list through
|
||||
// __sanderlingEnumerateBuiltin__. Neither policy can reach an action the other
|
||||
// cannot, because neither owns an enumeration of its own.
|
||||
//
|
||||
// PARITY CONTRACT - draw order.
|
||||
// Every random decision goes through the Pcg in a FIXED, pinned order. Changing
|
||||
// this order shifts the stream for a seed and breaks cross-engine
|
||||
// reproducibility, so treat it as load-bearing:
|
||||
//
|
||||
// pickUniform(list): NO draw for an empty or single-entry list, otherwise ONE
|
||||
// intN(list.length) draw. Both the actions node and the
|
||||
// builtin node select through it.
|
||||
// weighted node: ONE float64() draw, then an ASCENDING cumulative scan over
|
||||
// max(0, weight). (matches worker.go pickWeighted.)
|
||||
// actions node: the generator runs FIRST (any from(...).generate() inside
|
||||
// draws intN(itemCount) for >1 items, nothing otherwise),
|
||||
// THEN if the returned list has >1 entry, ONE intN(len) draw;
|
||||
// a 0- or 1-element list draws nothing. (pickFromResult.)
|
||||
// builtin node, per verb, in this exact sequence:
|
||||
// taps/doubleTaps/longPresses: intN(candidateCount) [1 draw]
|
||||
// typing: intN(candidateCount), intN(corpusLength)
|
||||
// swipes: intN(candidateCount),
|
||||
// 200 + intN(401) magnitude, intN(4) direction
|
||||
// scrolls: intN(candidateCount), intN(4) direction
|
||||
// pressKeys: intN(keyCount)
|
||||
// waitOnce: no draw
|
||||
// THEN pickUniform over the returned list.
|
||||
// builtin node: pickUniform over builtinCandidates(verb), THEN the one
|
||||
// value that verb's enumeration leaves to the policy:
|
||||
// `typing` ONE intN(corpusLength) draw for the text,
|
||||
// `swipes` ONE 200 + intN(401) draw for the drag distance.
|
||||
// Every other verb draws nothing further.
|
||||
//
|
||||
// The enumerated candidate count per verb, over the host targets the verb
|
||||
// accepts (targets.ts acceptsTarget), which is what pickUniform draws over:
|
||||
//
|
||||
// taps/doubleTaps/longPresses: one per accepted target
|
||||
// typing: one per accepted target
|
||||
// scrolls: two per scrollable container (down, up)
|
||||
// swipes: four per accepted target (down, up, left, right)
|
||||
// pressKeys: one per key in the platform's pool
|
||||
// waitOnce: exactly one, so it never draws
|
||||
//
|
||||
// Builtin targets are resolved to a {x, y} Point by the host BEFORE the picker
|
||||
// sees them, so no element handle crosses into this module.
|
||||
@@ -32,24 +47,114 @@ import type { Pcg } from "./pcg.ts";
|
||||
import type {
|
||||
ActionDescriptor,
|
||||
BuiltinVerb,
|
||||
Candidate,
|
||||
GeneratorNode,
|
||||
Host,
|
||||
} from "./action-tree.ts";
|
||||
import type { Direction, Point } from "./types.ts";
|
||||
import { INPUT_CORPUS, NATIVE_PRESS_KEYS, WEB_PRESS_KEYS } from "./corpus.ts";
|
||||
import { setSamplerRng } from "./sampler-rng.ts";
|
||||
import { acceptsTarget } from "./targets.ts";
|
||||
import { supports, warnUnsupportedOnce } from "./verbs.ts";
|
||||
|
||||
// SWIPE_MIN_MAGNITUDE / SWIPE_MAGNITUDE_SPAN reproduce worker.go's
|
||||
// `200 + rng.IntN(401)` swipe distance in pixels (200..600 inclusive).
|
||||
const WAIT_MILLIS = 500;
|
||||
|
||||
// SCROLL_DIRECTIONS and SWIPE_DIRECTIONS are the directions each gesture verb
|
||||
// enumerates, one candidate per (target, direction). Scrolls stay vertical:
|
||||
// they target every scrollable container, so they are what makes the enumerated
|
||||
// list long, and scrolling a mobile list means up and down. Swipes take all
|
||||
// four, because swipe-to-dismiss and swipe-to-delete are horizontal gestures on
|
||||
// list rows, and folding them away puts that defect class out of reach.
|
||||
const SCROLL_DIRECTIONS: readonly Direction[] = ["down", "up"];
|
||||
const SWIPE_DIRECTIONS: readonly Direction[] = ["down", "up", "left", "right"];
|
||||
|
||||
// SWIPE_MIN_MAGNITUDE / SWIPE_MAGNITUDE_SPAN are the free-form swipe distance in
|
||||
// pixels the seeded policy draws, 200..600 inclusive. SWIPE_NOMINAL_MAGNITUDE is
|
||||
// the distance the enumeration lists, so every enumerated candidate is already a
|
||||
// runnable gesture before any policy has drawn anything.
|
||||
const SWIPE_MIN_MAGNITUDE = 200;
|
||||
const SWIPE_MAGNITUDE_SPAN = 401;
|
||||
const SWIPE_NOMINAL_MAGNITUDE = 400;
|
||||
const SWIPE_DURATION_MILLIS = 250;
|
||||
|
||||
const DIRECTIONS: readonly Direction[] = ["up", "down", "left", "right"];
|
||||
|
||||
const MAX_RETRIES = 16;
|
||||
|
||||
// BuiltinCandidate is one enumerated action for a builtin verb, paired with the
|
||||
// index of the host target it acts on, in host.queryTargets() order (NOT the
|
||||
// verb-filtered order, so a host can resolve it without repeating the filter).
|
||||
// targetIndex is -1 for the untargeted verbs (a key press, a wait); the model
|
||||
// policy uses it to name the control the action lands on.
|
||||
export interface BuiltinCandidate {
|
||||
action: ActionDescriptor;
|
||||
targetIndex: number;
|
||||
// swipe is set on a `swipes` candidate only. The enumeration fixes where the
|
||||
// drag starts and which way it travels and lists a nominal distance; the
|
||||
// seeded policy rebuilds the gesture from these at a drawn distance, the way a
|
||||
// typing candidate names the field and leaves the text to the policy.
|
||||
swipe?: { origin: Point; direction: Direction };
|
||||
}
|
||||
|
||||
// builtinCandidates enumerates EVERY action a builtin verb can yield against the
|
||||
// host's current state. It is the single candidate producer both policies read,
|
||||
// over the single eligibility rule both hosts consume. An unsupported verb
|
||||
// reports once and enumerates nothing, so a platform that cannot dispatch a verb
|
||||
// never offers it to either policy.
|
||||
export function builtinCandidates(verb: BuiltinVerb, host: Host): BuiltinCandidate[] {
|
||||
if (!supports(verb, host.platform())) {
|
||||
warnUnsupportedOnce(host, verb);
|
||||
return [];
|
||||
}
|
||||
if (verb === "waitOnce") {
|
||||
return [{ action: { kind: "Wait", durationMillis: WAIT_MILLIS }, targetIndex: -1 }];
|
||||
}
|
||||
if (verb === "pressKeys") {
|
||||
const keys = host.platform() === "web" ? WEB_PRESS_KEYS : NATIVE_PRESS_KEYS;
|
||||
return keys.map((key) => ({
|
||||
action: { kind: "PressKey", key } as ActionDescriptor,
|
||||
targetIndex: -1,
|
||||
}));
|
||||
}
|
||||
|
||||
const candidates: BuiltinCandidate[] = [];
|
||||
host.queryTargets().forEach((target, targetIndex) => {
|
||||
if (!acceptsTarget(verb, target)) return;
|
||||
const point = withSelector({ x: target.x, y: target.y }, target.selector);
|
||||
const add = (action: ActionDescriptor) => candidates.push({ action, targetIndex });
|
||||
switch (verb) {
|
||||
case "taps":
|
||||
add({ kind: "Tap", on: point });
|
||||
break;
|
||||
case "doubleTaps":
|
||||
add({ kind: "DoubleTap", on: point });
|
||||
break;
|
||||
case "longPresses":
|
||||
add({ kind: "LongPress", on: point });
|
||||
break;
|
||||
case "typing":
|
||||
// text is left empty: it is the one value the policy supplies, drawn
|
||||
// from the corpus by the seeded arm and written by the model.
|
||||
add({ kind: "InputText", into: point, text: "" });
|
||||
break;
|
||||
case "scrolls":
|
||||
for (const direction of SCROLL_DIRECTIONS) {
|
||||
add(scrollDescriptor({ x: target.x, y: target.y }, direction, target));
|
||||
}
|
||||
break;
|
||||
case "swipes":
|
||||
for (const direction of SWIPE_DIRECTIONS) {
|
||||
const origin = { x: target.x, y: target.y };
|
||||
candidates.push({
|
||||
action: swipeDescriptor(origin, direction, SWIPE_NOMINAL_MAGNITUDE),
|
||||
targetIndex,
|
||||
swipe: { origin, direction },
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
});
|
||||
return candidates;
|
||||
}
|
||||
|
||||
// nextAction resolves an action for the current step, retrying walk() up to 16
|
||||
// times when it yields null (matches worker.go NextAction). Returns null when
|
||||
// every attempt comes up empty.
|
||||
@@ -79,7 +184,7 @@ export function walk(
|
||||
// generator so author sampling shares this single deterministic stream.
|
||||
setSamplerRng(rng);
|
||||
try {
|
||||
return walkActions(node.generate(), rng);
|
||||
return pickUniform(node.generate(), rng);
|
||||
} finally {
|
||||
setSamplerRng(null);
|
||||
}
|
||||
@@ -115,13 +220,13 @@ function walkWeighted(
|
||||
return last ? walk(last[1], rng, host) : null;
|
||||
}
|
||||
|
||||
function walkActions(
|
||||
generated: ActionDescriptor[],
|
||||
rng: Pcg,
|
||||
): ActionDescriptor | null {
|
||||
if (generated.length === 0) return null;
|
||||
if (generated.length === 1) return generated[0] ?? null;
|
||||
return generated[rng.intN(generated.length)] ?? null;
|
||||
// pickUniform selects one entry from a list, drawing nothing when there is no
|
||||
// choice to make. It is the only selection rule in this module: the seeded
|
||||
// policy is exactly "enumerate, then pickUniform".
|
||||
function pickUniform<T>(list: readonly T[], rng: Pcg): T | null {
|
||||
if (list.length === 0) return null;
|
||||
if (list.length === 1) return list[0] ?? null;
|
||||
return list[rng.intN(list.length)] ?? null;
|
||||
}
|
||||
|
||||
function walkBuiltin(
|
||||
@@ -129,41 +234,17 @@ function walkBuiltin(
|
||||
rng: Pcg,
|
||||
host: Host,
|
||||
): ActionDescriptor | null {
|
||||
if (!supports(verb, host.platform())) {
|
||||
warnUnsupportedOnce(host, verb);
|
||||
return null;
|
||||
}
|
||||
if (verb === "waitOnce") {
|
||||
return { kind: "Wait", durationMillis: 500 };
|
||||
}
|
||||
if (verb === "pressKeys") {
|
||||
return walkPressKey(rng, host);
|
||||
}
|
||||
|
||||
const candidates = host.queryCandidates(verb);
|
||||
if (candidates.length === 0) return null;
|
||||
const picked = candidates[rng.intN(candidates.length)];
|
||||
const picked = pickUniform(builtinCandidates(verb, host), rng);
|
||||
if (!picked) return null;
|
||||
const point: Point = { x: picked.x, y: picked.y };
|
||||
|
||||
switch (verb) {
|
||||
case "taps":
|
||||
return tapDescriptor("Tap", point, picked.selector);
|
||||
case "doubleTaps":
|
||||
return tapDescriptor("DoubleTap", point, picked.selector);
|
||||
case "longPresses":
|
||||
return tapDescriptor("LongPress", point, picked.selector);
|
||||
case "typing": {
|
||||
const text = INPUT_CORPUS[rng.intN(INPUT_CORPUS.length)] ?? "";
|
||||
return { kind: "InputText", into: withSelector(point, picked.selector), text };
|
||||
}
|
||||
case "swipes":
|
||||
return buildSwipe(point, rng);
|
||||
case "scrolls": {
|
||||
const direction = DIRECTIONS[rng.intN(DIRECTIONS.length)] ?? "down";
|
||||
return buildScroll(point, direction, picked, rng);
|
||||
}
|
||||
if (picked.action.kind === "InputText") {
|
||||
return { ...picked.action, text: INPUT_CORPUS[rng.intN(INPUT_CORPUS.length)] ?? "" };
|
||||
}
|
||||
if (picked.swipe) {
|
||||
const magnitude = SWIPE_MIN_MAGNITUDE + rng.intN(SWIPE_MAGNITUDE_SPAN);
|
||||
const { origin, direction } = picked.swipe;
|
||||
return swipeDescriptor(origin, direction, magnitude);
|
||||
}
|
||||
return picked.action;
|
||||
}
|
||||
|
||||
// withSelector attaches a native selector to a resolved Point so the runner can
|
||||
@@ -173,82 +254,43 @@ function withSelector(point: Point, selector?: string): Point {
|
||||
return { ...point, selector } as Point;
|
||||
}
|
||||
|
||||
function tapDescriptor(
|
||||
kind: "Tap" | "DoubleTap" | "LongPress",
|
||||
point: Point,
|
||||
selector?: string,
|
||||
): ActionDescriptor {
|
||||
return { kind, on: withSelector(point, selector) } as ActionDescriptor;
|
||||
}
|
||||
|
||||
// buildScroll lowers a scroll to a swipe over the container, matching
|
||||
// worker.go's geometry: the gesture drags opposite the named content motion,
|
||||
// scrollDescriptor lowers a scroll to a drag over the container, matching
|
||||
// runner.go's geometry: the gesture drags opposite the named content motion,
|
||||
// magnitude 40% of the container extent. Missing width/height (web root) yields
|
||||
// a zero-length endpoint, which the runner re-derives from container bounds.
|
||||
function buildScroll(
|
||||
function scrollDescriptor(
|
||||
from: Point,
|
||||
direction: Direction,
|
||||
candidate: { width?: number; height?: number },
|
||||
_rng: Pcg,
|
||||
candidate: Candidate,
|
||||
): ActionDescriptor {
|
||||
const width = candidate.width ?? 0;
|
||||
const height = candidate.height ?? 0;
|
||||
let toX = from.x;
|
||||
let toY = from.y;
|
||||
switch (direction) {
|
||||
case "down":
|
||||
toY = from.y - Math.trunc((4 * height) / 10);
|
||||
break;
|
||||
case "up":
|
||||
toY = from.y + Math.trunc((4 * height) / 10);
|
||||
break;
|
||||
case "left":
|
||||
toX = from.x + Math.trunc((4 * width) / 10);
|
||||
break;
|
||||
case "right":
|
||||
toX = from.x - Math.trunc((4 * width) / 10);
|
||||
break;
|
||||
}
|
||||
if (direction === "down") toY = from.y - Math.trunc((4 * height) / 10);
|
||||
if (direction === "up") toY = from.y + Math.trunc((4 * height) / 10);
|
||||
return {
|
||||
kind: "Scroll",
|
||||
direction,
|
||||
in: from,
|
||||
from,
|
||||
to: { x: Math.max(0, toX), y: Math.max(0, toY) },
|
||||
to: { x: from.x, y: Math.max(0, toY) },
|
||||
} as ActionDescriptor;
|
||||
}
|
||||
|
||||
function walkPressKey(rng: Pcg, host: Host): ActionDescriptor | null {
|
||||
const keys = host.platform() === "web" ? WEB_PRESS_KEYS : NATIVE_PRESS_KEYS;
|
||||
if (keys.length === 0) return null;
|
||||
const key = keys[rng.intN(keys.length)] ?? keys[0];
|
||||
if (key === undefined) return null;
|
||||
return { kind: "PressKey", key };
|
||||
// swipeDescriptor builds a free-form drag from a point over a raw pixel
|
||||
// distance, rather than a fraction of a container extent: `swipes` targets any
|
||||
// element with real bounds, and most of those have no scroll extent to size a
|
||||
// gesture against. `direction` names where the finger travels, because a swipe
|
||||
// IS the gesture; a scroll names content motion and drags the other way.
|
||||
function swipeDescriptor(
|
||||
from: Point,
|
||||
direction: Direction,
|
||||
magnitude: number,
|
||||
): ActionDescriptor {
|
||||
const horizontal = direction === "left" || direction === "right";
|
||||
const forward = direction === "down" || direction === "right";
|
||||
const travel = forward ? magnitude : -magnitude;
|
||||
const to = horizontal
|
||||
? { x: Math.max(0, from.x + travel), y: from.y }
|
||||
: { x: from.x, y: Math.max(0, from.y + travel) };
|
||||
return { kind: "Swipe", from, to, durationMillis: SWIPE_DURATION_MILLIS };
|
||||
}
|
||||
|
||||
function buildSwipe(from: Point, rng: Pcg): ActionDescriptor {
|
||||
const magnitude = SWIPE_MIN_MAGNITUDE + rng.intN(SWIPE_MAGNITUDE_SPAN);
|
||||
let toX = from.x;
|
||||
let toY = from.y;
|
||||
switch (rng.intN(4)) {
|
||||
case 0:
|
||||
toY = from.y - magnitude;
|
||||
break;
|
||||
case 1:
|
||||
toY = from.y + magnitude;
|
||||
break;
|
||||
case 2:
|
||||
toX = from.x - magnitude;
|
||||
break;
|
||||
case 3:
|
||||
toX = from.x + magnitude;
|
||||
break;
|
||||
}
|
||||
return {
|
||||
kind: "Swipe",
|
||||
from,
|
||||
to: { x: Math.max(0, toX), y: Math.max(0, toY) },
|
||||
durationMillis: SWIPE_DURATION_MILLIS,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -8,9 +8,9 @@
|
||||
// identical action stream by construction.
|
||||
|
||||
import { Pcg } from "./pcg.ts";
|
||||
import { nextAction, walk } from "./pick.ts";
|
||||
import { builtinCandidates, nextAction, walk } from "./pick.ts";
|
||||
import { INPUT_CORPUS } from "./corpus.ts";
|
||||
import type { ActionDescriptor, GeneratorNode, Host } from "./action-tree.ts";
|
||||
import type { ActionDescriptor, BuiltinVerb, GeneratorNode, Host } from "./action-tree.ts";
|
||||
import type { Point } from "./types.ts";
|
||||
|
||||
// SerializedAction is the flat, camelCase wire shape JS emits and Go decodes
|
||||
@@ -138,6 +138,16 @@ export function installRuntime(
|
||||
"__sanderlingSampleInput__",
|
||||
() => INPUT_CORPUS[rng.intN(INPUT_CORPUS.length)] ?? "",
|
||||
);
|
||||
// The model policy (Go) selects from the SAME enumeration the seeded picker
|
||||
// draws from, reached through here rather than reimplemented on the Go side.
|
||||
// Each entry is serialized with the wire contract Go already decodes, so the
|
||||
// two policies also agree on the action a chosen candidate executes.
|
||||
defineLockedGlobal("__sanderlingEnumerateBuiltin__", (verb: BuiltinVerb) =>
|
||||
builtinCandidates(verb, host).map((candidate) => ({
|
||||
action: serializeAction(candidate.action),
|
||||
targetIndex: candidate.targetIndex,
|
||||
})),
|
||||
);
|
||||
defineLockedGlobal("__sanderlingExtractors__", () => evaluateExtractors());
|
||||
// __sanderlingSetupAction__ walks ONLY the setup generator once, for the LLM
|
||||
// action generator (Go), which drives selection itself and must not run the
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
// Per-verb target eligibility: the ONE definition both hosts consume.
|
||||
//
|
||||
// A host enumerates every element it can offer (the goja verifier over the
|
||||
// hierarchy tree, the V8 web runtime over the DOM) and reports a fixed set of
|
||||
// facts about each. It does NOT decide which verb may act on which element:
|
||||
// acceptsTarget does, here, once. When each host routed verbs itself the two
|
||||
// drifted, and the same spec induced a different action space per platform --
|
||||
// web sent `swipes` to scrollable containers only, so swipe-to-dismiss on a
|
||||
// list row was reachable on native and unreachable on web.
|
||||
//
|
||||
// What stays platform-specific is how a fact is COMPUTED, because the source
|
||||
// models genuinely differ: `clickable` is an accessibility attribute on
|
||||
// Android/iOS and a CSS selector match on web. The fact vocabulary below is the
|
||||
// contract; the mapping onto it is each host's business, and it is the only
|
||||
// place the platforms are allowed to disagree.
|
||||
|
||||
import type { BuiltinVerb, TargetElement } from "./action-tree.ts";
|
||||
|
||||
// TargetFact names one property of a target a verb can require.
|
||||
export type TargetFact =
|
||||
| "clickable"
|
||||
| "enabled"
|
||||
| "editable"
|
||||
| "scrollable"
|
||||
| "positiveBounds";
|
||||
|
||||
// VERB_REQUIRED_FACTS lists the facts a target must have for a verb to act on
|
||||
// it. `null` marks a verb with no target at all: a key press and a wait are
|
||||
// enumerated by the picker without consulting the host.
|
||||
const VERB_REQUIRED_FACTS: Record<BuiltinVerb, readonly TargetFact[] | null> = {
|
||||
taps: ["clickable", "enabled", "positiveBounds"],
|
||||
doubleTaps: ["clickable", "enabled", "positiveBounds"],
|
||||
longPresses: ["clickable", "enabled", "positiveBounds"],
|
||||
typing: ["editable", "enabled", "positiveBounds"],
|
||||
// Scrolls target containers that can actually scroll, and enumerate down/up.
|
||||
scrolls: ["scrollable", "positiveBounds"],
|
||||
// Any visible element is a valid swipe origin. Swipe-to-dismiss and
|
||||
// swipe-to-delete live on list rows and cards, which are not scrollable
|
||||
// containers, so scoping swipes to containers would put that whole class of
|
||||
// interaction out of reach.
|
||||
swipes: ["positiveBounds"],
|
||||
pressKeys: null,
|
||||
waitOnce: null,
|
||||
};
|
||||
|
||||
// acceptsTarget is the per-verb element filter. Every targeted verb demands
|
||||
// real bounds: a zero-bounds node centers at (0,0), and a downward gesture from
|
||||
// the top-left corner is the system gesture that pulls down the notification
|
||||
// shade, dragging the fuzzer out of the app.
|
||||
export function acceptsTarget(verb: BuiltinVerb, target: TargetElement): boolean {
|
||||
const required = VERB_REQUIRED_FACTS[verb];
|
||||
if (required === null) return false;
|
||||
return required.every((fact) => hasFact(target, fact));
|
||||
}
|
||||
|
||||
function hasFact(target: TargetElement, fact: TargetFact): boolean {
|
||||
switch (fact) {
|
||||
case "clickable":
|
||||
return target.clickable;
|
||||
case "enabled":
|
||||
return target.enabled;
|
||||
case "editable":
|
||||
return target.editable;
|
||||
case "scrollable":
|
||||
return target.scrollable;
|
||||
case "positiveBounds":
|
||||
return (target.width ?? 0) > 0 && (target.height ?? 0) > 0;
|
||||
}
|
||||
}
|
||||
+50
-72
@@ -4,7 +4,7 @@
|
||||
//
|
||||
// This file is the WEB Host. It installs globalThis.__sanderling__ (extract +
|
||||
// LTL formula binds) before the spec evaluates, implements the Host interface
|
||||
// (platform/seed/queryCandidates/reportUnsupported) over the live DOM, and then
|
||||
// (platform/seed/queryTargets/reportUnsupported) over the live DOM, and then
|
||||
// delegates ALL action generation to the shared picker via installRuntime
|
||||
// (runtime-entry.ts -> pick.ts). The goja verifier runs the SAME picker over the
|
||||
// SAME Pcg, so a given seed yields an identical action stream by construction.
|
||||
@@ -13,11 +13,11 @@
|
||||
// window.__sanderlingNextAction__() over CDP each tick. LTL predicates are
|
||||
// stubbed: properties run host-side in goja, which loads its own bundle.
|
||||
//
|
||||
// Element references never cross V8/host. queryCandidates resolves each element
|
||||
// to a {x, y} Point via getBoundingClientRect before the picker sees it.
|
||||
// Element references never cross V8/host. queryTargets resolves each element to
|
||||
// a {x, y} Point via getBoundingClientRect before the picker sees it.
|
||||
|
||||
import { installRuntime } from "./runtime-entry.ts";
|
||||
import type { BuiltinVerb, Candidate, Host } from "./action-tree.ts";
|
||||
import type { BuiltinVerb, Candidate, Host, TargetElement } from "./action-tree.ts";
|
||||
|
||||
interface Handle {
|
||||
readonly current: unknown;
|
||||
@@ -467,9 +467,12 @@ function sanitizeAt(value: unknown, depth: number, seen: WeakSet<object>): unkno
|
||||
return out;
|
||||
}
|
||||
|
||||
// Per-verb DOM selector sets. The tappable set backs taps/doubleTaps/longPresses
|
||||
// (every tappable element is also a valid long-press target); the editable set
|
||||
// backs typing; swipes/scrolls target the scrollable element.
|
||||
// The DOM half of the fact mapping. Which verb may act on which target is NOT
|
||||
// decided here: queryTargets reports facts and targets.ts acceptsTarget applies
|
||||
// them, the same rule the native host's targets run through. These selectors are
|
||||
// only how the DOM answers "is this clickable" / "is this editable", the two
|
||||
// facts with no direct DOM equivalent of the accessibility attributes native
|
||||
// platforms expose.
|
||||
const TAPPABLE_SELECTOR = 'a, button, input, select, textarea, [role="button"], [onclick]';
|
||||
const EDITABLE_SELECTOR = "input, textarea, [contenteditable]";
|
||||
|
||||
@@ -477,12 +480,6 @@ const NON_TEXT_INPUT_TYPES = [
|
||||
"button", "submit", "checkbox", "radio", "range", "color", "file", "image", "reset",
|
||||
];
|
||||
|
||||
function isVisible(element: HTMLElement): boolean {
|
||||
if ((element as HTMLButtonElement).disabled) return false;
|
||||
const rect = element.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
}
|
||||
|
||||
function isEditableElement(element: HTMLElement): boolean {
|
||||
if (element.isContentEditable) return true;
|
||||
const tag = element.tagName.toLowerCase();
|
||||
@@ -494,6 +491,14 @@ function isEditableElement(element: HTMLElement): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
// isScrollable mirrors the native `scrollable` accessibility attribute: the
|
||||
// container can actually scroll, i.e. its content overflows its box. The
|
||||
// document scrolling root is not special-cased in: when the page does not
|
||||
// overflow there is no scroll to perform, and native would offer none either.
|
||||
function isScrollable(element: HTMLElement): boolean {
|
||||
return element.scrollHeight > element.clientHeight || element.scrollWidth > element.clientWidth;
|
||||
}
|
||||
|
||||
function pointOf(element: Element): Candidate {
|
||||
const rect = element.getBoundingClientRect();
|
||||
return {
|
||||
@@ -504,41 +509,33 @@ function pointOf(element: Element): Candidate {
|
||||
};
|
||||
}
|
||||
|
||||
function tappableCandidates(): Candidate[] {
|
||||
return Array.from(document.querySelectorAll<HTMLElement>(TAPPABLE_SELECTOR))
|
||||
.filter(isVisible)
|
||||
.map(pointOf);
|
||||
// collectTargets walks the document ONCE and reports every element with the facts
|
||||
// the shared eligibility rule reads. The tappable/editable membership sets are
|
||||
// resolved by selector first so the DOM's answer to "clickable" and "editable"
|
||||
// stays expressed in CSS, as it always was.
|
||||
function collectTargets(): TargetElement[] {
|
||||
const clickable = new Set<Element>(Array.from(document.querySelectorAll(TAPPABLE_SELECTOR)));
|
||||
const editable = new Set<Element>(
|
||||
Array.from(document.querySelectorAll<HTMLElement>(EDITABLE_SELECTOR)).filter(
|
||||
isEditableElement,
|
||||
),
|
||||
);
|
||||
return Array.from(document.querySelectorAll<HTMLElement>("*")).map((element) => ({
|
||||
...pointOf(element),
|
||||
clickable: clickable.has(element),
|
||||
enabled: !(element as HTMLButtonElement).disabled,
|
||||
editable: editable.has(element),
|
||||
scrollable: isScrollable(element),
|
||||
}));
|
||||
}
|
||||
|
||||
function editableCandidates(): Candidate[] {
|
||||
return Array.from(document.querySelectorAll<HTMLElement>(EDITABLE_SELECTOR))
|
||||
.filter((element) => isEditableElement(element) && isVisible(element))
|
||||
.map(pointOf);
|
||||
}
|
||||
|
||||
// Scrollable candidates back swipe/scroll: elements that overflow their box,
|
||||
// plus the scrolling root so a page-level scroll always has a target.
|
||||
function scrollableCandidates(): Candidate[] {
|
||||
const root = document.scrollingElement ?? document.documentElement;
|
||||
const candidates: Candidate[] = root ? [pointOf(root)] : [];
|
||||
for (const element of Array.from(document.querySelectorAll<HTMLElement>("*"))) {
|
||||
if (element === root) continue;
|
||||
if (element.scrollHeight <= element.clientHeight && element.scrollWidth <= element.clientWidth) {
|
||||
continue;
|
||||
}
|
||||
if (!isVisible(element)) continue;
|
||||
candidates.push(pointOf(element));
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
// Per-tick candidate cache: the picker's 16-attempt retry re-queries the same
|
||||
// verb, so we avoid re-walking the DOM and re-flushing layout within one tick.
|
||||
// Per-tick target cache: the picker's 16-attempt retry re-queries every tick, so
|
||||
// we avoid re-walking the DOM and re-flushing layout within one tick.
|
||||
// installRuntime resets it before each __sanderlingNextAction__ invocation.
|
||||
const candidateCache = new Map<BuiltinVerb, Candidate[]>();
|
||||
let cachedTargets: TargetElement[] | null = null;
|
||||
|
||||
function resetCandidateCache(): void {
|
||||
candidateCache.clear();
|
||||
function resetTargetCache(): void {
|
||||
cachedTargets = null;
|
||||
}
|
||||
|
||||
const host: Host = {
|
||||
@@ -546,28 +543,9 @@ const host: Host = {
|
||||
seedHi: () => SEED_HI,
|
||||
// lo = 0 matches the goja side's rand.NewPCG(seed, 0).
|
||||
seedLo: () => 0n,
|
||||
queryCandidates(verb: BuiltinVerb): Candidate[] {
|
||||
const cached = candidateCache.get(verb);
|
||||
if (cached) return cached;
|
||||
let candidates: Candidate[];
|
||||
switch (verb) {
|
||||
case "taps":
|
||||
case "doubleTaps":
|
||||
case "longPresses":
|
||||
candidates = tappableCandidates();
|
||||
break;
|
||||
case "typing":
|
||||
candidates = editableCandidates();
|
||||
break;
|
||||
case "swipes":
|
||||
case "scrolls":
|
||||
candidates = scrollableCandidates();
|
||||
break;
|
||||
default:
|
||||
candidates = [];
|
||||
}
|
||||
candidateCache.set(verb, candidates);
|
||||
return candidates;
|
||||
queryTargets(): TargetElement[] {
|
||||
if (!cachedTargets) cachedTargets = collectTargets();
|
||||
return cachedTargets;
|
||||
},
|
||||
reportUnsupported(verb: BuiltinVerb): void {
|
||||
console.warn(`[sanderling] verb ${verb} is unsupported on web`);
|
||||
@@ -578,11 +556,11 @@ const host: Host = {
|
||||
// this module (the web bundle imports the runtime first). Resolve the root
|
||||
// lazily so installRuntime captures it once the spec has evaluated. The root
|
||||
// resolver runs once per __sanderlingNextAction__ tick (before the retry loop),
|
||||
// so it is also where we reset the per-tick candidate cache.
|
||||
// so it is also where we reset the per-tick target cache.
|
||||
installRuntime(
|
||||
host,
|
||||
() => {
|
||||
resetCandidateCache();
|
||||
resetTargetCache();
|
||||
return (globalThis as { actions?: import("./action-tree.ts").GeneratorNode }).actions ?? null;
|
||||
},
|
||||
evaluateExtractors,
|
||||
@@ -593,10 +571,10 @@ installRuntime(
|
||||
export const __testing__ = {
|
||||
host,
|
||||
seedBigInt,
|
||||
tappableCandidates,
|
||||
editableCandidates,
|
||||
scrollableCandidates,
|
||||
resetCandidateCache,
|
||||
collectTargets,
|
||||
TAPPABLE_SELECTOR,
|
||||
EDITABLE_SELECTOR,
|
||||
resetTargetCache,
|
||||
runtime,
|
||||
extractors,
|
||||
evaluateExtractors,
|
||||
|
||||
Reference in new issue
Block a user