mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
fix(trace): an action names the generator that produced it
The setup exclusion landed for the model arm only, because only a model pick stamped a source. A seeded run returned setup's action through the same entry with no marker, so its denominator still counted the login while the model arm's did not, and the two are compared. serializeAction names setup and seeded on the wire, so both arms are counted by one rule. An already-recorded trace names nothing and keeps exactly the count it was reported with; unattributed_actions counts those steps so the old denominator cannot pass as the new one. TraceVersion is deliberately unbumped: oracle-reduction refuses a differing version, and a bump would make all 169 recorded runs unreplayable.
This commit is contained in:
1 parent
66fd5bce5d
commit
454988fbc8
19 files changed
+377
-82
No files matched your search
@@ -18,7 +18,9 @@ import type { Point } from "./types.ts";
|
||||
// (ONE decoder on each side). Builtin targets are already resolved to a Point,
|
||||
// and serializeAction collapses author targets that carry {x, y}; a target that
|
||||
// does not resolve to coordinates drops the action (returns null).
|
||||
export type SerializedAction =
|
||||
export type SerializedAction = SerializedActionShape & { source?: ActionSource };
|
||||
|
||||
type SerializedActionShape =
|
||||
| { kind: "Tap" | "DoubleTap" | "LongPress"; x: number; y: number; selector?: string }
|
||||
| { kind: "InputText"; x: number; y: number; text: string; selector?: string }
|
||||
| { kind: "Swipe"; fromX: number; fromY: number; toX: number; toY: number; durationMillis: number }
|
||||
@@ -35,6 +37,12 @@ export type SerializedAction =
|
||||
| { kind: "PressKey"; key: string }
|
||||
| { kind: "Wait"; durationMillis: number };
|
||||
|
||||
// ActionSource names which of the spec's two generators produced an action.
|
||||
// Both are dispatched the same way, but only the action root explores: setup
|
||||
// drives the app into its starting position, so a per-action rate that counts
|
||||
// its login steps measures a policy's exposure as larger than it was.
|
||||
export type ActionSource = "setup" | "seeded";
|
||||
|
||||
const DEFAULT_SWIPE_DURATION = 250;
|
||||
|
||||
// pointOf resolves a target to {x, y, selector?}. Builtins and resolved ax
|
||||
@@ -59,7 +67,19 @@ function pointOf(target: unknown): (Point & { selector?: string }) | undefined {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function serializeAction(action: ActionDescriptor | null): SerializedAction | null {
|
||||
// serializeAction emits the wire shape, tagged with the generator that produced
|
||||
// the action when the caller knows it. The candidate enumeration the model
|
||||
// policy reads passes no source: nothing has chosen those yet.
|
||||
export function serializeAction(
|
||||
action: ActionDescriptor | null,
|
||||
source?: ActionSource,
|
||||
): SerializedAction | null {
|
||||
const wire = serializeWire(action);
|
||||
if (!wire || !source) return wire;
|
||||
return { ...wire, source };
|
||||
}
|
||||
|
||||
function serializeWire(action: ActionDescriptor | null): SerializedAction | null {
|
||||
if (!action) return null;
|
||||
switch (action.kind) {
|
||||
case "Tap":
|
||||
@@ -184,7 +204,7 @@ export function installRuntime(
|
||||
resolveRoot();
|
||||
const setup = resolveSetup();
|
||||
if (!setup) return null;
|
||||
return serializeAction(walk(setup, rng, host));
|
||||
return serializeAction(walk(setup, rng, host), "setup");
|
||||
});
|
||||
defineLockedGlobal("__sanderlingNextAction__", () => {
|
||||
// resolveRoot runs first: on web it also resets the per-tick candidate
|
||||
@@ -193,10 +213,10 @@ export function installRuntime(
|
||||
const setup = resolveSetup();
|
||||
if (setup) {
|
||||
const setupAction = walk(setup, rng, host);
|
||||
if (setupAction) return serializeAction(setupAction);
|
||||
if (setupAction) return serializeAction(setupAction, "setup");
|
||||
}
|
||||
if (!current) return null;
|
||||
return serializeAction(nextAction(current, rng, host));
|
||||
return serializeAction(nextAction(current, rng, host), "seeded");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+40
-20
@@ -3,125 +3,145 @@
|
||||
"kind": "Tap",
|
||||
"x": 250,
|
||||
"y": 260,
|
||||
"selector": "id:gamma"
|
||||
"selector": "id:gamma",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "Tap",
|
||||
"x": 150,
|
||||
"y": 160,
|
||||
"selector": "id:beta"
|
||||
"selector": "id:beta",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "Tap",
|
||||
"x": 50,
|
||||
"y": 60,
|
||||
"selector": "id:alpha"
|
||||
"selector": "id:alpha",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "InputText",
|
||||
"x": 150,
|
||||
"y": 160,
|
||||
"text": "a",
|
||||
"selector": "id:beta"
|
||||
"selector": "id:beta",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "InputText",
|
||||
"x": 50,
|
||||
"y": 60,
|
||||
"text": "\t\n",
|
||||
"selector": "id:alpha"
|
||||
"selector": "id:alpha",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "Tap",
|
||||
"x": 250,
|
||||
"y": 260,
|
||||
"selector": "id:gamma"
|
||||
"selector": "id:gamma",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "Tap",
|
||||
"x": 150,
|
||||
"y": 160,
|
||||
"selector": "id:beta"
|
||||
"selector": "id:beta",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "Tap",
|
||||
"x": 250,
|
||||
"y": 260,
|
||||
"selector": "id:gamma"
|
||||
"selector": "id:gamma",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "Tap",
|
||||
"x": 150,
|
||||
"y": 160,
|
||||
"selector": "id:beta"
|
||||
"selector": "id:beta",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "Tap",
|
||||
"x": 250,
|
||||
"y": 260,
|
||||
"selector": "id:gamma"
|
||||
"selector": "id:gamma",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "Tap",
|
||||
"x": 250,
|
||||
"y": 260,
|
||||
"selector": "id:gamma"
|
||||
"selector": "id:gamma",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "Tap",
|
||||
"x": 150,
|
||||
"y": 160,
|
||||
"selector": "id:beta"
|
||||
"selector": "id:beta",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "Tap",
|
||||
"x": 250,
|
||||
"y": 260,
|
||||
"selector": "id:gamma"
|
||||
"selector": "id:gamma",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "Tap",
|
||||
"x": 50,
|
||||
"y": 60,
|
||||
"selector": "id:alpha"
|
||||
"selector": "id:alpha",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "InputText",
|
||||
"x": 150,
|
||||
"y": 160,
|
||||
"text": "999999999999999999999",
|
||||
"selector": "id:beta"
|
||||
"selector": "id:beta",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "InputText",
|
||||
"x": 150,
|
||||
"y": 160,
|
||||
"text": "-1",
|
||||
"selector": "id:beta"
|
||||
"selector": "id:beta",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "InputText",
|
||||
"x": 250,
|
||||
"y": 260,
|
||||
"text": "-1",
|
||||
"selector": "id:gamma"
|
||||
"selector": "id:gamma",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "Tap",
|
||||
"x": 50,
|
||||
"y": 60,
|
||||
"selector": "id:alpha"
|
||||
"selector": "id:alpha",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "Tap",
|
||||
"x": 250,
|
||||
"y": 260,
|
||||
"selector": "id:gamma"
|
||||
"selector": "id:gamma",
|
||||
"source": "seeded"
|
||||
},
|
||||
{
|
||||
"kind": "Tap",
|
||||
"x": 50,
|
||||
"y": 60,
|
||||
"selector": "id:alpha"
|
||||
"selector": "id:alpha",
|
||||
"source": "seeded"
|
||||
}
|
||||
]
|
||||
@@ -45,12 +45,14 @@ const HOST: Host = {
|
||||
|
||||
// runParity emits the parity scenario's action stream from a fresh Pcg. It
|
||||
// drives pick.ts directly (not installRuntime, whose globals are locked once)
|
||||
// so the caller can run it repeatedly to assert determinism.
|
||||
// so the caller can run it repeatedly to assert determinism. The scenario has
|
||||
// no setup, so it stands in for the entry's action-root branch and tags what it
|
||||
// emits the way that branch does.
|
||||
export function runParity(): (SerializedAction | null)[] {
|
||||
const rng = new Pcg(PARITY_SEED_HI, 0n);
|
||||
const stream: (SerializedAction | null)[] = [];
|
||||
for (let i = 0; i < PARITY_STEPS; i++) {
|
||||
stream.push(serializeAction(nextAction(PARITY_ROOT, rng, HOST)));
|
||||
stream.push(serializeAction(nextAction(PARITY_ROOT, rng, HOST), "seeded"));
|
||||
}
|
||||
return stream;
|
||||
}
|
||||
@@ -1129,3 +1129,31 @@ test("a nested undefined leaves the page as a dropped key, a nested null does no
|
||||
});
|
||||
assert.equal(wire, `{"0":{"value":{"empty":null,"present":1}}}`);
|
||||
});
|
||||
|
||||
// The web half of the cross-host marker contract: for the same spec shape, the
|
||||
// entry must name setup's action and the action root's differently, and by the
|
||||
// same two names the goja engine uses (TestNextActionNamesTheGeneratorThatProducedIt
|
||||
// in internal/verifier/setup_action_test.go asserts them there). A per-action
|
||||
// rate divides by the root's steps only, so an action that names no producer
|
||||
// puts a login's taps in the denominator of the policy's exploration.
|
||||
const { Tap, actions, taps } = await import("../src/actions.ts");
|
||||
|
||||
test("the next-action entry names the generator each action came from", () => {
|
||||
const button = fakeElement({
|
||||
tag: "button", x: 0, y: 0, width: 40, height: 20, id: "SignIn", clickable: true,
|
||||
});
|
||||
const g = globalThis as { actions?: unknown; setup?: unknown; __sanderlingNextAction__?: unknown };
|
||||
const nextAction = g.__sanderlingNextAction__ as () => { source?: string } | null;
|
||||
withFakeDocument([button], () => {
|
||||
try {
|
||||
g.actions = taps;
|
||||
g.setup = actions(() => [Tap({ on: "id:SignIn" })]);
|
||||
assert.equal(nextAction()?.source, "setup");
|
||||
g.setup = undefined;
|
||||
assert.equal(nextAction()?.source, "seeded");
|
||||
} finally {
|
||||
g.actions = undefined;
|
||||
g.setup = undefined;
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user