diff --git a/pkg/spec/src/actions.ts b/pkg/spec/src/actions.ts index 317fe16..2154d1d 100644 --- a/pkg/spec/src/actions.ts +++ b/pkg/spec/src/actions.ts @@ -22,7 +22,7 @@ import type { WeightedEntry, } from "./types.ts"; import type { ActionDescriptor, BuiltinVerb, GeneratorNode } from "./action-tree.ts"; -import { getSamplerRng } from "./sampler-rng.ts"; +import { getSamplerRng, refuseWhileEnumerating } from "./sampler-rng.ts"; export { setSamplerRng } from "./sampler-rng.ts"; @@ -67,6 +67,7 @@ export function from(items: readonly T[]): Sampler { return { generate(): T { if (items.length <= 1) return items[0] as T; + refuseWhileEnumerating(items.length); const rng = getSamplerRng(); const index = rng ? rng.intN(items.length) : 0; return items[index] as T; diff --git a/pkg/spec/src/runtime-entry.ts b/pkg/spec/src/runtime-entry.ts index c1f0f87..a14bba1 100644 --- a/pkg/spec/src/runtime-entry.ts +++ b/pkg/spec/src/runtime-entry.ts @@ -10,6 +10,7 @@ import { Pcg } from "./pcg.ts"; import { builtinCandidates, nextAction, walk } from "./pick.ts"; import { INPUT_CORPUS } from "./corpus.ts"; +import { setEnumeratingCandidates } from "./sampler-rng.ts"; import type { ActionDescriptor, BuiltinVerb, GeneratorNode, Host } from "./action-tree.ts"; import type { Point } from "./types.ts"; @@ -156,6 +157,10 @@ export function installRuntime( targetIndex: candidate.targetIndex, })), ); + // The model policy calls the authored leaves itself, from Go, outside the + // picker's rng scope. It brackets those calls with this so a multi-item + // sampler refuses rather than handing back its first item forever. + defineLockedGlobal("__sanderlingSetEnumeratingCandidates__", setEnumeratingCandidates); defineLockedGlobal("__sanderlingExtractors__", () => evaluateExtractors()); // __sanderlingSetupAction__ walks ONLY the setup generator once, for the LLM // action generator (Go), which drives selection itself and must not run the diff --git a/pkg/spec/src/sampler-rng.ts b/pkg/spec/src/sampler-rng.ts index 0924dbe..29e29e5 100644 --- a/pkg/spec/src/sampler-rng.ts +++ b/pkg/spec/src/sampler-rng.ts @@ -15,3 +15,33 @@ export function setSamplerRng(rng: Pcg | null): void { export function getSamplerRng(): Pcg | null { return samplerRng; } + +// enumeratingCandidates is set while the Go host enumerates the authored leaves +// for the model policy (internal/verifier/llm.go collectActions). That walk runs +// outside the picker's rng scope, so a draw there would silently collapse to +// item 0. +let enumeratingCandidates = false; + +export function setEnumeratingCandidates(enumerating: boolean): void { + enumeratingCandidates = enumerating; +} + +// SAMPLER_REFUSAL_NAME marks the refusal below so the Go host can tell it from +// any other error a spec's generator throws, which it still tolerates by +// skipping the leaf. +export const SAMPLER_REFUSAL_NAME = "SanderlingSamplerRefusal"; + +// refuseWhileEnumerating stops a draw the model policy cannot make. Collapsing +// to item 0 instead would offer the model one fixed target while the seeded +// picker reaches every item, and a comparison between the two policies would +// then be measuring the sampler rather than the policies. +export function refuseWhileEnumerating(itemCount: number): void { + if (!enumeratingCandidates) return; + const refusal = new Error( + `draws 1 of ${itemCount} sampled items, which only the seeded picker can do: ` + + "the model policy would be offered the first item every time. " + + "Return one action per item instead of calling generate().", + ); + refusal.name = SAMPLER_REFUSAL_NAME; + throw refusal; +} diff --git a/pkg/spec/test/api.test.ts b/pkg/spec/test/api.test.ts index eb38207..60647fd 100644 --- a/pkg/spec/test/api.test.ts +++ b/pkg/spec/test/api.test.ts @@ -30,6 +30,7 @@ import { whenRoute, } from "../src/index.ts"; import { setSamplerRng } from "../src/actions.ts"; +import { SAMPLER_REFUSAL_NAME, setEnumeratingCandidates } from "../src/sampler-rng.ts"; import { Pcg } from "../src/pcg.ts"; import type { GeneratorNode } from "../src/action-tree.ts"; import type { @@ -332,6 +333,27 @@ test("from falls back to the first item outside a picker walk", () => { assert.equal(from(["a", "b", "c"]).generate(), "a"); }); +test("from refuses a multi-item draw while the model policy enumerates", () => { + setEnumeratingCandidates(true); + try { + assert.throws(() => from(["a", "b", "c"]).generate(), { + name: SAMPLER_REFUSAL_NAME, + message: /draws 1 of 3 sampled items/, + }); + } finally { + setEnumeratingCandidates(false); + } +}); + +test("from keeps serving a single item while the model policy enumerates", () => { + setEnumeratingCandidates(true); + try { + assert.equal(from(["only"]).generate(), "only"); + } finally { + setEnumeratingCandidates(false); + } +}); + function elementWithChildren(cells: Record): AccessibilityElement { return { find: selector => {