From c76ba4b497e28c1c306089ee4c92d747ec7c74e8 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 13 Aug 2026 00:44:00 +0530 Subject: [PATCH] feat(spec): refuse a multi-item authored sampler while enumerating from().generate() draws from the picker's rng, which exists only inside walkActions. The model policy enumerates authored leaves outside that walk, so the sampler silently yielded its first item on every step: measured over 30 draws the seeded arm reached three targets in roughly equal proportion and the model was offered only the first. The two policies had different action spaces and nothing said so. A single-item sampler short-circuits before the rng, so both policies get the same value and it is not refused. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX --- pkg/spec/src/actions.ts | 3 ++- pkg/spec/src/runtime-entry.ts | 5 +++++ pkg/spec/src/sampler-rng.ts | 30 ++++++++++++++++++++++++++++++ pkg/spec/test/api.test.ts | 22 ++++++++++++++++++++++ 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/pkg/spec/src/actions.ts b/pkg/spec/src/actions.ts index 317fe16..2154d1d 100644 --- a/pkg/spec/src/actions.ts +++ b/pkg/spec/src/actions.ts @@ -22,7 +22,7 @@ import type { WeightedEntry, } from "./types.ts"; import type { ActionDescriptor, BuiltinVerb, GeneratorNode } from "./action-tree.ts"; -import { getSamplerRng } from "./sampler-rng.ts"; +import { getSamplerRng, refuseWhileEnumerating } from "./sampler-rng.ts"; export { setSamplerRng } from "./sampler-rng.ts"; @@ -67,6 +67,7 @@ export function from(items: readonly T[]): Sampler { return { generate(): T { if (items.length <= 1) return items[0] as T; + refuseWhileEnumerating(items.length); const rng = getSamplerRng(); const index = rng ? rng.intN(items.length) : 0; return items[index] as T; diff --git a/pkg/spec/src/runtime-entry.ts b/pkg/spec/src/runtime-entry.ts index c1f0f87..a14bba1 100644 --- a/pkg/spec/src/runtime-entry.ts +++ b/pkg/spec/src/runtime-entry.ts @@ -10,6 +10,7 @@ import { Pcg } from "./pcg.ts"; import { builtinCandidates, nextAction, walk } from "./pick.ts"; import { INPUT_CORPUS } from "./corpus.ts"; +import { setEnumeratingCandidates } from "./sampler-rng.ts"; import type { ActionDescriptor, BuiltinVerb, GeneratorNode, Host } from "./action-tree.ts"; import type { Point } from "./types.ts"; @@ -156,6 +157,10 @@ export function installRuntime( targetIndex: candidate.targetIndex, })), ); + // The model policy calls the authored leaves itself, from Go, outside the + // picker's rng scope. It brackets those calls with this so a multi-item + // sampler refuses rather than handing back its first item forever. + defineLockedGlobal("__sanderlingSetEnumeratingCandidates__", setEnumeratingCandidates); defineLockedGlobal("__sanderlingExtractors__", () => evaluateExtractors()); // __sanderlingSetupAction__ walks ONLY the setup generator once, for the LLM // action generator (Go), which drives selection itself and must not run the diff --git a/pkg/spec/src/sampler-rng.ts b/pkg/spec/src/sampler-rng.ts index 0924dbe..29e29e5 100644 --- a/pkg/spec/src/sampler-rng.ts +++ b/pkg/spec/src/sampler-rng.ts @@ -15,3 +15,33 @@ export function setSamplerRng(rng: Pcg | null): void { export function getSamplerRng(): Pcg | null { return samplerRng; } + +// enumeratingCandidates is set while the Go host enumerates the authored leaves +// for the model policy (internal/verifier/llm.go collectActions). That walk runs +// outside the picker's rng scope, so a draw there would silently collapse to +// item 0. +let enumeratingCandidates = false; + +export function setEnumeratingCandidates(enumerating: boolean): void { + enumeratingCandidates = enumerating; +} + +// SAMPLER_REFUSAL_NAME marks the refusal below so the Go host can tell it from +// any other error a spec's generator throws, which it still tolerates by +// skipping the leaf. +export const SAMPLER_REFUSAL_NAME = "SanderlingSamplerRefusal"; + +// refuseWhileEnumerating stops a draw the model policy cannot make. Collapsing +// to item 0 instead would offer the model one fixed target while the seeded +// picker reaches every item, and a comparison between the two policies would +// then be measuring the sampler rather than the policies. +export function refuseWhileEnumerating(itemCount: number): void { + if (!enumeratingCandidates) return; + const refusal = new Error( + `draws 1 of ${itemCount} sampled items, which only the seeded picker can do: ` + + "the model policy would be offered the first item every time. " + + "Return one action per item instead of calling generate().", + ); + refusal.name = SAMPLER_REFUSAL_NAME; + throw refusal; +} diff --git a/pkg/spec/test/api.test.ts b/pkg/spec/test/api.test.ts index eb38207..60647fd 100644 --- a/pkg/spec/test/api.test.ts +++ b/pkg/spec/test/api.test.ts @@ -30,6 +30,7 @@ import { whenRoute, } from "../src/index.ts"; import { setSamplerRng } from "../src/actions.ts"; +import { SAMPLER_REFUSAL_NAME, setEnumeratingCandidates } from "../src/sampler-rng.ts"; import { Pcg } from "../src/pcg.ts"; import type { GeneratorNode } from "../src/action-tree.ts"; import type { @@ -332,6 +333,27 @@ test("from falls back to the first item outside a picker walk", () => { assert.equal(from(["a", "b", "c"]).generate(), "a"); }); +test("from refuses a multi-item draw while the model policy enumerates", () => { + setEnumeratingCandidates(true); + try { + assert.throws(() => from(["a", "b", "c"]).generate(), { + name: SAMPLER_REFUSAL_NAME, + message: /draws 1 of 3 sampled items/, + }); + } finally { + setEnumeratingCandidates(false); + } +}); + +test("from keeps serving a single item while the model policy enumerates", () => { + setEnumeratingCandidates(true); + try { + assert.equal(from(["only"]).generate(), "only"); + } finally { + setEnumeratingCandidates(false); + } +}); + function elementWithChildren(cells: Record): AccessibilityElement { return { find: selector => {