feat(spec): refuse a multi-value generator while enumerating

integers, strings, emails and edgeCaseText read the same rng from() does, so
under the model policy an authored InputText typed the same value on every
step while the seeded arm varied it. That is a silently different experiment,
not just a silently different action space.

Single-valued spans are exempt, because both policies then get the same value:
between(7,7), a zero-length string, and a one-entry corpus. length(4,4) is
still refused, since the length is pinned but each character is drawn from 62.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
pj committed 2026-08-13 00:58:17 +05:30
1 parent 7880e3e70f
commit 0a71e05588
3 files changed
+101 -5

No files matched your search

+20 -3
View File
@@ -26,8 +26,8 @@ export function setEnumeratingCandidates(enumerating: boolean): void {
enumeratingCandidates = enumerating;
}
// SAMPLER_REFUSAL_NAME marks the refusal below so the Go host can tell it from
// any other error a spec's generator throws, which it still tolerates by
// SAMPLER_REFUSAL_NAME marks the refusals below so the Go host can tell them
// from any other error a spec's generator throws, which it still tolerates by
// skipping the leaf.
export const SAMPLER_REFUSAL_NAME = "SanderlingSamplerRefusal";
@@ -37,11 +37,28 @@ export const SAMPLER_REFUSAL_NAME = "SanderlingSamplerRefusal";
// then be measuring the sampler rather than the policies.
export function refuseWhileEnumerating(itemCount: number): void {
if (!enumeratingCandidates) return;
const refusal = new Error(
refuse(
`draws 1 of ${itemCount} sampled items, which only the seeded picker can do: ` +
"the model policy would be offered the first item every time. " +
"Return one action per item instead of calling generate().",
);
}
// refuseValueWhileEnumerating is the same refusal for the values.ts generators,
// whose span is a range rather than a list of actions. Collapsing there is
// quieter and worse: the action space still matches, so both arms look like the
// same experiment while the model types one fixed value on every step.
export function refuseValueWhileEnumerating(generator: string): void {
if (!enumeratingCandidates) return;
refuse(
`draws a random value from ${generator}, which only the seeded picker can do: ` +
"the model policy would be handed the same value on every step. " +
"Use a fixed value instead of calling generate().",
);
}
function refuse(message: string): never {
const refusal = new Error(message);
refusal.name = SAMPLER_REFUSAL_NAME;
throw refusal;
}