feat(spec): refuse a multi-value generator while enumerating

integers, strings, emails and edgeCaseText read the same rng from() does, so
under the model policy an authored InputText typed the same value on every
step while the seeded arm varied it. That is a silently different experiment,
not just a silently different action space.

Single-valued spans are exempt, because both policies then get the same value:
between(7,7), a zero-length string, and a one-entry corpus. length(4,4) is
still refused, since the length is pinned but each character is drawn from 62.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
pj committed 2026-08-13 00:58:17 +05:30
1 parent 7880e3e70f
commit 0a71e05588
3 files changed
+101 -5

No files matched your search

+20 -3
View File
@@ -26,8 +26,8 @@ export function setEnumeratingCandidates(enumerating: boolean): void {
enumeratingCandidates = enumerating; enumeratingCandidates = enumerating;
} }
// SAMPLER_REFUSAL_NAME marks the refusal below so the Go host can tell it from // SAMPLER_REFUSAL_NAME marks the refusals below so the Go host can tell them
// any other error a spec's generator throws, which it still tolerates by // from any other error a spec's generator throws, which it still tolerates by
// skipping the leaf. // skipping the leaf.
export const SAMPLER_REFUSAL_NAME = "SanderlingSamplerRefusal"; export const SAMPLER_REFUSAL_NAME = "SanderlingSamplerRefusal";
@@ -37,11 +37,28 @@ export const SAMPLER_REFUSAL_NAME = "SanderlingSamplerRefusal";
// then be measuring the sampler rather than the policies. // then be measuring the sampler rather than the policies.
export function refuseWhileEnumerating(itemCount: number): void { export function refuseWhileEnumerating(itemCount: number): void {
if (!enumeratingCandidates) return; if (!enumeratingCandidates) return;
const refusal = new Error( refuse(
`draws 1 of ${itemCount} sampled items, which only the seeded picker can do: ` + `draws 1 of ${itemCount} sampled items, which only the seeded picker can do: ` +
"the model policy would be offered the first item every time. " + "the model policy would be offered the first item every time. " +
"Return one action per item instead of calling generate().", "Return one action per item instead of calling generate().",
); );
}
// refuseValueWhileEnumerating is the same refusal for the values.ts generators,
// whose span is a range rather than a list of actions. Collapsing there is
// quieter and worse: the action space still matches, so both arms look like the
// same experiment while the model types one fixed value on every step.
export function refuseValueWhileEnumerating(generator: string): void {
if (!enumeratingCandidates) return;
refuse(
`draws a random value from ${generator}, which only the seeded picker can do: ` +
"the model policy would be handed the same value on every step. " +
"Use a fixed value instead of calling generate().",
);
}
function refuse(message: string): never {
const refusal = new Error(message);
refusal.name = SAMPLER_REFUSAL_NAME; refusal.name = SAMPLER_REFUSAL_NAME;
throw refusal; throw refusal;
} }
+21 -1
View File
@@ -7,10 +7,15 @@
// and the V8 web runtime. Outside an actions() walk samplerRng is null and each // and the V8 web runtime. Outside an actions() walk samplerRng is null and each
// generator returns a fixed deterministic default (mirroring from()'s index 0), // generator returns a fixed deterministic default (mirroring from()'s index 0),
// so module-load-time calls never throw and never use an unseeded source. // so module-load-time calls never throw and never use an unseeded source.
//
// The model policy is the one caller that must not take that default: it walks
// the authored leaves on every step, outside the rng, so the default would be
// the value it types forever while the seeded arm varies it. A generator that
// spans more than one value refuses there instead.
import type { Pcg } from "./pcg.ts"; import type { Pcg } from "./pcg.ts";
import type { Sampler } from "./types.ts"; import type { Sampler } from "./types.ts";
import { getSamplerRng } from "./sampler-rng.ts"; import { getSamplerRng, refuseValueWhileEnumerating } from "./sampler-rng.ts";
import { INPUT_CORPUS } from "./corpus.ts"; import { INPUT_CORPUS } from "./corpus.ts";
const ALPHA = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; const ALPHA = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
@@ -37,7 +42,17 @@ class StringBuilder implements Sampler<string> {
return this; return this;
} }
// A max below the min never reaches the draw, so the count is minLength, and
// a count below zero emits nothing: length(-2, 0) is the empty string either
// way, and both policies agree on it.
private spansMoreThanOneString(): boolean {
const shortest = Math.max(this.minLength, 0);
const longest = Math.max(this.maxLength, this.minLength, 0);
return longest > shortest || (longest > 0 && this.charset.length > 1);
}
generate(): string { generate(): string {
if (this.spansMoreThanOneString()) refuseValueWhileEnumerating("strings()");
const rng = getSamplerRng(); const rng = getSamplerRng();
const span = this.maxLength - this.minLength + 1; const span = this.maxLength - this.minLength + 1;
const count = this.minLength + draw(rng, span); const count = this.minLength + draw(rng, span);
@@ -62,6 +77,7 @@ class IntegerBuilder implements Sampler<number> {
generate(): number { generate(): number {
const rng = getSamplerRng(); const rng = getSamplerRng();
const span = this.maxValue - this.minValue + 1; const span = this.maxValue - this.minValue + 1;
if (span > 1) refuseValueWhileEnumerating("integers()");
return this.minValue + draw(rng, span); return this.minValue + draw(rng, span);
} }
} }
@@ -74,7 +90,10 @@ class EmailBuilder implements Sampler<string> {
return this; return this;
} }
// The local part always spans 3 to 8 alphabetic characters and only the host
// is author-set, so an address is never a single value.
generate(): string { generate(): string {
refuseValueWhileEnumerating("emails()");
const local = new StringBuilder().length(3, 8).alpha().generate(); const local = new StringBuilder().length(3, 8).alpha().generate();
return `${local}@${this.host}`; return `${local}@${this.host}`;
} }
@@ -82,6 +101,7 @@ class EmailBuilder implements Sampler<string> {
class EdgeCaseTextBuilder implements Sampler<string> { class EdgeCaseTextBuilder implements Sampler<string> {
generate(): string { generate(): string {
if (INPUT_CORPUS.length > 1) refuseValueWhileEnumerating("edgeCaseText()");
const rng = getSamplerRng(); const rng = getSamplerRng();
return INPUT_CORPUS[draw(rng, INPUT_CORPUS.length)] ?? ""; return INPUT_CORPUS[draw(rng, INPUT_CORPUS.length)] ?? "";
} }
+60 -1
View File
@@ -1,7 +1,11 @@
import { test } from "node:test"; import { test } from "node:test";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { Pcg } from "../src/pcg.ts"; import { Pcg } from "../src/pcg.ts";
import { setSamplerRng } from "../src/sampler-rng.ts"; import {
SAMPLER_REFUSAL_NAME,
setEnumeratingCandidates,
setSamplerRng,
} from "../src/sampler-rng.ts";
import { edgeCaseText, emails, integers, strings } from "../src/values.ts"; import { edgeCaseText, emails, integers, strings } from "../src/values.ts";
import { INPUT_CORPUS } from "../src/corpus.ts"; import { INPUT_CORPUS } from "../src/corpus.ts";
import type { Sampler } from "../src/types.ts"; import type { Sampler } from "../src/types.ts";
@@ -73,3 +77,58 @@ test("outside a walk generators return a fixed deterministic default", () => {
assert.equal(emails().domain("folio.app").generate(), "[email protected]"); assert.equal(emails().domain("folio.app").generate(), "[email protected]");
assert.equal(edgeCaseText().generate(), INPUT_CORPUS[0]); assert.equal(edgeCaseText().generate(), INPUT_CORPUS[0]);
}); });
function whileEnumerating<T>(body: () => T): T {
setEnumeratingCandidates(true);
try {
return body();
} finally {
setEnumeratingCandidates(false);
}
}
test("every value generator refuses a multi-value draw while the model policy enumerates", () => {
whileEnumerating(() => {
assert.throws(() => integers().between(1, 500).generate(), {
name: SAMPLER_REFUSAL_NAME,
message: /random value from integers\(\)/,
});
assert.throws(() => strings().length(3, 6).alpha().generate(), {
name: SAMPLER_REFUSAL_NAME,
message: /random value from strings\(\)/,
});
assert.throws(() => emails().domain("folio.app").generate(), {
name: SAMPLER_REFUSAL_NAME,
message: /random value from emails\(\)/,
});
assert.throws(() => edgeCaseText().generate(), {
name: SAMPLER_REFUSAL_NAME,
message: /random value from edgeCaseText\(\)/,
});
});
});
test("a fixed-length string still refuses, because its characters vary", () => {
whileEnumerating(() => {
assert.throws(() => strings().length(4, 4).alpha().generate(), {
name: SAMPLER_REFUSAL_NAME,
});
});
});
test("a single-valued generator keeps drawing while the model policy enumerates", () => {
whileEnumerating(() => {
assert.equal(integers().between(7, 7).generate(), 7);
assert.equal(strings().length(0, 0).generate(), "");
});
});
test("a refused generator draws again once enumeration ends", () => {
whileEnumerating(() => {
assert.throws(() => integers().between(1, 500).generate());
});
withRng(5n, () => {
const value = integers().between(1, 500).generate();
assert.ok(value >= 1 && value <= 500);
});
});