From 0a71e055885516969f8ab889145fe7efcadf1b2f Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 13 Aug 2026 00:58:17 +0530 Subject: [PATCH] feat(spec): refuse a multi-value generator while enumerating integers, strings, emails and edgeCaseText read the same rng from() does, so under the model policy an authored InputText typed the same value on every step while the seeded arm varied it. That is a silently different experiment, not just a silently different action space. Single-valued spans are exempt, because both policies then get the same value: between(7,7), a zero-length string, and a one-entry corpus. length(4,4) is still refused, since the length is pinned but each character is drawn from 62. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX --- pkg/spec/src/sampler-rng.ts | 23 ++++++++++++-- pkg/spec/src/values.ts | 22 ++++++++++++- pkg/spec/test/values.test.ts | 61 +++++++++++++++++++++++++++++++++++- 3 files changed, 101 insertions(+), 5 deletions(-) diff --git a/pkg/spec/src/sampler-rng.ts b/pkg/spec/src/sampler-rng.ts index 29e29e5..a894c55 100644 --- a/pkg/spec/src/sampler-rng.ts +++ b/pkg/spec/src/sampler-rng.ts @@ -26,8 +26,8 @@ export function setEnumeratingCandidates(enumerating: boolean): void { enumeratingCandidates = enumerating; } -// SAMPLER_REFUSAL_NAME marks the refusal below so the Go host can tell it from -// any other error a spec's generator throws, which it still tolerates by +// SAMPLER_REFUSAL_NAME marks the refusals below so the Go host can tell them +// from any other error a spec's generator throws, which it still tolerates by // skipping the leaf. export const SAMPLER_REFUSAL_NAME = "SanderlingSamplerRefusal"; @@ -37,11 +37,28 @@ export const SAMPLER_REFUSAL_NAME = "SanderlingSamplerRefusal"; // then be measuring the sampler rather than the policies. export function refuseWhileEnumerating(itemCount: number): void { if (!enumeratingCandidates) return; - const refusal = new Error( + refuse( `draws 1 of ${itemCount} sampled items, which only the seeded picker can do: ` + "the model policy would be offered the first item every time. " + "Return one action per item instead of calling generate().", ); +} + +// refuseValueWhileEnumerating is the same refusal for the values.ts generators, +// whose span is a range rather than a list of actions. Collapsing there is +// quieter and worse: the action space still matches, so both arms look like the +// same experiment while the model types one fixed value on every step. +export function refuseValueWhileEnumerating(generator: string): void { + if (!enumeratingCandidates) return; + refuse( + `draws a random value from ${generator}, which only the seeded picker can do: ` + + "the model policy would be handed the same value on every step. " + + "Use a fixed value instead of calling generate().", + ); +} + +function refuse(message: string): never { + const refusal = new Error(message); refusal.name = SAMPLER_REFUSAL_NAME; throw refusal; } diff --git a/pkg/spec/src/values.ts b/pkg/spec/src/values.ts index 818c386..34a6183 100644 --- a/pkg/spec/src/values.ts +++ b/pkg/spec/src/values.ts @@ -7,10 +7,15 @@ // and the V8 web runtime. Outside an actions() walk samplerRng is null and each // generator returns a fixed deterministic default (mirroring from()'s index 0), // so module-load-time calls never throw and never use an unseeded source. +// +// The model policy is the one caller that must not take that default: it walks +// the authored leaves on every step, outside the rng, so the default would be +// the value it types forever while the seeded arm varies it. A generator that +// spans more than one value refuses there instead. import type { Pcg } from "./pcg.ts"; import type { Sampler } from "./types.ts"; -import { getSamplerRng } from "./sampler-rng.ts"; +import { getSamplerRng, refuseValueWhileEnumerating } from "./sampler-rng.ts"; import { INPUT_CORPUS } from "./corpus.ts"; const ALPHA = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; @@ -37,7 +42,17 @@ class StringBuilder implements Sampler { return this; } + // A max below the min never reaches the draw, so the count is minLength, and + // a count below zero emits nothing: length(-2, 0) is the empty string either + // way, and both policies agree on it. + private spansMoreThanOneString(): boolean { + const shortest = Math.max(this.minLength, 0); + const longest = Math.max(this.maxLength, this.minLength, 0); + return longest > shortest || (longest > 0 && this.charset.length > 1); + } + generate(): string { + if (this.spansMoreThanOneString()) refuseValueWhileEnumerating("strings()"); const rng = getSamplerRng(); const span = this.maxLength - this.minLength + 1; const count = this.minLength + draw(rng, span); @@ -62,6 +77,7 @@ class IntegerBuilder implements Sampler { generate(): number { const rng = getSamplerRng(); const span = this.maxValue - this.minValue + 1; + if (span > 1) refuseValueWhileEnumerating("integers()"); return this.minValue + draw(rng, span); } } @@ -74,7 +90,10 @@ class EmailBuilder implements Sampler { return this; } + // The local part always spans 3 to 8 alphabetic characters and only the host + // is author-set, so an address is never a single value. generate(): string { + refuseValueWhileEnumerating("emails()"); const local = new StringBuilder().length(3, 8).alpha().generate(); return `${local}@${this.host}`; } @@ -82,6 +101,7 @@ class EmailBuilder implements Sampler { class EdgeCaseTextBuilder implements Sampler { generate(): string { + if (INPUT_CORPUS.length > 1) refuseValueWhileEnumerating("edgeCaseText()"); const rng = getSamplerRng(); return INPUT_CORPUS[draw(rng, INPUT_CORPUS.length)] ?? ""; } diff --git a/pkg/spec/test/values.test.ts b/pkg/spec/test/values.test.ts index 153c620..cdbf928 100644 --- a/pkg/spec/test/values.test.ts +++ b/pkg/spec/test/values.test.ts @@ -1,7 +1,11 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { Pcg } from "../src/pcg.ts"; -import { setSamplerRng } from "../src/sampler-rng.ts"; +import { + SAMPLER_REFUSAL_NAME, + setEnumeratingCandidates, + setSamplerRng, +} from "../src/sampler-rng.ts"; import { edgeCaseText, emails, integers, strings } from "../src/values.ts"; import { INPUT_CORPUS } from "../src/corpus.ts"; import type { Sampler } from "../src/types.ts"; @@ -73,3 +77,58 @@ test("outside a walk generators return a fixed deterministic default", () => { assert.equal(emails().domain("folio.app").generate(), "aaa@folio.app"); assert.equal(edgeCaseText().generate(), INPUT_CORPUS[0]); }); + +function whileEnumerating(body: () => T): T { + setEnumeratingCandidates(true); + try { + return body(); + } finally { + setEnumeratingCandidates(false); + } +} + +test("every value generator refuses a multi-value draw while the model policy enumerates", () => { + whileEnumerating(() => { + assert.throws(() => integers().between(1, 500).generate(), { + name: SAMPLER_REFUSAL_NAME, + message: /random value from integers\(\)/, + }); + assert.throws(() => strings().length(3, 6).alpha().generate(), { + name: SAMPLER_REFUSAL_NAME, + message: /random value from strings\(\)/, + }); + assert.throws(() => emails().domain("folio.app").generate(), { + name: SAMPLER_REFUSAL_NAME, + message: /random value from emails\(\)/, + }); + assert.throws(() => edgeCaseText().generate(), { + name: SAMPLER_REFUSAL_NAME, + message: /random value from edgeCaseText\(\)/, + }); + }); +}); + +test("a fixed-length string still refuses, because its characters vary", () => { + whileEnumerating(() => { + assert.throws(() => strings().length(4, 4).alpha().generate(), { + name: SAMPLER_REFUSAL_NAME, + }); + }); +}); + +test("a single-valued generator keeps drawing while the model policy enumerates", () => { + whileEnumerating(() => { + assert.equal(integers().between(7, 7).generate(), 7); + assert.equal(strings().length(0, 0).generate(), ""); + }); +}); + +test("a refused generator draws again once enumeration ends", () => { + whileEnumerating(() => { + assert.throws(() => integers().between(1, 500).generate()); + }); + withRng(5n, () => { + const value = integers().between(1, 500).generate(); + assert.ok(value >= 1 && value <= 500); + }); +});