diff --git a/pkg/spec/src/sampler-rng.ts b/pkg/spec/src/sampler-rng.ts index 29e29e5..a894c55 100644 --- a/pkg/spec/src/sampler-rng.ts +++ b/pkg/spec/src/sampler-rng.ts @@ -26,8 +26,8 @@ export function setEnumeratingCandidates(enumerating: boolean): void { enumeratingCandidates = enumerating; } -// SAMPLER_REFUSAL_NAME marks the refusal below so the Go host can tell it from -// any other error a spec's generator throws, which it still tolerates by +// SAMPLER_REFUSAL_NAME marks the refusals below so the Go host can tell them +// from any other error a spec's generator throws, which it still tolerates by // skipping the leaf. export const SAMPLER_REFUSAL_NAME = "SanderlingSamplerRefusal"; @@ -37,11 +37,28 @@ export const SAMPLER_REFUSAL_NAME = "SanderlingSamplerRefusal"; // then be measuring the sampler rather than the policies. export function refuseWhileEnumerating(itemCount: number): void { if (!enumeratingCandidates) return; - const refusal = new Error( + refuse( `draws 1 of ${itemCount} sampled items, which only the seeded picker can do: ` + "the model policy would be offered the first item every time. " + "Return one action per item instead of calling generate().", ); +} + +// refuseValueWhileEnumerating is the same refusal for the values.ts generators, +// whose span is a range rather than a list of actions. Collapsing there is +// quieter and worse: the action space still matches, so both arms look like the +// same experiment while the model types one fixed value on every step. +export function refuseValueWhileEnumerating(generator: string): void { + if (!enumeratingCandidates) return; + refuse( + `draws a random value from ${generator}, which only the seeded picker can do: ` + + "the model policy would be handed the same value on every step. " + + "Use a fixed value instead of calling generate().", + ); +} + +function refuse(message: string): never { + const refusal = new Error(message); refusal.name = SAMPLER_REFUSAL_NAME; throw refusal; } diff --git a/pkg/spec/src/values.ts b/pkg/spec/src/values.ts index 818c386..34a6183 100644 --- a/pkg/spec/src/values.ts +++ b/pkg/spec/src/values.ts @@ -7,10 +7,15 @@ // and the V8 web runtime. Outside an actions() walk samplerRng is null and each // generator returns a fixed deterministic default (mirroring from()'s index 0), // so module-load-time calls never throw and never use an unseeded source. +// +// The model policy is the one caller that must not take that default: it walks +// the authored leaves on every step, outside the rng, so the default would be +// the value it types forever while the seeded arm varies it. A generator that +// spans more than one value refuses there instead. import type { Pcg } from "./pcg.ts"; import type { Sampler } from "./types.ts"; -import { getSamplerRng } from "./sampler-rng.ts"; +import { getSamplerRng, refuseValueWhileEnumerating } from "./sampler-rng.ts"; import { INPUT_CORPUS } from "./corpus.ts"; const ALPHA = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; @@ -37,7 +42,17 @@ class StringBuilder implements Sampler { return this; } + // A max below the min never reaches the draw, so the count is minLength, and + // a count below zero emits nothing: length(-2, 0) is the empty string either + // way, and both policies agree on it. + private spansMoreThanOneString(): boolean { + const shortest = Math.max(this.minLength, 0); + const longest = Math.max(this.maxLength, this.minLength, 0); + return longest > shortest || (longest > 0 && this.charset.length > 1); + } + generate(): string { + if (this.spansMoreThanOneString()) refuseValueWhileEnumerating("strings()"); const rng = getSamplerRng(); const span = this.maxLength - this.minLength + 1; const count = this.minLength + draw(rng, span); @@ -62,6 +77,7 @@ class IntegerBuilder implements Sampler { generate(): number { const rng = getSamplerRng(); const span = this.maxValue - this.minValue + 1; + if (span > 1) refuseValueWhileEnumerating("integers()"); return this.minValue + draw(rng, span); } } @@ -74,7 +90,10 @@ class EmailBuilder implements Sampler { return this; } + // The local part always spans 3 to 8 alphabetic characters and only the host + // is author-set, so an address is never a single value. generate(): string { + refuseValueWhileEnumerating("emails()"); const local = new StringBuilder().length(3, 8).alpha().generate(); return `${local}@${this.host}`; } @@ -82,6 +101,7 @@ class EmailBuilder implements Sampler { class EdgeCaseTextBuilder implements Sampler { generate(): string { + if (INPUT_CORPUS.length > 1) refuseValueWhileEnumerating("edgeCaseText()"); const rng = getSamplerRng(); return INPUT_CORPUS[draw(rng, INPUT_CORPUS.length)] ?? ""; } diff --git a/pkg/spec/test/values.test.ts b/pkg/spec/test/values.test.ts index 153c620..cdbf928 100644 --- a/pkg/spec/test/values.test.ts +++ b/pkg/spec/test/values.test.ts @@ -1,7 +1,11 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { Pcg } from "../src/pcg.ts"; -import { setSamplerRng } from "../src/sampler-rng.ts"; +import { + SAMPLER_REFUSAL_NAME, + setEnumeratingCandidates, + setSamplerRng, +} from "../src/sampler-rng.ts"; import { edgeCaseText, emails, integers, strings } from "../src/values.ts"; import { INPUT_CORPUS } from "../src/corpus.ts"; import type { Sampler } from "../src/types.ts"; @@ -73,3 +77,58 @@ test("outside a walk generators return a fixed deterministic default", () => { assert.equal(emails().domain("folio.app").generate(), "aaa@folio.app"); assert.equal(edgeCaseText().generate(), INPUT_CORPUS[0]); }); + +function whileEnumerating(body: () => T): T { + setEnumeratingCandidates(true); + try { + return body(); + } finally { + setEnumeratingCandidates(false); + } +} + +test("every value generator refuses a multi-value draw while the model policy enumerates", () => { + whileEnumerating(() => { + assert.throws(() => integers().between(1, 500).generate(), { + name: SAMPLER_REFUSAL_NAME, + message: /random value from integers\(\)/, + }); + assert.throws(() => strings().length(3, 6).alpha().generate(), { + name: SAMPLER_REFUSAL_NAME, + message: /random value from strings\(\)/, + }); + assert.throws(() => emails().domain("folio.app").generate(), { + name: SAMPLER_REFUSAL_NAME, + message: /random value from emails\(\)/, + }); + assert.throws(() => edgeCaseText().generate(), { + name: SAMPLER_REFUSAL_NAME, + message: /random value from edgeCaseText\(\)/, + }); + }); +}); + +test("a fixed-length string still refuses, because its characters vary", () => { + whileEnumerating(() => { + assert.throws(() => strings().length(4, 4).alpha().generate(), { + name: SAMPLER_REFUSAL_NAME, + }); + }); +}); + +test("a single-valued generator keeps drawing while the model policy enumerates", () => { + whileEnumerating(() => { + assert.equal(integers().between(7, 7).generate(), 7); + assert.equal(strings().length(0, 0).generate(), ""); + }); +}); + +test("a refused generator draws again once enumeration ends", () => { + whileEnumerating(() => { + assert.throws(() => integers().between(1, 500).generate()); + }); + withRng(5n, () => { + const value = integers().between(1, 500).generate(); + assert.ok(value >= 1 && value <= 500); + }); +});