mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 11:07:10 +00:00
feat(spec): refuse a multi-value generator while enumerating
integers, strings, emails and edgeCaseText read the same rng from() does, so under the model policy an authored InputText typed the same value on every step while the seeded arm varied it. That is a silently different experiment, not just a silently different action space. Single-valued spans are exempt, because both policies then get the same value: between(7,7), a zero-length string, and a one-entry corpus. length(4,4) is still refused, since the length is pinned but each character is drawn from 62. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
7880e3e70f
commit
0a71e05588
3 files changed
+101
-5
No files matched your search
@@ -26,8 +26,8 @@ export function setEnumeratingCandidates(enumerating: boolean): void {
|
||||
enumeratingCandidates = enumerating;
|
||||
}
|
||||
|
||||
// SAMPLER_REFUSAL_NAME marks the refusal below so the Go host can tell it from
|
||||
// any other error a spec's generator throws, which it still tolerates by
|
||||
// SAMPLER_REFUSAL_NAME marks the refusals below so the Go host can tell them
|
||||
// from any other error a spec's generator throws, which it still tolerates by
|
||||
// skipping the leaf.
|
||||
export const SAMPLER_REFUSAL_NAME = "SanderlingSamplerRefusal";
|
||||
|
||||
@@ -37,11 +37,28 @@ export const SAMPLER_REFUSAL_NAME = "SanderlingSamplerRefusal";
|
||||
// then be measuring the sampler rather than the policies.
|
||||
export function refuseWhileEnumerating(itemCount: number): void {
|
||||
if (!enumeratingCandidates) return;
|
||||
const refusal = new Error(
|
||||
refuse(
|
||||
`draws 1 of ${itemCount} sampled items, which only the seeded picker can do: ` +
|
||||
"the model policy would be offered the first item every time. " +
|
||||
"Return one action per item instead of calling generate().",
|
||||
);
|
||||
}
|
||||
|
||||
// refuseValueWhileEnumerating is the same refusal for the values.ts generators,
|
||||
// whose span is a range rather than a list of actions. Collapsing there is
|
||||
// quieter and worse: the action space still matches, so both arms look like the
|
||||
// same experiment while the model types one fixed value on every step.
|
||||
export function refuseValueWhileEnumerating(generator: string): void {
|
||||
if (!enumeratingCandidates) return;
|
||||
refuse(
|
||||
`draws a random value from ${generator}, which only the seeded picker can do: ` +
|
||||
"the model policy would be handed the same value on every step. " +
|
||||
"Use a fixed value instead of calling generate().",
|
||||
);
|
||||
}
|
||||
|
||||
function refuse(message: string): never {
|
||||
const refusal = new Error(message);
|
||||
refusal.name = SAMPLER_REFUSAL_NAME;
|
||||
throw refusal;
|
||||
}
|
||||
+21
-1
@@ -7,10 +7,15 @@
|
||||
// and the V8 web runtime. Outside an actions() walk samplerRng is null and each
|
||||
// generator returns a fixed deterministic default (mirroring from()'s index 0),
|
||||
// so module-load-time calls never throw and never use an unseeded source.
|
||||
//
|
||||
// The model policy is the one caller that must not take that default: it walks
|
||||
// the authored leaves on every step, outside the rng, so the default would be
|
||||
// the value it types forever while the seeded arm varies it. A generator that
|
||||
// spans more than one value refuses there instead.
|
||||
|
||||
import type { Pcg } from "./pcg.ts";
|
||||
import type { Sampler } from "./types.ts";
|
||||
import { getSamplerRng } from "./sampler-rng.ts";
|
||||
import { getSamplerRng, refuseValueWhileEnumerating } from "./sampler-rng.ts";
|
||||
import { INPUT_CORPUS } from "./corpus.ts";
|
||||
|
||||
const ALPHA = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
|
||||
@@ -37,7 +42,17 @@ class StringBuilder implements Sampler<string> {
|
||||
return this;
|
||||
}
|
||||
|
||||
// A max below the min never reaches the draw, so the count is minLength, and
|
||||
// a count below zero emits nothing: length(-2, 0) is the empty string either
|
||||
// way, and both policies agree on it.
|
||||
private spansMoreThanOneString(): boolean {
|
||||
const shortest = Math.max(this.minLength, 0);
|
||||
const longest = Math.max(this.maxLength, this.minLength, 0);
|
||||
return longest > shortest || (longest > 0 && this.charset.length > 1);
|
||||
}
|
||||
|
||||
generate(): string {
|
||||
if (this.spansMoreThanOneString()) refuseValueWhileEnumerating("strings()");
|
||||
const rng = getSamplerRng();
|
||||
const span = this.maxLength - this.minLength + 1;
|
||||
const count = this.minLength + draw(rng, span);
|
||||
@@ -62,6 +77,7 @@ class IntegerBuilder implements Sampler<number> {
|
||||
generate(): number {
|
||||
const rng = getSamplerRng();
|
||||
const span = this.maxValue - this.minValue + 1;
|
||||
if (span > 1) refuseValueWhileEnumerating("integers()");
|
||||
return this.minValue + draw(rng, span);
|
||||
}
|
||||
}
|
||||
@@ -74,7 +90,10 @@ class EmailBuilder implements Sampler<string> {
|
||||
return this;
|
||||
}
|
||||
|
||||
// The local part always spans 3 to 8 alphabetic characters and only the host
|
||||
// is author-set, so an address is never a single value.
|
||||
generate(): string {
|
||||
refuseValueWhileEnumerating("emails()");
|
||||
const local = new StringBuilder().length(3, 8).alpha().generate();
|
||||
return `${local}@${this.host}`;
|
||||
}
|
||||
@@ -82,6 +101,7 @@ class EmailBuilder implements Sampler<string> {
|
||||
|
||||
class EdgeCaseTextBuilder implements Sampler<string> {
|
||||
generate(): string {
|
||||
if (INPUT_CORPUS.length > 1) refuseValueWhileEnumerating("edgeCaseText()");
|
||||
const rng = getSamplerRng();
|
||||
return INPUT_CORPUS[draw(rng, INPUT_CORPUS.length)] ?? "";
|
||||
}
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { Pcg } from "../src/pcg.ts";
|
||||
import { setSamplerRng } from "../src/sampler-rng.ts";
|
||||
import {
|
||||
SAMPLER_REFUSAL_NAME,
|
||||
setEnumeratingCandidates,
|
||||
setSamplerRng,
|
||||
} from "../src/sampler-rng.ts";
|
||||
import { edgeCaseText, emails, integers, strings } from "../src/values.ts";
|
||||
import { INPUT_CORPUS } from "../src/corpus.ts";
|
||||
import type { Sampler } from "../src/types.ts";
|
||||
@@ -73,3 +77,58 @@ test("outside a walk generators return a fixed deterministic default", () => {
|
||||
assert.equal(emails().domain("folio.app").generate(), "[email protected]");
|
||||
assert.equal(edgeCaseText().generate(), INPUT_CORPUS[0]);
|
||||
});
|
||||
|
||||
function whileEnumerating<T>(body: () => T): T {
|
||||
setEnumeratingCandidates(true);
|
||||
try {
|
||||
return body();
|
||||
} finally {
|
||||
setEnumeratingCandidates(false);
|
||||
}
|
||||
}
|
||||
|
||||
test("every value generator refuses a multi-value draw while the model policy enumerates", () => {
|
||||
whileEnumerating(() => {
|
||||
assert.throws(() => integers().between(1, 500).generate(), {
|
||||
name: SAMPLER_REFUSAL_NAME,
|
||||
message: /random value from integers\(\)/,
|
||||
});
|
||||
assert.throws(() => strings().length(3, 6).alpha().generate(), {
|
||||
name: SAMPLER_REFUSAL_NAME,
|
||||
message: /random value from strings\(\)/,
|
||||
});
|
||||
assert.throws(() => emails().domain("folio.app").generate(), {
|
||||
name: SAMPLER_REFUSAL_NAME,
|
||||
message: /random value from emails\(\)/,
|
||||
});
|
||||
assert.throws(() => edgeCaseText().generate(), {
|
||||
name: SAMPLER_REFUSAL_NAME,
|
||||
message: /random value from edgeCaseText\(\)/,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("a fixed-length string still refuses, because its characters vary", () => {
|
||||
whileEnumerating(() => {
|
||||
assert.throws(() => strings().length(4, 4).alpha().generate(), {
|
||||
name: SAMPLER_REFUSAL_NAME,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("a single-valued generator keeps drawing while the model policy enumerates", () => {
|
||||
whileEnumerating(() => {
|
||||
assert.equal(integers().between(7, 7).generate(), 7);
|
||||
assert.equal(strings().length(0, 0).generate(), "");
|
||||
});
|
||||
});
|
||||
|
||||
test("a refused generator draws again once enumeration ends", () => {
|
||||
whileEnumerating(() => {
|
||||
assert.throws(() => integers().between(1, 500).generate());
|
||||
});
|
||||
withRng(5n, () => {
|
||||
const value = integers().between(1, 500).generate();
|
||||
assert.ok(value >= 1 && value <= 500);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user