mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 20:17:09 +00:00
test(verifier): setup still draws, and the seeded stream is unmoved
Setup runs through the picker with the rng under both policies, so a generator there is legitimate and must keep working. Interleaving enumeration and setup catches the flag leaking out of the model's walk. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
0a71e05588
commit
3cab483b96
2 files changed
+107
-4
No files matched your search
@@ -5,6 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"maps"
|
"maps"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -220,8 +221,8 @@ globalThis.actions = actions(() => [Tap({ on: targets.generate() })]);
|
|||||||
// the seeded stream must be identical whether or not the model policy tried and
|
// the seeded stream must be identical whether or not the model policy tried and
|
||||||
// failed to enumerate the same leaf first, and it must still reach every item.
|
// failed to enumerate the same leaf first, and it must still reach every item.
|
||||||
func TestSeededSamplingSurvivesTheModelPolicysRefusal(t *testing.T) {
|
func TestSeededSamplingSurvivesTheModelPolicysRefusal(t *testing.T) {
|
||||||
alone := seededSamplerStream(t, false)
|
alone := seededSamplerStream(t, samplerParitySpec, false)
|
||||||
afterRefusal := seededSamplerStream(t, true)
|
afterRefusal := seededSamplerStream(t, samplerParitySpec, true)
|
||||||
if !slices.Equal(alone, afterRefusal) {
|
if !slices.Equal(alone, afterRefusal) {
|
||||||
t.Error("a refused enumeration moved the seeded draw stream")
|
t.Error("a refused enumeration moved the seeded draw stream")
|
||||||
}
|
}
|
||||||
@@ -236,10 +237,10 @@ func TestSeededSamplingSurvivesTheModelPolicysRefusal(t *testing.T) {
|
|||||||
|
|
||||||
// seededSamplerStream drives the seeded picker over the draw budget, optionally
|
// seededSamplerStream drives the seeded picker over the draw budget, optionally
|
||||||
// letting the model policy refuse the same spec before every draw.
|
// letting the model policy refuse the same spec before every draw.
|
||||||
func seededSamplerStream(t *testing.T, enumerateFirst bool) []string {
|
func seededSamplerStream(t *testing.T, specSource string, enumerateFirst bool) []string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
verifier := newVerifier(t, WithSeed(0x5eed))
|
verifier := newVerifier(t, WithSeed(0x5eed))
|
||||||
loadActionSpec(t, verifier, samplerParitySpec)
|
loadActionSpec(t, verifier, specSource)
|
||||||
pushTree(t, verifier, policyTreeJSON)
|
pushTree(t, verifier, policyTreeJSON)
|
||||||
stream := make([]string, 0, seededDrawBudget)
|
stream := make([]string, 0, seededDrawBudget)
|
||||||
for range seededDrawBudget {
|
for range seededDrawBudget {
|
||||||
@@ -256,3 +257,75 @@ func seededSamplerStream(t *testing.T, enumerateFirst bool) []string {
|
|||||||
}
|
}
|
||||||
return stream
|
return stream
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// valueGeneratorSpec authors one leaf that types a drawn value, which is the
|
||||||
|
// same divergence from() has: the draw reaches the seeded picker's rng and never
|
||||||
|
// this enumeration, so the model would be handed one fixed value forever.
|
||||||
|
func valueGeneratorSpec(generator string) string {
|
||||||
|
return fmt.Sprintf(`
|
||||||
|
import { actions, InputText, integers, strings, emails, edgeCaseText } from "@sanderling/spec";
|
||||||
|
const authoredValues = %s;
|
||||||
|
globalThis.actions = actions(() => [InputText({ into: "id:Amount", text: String(authoredValues.generate()) })]);
|
||||||
|
`, generator)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestModelPolicyRefusesAnAuthoredValueGenerator covers every generator in
|
||||||
|
// values.ts whose span is wider than one value.
|
||||||
|
func TestModelPolicyRefusesAnAuthoredValueGenerator(t *testing.T) {
|
||||||
|
for _, generator := range []struct{ name, expression string }{
|
||||||
|
{"integers", "integers().between(1, 500)"},
|
||||||
|
{"strings", "strings().length(3, 6).alpha()"},
|
||||||
|
{"emails", `emails().domain("folio.app")`},
|
||||||
|
{"edgeCaseText", "edgeCaseText()"},
|
||||||
|
} {
|
||||||
|
t.Run(generator.name, func(t *testing.T) {
|
||||||
|
verifier := newVerifier(t, WithSeed(0x5eed))
|
||||||
|
loadActionSpec(t, verifier, valueGeneratorSpec(generator.expression))
|
||||||
|
pushTree(t, verifier, policyTreeJSON)
|
||||||
|
|
||||||
|
_, err := verifier.Candidates(LabelSourceVisibleText)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("%s was enumerated for the model policy, which cannot draw it", generator.name)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "authoredValues.generate()") {
|
||||||
|
t.Errorf("error does not name the offending leaf: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), generator.name+"()") {
|
||||||
|
t.Errorf("error does not name %s(): %v", generator.name, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestModelPolicyAcceptsASingleValuedGenerator is the boundary: a generator that
|
||||||
|
// spans one value hands both policies the same value, so refusing it would stop
|
||||||
|
// runs that have nothing wrong with them.
|
||||||
|
func TestModelPolicyAcceptsASingleValuedGenerator(t *testing.T) {
|
||||||
|
verifier := newVerifier(t, WithSeed(0x5eed))
|
||||||
|
loadActionSpec(t, verifier, valueGeneratorSpec("integers().between(7, 7)"))
|
||||||
|
pushTree(t, verifier, policyTreeJSON)
|
||||||
|
|
||||||
|
candidates := mustCandidates(t, verifier, LabelSourceVisibleText)
|
||||||
|
if len(candidates) != 1 || candidates[0].Action.Text != "7" {
|
||||||
|
t.Fatalf("model was offered %+v, want the one authored InputText typing 7", candidates)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSeededValueDrawsSurviveTheModelPolicysRefusal is the values.ts half of the
|
||||||
|
// guard above: the seeded arm keeps its whole range, and its draw stream does not
|
||||||
|
// move because the model policy refused the same spec first.
|
||||||
|
func TestSeededValueDrawsSurviveTheModelPolicysRefusal(t *testing.T) {
|
||||||
|
spec := valueGeneratorSpec("integers().between(1, 500)")
|
||||||
|
alone := seededSamplerStream(t, spec, false)
|
||||||
|
afterRefusal := seededSamplerStream(t, spec, true)
|
||||||
|
if !slices.Equal(alone, afterRefusal) {
|
||||||
|
t.Error("a refused enumeration moved the seeded draw stream")
|
||||||
|
}
|
||||||
|
drawn := map[string]bool{}
|
||||||
|
for _, action := range alone {
|
||||||
|
drawn[action] = true
|
||||||
|
}
|
||||||
|
if len(drawn) < 100 {
|
||||||
|
t.Errorf("seeded picker typed %d distinct values over %d draws", len(drawn), seededDrawBudget)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,8 +2,10 @@ package verifier
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"maps"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/priyanshujain/sanderling/internal/bundler"
|
"github.com/priyanshujain/sanderling/internal/bundler"
|
||||||
@@ -93,3 +95,31 @@ export const actionsRoot = taps;
|
|||||||
t.Fatalf("NextAction should draw from actionsRoot: %v", err)
|
t.Fatalf("NextAction should draw from actionsRoot: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestSetupGeneratorDrawsUnderTheModelPolicy: setup walks through the picker
|
||||||
|
// with its rng under both policies, so a generator there is not the divergence
|
||||||
|
// the enumeration refuses and must keep drawing. Enumerating before every setup
|
||||||
|
// step is what a model-driven run does, and the refusal must not leak out of it.
|
||||||
|
func TestSetupGeneratorDrawsUnderTheModelPolicy(t *testing.T) {
|
||||||
|
spec := `
|
||||||
|
import { InputText, actions, integers, taps } from "@sanderling/spec";
|
||||||
|
const setupValues = integers().between(1, 500);
|
||||||
|
export const setup = actions(() => [InputText({ into: "id:Amount", text: String(setupValues.generate()) })]);
|
||||||
|
export const actionsRoot = taps;
|
||||||
|
`
|
||||||
|
v := loadBundled(t, spec, policyTreeJSON)
|
||||||
|
typed := map[string]bool{}
|
||||||
|
for range 16 {
|
||||||
|
if _, err := v.Candidates(LabelSourceVisibleText); err != nil {
|
||||||
|
t.Fatalf("Candidates: %v", err)
|
||||||
|
}
|
||||||
|
action, err := v.SetupAction()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("SetupAction: %v", err)
|
||||||
|
}
|
||||||
|
typed[action.Text] = true
|
||||||
|
}
|
||||||
|
if len(typed) < 2 {
|
||||||
|
t.Errorf("setup typed %v on every step; the picker's rng did not reach it", slices.Sorted(maps.Keys(typed)))
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user