mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 12:07:09 +00:00
fix(verifier): stop the run on a sampler the model cannot draw, and offer disabled targets
Candidates returns an error now. The refusal is thrown at the draw and wrapped with the source of the leaf that made it, since generate() cannot know which leaf it is inside. Only that marked refusal is fatal: this walk calls every leaf on every step, so promoting the rest would kill model runs the seeded arm survives. Authored actions on a disabled target are no longer dropped from the model's candidate list. The seeded picker executes whatever the leaf authored, and a control the application forgot to re-enable is exactly where boundary defects live, so a policy that cannot attempt it cannot find them. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
c76ba4b497
commit
0094a7fc64
4 files changed
+269
-84
No files matched your search
@@ -62,7 +62,7 @@ func TestModelCandidateDescriptionsAreUniqueAndNamed(t *testing.T) {
|
||||
t.Run(verb, func(t *testing.T) {
|
||||
verifier := loadVerbSpec(t, verb)
|
||||
seen := map[string]bool{}
|
||||
for _, candidate := range verifier.Candidates(LabelSourceVisibleText) {
|
||||
for _, candidate := range mustCandidates(t, verifier, LabelSourceVisibleText) {
|
||||
if candidate.Description == "" {
|
||||
t.Fatalf("%s produced a candidate with no description: %+v", verb, candidate.Action)
|
||||
}
|
||||
@@ -81,14 +81,14 @@ func TestModelCandidateDescriptionsAreUniqueAndNamed(t *testing.T) {
|
||||
// dropped, so the model could never navigate back or let the app settle.
|
||||
func TestModelIsOfferedTheUntargetedVerbs(t *testing.T) {
|
||||
verifier := loadVerbSpec(t, "pressKeys")
|
||||
if !hasCandidate(verifier.Candidates(LabelSourceVisibleText), "Press back") {
|
||||
if !hasCandidate(mustCandidates(t, verifier, LabelSourceVisibleText), "Press back") {
|
||||
t.Errorf("pressKeys missing from the model's candidates: %v",
|
||||
descriptions(verifier.Candidates(LabelSourceVisibleText)))
|
||||
descriptions(mustCandidates(t, verifier, LabelSourceVisibleText)))
|
||||
}
|
||||
verifier = loadVerbSpec(t, "waitOnce")
|
||||
if !hasCandidate(verifier.Candidates(LabelSourceVisibleText), "Wait") {
|
||||
if !hasCandidate(mustCandidates(t, verifier, LabelSourceVisibleText), "Wait") {
|
||||
t.Errorf("waitOnce missing from the model's candidates: %v",
|
||||
descriptions(verifier.Candidates(LabelSourceVisibleText)))
|
||||
descriptions(mustCandidates(t, verifier, LabelSourceVisibleText)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,7 +124,7 @@ func seededDrawStream(t *testing.T, verb, labelSource string) []string {
|
||||
stream := make([]string, 0, seededDrawBudget)
|
||||
for range seededDrawBudget {
|
||||
if labelSource != "" {
|
||||
verifier.Candidates(labelSource)
|
||||
mustCandidates(t, verifier, labelSource)
|
||||
}
|
||||
action, err := verifier.NextAction()
|
||||
if errors.Is(err, ErrNoAction) {
|
||||
@@ -175,7 +175,7 @@ func modelOfferedActions(t *testing.T, verb string) map[string]Action {
|
||||
t.Helper()
|
||||
verifier := loadVerbSpec(t, verb)
|
||||
offered := map[string]Action{}
|
||||
for _, candidate := range verifier.Candidates(LabelSourceVisibleText) {
|
||||
for _, candidate := range mustCandidates(t, verifier, LabelSourceVisibleText) {
|
||||
offered[actionIdentity(candidate.Action)] = candidate.Action
|
||||
}
|
||||
return offered
|
||||
@@ -206,3 +206,53 @@ func sign(value int) int {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// samplerParitySpec authors one leaf that taps a target drawn from three: the
|
||||
// pattern the model policy refuses and the seeded picker exists to draw.
|
||||
const samplerParitySpec = `
|
||||
import { actions, from, Tap } from "@sanderling/spec";
|
||||
const targets = from(["id:Save", "id:Cancel", "id:Amount"]);
|
||||
globalThis.actions = actions(() => [Tap({ on: targets.generate() })]);
|
||||
`
|
||||
|
||||
// TestSeededSamplingSurvivesTheModelPolicysRefusal keeps the refusal on the one
|
||||
// policy it belongs to. Sampling inside the picker's rng scope is correct, so
|
||||
// the seeded stream must be identical whether or not the model policy tried and
|
||||
// failed to enumerate the same leaf first, and it must still reach every item.
|
||||
func TestSeededSamplingSurvivesTheModelPolicysRefusal(t *testing.T) {
|
||||
alone := seededSamplerStream(t, false)
|
||||
afterRefusal := seededSamplerStream(t, true)
|
||||
if !slices.Equal(alone, afterRefusal) {
|
||||
t.Error("a refused enumeration moved the seeded draw stream")
|
||||
}
|
||||
drawn := map[string]bool{}
|
||||
for _, action := range alone {
|
||||
drawn[action] = true
|
||||
}
|
||||
if len(drawn) != 3 {
|
||||
t.Errorf("seeded picker reached %d of the 3 sampled targets: %v", len(drawn), slices.Sorted(maps.Keys(drawn)))
|
||||
}
|
||||
}
|
||||
|
||||
// seededSamplerStream drives the seeded picker over the draw budget, optionally
|
||||
// letting the model policy refuse the same spec before every draw.
|
||||
func seededSamplerStream(t *testing.T, enumerateFirst bool) []string {
|
||||
t.Helper()
|
||||
verifier := newVerifier(t, WithSeed(0x5eed))
|
||||
loadActionSpec(t, verifier, samplerParitySpec)
|
||||
pushTree(t, verifier, policyTreeJSON)
|
||||
stream := make([]string, 0, seededDrawBudget)
|
||||
for range seededDrawBudget {
|
||||
if enumerateFirst {
|
||||
if _, err := verifier.Candidates(LabelSourceVisibleText); err == nil {
|
||||
t.Fatal("the model policy must refuse a spec that samples")
|
||||
}
|
||||
}
|
||||
action, err := verifier.NextAction()
|
||||
if err != nil {
|
||||
t.Fatalf("seeded picker declined the sampled tap: %v", err)
|
||||
}
|
||||
stream = append(stream, fmt.Sprintf("%+v", action))
|
||||
}
|
||||
return stream
|
||||
}
|
||||
Reference in new issue
Block a user