From 0094a7fc641caa848c3f40611ac8dabbf3c7166b Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 13 Aug 2026 00:44:00 +0530 Subject: [PATCH] fix(verifier): stop the run on a sampler the model cannot draw, and offer disabled targets Candidates returns an error now. The refusal is thrown at the draw and wrapped with the source of the leaf that made it, since generate() cannot know which leaf it is inside. Only that marked refusal is fatal: this walk calls every leaf on every step, so promoting the rest would kill model runs the seeded arm survives. Authored actions on a disabled target are no longer dropped from the model's candidate list. The seeded picker executes whatever the leaf authored, and a control the application forgot to re-enable is exactly where boundary defects live, so a policy that cannot attempt it cannot find them. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX --- internal/verifier/llm.go | 136 +++++++++++++++++------ internal/verifier/llm_test.go | 140 ++++++++++++++++-------- internal/verifier/policy_parity_test.go | 64 +++++++++-- internal/verifier/worker.go | 13 +++ 4 files changed, 269 insertions(+), 84 deletions(-) diff --git a/internal/verifier/llm.go b/internal/verifier/llm.go index 31f3400..54851e1 100644 --- a/internal/verifier/llm.go +++ b/internal/verifier/llm.go @@ -176,38 +176,59 @@ const ( // labelSource selects the channel each target is named by. An unrecognized // value (including the zero value) names targets by visible text; the CLI // rejects an unknown mode before a run starts, so only a test reaches that. -func (v *Verifier) Candidates(labelSource string) []ActionCandidate { +// +// The error is the spec refusing to be run by this policy at all: an authored +// leaf that samples one of several items reaches the seeded picker's rng but +// never this walk, so the model would be offered a fixed first item forever. It +// names the leaf and it is fatal, because degrading to that fixed item silently +// is what makes a policy comparison meaningless. +func (v *Verifier) Candidates(labelSource string) ([]ActionCandidate, error) { if v.lastTree == nil { - return nil + return nil, nil } root := v.runtime.GlobalObject().Get("actions") if root == nil || goja.IsUndefined(root) || goja.IsNull(root) { - return nil + return nil, nil } labels := labelContext{nodeIndex: buildNodeIndex(v.lastTree), source: labelSource} var raw []ActionCandidate - v.collectNode(root, 1.0, false, labels, &raw) - return finalizeCandidates(raw) + v.setEnumeratingCandidates(true) + defer v.setEnumeratingCandidates(false) + if err := v.collectNode(root, 1.0, false, labels, &raw); err != nil { + return nil, err + } + return finalizeCandidates(raw), nil +} + +// setEnumeratingCandidates tells the spec bundle that the authored leaves are +// being called by this policy rather than by the picker. A spec loaded without +// the runtime entry (a raw-JS unit fixture) has no such callable, and no +// sampler to refuse either. +func (v *Verifier) setEnumeratingCandidates(enumerating bool) { + if v.setEnumeratingCandidatesFn == nil { + return + } + _, _ = v.setEnumeratingCandidatesFn(goja.Undefined(), v.runtime.ToValue(enumerating)) } // collectNode dispatches one GeneratorNode of the action tree. prob is the // accumulated probability the seeded picker reaches this node; weighted records // whether any weighted node lies on the path (so weights are shown only when the // spec actually declared them). -func (v *Verifier) collectNode(node goja.Value, prob float64, weighted bool, labels labelContext, out *[]ActionCandidate) { +func (v *Verifier) collectNode(node goja.Value, prob float64, weighted bool, labels labelContext, out *[]ActionCandidate) error { object := node.ToObject(v.runtime) if object == nil { - return + return nil } kind := object.Get("kind") if kind == nil || goja.IsUndefined(kind) { - return + return nil } switch kind.String() { case "weighted": - v.collectWeighted(object, prob, labels, out) + return v.collectWeighted(object, prob, labels, out) case "actions": - v.collectActions(object, prob, weighted, labels, out) + return v.collectActions(object, prob, weighted, labels, out) case "builtin": verb := object.Get("verb") if verb != nil && !goja.IsUndefined(verb) { @@ -216,19 +237,20 @@ func (v *Verifier) collectNode(node goja.Value, prob float64, weighted bool, lab case "llm": // The llm marker is the generator, not part of the candidate tree. } + return nil } // collectWeighted recurses each branch, splitting the incoming probability by // the branch weight over the sibling total (matching the seeded picker's single // weighted draw). -func (v *Verifier) collectWeighted(object *goja.Object, prob float64, labels labelContext, out *[]ActionCandidate) { +func (v *Verifier) collectWeighted(object *goja.Object, prob float64, labels labelContext, out *[]ActionCandidate) error { branches := object.Get("branches") if branches == nil { - return + return nil } array := branches.ToObject(v.runtime) if array == nil { - return + return nil } length := int(array.Get("length").ToInteger()) weights := make([]float64, length) @@ -249,32 +271,47 @@ func (v *Verifier) collectWeighted(object *goja.Object, prob float64, labels lab total += weight } if total <= 0 { - return + return nil } for i := range length { if children[i] == nil { continue } - v.collectNode(children[i], prob*weights[i]/total, true, labels, out) + // The branch number is the author's own path to a refused leaf, which + // its closure source alone does not give when the leaf is a whenRoute + // (whose closure belongs to the library, not the spec). + if err := v.collectNode(children[i], prob*weights[i]/total, true, labels, out); err != nil { + return fmt.Errorf("branch %d: %w", i+1, err) + } } + return nil } // collectActions calls an authored leaf's generator once (safe: it reads state // and, off-route, returns []), turning each concrete descriptor into a // candidate. It runs OUTSIDE the picker's rng scope, so from(...).generate() // draws nothing and no seed advances. -func (v *Verifier) collectActions(object *goja.Object, prob float64, weighted bool, labels labelContext, out *[]ActionCandidate) { - generate, ok := goja.AssertFunction(object.Get("generate")) +// +// A generator that throws for its own reasons still contributes nothing and +// nothing more: this walk calls EVERY leaf every step, including leaves the +// seeded picker would have walked once in a hundred steps, so promoting those +// throws would kill runs the seeded arm survives. +func (v *Verifier) collectActions(object *goja.Object, prob float64, weighted bool, labels labelContext, out *[]ActionCandidate) error { + generatorValue := object.Get("generate") + generate, ok := goja.AssertFunction(generatorValue) if !ok { - return + return nil } result, err := generate(goja.Undefined()) if err != nil { - return + if refusal, refused := v.samplerRefusal(err); refused { + return fmt.Errorf("authored action %s %s", authoredLeafIdentity(generatorValue), refusal) + } + return nil } array := result.ToObject(v.runtime) if array == nil { - return + return nil } length := int(array.Get("length").ToInteger()) for i := range length { @@ -286,11 +323,54 @@ func (v *Verifier) collectActions(object *goja.Object, prob float64, weighted bo candidate.Weighted = weighted *out = append(*out, candidate) } + return nil +} + +// samplerRefusalName is the error name pkg/spec/src/sampler-rng.ts stamps on the +// refusal it throws, which is what tells that refusal apart from a spec's own +// runtime errors. +const samplerRefusalName = "SanderlingSamplerRefusal" + +// samplerRefusal reports the refusal message when the authored leaf declined to +// sample for this policy. +func (v *Verifier) samplerRefusal(err error) (string, bool) { + var exception *goja.Exception + if !errors.As(err, &exception) { + return "", false + } + value := exception.Value() + if value == nil || goja.IsUndefined(value) || goja.IsNull(value) { + return "", false + } + thrown := value.ToObject(v.runtime) + if thrown == nil || stringField(thrown, "name") != samplerRefusalName { + return "", false + } + return stringField(thrown, "message"), true +} + +// maxLeafSourceRunes caps the generator excerpt that names a leaf in an error. +const maxLeafSourceRunes = 160 + +// authoredLeafIdentity renders the leaf's generator source on one line. An +// authored leaf is an anonymous closure among identical-looking tree nodes, so +// its source is the handle an author can search the spec for. +func authoredLeafIdentity(generator goja.Value) string { + source := []rune(strings.Join(strings.Fields(generator.String()), " ")) + if len(source) > maxLeafSourceRunes { + return strconv.Quote(string(source[:maxLeafSourceRunes]) + "...") + } + return strconv.Quote(string(source)) } // candidateFromDescriptor lowers one authored ActionDescriptor (as a goja // object) into a ready-to-run candidate, resolving the target's coordinates, -// selector, and label. Actions on a disabled control are dropped. +// selector, and label. +// +// A disabled target is offered like any other. The seeded picker executes +// whatever the leaf authored, disabled or not, and attempting a disabled +// control is where boundary defects live: a control the app forgot to re-enable +// reads as disabled, and a policy that cannot attempt it cannot find that. func (v *Verifier) candidateFromDescriptor(value goja.Value, labels labelContext) (ActionCandidate, bool) { object := value.ToObject(v.runtime) if object == nil { @@ -304,7 +384,7 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, labels labelContext switch kind { case ActionKindTap, ActionKindDoubleTap, ActionKindLongPress: target, ok := v.resolveTarget(object.Get("on"), labels) - if !ok || target.disabled { + if !ok { return ActionCandidate{}, false } return ActionCandidate{ @@ -314,7 +394,7 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, labels labelContext }, true case ActionKindInputText: target, ok := v.resolveTarget(object.Get("into"), labels) - if !ok || target.disabled { + if !ok { return ActionCandidate{}, false } text := stringField(object, "text") @@ -384,7 +464,6 @@ type resolvedTarget struct { selector string label string inputType string - disabled bool } // resolveTarget reads an authored action's target. Ax element handles carry @@ -423,7 +502,6 @@ func (v *Verifier) resolveTarget(value goja.Value, labels labelContext) (resolve if element := v.findBySelector(selector); element != nil { target.label = labels.label(element) target.inputType = inputTypeHint(element) - target.disabled = !element.Enabled && hasEnabled(element) } if target.label == "" { target.label = truncateLabel(stringField(object, labels.handleField())) @@ -441,7 +519,6 @@ func (v *Verifier) targetFromSelector(selector string, labels labelContext) reso target.x, target.y = element.Bounds.Center() target.label = labels.label(element) target.inputType = inputTypeHint(element) - target.disabled = !element.Enabled && hasEnabled(element) return target } @@ -754,13 +831,6 @@ func inputTypeHint(element *hierarchy.Element) string { } } -// hasEnabled reports whether the source tree carried an explicit enabled flag -// for the element, so a missing flag is not mistaken for "disabled". -func hasEnabled(element *hierarchy.Element) bool { - _, ok := element.Attributes["enabled"] - return ok -} - // stringField reads a string property off a goja object, returning "" when // absent, null, or undefined. func stringField(object *goja.Object, key string) string { diff --git a/internal/verifier/llm_test.go b/internal/verifier/llm_test.go index e4502e3..30005eb 100644 --- a/internal/verifier/llm_test.go +++ b/internal/verifier/llm_test.go @@ -69,6 +69,17 @@ func enumVerifier(t *testing.T, actionsJS, treeJSON string) *Verifier { return v } +// mustCandidates enumerates the model policy's list, failing the test on the +// refusal an authored multi-item sampler raises. +func mustCandidates(t *testing.T, v *Verifier, labelSource string) []ActionCandidate { + t.Helper() + candidates, err := v.Candidates(labelSource) + if err != nil { + t.Fatalf("Candidates: %v", err) + } + return candidates +} + func findCandidate(candidates []ActionCandidate, description string) (ActionCandidate, bool) { for _, candidate := range candidates { if candidate.Description == description { @@ -85,7 +96,7 @@ func hasCandidate(candidates []ActionCandidate, description string) bool { func TestCandidatesLabelsControlsByVisibleText(t *testing.T) { v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", enumTreeJSON) - candidates := v.Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, v, LabelSourceVisibleText) // The empty-text clickable wrapper is labeled by its child Text, NOT its // resource-id. @@ -109,8 +120,7 @@ func TestCandidatesLabelsControlsByVisibleText(t *testing.T) { } func TestCandidatesLabelsControlsByResourceIdentifier(t *testing.T) { - candidates := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). - Candidates(LabelSourceResourceID) + candidates := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON), LabelSourceResourceID) if !hasCandidate(candidates, `Tap "add_credit_button"`) { t.Errorf("want the control named by its identifier, got %v", descriptions(candidates)) @@ -125,8 +135,7 @@ func TestCandidatesLabelsControlsByResourceIdentifier(t *testing.T) { } func TestCandidatesIdentifierChannelFallsBackToClassThenBareControl(t *testing.T) { - candidates := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). - Candidates(LabelSourceResourceID) + candidates := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON), LabelSourceResourceID) if !hasCandidate(candidates, `Tap "android.widget.CheckBox"`) { t.Errorf("a control with no identifier falls back to its class, got %v", descriptions(candidates)) @@ -143,10 +152,8 @@ func TestCandidatesIdentifierChannelFallsBackToClassThenBareControl(t *testing.T // own action, so the model can act on either by number. A channel that renames // controls must never shrink the action space. func TestCandidatesIdentifierChannelKeepsControlsItCannotNameApartReachable(t *testing.T) { - text := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). - Candidates(LabelSourceVisibleText) - identifier := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). - Candidates(LabelSourceResourceID) + text := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON), LabelSourceVisibleText) + identifier := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON), LabelSourceResourceID) if count(text, `Tap "Remember me"`) != 1 || count(text, `Tap "Stay signed in"`) != 1 { t.Fatalf("the text channel should address both checkboxes, got %v", descriptions(text)) @@ -171,8 +178,7 @@ func TestCandidatesIdentifierChannelKeepsControlsItCannotNameApartReachable(t *t // on the rendered line dropped the second one, putting it out of reach of any // prompt or policy. func TestCandidatesReachBothControlsSharingOneVisibleLabel(t *testing.T) { - candidates := enumVerifier(t, "{kind:'builtin', verb:'taps'}", sharedLabelTreeJSON). - Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'taps'}", sharedLabelTreeJSON), LabelSourceVisibleText) deletes := candidatesMatching(candidates, `Tap "Delete"`) if len(deletes) != 2 { @@ -193,8 +199,7 @@ func TestCandidatesReachBothControlsSharingOneVisibleLabel(t *testing.T) { // agree: a control carrying nothing readable is named by its identifier in both, // so a screen built entirely from such controls is one cell, not two. func TestCandidatesVisibleTextFallsBackToTheIdentifier(t *testing.T) { - candidates := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). - Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON), LabelSourceVisibleText) if !hasCandidate(candidates, `Tap "silent_row"`) { t.Errorf("a control with no readable text falls back to its identifier, got %v", @@ -203,14 +208,12 @@ func TestCandidatesVisibleTextFallsBackToTheIdentifier(t *testing.T) { } func TestCandidatesTypingLabelFollowsTheLabelSource(t *testing.T) { - text := enumVerifier(t, "{kind:'builtin', verb:'typing'}", labelChannelTreeJSON). - Candidates(LabelSourceVisibleText) + text := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'typing'}", labelChannelTreeJSON), LabelSourceVisibleText) if !hasCandidate(text, `Type into "Amount" (number)`) { t.Errorf("want the field named by its hint, got %v", descriptions(text)) } - identifier := enumVerifier(t, "{kind:'builtin', verb:'typing'}", labelChannelTreeJSON). - Candidates(LabelSourceResourceID) + identifier := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'typing'}", labelChannelTreeJSON), LabelSourceResourceID) if !hasCandidate(identifier, `Type into "amount_field" (number)`) { t.Errorf("want the field named by its identifier, got %v", descriptions(identifier)) } @@ -243,8 +246,8 @@ func TestLabelSourceChangesOnlyTheDescription(t *testing.T) { } for _, fixture := range fixtures { t.Run(fixture.name, func(t *testing.T) { - text := enumVerifier(t, everyLabelledVerb, fixture.tree).Candidates(LabelSourceVisibleText) - identifier := enumVerifier(t, everyLabelledVerb, fixture.tree).Candidates(LabelSourceResourceID) + text := mustCandidates(t, enumVerifier(t, everyLabelledVerb, fixture.tree), LabelSourceVisibleText) + identifier := mustCandidates(t, enumVerifier(t, everyLabelledVerb, fixture.tree), LabelSourceResourceID) if len(text) == 0 { t.Fatal("fixture yielded no candidates") } @@ -269,9 +272,9 @@ func TestLabelSourceChangesOnlyTheDescription(t *testing.T) { } } -func TestCandidatesDropsDisabledControls(t *testing.T) { +func TestCandidatesDropsDisabledControlsFromBuiltinVerbs(t *testing.T) { v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", enumTreeJSON) - for _, candidate := range v.Candidates(LabelSourceVisibleText) { + for _, candidate := range mustCandidates(t, v, LabelSourceVisibleText) { if strings.Contains(candidate.Description, "Off") { t.Errorf("disabled control surfaced as %q", candidate.Description) } @@ -280,7 +283,7 @@ func TestCandidatesDropsDisabledControls(t *testing.T) { func TestCandidatesTypingExposesInputType(t *testing.T) { v := enumVerifier(t, "{kind:'builtin', verb:'typing'}", enumTreeJSON) - candidates := v.Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, v, LabelSourceVisibleText) candidate, ok := findCandidate(candidates, `Type into "Amount" (number)`) if !ok { t.Fatalf("want typing candidate with input type, got %v", descriptions(candidates)) @@ -303,7 +306,7 @@ func TestCandidatesLabelsEditableFieldByHintNotTypedValue(t *testing.T) { ] }` v := enumVerifier(t, "{kind:'builtin', verb:'typing'}", tree) - candidates := v.Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, v, LabelSourceVisibleText) if hasCandidate(candidates, `Type into "99" (number)`) || hasCandidate(candidates, `Type into "99"`) { t.Errorf("editable field labeled by its typed value: %v", descriptions(candidates)) } @@ -316,7 +319,7 @@ func TestCandidatesKeepsGestureVerbsDistinct(t *testing.T) { v := enumVerifier(t, "{kind:'weighted', branches:[[1,{kind:'builtin',verb:'scrolls'}],[1,{kind:'builtin',verb:'swipes'}]]}", enumTreeJSON) - candidates := v.Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, v, LabelSourceVisibleText) // `scrolls` folds to one directional pair over the single scrollable // container, which is what keeps the list short. @@ -358,7 +361,7 @@ func TestCandidatesWeightsCombineAcrossPaths(t *testing.T) { v := enumVerifier(t, "{kind:'weighted', branches:[[1,{kind:'builtin',verb:'taps'}],[1,{kind:'builtin',verb:'taps'}]]}", oneClickable) - candidates := v.Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, v, LabelSourceVisibleText) if len(candidates) != 1 { t.Fatalf("want one deduped candidate, got %v", descriptions(candidates)) } @@ -375,7 +378,7 @@ func TestCandidatesWeightReflectsBranchShare(t *testing.T) { v := enumVerifier(t, "{kind:'weighted', branches:[[1,{kind:'builtin',verb:'taps'}],[3,{kind:'builtin',verb:'typing'}]]}", enumTreeJSON) - candidates := v.Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, v, LabelSourceVisibleText) tap, ok := findCandidate(candidates, `Tap "Sign in"`) if !ok { t.Fatalf("missing tap candidate: %v", descriptions(candidates)) @@ -394,7 +397,7 @@ func TestCandidatesWeightReflectsBranchShare(t *testing.T) { func TestCandidatesUnweightedTreeShowsNoWeight(t *testing.T) { v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", enumTreeJSON) - for _, candidate := range v.Candidates(LabelSourceVisibleText) { + for _, candidate := range mustCandidates(t, v, LabelSourceVisibleText) { if candidate.Weighted || candidate.Weight != 0 { t.Errorf("%q carries a weight despite no weighted node", candidate.Description) } @@ -408,17 +411,17 @@ func TestCandidatesCallsAuthoredLeafOnce(t *testing.T) { {kind:'InputText', into:'id:Amount', text:'42'} ]}` v := enumVerifier(t, actions, enumTreeJSON) - candidates := v.Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, v, LabelSourceVisibleText) // Authored Tap resolves its selector to the visible-text label. if !hasCandidate(candidates, `Tap "Sign in"`) { t.Errorf("authored tap missing: %v", descriptions(candidates)) } - // A disabled authored target is dropped. - for _, candidate := range candidates { - if strings.Contains(candidate.Description, "Off") { - t.Errorf("authored action on disabled control surfaced: %q", candidate.Description) - } + // A disabled authored target is offered, not dropped: the seeded picker + // executes it, and attempting a disabled control is where boundary defects + // live, so a policy that cannot attempt it cannot find them. + if !hasCandidate(candidates, `Tap "Off"`) { + t.Errorf("authored action on a disabled control was dropped: %v", descriptions(candidates)) } // Authored InputText replays its own sampled value (LLM does not supply it). authored, ok := findCandidate(candidates, `Type "42" into "Amount"`) @@ -442,7 +445,7 @@ func TestCandidatesSurfaceAuthoredUntargetedActions(t *testing.T) { {kind:'PressKey', key:'back'}, {kind:'Wait'} ]}` - candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, actions, enumTreeJSON), LabelSourceVisibleText) for _, want := range []string{"Swipe from (10,600) to (10,100)", "Press back", "Wait"} { if !hasCandidate(candidates, want) { t.Errorf("authored %q missing: %v", want, descriptions(candidates)) @@ -455,7 +458,7 @@ func TestCandidatesSurfaceAuthoredUntargetedActions(t *testing.T) { // idling on paper while the seeded arm really waits. func TestCandidatesAuthoredWaitKeepsItsDuration(t *testing.T) { actions := `{kind:'actions', generate: () => [{kind:'Wait', durationMillis: 500}]}` - candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, actions, enumTreeJSON), LabelSourceVisibleText) candidate, ok := findCandidate(candidates, "Wait") if !ok { t.Fatalf("authored wait missing: %v", descriptions(candidates)) @@ -470,7 +473,7 @@ func TestCandidatesAuthoredWaitKeepsItsDuration(t *testing.T) { // the screen and the scroll lands on whatever else is scrollable. func TestCandidatesAuthoredScrollNamesItsContainer(t *testing.T) { actions := `{kind:'actions', generate: () => [{kind:'Scroll', direction:'down', in:'id:List'}]}` - candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, actions, enumTreeJSON), LabelSourceVisibleText) candidate, ok := findCandidate(candidates, "Scroll down") if !ok { t.Fatalf("authored scroll missing: %v", descriptions(candidates)) @@ -489,7 +492,7 @@ func TestCandidatesAuthoredScrollKeepsPrecomputedEndpoints(t *testing.T) { actions := `{kind:'actions', generate: () => [ {kind:'Scroll', direction:'down', in:'id:List', from:{x:540,y:1400}, to:{x:540,y:920}} ]}` - candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, actions, enumTreeJSON), LabelSourceVisibleText) candidate, ok := findCandidate(candidates, "Scroll down") if !ok { t.Fatalf("authored scroll missing: %v", descriptions(candidates)) @@ -510,7 +513,7 @@ func TestCandidatesAuthoredSwipeDefaultsItsDuration(t *testing.T) { {kind:'Swipe', from:{x:10,y:600}, to:{x:10,y:100}}, {kind:'Swipe', from:{x:20,y:600}, to:{x:20,y:100}, durationMillis: 400} ]}` - candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, actions, enumTreeJSON), LabelSourceVisibleText) omitted, ok := findCandidate(candidates, "Swipe from (10,600) to (10,100)") if !ok { t.Fatalf("authored swipe missing: %v", descriptions(candidates)) @@ -538,7 +541,7 @@ func TestCandidatesDropTargetsThatResolveToNothing(t *testing.T) { {kind:'InputText', into: {}, text:'x'}, {kind:'Swipe', from:{x:1,y:2}, to:{}} ]}` - candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, actions, enumTreeJSON), LabelSourceVisibleText) if len(candidates) != 0 { t.Errorf("targetless actions reached the model: %v", descriptions(candidates)) } @@ -549,7 +552,7 @@ func TestCandidatesDropTargetsThatResolveToNothing(t *testing.T) { // a target with no coordinates rather than on coordinates that are zero. func TestCandidatesKeepATargetOnTheScreenOrigin(t *testing.T) { actions := `{kind:'actions', generate: () => [{kind:'Tap', on: {x: 0, y: 0}}]}` - candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, actions, enumTreeJSON), LabelSourceVisibleText) if len(candidates) != 1 { t.Fatalf("want the origin tap kept, got %v", descriptions(candidates)) } @@ -557,7 +560,7 @@ func TestCandidatesKeepATargetOnTheScreenOrigin(t *testing.T) { func TestCandidatesOffRouteLeafYieldsNothing(t *testing.T) { v := enumVerifier(t, "{kind:'actions', generate: () => []}", enumTreeJSON) - if got := v.Candidates(LabelSourceVisibleText); len(got) != 0 { + if got := mustCandidates(t, v, LabelSourceVisibleText); len(got) != 0 { t.Errorf("off-route leaf should yield no candidates, got %v", descriptions(got)) } } @@ -575,7 +578,7 @@ func TestCandidatesSkipsCrossFadeFrames(t *testing.T) { ] }` v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", crossFade) - if got := v.Candidates(LabelSourceVisibleText); len(got) != 0 { + if got := mustCandidates(t, v, LabelSourceVisibleText); len(got) != 0 { t.Errorf("cross-fade frame should yield no candidates, got %v", descriptions(got)) } // The seeded policy is skipped by the SAME guard, in the shared producer, @@ -587,13 +590,13 @@ func TestCandidatesSkipsCrossFadeFrames(t *testing.T) { func TestCandidatesNilWithoutTreeOrActions(t *testing.T) { withActions := newLoadedVerifier(t, "globalThis.actions = {kind:'builtin', verb:'taps'};") - if got := withActions.Candidates(LabelSourceVisibleText); got != nil { + if got := mustCandidates(t, withActions, LabelSourceVisibleText); got != nil { t.Errorf("Candidates with no tree = %v, want nil", got) } noActions := newLoadedVerifier(t, "globalThis.properties = {};") tree, _ := hierarchy.Parse(enumTreeJSON) noActions.lastTree = tree - if got := noActions.Candidates(LabelSourceVisibleText); got != nil { + if got := mustCandidates(t, noActions, LabelSourceVisibleText); got != nil { t.Errorf("Candidates with no actions root = %v, want nil", got) } } @@ -687,3 +690,52 @@ func newLoadedVerifier(t *testing.T, source string) *Verifier { } return v } + +// samplerSpec authors one leaf that taps a target drawn from the given list. +func samplerSpec(items string) string { + return ` +import { actions, from, Tap } from "@sanderling/spec"; +const targets = from(` + items + `); +globalThis.actions = actions(() => [Tap({ on: targets.generate() })]); +` +} + +// TestCandidatesRefuseAMultiItemAuthoredSampler pins the refusal: a sampler +// reads the picker's rng, which this policy has no way to enter, so the draw +// would collapse to the first item on every step while the seeded picker keeps +// reaching all three. Offering that silently is what would make a comparison of +// the two policies meaningless, so the spec is refused instead. +func TestCandidatesRefuseAMultiItemAuthoredSampler(t *testing.T) { + v := newVerifier(t) + loadActionSpec(t, v, samplerSpec(`["id:SignIn", "id:Amount", "id:List"]`)) + pushTree(t, v, enumTreeJSON) + + _, err := v.Candidates(LabelSourceVisibleText) + if err == nil { + t.Fatal("a multi-item authored sampler must refuse to run under the model policy") + } + message := err.Error() + if !strings.Contains(message, "targets.generate()") { + t.Errorf("error does not name the offending leaf, so the author cannot find it: %s", message) + } + if !strings.Contains(message, "draws 1 of 3 sampled items") { + t.Errorf("error does not say what the leaf did: %s", message) + } +} + +// TestCandidatesAcceptASingleItemAuthoredSampler: a one-item sampler short +// circuits before the rng, so both policies get that one value and there is no +// divergence to refuse. +func TestCandidatesAcceptASingleItemAuthoredSampler(t *testing.T) { + v := newVerifier(t) + loadActionSpec(t, v, samplerSpec(`["id:SignIn"]`)) + pushTree(t, v, enumTreeJSON) + + candidates, err := v.Candidates(LabelSourceVisibleText) + if err != nil { + t.Fatalf("a single-item sampler is not a divergence: %v", err) + } + if !hasCandidate(candidates, `Tap "Sign in"`) { + t.Errorf("sampled tap missing: %v", descriptions(candidates)) + } +} diff --git a/internal/verifier/policy_parity_test.go b/internal/verifier/policy_parity_test.go index 5c477ee..c71fa9c 100644 --- a/internal/verifier/policy_parity_test.go +++ b/internal/verifier/policy_parity_test.go @@ -62,7 +62,7 @@ func TestModelCandidateDescriptionsAreUniqueAndNamed(t *testing.T) { t.Run(verb, func(t *testing.T) { verifier := loadVerbSpec(t, verb) seen := map[string]bool{} - for _, candidate := range verifier.Candidates(LabelSourceVisibleText) { + for _, candidate := range mustCandidates(t, verifier, LabelSourceVisibleText) { if candidate.Description == "" { t.Fatalf("%s produced a candidate with no description: %+v", verb, candidate.Action) } @@ -81,14 +81,14 @@ func TestModelCandidateDescriptionsAreUniqueAndNamed(t *testing.T) { // dropped, so the model could never navigate back or let the app settle. func TestModelIsOfferedTheUntargetedVerbs(t *testing.T) { verifier := loadVerbSpec(t, "pressKeys") - if !hasCandidate(verifier.Candidates(LabelSourceVisibleText), "Press back") { + if !hasCandidate(mustCandidates(t, verifier, LabelSourceVisibleText), "Press back") { t.Errorf("pressKeys missing from the model's candidates: %v", - descriptions(verifier.Candidates(LabelSourceVisibleText))) + descriptions(mustCandidates(t, verifier, LabelSourceVisibleText))) } verifier = loadVerbSpec(t, "waitOnce") - if !hasCandidate(verifier.Candidates(LabelSourceVisibleText), "Wait") { + if !hasCandidate(mustCandidates(t, verifier, LabelSourceVisibleText), "Wait") { t.Errorf("waitOnce missing from the model's candidates: %v", - descriptions(verifier.Candidates(LabelSourceVisibleText))) + descriptions(mustCandidates(t, verifier, LabelSourceVisibleText))) } } @@ -124,7 +124,7 @@ func seededDrawStream(t *testing.T, verb, labelSource string) []string { stream := make([]string, 0, seededDrawBudget) for range seededDrawBudget { if labelSource != "" { - verifier.Candidates(labelSource) + mustCandidates(t, verifier, labelSource) } action, err := verifier.NextAction() if errors.Is(err, ErrNoAction) { @@ -175,7 +175,7 @@ func modelOfferedActions(t *testing.T, verb string) map[string]Action { t.Helper() verifier := loadVerbSpec(t, verb) offered := map[string]Action{} - for _, candidate := range verifier.Candidates(LabelSourceVisibleText) { + for _, candidate := range mustCandidates(t, verifier, LabelSourceVisibleText) { offered[actionIdentity(candidate.Action)] = candidate.Action } return offered @@ -206,3 +206,53 @@ func sign(value int) int { return 0 } } + +// samplerParitySpec authors one leaf that taps a target drawn from three: the +// pattern the model policy refuses and the seeded picker exists to draw. +const samplerParitySpec = ` +import { actions, from, Tap } from "@sanderling/spec"; +const targets = from(["id:Save", "id:Cancel", "id:Amount"]); +globalThis.actions = actions(() => [Tap({ on: targets.generate() })]); +` + +// TestSeededSamplingSurvivesTheModelPolicysRefusal keeps the refusal on the one +// policy it belongs to. Sampling inside the picker's rng scope is correct, so +// the seeded stream must be identical whether or not the model policy tried and +// failed to enumerate the same leaf first, and it must still reach every item. +func TestSeededSamplingSurvivesTheModelPolicysRefusal(t *testing.T) { + alone := seededSamplerStream(t, false) + afterRefusal := seededSamplerStream(t, true) + if !slices.Equal(alone, afterRefusal) { + t.Error("a refused enumeration moved the seeded draw stream") + } + drawn := map[string]bool{} + for _, action := range alone { + drawn[action] = true + } + if len(drawn) != 3 { + t.Errorf("seeded picker reached %d of the 3 sampled targets: %v", len(drawn), slices.Sorted(maps.Keys(drawn))) + } +} + +// seededSamplerStream drives the seeded picker over the draw budget, optionally +// letting the model policy refuse the same spec before every draw. +func seededSamplerStream(t *testing.T, enumerateFirst bool) []string { + t.Helper() + verifier := newVerifier(t, WithSeed(0x5eed)) + loadActionSpec(t, verifier, samplerParitySpec) + pushTree(t, verifier, policyTreeJSON) + stream := make([]string, 0, seededDrawBudget) + for range seededDrawBudget { + if enumerateFirst { + if _, err := verifier.Candidates(LabelSourceVisibleText); err == nil { + t.Fatal("the model policy must refuse a spec that samples") + } + } + action, err := verifier.NextAction() + if err != nil { + t.Fatalf("seeded picker declined the sampled tap: %v", err) + } + stream = append(stream, fmt.Sprintf("%+v", action)) + } + return stream +} diff --git a/internal/verifier/worker.go b/internal/verifier/worker.go index 82ac57b..793a279 100644 --- a/internal/verifier/worker.go +++ b/internal/verifier/worker.go @@ -45,6 +45,13 @@ type Verifier struct { // over the picker's action space rather than one of its own. enumerateBuiltinFn goja.Callable + // setEnumeratingCandidatesFn is the bundle-installed + // __sanderlingSetEnumeratingCandidates__, which brackets the model policy's + // authored-leaf calls. Those run outside the picker's rng scope, where a + // sampler would quietly hand back its first item, so the bundle refuses to + // sample while it is set. + setEnumeratingCandidatesFn goja.Callable + evaluators map[string]*ltl.Evaluator priorVerdicts map[string]ltl.Verdict @@ -192,6 +199,12 @@ func (v *Verifier) Load(source string) error { } } + if fn := v.runtime.GlobalObject().Get("__sanderlingSetEnumeratingCandidates__"); fn != nil { + if callable, ok := goja.AssertFunction(fn); ok { + v.setEnumeratingCandidatesFn = callable + } + } + return nil }