diff --git a/internal/verifier/llm.go b/internal/verifier/llm.go index 31f3400..54851e1 100644 --- a/internal/verifier/llm.go +++ b/internal/verifier/llm.go @@ -176,38 +176,59 @@ const ( // labelSource selects the channel each target is named by. An unrecognized // value (including the zero value) names targets by visible text; the CLI // rejects an unknown mode before a run starts, so only a test reaches that. -func (v *Verifier) Candidates(labelSource string) []ActionCandidate { +// +// The error is the spec refusing to be run by this policy at all: an authored +// leaf that samples one of several items reaches the seeded picker's rng but +// never this walk, so the model would be offered a fixed first item forever. It +// names the leaf and it is fatal, because degrading to that fixed item silently +// is what makes a policy comparison meaningless. +func (v *Verifier) Candidates(labelSource string) ([]ActionCandidate, error) { if v.lastTree == nil { - return nil + return nil, nil } root := v.runtime.GlobalObject().Get("actions") if root == nil || goja.IsUndefined(root) || goja.IsNull(root) { - return nil + return nil, nil } labels := labelContext{nodeIndex: buildNodeIndex(v.lastTree), source: labelSource} var raw []ActionCandidate - v.collectNode(root, 1.0, false, labels, &raw) - return finalizeCandidates(raw) + v.setEnumeratingCandidates(true) + defer v.setEnumeratingCandidates(false) + if err := v.collectNode(root, 1.0, false, labels, &raw); err != nil { + return nil, err + } + return finalizeCandidates(raw), nil +} + +// setEnumeratingCandidates tells the spec bundle that the authored leaves are +// being called by this policy rather than by the picker. A spec loaded without +// the runtime entry (a raw-JS unit fixture) has no such callable, and no +// sampler to refuse either. +func (v *Verifier) setEnumeratingCandidates(enumerating bool) { + if v.setEnumeratingCandidatesFn == nil { + return + } + _, _ = v.setEnumeratingCandidatesFn(goja.Undefined(), v.runtime.ToValue(enumerating)) } // collectNode dispatches one GeneratorNode of the action tree. prob is the // accumulated probability the seeded picker reaches this node; weighted records // whether any weighted node lies on the path (so weights are shown only when the // spec actually declared them). -func (v *Verifier) collectNode(node goja.Value, prob float64, weighted bool, labels labelContext, out *[]ActionCandidate) { +func (v *Verifier) collectNode(node goja.Value, prob float64, weighted bool, labels labelContext, out *[]ActionCandidate) error { object := node.ToObject(v.runtime) if object == nil { - return + return nil } kind := object.Get("kind") if kind == nil || goja.IsUndefined(kind) { - return + return nil } switch kind.String() { case "weighted": - v.collectWeighted(object, prob, labels, out) + return v.collectWeighted(object, prob, labels, out) case "actions": - v.collectActions(object, prob, weighted, labels, out) + return v.collectActions(object, prob, weighted, labels, out) case "builtin": verb := object.Get("verb") if verb != nil && !goja.IsUndefined(verb) { @@ -216,19 +237,20 @@ func (v *Verifier) collectNode(node goja.Value, prob float64, weighted bool, lab case "llm": // The llm marker is the generator, not part of the candidate tree. } + return nil } // collectWeighted recurses each branch, splitting the incoming probability by // the branch weight over the sibling total (matching the seeded picker's single // weighted draw). -func (v *Verifier) collectWeighted(object *goja.Object, prob float64, labels labelContext, out *[]ActionCandidate) { +func (v *Verifier) collectWeighted(object *goja.Object, prob float64, labels labelContext, out *[]ActionCandidate) error { branches := object.Get("branches") if branches == nil { - return + return nil } array := branches.ToObject(v.runtime) if array == nil { - return + return nil } length := int(array.Get("length").ToInteger()) weights := make([]float64, length) @@ -249,32 +271,47 @@ func (v *Verifier) collectWeighted(object *goja.Object, prob float64, labels lab total += weight } if total <= 0 { - return + return nil } for i := range length { if children[i] == nil { continue } - v.collectNode(children[i], prob*weights[i]/total, true, labels, out) + // The branch number is the author's own path to a refused leaf, which + // its closure source alone does not give when the leaf is a whenRoute + // (whose closure belongs to the library, not the spec). + if err := v.collectNode(children[i], prob*weights[i]/total, true, labels, out); err != nil { + return fmt.Errorf("branch %d: %w", i+1, err) + } } + return nil } // collectActions calls an authored leaf's generator once (safe: it reads state // and, off-route, returns []), turning each concrete descriptor into a // candidate. It runs OUTSIDE the picker's rng scope, so from(...).generate() // draws nothing and no seed advances. -func (v *Verifier) collectActions(object *goja.Object, prob float64, weighted bool, labels labelContext, out *[]ActionCandidate) { - generate, ok := goja.AssertFunction(object.Get("generate")) +// +// A generator that throws for its own reasons still contributes nothing and +// nothing more: this walk calls EVERY leaf every step, including leaves the +// seeded picker would have walked once in a hundred steps, so promoting those +// throws would kill runs the seeded arm survives. +func (v *Verifier) collectActions(object *goja.Object, prob float64, weighted bool, labels labelContext, out *[]ActionCandidate) error { + generatorValue := object.Get("generate") + generate, ok := goja.AssertFunction(generatorValue) if !ok { - return + return nil } result, err := generate(goja.Undefined()) if err != nil { - return + if refusal, refused := v.samplerRefusal(err); refused { + return fmt.Errorf("authored action %s %s", authoredLeafIdentity(generatorValue), refusal) + } + return nil } array := result.ToObject(v.runtime) if array == nil { - return + return nil } length := int(array.Get("length").ToInteger()) for i := range length { @@ -286,11 +323,54 @@ func (v *Verifier) collectActions(object *goja.Object, prob float64, weighted bo candidate.Weighted = weighted *out = append(*out, candidate) } + return nil +} + +// samplerRefusalName is the error name pkg/spec/src/sampler-rng.ts stamps on the +// refusal it throws, which is what tells that refusal apart from a spec's own +// runtime errors. +const samplerRefusalName = "SanderlingSamplerRefusal" + +// samplerRefusal reports the refusal message when the authored leaf declined to +// sample for this policy. +func (v *Verifier) samplerRefusal(err error) (string, bool) { + var exception *goja.Exception + if !errors.As(err, &exception) { + return "", false + } + value := exception.Value() + if value == nil || goja.IsUndefined(value) || goja.IsNull(value) { + return "", false + } + thrown := value.ToObject(v.runtime) + if thrown == nil || stringField(thrown, "name") != samplerRefusalName { + return "", false + } + return stringField(thrown, "message"), true +} + +// maxLeafSourceRunes caps the generator excerpt that names a leaf in an error. +const maxLeafSourceRunes = 160 + +// authoredLeafIdentity renders the leaf's generator source on one line. An +// authored leaf is an anonymous closure among identical-looking tree nodes, so +// its source is the handle an author can search the spec for. +func authoredLeafIdentity(generator goja.Value) string { + source := []rune(strings.Join(strings.Fields(generator.String()), " ")) + if len(source) > maxLeafSourceRunes { + return strconv.Quote(string(source[:maxLeafSourceRunes]) + "...") + } + return strconv.Quote(string(source)) } // candidateFromDescriptor lowers one authored ActionDescriptor (as a goja // object) into a ready-to-run candidate, resolving the target's coordinates, -// selector, and label. Actions on a disabled control are dropped. +// selector, and label. +// +// A disabled target is offered like any other. The seeded picker executes +// whatever the leaf authored, disabled or not, and attempting a disabled +// control is where boundary defects live: a control the app forgot to re-enable +// reads as disabled, and a policy that cannot attempt it cannot find that. func (v *Verifier) candidateFromDescriptor(value goja.Value, labels labelContext) (ActionCandidate, bool) { object := value.ToObject(v.runtime) if object == nil { @@ -304,7 +384,7 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, labels labelContext switch kind { case ActionKindTap, ActionKindDoubleTap, ActionKindLongPress: target, ok := v.resolveTarget(object.Get("on"), labels) - if !ok || target.disabled { + if !ok { return ActionCandidate{}, false } return ActionCandidate{ @@ -314,7 +394,7 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, labels labelContext }, true case ActionKindInputText: target, ok := v.resolveTarget(object.Get("into"), labels) - if !ok || target.disabled { + if !ok { return ActionCandidate{}, false } text := stringField(object, "text") @@ -384,7 +464,6 @@ type resolvedTarget struct { selector string label string inputType string - disabled bool } // resolveTarget reads an authored action's target. Ax element handles carry @@ -423,7 +502,6 @@ func (v *Verifier) resolveTarget(value goja.Value, labels labelContext) (resolve if element := v.findBySelector(selector); element != nil { target.label = labels.label(element) target.inputType = inputTypeHint(element) - target.disabled = !element.Enabled && hasEnabled(element) } if target.label == "" { target.label = truncateLabel(stringField(object, labels.handleField())) @@ -441,7 +519,6 @@ func (v *Verifier) targetFromSelector(selector string, labels labelContext) reso target.x, target.y = element.Bounds.Center() target.label = labels.label(element) target.inputType = inputTypeHint(element) - target.disabled = !element.Enabled && hasEnabled(element) return target } @@ -754,13 +831,6 @@ func inputTypeHint(element *hierarchy.Element) string { } } -// hasEnabled reports whether the source tree carried an explicit enabled flag -// for the element, so a missing flag is not mistaken for "disabled". -func hasEnabled(element *hierarchy.Element) bool { - _, ok := element.Attributes["enabled"] - return ok -} - // stringField reads a string property off a goja object, returning "" when // absent, null, or undefined. func stringField(object *goja.Object, key string) string { diff --git a/internal/verifier/llm_test.go b/internal/verifier/llm_test.go index e4502e3..30005eb 100644 --- a/internal/verifier/llm_test.go +++ b/internal/verifier/llm_test.go @@ -69,6 +69,17 @@ func enumVerifier(t *testing.T, actionsJS, treeJSON string) *Verifier { return v } +// mustCandidates enumerates the model policy's list, failing the test on the +// refusal an authored multi-item sampler raises. +func mustCandidates(t *testing.T, v *Verifier, labelSource string) []ActionCandidate { + t.Helper() + candidates, err := v.Candidates(labelSource) + if err != nil { + t.Fatalf("Candidates: %v", err) + } + return candidates +} + func findCandidate(candidates []ActionCandidate, description string) (ActionCandidate, bool) { for _, candidate := range candidates { if candidate.Description == description { @@ -85,7 +96,7 @@ func hasCandidate(candidates []ActionCandidate, description string) bool { func TestCandidatesLabelsControlsByVisibleText(t *testing.T) { v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", enumTreeJSON) - candidates := v.Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, v, LabelSourceVisibleText) // The empty-text clickable wrapper is labeled by its child Text, NOT its // resource-id. @@ -109,8 +120,7 @@ func TestCandidatesLabelsControlsByVisibleText(t *testing.T) { } func TestCandidatesLabelsControlsByResourceIdentifier(t *testing.T) { - candidates := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). - Candidates(LabelSourceResourceID) + candidates := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON), LabelSourceResourceID) if !hasCandidate(candidates, `Tap "add_credit_button"`) { t.Errorf("want the control named by its identifier, got %v", descriptions(candidates)) @@ -125,8 +135,7 @@ func TestCandidatesLabelsControlsByResourceIdentifier(t *testing.T) { } func TestCandidatesIdentifierChannelFallsBackToClassThenBareControl(t *testing.T) { - candidates := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). - Candidates(LabelSourceResourceID) + candidates := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON), LabelSourceResourceID) if !hasCandidate(candidates, `Tap "android.widget.CheckBox"`) { t.Errorf("a control with no identifier falls back to its class, got %v", descriptions(candidates)) @@ -143,10 +152,8 @@ func TestCandidatesIdentifierChannelFallsBackToClassThenBareControl(t *testing.T // own action, so the model can act on either by number. A channel that renames // controls must never shrink the action space. func TestCandidatesIdentifierChannelKeepsControlsItCannotNameApartReachable(t *testing.T) { - text := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). - Candidates(LabelSourceVisibleText) - identifier := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). - Candidates(LabelSourceResourceID) + text := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON), LabelSourceVisibleText) + identifier := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON), LabelSourceResourceID) if count(text, `Tap "Remember me"`) != 1 || count(text, `Tap "Stay signed in"`) != 1 { t.Fatalf("the text channel should address both checkboxes, got %v", descriptions(text)) @@ -171,8 +178,7 @@ func TestCandidatesIdentifierChannelKeepsControlsItCannotNameApartReachable(t *t // on the rendered line dropped the second one, putting it out of reach of any // prompt or policy. func TestCandidatesReachBothControlsSharingOneVisibleLabel(t *testing.T) { - candidates := enumVerifier(t, "{kind:'builtin', verb:'taps'}", sharedLabelTreeJSON). - Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'taps'}", sharedLabelTreeJSON), LabelSourceVisibleText) deletes := candidatesMatching(candidates, `Tap "Delete"`) if len(deletes) != 2 { @@ -193,8 +199,7 @@ func TestCandidatesReachBothControlsSharingOneVisibleLabel(t *testing.T) { // agree: a control carrying nothing readable is named by its identifier in both, // so a screen built entirely from such controls is one cell, not two. func TestCandidatesVisibleTextFallsBackToTheIdentifier(t *testing.T) { - candidates := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). - Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON), LabelSourceVisibleText) if !hasCandidate(candidates, `Tap "silent_row"`) { t.Errorf("a control with no readable text falls back to its identifier, got %v", @@ -203,14 +208,12 @@ func TestCandidatesVisibleTextFallsBackToTheIdentifier(t *testing.T) { } func TestCandidatesTypingLabelFollowsTheLabelSource(t *testing.T) { - text := enumVerifier(t, "{kind:'builtin', verb:'typing'}", labelChannelTreeJSON). - Candidates(LabelSourceVisibleText) + text := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'typing'}", labelChannelTreeJSON), LabelSourceVisibleText) if !hasCandidate(text, `Type into "Amount" (number)`) { t.Errorf("want the field named by its hint, got %v", descriptions(text)) } - identifier := enumVerifier(t, "{kind:'builtin', verb:'typing'}", labelChannelTreeJSON). - Candidates(LabelSourceResourceID) + identifier := mustCandidates(t, enumVerifier(t, "{kind:'builtin', verb:'typing'}", labelChannelTreeJSON), LabelSourceResourceID) if !hasCandidate(identifier, `Type into "amount_field" (number)`) { t.Errorf("want the field named by its identifier, got %v", descriptions(identifier)) } @@ -243,8 +246,8 @@ func TestLabelSourceChangesOnlyTheDescription(t *testing.T) { } for _, fixture := range fixtures { t.Run(fixture.name, func(t *testing.T) { - text := enumVerifier(t, everyLabelledVerb, fixture.tree).Candidates(LabelSourceVisibleText) - identifier := enumVerifier(t, everyLabelledVerb, fixture.tree).Candidates(LabelSourceResourceID) + text := mustCandidates(t, enumVerifier(t, everyLabelledVerb, fixture.tree), LabelSourceVisibleText) + identifier := mustCandidates(t, enumVerifier(t, everyLabelledVerb, fixture.tree), LabelSourceResourceID) if len(text) == 0 { t.Fatal("fixture yielded no candidates") } @@ -269,9 +272,9 @@ func TestLabelSourceChangesOnlyTheDescription(t *testing.T) { } } -func TestCandidatesDropsDisabledControls(t *testing.T) { +func TestCandidatesDropsDisabledControlsFromBuiltinVerbs(t *testing.T) { v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", enumTreeJSON) - for _, candidate := range v.Candidates(LabelSourceVisibleText) { + for _, candidate := range mustCandidates(t, v, LabelSourceVisibleText) { if strings.Contains(candidate.Description, "Off") { t.Errorf("disabled control surfaced as %q", candidate.Description) } @@ -280,7 +283,7 @@ func TestCandidatesDropsDisabledControls(t *testing.T) { func TestCandidatesTypingExposesInputType(t *testing.T) { v := enumVerifier(t, "{kind:'builtin', verb:'typing'}", enumTreeJSON) - candidates := v.Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, v, LabelSourceVisibleText) candidate, ok := findCandidate(candidates, `Type into "Amount" (number)`) if !ok { t.Fatalf("want typing candidate with input type, got %v", descriptions(candidates)) @@ -303,7 +306,7 @@ func TestCandidatesLabelsEditableFieldByHintNotTypedValue(t *testing.T) { ] }` v := enumVerifier(t, "{kind:'builtin', verb:'typing'}", tree) - candidates := v.Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, v, LabelSourceVisibleText) if hasCandidate(candidates, `Type into "99" (number)`) || hasCandidate(candidates, `Type into "99"`) { t.Errorf("editable field labeled by its typed value: %v", descriptions(candidates)) } @@ -316,7 +319,7 @@ func TestCandidatesKeepsGestureVerbsDistinct(t *testing.T) { v := enumVerifier(t, "{kind:'weighted', branches:[[1,{kind:'builtin',verb:'scrolls'}],[1,{kind:'builtin',verb:'swipes'}]]}", enumTreeJSON) - candidates := v.Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, v, LabelSourceVisibleText) // `scrolls` folds to one directional pair over the single scrollable // container, which is what keeps the list short. @@ -358,7 +361,7 @@ func TestCandidatesWeightsCombineAcrossPaths(t *testing.T) { v := enumVerifier(t, "{kind:'weighted', branches:[[1,{kind:'builtin',verb:'taps'}],[1,{kind:'builtin',verb:'taps'}]]}", oneClickable) - candidates := v.Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, v, LabelSourceVisibleText) if len(candidates) != 1 { t.Fatalf("want one deduped candidate, got %v", descriptions(candidates)) } @@ -375,7 +378,7 @@ func TestCandidatesWeightReflectsBranchShare(t *testing.T) { v := enumVerifier(t, "{kind:'weighted', branches:[[1,{kind:'builtin',verb:'taps'}],[3,{kind:'builtin',verb:'typing'}]]}", enumTreeJSON) - candidates := v.Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, v, LabelSourceVisibleText) tap, ok := findCandidate(candidates, `Tap "Sign in"`) if !ok { t.Fatalf("missing tap candidate: %v", descriptions(candidates)) @@ -394,7 +397,7 @@ func TestCandidatesWeightReflectsBranchShare(t *testing.T) { func TestCandidatesUnweightedTreeShowsNoWeight(t *testing.T) { v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", enumTreeJSON) - for _, candidate := range v.Candidates(LabelSourceVisibleText) { + for _, candidate := range mustCandidates(t, v, LabelSourceVisibleText) { if candidate.Weighted || candidate.Weight != 0 { t.Errorf("%q carries a weight despite no weighted node", candidate.Description) } @@ -408,17 +411,17 @@ func TestCandidatesCallsAuthoredLeafOnce(t *testing.T) { {kind:'InputText', into:'id:Amount', text:'42'} ]}` v := enumVerifier(t, actions, enumTreeJSON) - candidates := v.Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, v, LabelSourceVisibleText) // Authored Tap resolves its selector to the visible-text label. if !hasCandidate(candidates, `Tap "Sign in"`) { t.Errorf("authored tap missing: %v", descriptions(candidates)) } - // A disabled authored target is dropped. - for _, candidate := range candidates { - if strings.Contains(candidate.Description, "Off") { - t.Errorf("authored action on disabled control surfaced: %q", candidate.Description) - } + // A disabled authored target is offered, not dropped: the seeded picker + // executes it, and attempting a disabled control is where boundary defects + // live, so a policy that cannot attempt it cannot find them. + if !hasCandidate(candidates, `Tap "Off"`) { + t.Errorf("authored action on a disabled control was dropped: %v", descriptions(candidates)) } // Authored InputText replays its own sampled value (LLM does not supply it). authored, ok := findCandidate(candidates, `Type "42" into "Amount"`) @@ -442,7 +445,7 @@ func TestCandidatesSurfaceAuthoredUntargetedActions(t *testing.T) { {kind:'PressKey', key:'back'}, {kind:'Wait'} ]}` - candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, actions, enumTreeJSON), LabelSourceVisibleText) for _, want := range []string{"Swipe from (10,600) to (10,100)", "Press back", "Wait"} { if !hasCandidate(candidates, want) { t.Errorf("authored %q missing: %v", want, descriptions(candidates)) @@ -455,7 +458,7 @@ func TestCandidatesSurfaceAuthoredUntargetedActions(t *testing.T) { // idling on paper while the seeded arm really waits. func TestCandidatesAuthoredWaitKeepsItsDuration(t *testing.T) { actions := `{kind:'actions', generate: () => [{kind:'Wait', durationMillis: 500}]}` - candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, actions, enumTreeJSON), LabelSourceVisibleText) candidate, ok := findCandidate(candidates, "Wait") if !ok { t.Fatalf("authored wait missing: %v", descriptions(candidates)) @@ -470,7 +473,7 @@ func TestCandidatesAuthoredWaitKeepsItsDuration(t *testing.T) { // the screen and the scroll lands on whatever else is scrollable. func TestCandidatesAuthoredScrollNamesItsContainer(t *testing.T) { actions := `{kind:'actions', generate: () => [{kind:'Scroll', direction:'down', in:'id:List'}]}` - candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, actions, enumTreeJSON), LabelSourceVisibleText) candidate, ok := findCandidate(candidates, "Scroll down") if !ok { t.Fatalf("authored scroll missing: %v", descriptions(candidates)) @@ -489,7 +492,7 @@ func TestCandidatesAuthoredScrollKeepsPrecomputedEndpoints(t *testing.T) { actions := `{kind:'actions', generate: () => [ {kind:'Scroll', direction:'down', in:'id:List', from:{x:540,y:1400}, to:{x:540,y:920}} ]}` - candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, actions, enumTreeJSON), LabelSourceVisibleText) candidate, ok := findCandidate(candidates, "Scroll down") if !ok { t.Fatalf("authored scroll missing: %v", descriptions(candidates)) @@ -510,7 +513,7 @@ func TestCandidatesAuthoredSwipeDefaultsItsDuration(t *testing.T) { {kind:'Swipe', from:{x:10,y:600}, to:{x:10,y:100}}, {kind:'Swipe', from:{x:20,y:600}, to:{x:20,y:100}, durationMillis: 400} ]}` - candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, actions, enumTreeJSON), LabelSourceVisibleText) omitted, ok := findCandidate(candidates, "Swipe from (10,600) to (10,100)") if !ok { t.Fatalf("authored swipe missing: %v", descriptions(candidates)) @@ -538,7 +541,7 @@ func TestCandidatesDropTargetsThatResolveToNothing(t *testing.T) { {kind:'InputText', into: {}, text:'x'}, {kind:'Swipe', from:{x:1,y:2}, to:{}} ]}` - candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, actions, enumTreeJSON), LabelSourceVisibleText) if len(candidates) != 0 { t.Errorf("targetless actions reached the model: %v", descriptions(candidates)) } @@ -549,7 +552,7 @@ func TestCandidatesDropTargetsThatResolveToNothing(t *testing.T) { // a target with no coordinates rather than on coordinates that are zero. func TestCandidatesKeepATargetOnTheScreenOrigin(t *testing.T) { actions := `{kind:'actions', generate: () => [{kind:'Tap', on: {x: 0, y: 0}}]}` - candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) + candidates := mustCandidates(t, enumVerifier(t, actions, enumTreeJSON), LabelSourceVisibleText) if len(candidates) != 1 { t.Fatalf("want the origin tap kept, got %v", descriptions(candidates)) } @@ -557,7 +560,7 @@ func TestCandidatesKeepATargetOnTheScreenOrigin(t *testing.T) { func TestCandidatesOffRouteLeafYieldsNothing(t *testing.T) { v := enumVerifier(t, "{kind:'actions', generate: () => []}", enumTreeJSON) - if got := v.Candidates(LabelSourceVisibleText); len(got) != 0 { + if got := mustCandidates(t, v, LabelSourceVisibleText); len(got) != 0 { t.Errorf("off-route leaf should yield no candidates, got %v", descriptions(got)) } } @@ -575,7 +578,7 @@ func TestCandidatesSkipsCrossFadeFrames(t *testing.T) { ] }` v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", crossFade) - if got := v.Candidates(LabelSourceVisibleText); len(got) != 0 { + if got := mustCandidates(t, v, LabelSourceVisibleText); len(got) != 0 { t.Errorf("cross-fade frame should yield no candidates, got %v", descriptions(got)) } // The seeded policy is skipped by the SAME guard, in the shared producer, @@ -587,13 +590,13 @@ func TestCandidatesSkipsCrossFadeFrames(t *testing.T) { func TestCandidatesNilWithoutTreeOrActions(t *testing.T) { withActions := newLoadedVerifier(t, "globalThis.actions = {kind:'builtin', verb:'taps'};") - if got := withActions.Candidates(LabelSourceVisibleText); got != nil { + if got := mustCandidates(t, withActions, LabelSourceVisibleText); got != nil { t.Errorf("Candidates with no tree = %v, want nil", got) } noActions := newLoadedVerifier(t, "globalThis.properties = {};") tree, _ := hierarchy.Parse(enumTreeJSON) noActions.lastTree = tree - if got := noActions.Candidates(LabelSourceVisibleText); got != nil { + if got := mustCandidates(t, noActions, LabelSourceVisibleText); got != nil { t.Errorf("Candidates with no actions root = %v, want nil", got) } } @@ -687,3 +690,52 @@ func newLoadedVerifier(t *testing.T, source string) *Verifier { } return v } + +// samplerSpec authors one leaf that taps a target drawn from the given list. +func samplerSpec(items string) string { + return ` +import { actions, from, Tap } from "@sanderling/spec"; +const targets = from(` + items + `); +globalThis.actions = actions(() => [Tap({ on: targets.generate() })]); +` +} + +// TestCandidatesRefuseAMultiItemAuthoredSampler pins the refusal: a sampler +// reads the picker's rng, which this policy has no way to enter, so the draw +// would collapse to the first item on every step while the seeded picker keeps +// reaching all three. Offering that silently is what would make a comparison of +// the two policies meaningless, so the spec is refused instead. +func TestCandidatesRefuseAMultiItemAuthoredSampler(t *testing.T) { + v := newVerifier(t) + loadActionSpec(t, v, samplerSpec(`["id:SignIn", "id:Amount", "id:List"]`)) + pushTree(t, v, enumTreeJSON) + + _, err := v.Candidates(LabelSourceVisibleText) + if err == nil { + t.Fatal("a multi-item authored sampler must refuse to run under the model policy") + } + message := err.Error() + if !strings.Contains(message, "targets.generate()") { + t.Errorf("error does not name the offending leaf, so the author cannot find it: %s", message) + } + if !strings.Contains(message, "draws 1 of 3 sampled items") { + t.Errorf("error does not say what the leaf did: %s", message) + } +} + +// TestCandidatesAcceptASingleItemAuthoredSampler: a one-item sampler short +// circuits before the rng, so both policies get that one value and there is no +// divergence to refuse. +func TestCandidatesAcceptASingleItemAuthoredSampler(t *testing.T) { + v := newVerifier(t) + loadActionSpec(t, v, samplerSpec(`["id:SignIn"]`)) + pushTree(t, v, enumTreeJSON) + + candidates, err := v.Candidates(LabelSourceVisibleText) + if err != nil { + t.Fatalf("a single-item sampler is not a divergence: %v", err) + } + if !hasCandidate(candidates, `Tap "Sign in"`) { + t.Errorf("sampled tap missing: %v", descriptions(candidates)) + } +} diff --git a/internal/verifier/policy_parity_test.go b/internal/verifier/policy_parity_test.go index 5c477ee..c71fa9c 100644 --- a/internal/verifier/policy_parity_test.go +++ b/internal/verifier/policy_parity_test.go @@ -62,7 +62,7 @@ func TestModelCandidateDescriptionsAreUniqueAndNamed(t *testing.T) { t.Run(verb, func(t *testing.T) { verifier := loadVerbSpec(t, verb) seen := map[string]bool{} - for _, candidate := range verifier.Candidates(LabelSourceVisibleText) { + for _, candidate := range mustCandidates(t, verifier, LabelSourceVisibleText) { if candidate.Description == "" { t.Fatalf("%s produced a candidate with no description: %+v", verb, candidate.Action) } @@ -81,14 +81,14 @@ func TestModelCandidateDescriptionsAreUniqueAndNamed(t *testing.T) { // dropped, so the model could never navigate back or let the app settle. func TestModelIsOfferedTheUntargetedVerbs(t *testing.T) { verifier := loadVerbSpec(t, "pressKeys") - if !hasCandidate(verifier.Candidates(LabelSourceVisibleText), "Press back") { + if !hasCandidate(mustCandidates(t, verifier, LabelSourceVisibleText), "Press back") { t.Errorf("pressKeys missing from the model's candidates: %v", - descriptions(verifier.Candidates(LabelSourceVisibleText))) + descriptions(mustCandidates(t, verifier, LabelSourceVisibleText))) } verifier = loadVerbSpec(t, "waitOnce") - if !hasCandidate(verifier.Candidates(LabelSourceVisibleText), "Wait") { + if !hasCandidate(mustCandidates(t, verifier, LabelSourceVisibleText), "Wait") { t.Errorf("waitOnce missing from the model's candidates: %v", - descriptions(verifier.Candidates(LabelSourceVisibleText))) + descriptions(mustCandidates(t, verifier, LabelSourceVisibleText))) } } @@ -124,7 +124,7 @@ func seededDrawStream(t *testing.T, verb, labelSource string) []string { stream := make([]string, 0, seededDrawBudget) for range seededDrawBudget { if labelSource != "" { - verifier.Candidates(labelSource) + mustCandidates(t, verifier, labelSource) } action, err := verifier.NextAction() if errors.Is(err, ErrNoAction) { @@ -175,7 +175,7 @@ func modelOfferedActions(t *testing.T, verb string) map[string]Action { t.Helper() verifier := loadVerbSpec(t, verb) offered := map[string]Action{} - for _, candidate := range verifier.Candidates(LabelSourceVisibleText) { + for _, candidate := range mustCandidates(t, verifier, LabelSourceVisibleText) { offered[actionIdentity(candidate.Action)] = candidate.Action } return offered @@ -206,3 +206,53 @@ func sign(value int) int { return 0 } } + +// samplerParitySpec authors one leaf that taps a target drawn from three: the +// pattern the model policy refuses and the seeded picker exists to draw. +const samplerParitySpec = ` +import { actions, from, Tap } from "@sanderling/spec"; +const targets = from(["id:Save", "id:Cancel", "id:Amount"]); +globalThis.actions = actions(() => [Tap({ on: targets.generate() })]); +` + +// TestSeededSamplingSurvivesTheModelPolicysRefusal keeps the refusal on the one +// policy it belongs to. Sampling inside the picker's rng scope is correct, so +// the seeded stream must be identical whether or not the model policy tried and +// failed to enumerate the same leaf first, and it must still reach every item. +func TestSeededSamplingSurvivesTheModelPolicysRefusal(t *testing.T) { + alone := seededSamplerStream(t, false) + afterRefusal := seededSamplerStream(t, true) + if !slices.Equal(alone, afterRefusal) { + t.Error("a refused enumeration moved the seeded draw stream") + } + drawn := map[string]bool{} + for _, action := range alone { + drawn[action] = true + } + if len(drawn) != 3 { + t.Errorf("seeded picker reached %d of the 3 sampled targets: %v", len(drawn), slices.Sorted(maps.Keys(drawn))) + } +} + +// seededSamplerStream drives the seeded picker over the draw budget, optionally +// letting the model policy refuse the same spec before every draw. +func seededSamplerStream(t *testing.T, enumerateFirst bool) []string { + t.Helper() + verifier := newVerifier(t, WithSeed(0x5eed)) + loadActionSpec(t, verifier, samplerParitySpec) + pushTree(t, verifier, policyTreeJSON) + stream := make([]string, 0, seededDrawBudget) + for range seededDrawBudget { + if enumerateFirst { + if _, err := verifier.Candidates(LabelSourceVisibleText); err == nil { + t.Fatal("the model policy must refuse a spec that samples") + } + } + action, err := verifier.NextAction() + if err != nil { + t.Fatalf("seeded picker declined the sampled tap: %v", err) + } + stream = append(stream, fmt.Sprintf("%+v", action)) + } + return stream +} diff --git a/internal/verifier/worker.go b/internal/verifier/worker.go index 82ac57b..793a279 100644 --- a/internal/verifier/worker.go +++ b/internal/verifier/worker.go @@ -45,6 +45,13 @@ type Verifier struct { // over the picker's action space rather than one of its own. enumerateBuiltinFn goja.Callable + // setEnumeratingCandidatesFn is the bundle-installed + // __sanderlingSetEnumeratingCandidates__, which brackets the model policy's + // authored-leaf calls. Those run outside the picker's rng scope, where a + // sampler would quietly hand back its first item, so the bundle refuses to + // sample while it is set. + setEnumeratingCandidatesFn goja.Callable + evaluators map[string]*ltl.Evaluator priorVerdicts map[string]ltl.Verdict @@ -192,6 +199,12 @@ func (v *Verifier) Load(source string) error { } } + if fn := v.runtime.GlobalObject().Get("__sanderlingSetEnumeratingCandidates__"); fn != nil { + if callable, ok := goja.AssertFunction(fn); ok { + v.setEnumeratingCandidatesFn = callable + } + } + return nil }