From 8bb20c472160ac9a0edf79ef9f5f81df197108bc Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 12 Aug 2026 23:19:24 +0530 Subject: [PATCH] feat(verifier): select the candidate label source Candidates takes the label source as an argument rather than storing it, which is what keeps the asymmetry structural: the seeded picker selects by index and never calls Candidates, so the mode cannot reach it. That asymmetry is load-bearing, because it makes the two seeded cells of the factorial a manipulation check with identical draw streams. The identifier ladder deliberately has no text rung. A fallback that reached for text would silently turn one arm back into the other. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX --- internal/verifier/llm.go | 116 ++++++++++++++----- internal/verifier/llm_test.go | 146 +++++++++++++++++++++--- internal/verifier/policy_parity_test.go | 59 +++++++++- 3 files changed, 272 insertions(+), 49 deletions(-) diff --git a/internal/verifier/llm.go b/internal/verifier/llm.go index c27cdf6..8c8fabd 100644 --- a/internal/verifier/llm.go +++ b/internal/verifier/llm.go @@ -115,7 +115,8 @@ type ActionCandidate struct { // Description is the rendered action shown to the model and echoed back as // chosen_action, e.g. `Tap "Add credit"`. Dedup keys on it, so it is unique. Description string - // Label is the visible-text target label (empty for gestures). + // Label is the target label the selected LabelSource named (empty for + // gestures). Label string // Weight is the effective selection weight as a percentage (1..100), // meaningful only when Weighted is true (the tree used `weighted`). @@ -142,6 +143,19 @@ type ActionCandidate struct { // enough to render on one numbered line. const maxLabelRunes = 40 +// The label sources a candidate's target can be named by. This is the +// observation channel the model reads, and nothing else: the seeded picker +// selects by index and never asks for a label, so the two seeded cells of a +// labelling factorial draw the identical stream. +const ( + // LabelSourceVisibleText names a control by what a user would read. It is + // the default, and the channel every run so far was produced with. + LabelSourceVisibleText = "visible-text" + // LabelSourceResourceID names a control by the identifier the app assigned + // it, which no user ever sees. + LabelSourceResourceID = "resource-id" +) + // Candidates enumerates every action the spec's weighted actionsRoot yields at // the current step, each tagged with a plainly-worded description and its // effective weight, for the LLM generator to pick one number from. It walks the @@ -149,7 +163,11 @@ const maxLabelRunes = 40 // selection probability), authored actions()/whenRoute leaves are called once // for their concrete actions, and builtin verbs come straight from the picker's // own enumeration. Identical descriptions dedup, summing weight. -func (v *Verifier) Candidates() []ActionCandidate { +// +// labelSource selects the channel each target is named by. An unrecognized +// value (including the zero value) names targets by visible text; the CLI +// rejects an unknown mode before a run starts, so only a test reaches that. +func (v *Verifier) Candidates(labelSource string) []ActionCandidate { if v.lastTree == nil { return nil } @@ -157,9 +175,9 @@ func (v *Verifier) Candidates() []ActionCandidate { if root == nil || goja.IsUndefined(root) || goja.IsNull(root) { return nil } - nodeIndex := buildNodeIndex(v.lastTree) + labels := labelContext{nodeIndex: buildNodeIndex(v.lastTree), source: labelSource} var raw []ActionCandidate - v.collectNode(root, 1.0, false, nodeIndex, &raw) + v.collectNode(root, 1.0, false, labels, &raw) return finalizeCandidates(raw) } @@ -167,7 +185,7 @@ func (v *Verifier) Candidates() []ActionCandidate { // accumulated probability the seeded picker reaches this node; weighted records // whether any weighted node lies on the path (so weights are shown only when the // spec actually declared them). -func (v *Verifier) collectNode(node goja.Value, prob float64, weighted bool, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) { +func (v *Verifier) collectNode(node goja.Value, prob float64, weighted bool, labels labelContext, out *[]ActionCandidate) { object := node.ToObject(v.runtime) if object == nil { return @@ -178,13 +196,13 @@ func (v *Verifier) collectNode(node goja.Value, prob float64, weighted bool, nod } switch kind.String() { case "weighted": - v.collectWeighted(object, prob, nodeIndex, out) + v.collectWeighted(object, prob, labels, out) case "actions": - v.collectActions(object, prob, weighted, nodeIndex, out) + v.collectActions(object, prob, weighted, labels, out) case "builtin": verb := object.Get("verb") if verb != nil && !goja.IsUndefined(verb) { - v.collectBuiltin(verb.String(), prob, weighted, nodeIndex, out) + v.collectBuiltin(verb.String(), prob, weighted, labels, out) } case "llm": // The llm marker is the generator, not part of the candidate tree. @@ -194,7 +212,7 @@ func (v *Verifier) collectNode(node goja.Value, prob float64, weighted bool, nod // collectWeighted recurses each branch, splitting the incoming probability by // the branch weight over the sibling total (matching the seeded picker's single // weighted draw). -func (v *Verifier) collectWeighted(object *goja.Object, prob float64, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) { +func (v *Verifier) collectWeighted(object *goja.Object, prob float64, labels labelContext, out *[]ActionCandidate) { branches := object.Get("branches") if branches == nil { return @@ -228,7 +246,7 @@ func (v *Verifier) collectWeighted(object *goja.Object, prob float64, nodeIndex if children[i] == nil { continue } - v.collectNode(children[i], prob*weights[i]/total, true, nodeIndex, out) + v.collectNode(children[i], prob*weights[i]/total, true, labels, out) } } @@ -236,7 +254,7 @@ func (v *Verifier) collectWeighted(object *goja.Object, prob float64, nodeIndex // and, off-route, returns []), turning each concrete descriptor into a // candidate. It runs OUTSIDE the picker's rng scope, so from(...).generate() // draws nothing and no seed advances. -func (v *Verifier) collectActions(object *goja.Object, prob float64, weighted bool, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) { +func (v *Verifier) collectActions(object *goja.Object, prob float64, weighted bool, labels labelContext, out *[]ActionCandidate) { generate, ok := goja.AssertFunction(object.Get("generate")) if !ok { return @@ -251,7 +269,7 @@ func (v *Verifier) collectActions(object *goja.Object, prob float64, weighted bo } length := int(array.Get("length").ToInteger()) for i := range length { - candidate, ok := v.candidateFromDescriptor(array.Get(strconv.Itoa(i)), nodeIndex) + candidate, ok := v.candidateFromDescriptor(array.Get(strconv.Itoa(i)), labels) if !ok { continue } @@ -263,8 +281,8 @@ func (v *Verifier) collectActions(object *goja.Object, prob float64, weighted bo // candidateFromDescriptor lowers one authored ActionDescriptor (as a goja // object) into a ready-to-run candidate, resolving the target's coordinates, -// selector, and visible-text label. Actions on a disabled control are dropped. -func (v *Verifier) candidateFromDescriptor(value goja.Value, nodeIndex map[*hierarchy.Element]*hierarchy.Node) (ActionCandidate, bool) { +// selector, and label. Actions on a disabled control are dropped. +func (v *Verifier) candidateFromDescriptor(value goja.Value, labels labelContext) (ActionCandidate, bool) { object := value.ToObject(v.runtime) if object == nil { return ActionCandidate{}, false @@ -276,7 +294,7 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, nodeIndex map[*hier kind := ActionKind(kindValue.String()) switch kind { case ActionKindTap, ActionKindDoubleTap, ActionKindLongPress: - target := v.resolveTarget(object.Get("on"), nodeIndex) + target := v.resolveTarget(object.Get("on"), labels) if target.disabled { return ActionCandidate{}, false } @@ -286,7 +304,7 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, nodeIndex map[*hier Action: Action{Kind: kind, On: target.selector, X: target.x, Y: target.y}, }, true case ActionKindInputText: - target := v.resolveTarget(object.Get("into"), nodeIndex) + target := v.resolveTarget(object.Get("into"), labels) if target.disabled { return ActionCandidate{}, false } @@ -298,7 +316,7 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, nodeIndex map[*hier Action: Action{Kind: kind, On: target.selector, X: target.x, Y: target.y, Text: text}, }, true case ActionKindScroll: - target := v.resolveTarget(object.Get("in"), nodeIndex) + target := v.resolveTarget(object.Get("in"), labels) direction := stringField(object, "direction") if direction == "" { direction = "down" @@ -309,8 +327,8 @@ func (v *Verifier) candidateFromDescriptor(value goja.Value, nodeIndex map[*hier Action: Action{Kind: kind, On: target.selector, Direction: direction}, }, true case ActionKindSwipe: - from := v.resolveTarget(object.Get("from"), nodeIndex) - to := v.resolveTarget(object.Get("to"), nodeIndex) + from := v.resolveTarget(object.Get("from"), labels) + to := v.resolveTarget(object.Get("to"), labels) return ActionCandidate{ Kind: kind, Action: Action{ @@ -343,14 +361,14 @@ type resolvedTarget struct { } // resolveTarget reads an authored action's target. Ax element handles carry -// x/y/__sanderlingSelector plus their own text; a bare selector string resolves -// against the current tree; a point carries geometry only. -func (v *Verifier) resolveTarget(value goja.Value, nodeIndex map[*hierarchy.Element]*hierarchy.Node) resolvedTarget { +// x/y/__sanderlingSelector plus their own text and id; a bare selector string +// resolves against the current tree; a point carries geometry only. +func (v *Verifier) resolveTarget(value goja.Value, labels labelContext) resolvedTarget { if value == nil || goja.IsUndefined(value) || goja.IsNull(value) { return resolvedTarget{} } if selector, ok := value.Export().(string); ok { - return v.targetFromSelector(selector, nodeIndex) + return v.targetFromSelector(selector, labels) } object := value.ToObject(v.runtime) if object == nil { @@ -366,25 +384,25 @@ func (v *Verifier) resolveTarget(value goja.Value, nodeIndex map[*hierarchy.Elem selector: selector, } if element := v.findBySelector(selector); element != nil { - target.label = visibleLabel(element, nodeIndex) + target.label = labels.label(element) target.inputType = inputTypeHint(element) target.disabled = !element.Enabled && hasEnabled(element) } if target.label == "" { - target.label = truncateLabel(stringField(object, "text")) + target.label = truncateLabel(stringField(object, labels.handleField())) } return target } // targetFromSelector resolves a bare selector-string target against the tree. -func (v *Verifier) targetFromSelector(selector string, nodeIndex map[*hierarchy.Element]*hierarchy.Node) resolvedTarget { +func (v *Verifier) targetFromSelector(selector string, labels labelContext) resolvedTarget { target := resolvedTarget{selector: selector} element := v.findBySelector(selector) if element == nil { return target } target.x, target.y = element.Bounds.Center() - target.label = visibleLabel(element, nodeIndex) + target.label = labels.label(element) target.inputType = inputTypeHint(element) target.disabled = !element.Enabled && hasEnabled(element) return target @@ -403,7 +421,7 @@ func (v *Verifier) findBySelector(selector string) *hierarchy.Element { // the two policies select over one action space and cannot drift apart. Each // entry's action arrives on the wire contract DecodeAction already reads, so a // chosen candidate executes the action the seeded draw would have executed. -func (v *Verifier) collectBuiltin(verb string, prob float64, weighted bool, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) { +func (v *Verifier) collectBuiltin(verb string, prob float64, weighted bool, labels labelContext, out *[]ActionCandidate) { entries, err := v.enumerateBuiltin(verb) if err != nil { return @@ -422,7 +440,7 @@ func (v *Verifier) collectBuiltin(verb string, prob float64, weighted bool, node } if entry.targetIndex >= 0 && entry.targetIndex < len(targets) { element := targets[entry.targetIndex].element - candidate.Label = visibleLabel(element, nodeIndex) + candidate.Label = labels.label(element) candidate.InputType = inputTypeHint(element) } *out = append(*out, candidate) @@ -558,6 +576,46 @@ func buildNodeIndex(tree *hierarchy.Tree) map[*hierarchy.Element]*hierarchy.Node return index } +// labelContext carries what naming a candidate's target takes: the node index +// descendant text is borrowed through, and the channel the name comes from. +type labelContext struct { + nodeIndex map[*hierarchy.Element]*hierarchy.Node + source string +} + +func (l labelContext) label(element *hierarchy.Element) string { + if l.source == LabelSourceResourceID { + return resourceIdentifierLabel(element) + } + return visibleLabel(element, l.nodeIndex) +} + +// handleField is the ax-element handle field a label falls back to when the +// target's selector no longer resolves against the current tree. Reading the +// handle's text there would leak visible text into an identifier-labelled run, +// which is the one thing that arm must not see. +func (l labelContext) handleField() string { + if l.source == LabelSourceResourceID { + return "id" + } + return "text" +} + +// resourceIdentifierLabel names a control by the identifier the app assigned it, +// then by its class, then by a bare word. Every rung a user could read (text, +// description, hint, descendant text) is deliberately absent: the point of this +// channel is that the model sees no visible text at all, so a fallback that +// reached for text would silently turn the arm back into the default one. +func resourceIdentifierLabel(element *hierarchy.Element) string { + if element.ResourceID != "" { + return truncateLabel(element.ResourceID) + } + if element.Class != "" { + return element.Class + } + return "control" +} + // visibleLabel names a control by what a user would read: its own text, then // description, then a field hint, then text borrowed from its descendants (the // case that fixes empty-text Compose buttons whose word lives on a child), then diff --git a/internal/verifier/llm_test.go b/internal/verifier/llm_test.go index 4c0ce88..5fbe69b 100644 --- a/internal/verifier/llm_test.go +++ b/internal/verifier/llm_test.go @@ -24,6 +24,23 @@ const enumTreeJSON = `{ ] }` +// labelChannelTreeJSON pulls the two label channels apart: every identifier +// differs from the text a user reads, one control has a class but no identifier, +// one has neither, and one has an identifier but nothing readable at all. +const labelChannelTreeJSON = `{ + "attributes": {"bounds": "[0,0,1080,2400]"}, + "children": [ + {"attributes": {"resource-id": "add_credit_button", "class": "android.widget.Button", "bounds": "[0,100,1080,200]"}, "clickable": true, "enabled": true, "children": [ + {"attributes": {"text": "Add credit", "bounds": "[0,100,540,200]"}, "children": []} + ]}, + {"attributes": {"class": "android.widget.CheckBox", "text": "Remember me", "bounds": "[0,250,1080,300]"}, "clickable": true, "enabled": true, "children": []}, + {"attributes": {"class": "android.widget.CheckBox", "text": "Stay signed in", "bounds": "[0,300,1080,350]"}, "clickable": true, "enabled": true, "children": []}, + {"attributes": {"text": "Sign in", "bounds": "[0,400,1080,450]"}, "clickable": true, "enabled": true, "children": []}, + {"attributes": {"resource-id": "amount_field", "class": "EditText", "hintText": "Amount", "bounds": "[0,500,1080,600]"}, "enabled": true, "children": []}, + {"attributes": {"resource-id": "silent_row", "bounds": "[0,650,1080,700]"}, "clickable": true, "enabled": true, "children": []} + ] +}` + // enumVerifier loads a spec whose actions root is the given plain-object graph // and stages the given tree, so Candidates walks a controlled action tree. The // spec is bundled with the goja runtime entry because the model arm reads the @@ -56,7 +73,7 @@ func hasCandidate(candidates []ActionCandidate, description string) bool { func TestCandidatesLabelsControlsByVisibleText(t *testing.T) { v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", enumTreeJSON) - candidates := v.Candidates() + candidates := v.Candidates(LabelSourceVisibleText) // The empty-text clickable wrapper is labeled by its child Text, NOT its // resource-id. @@ -79,9 +96,110 @@ func TestCandidatesLabelsControlsByVisibleText(t *testing.T) { } } +func TestCandidatesLabelsControlsByResourceIdentifier(t *testing.T) { + candidates := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). + Candidates(LabelSourceResourceID) + + if !hasCandidate(candidates, `Tap "add_credit_button"`) { + t.Errorf("want the control named by its identifier, got %v", descriptions(candidates)) + } + // Nothing a user could read may reach this channel, including through a + // fallback rung: an arm that sees the text is the other arm. + for _, readable := range []string{`Tap "Add credit"`, `Tap "Remember me"`, `Tap "Sign in"`} { + if hasCandidate(candidates, readable) { + t.Errorf("visible text leaked in as %s: %v", readable, descriptions(candidates)) + } + } +} + +func TestCandidatesIdentifierChannelFallsBackToClassThenBareControl(t *testing.T) { + candidates := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). + Candidates(LabelSourceResourceID) + + if !hasCandidate(candidates, `Tap "android.widget.CheckBox"`) { + t.Errorf("a control with no identifier falls back to its class, got %v", descriptions(candidates)) + } + if !hasCandidate(candidates, `Tap "control"`) { + t.Errorf("a control with neither identifier nor class falls back to a bare word, got %v", + descriptions(candidates)) + } +} + +// TestCandidatesIdentifierChannelMergesControlsItCannotTellApart pins the cost +// of the channel rather than a defect in it: dedup keys on the rendered line, so +// two identifier-less controls of one class arrive as ONE numbered entry and the +// model can only reach the first. The list the identifier arm picks from is +// therefore shorter than the text arm's on the same screen. +func TestCandidatesIdentifierChannelMergesControlsItCannotTellApart(t *testing.T) { + text := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). + Candidates(LabelSourceVisibleText) + identifier := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). + Candidates(LabelSourceResourceID) + + if count(text, `Tap "Remember me"`) != 1 || count(text, `Tap "Stay signed in"`) != 1 { + t.Fatalf("the text channel should address both checkboxes, got %v", descriptions(text)) + } + if got := count(identifier, `Tap "android.widget.CheckBox"`); got != 1 { + t.Errorf("the two checkboxes should merge into one line, got %d: %v", got, descriptions(identifier)) + } + if len(identifier) >= len(text) { + t.Errorf("identifier list (%d) should be shorter than the text list (%d): %v vs %v", + len(identifier), len(text), descriptions(identifier), descriptions(text)) + } +} + +// TestCandidatesVisibleTextFallsBackToTheIdentifier is where the two channels +// agree: a control carrying nothing readable is named by its identifier in both, +// so a screen built entirely from such controls is one cell, not two. +func TestCandidatesVisibleTextFallsBackToTheIdentifier(t *testing.T) { + candidates := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). + Candidates(LabelSourceVisibleText) + + if !hasCandidate(candidates, `Tap "silent_row"`) { + t.Errorf("a control with no readable text falls back to its identifier, got %v", + descriptions(candidates)) + } +} + +func TestCandidatesTypingLabelFollowsTheLabelSource(t *testing.T) { + text := enumVerifier(t, "{kind:'builtin', verb:'typing'}", labelChannelTreeJSON). + Candidates(LabelSourceVisibleText) + if !hasCandidate(text, `Type into "Amount" (number)`) { + t.Errorf("want the field named by its hint, got %v", descriptions(text)) + } + + identifier := enumVerifier(t, "{kind:'builtin', verb:'typing'}", labelChannelTreeJSON). + Candidates(LabelSourceResourceID) + if !hasCandidate(identifier, `Type into "amount_field" (number)`) { + t.Errorf("want the field named by its identifier, got %v", descriptions(identifier)) + } +} + +// TestLabelSourceChangesOnlyTheDescription is the claim the factorial rests on: +// the channel renames the target and does nothing else, so a difference in +// defect yield cannot come from the two arms executing different actions. +func TestLabelSourceChangesOnlyTheDescription(t *testing.T) { + text := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). + Candidates(LabelSourceVisibleText) + identifier := enumVerifier(t, "{kind:'builtin', verb:'taps'}", labelChannelTreeJSON). + Candidates(LabelSourceResourceID) + + readable, ok := findCandidate(text, `Tap "Add credit"`) + if !ok { + t.Fatalf("missing the text-labelled tap: %v", descriptions(text)) + } + named, ok := findCandidate(identifier, `Tap "add_credit_button"`) + if !ok { + t.Fatalf("missing the identifier-labelled tap: %v", descriptions(identifier)) + } + if readable.Action != named.Action { + t.Errorf("same control, different action:\n text=%+v\n id=%+v", readable.Action, named.Action) + } +} + func TestCandidatesDropsDisabledControls(t *testing.T) { v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", enumTreeJSON) - for _, candidate := range v.Candidates() { + for _, candidate := range v.Candidates(LabelSourceVisibleText) { if strings.Contains(candidate.Description, "Off") { t.Errorf("disabled control surfaced as %q", candidate.Description) } @@ -90,7 +208,7 @@ func TestCandidatesDropsDisabledControls(t *testing.T) { func TestCandidatesTypingExposesInputType(t *testing.T) { v := enumVerifier(t, "{kind:'builtin', verb:'typing'}", enumTreeJSON) - candidates := v.Candidates() + candidates := v.Candidates(LabelSourceVisibleText) candidate, ok := findCandidate(candidates, `Type into "Amount" (number)`) if !ok { t.Fatalf("want typing candidate with input type, got %v", descriptions(candidates)) @@ -113,7 +231,7 @@ func TestCandidatesLabelsEditableFieldByHintNotTypedValue(t *testing.T) { ] }` v := enumVerifier(t, "{kind:'builtin', verb:'typing'}", tree) - candidates := v.Candidates() + candidates := v.Candidates(LabelSourceVisibleText) if hasCandidate(candidates, `Type into "99" (number)`) || hasCandidate(candidates, `Type into "99"`) { t.Errorf("editable field labeled by its typed value: %v", descriptions(candidates)) } @@ -126,7 +244,7 @@ func TestCandidatesKeepsGestureVerbsDistinct(t *testing.T) { v := enumVerifier(t, "{kind:'weighted', branches:[[1,{kind:'builtin',verb:'scrolls'}],[1,{kind:'builtin',verb:'swipes'}]]}", enumTreeJSON) - candidates := v.Candidates() + candidates := v.Candidates(LabelSourceVisibleText) // `scrolls` folds to one directional pair over the single scrollable // container, which is what keeps the list short. @@ -168,7 +286,7 @@ func TestCandidatesWeightsCombineAcrossPaths(t *testing.T) { v := enumVerifier(t, "{kind:'weighted', branches:[[1,{kind:'builtin',verb:'taps'}],[1,{kind:'builtin',verb:'taps'}]]}", oneClickable) - candidates := v.Candidates() + candidates := v.Candidates(LabelSourceVisibleText) if len(candidates) != 1 { t.Fatalf("want one deduped candidate, got %v", descriptions(candidates)) } @@ -185,7 +303,7 @@ func TestCandidatesWeightReflectsBranchShare(t *testing.T) { v := enumVerifier(t, "{kind:'weighted', branches:[[1,{kind:'builtin',verb:'taps'}],[3,{kind:'builtin',verb:'typing'}]]}", enumTreeJSON) - candidates := v.Candidates() + candidates := v.Candidates(LabelSourceVisibleText) tap, ok := findCandidate(candidates, `Tap "Sign in"`) if !ok { t.Fatalf("missing tap candidate: %v", descriptions(candidates)) @@ -204,7 +322,7 @@ func TestCandidatesWeightReflectsBranchShare(t *testing.T) { func TestCandidatesUnweightedTreeShowsNoWeight(t *testing.T) { v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", enumTreeJSON) - for _, candidate := range v.Candidates() { + for _, candidate := range v.Candidates(LabelSourceVisibleText) { if candidate.Weighted || candidate.Weight != 0 { t.Errorf("%q carries a weight despite no weighted node", candidate.Description) } @@ -218,7 +336,7 @@ func TestCandidatesCallsAuthoredLeafOnce(t *testing.T) { {kind:'InputText', into:'id:Amount', text:'42'} ]}` v := enumVerifier(t, actions, enumTreeJSON) - candidates := v.Candidates() + candidates := v.Candidates(LabelSourceVisibleText) // Authored Tap resolves its selector to the visible-text label. if !hasCandidate(candidates, `Tap "Sign in"`) { @@ -252,7 +370,7 @@ func TestCandidatesSurfaceAuthoredUntargetedActions(t *testing.T) { {kind:'PressKey', key:'back'}, {kind:'Wait'} ]}` - candidates := enumVerifier(t, actions, enumTreeJSON).Candidates() + candidates := enumVerifier(t, actions, enumTreeJSON).Candidates(LabelSourceVisibleText) for _, want := range []string{"Swipe from (10,600) to (10,100)", "Press back", "Wait"} { if !hasCandidate(candidates, want) { t.Errorf("authored %q missing: %v", want, descriptions(candidates)) @@ -262,7 +380,7 @@ func TestCandidatesSurfaceAuthoredUntargetedActions(t *testing.T) { func TestCandidatesOffRouteLeafYieldsNothing(t *testing.T) { v := enumVerifier(t, "{kind:'actions', generate: () => []}", enumTreeJSON) - if got := v.Candidates(); len(got) != 0 { + if got := v.Candidates(LabelSourceVisibleText); len(got) != 0 { t.Errorf("off-route leaf should yield no candidates, got %v", descriptions(got)) } } @@ -280,7 +398,7 @@ func TestCandidatesSkipsCrossFadeFrames(t *testing.T) { ] }` v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", crossFade) - if got := v.Candidates(); len(got) != 0 { + if got := v.Candidates(LabelSourceVisibleText); len(got) != 0 { t.Errorf("cross-fade frame should yield no candidates, got %v", descriptions(got)) } // The seeded policy is skipped by the SAME guard, in the shared producer, @@ -292,13 +410,13 @@ func TestCandidatesSkipsCrossFadeFrames(t *testing.T) { func TestCandidatesNilWithoutTreeOrActions(t *testing.T) { withActions := newLoadedVerifier(t, "globalThis.actions = {kind:'builtin', verb:'taps'};") - if got := withActions.Candidates(); got != nil { + if got := withActions.Candidates(LabelSourceVisibleText); got != nil { t.Errorf("Candidates with no tree = %v, want nil", got) } noActions := newLoadedVerifier(t, "globalThis.properties = {};") tree, _ := hierarchy.Parse(enumTreeJSON) noActions.lastTree = tree - if got := noActions.Candidates(); got != nil { + if got := noActions.Candidates(LabelSourceVisibleText); got != nil { t.Errorf("Candidates with no actions root = %v, want nil", got) } } diff --git a/internal/verifier/policy_parity_test.go b/internal/verifier/policy_parity_test.go index 3dca35b..5c477ee 100644 --- a/internal/verifier/policy_parity_test.go +++ b/internal/verifier/policy_parity_test.go @@ -62,7 +62,7 @@ func TestModelCandidateDescriptionsAreUniqueAndNamed(t *testing.T) { t.Run(verb, func(t *testing.T) { verifier := loadVerbSpec(t, verb) seen := map[string]bool{} - for _, candidate := range verifier.Candidates() { + for _, candidate := range verifier.Candidates(LabelSourceVisibleText) { if candidate.Description == "" { t.Fatalf("%s produced a candidate with no description: %+v", verb, candidate.Action) } @@ -81,17 +81,64 @@ func TestModelCandidateDescriptionsAreUniqueAndNamed(t *testing.T) { // dropped, so the model could never navigate back or let the app settle. func TestModelIsOfferedTheUntargetedVerbs(t *testing.T) { verifier := loadVerbSpec(t, "pressKeys") - if !hasCandidate(verifier.Candidates(), "Press back") { + if !hasCandidate(verifier.Candidates(LabelSourceVisibleText), "Press back") { t.Errorf("pressKeys missing from the model's candidates: %v", - descriptions(verifier.Candidates())) + descriptions(verifier.Candidates(LabelSourceVisibleText))) } verifier = loadVerbSpec(t, "waitOnce") - if !hasCandidate(verifier.Candidates(), "Wait") { + if !hasCandidate(verifier.Candidates(LabelSourceVisibleText), "Wait") { t.Errorf("waitOnce missing from the model's candidates: %v", - descriptions(verifier.Candidates())) + descriptions(verifier.Candidates(LabelSourceVisibleText))) } } +// TestSeededDrawStreamIgnoresLabelSource is the manipulation check the +// labelling factorial needs: the seeded picker selects by index and never asks +// for a label, so its draw stream must be bit-identical whichever channel the +// model policy would have been given, and identical again to a run where the +// candidate list was never enumerated at all. Any difference between two seeded +// cells is then the application and the harness, not the factor. +func TestSeededDrawStreamIgnoresLabelSource(t *testing.T) { + for _, verb := range policyVerbs { + t.Run(verb, func(t *testing.T) { + never := seededDrawStream(t, verb, "") + text := seededDrawStream(t, verb, LabelSourceVisibleText) + identifier := seededDrawStream(t, verb, LabelSourceResourceID) + if !slices.Equal(never, text) { + t.Errorf("enumerating visible-text candidates moved the seeded stream for %s", verb) + } + if !slices.Equal(never, identifier) { + t.Errorf("enumerating identifier candidates moved the seeded stream for %s", verb) + } + }) + } +} + +// seededDrawStream drives the seeded picker for the draw budget and returns +// every action in order. An empty labelSource enumerates nothing; otherwise the +// model's candidate list is built under that channel before each draw, which is +// the only way the two could ever touch. +func seededDrawStream(t *testing.T, verb, labelSource string) []string { + t.Helper() + verifier := loadVerbSpec(t, verb) + stream := make([]string, 0, seededDrawBudget) + for range seededDrawBudget { + if labelSource != "" { + verifier.Candidates(labelSource) + } + action, err := verifier.NextAction() + if errors.Is(err, ErrNoAction) { + stream = append(stream, "no action") + continue + } + if err != nil { + t.Fatalf("%s next action: %v", verb, err) + } + stream = append(stream, fmt.Sprintf("%+v", action)) + } + return stream +} + // loadVerbSpec builds a verifier whose whole action tree is one builtin verb, // with policyTreeJSON pushed as the current state. func loadVerbSpec(t *testing.T, verb string) *Verifier { @@ -128,7 +175,7 @@ func modelOfferedActions(t *testing.T, verb string) map[string]Action { t.Helper() verifier := loadVerbSpec(t, verb) offered := map[string]Action{} - for _, candidate := range verifier.Candidates() { + for _, candidate := range verifier.Candidates(LabelSourceVisibleText) { offered[actionIdentity(candidate.Action)] = candidate.Action } return offered