package verifier import ( "errors" "fmt" "maps" "slices" "testing" ) // policyTreeJSON gives every builtin verb something to act on, with every label // distinct so no two candidates render the same way. const policyTreeJSON = `{ "attributes": {"bounds": "[0,0,400,800]"}, "children": [ {"attributes": {"resource-id": "Save", "text": "Save", "bounds": "[0,0,200,60]"}, "clickable": true, "enabled": true, "children": []}, {"attributes": {"resource-id": "Cancel", "text": "Cancel", "bounds": "[200,0,400,60]"}, "clickable": true, "enabled": true, "children": []}, {"attributes": {"resource-id": "Amount", "class": "EditText", "hintText": "Amount", "bounds": "[0,100,400,160]"}, "enabled": true, "children": []}, {"attributes": {"resource-id": "Note", "class": "EditText", "hintText": "Note", "bounds": "[0,200,400,260]"}, "enabled": true, "children": []}, {"attributes": {"resource-id": "List", "scrollable": "true", "bounds": "[0,300,400,700]"}, "children": []} ] }` // policyVerbs is every builtin verb a spec can put in its action tree. var policyVerbs = []string{ "taps", "doubleTaps", "longPresses", "typing", "scrolls", "swipes", "pressKeys", "waitOnce", } // seededDrawBudget is how many times the seeded picker is driven per verb. The // candidate sets here are a handful of entries wide, so this exhausts them many // times over; a miss would mean the picker cannot reach one of its own // candidates, which is itself the bug worth failing on. const seededDrawBudget = 300 // TestPoliciesEnumerateTheSameActions is the guard on the claim the paper makes // about the two action policies: they differ in the pick and in nothing else. // For every builtin verb, the actions the seeded picker can draw and the // candidates the model policy is offered must be the same set. Enumeration lives // in one place (pick.ts builtinCandidates) precisely so this cannot drift, and // this test is what notices if a second one ever grows back. func TestPoliciesEnumerateTheSameActions(t *testing.T) { for _, verb := range policyVerbs { t.Run(verb, func(t *testing.T) { seeded := seededReachableActions(t, verb) model := modelOfferedActions(t, verb) if len(model) == 0 { t.Fatalf("%s offered the model no candidates at all", verb) } if !slices.Equal(slices.Sorted(maps.Keys(seeded)), slices.Sorted(maps.Keys(model))) { t.Errorf("action spaces differ for %s\n seeded=%v\n model=%v", verb, slices.Sorted(maps.Keys(seeded)), slices.Sorted(maps.Keys(model))) } }) } } // TestModelCandidateDescriptionsAreUniqueAndNamed checks the rendering half of // the contract: every enumerated action reaches the model as its own distinctly // named line, so the number it picks and the action that executes agree. func TestModelCandidateDescriptionsAreUniqueAndNamed(t *testing.T) { for _, verb := range policyVerbs { t.Run(verb, func(t *testing.T) { verifier := loadVerbSpec(t, verb) seen := map[string]bool{} for _, candidate := range verifier.Candidates() { if candidate.Description == "" { t.Fatalf("%s produced a candidate with no description: %+v", verb, candidate.Action) } if seen[candidate.Description] { t.Errorf("%s rendered %q twice, so the model cannot address both", verb, candidate.Description) } seen[candidate.Description] = true } }) } } // TestModelIsOfferedTheUntargetedVerbs pins the two verbs the model arm used to // be blind to: with no element to enumerate over, a key press and a wait were // dropped, so the model could never navigate back or let the app settle. func TestModelIsOfferedTheUntargetedVerbs(t *testing.T) { verifier := loadVerbSpec(t, "pressKeys") if !hasCandidate(verifier.Candidates(), "Press back") { t.Errorf("pressKeys missing from the model's candidates: %v", descriptions(verifier.Candidates())) } verifier = loadVerbSpec(t, "waitOnce") if !hasCandidate(verifier.Candidates(), "Wait") { t.Errorf("waitOnce missing from the model's candidates: %v", descriptions(verifier.Candidates())) } } // loadVerbSpec builds a verifier whose whole action tree is one builtin verb, // with policyTreeJSON pushed as the current state. func loadVerbSpec(t *testing.T, verb string) *Verifier { t.Helper() verifier := newVerifier(t, WithSeed(0x5eed)) loadActionSpec(t, verifier, fmt.Sprintf( "import { %s } from \"@sanderling/spec\";\nglobalThis.actions = %s;", verb, verb)) pushTree(t, verifier, policyTreeJSON) return verifier } // seededReachableActions drives the seeded picker over its draw budget and // returns every distinct action it produced. func seededReachableActions(t *testing.T, verb string) map[string]Action { t.Helper() verifier := loadVerbSpec(t, verb) reachable := map[string]Action{} for range seededDrawBudget { action, err := verifier.NextAction() if errors.Is(err, ErrNoAction) { continue } if err != nil { t.Fatalf("%s next action: %v", verb, err) } reachable[actionIdentity(action)] = action } return reachable } // modelOfferedActions returns the actions behind the numbered list the model // policy picks from. func modelOfferedActions(t *testing.T, verb string) map[string]Action { t.Helper() verifier := loadVerbSpec(t, verb) offered := map[string]Action{} for _, candidate := range verifier.Candidates() { offered[actionIdentity(candidate.Action)] = candidate.Action } return offered } // actionIdentity keys an action by everything except the values the policy owns // rather than the candidate set: the typed text, which the seeded arm draws from // the edge-case corpus and the model writes itself, and a swipe's drag distance, // which the seeded arm draws and the enumeration lists at a nominal length. // Comparing those would compare policies instead of action spaces. A swipe's // direction is NOT policy-owned, so it survives as the sign of the drag. func actionIdentity(action Action) string { action.Text = "" if action.Kind == ActionKindSwipe { action.ToX = sign(action.ToX - action.FromX) action.ToY = sign(action.ToY - action.FromY) } return fmt.Sprintf("%+v", action) } func sign(value int) int { switch { case value > 0: return 1 case value < 0: return -1 default: return 0 } }