From 17a0cd5fe53877945c15b7b3b76cfeab2690e102 Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 13 Jul 2026 07:23:21 +0530 Subject: [PATCH] test(runner): cover choice schema, strict-skip, and setup precedence --- internal/runner/llm_source_test.go | 260 ++++++++++++++--------------- 1 file changed, 129 insertions(+), 131 deletions(-) diff --git a/internal/runner/llm_source_test.go b/internal/runner/llm_source_test.go index b0e936d..2c9c9a2 100644 --- a/internal/runner/llm_source_test.go +++ b/internal/runner/llm_source_test.go @@ -29,8 +29,9 @@ var llmInputCorpus = []string{ } const llmFixtureSpec = ` -import { llm, always } from "@sanderling/spec"; +import { llm, always, taps, typing, weighted } from "@sanderling/spec"; globalThis.properties = { ok: always(() => true) }; +globalThis.actions = weighted([1, taps], [1, typing]); globalThis.generator = llm({ model: "test/model" }); ` @@ -42,101 +43,89 @@ const llmTreeJSON = `{ ] }` -func TestActionFromCandidateMapping(t *testing.T) { - sampler := func() (string, error) { return "typed-value", nil } - cases := []struct { - name string - input verifier.ActionCandidate - assert func(t *testing.T, action verifier.Action) - }{ - { - name: "tap keeps coordinates and selector", - input: verifier.ActionCandidate{Kind: verifier.ActionKindTap, X: 10, Y: 20, Selector: "id:Submit"}, - assert: func(t *testing.T, action verifier.Action) { - if action.Kind != verifier.ActionKindTap || action.X != 10 || action.Y != 20 || action.On != "id:Submit" { - t.Errorf("tap = %+v", action) - } - }, - }, - { - name: "typing fills text from the sampler", - input: verifier.ActionCandidate{Kind: verifier.ActionKindInputText, X: 5, Y: 6, Selector: "id:Name"}, - assert: func(t *testing.T, action verifier.Action) { - if action.Kind != verifier.ActionKindInputText || action.Text != "typed-value" || action.On != "id:Name" { - t.Errorf("inputText = %+v", action) - } - }, - }, - { - name: "scroll defaults down with derived endpoints", - input: verifier.ActionCandidate{Kind: verifier.ActionKindScroll, X: 50, Y: 50, Selector: "id:List"}, - assert: func(t *testing.T, action verifier.Action) { - if action.Direction != "down" { - t.Errorf("scroll direction = %q, want down", action.Direction) - } - if action.X != 0 || action.Y != 0 || action.FromX != 0 || action.ToY != 0 { - t.Errorf("scroll endpoints should be left zero for runner derivation, got %+v", action) - } - }, - }, - { - name: "swipe builds a vertical gesture sized off height", - input: verifier.ActionCandidate{Kind: verifier.ActionKindSwipe, X: 100, Y: 1000, Height: 1000}, - assert: func(t *testing.T, action verifier.Action) { - if action.FromX != 100 || action.FromY != 1000 || action.ToX != 100 { - t.Errorf("swipe origin = %+v", action) - } - if action.ToY != 1000-400 { - t.Errorf("swipe ToY = %d, want 600", action.ToY) - } - }, - }, - { - name: "swipe magnitude floors at the minimum", - input: verifier.ActionCandidate{Kind: verifier.ActionKindSwipe, X: 10, Y: 500, Height: 100}, - assert: func(t *testing.T, action verifier.Action) { - // 100*4/10 = 40 < 200, so the floor applies. - if action.ToY != 500-swipeMinMagnitude { - t.Errorf("swipe ToY = %d, want %d", action.ToY, 500-swipeMinMagnitude) - } - }, - }, +func TestActionForCandidatePassesNonTypingThrough(t *testing.T) { + source := &llmSource{} + candidate := verifier.ActionCandidate{ + Kind: verifier.ActionKindTap, + Action: verifier.Action{Kind: verifier.ActionKindTap, On: "id:Submit", X: 10, Y: 20}, } - for _, testCase := range cases { - t.Run(testCase.name, func(t *testing.T) { - action, err := actionFromCandidate(testCase.input, sampler) - if err != nil { - t.Fatalf("actionFromCandidate: %v", err) - } - testCase.assert(t, action) - }) - } -} - -func TestActionFromCandidateSamplerError(t *testing.T) { - failing := func() (string, error) { return "", errors.New("no sampler") } - _, err := actionFromCandidate(verifier.ActionCandidate{Kind: verifier.ActionKindInputText}, failing) - if err == nil { - t.Fatal("expected sampler error to propagate") - } -} - -func TestParseRanked(t *testing.T) { - ranked, reasoning, err := parseRanked(`{"reasoning":"go home","ranked":[3,1,0]}`) + action, err := source.actionForCandidate(candidate, "ignored") if err != nil { - t.Fatalf("parseRanked: %v", err) + t.Fatalf("actionForCandidate: %v", err) } - if reasoning != "go home" || !slices.Equal(ranked, []int{3, 1, 0}) { - t.Errorf("parseRanked = %v %q", ranked, reasoning) + if action.Kind != verifier.ActionKindTap || action.On != "id:Submit" || action.X != 10 || action.Y != 20 { + t.Errorf("tap = %+v, want the candidate action verbatim", action) + } + if action.Text != "" { + t.Errorf("non-typing action must not carry text, got %q", action.Text) + } +} + +func TestActionForCandidateUsesModelText(t *testing.T) { + source := &llmSource{} + candidate := verifier.ActionCandidate{ + Kind: verifier.ActionKindInputText, + LLMText: true, + Action: verifier.Action{Kind: verifier.ActionKindInputText, On: "id:Name"}, + } + action, err := source.actionForCandidate(candidate, "Priya") + if err != nil { + t.Fatalf("actionForCandidate: %v", err) + } + if action.Text != "Priya" { + t.Errorf("text = %q, want the model-supplied value", action.Text) + } +} + +func TestActionForCandidateAuthoredTypingKeepsSampledValue(t *testing.T) { + source := &llmSource{} + candidate := verifier.ActionCandidate{ + Kind: verifier.ActionKindInputText, + LLMText: false, // authored InputText: replay the spec's sampled value + Action: verifier.Action{Kind: verifier.ActionKindInputText, On: "id:Amount", Text: "42"}, + } + action, err := source.actionForCandidate(candidate, "ignored") + if err != nil { + t.Fatalf("actionForCandidate: %v", err) + } + if action.Text != "42" { + t.Errorf("text = %q, want the authored value 42", action.Text) + } +} + +func TestActionForCandidateFallsBackToSampler(t *testing.T) { + fake := newFakeOpenRouter(t) + source, _ := newLLMSource(t, fake) + candidate := verifier.ActionCandidate{ + Kind: verifier.ActionKindInputText, + LLMText: true, + Action: verifier.Action{Kind: verifier.ActionKindInputText, On: "id:Name"}, + } + action, err := source.actionForCandidate(candidate, " ") + if err != nil { + t.Fatalf("actionForCandidate: %v", err) + } + if !slices.Contains(llmInputCorpus, action.Text) { + t.Errorf("empty model text should fall back to the corpus, got %q", action.Text) + } +} + +func TestParseChoice(t *testing.T) { + out, err := parseChoice(`{"reasoning":"go home","choice":3,"chosen_action":"Tap \"Home\"","text":""}`) + if err != nil { + t.Fatalf("parseChoice: %v", err) + } + if out.Reasoning != "go home" || out.Choice != 3 || out.ChosenAction != `Tap "Home"` { + t.Errorf("parseChoice = %+v", out) } - if _, _, err := parseRanked(""); err == nil { + if _, err := parseChoice(""); err == nil { t.Error("expected error for empty content") } - if _, _, err := parseRanked(`{"reasoning":"x","ranked":[]}`); err == nil { - t.Error("expected error for empty ranked list") + if _, err := parseChoice(`{"reasoning":"x","choice":0,"chosen_action":"","text":""}`); err == nil { + t.Error("expected error for a zero choice") } - if _, _, err := parseRanked(`not json`); err == nil { + if _, err := parseChoice(`not json`); err == nil { t.Error("expected error for malformed JSON") } } @@ -157,10 +146,12 @@ func TestSystemPromptAppendsInstructions(t *testing.T) { // fakeOpenRouter is a configurable in-process OpenRouter server. Set ranked / // reasoning before each call; it echoes them as a json_schema content body. type fakeOpenRouter struct { - server *httptest.Server - ranked []int - reasoning string - lastRequest map[string]any + server *httptest.Server + choice int + chosenAction string + text string + reasoning string + lastRequest map[string]any } func newFakeOpenRouter(t *testing.T) *fakeOpenRouter { @@ -169,7 +160,12 @@ func newFakeOpenRouter(t *testing.T) *fakeOpenRouter { fake.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) _ = json.Unmarshal(body, &fake.lastRequest) - content, _ := json.Marshal(map[string]any{"reasoning": fake.reasoning, "ranked": fake.ranked}) + content, _ := json.Marshal(map[string]any{ + "reasoning": fake.reasoning, + "choice": fake.choice, + "chosen_action": fake.chosenAction, + "text": fake.text, + }) response, _ := json.Marshal(llmclient.Response{ Choices: []llmclient.Choice{{Message: llmclient.ResponseMessage{Content: string(content)}}}, }) @@ -221,13 +217,15 @@ func pushLLMSnapshot(t *testing.T, v *verifier.Verifier) { } } -func candidateIndex(candidates []verifier.ActionCandidate, kind verifier.ActionKind) int { +func candidateByKind(t *testing.T, candidates []verifier.ActionCandidate, kind verifier.ActionKind) verifier.ActionCandidate { + t.Helper() for _, candidate := range candidates { if candidate.Kind == kind { - return candidate.Index + return candidate } } - return -1 + t.Fatalf("no candidate of kind %q in %v", kind, candidates) + return verifier.ActionCandidate{} } func TestPickSourcesSelectsLLMWhenRequested(t *testing.T) { @@ -267,11 +265,15 @@ func TestLLMSourceDrivesExecutedActions(t *testing.T) { fake := newFakeOpenRouter(t) source, verifierInstance := newLLMSource(t, fake) pushLLMSnapshot(t, verifierInstance) - candidates := verifierInstance.AllCandidates() + candidates := verifierInstance.Candidates() - // Step 1: the model ranks the Tap on Submit first. - fake.ranked = []int{candidateIndex(candidates, verifier.ActionKindTap)} + // Step 1: the model picks the Tap on Submit by its number, echoing its + // description. + tap := candidateByKind(t, candidates, verifier.ActionKindTap) + fake.choice = tap.Index + fake.chosenAction = tap.Description fake.reasoning = "tap submit" + fake.text = "" action, err := source.NextAction(context.Background()) if err != nil { t.Fatalf("NextAction: %v", err) @@ -291,11 +293,13 @@ func TestLLMSourceDrivesExecutedActions(t *testing.T) { t.Error("request carried no screenshot image part") } - // Step 2: the model ranks the InputText on Name. Its text must come from the - // shared corpus sampler, not the model. + // Step 2: the model picks the typing candidate and supplies the value. pushLLMSnapshot(t, verifierInstance) - fake.ranked = []int{candidateIndex(candidates, verifier.ActionKindInputText)} + typing := candidateByKind(t, candidates, verifier.ActionKindInputText) + fake.choice = typing.Index + fake.chosenAction = typing.Description fake.reasoning = "type a name" + fake.text = "Priya" action, err = source.NextAction(context.Background()) if err != nil { t.Fatalf("NextAction: %v", err) @@ -303,60 +307,54 @@ func TestLLMSourceDrivesExecutedActions(t *testing.T) { if action.Kind != verifier.ActionKindInputText || action.On != "id:Name" { t.Errorf("step 2 action = %+v, want InputText on id:Name", action) } - if !slices.Contains(llmInputCorpus, action.Text) { - t.Errorf("InputText text %q was not drawn from the corpus", action.Text) + if action.Text != "Priya" { + t.Errorf("InputText text = %q, want the model-supplied Priya", action.Text) } - // The trace records source=llm and the reasoning. + // The trace records source=llm, the reasoning, the choice, and the echo. traceAction := traceActionFor(action, nil) stampActionSource(traceAction, source) if traceAction.Source != "llm" || traceAction.LLMReasoning != "type a name" { t.Errorf("trace action = %+v, want source=llm reasoning=type a name", traceAction) } + if traceAction.LLMChoice != typing.Index || traceAction.LLMChosenAction != typing.Description { + t.Errorf("trace choice = %d/%q, want %d/%q", traceAction.LLMChoice, traceAction.LLMChosenAction, typing.Index, typing.Description) + } } -func TestLLMSourceSkipsOnInvalidIndex(t *testing.T) { +func TestLLMSourceSkipsOnOutOfRangeChoice(t *testing.T) { fake := newFakeOpenRouter(t) source, verifierInstance := newLLMSource(t, fake) pushLLMSnapshot(t, verifierInstance) - fake.ranked = []int{9999} + fake.choice = 9999 + fake.chosenAction = "whatever" _, err := source.NextAction(context.Background()) if !errors.Is(err, verifier.ErrNoAction) { - t.Fatalf("NextAction err = %v, want ErrNoAction for an out-of-range index", err) + t.Fatalf("NextAction err = %v, want ErrNoAction for an out-of-range choice", err) } if source.lastSource != "" { t.Errorf("lastSource = %q, want empty after a skipped step", source.lastSource) } } -func TestLLMSourceFirstValidIndexWins(t *testing.T) { +func TestLLMSourceStrictSkipsOnEchoMismatch(t *testing.T) { fake := newFakeOpenRouter(t) source, verifierInstance := newLLMSource(t, fake) pushLLMSnapshot(t, verifierInstance) - candidates := verifierInstance.AllCandidates() + candidates := verifierInstance.Candidates() - tapIndex := candidateIndex(candidates, verifier.ActionKindTap) - // First index is invalid; the second is the valid Tap candidate. - fake.ranked = []int{-1, tapIndex} - action, err := source.NextAction(context.Background()) - if err != nil { - t.Fatalf("NextAction: %v", err) + // A valid number, but the echoed action disagrees with that numbered entry: + // the model reasoned about one control and picked another's number. + tap := candidateByKind(t, candidates, verifier.ActionKindTap) + fake.choice = tap.Index + fake.chosenAction = "Tap \"Something Else\"" + _, err := source.NextAction(context.Background()) + if !errors.Is(err, verifier.ErrNoAction) { + t.Fatalf("NextAction err = %v, want ErrNoAction on chosen_action mismatch", err) } - if action.Kind != verifier.ActionKindTap { - t.Errorf("action = %+v, want Tap from the first valid index", action) - } - - // The trace records the full ranked list and the 1-based rank that won, so - // the reasoning (which describes the top pick) can be reconciled with the - // executed action when an earlier pick was skipped. - traceAction := traceActionFor(action, nil) - stampActionSource(traceAction, source) - if !slices.Equal(traceAction.LLMRanked, []int{-1, tapIndex}) { - t.Errorf("trace ranked = %v, want [-1 %d]", traceAction.LLMRanked, tapIndex) - } - if traceAction.LLMChosenRank != 2 { - t.Errorf("trace chosen rank = %d, want 2 (the second pick won)", traceAction.LLMChosenRank) + if source.lastSource != "" { + t.Errorf("lastSource = %q, want empty after a strict skip", source.lastSource) } } @@ -369,7 +367,7 @@ func TestLLMSourceSkipsOnHTTPError(t *testing.T) { source, verifierInstance := newLLMSource(t, fake) pushLLMSnapshot(t, verifierInstance) - fake.ranked = []int{0} + fake.choice = 1 _, err := source.NextAction(context.Background()) if !errors.Is(err, verifier.ErrNoAction) { t.Fatalf("NextAction err = %v, want ErrNoAction on HTTP failure", err)