feat(runner): llm picks one number from weighted candidates

drop the seeded-root call for a setup-only precedence path, render a
numbered weighted candidate list, pin a reasoning-first choice schema,
strict-skip when chosen_action does not echo the numbered entry, and let
the model supply typed values (corpus fallback when empty).

internal/runner/llm_source.go

@@ -23,25 +23,18 @@ const (
 	// while keeping the UI legible.
 	llmMaxImageEdge = 1024
 	// llmHistorySize is how many recent actions (and the screen each led to) the
-	// prompt carries to discourage loops.
+	// prompt carries as context.
 	llmHistorySize = 5
-	// llmMaxRanked caps the ranked-index list the model returns.
-	llmMaxRanked = 5
-	// swipeMinMagnitude is the floor for an LLM-chosen swipe distance, matching
-	// the seeded swipe builder's minimum.
-	swipeMinMagnitude = 200
 )
 
-// llmSystemPrompt frames the selection task. The model only ranks the numbered
+// llmSystemPrompt frames the selection task: a short, generic bug-hunting
-// candidates the system already enumerated; it never invents actions. The kind
+// instruction. Each candidate is already a concrete, correctly-labeled action
-// semantics matter: every visible element doubles as a Swipe origin, so a
+// with a weight hinting the spec's testing priority; the model reads the
-// control whose only candidate is Swipe is NOT pressable — without the
+// screenshot, picks ONE number, and echoes that action so a mismatch can be
-// explanation models pick `Swipe "Submit"` intending to press Submit and loop
+// caught. The app-specific description (spec instructions) is appended.
-// forever on a disabled button.
+const llmSystemPrompt = "You are exercising a UI to find bugs. Each turn you get a screenshot and a numbered list of concrete actions, each with a weight hinting how much the test author wants it exercised (higher = more). " +
-const llmSystemPrompt = "You are exploring this app to surface bugs. Choose the most useful next action from the numbered candidates. " +
+	"Pick the ONE action most likely to make progress or expose a defect, and feel free to repeat an action when repetition is what would trip a bug. " +
-	"Candidate kinds: Tap/DoubleTap/LongPress press a control; InputText types into a field; Scroll and Swipe only pan the view — they never press the element they are labeled with. " +
+	"Respond with your reasoning, the chosen number, and chosen_action copied verbatim from that line. For a typing action, also provide the text to enter."
-	"A button that has no Tap candidate is disabled; satisfy its preconditions first (usually InputText into a field) instead of swiping it. " +
-	"Avoid repeating recent actions; prefer progress into new screens. Return only your ranked choices."
 
 // llmSource selects each step's action with an OpenAI-compatible vision model
 // instead of the seeded random pick. It replaces ONLY the pick: the candidate list, the input
@@ -60,21 +53,21 @@ type llmSource struct {
 
 	// lastSource/lastReasoning describe the most recent NextAction so the runner
 	// can stamp the trace. lastSource is "llm" only when the LLM (not setup)
-	// chose the action; lastReasoning is the model's rationale. lastRanked is
+	// chose the action; lastReasoning is the model's rationale. lastChoice is the
-	// the model's full ranked list and lastChosenRank the 1-based position in it
+	// 1-based number it picked and lastChosenAction the description it echoed, so
-	// that won (1 = top pick), so the trace can reconcile reasoning with action.
+	// the trace shows what the model believed it was doing.
-	lastSource     string
+	lastSource       string
-	lastReasoning  string
+	lastReasoning    string
-	lastRanked     []int
+	lastChoice       int
-	lastChosenRank int
+	lastChosenAction string
 }
 
 // llmSelection is the outcome of one LLM selection call.
 type llmSelection struct {
-	action     verifier.Action
+	action       verifier.Action
-	reasoning  string
+	reasoning    string
-	ranked     []int
+	choice       int
-	chosenRank int // 1-based position in ranked that produced action
+	chosenAction string
 }
 
 // NextAction returns the step's action. Setup precedence is preserved by
@@ -83,11 +76,13 @@ type llmSelection struct {
 func (s *llmSource) NextAction(ctx context.Context) (verifier.Action, error) {
 	s.lastSource = ""
 	s.lastReasoning = ""
-	s.lastRanked = nil
+	s.lastChoice = 0
-	s.lastChosenRank = 0
+	s.lastChosenAction = ""
 	s.history.completeLast(s.verifier.CurrentScreen())
 
-	action, err := s.verifier.NextAction()
+	// Setup precedence only: the LLM replaces the seeded action root, so we run
+	// setup (e.g. login) first but never the weighted picker.
+	action, err := s.verifier.SetupAction()
 	if err == nil {
 		s.history.add(describeAction(action))
 		return action, nil
@@ -98,23 +93,23 @@ func (s *llmSource) NextAction(ctx context.Context) (verifier.Action, error) {
 
 	selection, ok := s.selectViaLLM(ctx)
 	if !ok {
-		// Any failure (HTTP error, unusable output, no valid index) skips the
+		// Any failure (HTTP error, unusable output, invalid choice, echo
-		// step; the next step re-observes and tries again. No backend mixing.
+		// mismatch) skips the step; the next step re-observes and tries again.
 		return verifier.Action{}, verifier.ErrNoAction
 	}
 	s.lastSource = "llm"
 	s.lastReasoning = selection.reasoning
-	s.lastRanked = selection.ranked
+	s.lastChoice = selection.choice
-	s.lastChosenRank = selection.chosenRank
+	s.lastChosenAction = selection.chosenAction
 	s.history.add(describeAction(selection.action))
 	return selection.action, nil
 }
 
-// selectViaLLM runs one multimodal call and maps the first valid ranked index
+// selectViaLLM runs one multimodal call and maps the chosen number to an action.
-// to an action. It returns ok=false on any error/empty/invalid output, logging
+// It returns ok=false on any error/empty/invalid output, logging the cause; the
-// the cause; the caller turns that into a skipped step.
+// caller turns that into a skipped step.
 func (s *llmSource) selectViaLLM(ctx context.Context) (llmSelection, bool) {
-	candidates := s.verifier.AllCandidates()
+	candidates := s.verifier.Candidates()
 	if len(candidates) == 0 {
 		return llmSelection{}, false
 	}
@@ -129,24 +124,56 @@ func (s *llmSource) selectViaLLM(ctx context.Context) (llmSelection, bool) {
 		return llmSelection{}, false
 	}
 
-	ranked, reasoning, err := parseRanked(response.Choices[0].Message.Content)
+	output, err := parseChoice(response.Choices[0].Message.Content)
 	if err != nil {
 		s.logger.Warn("llm output unusable", "err", err)
 		return llmSelection{}, false
 	}
-	for position, index := range ranked {
+	// choice is 1-based into the numbered list.
-		if index < 0 || index >= len(candidates) {
+	if output.Choice < 1 || output.Choice > len(candidates) {
-			continue
+		s.logger.Warn("llm choice out of range", "choice", output.Choice, "candidates", len(candidates))
-		}
+		return llmSelection{}, false
-		action, err := actionFromCandidate(candidates[index], s.verifier.SampleInput)
-		if err != nil {
-			s.logger.Warn("building action from candidate failed", "index", index, "err", err)
-			continue
-		}
-		return llmSelection{action: action, reasoning: reasoning, ranked: ranked, chosenRank: position + 1}, true
 	}
-	s.logger.Warn("llm returned no valid candidate index", "ranked", ranked, "candidates", len(candidates))
+	candidate := candidates[output.Choice-1]
-	return llmSelection{}, false
+	// Strict skip: the echoed action must match the numbered entry, so a model
+	// that reasoned about one target but named a number for another cannot act.
+	if strings.TrimSpace(output.ChosenAction) != candidate.Description {
+		s.logger.Warn("llm chosen_action mismatch; skipping",
+			"choice", output.Choice, "echoed", output.ChosenAction, "candidate", candidate.Description)
+		return llmSelection{}, false
+	}
+	action, err := s.actionForCandidate(candidate, output.Text)
+	if err != nil {
+		s.logger.Warn("building action from candidate failed", "choice", output.Choice, "err", err)
+		return llmSelection{}, false
+	}
+	return llmSelection{
+		action:       action,
+		reasoning:    output.Reasoning,
+		choice:       output.Choice,
+		chosenAction: candidate.Description,
+	}, true
+}
+
+// actionForCandidate turns a chosen candidate into the executable action. The
+// candidate already carries a ready action; only builtin typing needs the
+// model's value spliced in (authored InputText keeps its sampled value, and any
+// other kind runs verbatim).
+func (s *llmSource) actionForCandidate(candidate verifier.ActionCandidate, text string) (verifier.Action, error) {
+	action := candidate.Action
+	if candidate.Kind == verifier.ActionKindInputText && candidate.LLMText {
+		if strings.TrimSpace(text) == "" {
+			// The model omitted a value; fall back to the shared corpus sampler
+			// so typing still exercises an edge-case string.
+			sampled, err := s.verifier.SampleInput()
+			if err != nil {
+				return verifier.Action{}, err
+			}
+			text = sampled
+		}
+		action.Text = text
+	}
+	return action, nil
 }
 
 // buildRequest assembles the one-shot multimodal request: a system frame, the
@@ -165,7 +192,7 @@ func (s *llmSource) buildRequest(candidates []verifier.ActionCandidate) llmclien
 			{Role: "system", Content: []llmclient.ContentPart{llmclient.TextPart(s.systemPrompt())}},
 			{Role: "user", Content: userParts},
 		},
-		ResponseFormat: rankedResponseFormat(),
+		ResponseFormat: choiceResponseFormat(len(candidates)),
 	}
 }
 
@@ -179,12 +206,17 @@ func (s *llmSource) systemPrompt() string {
 	return llmSystemPrompt + "\n\n" + s.instructions
 }
 
-// userPrompt renders the numbered candidate list and the recent-action memory.
+// userPrompt renders the numbered candidate list (with weights) and the
+// recent-action memory.
 func (s *llmSource) userPrompt(candidates []verifier.ActionCandidate) string {
 	var builder strings.Builder
-	builder.WriteString("Candidate actions on the current screen:\n")
+	builder.WriteString("Actions available on the current screen:\n")
 	for _, candidate := range candidates {
-		fmt.Fprintf(&builder, "#%d %s %q\n", candidate.Index, candidate.Kind, candidate.Label)
+		fmt.Fprintf(&builder, "%d. %s", candidate.Index, candidate.Description)
+		if candidate.Weighted {
+			fmt.Fprintf(&builder, "  (w%d)", candidate.Weight)
+		}
+		builder.WriteByte('\n')
 	}
 	if recent := s.history.recent(); len(recent) > 0 {
 		builder.WriteString("\nYour recent actions (oldest first) and the screen each led to:\n")
@@ -196,88 +228,58 @@ func (s *llmSource) userPrompt(candidates []verifier.ActionCandidate) string {
 			fmt.Fprintf(&builder, "- %s -> %s\n", entry.action, screen)
 		}
 	}
-	builder.WriteString("\nReturn your ranked candidate indices, most useful first.")
+	builder.WriteString("\nPick one action by its number.")
 	return builder.String()
 }
 
-// rankedResponseFormat is the strict structured-output schema: a short
+// choiceResponseFormat is the strict structured-output schema. Field order is
-// reasoning string and a ranked list of candidate indices.
+// pinned via raw JSON with reasoning FIRST, so the model reasons before it
-func rankedResponseFormat() *llmclient.ResponseFormat {
+// commits to a number (a materially better ordering than answer-first). text is
+// required by strict mode but empty for non-typing actions.
+func choiceResponseFormat(candidateCount int) *llmclient.ResponseFormat {
+	schema := fmt.Sprintf(`{
+  "type": "object",
+  "properties": {
+    "reasoning": {"type": "string", "description": "One short sentence on what you are trying to do and why this action."},
+    "choice": {"type": "integer", "minimum": 1, "maximum": %d, "description": "The number of the chosen action."},
+    "chosen_action": {"type": "string", "description": "The chosen action's text, copied verbatim from its numbered line."},
+    "text": {"type": "string", "description": "For a typing action, the text to enter; otherwise an empty string."}
+  },
+  "required": ["reasoning", "choice", "chosen_action", "text"],
+  "additionalProperties": false
+}`, candidateCount)
 	return &llmclient.ResponseFormat{
 		Type: "json_schema",
 		JSONSchema: llmclient.JSONSchema{
-			Name:   "ranked_actions",
+			Name:   "action_choice",
 			Strict: true,
-			Schema: map[string]any{
+			Schema: json.RawMessage(schema),
-				"type": "object",
-				"properties": map[string]any{
-					"reasoning": map[string]any{
-						"type":        "string",
-						"description": "One short sentence on why the top choice is most useful.",
-					},
-					"ranked": map[string]any{
-						"type":     "array",
-						"items":    map[string]any{"type": "integer"},
-						"minItems": 1,
-						"maxItems": llmMaxRanked,
-					},
-				},
-				"required":             []string{"reasoning", "ranked"},
-				"additionalProperties": false,
-			},
 		},
 	}
 }
 
-// rankedOutput is the model's structured response.
+// choiceOutput is the model's structured response, reasoning first.
-type rankedOutput struct {
+type choiceOutput struct {
-	Reasoning string `json:"reasoning"`
+	Reasoning    string `json:"reasoning"`
-	Ranked    []int  `json:"ranked"`
+	Choice       int    `json:"choice"`
+	ChosenAction string `json:"chosen_action"`
+	Text         string `json:"text"`
 }
 
-// parseRanked decodes the model's JSON content into ranked indices + reasoning.
+// parseChoice decodes the model's JSON content into the structured choice.
-func parseRanked(content string) ([]int, string, error) {
+func parseChoice(content string) (choiceOutput, error) {
 	content = strings.TrimSpace(content)
 	if content == "" {
-		return nil, "", errors.New("empty content")
+		return choiceOutput{}, errors.New("empty content")
 	}
-	var out rankedOutput
+	var out choiceOutput
 	if err := json.Unmarshal([]byte(content), &out); err != nil {
-		return nil, "", err
+		return choiceOutput{}, err
 	}
-	if len(out.Ranked) == 0 {
+	if out.Choice == 0 {
-		return nil, "", errors.New("no ranked indices")
+		return choiceOutput{}, errors.New("no choice")
 	}
-	return out.Ranked, out.Reasoning, nil
+	return out, nil
-}
-
-// actionFromCandidate maps a chosen candidate to a concrete action, reusing the
-// corpus sampler for InputText text and the seeded gesture geometry for
-// swipe/scroll. sampleInput is verifier.SampleInput, injected for testability.
-func actionFromCandidate(candidate verifier.ActionCandidate, sampleInput func() (string, error)) (verifier.Action, error) {
-	action := verifier.Action{Kind: candidate.Kind, On: candidate.Selector, X: candidate.X, Y: candidate.Y}
-	switch candidate.Kind {
-	case verifier.ActionKindInputText:
-		text, err := sampleInput()
-		if err != nil {
-			return verifier.Action{}, err
-		}
-		action.Text = text
-	case verifier.ActionKindScroll:
-		action.Direction = "down"
-		// Leave endpoints zero so the runner derives the gesture from the target
-		// bounds (scrollEndpoints), exactly as for an authored Scroll.
-		action.X, action.Y = 0, 0
-	case verifier.ActionKindSwipe:
-		// A vertical drag upward from the center reveals lower content, sized off
-		// the element height like the seeded swipe builder.
-		magnitude := max(swipeMinMagnitude, candidate.Height*4/10)
-		action.FromX, action.FromY = candidate.X, candidate.Y
-		action.ToX = candidate.X
-		action.ToY = max(0, candidate.Y-magnitude)
-		action.X, action.Y = 0, 0
-	}
-	return action, nil
 }
 
 // describeAction renders a short action summary for the recent-action memory.
@@ -357,8 +359,8 @@ func stampActionSource(traceAction *trace.Action, source ActionSource) {
 	}
 	traceAction.Source = llm.lastSource
 	traceAction.LLMReasoning = llm.lastReasoning
-	traceAction.LLMRanked = llm.lastRanked
+	traceAction.LLMChoice = llm.lastChoice
-	traceAction.LLMChosenRank = llm.lastChosenRank
+	traceAction.LLMChosenAction = llm.lastChosenAction
 }
 
 // screenshotDataURL downscales the PNG and encodes it as a data URL for the