package runner import ( "bytes" "context" "encoding/base64" "encoding/json" "errors" "fmt" "image" "image/color" "image/png" "log/slog" "regexp" "strings" "github.com/priyanshujain/sanderling/internal/llmclient" "github.com/priyanshujain/sanderling/internal/trace" "github.com/priyanshujain/sanderling/internal/verifier" ) const ( // llmMaxImageEdge downscales the screenshot's long edge to bound the payload // while keeping the UI legible. llmMaxImageEdge = 1024 // llmHistorySize is how many recent actions (and the screen each led to) the // prompt carries as context. llmHistorySize = 5 ) // llmSystemPrompt frames the selection task: a short, generic bug-hunting // instruction. Each candidate is already a concrete, correctly-labeled action // with a weight hinting the spec's testing priority; the model reads the // screenshot, picks ONE number, and echoes that action so a mismatch can be // caught. The app-specific description (spec instructions) is appended. const llmSystemPrompt = "You are exercising a UI to find bugs. Each turn you get a screenshot and a numbered list of concrete actions, each with a weight hinting how much the test author wants it exercised (higher = more). " + "Pick the ONE action most likely to make progress or expose a defect. Bugs often hide in repeated or rapid actions, so once a screen works, deliberately stress it (for example submitting the same form twice in a row to check it is not applied more than once) rather than only advancing. " + "Respond with your reasoning, the chosen number, and chosen_action copied verbatim from that line. For a typing action, also provide the text to enter." // llmSource selects each step's action with an OpenAI-compatible vision model // instead of the seeded random pick. It replaces ONLY the pick: the candidate list, the input // values, and action execution are all reused unchanged. The spec's JS setup // still runs first each tick (setup precedence), and the LLM drives once setup // yields nothing. type llmSource struct { verifier *verifier.Verifier client *llmclient.Client model string // instructions is optional spec-level guidance appended to the system prompt // to steer the model's bug-hunting (empty when unset). instructions string logger *slog.Logger history *actionHistory // lastSource/lastReasoning describe the most recent NextAction so the runner // can stamp the trace. lastSource is "llm" only when the LLM (not setup) // chose the action; lastReasoning is the model's rationale. lastChoice is the // 1-based number it picked and lastChosenAction the description it echoed, so // the trace shows what the model believed it was doing. lastSource string lastReasoning string lastChoice int lastChosenAction string } // llmSelection is the outcome of one LLM selection call. type llmSelection struct { action verifier.Action reasoning string choice int chosenAction string } // NextAction returns the step's action. Setup precedence is preserved by // running the JS path first (the llm marker is inert there, so a null result // means setup yielded nothing); the LLM selection then takes over. func (s *llmSource) NextAction(ctx context.Context) (verifier.Action, error) { s.lastSource = "" s.lastReasoning = "" s.lastChoice = 0 s.lastChosenAction = "" s.history.completeLast(s.verifier.CurrentScreen()) // Setup precedence only: the LLM replaces the seeded action root, so we run // setup (e.g. login) first but never the weighted picker. action, err := s.verifier.SetupAction() if err == nil { s.history.add(describeAction(action)) return action, nil } if !errors.Is(err, verifier.ErrNoAction) { return verifier.Action{}, err } selection, ok := s.selectViaLLM(ctx) if !ok { // Any failure (HTTP error, unusable output, invalid choice, echo // mismatch) skips the step; the next step re-observes and tries again. return verifier.Action{}, verifier.ErrNoAction } s.lastSource = "llm" s.lastReasoning = selection.reasoning s.lastChoice = selection.choice s.lastChosenAction = selection.chosenAction s.history.add(describeAction(selection.action)) return selection.action, nil } // selectViaLLM runs one multimodal call and maps the chosen number to an action. // It returns ok=false on any error/empty/invalid output, logging the cause; the // caller turns that into a skipped step. func (s *llmSource) selectViaLLM(ctx context.Context) (llmSelection, bool) { candidates := s.verifier.Candidates() if len(candidates) == 0 { return llmSelection{}, false } response, err := s.client.ChatCompletion(ctx, s.buildRequest(candidates)) if err != nil { s.logger.Warn("llm action selection failed", "err", err) return llmSelection{}, false } if len(response.Choices) == 0 { s.logger.Warn("llm returned no choices") return llmSelection{}, false } output, err := parseChoice(response.Choices[0].Message.Content) if err != nil { s.logger.Warn("llm output unusable", "err", err) return llmSelection{}, false } // choice is 1-based into the numbered list. if output.Choice < 1 || output.Choice > len(candidates) { s.logger.Warn("llm choice out of range", "choice", output.Choice, "candidates", len(candidates)) return llmSelection{}, false } candidate := candidates[output.Choice-1] // Strict skip: the echoed action must match the numbered entry, so a model // that reasoned about one target but named a number for another cannot act. // Models copy the whole rendered line including its trailing "(w34)" weight // annotation, so strip that before comparing to the (weight-free) description. if stripWeightSuffix(output.ChosenAction) != candidate.Description { s.logger.Warn("llm chosen_action mismatch; skipping", "choice", output.Choice, "echoed", output.ChosenAction, "candidate", candidate.Description) return llmSelection{}, false } action, err := s.actionForCandidate(candidate, output.Text) if err != nil { s.logger.Warn("building action from candidate failed", "choice", output.Choice, "err", err) return llmSelection{}, false } return llmSelection{ action: action, reasoning: output.Reasoning, choice: output.Choice, chosenAction: candidate.Description, }, true } // actionForCandidate turns a chosen candidate into the executable action. The // candidate already carries a ready action; only builtin typing needs the // model's value spliced in (authored InputText keeps its sampled value, and any // other kind runs verbatim). func (s *llmSource) actionForCandidate(candidate verifier.ActionCandidate, text string) (verifier.Action, error) { action := candidate.Action if candidate.Kind == verifier.ActionKindInputText && candidate.LLMText { if strings.TrimSpace(text) == "" { // The model omitted a value; fall back to the shared corpus sampler // so typing still exercises an edge-case string. sampled, err := s.verifier.SampleInput() if err != nil { return verifier.Action{}, err } text = sampled } action.Text = text } return action, nil } // buildRequest assembles the one-shot multimodal request: a system frame, the // numbered candidate list plus recent-action memory, and the downscaled // screenshot. The strict json_schema response format pins the ranked output. func (s *llmSource) buildRequest(candidates []verifier.ActionCandidate) llmclient.Request { userParts := []llmclient.ContentPart{llmclient.TextPart(s.userPrompt(candidates))} if screenshot := s.verifier.Screenshot(); len(screenshot) > 0 { if dataURL, ok := screenshotDataURL(screenshot, llmMaxImageEdge); ok { userParts = append(userParts, llmclient.ImagePart(dataURL)) } } return llmclient.Request{ Model: s.model, Messages: []llmclient.Message{ {Role: "system", Content: []llmclient.ContentPart{llmclient.TextPart(s.systemPrompt())}}, {Role: "user", Content: userParts}, }, ResponseFormat: choiceResponseFormat(len(candidates)), } } // systemPrompt is the base framing plus any spec-level instructions, appended as // extra guidance so a spec can steer the model's bug-hunting without losing the // candidate-kind semantics the base prompt establishes. func (s *llmSource) systemPrompt() string { if strings.TrimSpace(s.instructions) == "" { return llmSystemPrompt } return llmSystemPrompt + "\n\n" + s.instructions } // userPrompt renders the numbered candidate list (with weights) and the // recent-action memory. func (s *llmSource) userPrompt(candidates []verifier.ActionCandidate) string { var builder strings.Builder builder.WriteString("Actions available on the current screen:\n") for _, candidate := range candidates { fmt.Fprintf(&builder, "%d. %s", candidate.Index, candidate.Description) if candidate.Weighted { fmt.Fprintf(&builder, " (w%d)", candidate.Weight) } builder.WriteByte('\n') } if recent := s.history.recent(); len(recent) > 0 { builder.WriteString("\nYour recent actions (oldest first) and the screen each led to:\n") for _, entry := range recent { screen := entry.screen if screen == "" { screen = "(current screen)" } fmt.Fprintf(&builder, "- %s -> %s\n", entry.action, screen) } } builder.WriteString("\nPick one action by its number.") return builder.String() } // choiceResponseFormat is the strict structured-output schema. Field order is // pinned via raw JSON with reasoning FIRST, so the model reasons before it // commits to a number (a materially better ordering than answer-first). text is // required by strict mode but empty for non-typing actions. func choiceResponseFormat(candidateCount int) *llmclient.ResponseFormat { schema := fmt.Sprintf(`{ "type": "object", "properties": { "reasoning": {"type": "string", "description": "One short sentence on what you are trying to do and why this action."}, "choice": {"type": "integer", "minimum": 1, "maximum": %d, "description": "The number of the chosen action."}, "chosen_action": {"type": "string", "description": "The chosen action's text, copied verbatim from its numbered line."}, "text": {"type": "string", "description": "For a typing action, the text to enter; otherwise an empty string."} }, "required": ["reasoning", "choice", "chosen_action", "text"], "additionalProperties": false }`, candidateCount) return &llmclient.ResponseFormat{ Type: "json_schema", JSONSchema: llmclient.JSONSchema{ Name: "action_choice", Strict: true, Schema: json.RawMessage(schema), }, } } // choiceOutput is the model's structured response, reasoning first. type choiceOutput struct { Reasoning string `json:"reasoning"` Choice int `json:"choice"` ChosenAction string `json:"chosen_action"` Text string `json:"text"` } // weightSuffix matches the trailing " (w34)" annotation appended to each // numbered line, which models copy verbatim into chosen_action. var weightSuffix = regexp.MustCompile(`\s*\(w\d+\)$`) // stripWeightSuffix trims surrounding whitespace and a trailing weight // annotation from the model's echoed action so it can be compared to the // weight-free candidate description. func stripWeightSuffix(echo string) string { return strings.TrimSpace(weightSuffix.ReplaceAllString(strings.TrimSpace(echo), "")) } // parseChoice decodes the model's JSON content into the structured choice. func parseChoice(content string) (choiceOutput, error) { content = strings.TrimSpace(content) if content == "" { return choiceOutput{}, errors.New("empty content") } var out choiceOutput if err := json.Unmarshal([]byte(content), &out); err != nil { return choiceOutput{}, err } if out.Choice == 0 { return choiceOutput{}, errors.New("no choice") } return out, nil } // describeAction renders a short action summary for the recent-action memory. func describeAction(action verifier.Action) string { switch action.Kind { case verifier.ActionKindInputText: return fmt.Sprintf("InputText %s = %q", actionTarget(action), action.Text) case verifier.ActionKindScroll: // A builtin gesture carries endpoints rather than a selector, so name the // container by where the drag starts; that is what tells two scrollable // regions apart in the recent-action memory. target := action.On if target == "" { target = fmt.Sprintf("(%d,%d)", action.FromX, action.FromY) } return fmt.Sprintf("Scroll %s %s", action.Direction, target) case verifier.ActionKindSwipe: // Coordinates make a repeated identical swipe recognizable in the // prompt's recent-action memory. return fmt.Sprintf("Swipe from (%d,%d)", action.FromX, action.FromY) case verifier.ActionKindPressKey: return "PressKey " + action.Key case verifier.ActionKindWait: return "Wait" default: return fmt.Sprintf("%s %s", action.Kind, actionTarget(action)) } } func actionTarget(action verifier.Action) string { if action.On != "" { return action.On } return fmt.Sprintf("(%d,%d)", action.X, action.Y) } // historyEntry records one performed action and the screen it led to (filled on // the following step, once that screen is observed). type historyEntry struct { action string screen string } // actionHistory is a bounded ring of recent actions for the prompt. type actionHistory struct { entries []historyEntry size int } func newActionHistory(size int) *actionHistory { return &actionHistory{size: size} } // completeLast fills the most recent action's led-to screen with the // just-observed screen, if it was still pending. func (h *actionHistory) completeLast(screen string) { if n := len(h.entries); n > 0 && h.entries[n-1].screen == "" { h.entries[n-1].screen = screen } } // add appends an action (its led-to screen pending) and trims to size. func (h *actionHistory) add(action string) { h.entries = append(h.entries, historyEntry{action: action}) if len(h.entries) > h.size { h.entries = h.entries[len(h.entries)-h.size:] } } func (h *actionHistory) recent() []historyEntry { return h.entries } // stampActionSource records the backend that chose an action on the trace. // Only an LLM-selected action (not a setup action the JS path produced) carries // source="llm" and the model's reasoning. func stampActionSource(traceAction *trace.Action, source ActionSource) { if traceAction == nil { return } llm, ok := source.(*llmSource) if !ok || llm.lastSource == "" { return } traceAction.Source = llm.lastSource traceAction.LLMReasoning = llm.lastReasoning traceAction.LLMChoice = llm.lastChoice traceAction.LLMChosenAction = llm.lastChosenAction } // screenshotDataURL downscales the PNG and encodes it as a data URL for the // image content part. func screenshotDataURL(pngBytes []byte, maxEdge int) (string, bool) { scaled := downscalePNG(pngBytes, maxEdge) if len(scaled) == 0 { return "", false } return "data:image/png;base64," + base64.StdEncoding.EncodeToString(scaled), true } // downscalePNG shrinks the image so its long edge is at most maxEdge, returning // the original bytes when it is already small enough and nil on decode failure. func downscalePNG(pngBytes []byte, maxEdge int) []byte { source, err := png.Decode(bytes.NewReader(pngBytes)) if err != nil { return nil } bounds := source.Bounds() width, height := bounds.Dx(), bounds.Dy() if width <= 0 || height <= 0 { return nil } longEdge := max(width, height) if longEdge <= maxEdge { return pngBytes } scale := float64(maxEdge) / float64(longEdge) newWidth := max(1, int(float64(width)*scale)) newHeight := max(1, int(float64(height)*scale)) var buffer bytes.Buffer if err := png.Encode(&buffer, boxDownscale(source, newWidth, newHeight)); err != nil { return nil } return buffer.Bytes() } // boxDownscale averages each destination pixel over its source box, a cheap // dependency-free downscale that keeps text legible enough for the model. func boxDownscale(source image.Image, newWidth, newHeight int) image.Image { bounds := source.Bounds() width, height := bounds.Dx(), bounds.Dy() dest := image.NewRGBA(image.Rect(0, 0, newWidth, newHeight)) for dy := range newHeight { sy0 := dy * height / newHeight sy1 := max((dy+1)*height/newHeight, sy0+1) for dx := range newWidth { sx0 := dx * width / newWidth sx1 := max((dx+1)*width/newWidth, sx0+1) var r, g, b, a, count uint64 for sy := sy0; sy < sy1; sy++ { for sx := sx0; sx < sx1; sx++ { pr, pg, pb, pa := source.At(bounds.Min.X+sx, bounds.Min.Y+sy).RGBA() r += uint64(pr) g += uint64(pg) b += uint64(pb) a += uint64(pa) count++ } } count = max(1, count) dest.Set(dx, dy, color.RGBA64{ R: uint16(r / count), G: uint16(g / count), B: uint16(b / count), A: uint16(a / count), }) } } return dest }