sanderling/internal/verifier/llm.go

package verifier

import (
	"errors"

	"github.com/dop251/goja"

	"github.com/priyanshujain/sanderling/internal/hierarchy"
)

// LLMConfig is the spec-declared configuration for the LLM action backend,
// read off globalThis.actions when the spec assigned `actions = llm({...})`.
type LLMConfig struct {
	Model string
	// Instructions is optional spec-level guidance appended to the prompt to
	// steer the model toward bug-hunting (empty when unset).
	Instructions string
}

// LLMConfig reports the LLM action backend config when the spec selected it
// (globalThis.actions.kind === "llm"). The second return is false for every
// other spec, so the runner falls back to the seeded picker.
func (v *Verifier) LLMConfig() (LLMConfig, bool) {
	actions := v.runtime.GlobalObject().Get("actions")
	if actions == nil || goja.IsUndefined(actions) || goja.IsNull(actions) {
		return LLMConfig{}, false
	}
	object := actions.ToObject(v.runtime)
	if object == nil {
		return LLMConfig{}, false
	}
	kind := object.Get("kind")
	if kind == nil || kind.String() != "llm" {
		return LLMConfig{}, false
	}
	config := object.Get("config")
	if config == nil || goja.IsUndefined(config) || goja.IsNull(config) {
		return LLMConfig{}, false
	}
	configObject := config.ToObject(v.runtime)
	if configObject == nil {
		return LLMConfig{}, false
	}
	model := ""
	if value := configObject.Get("model"); value != nil && !goja.IsUndefined(value) {
		model = value.String()
	}
	instructions := ""
	if value := configObject.Get("instructions"); value != nil && !goja.IsUndefined(value) && !goja.IsNull(value) {
		instructions = value.String()
	}
	return LLMConfig{Model: model, Instructions: instructions}, true
}

// Screenshot returns the most recent step's screenshot PNG (set by
// PushSnapshot), or nil if none was captured.
func (v *Verifier) Screenshot() []byte {
	return v.lastScreenshot
}

// CurrentScreen returns the screen id of the most recent snapshot's first
// element, matching the runner's own screen labeling. Empty when no tree is
// loaded.
func (v *Verifier) CurrentScreen() string {
	if v.lastTree == nil || len(v.lastTree.Elements) == 0 {
		return ""
	}
	return v.lastTree.Elements[0].Screen
}

// SampleInput draws one InputText value from the shared corpus via the bundled
// __sanderlingSampleInput__. It errors when the bundle did not install the
// callable (a raw-JS fixture) so the caller can skip typing rather than send an
// empty string.
func (v *Verifier) SampleInput() (string, error) {
	if v.sampleInputFn == nil {
		return "", errors.New("verifier: input sampler not available")
	}
	value, err := v.sampleInputFn(goja.Undefined())
	if err != nil {
		return "", err
	}
	if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
		return "", nil
	}
	return value.String(), nil
}

// ActionCandidate is one selectable action the LLM backend may choose from: an
// applicable verb on an in-scope element, resolved to coordinates and a
// selector. It is the existing per-verb enumeration (candidatesForVerb),
// unioned across verbs and flattened into one indexed list.
type ActionCandidate struct {
	// Index is the candidate's position in the AllCandidates list; the LLM
	// returns these indices.
	Index int
	// Verb is the picker verb that produced this candidate
	// (taps/doubleTaps/longPresses/typing/scrolls/swipes).
	Verb string
	// Kind is the resulting action kind, shown to the model as the candidate's
	// kind label.
	Kind ActionKind
	// Label is a short human-readable target description (text, then
	// description, then resource-id, then class).
	Label string
	// X, Y are the target element center.
	X, Y int
	// Width, Height are the target bounds, used to size swipe/scroll gestures.
	Width, Height int
	// Selector resolves the target back to an element by id/text, or "" when no
	// selector uniquely resolves.
	Selector string
}

// llmVerbs lists the verbs AllCandidates enumerates, in the order they are
// emitted per element. Mirrors verbAccepts; no new filtering logic.
var llmVerbs = []string{"taps", "doubleTaps", "longPresses", "typing", "scrolls", "swipes"}

// AllCandidates flattens the per-verb candidate enumeration into one indexed
// list the LLM backend chooses from. It reuses scopedElements/verbAccepts/
// selectorForElement exactly as the seeded picker does, walking the tree once
// and emitting an entry for every (in-scope element, applicable verb) pair.
func (v *Verifier) AllCandidates() []ActionCandidate {
	if v.lastTree == nil {
		return nil
	}
	scope := v.scopedElements()
	var result []ActionCandidate
	for _, element := range v.lastTree.Elements {
		if !scope[element] {
			continue
		}
		for _, verb := range llmVerbs {
			if !verbAccepts(verb, element) {
				continue
			}
			x, y := element.Bounds.Center()
			result = append(result, ActionCandidate{
				Index:    len(result),
				Verb:     verb,
				Kind:     verbActionKind(verb),
				Label:    candidateLabel(element),
				X:        x,
				Y:        y,
				Width:    element.Bounds.Width(),
				Height:   element.Bounds.Height(),
				Selector: selectorForElement(v.lastTree, element),
			})
		}
	}
	return result
}

// verbActionKind maps a picker verb to the action kind it dispatches.
func verbActionKind(verb string) ActionKind {
	switch verb {
	case "taps":
		return ActionKindTap
	case "doubleTaps":
		return ActionKindDoubleTap
	case "longPresses":
		return ActionKindLongPress
	case "typing":
		return ActionKindInputText
	case "scrolls":
		return ActionKindScroll
	case "swipes":
		return ActionKindSwipe
	default:
		return ""
	}
}

// candidateLabel builds a short target description, preferring the most
// human-meaningful field available.
func candidateLabel(element *hierarchy.Element) string {
	switch {
	case element.Text != "":
		return element.Text
	case element.Description != "":
		return element.Description
	case element.ResourceID != "":
		return element.ResourceID
	default:
		return element.Class
	}
}