sanderling/internal/verifier/llm.go

package verifier

import (
	"encoding/json"
	"errors"
	"fmt"
	"math"
	"strconv"
	"strings"

	"github.com/dop251/goja"

	"github.com/priyanshujain/sanderling/internal/hierarchy"
)

// LLMConfig is the spec-declared configuration for the LLM action generator,
// read off globalThis.generator when the spec assigned `generator = llm({...})`.
// It is orthogonal to globalThis.actions (the weighted tree the LLM picks from);
// only the picker differs.
type LLMConfig struct {
	Model string
	// Instructions is optional spec-level guidance appended to the prompt to
	// steer the model toward bug-hunting (empty when unset).
	Instructions string
}

// LLMConfig reports the LLM action-generator config when the spec declared one
// (globalThis.generator.kind === "llm"). The second return is false for every
// other spec, so the runner falls back to the seeded picker.
func (v *Verifier) LLMConfig() (LLMConfig, bool) {
	generator := v.runtime.GlobalObject().Get("generator")
	if generator == nil || goja.IsUndefined(generator) || goja.IsNull(generator) {
		return LLMConfig{}, false
	}
	object := generator.ToObject(v.runtime)
	if object == nil {
		return LLMConfig{}, false
	}
	kind := object.Get("kind")
	if kind == nil || kind.String() != "llm" {
		return LLMConfig{}, false
	}
	config := object.Get("config")
	if config == nil || goja.IsUndefined(config) || goja.IsNull(config) {
		return LLMConfig{}, false
	}
	configObject := config.ToObject(v.runtime)
	if configObject == nil {
		return LLMConfig{}, false
	}
	model := ""
	if value := configObject.Get("model"); value != nil && !goja.IsUndefined(value) {
		model = value.String()
	}
	instructions := ""
	if value := configObject.Get("instructions"); value != nil && !goja.IsUndefined(value) && !goja.IsNull(value) {
		instructions = value.String()
	}
	return LLMConfig{Model: model, Instructions: instructions}, true
}

// Screenshot returns the most recent step's screenshot PNG (set by
// PushSnapshot), or nil if none was captured.
func (v *Verifier) Screenshot() []byte {
	return v.lastScreenshot
}

// CurrentScreen returns the screen id of the most recent snapshot's first
// element, matching the runner's own screen labeling. Empty when no tree is
// loaded.
func (v *Verifier) CurrentScreen() string {
	if v.lastTree == nil || len(v.lastTree.Elements) == 0 {
		return ""
	}
	return v.lastTree.Elements[0].Screen
}

// SampleInput draws one InputText value from the shared corpus via the bundled
// __sanderlingSampleInput__. It errors when the bundle did not install the
// callable (a raw-JS fixture) so the caller can skip typing rather than send an
// empty string.
func (v *Verifier) SampleInput() (string, error) {
	if v.sampleInputFn == nil {
		return "", errors.New("verifier: input sampler not available")
	}
	value, err := v.sampleInputFn(goja.Undefined())
	if err != nil {
		return "", err
	}
	if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
		return "", nil
	}
	return value.String(), nil
}

// ActionCandidate is one selectable action the LLM generator may choose from,
// enumerated by collect-walking the spec's weighted actionsRoot (the same tree
// the seeded picker draws). Each candidate is a concrete, ready-to-execute
// action carrying a plainly-worded Description (numbered and echoed for
// strict-skip) plus its effective Weight so the model sees the spec's testing
// priorities.
type ActionCandidate struct {
	// Index is the candidate's 1-based position in the numbered list the model
	// picks a number from.
	Index int
	// Kind is the resulting action kind.
	Kind ActionKind
	// Description is the rendered action shown to the model and echoed back as
	// chosen_action, e.g. `Tap "Add credit"`. Dedup keys on it, so it is unique.
	Description string
	// Label is the visible-text target label (empty for gestures).
	Label string
	// Weight is the effective selection weight as a percentage (1..100),
	// meaningful only when Weighted is true (the tree used `weighted`).
	Weight   int
	Weighted bool
	// InputType hints a typing field's expected input (e.g. "number", or the
	// field's hint); empty when unknown or not a typing candidate.
	InputType string
	// Direction is up/down/left/right for gesture (Scroll) candidates, else "".
	Direction string
	// LLMText is true for builtin typing, where the model supplies the value;
	// false for authored InputText, whose sampled Action.Text is replayed as-is.
	LLMText bool
	// Action is the concrete action executed when this candidate is chosen. For
	// builtin typing it carries no text until the model's value is filled in.
	Action Action

	// prob is the internal accumulated selection probability, summed across
	// dedup, then rounded into Weight. Not exposed in the prompt directly.
	prob float64
}

// maxLabelRunes caps a visible-text label so joined descendant text stays short
// enough to render on one numbered line.
const maxLabelRunes = 40

// Candidates enumerates every action the spec's weighted actionsRoot yields at
// the current step, each tagged with a plainly-worded description and its
// effective weight, for the LLM generator to pick one number from. It walks the
// SAME tree the seeded picker draws: weighted branches recurse (accumulating the
// selection probability), authored actions()/whenRoute leaves are called once
// for their concrete actions, and builtin verbs come straight from the picker's
// own enumeration. Identical descriptions dedup, summing weight.
func (v *Verifier) Candidates() []ActionCandidate {
	if v.lastTree == nil {
		return nil
	}
	root := v.runtime.GlobalObject().Get("actions")
	if root == nil || goja.IsUndefined(root) || goja.IsNull(root) {
		return nil
	}
	nodeIndex := buildNodeIndex(v.lastTree)
	var raw []ActionCandidate
	v.collectNode(root, 1.0, false, nodeIndex, &raw)
	return finalizeCandidates(raw)
}

// collectNode dispatches one GeneratorNode of the action tree. prob is the
// accumulated probability the seeded picker reaches this node; weighted records
// whether any weighted node lies on the path (so weights are shown only when the
// spec actually declared them).
func (v *Verifier) collectNode(node goja.Value, prob float64, weighted bool, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) {
	object := node.ToObject(v.runtime)
	if object == nil {
		return
	}
	kind := object.Get("kind")
	if kind == nil || goja.IsUndefined(kind) {
		return
	}
	switch kind.String() {
	case "weighted":
		v.collectWeighted(object, prob, nodeIndex, out)
	case "actions":
		v.collectActions(object, prob, weighted, nodeIndex, out)
	case "builtin":
		verb := object.Get("verb")
		if verb != nil && !goja.IsUndefined(verb) {
			v.collectBuiltin(verb.String(), prob, weighted, nodeIndex, out)
		}
	case "llm":
		// The llm marker is the generator, not part of the candidate tree.
	}
}

// collectWeighted recurses each branch, splitting the incoming probability by
// the branch weight over the sibling total (matching the seeded picker's single
// weighted draw).
func (v *Verifier) collectWeighted(object *goja.Object, prob float64, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) {
	branches := object.Get("branches")
	if branches == nil {
		return
	}
	array := branches.ToObject(v.runtime)
	if array == nil {
		return
	}
	length := int(array.Get("length").ToInteger())
	weights := make([]float64, length)
	children := make([]goja.Value, length)
	total := 0.0
	for i := range length {
		entry := array.Get(strconv.Itoa(i))
		pair := entry.ToObject(v.runtime)
		if pair == nil {
			continue
		}
		weight := pair.Get("0").ToFloat()
		if weight < 0 || math.IsNaN(weight) {
			weight = 0
		}
		weights[i] = weight
		children[i] = pair.Get("1")
		total += weight
	}
	if total <= 0 {
		return
	}
	for i := range length {
		if children[i] == nil {
			continue
		}
		v.collectNode(children[i], prob*weights[i]/total, true, nodeIndex, out)
	}
}

// collectActions calls an authored leaf's generator once (safe: it reads state
// and, off-route, returns []), turning each concrete descriptor into a
// candidate. It runs OUTSIDE the picker's rng scope, so from(...).generate()
// draws nothing and no seed advances.
func (v *Verifier) collectActions(object *goja.Object, prob float64, weighted bool, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) {
	generate, ok := goja.AssertFunction(object.Get("generate"))
	if !ok {
		return
	}
	result, err := generate(goja.Undefined())
	if err != nil {
		return
	}
	array := result.ToObject(v.runtime)
	if array == nil {
		return
	}
	length := int(array.Get("length").ToInteger())
	for i := range length {
		candidate, ok := v.candidateFromDescriptor(array.Get(strconv.Itoa(i)), nodeIndex)
		if !ok {
			continue
		}
		candidate.prob = prob
		candidate.Weighted = weighted
		*out = append(*out, candidate)
	}
}

// candidateFromDescriptor lowers one authored ActionDescriptor (as a goja
// object) into a ready-to-run candidate, resolving the target's coordinates,
// selector, and visible-text label. Actions on a disabled control are dropped.
func (v *Verifier) candidateFromDescriptor(value goja.Value, nodeIndex map[*hierarchy.Element]*hierarchy.Node) (ActionCandidate, bool) {
	object := value.ToObject(v.runtime)
	if object == nil {
		return ActionCandidate{}, false
	}
	kindValue := object.Get("kind")
	if kindValue == nil || goja.IsUndefined(kindValue) {
		return ActionCandidate{}, false
	}
	kind := ActionKind(kindValue.String())
	switch kind {
	case ActionKindTap, ActionKindDoubleTap, ActionKindLongPress:
		target := v.resolveTarget(object.Get("on"), nodeIndex)
		if target.disabled {
			return ActionCandidate{}, false
		}
		return ActionCandidate{
			Kind:   kind,
			Label:  target.label,
			Action: Action{Kind: kind, On: target.selector, X: target.x, Y: target.y},
		}, true
	case ActionKindInputText:
		target := v.resolveTarget(object.Get("into"), nodeIndex)
		if target.disabled {
			return ActionCandidate{}, false
		}
		text := stringField(object, "text")
		return ActionCandidate{
			Kind:      kind,
			Label:     target.label,
			InputType: target.inputType,
			Action:    Action{Kind: kind, On: target.selector, X: target.x, Y: target.y, Text: text},
		}, true
	case ActionKindScroll:
		target := v.resolveTarget(object.Get("in"), nodeIndex)
		direction := stringField(object, "direction")
		if direction == "" {
			direction = "down"
		}
		return ActionCandidate{
			Kind:      kind,
			Direction: direction,
			Action:    Action{Kind: kind, On: target.selector, Direction: direction},
		}, true
	case ActionKindSwipe:
		from := v.resolveTarget(object.Get("from"), nodeIndex)
		to := v.resolveTarget(object.Get("to"), nodeIndex)
		return ActionCandidate{
			Kind: kind,
			Action: Action{
				Kind:  kind,
				FromX: from.x, FromY: from.y,
				ToX: to.x, ToY: to.y,
				DurationMillis: intField(object, "durationMillis"),
			},
		}, true
	case ActionKindPressKey:
		return ActionCandidate{
			Kind:   kind,
			Action: Action{Kind: kind, Key: stringField(object, "key")},
		}, true
	case ActionKindWait:
		return ActionCandidate{Kind: kind, Action: Action{Kind: kind}}, true
	default:
		return ActionCandidate{}, false
	}
}

// resolvedTarget is the geometry, selector, label, and input hint a target
// (ax element, selector string, or bare point) resolves to.
type resolvedTarget struct {
	x, y      int
	selector  string
	label     string
	inputType string
	disabled  bool
}

// resolveTarget reads an authored action's target. Ax element handles carry
// x/y/__sanderlingSelector plus their own text; a bare selector string resolves
// against the current tree; a point carries geometry only.
func (v *Verifier) resolveTarget(value goja.Value, nodeIndex map[*hierarchy.Element]*hierarchy.Node) resolvedTarget {
	if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
		return resolvedTarget{}
	}
	if selector, ok := value.Export().(string); ok {
		return v.targetFromSelector(selector, nodeIndex)
	}
	object := value.ToObject(v.runtime)
	if object == nil {
		return resolvedTarget{}
	}
	selector := stringField(object, tagSelector)
	if selector == "" {
		selector = stringField(object, "selector")
	}
	target := resolvedTarget{
		x:        int(object.Get("x").ToInteger()),
		y:        int(object.Get("y").ToInteger()),
		selector: selector,
	}
	if element := v.findBySelector(selector); element != nil {
		target.label = visibleLabel(element, nodeIndex)
		target.inputType = inputTypeHint(element)
		target.disabled = !element.Enabled && hasEnabled(element)
	}
	if target.label == "" {
		target.label = truncateLabel(stringField(object, "text"))
	}
	return target
}

// targetFromSelector resolves a bare selector-string target against the tree.
func (v *Verifier) targetFromSelector(selector string, nodeIndex map[*hierarchy.Element]*hierarchy.Node) resolvedTarget {
	target := resolvedTarget{selector: selector}
	element := v.findBySelector(selector)
	if element == nil {
		return target
	}
	target.x, target.y = element.Bounds.Center()
	target.label = visibleLabel(element, nodeIndex)
	target.inputType = inputTypeHint(element)
	target.disabled = !element.Enabled && hasEnabled(element)
	return target
}

func (v *Verifier) findBySelector(selector string) *hierarchy.Element {
	if selector == "" || v.lastTree == nil {
		return nil
	}
	return v.lastTree.Find(selector)
}

// collectBuiltin turns the picker's own enumeration of a builtin verb into
// candidates. The list comes from the bundle's __sanderlingEnumerateBuiltin__
// (pick.ts builtinCandidates), which is what the seeded policy draws from, so
// the two policies select over one action space and cannot drift apart. Each
// entry's action arrives on the wire contract DecodeAction already reads, so a
// chosen candidate executes the action the seeded draw would have executed.
func (v *Verifier) collectBuiltin(verb string, prob float64, weighted bool, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) {
	entries, err := v.enumerateBuiltin(verb)
	if err != nil {
		return
	}
	targets := v.targets()
	for _, entry := range entries {
		candidate := ActionCandidate{
			Kind:      entry.action.Kind,
			Direction: entry.action.Direction,
			// Builtin typing enumerates the field, not the value: the seeded
			// policy draws its text from the corpus and the model writes its own.
			LLMText:  entry.action.Kind == ActionKindInputText,
			Action:   entry.action,
			prob:     prob,
			Weighted: weighted,
		}
		if entry.targetIndex >= 0 && entry.targetIndex < len(targets) {
			element := targets[entry.targetIndex].element
			candidate.Label = visibleLabel(element, nodeIndex)
			candidate.InputType = inputTypeHint(element)
		}
		*out = append(*out, candidate)
	}
}

// builtinCandidate is one entry of the shared builtin enumeration: the concrete
// action, plus the index of the host candidate it targets (-1 when the verb has
// no target, as for a key press or a wait).
type builtinCandidate struct {
	action      Action
	targetIndex int
}

// enumerateBuiltin invokes the bundle's shared enumeration for one verb. A spec
// loaded without the runtime entry (a raw-JS unit fixture) has no enumerator, so
// the verb contributes nothing rather than falling back to a second enumeration.
func (v *Verifier) enumerateBuiltin(verb string) ([]builtinCandidate, error) {
	if v.enumerateBuiltinFn == nil {
		return nil, errors.New("verifier: builtin enumeration not available")
	}
	value, err := v.enumerateBuiltinFn(goja.Undefined(), v.runtime.ToValue(verb))
	if err != nil {
		return nil, fmt.Errorf("enumerate %s: %w", verb, err)
	}
	if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
		return nil, nil
	}
	raw, err := json.Marshal(value.Export())
	if err != nil {
		return nil, fmt.Errorf("marshal %s enumeration: %w", verb, err)
	}
	var wire []struct {
		Action      json.RawMessage `json:"action"`
		TargetIndex int             `json:"targetIndex"`
	}
	if err := json.Unmarshal(raw, &wire); err != nil {
		return nil, fmt.Errorf("decode %s enumeration: %w", verb, err)
	}
	entries := make([]builtinCandidate, 0, len(wire))
	for _, item := range wire {
		action, err := DecodeAction(item.Action)
		if err != nil {
			continue
		}
		entries = append(entries, builtinCandidate{action: action, targetIndex: item.TargetIndex})
	}
	return entries, nil
}

// finalizeCandidates renders each candidate's description, dedups identical
// descriptions (summing weight), numbers the survivors 1..N, and rounds the
// accumulated probability into a percentage Weight.
func finalizeCandidates(raw []ActionCandidate) []ActionCandidate {
	seen := make(map[string]int, len(raw))
	result := make([]ActionCandidate, 0, len(raw))
	for _, candidate := range raw {
		candidate.Description = describeCandidate(candidate)
		if index, ok := seen[candidate.Description]; ok {
			result[index].prob += candidate.prob
			result[index].Weighted = result[index].Weighted || candidate.Weighted
			continue
		}
		seen[candidate.Description] = len(result)
		result = append(result, candidate)
	}
	for i := range result {
		result[i].Index = i + 1
		if result[i].Weighted {
			result[i].Weight = max(1, int(math.Round(result[i].prob*100)))
		}
	}
	return result
}

// describeCandidate renders the plain, echo-friendly description shown in the
// numbered list. It is the dedup key, so it must be stable and unique per
// distinct action.
func describeCandidate(candidate ActionCandidate) string {
	switch candidate.Kind {
	case ActionKindTap:
		return fmt.Sprintf("Tap %q", candidate.Label)
	case ActionKindDoubleTap:
		return fmt.Sprintf("Double-tap %q", candidate.Label)
	case ActionKindLongPress:
		return fmt.Sprintf("Long-press %q", candidate.Label)
	case ActionKindInputText:
		if candidate.LLMText {
			if candidate.InputType != "" {
				return fmt.Sprintf("Type into %q (%s)", candidate.Label, candidate.InputType)
			}
			return fmt.Sprintf("Type into %q", candidate.Label)
		}
		return fmt.Sprintf("Type %q into %q", candidate.Action.Text, candidate.Label)
	case ActionKindScroll:
		return "Scroll " + candidate.Direction
	case ActionKindSwipe:
		// A swipe carries endpoints and no selector, so the coordinates are what
		// keep two swipes distinct. The label is prepended when the origin
		// element has one, because "swipe that row" is the interaction a model
		// reaches for and a bare pair of points does not say which row.
		where := fmt.Sprintf("from (%d,%d) to (%d,%d)",
			candidate.Action.FromX, candidate.Action.FromY,
			candidate.Action.ToX, candidate.Action.ToY)
		if candidate.Label == "" {
			return "Swipe " + where
		}
		return fmt.Sprintf("Swipe %q %s", candidate.Label, where)
	case ActionKindPressKey:
		return "Press " + candidate.Action.Key
	case ActionKindWait:
		return "Wait"
	default:
		return string(candidate.Kind)
	}
}

// buildNodeIndex maps each Element pointer to its Node so descendant text can be
// borrowed for a control whose own text is empty.
func buildNodeIndex(tree *hierarchy.Tree) map[*hierarchy.Element]*hierarchy.Node {
	index := make(map[*hierarchy.Element]*hierarchy.Node)
	if tree == nil || tree.Root == nil {
		return index
	}
	var walk func(node *hierarchy.Node)
	walk = func(node *hierarchy.Node) {
		index[&node.Element] = node
		for _, child := range node.Children {
			walk(child)
		}
	}
	walk(tree.Root)
	return index
}

// visibleLabel names a control by what a user would read: its own text, then
// description, then a field hint, then text borrowed from its descendants (the
// case that fixes empty-text Compose buttons whose word lives on a child), then
// its class as a last resort.
func visibleLabel(element *hierarchy.Element, nodeIndex map[*hierarchy.Element]*hierarchy.Node) string {
	// An editable field's own text is the transient typed value ("1"); its hint
	// names its purpose ("Amount") and stays stable, so prefer the hint there.
	if element.Editable {
		if hint := element.Attributes["hintText"]; hint != "" {
			return truncateLabel(hint)
		}
	}
	if element.Text != "" {
		return truncateLabel(element.Text)
	}
	if element.Description != "" {
		return truncateLabel(element.Description)
	}
	if hint := element.Attributes["hintText"]; hint != "" {
		return truncateLabel(hint)
	}
	if node := nodeIndex[element]; node != nil {
		if text := descendantText(node); text != "" {
			return truncateLabel(text)
		}
	}
	if element.Class != "" {
		return element.Class
	}
	if element.ResourceID != "" {
		return element.ResourceID
	}
	return "control"
}

// descendantText joins the visible text of a node's descendants in tree order,
// so a clickable wrapper borrows the label of the Text child it contains.
func descendantText(node *hierarchy.Node) string {
	var parts []string
	var walk func(node *hierarchy.Node)
	walk = func(node *hierarchy.Node) {
		for _, child := range node.Children {
			switch {
			case child.Element.Text != "":
				parts = append(parts, child.Element.Text)
			case child.Element.Description != "":
				parts = append(parts, child.Element.Description)
			}
			walk(child)
		}
	}
	walk(node)
	return strings.Join(parts, " ")
}

// truncateLabel trims and shortens a label to one line's worth of runes.
func truncateLabel(text string) string {
	text = strings.TrimSpace(strings.ReplaceAll(text, "\n", " "))
	runes := []rune(text)
	if len(runes) <= maxLabelRunes {
		return text
	}
	return strings.TrimSpace(string(runes[:maxLabelRunes])) + "…"
}

// inputTypeHint reports a typing field's expected input as a short word the
// model can use to synthesize a value, or "" when nothing distinguishes it.
func inputTypeHint(element *hierarchy.Element) string {
	haystack := strings.ToLower(element.Class + " " +
		element.Attributes["inputType"] + " " + element.Attributes["hintText"])
	switch {
	case strings.Contains(haystack, "number") || strings.Contains(haystack, "amount") || strings.Contains(haystack, "numeric"):
		return "number"
	case strings.Contains(haystack, "email"):
		return "email"
	case strings.Contains(haystack, "password"):
		return "password"
	case strings.Contains(haystack, "phone"):
		return "phone"
	default:
		return ""
	}
}

// hasEnabled reports whether the source tree carried an explicit enabled flag
// for the element, so a missing flag is not mistaken for "disabled".
func hasEnabled(element *hierarchy.Element) bool {
	_, ok := element.Attributes["enabled"]
	return ok
}

// stringField reads a string property off a goja object, returning "" when
// absent, null, or undefined.
func stringField(object *goja.Object, key string) string {
	value := object.Get(key)
	if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
		return ""
	}
	return value.String()
}

// intField reads a numeric property off a goja object, returning 0 when absent,
// null, or undefined.
func intField(object *goja.Object, key string) int {
	value := object.Get(key)
	if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
		return 0
	}
	return int(value.ToInteger())
}