sanderling/internal/driver/ioscompanion/input.go

// Package ioscompanion drives an iOS simulator through the native simulator
// companion. This file composes text input and gestures into HID streams and
// implements the pasteboard fallback for text that the hardware keyboard
// cannot type (anything outside the mappable rune set, such as accented
// letters or emoji).
package ioscompanion

import (
	"context"
	"encoding/json"
	"fmt"
	"os/exec"
	"strings"
	"time"

	"github.com/priyanshujain/sanderling/internal/driver/ioscompanion/transport"
)

// DefaultDoubleTapGapMilliseconds is a sensible inter-tap gap for a synthesized
// double tap. The driver owns the real default; gesture composers take the gap
// as a parameter so the value stays configurable.
const DefaultDoubleTapGapMilliseconds = 70

// pasteAttempts bounds the paste-and-dismiss-dialog retry loop. Each attempt
// resends the paste chord and checks for the permission dialog or a landed
// value, so eight attempts comfortably covers the one-to-two dialogs iOS shows
// per app session plus a couple of refocus retries.
const pasteAttempts = 8

// pasteSettle is how long to wait after a paste chord before reading the
// hierarchy. The warm-paste measurement in the validated spike settled around
// 124ms; this leaves margin for a cold paste while keeping the loop tight.
const pasteSettle = 350 * time.Millisecond

// dialogSettle is the pause after tapping the dialog's allow button and after
// refocusing the field, before the next paste attempt.
const dialogSettle = 200 * time.Millisecond

// warmUpPrimer is the throwaway string pasted at session start so the
// permission dialog fires and gets handled before any real input.
const warmUpPrimer = "sanderling"

// runner abstracts the simulator-companion side effects InputText needs so the
// decision logic stays testable without a live device. The driver supplies a
// real implementation; tests supply a fake.
type runner interface {
	// setPasteboard places text on the simulator pasteboard.
	setPasteboard(ctx context.Context, text string) error
	// sendHID sends one HID stream to the companion.
	sendHID(ctx context.Context, events ...transport.HIDEvent) error
	// describeAll returns the flat describe-all accessibility dump.
	describeAll(ctx context.Context) ([]byte, error)
	// sleep waits, respecting context cancellation.
	sleep(ctx context.Context, duration time.Duration) error
}

// simctlRunner is the production runner. It shells out to simctl for the
// pasteboard and uses the transport companion for everything else.
type simctlRunner struct {
	companion transport.Companion
	udid      string
}

func (r simctlRunner) setPasteboard(ctx context.Context, text string) error {
	command := exec.CommandContext(ctx, "xcrun", "simctl", "pbcopy", r.udid)
	command.Stdin = strings.NewReader(text)
	return command.Run()
}

func (r simctlRunner) sendHID(ctx context.Context, events ...transport.HIDEvent) error {
	return r.companion.SendHID(ctx, events...)
}

func (r simctlRunner) describeAll(ctx context.Context) ([]byte, error) {
	info, err := r.companion.AccessibilityInfo(ctx)
	if err != nil {
		return nil, err
	}
	return []byte(info), nil
}

func (r simctlRunner) sleep(ctx context.Context, duration time.Duration) error {
	timer := time.NewTimer(duration)
	defer timer.Stop()
	select {
	case <-ctx.Done():
		return ctx.Err()
	case <-timer.C:
		return nil
	}
}

// fieldTarget identifies the focused field for the pasteboard path: its
// AXUniqueId (to confirm the paste landed) and its on-screen center (to refocus
// after dismissing the permission dialog).
type fieldTarget struct {
	identifier string
	centerX    float64
	centerY    float64
}

// pasteLengthThreshold is the rune count above which mappable text still goes
// through the pasteboard. Typed keys render progressively on the simulator
// (roughly 75ms per character), so a long typed string keeps the screen
// churning well past the HID call and stretches the post-action settle; a
// paste lands the whole value in one frame.
const pasteLengthThreshold = 3

// usesPasteboard reports whether text takes the pasteboard path: any
// unmappable rune forces it, and longer mappable text uses it so the value
// lands atomically.
func usesPasteboard(text string) bool {
	_, skipped := typeString(text)
	return len(skipped) > 0 || len([]rune(text)) > pasteLengthThreshold
}

// inputText types text into the focused field. Short mappable text goes
// through the hardware keyboard in one HID stream; everything else goes
// through the pasteboard. The field target is only consulted on the
// pasteboard path.
func inputText(ctx context.Context, run runner, text string, field fieldTarget) error {
	if !usesPasteboard(text) {
		presses, _ := typeString(text)
		return run.sendHID(ctx, keyPressEvents(presses)...)
	}
	return pasteText(ctx, run, text, field)
}

// pasteText copies the full text to the pasteboard, sends the paste chord, and
// loops past the permission dialog until the field reflects the text or the
// attempts are exhausted.
func pasteText(ctx context.Context, run runner, text string, field fieldTarget) error {
	if err := run.setPasteboard(ctx, text); err != nil {
		return fmt.Errorf("set pasteboard: %w", err)
	}
	for attempt := 0; attempt < pasteAttempts; attempt++ {
		if err := run.sendHID(ctx, pasteChordEvents()...); err != nil {
			return fmt.Errorf("send paste chord: %w", err)
		}
		// A warm paste lands within a frame or two; check once before paying
		// the settle sleep so the common case stays fast.
		quick, err := run.describeAll(ctx)
		if err != nil {
			return fmt.Errorf("describe accessibility: %w", err)
		}
		if pasteLanded(quick, field.identifier, text) {
			return nil
		}
		if err := run.sleep(ctx, pasteSettle); err != nil {
			return err
		}
		dump, err := run.describeAll(ctx)
		if err != nil {
			return fmt.Errorf("describe accessibility: %w", err)
		}
		if pasteLanded(dump, field.identifier, text) {
			return nil
		}
		button, found := findAllowPasteButton(dump)
		if !found {
			if err := run.sleep(ctx, dialogSettle); err != nil {
				return err
			}
			continue
		}
		if err := run.sendHID(ctx, tapEvents(button.centerX, button.centerY)...); err != nil {
			return fmt.Errorf("tap allow button: %w", err)
		}
		if err := run.sleep(ctx, dialogSettle); err != nil {
			return err
		}
		if err := run.sendHID(ctx, tapEvents(field.centerX, field.centerY)...); err != nil {
			return fmt.Errorf("refocus field: %w", err)
		}
		if err := run.sleep(ctx, dialogSettle); err != nil {
			return err
		}
	}
	return fmt.Errorf("paste did not land after %d attempts", pasteAttempts)
}

// warmUpPaste pastes a throwaway primer once so the iOS pasteboard-permission
// dialog fires and gets dismissed at a moment the driver chooses (session
// start) rather than mid-run. It does not assert the paste landed: the goal is
// only to clear the dialog. A best-effort allow-button tap handles the prompt.
func warmUpPaste(ctx context.Context, run runner) error {
	if err := run.setPasteboard(ctx, warmUpPrimer); err != nil {
		return fmt.Errorf("set pasteboard: %w", err)
	}
	if err := run.sendHID(ctx, pasteChordEvents()...); err != nil {
		return fmt.Errorf("send paste chord: %w", err)
	}
	if err := run.sleep(ctx, pasteSettle); err != nil {
		return err
	}
	dump, err := run.describeAll(ctx)
	if err != nil {
		return fmt.Errorf("describe accessibility: %w", err)
	}
	if button, found := findAllowPasteButton(dump); found {
		if err := run.sendHID(ctx, tapEvents(button.centerX, button.centerY)...); err != nil {
			return fmt.Errorf("tap allow button: %w", err)
		}
	}
	return nil
}

// eraseText deletes characterCount characters from the focused field by sending
// that many backspaces in one HID stream.
func eraseText(ctx context.Context, run runner, characterCount int) error {
	if characterCount <= 0 {
		return nil
	}
	return run.sendHID(ctx, keyPressEvents(backspaces(characterCount))...)
}

// keyPressEvents flattens key presses into a HID event stream. A shifted press
// is wrapped with left-shift down before and up after, so the shift modifier is
// held only for that key.
func keyPressEvents(presses []KeyPress) []transport.HIDEvent {
	events := make([]transport.HIDEvent, 0, len(presses)*2)
	for _, press := range presses {
		if press.Shift {
			events = append(events, transport.KeyDown(usageLeftShift))
		}
		events = append(events, transport.KeyDown(press.Usage), transport.KeyUp(press.Usage))
		if press.Shift {
			events = append(events, transport.KeyUp(usageLeftShift))
		}
	}
	return events
}

// pasteChordEvents is the command+V chord: command down, V down, V up,
// command up.
func pasteChordEvents() []transport.HIDEvent {
	return []transport.HIDEvent{
		transport.KeyDown(LeftGUI),
		transport.KeyDown(VKey),
		transport.KeyUp(VKey),
		transport.KeyUp(LeftGUI),
	}
}

// tapEvents is a single tap: finger down then up at one point.
func tapEvents(x, y float64) []transport.HIDEvent {
	return []transport.HIDEvent{transport.TouchDown(x, y), transport.TouchUp(x, y)}
}

// doubleTapEvents is two taps in one stream separated by gapMilliseconds.
func doubleTapEvents(x, y float64, gapMilliseconds float64) []transport.HIDEvent {
	return []transport.HIDEvent{
		transport.TouchDown(x, y), transport.TouchUp(x, y),
		transport.Delay(gapMilliseconds),
		transport.TouchDown(x, y), transport.TouchUp(x, y),
	}
}

// longPressEvents is a finger held down for holdMilliseconds before lifting.
func longPressEvents(x, y float64, holdMilliseconds float64) []transport.HIDEvent {
	return []transport.HIDEvent{
		transport.TouchDown(x, y),
		transport.Delay(holdMilliseconds),
		transport.TouchUp(x, y),
	}
}

// allowPasteButton is the located allow button of the paste-permission dialog.
type allowPasteButton struct {
	centerX float64
	centerY float64
}

// allowPasteLabels are the known en-US labels of the paste dialog's accept
// button. iOS also surfaces a reject button ("Don't Allow Paste"), so a plain
// "Allow" substring match would be ambiguous; the located labels are matched
// exactly against the trimmed AXLabel.
var allowPasteLabels = []string{"Allow Paste", "Allow"}

// findAllowPasteButton locates the allow button of the paste-permission dialog
// in a describe-all dump. It first matches a button whose label is a known
// allow label. Failing that, it applies a conservative fallback: if exactly one
// enabled button is present in the dump, that lone button is taken to be the
// dialog's allow control. The fallback is deliberately narrow so it cannot fire
// on an ordinary screen full of buttons; the dialog is modal and collapses the
// dump to its own controls.
func findAllowPasteButton(dump []byte) (allowPasteButton, bool) {
	elements := decodeDump(dump)

	for _, element := range elements {
		if element.Type != "Button" {
			continue
		}
		label := strings.TrimSpace(stringValue(element.AXLabel))
		if isRejectPasteLabel(label) {
			continue
		}
		for _, allow := range allowPasteLabels {
			if label == allow {
				if center, ok := buttonCenter(element); ok {
					return center, true
				}
			}
		}
	}

	var soleEnabled allowPasteButton
	enabledButtons := 0
	for _, element := range elements {
		if element.Type != "Button" || !element.Enabled {
			continue
		}
		if isRejectPasteLabel(strings.TrimSpace(stringValue(element.AXLabel))) {
			continue
		}
		center, ok := buttonCenter(element)
		if !ok {
			continue
		}
		enabledButtons++
		soleEnabled = center
	}
	if enabledButtons == 1 {
		return soleEnabled, true
	}
	return allowPasteButton{}, false
}

// isRejectPasteLabel reports whether label is the dialog's reject button. iOS
// renders the apostrophe as a right single quotation mark (U+2019), so both the
// curly and straight forms are checked.
func isRejectPasteLabel(label string) bool {
	return strings.Contains(label, "Don’t Allow Paste") ||
		strings.Contains(label, "Don't Allow Paste")
}

func buttonCenter(element rawElement) (allowPasteButton, bool) {
	frame := element.Frame
	if !finite(frame.X) || !finite(frame.Y) || !finite(frame.Width) || !finite(frame.Height) {
		return allowPasteButton{}, false
	}
	if frame.Width == 0 && frame.Height == 0 {
		return allowPasteButton{}, false
	}
	return allowPasteButton{
		centerX: frame.X + frame.Width/2,
		centerY: frame.Y + frame.Height/2,
	}, true
}

// pasteLanded reports whether the field identified by fieldIdentifier now shows
// expectedText in its AXValue. The paste appends at the cursor, so a substring
// match (rather than equality) is used: the field may already hold text.
func pasteLanded(dump []byte, fieldIdentifier, expectedText string) bool {
	if fieldIdentifier == "" || expectedText == "" {
		return false
	}
	for _, element := range decodeDump(dump) {
		if stringValue(element.AXUniqueID) != fieldIdentifier {
			continue
		}
		return strings.Contains(stringValue(element.AXValue), expectedText)
	}
	return false
}

// decodeDump parses a flat describe-all dump into elements, reusing the same
// element shape and per-element tolerance as the hierarchy mapper: a single
// malformed entry is skipped rather than discarding the whole dump.
func decodeDump(dump []byte) []rawElement {
	var rawElements []json.RawMessage
	if len(dump) > 0 {
		_ = json.Unmarshal(dump, &rawElements)
	}
	elements := make([]rawElement, 0, len(rawElements))
	for _, raw := range rawElements {
		var element rawElement
		if err := json.Unmarshal(raw, &element); err != nil {
			continue
		}
		elements = append(elements, element)
	}
	return elements
}