iOS simulator driver: Go-native companion-backed backend (#62)

* perf(ios): use prebuilt XCTest runner to cut startup

* chore(ioscompanion): add companion asset prepare script

* feat(ioscompanion): embed and extract simulator companion bundle

* test(ioscompanion): cover companion stub and embedded extraction

* docs: add third party notices for vendored companion

* chore: ignore vendored companion bundle artifact

* build(proto): pin simulator companion proto v1.1.8

* build(proto): add dedicated buf module and gen template for pinned proto

* build(proto): exclude pinned companion proto from root buf workspace

* feat(ioscompanion): commit generated companion gRPC stubs

* feat(ioscompanion): map flat companion describe dump to TreeNode JSON

* test(ioscompanion): add hierarchy-map golden and unit tests

* feat(ioscompanion): port screen-settle stability polling to Go

* test(ioscompanion): cover settle transitional, hash, streak, and cap rules

* feat(ioscompanion): add USB HID keymap module

* test(ioscompanion): cover keymap branches and paste-chord constants

* build: embed companion assets via withcompanion tag

* feat(ioscompanion): add transport companion interface

* feat(ioscompanion): add HID event wrapper and builders

* feat(ioscompanion): wire gRPC companion client and Dial

* test(ioscompanion): cover HID builders and unit conversions

* test(ioscompanion): cover Dial, process-state mapping, and install archive

* test(ioscompanion): add gated simulator integration smoke test

* feat(ioscompanion): text input and gesture HID composition with pasteboard fallback

* test(ioscompanion): cover input composers, paste dialog loop, and pure helpers

* feat(ioscompanion): add Describe to companion transport

* feat(ioscompanion): implement DeviceDriver with companion supervision

* test(ioscompanion): unit tests with fake companion transport

* test(ioscompanion): gated companion smoke test

* feat(ios): add ResolveTarget for simulator vs physical-device routing

* feat(testrun): route iOS simulators through the native companion driver

* refactor(testrun): defer the java preflight check to the physical-device path

* feat(cli): add --ios-app-path flag

* feat(doctor): split iOS checks into simulator and physical-device paths

* test(folio): add gate-analyzer fixtures for G1-G5

* feat(folio): add iOS conformance gate script

* chore(folio): wire gates recipe, app path, and ignore gate output

* style: gofmt struct alignment drift

* fix(doctor): probe simctl via xcrun instead of PATH lookup

* fix(ioscompanion): spawn companion under driver-lifetime context

* test(ioscompanion): prove companion child outlives startup context

* fix(ioscompanion): chunk install payload under companion message cap

* test(ioscompanion): cover install payload chunking

* fix(ioscompanion): reinstall via simctl and sanitize companion env

* fix(ioscompanion): wait out unresolved accessibility values after launch

* perf(ioscompanion): paste long text for atomic landing

* test(ioscompanion): cover paste threshold, retry flow, and sentinel detection

* fix(ioscompanion): treat unresolved bridge values as transitional, never as content

* fix(ioscompanion): accept masked secure-field values as paste landing

* test(ioscompanion): cover sentinel mapping and masked-field landing

* fix(ioscompanion): atomic erase and single-send paste to prevent doubling

* test(ioscompanion): cover atomic erase, single chord, unverifiable field

* fix(ioscompanion): verify paste on a time budget that outlasts the bridge blackout

* test(ioscompanion): cover bridge-blackout paste verification

* fix(ioscompanion): drop unresolved-value settle gate that never let empty-field screens settle

* refactor(ioscompanion): name the empty-editable-field sentinel for what it is

* perf(ioscompanion): tighten settle streak for the fast companion transport

* feat(ioscompanion): pre-grant pasteboard access so unicode input skips the OS prompt

* refactor(ioscompanion): drop paste warm-up now that the grant suppresses the prompt

* test(ioscompanion): cover pasteboard grant on launch, drop warm-up tests

* fix(ioscompanion): retry describe past transient collapsed accessibility dumps

* test(ioscompanion): cover collapsed-dump detection

* perf(ioscompanion): split raw and retrying describe so settle does not double-wait collapses

* perf(ioscompanion): tighten settle now that collapses are handled separately

* fix(ioscompanion): replace field content on input so blackout-skipped erase cannot accumulate text

* test(ioscompanion): cover replace-on-input and TextReplacer capability

* refactor(ioscompanion): neutralize HID events behind the transport seam

* feat(companion): add simulator runner project skeleton

* feat(companion): serve accessibility snapshots over the wire protocol

* feat(companion): synthesize timestamped touch gestures

* feat(companion): type text with replace semantics

* feat(companion): serve the wire protocol from a parked runner

* feat(ioscompanion): add TextEditor capability and unavailable sentinel to the transport seam

* feat(ioscompanion): route text input through a text-editing companion when available

* fix(companion): bind listener by port and source screen size from snapshot

* feat(ioscompanion): add runner companion JSON transport

* test(ioscompanion): cover runner transport protocol mapping

* fix(companion): synthesize gestures synchronously to avoid the async completion crash

* fix(companion): type on the main thread and recover from focus assertions

* fix(companion): keep serving after an automation failure

* refactor(companion): tidy snapshot serialization

* fix(companion): honor sequential tap gaps and survive synthesis exceptions

* feat(ioscompanion): expose native typing with an explicit replace flag

* chore(companion): add runner asset prepare script

* feat(ioscompanion): embed and extract the runner test bundle

* test(ioscompanion): cover runner asset extraction

* build(ioscompanion): commit runner asset archive

* feat(ioscompanion): pair the legacy companion with the in-simulator runner

* test(ioscompanion): cover hybrid routing, paste-grant skip, and port binding

* fix(ioscompanion): reconnect after interrupted runner calls instead of restarting

* fix(ioscompanion): route hybrid lifecycle through the runner and harden restarts

* feat(companion): launch and terminate apps through the automation session

* build(ioscompanion): refresh runner asset with session lifecycle

* fix(ioscompanion): classify connection deadline expiry as caller budget

* fix(companion): capture snapshots on the main thread inside the catch bridge

* build(ioscompanion): refresh runner asset with main-thread snapshots

* perf(ioscompanion): count read spans toward settle and capture snapshots concurrently

* feat(ioscompanion): make the hybrid simulator companion the default

* test(folio): cover runner-session orphans in the gate harness

* test(ioscompanion): pin the child-lifetime test to the legacy path

* fix(ioscompanion): keep mappable text on one HID stream and verify unicode clears

* fix(ioscompanion): pause the clear chord so selection applies before the delete

* fix(companion): prune the keyboard subtree from snapshots

* build(ioscompanion): refresh runner asset without keyboard elements

* fix(ioscompanion): capture the screenshot transport before a recovery can reassign it

* fix(companion): pin the runner listener to loopback

* fix(companion): size the replace delete prefix to cover any focused field

* build(ioscompanion): refresh runner asset with loopback bind and replace fix

* fix(cli): cancel the run context on SIGINT so spawned children are reaped

* fix(testrun): point the device java preflight hint at the ios-device doctor

* fix(folio): word-bound the G2 ERROR scan and drop the dead objc allowlist glob

* test(ioscompanion): cover stopProcess, restart, and failed bring-up supervision

* chore: add test-companion target for the withcompanion-tagged suite

* chore(ioscompanion): stop tracking the runner archive build artifact

* build: produce the runner archive from source like the companion bundle

* refactor(conformance): move the gate harness out of examples/folio

* chore(folio): drop the gate harness wiring from the example app
This commit is contained in:
pj authored and GitHub committed 2026-06-08 19:10:54 +05:30
1 parent 94d9511312
commit 406b7516b3
97 files changed
+22104 -83

No files matched your search

+433
View File
@@ -0,0 +1,433 @@
package ioscompanion
import (
"context"
"os"
"path/filepath"
"reflect"
"testing"
"time"
"github.com/priyanshujain/sanderling/internal/driver/ioscompanion/transport"
)
func eventsEqual(t *testing.T, got, want []transport.HIDEvent) {
t.Helper()
if len(got) != len(want) {
t.Fatalf("event count: got %d, want %d", len(got), len(want))
}
for i := range want {
if !reflect.DeepEqual(got[i], want[i]) {
t.Fatalf("event %d differs", i)
}
}
}
func TestKeyPressEvents(t *testing.T) {
tests := []struct {
name string
presses []KeyPress
want []transport.HIDEvent
}{
{
name: "lowercase letter is down then up, no shift",
presses: []KeyPress{{Usage: usageA}},
want: []transport.HIDEvent{transport.KeyDown(usageA), transport.KeyUp(usageA)},
},
{
name: "shifted letter wraps with left shift down and up",
presses: []KeyPress{{Usage: usageA, Shift: true}},
want: []transport.HIDEvent{
transport.KeyDown(usageLeftShift),
transport.KeyDown(usageA),
transport.KeyUp(usageA),
transport.KeyUp(usageLeftShift),
},
},
{
name: "empty input yields no events",
presses: nil,
want: []transport.HIDEvent{},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
eventsEqual(t, keyPressEvents(test.presses), test.want)
})
}
}
func TestKeyPressEventsMixedString(t *testing.T) {
presses, skipped := typeString("aB")
if len(skipped) != 0 {
t.Fatalf("unexpected skipped runes: %v", skipped)
}
got := keyPressEvents(presses)
want := []transport.HIDEvent{
transport.KeyDown(usageA), transport.KeyUp(usageA),
transport.KeyDown(usageLeftShift),
transport.KeyDown(usageA + 1),
transport.KeyUp(usageA + 1),
transport.KeyUp(usageLeftShift),
}
eventsEqual(t, got, want)
}
func TestPasteChordEvents(t *testing.T) {
want := []transport.HIDEvent{
transport.KeyDown(LeftGUI),
transport.KeyDown(VKey),
transport.KeyUp(VKey),
transport.KeyUp(LeftGUI),
}
eventsEqual(t, pasteChordEvents(), want)
}
func TestTapEvents(t *testing.T) {
want := []transport.HIDEvent{transport.TouchDown(12, 34), transport.TouchUp(12, 34)}
eventsEqual(t, tapEvents(12, 34), want)
}
func TestDoubleTapEvents(t *testing.T) {
want := []transport.HIDEvent{
transport.TouchDown(5, 6), transport.TouchUp(5, 6),
transport.Delay(70),
transport.TouchDown(5, 6), transport.TouchUp(5, 6),
}
eventsEqual(t, doubleTapEvents(5, 6, DefaultDoubleTapGapMilliseconds), want)
}
func TestLongPressEvents(t *testing.T) {
want := []transport.HIDEvent{
transport.TouchDown(8, 9),
transport.Delay(500),
transport.TouchUp(8, 9),
}
eventsEqual(t, longPressEvents(8, 9, 500), want)
}
func loadDialogDump(t *testing.T) []byte {
t.Helper()
dump, err := os.ReadFile(filepath.Join("testdata", "paste-dialog.json"))
if err != nil {
t.Fatalf("read testdata: %v", err)
}
return dump
}
func TestFindAllowPasteButton(t *testing.T) {
tests := []struct {
name string
dump string
wantFound bool
wantX float64
wantY float64
}{
{
name: "exact Allow Paste wins over Don't Allow Paste",
dump: string(loadDialogDump(t)),
wantFound: true,
wantX: 280, // 210 + 140/2
wantY: 465, // 440 + 50/2
},
{
name: "plain Allow label matches",
dump: `[{"type":"Button","AXLabel":"Allow","frame":{"x":100,"y":100,"width":40,"height":20},"enabled":true}]`,
wantFound: true,
wantX: 120,
wantY: 110,
},
{
name: "sole enabled button fallback",
dump: `[{"type":"StaticText","AXLabel":"Paste?","frame":{"x":0,"y":0,"width":10,"height":10},"enabled":true},{"type":"Button","AXLabel":"OK","frame":{"x":50,"y":60,"width":100,"height":40},"enabled":true}]`,
wantFound: true,
wantX: 100,
wantY: 80,
},
{
name: "no fallback when several enabled buttons present",
dump: `[{"type":"Button","AXLabel":"A","frame":{"x":0,"y":0,"width":10,"height":10},"enabled":true},{"type":"Button","AXLabel":"B","frame":{"x":20,"y":0,"width":10,"height":10},"enabled":true}]`,
wantFound: false,
},
{
name: "reject button alone does not match",
dump: `[{"type":"Button","AXLabel":"Don’t Allow Paste","frame":{"x":0,"y":0,"width":10,"height":10},"enabled":true}]`,
wantFound: false,
},
{
name: "empty dump finds nothing",
dump: ``,
wantFound: false,
},
{
name: "malformed json finds nothing",
dump: `not json`,
wantFound: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
button, found := findAllowPasteButton([]byte(test.dump))
if found != test.wantFound {
t.Fatalf("found: got %v, want %v", found, test.wantFound)
}
if !found {
return
}
if button.centerX != test.wantX || button.centerY != test.wantY {
t.Fatalf("center: got (%v,%v), want (%v,%v)", button.centerX, button.centerY, test.wantX, test.wantY)
}
})
}
}
func TestPasteLanded(t *testing.T) {
dump := `[{"type":"TextField","AXUniqueId":"NoteField","AXValue":"prefix Café ☕","frame":{"x":0,"y":0,"width":10,"height":10}}]`
tests := []struct {
name string
dump string
identifier string
expected string
want bool
}{
{name: "substring present", dump: dump, identifier: "NoteField", expected: "Café ☕", want: true},
{name: "value not yet landed", dump: `[{"type":"TextField","AXUniqueId":"NoteField","AXValue":"prefix"}]`, identifier: "NoteField", expected: "Café", want: false},
{name: "field absent", dump: dump, identifier: "MissingField", expected: "Café", want: false},
{name: "empty identifier never matches", dump: dump, identifier: "", expected: "Café", want: false},
{name: "empty expected never matches", dump: dump, identifier: "NoteField", expected: "", want: false},
{name: "malformed dump never matches", dump: `nope`, identifier: "NoteField", expected: "Café", want: false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := pasteLanded([]byte(test.dump), test.identifier, test.expected); got != test.want {
t.Fatalf("got %v, want %v", got, test.want)
}
})
}
}
// fakeRunner records side effects and serves scripted describe-all dumps.
type fakeRunner struct {
pasteboard string
hidStreams [][]transport.HIDEvent
dumps [][]byte
dumpIndex int
setError error
sendError error
describeErr error
sleepCount int
}
func (f *fakeRunner) setPasteboard(ctx context.Context, text string) error {
if f.setError != nil {
return f.setError
}
f.pasteboard = text
return nil
}
func (f *fakeRunner) sendHID(ctx context.Context, events ...transport.HIDEvent) error {
if f.sendError != nil {
return f.sendError
}
f.hidStreams = append(f.hidStreams, events)
return nil
}
func (f *fakeRunner) describeAll(ctx context.Context) ([]byte, error) {
if f.describeErr != nil {
return nil, f.describeErr
}
if f.dumpIndex >= len(f.dumps) {
return f.dumps[len(f.dumps)-1], nil
}
dump := f.dumps[f.dumpIndex]
f.dumpIndex++
return dump, nil
}
func (f *fakeRunner) sleep(ctx context.Context, duration time.Duration) error {
f.sleepCount++
return ctx.Err()
}
func TestInputTextFastPathUsesHardwareKeyboard(t *testing.T) {
fake := &fakeRunner{}
if err := inputText(context.Background(), fake, "hi", fieldTarget{}); err != nil {
t.Fatalf("inputText: %v", err)
}
if fake.pasteboard != "" {
t.Fatalf("fast path must not touch pasteboard, got %q", fake.pasteboard)
}
if len(fake.hidStreams) != 1 {
t.Fatalf("fast path must send one HID stream, got %d", len(fake.hidStreams))
}
// One stream: clear the field, then type the runes.
want := append(clearFieldEvents(),
keyPressEvents([]KeyPress{{Usage: usageA + ('h' - 'a')}, {Usage: usageA + ('i' - 'a')}})...)
eventsEqual(t, fake.hidStreams[0], want)
}
func TestInputTextPasteLandsImmediately(t *testing.T) {
landed := `[{"type":"TextField","AXUniqueId":"F","AXValue":"Café"}]`
fake := &fakeRunner{dumps: [][]byte{[]byte(landed)}}
field := fieldTarget{identifier: "F", centerX: 100, centerY: 200}
if err := inputText(context.Background(), fake, "Café", field); err != nil {
t.Fatalf("inputText: %v", err)
}
if fake.pasteboard != "Café" {
t.Fatalf("pasteboard: got %q", fake.pasteboard)
}
// Streams: clear field, then paste chord.
if len(fake.hidStreams) != 2 {
t.Fatalf("expected clear then paste chord, got %d streams", len(fake.hidStreams))
}
eventsEqual(t, fake.hidStreams[0], clearFieldEvents())
eventsEqual(t, fake.hidStreams[1], pasteChordEvents())
}
func TestInputTextPasteDismissesDialogThenLands(t *testing.T) {
dialog := string(loadDialogDump(t))
landed := `[{"type":"TextField","AXUniqueId":"TxnNoteField","AXValue":"Café ☕ 😀"}]`
// First check sees the dialog (which swallowed the initial chord); the
// check after the retried chord sees the landed value.
fake := &fakeRunner{dumps: [][]byte{[]byte(dialog), []byte(landed)}}
field := fieldTarget{identifier: "TxnNoteField", centerX: 195, centerY: 222}
if err := inputText(context.Background(), fake, "Café ☕ 😀", field); err != nil {
t.Fatalf("inputText: %v", err)
}
// Streams: clear field, paste chord, tap allow, refocus field, paste chord again.
if len(fake.hidStreams) != 5 {
t.Fatalf("expected 5 HID streams, got %d", len(fake.hidStreams))
}
eventsEqual(t, fake.hidStreams[0], clearFieldEvents())
eventsEqual(t, fake.hidStreams[1], pasteChordEvents())
eventsEqual(t, fake.hidStreams[2], tapEvents(280, 465))
eventsEqual(t, fake.hidStreams[3], tapEvents(195, 222))
eventsEqual(t, fake.hidStreams[4], pasteChordEvents())
}
func TestInputTextPasteFailsAfterAllAttempts(t *testing.T) {
stuck := `[{"type":"TextField","AXUniqueId":"F","AXValue":""}]`
fake := &fakeRunner{dumps: [][]byte{[]byte(stuck)}}
field := fieldTarget{identifier: "F", centerX: 1, centerY: 2}
err := inputText(context.Background(), fake, "😀", field)
if err == nil {
t.Fatal("expected error after exhausting the verify budget")
}
// Streams: clear field, then exactly one paste chord (no dialog, so it is
// not re-sent: re-sending on a slow render would paste twice).
if len(fake.hidStreams) != 2 {
t.Fatalf("expected clear then one paste chord, got %d streams", len(fake.hidStreams))
}
eventsEqual(t, fake.hidStreams[0], clearFieldEvents())
wantPolls := int(pasteVerifyTimeout / pastePoll)
if fake.sleepCount != wantPolls {
t.Fatalf("expected %d verify polls, got %d", wantPolls, fake.sleepCount)
}
}
func TestInputTextPasteLandsAfterBridgeBlackout(t *testing.T) {
// The field is absent (bridge blacked out by the dialog) for several
// polls, then the value lands. The loop must keep waiting through the
// blackout instead of failing.
blacked := `[{"type":"Application"}]`
landed := `[{"type":"TextField","AXUniqueId":"F","AXValue":"😀"}]`
dumps := [][]byte{[]byte(blacked), []byte(blacked), []byte(blacked), []byte(landed)}
fake := &fakeRunner{dumps: dumps}
field := fieldTarget{identifier: "F", centerX: 1, centerY: 2}
if err := inputText(context.Background(), fake, "😀", field); err != nil {
t.Fatalf("inputText: %v", err)
}
// Clear field, one chord, no dialog seen, success once the value appears.
if len(fake.hidStreams) != 2 {
t.Fatalf("expected clear then one paste chord, got %d streams", len(fake.hidStreams))
}
}
func TestInputTextPasteUnverifiableFieldSingleChord(t *testing.T) {
// No field identifier resolved: one chord, no dialog, success after the
// settle without endless retries.
noField := `[{"type":"StaticText","AXLabel":"whatever"}]`
fake := &fakeRunner{dumps: [][]byte{[]byte(noField)}}
if err := inputText(context.Background(), fake, "😀", fieldTarget{}); err != nil {
t.Fatalf("inputText: %v", err)
}
// Clear field, then one paste chord.
if len(fake.hidStreams) != 2 {
t.Fatalf("expected clear then one paste chord, got %d streams", len(fake.hidStreams))
}
}
func TestEraseTextLargeCountClearsAtomically(t *testing.T) {
fake := &fakeRunner{}
if err := eraseText(context.Background(), fake, 40); err != nil {
t.Fatalf("eraseText: %v", err)
}
if len(fake.hidStreams) != 1 {
t.Fatalf("expected one stream, got %d", len(fake.hidStreams))
}
eventsEqual(t, fake.hidStreams[0], clearFieldEvents())
}
func TestEraseTextSendsBackspaces(t *testing.T) {
fake := &fakeRunner{}
if err := eraseText(context.Background(), fake, 3); err != nil {
t.Fatalf("eraseText: %v", err)
}
if len(fake.hidStreams) != 1 {
t.Fatalf("expected one stream, got %d", len(fake.hidStreams))
}
want := keyPressEvents(backspaces(3))
eventsEqual(t, fake.hidStreams[0], want)
}
func TestEraseTextZeroIsNoOp(t *testing.T) {
fake := &fakeRunner{}
if err := eraseText(context.Background(), fake, 0); err != nil {
t.Fatalf("eraseText: %v", err)
}
if len(fake.hidStreams) != 0 {
t.Fatalf("zero count must send nothing, got %d streams", len(fake.hidStreams))
}
}
func TestUsesPasteboard(t *testing.T) {
cases := []struct {
text string
want bool
}{
{"", false},
{"a", false},
{"abc", false},
{"a long but fully mappable ascii string", false},
{"-1", false},
{"%s%n", false},
{"😀", true},
{"Café", true},
}
for _, c := range cases {
if got := usesPasteboard(c.text); got != c.want {
t.Errorf("usesPasteboard(%q) = %v, want %v", c.text, got, c.want)
}
}
}
func TestPasteLandedMaskedSecureField(t *testing.T) {
cases := []struct {
name string
dump string
want bool
}{
{name: "all bullets counts as landed", dump: `[{"type":"TextField","AXUniqueId":"PW","AXValue":"•••••"}]`, want: true},
{name: "empty secure field not landed", dump: `[{"type":"TextField","AXUniqueId":"PW","AXValue":""}]`, want: false},
{name: "mixed bullets and text not masked", dump: `[{"type":"TextField","AXUniqueId":"PW","AXValue":"••a"}]`, want: false},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if got := pasteLanded([]byte(c.dump), "PW", "secret9"); got != c.want {
t.Fatalf("got %v, want %v", got, c.want)
}
})
}
}