From 6244a056526999d9b2015d85302e2b90954a7096 Mon Sep 17 00:00:00 2001 From: PJ Date: Sun, 7 Jun 2026 17:32:38 +0530 Subject: [PATCH] fix(ioscompanion): keep mappable text on one HID stream and verify unicode clears --- internal/driver/ioscompanion/driver.go | 70 +++++++++++++++++++-- internal/driver/ioscompanion/driver_test.go | 59 +++++++++++++++++ 2 files changed, 124 insertions(+), 5 deletions(-) diff --git a/internal/driver/ioscompanion/driver.go b/internal/driver/ioscompanion/driver.go index d88fbda..2373df8 100644 --- a/internal/driver/ioscompanion/driver.go +++ b/internal/driver/ioscompanion/driver.go @@ -638,15 +638,24 @@ func (d *Driver) resolveSelectorCenter(ctx context.Context, selector string) (in } func (d *Driver) InputText(ctx context.Context, text string) error { - // Hybrid path: clear the focused field atomically with the legacy HID - // select-all chord, then let the runner type the text natively. Typing - // covers unicode without the pasteboard and its permission dialog; the - // chord clears regardless of where a tap left the cursor. - if d.runnerTyper() != nil { + // Hybrid path. Mappable text rides one HID stream: select-all chord plus + // keystrokes, atomic and strictly ordered on a single channel. Unicode + // (which HID cannot express) is typed natively by the runner after the + // chord; chord and typing ride different channels with no ordering + // guarantee between them, so the clear is verified through a snapshot + // before the first keystroke goes out. + if typer := d.runnerTyper(); typer != nil { + if !usesPasteboard(text) { + events := append(clearFieldEvents(), keyPressEvents(typeStringPresses(text))...) + return d.withRecovery(ctx, func() error { + return d.companion.SendHID(ctx, events...) + }) + } return d.withRecovery(ctx, func() error { if err := d.companion.SendHID(ctx, clearFieldEvents()...); err != nil { return fmt.Errorf("clear field: %w", err) } + d.waitFieldCleared(ctx) return d.runnerTyper().TypeText(ctx, text, false) }) } @@ -667,6 +676,57 @@ func (d *Driver) InputText(ctx context.Context, text string) error { return inputText(ctx, d.makeRunner(), text, field) } +// fieldClearedWaitCap and fieldClearedPoll bound the verify-cleared loop +// between the HID clear chord and the runner's native typing. +const fieldClearedWaitCap = 1200 * time.Millisecond +const fieldClearedPoll = 150 * time.Millisecond + +// waitFieldCleared polls the focused field (the editable element under the +// last tap) until its value reads empty, so the clear chord has demonstrably +// landed before typing starts on the other channel. Best effort: when the +// field cannot be resolved or the cap elapses, typing proceeds anyway. +func (d *Driver) waitFieldCleared(ctx context.Context) { + d.mu.Lock() + tap := d.lastTap + d.mu.Unlock() + if !tap.set { + return + } + deadline := time.Now().Add(fieldClearedWaitCap) + for time.Now().Before(deadline) { + dump, err := d.describeAllRaw(ctx) + if err != nil { + return + } + cleared := true + for _, element := range decodeDump(dump) { + if !isEditable(element.Type) { + continue + } + frame := element.Frame + if !finite(frame.X) || !finite(frame.Y) || !finite(frame.Width) || !finite(frame.Height) { + continue + } + if tap.x < frame.X || tap.x > frame.X+frame.Width || + tap.y < frame.Y || tap.y > frame.Y+frame.Height { + continue + } + if value := stringValue(element.AXValue); value != "" && value != emptyFieldValueSentinel { + cleared = false + } + break + } + if cleared { + return + } + select { + case <-ctx.Done(): + return + case <-time.After(fieldClearedPoll): + } + } +} + // resolveInputField finds the editable element under the last tap so the // pasteboard fallback can confirm the paste landed and refocus after dismissing // the permission dialog. The runner always taps a field before typing, so diff --git a/internal/driver/ioscompanion/driver_test.go b/internal/driver/ioscompanion/driver_test.go index ca38e6b..c1b290a 100644 --- a/internal/driver/ioscompanion/driver_test.go +++ b/internal/driver/ioscompanion/driver_test.go @@ -749,3 +749,62 @@ func TestBindTestRunPortRejectsMissingPlaceholder(t *testing.T) { t.Fatal("expected an error for a configuration without the placeholder") } } + +func TestHybridMappableTextRidesOneHIDStream(t *testing.T) { + legacy := &fakeCompanion{accessibilityJSON: "[]"} + runner := &fakeRunnerCompanion{} + d := newHybridTestDriver(legacy, runner) + + if err := d.InputText(context.Background(), "Travel 42"); err != nil { + t.Fatal(err) + } + if len(legacy.recorded()) != 1 || legacy.recorded()[0] != "hid" { + t.Fatalf("legacy calls = %v, want one combined chord-and-keystrokes stream", legacy.recorded()) + } + if len(runner.typed) != 0 { + t.Fatalf("typed = %+v, want no native typing for mappable text", runner.typed) + } +} + +func TestHybridUnicodeWaitsForClearedFieldBeforeTyping(t *testing.T) { + legacy := &fakeCompanion{accessibilityJSON: "[]"} + runner := &fakeRunnerCompanion{} + // The focused field still shows old content on the first read and is + // empty on the second; typing must come after the cleared read. + first := `[{"type":"TextField","AXUniqueId":"F","AXValue":"old","frame":{"x":0,"y":0,"width":100,"height":40},"enabled":true}]` + second := `[{"type":"TextField","AXUniqueId":"F","AXValue":"","frame":{"x":0,"y":0,"width":100,"height":40},"enabled":true}]` + reads := 0 + d := newHybridTestDriver(legacy, runner) + d.mu.Lock() + d.lastTap.x, d.lastTap.y, d.lastTap.set = 50, 20, true + d.mu.Unlock() + // Swap the dump after the first read through a wrapper companion. + wrapped := &sequencedDumpCompanion{fakeRunnerCompanion: runner, dumps: []string{first, second}, reads: &reads} + d.runnerClient = wrapped + + if err := d.InputText(context.Background(), "héllo 🌟"); err != nil { + t.Fatal(err) + } + if reads < 2 { + t.Fatalf("reads = %d, want at least 2 (poll until cleared)", reads) + } + if len(wrapped.typed) != 1 || wrapped.typed[0].text != "héllo 🌟" || wrapped.typed[0].replace { + t.Fatalf("typed = %+v", wrapped.typed) + } +} + +// sequencedDumpCompanion serves scripted dumps in order, repeating the last. +type sequencedDumpCompanion struct { + *fakeRunnerCompanion + dumps []string + reads *int +} + +func (s *sequencedDumpCompanion) AccessibilityInfo(context.Context) (string, error) { + index := *s.reads + if index >= len(s.dumps) { + index = len(s.dumps) - 1 + } + *s.reads++ + return s.dumps[index], nil +}