fix(ioscompanion): keep mappable text on one HID stream and verify unicode clears

This commit is contained in:
pj committed 2026-06-07 17:32:38 +05:30
1 parent dc3a407069
commit 6244a05652
2 files changed
+124 -5

No files matched your search

+65 -5
View File
@@ -638,15 +638,24 @@ func (d *Driver) resolveSelectorCenter(ctx context.Context, selector string) (in
}
func (d *Driver) InputText(ctx context.Context, text string) error {
// Hybrid path: clear the focused field atomically with the legacy HID
// select-all chord, then let the runner type the text natively. Typing
// covers unicode without the pasteboard and its permission dialog; the
// chord clears regardless of where a tap left the cursor.
if d.runnerTyper() != nil {
// Hybrid path. Mappable text rides one HID stream: select-all chord plus
// keystrokes, atomic and strictly ordered on a single channel. Unicode
// (which HID cannot express) is typed natively by the runner after the
// chord; chord and typing ride different channels with no ordering
// guarantee between them, so the clear is verified through a snapshot
// before the first keystroke goes out.
if typer := d.runnerTyper(); typer != nil {
if !usesPasteboard(text) {
events := append(clearFieldEvents(), keyPressEvents(typeStringPresses(text))...)
return d.withRecovery(ctx, func() error {
return d.companion.SendHID(ctx, events...)
})
}
return d.withRecovery(ctx, func() error {
if err := d.companion.SendHID(ctx, clearFieldEvents()...); err != nil {
return fmt.Errorf("clear field: %w", err)
}
d.waitFieldCleared(ctx)
return d.runnerTyper().TypeText(ctx, text, false)
})
}
@@ -667,6 +676,57 @@ func (d *Driver) InputText(ctx context.Context, text string) error {
return inputText(ctx, d.makeRunner(), text, field)
}
// fieldClearedWaitCap and fieldClearedPoll bound the verify-cleared loop
// between the HID clear chord and the runner's native typing.
const fieldClearedWaitCap = 1200 * time.Millisecond
const fieldClearedPoll = 150 * time.Millisecond
// waitFieldCleared polls the focused field (the editable element under the
// last tap) until its value reads empty, so the clear chord has demonstrably
// landed before typing starts on the other channel. Best effort: when the
// field cannot be resolved or the cap elapses, typing proceeds anyway.
func (d *Driver) waitFieldCleared(ctx context.Context) {
d.mu.Lock()
tap := d.lastTap
d.mu.Unlock()
if !tap.set {
return
}
deadline := time.Now().Add(fieldClearedWaitCap)
for time.Now().Before(deadline) {
dump, err := d.describeAllRaw(ctx)
if err != nil {
return
}
cleared := true
for _, element := range decodeDump(dump) {
if !isEditable(element.Type) {
continue
}
frame := element.Frame
if !finite(frame.X) || !finite(frame.Y) || !finite(frame.Width) || !finite(frame.Height) {
continue
}
if tap.x < frame.X || tap.x > frame.X+frame.Width ||
tap.y < frame.Y || tap.y > frame.Y+frame.Height {
continue
}
if value := stringValue(element.AXValue); value != "" && value != emptyFieldValueSentinel {
cleared = false
}
break
}
if cleared {
return
}
select {
case <-ctx.Done():
return
case <-time.After(fieldClearedPoll):
}
}
}
// resolveInputField finds the editable element under the last tap so the
// pasteboard fallback can confirm the paste landed and refocus after dismissing
// the permission dialog. The runner always taps a field before typing, so
@@ -749,3 +749,62 @@ func TestBindTestRunPortRejectsMissingPlaceholder(t *testing.T) {
t.Fatal("expected an error for a configuration without the placeholder")
}
}
func TestHybridMappableTextRidesOneHIDStream(t *testing.T) {
legacy := &fakeCompanion{accessibilityJSON: "[]"}
runner := &fakeRunnerCompanion{}
d := newHybridTestDriver(legacy, runner)
if err := d.InputText(context.Background(), "Travel 42"); err != nil {
t.Fatal(err)
}
if len(legacy.recorded()) != 1 || legacy.recorded()[0] != "hid" {
t.Fatalf("legacy calls = %v, want one combined chord-and-keystrokes stream", legacy.recorded())
}
if len(runner.typed) != 0 {
t.Fatalf("typed = %+v, want no native typing for mappable text", runner.typed)
}
}
func TestHybridUnicodeWaitsForClearedFieldBeforeTyping(t *testing.T) {
legacy := &fakeCompanion{accessibilityJSON: "[]"}
runner := &fakeRunnerCompanion{}
// The focused field still shows old content on the first read and is
// empty on the second; typing must come after the cleared read.
first := `[{"type":"TextField","AXUniqueId":"F","AXValue":"old","frame":{"x":0,"y":0,"width":100,"height":40},"enabled":true}]`
second := `[{"type":"TextField","AXUniqueId":"F","AXValue":"","frame":{"x":0,"y":0,"width":100,"height":40},"enabled":true}]`
reads := 0
d := newHybridTestDriver(legacy, runner)
d.mu.Lock()
d.lastTap.x, d.lastTap.y, d.lastTap.set = 50, 20, true
d.mu.Unlock()
// Swap the dump after the first read through a wrapper companion.
wrapped := &sequencedDumpCompanion{fakeRunnerCompanion: runner, dumps: []string{first, second}, reads: &reads}
d.runnerClient = wrapped
if err := d.InputText(context.Background(), "héllo 🌟"); err != nil {
t.Fatal(err)
}
if reads < 2 {
t.Fatalf("reads = %d, want at least 2 (poll until cleared)", reads)
}
if len(wrapped.typed) != 1 || wrapped.typed[0].text != "héllo 🌟" || wrapped.typed[0].replace {
t.Fatalf("typed = %+v", wrapped.typed)
}
}
// sequencedDumpCompanion serves scripted dumps in order, repeating the last.
type sequencedDumpCompanion struct {
*fakeRunnerCompanion
dumps []string
reads *int
}
func (s *sequencedDumpCompanion) AccessibilityInfo(context.Context) (string, error) {
index := *s.reads
if index >= len(s.dumps) {
index = len(s.dumps) - 1
}
*s.reads++
return s.dumps[index], nil
}