diff --git a/internal/driver/ioscompanion/hierarchymap.go b/internal/driver/ioscompanion/hierarchymap.go index 8b0db9d..3d5f0b5 100644 --- a/internal/driver/ioscompanion/hierarchymap.go +++ b/internal/driver/ioscompanion/hierarchymap.go @@ -129,6 +129,9 @@ func mapElement(element *rawElement) (treeNode, bool) { attributes["hintText"] = label } else if label != "" { attributes["accessibilityText"] = label + if labelIsDisplayedText(element.Type) { + attributes["text"] = label + } } enabled := element.Enabled @@ -150,6 +153,22 @@ func isEditable(elementType string) bool { return elementType == "TextArea" || elementType == "TextField" } +// labelIsDisplayedText reports whether an element type's AXLabel is the string +// drawn on screen rather than an accessibility annotation about it. Only +// StaticText qualifies: a text element's label IS what it renders, so it +// belongs in `text`, matching a TextView on Android and a text node on web. +// +// Buttons and images are deliberately excluded even though a titled button's +// label is also its visible title. The snapshot cannot tell that button apart +// from an icon-only one whose label exists purely for VoiceOver, nor from a +// container whose label is a comma-joined reading of its children ("CH, +// Checking, $0.00, 0 transactions"). Inventing `text` for those would put +// strings in `text` that no user can read, and would diverge from Android, +// which leaves `text` empty and reports a contentDescription as `description`. +func labelIsDisplayedText(elementType string) bool { + return elementType == "StaticText" +} + func stringValue(pointer *string) string { if pointer == nil { return "" diff --git a/internal/driver/ioscompanion/hierarchymap_test.go b/internal/driver/ioscompanion/hierarchymap_test.go index 7c488c0..1c7a938 100644 --- a/internal/driver/ioscompanion/hierarchymap_test.go +++ b/internal/driver/ioscompanion/hierarchymap_test.go @@ -132,6 +132,79 @@ func TestNonEmptyValueMapsToText(t *testing.T) { } } +func TestStaticTextLabelMapsToText(t *testing.T) { + dump := `[{"type":"StaticText","frame":{"x":0,"y":0,"width":50,"height":18}, + "AXLabel":"$239.00","AXValue":null,"enabled":true}]` + element := parseSingle(t, dump) + // A StaticText renders its label, so a spec reading .text must see it. + if element.Text != "$239.00" { + t.Fatalf("text = %q, want $239.00", element.Text) + } + // The raw label stays available: desc:/label:/content-desc: selectors and + // the settle hash read it on the iOS path. + if element.Description != "$239.00" { + t.Fatalf("description = %q, want $239.00", element.Description) + } +} + +func TestNonTextLabelStaysDescriptionOnly(t *testing.T) { + // An icon-only button's label is a VoiceOver annotation, and a container's + // is a comma-joined reading of its children. Neither is on screen, so + // neither may become text; Android reports both as description too. + cases := []struct{ elementType, label string }{ + {"Button", "Log out"}, + {"Image", "Avatar"}, + {"Other", "CH, Checking, $0.00, 0 transactions"}, + } + for _, testCase := range cases { + dump := `[{"type":"` + testCase.elementType + `","frame":{"x":0,"y":0,"width":10,"height":10}, + "AXLabel":"` + testCase.label + `","AXValue":null,"enabled":true}]` + element := parseSingle(t, dump) + if element.Text != "" { + t.Errorf("%s text = %q, want empty", testCase.elementType, element.Text) + } + if element.Description != testCase.label { + t.Errorf("%s description = %q, want %q", testCase.elementType, element.Description, testCase.label) + } + } +} + +// Mirrors the folio Home screen as the companion actually dumps it: each +// AccountCard is a Button carrying a merged VoiceOver label, with the name and +// balance as StaticText siblings that the flat tree re-parents by containment. +// Reading a card's balance is what folio's totalBalance extractor does, and it +// read nothing on iOS until StaticText labels became text. +func TestGoldenHomeCardBalancesAreReadableAsText(t *testing.T) { + tree := mapAndParse(t, readDump(t, "home-cards-describe.json"), 402, 874) + + cards := tree.FindAllNodes("id:HomeScreen > id:AccountCard") + if len(cards) != 2 { + t.Fatalf("account cards = %d, want 2", len(cards)) + } + want := []struct{ name, balance string }{{"Checking", "$12.34"}, {"Savings", "$500.00"}} + for i, card := range cards { + balance := card.Find("id:AccountBalance") + if balance == nil { + t.Fatalf("card %d: id:AccountBalance did not resolve", i) + } + if balance.Text != want[i].balance { + t.Errorf("card %d: balance text = %q, want %q", i, balance.Text, want[i].balance) + } + name := card.Find("id:AccountName") + if name == nil { + t.Fatalf("card %d: id:AccountName did not resolve", i) + } + if name.Text != want[i].name { + t.Errorf("card %d: name text = %q, want %q", i, name.Text, want[i].name) + } + // The card's own merged label is not visible text; the web fallback in + // the folio spec parses cardText and must not see a VoiceOver reading. + if card.Element.Text != "" { + t.Errorf("card %d: card text = %q, want empty", i, card.Element.Text) + } + } +} + func TestEditableAndClickableFlags(t *testing.T) { cases := []struct { elementType string diff --git a/internal/driver/ioscompanion/testdata/home-cards-describe.json b/internal/driver/ioscompanion/testdata/home-cards-describe.json new file mode 100644 index 0000000..ffc342d --- /dev/null +++ b/internal/driver/ioscompanion/testdata/home-cards-describe.json @@ -0,0 +1,16 @@ +[ + {"type":"Application","frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":"Folio","AXValue":null,"AXUniqueId":null}, + {"type":"Other","frame":{"x":0,"y":62,"width":402,"height":778},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":"HomeScreen"}, + {"type":"StaticText","frame":{"x":20,"y":76,"width":97,"height":24},"enabled":true,"AXLabel":"Accounts","AXValue":null,"AXUniqueId":null}, + {"type":"Button","frame":{"x":340,"y":71,"width":48,"height":48},"enabled":true,"AXLabel":"Log out","AXValue":null,"AXUniqueId":"LogoutButton"}, + {"type":"Button","frame":{"x":20,"y":130,"width":362,"height":72},"enabled":true,"AXLabel":"CH, Checking, $12.34, 0 transactions","AXValue":null,"AXUniqueId":"AccountCard"}, + {"type":"StaticText","frame":{"x":90,"y":150,"width":74,"height":18},"enabled":true,"AXLabel":"Checking","AXValue":null,"AXUniqueId":"AccountName"}, + {"type":"StaticText","frame":{"x":313,"y":157,"width":53,"height":18},"enabled":true,"AXLabel":"$12.34","AXValue":null,"AXUniqueId":"AccountBalance"}, + {"type":"Button","frame":{"x":20,"y":210,"width":362,"height":72},"enabled":true,"AXLabel":"SA, Savings, $500.00, 2 transactions","AXValue":null,"AXUniqueId":"AccountCard"}, + {"type":"StaticText","frame":{"x":90,"y":230,"width":66,"height":18},"enabled":true,"AXLabel":"Savings","AXValue":null,"AXUniqueId":"AccountName"}, + {"type":"StaticText","frame":{"x":306,"y":237,"width":60,"height":18},"enabled":true,"AXLabel":"$500.00","AXValue":null,"AXUniqueId":"AccountBalance"}, + {"type":"StaticText","frame":{"x":20,"y":716,"width":106,"height":14},"enabled":true,"AXLabel":"TOTAL BALANCE","AXValue":null,"AXUniqueId":null}, + {"type":"StaticText","frame":{"x":20,"y":731,"width":85,"height":33},"enabled":true,"AXLabel":"$512.34","AXValue":null,"AXUniqueId":null}, + {"type":"Button","frame":{"x":20,"y":777,"width":362,"height":48},"enabled":true,"AXLabel":"+ Add account","AXValue":null,"AXUniqueId":"AddAccountButton"}, + {"type":"StaticText","frame":{"x":141,"y":792,"width":121,"height":18},"enabled":true,"AXLabel":"+ Add account","AXValue":null,"AXUniqueId":null} +]