fix(ios): read a StaticText's label as its text

AXValue was the only source for text, but a StaticText carries its
string in AXLabel, so nothing on screen had .text on ios: a spec reading
it saw everything on android and nothing here.
This commit is contained in:
pj committed 2026-08-14 11:04:05 +05:30
1 parent 2f782608c5
commit 3ce9455928
3 files changed
+108

No files matched your search

@@ -129,6 +129,9 @@ func mapElement(element *rawElement) (treeNode, bool) {
attributes["hintText"] = label
} else if label != "" {
attributes["accessibilityText"] = label
if labelIsDisplayedText(element.Type) {
attributes["text"] = label
}
}
enabled := element.Enabled
@@ -150,6 +153,22 @@ func isEditable(elementType string) bool {
return elementType == "TextArea" || elementType == "TextField"
}
// labelIsDisplayedText reports whether an element type's AXLabel is the string
// drawn on screen rather than an accessibility annotation about it. Only
// StaticText qualifies: a text element's label IS what it renders, so it
// belongs in `text`, matching a TextView on Android and a text node on web.
//
// Buttons and images are deliberately excluded even though a titled button's
// label is also its visible title. The snapshot cannot tell that button apart
// from an icon-only one whose label exists purely for VoiceOver, nor from a
// container whose label is a comma-joined reading of its children ("CH,
// Checking, $0.00, 0 transactions"). Inventing `text` for those would put
// strings in `text` that no user can read, and would diverge from Android,
// which leaves `text` empty and reports a contentDescription as `description`.
func labelIsDisplayedText(elementType string) bool {
return elementType == "StaticText"
}
func stringValue(pointer *string) string {
if pointer == nil {
return ""
@@ -132,6 +132,79 @@ func TestNonEmptyValueMapsToText(t *testing.T) {
}
}
func TestStaticTextLabelMapsToText(t *testing.T) {
dump := `[{"type":"StaticText","frame":{"x":0,"y":0,"width":50,"height":18},
"AXLabel":"$239.00","AXValue":null,"enabled":true}]`
element := parseSingle(t, dump)
// A StaticText renders its label, so a spec reading .text must see it.
if element.Text != "$239.00" {
t.Fatalf("text = %q, want $239.00", element.Text)
}
// The raw label stays available: desc:/label:/content-desc: selectors and
// the settle hash read it on the iOS path.
if element.Description != "$239.00" {
t.Fatalf("description = %q, want $239.00", element.Description)
}
}
func TestNonTextLabelStaysDescriptionOnly(t *testing.T) {
// An icon-only button's label is a VoiceOver annotation, and a container's
// is a comma-joined reading of its children. Neither is on screen, so
// neither may become text; Android reports both as description too.
cases := []struct{ elementType, label string }{
{"Button", "Log out"},
{"Image", "Avatar"},
{"Other", "CH, Checking, $0.00, 0 transactions"},
}
for _, testCase := range cases {
dump := `[{"type":"` + testCase.elementType + `","frame":{"x":0,"y":0,"width":10,"height":10},
"AXLabel":"` + testCase.label + `","AXValue":null,"enabled":true}]`
element := parseSingle(t, dump)
if element.Text != "" {
t.Errorf("%s text = %q, want empty", testCase.elementType, element.Text)
}
if element.Description != testCase.label {
t.Errorf("%s description = %q, want %q", testCase.elementType, element.Description, testCase.label)
}
}
}
// Mirrors the folio Home screen as the companion actually dumps it: each
// AccountCard is a Button carrying a merged VoiceOver label, with the name and
// balance as StaticText siblings that the flat tree re-parents by containment.
// Reading a card's balance is what folio's totalBalance extractor does, and it
// read nothing on iOS until StaticText labels became text.
func TestGoldenHomeCardBalancesAreReadableAsText(t *testing.T) {
tree := mapAndParse(t, readDump(t, "home-cards-describe.json"), 402, 874)
cards := tree.FindAllNodes("id:HomeScreen > id:AccountCard")
if len(cards) != 2 {
t.Fatalf("account cards = %d, want 2", len(cards))
}
want := []struct{ name, balance string }{{"Checking", "$12.34"}, {"Savings", "$500.00"}}
for i, card := range cards {
balance := card.Find("id:AccountBalance")
if balance == nil {
t.Fatalf("card %d: id:AccountBalance did not resolve", i)
}
if balance.Text != want[i].balance {
t.Errorf("card %d: balance text = %q, want %q", i, balance.Text, want[i].balance)
}
name := card.Find("id:AccountName")
if name == nil {
t.Fatalf("card %d: id:AccountName did not resolve", i)
}
if name.Text != want[i].name {
t.Errorf("card %d: name text = %q, want %q", i, name.Text, want[i].name)
}
// The card's own merged label is not visible text; the web fallback in
// the folio spec parses cardText and must not see a VoiceOver reading.
if card.Element.Text != "" {
t.Errorf("card %d: card text = %q, want empty", i, card.Element.Text)
}
}
}
func TestEditableAndClickableFlags(t *testing.T) {
cases := []struct {
elementType string
@@ -0,0 +1,16 @@
[
{"type":"Application","frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":"Folio","AXValue":null,"AXUniqueId":null},
{"type":"Other","frame":{"x":0,"y":62,"width":402,"height":778},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":"HomeScreen"},
{"type":"StaticText","frame":{"x":20,"y":76,"width":97,"height":24},"enabled":true,"AXLabel":"Accounts","AXValue":null,"AXUniqueId":null},
{"type":"Button","frame":{"x":340,"y":71,"width":48,"height":48},"enabled":true,"AXLabel":"Log out","AXValue":null,"AXUniqueId":"LogoutButton"},
{"type":"Button","frame":{"x":20,"y":130,"width":362,"height":72},"enabled":true,"AXLabel":"CH, Checking, $12.34, 0 transactions","AXValue":null,"AXUniqueId":"AccountCard"},
{"type":"StaticText","frame":{"x":90,"y":150,"width":74,"height":18},"enabled":true,"AXLabel":"Checking","AXValue":null,"AXUniqueId":"AccountName"},
{"type":"StaticText","frame":{"x":313,"y":157,"width":53,"height":18},"enabled":true,"AXLabel":"$12.34","AXValue":null,"AXUniqueId":"AccountBalance"},
{"type":"Button","frame":{"x":20,"y":210,"width":362,"height":72},"enabled":true,"AXLabel":"SA, Savings, $500.00, 2 transactions","AXValue":null,"AXUniqueId":"AccountCard"},
{"type":"StaticText","frame":{"x":90,"y":230,"width":66,"height":18},"enabled":true,"AXLabel":"Savings","AXValue":null,"AXUniqueId":"AccountName"},
{"type":"StaticText","frame":{"x":306,"y":237,"width":60,"height":18},"enabled":true,"AXLabel":"$500.00","AXValue":null,"AXUniqueId":"AccountBalance"},
{"type":"StaticText","frame":{"x":20,"y":716,"width":106,"height":14},"enabled":true,"AXLabel":"TOTAL BALANCE","AXValue":null,"AXUniqueId":null},
{"type":"StaticText","frame":{"x":20,"y":731,"width":85,"height":33},"enabled":true,"AXLabel":"$512.34","AXValue":null,"AXUniqueId":null},
{"type":"Button","frame":{"x":20,"y":777,"width":362,"height":48},"enabled":true,"AXLabel":"+ Add account","AXValue":null,"AXUniqueId":"AddAccountButton"},
{"type":"StaticText","frame":{"x":141,"y":792,"width":121,"height":18},"enabled":true,"AXLabel":"+ Add account","AXValue":null,"AXUniqueId":null}
]