diff --git a/examples/folio/composeApp/src/commonMain/kotlin/app/folio/feature/home/HomeScreen.kt b/examples/folio/composeApp/src/commonMain/kotlin/app/folio/feature/home/HomeScreen.kt index f6345c9..ce18006 100644 --- a/examples/folio/composeApp/src/commonMain/kotlin/app/folio/feature/home/HomeScreen.kt +++ b/examples/folio/composeApp/src/commonMain/kotlin/app/folio/feature/home/HomeScreen.kt @@ -123,7 +123,7 @@ private fun AccountCard( .clip(RoundedCornerShape(RadiusLg)) .background(t.surface) .border(1.dp, t.border, RoundedCornerShape(RadiusLg)) - .semantics(mergeDescendants = true) { contentDescription = "account:$id:$balance" } + .semantics { contentDescription = "account_card:$id" } .clickable(role = Role.Button, onClick = onClick) .padding(16.dp), verticalAlignment = Alignment.CenterVertically, @@ -148,6 +148,11 @@ private fun AccountCard( ) Text(txnLabel, style = Type.caption, color = t.textMuted) } - Text(formatCents(balance), style = Type.bodyStrong, color = t.text) + Text( + formatCents(balance), + style = Type.bodyStrong, + color = t.text, + modifier = Modifier.semantics { contentDescription = "account_balance" }, + ) } } diff --git a/examples/folio/composeApp/src/commonMain/kotlin/app/folio/feature/ledger/LedgerScreen.kt b/examples/folio/composeApp/src/commonMain/kotlin/app/folio/feature/ledger/LedgerScreen.kt index 6412e21..f7f60df 100644 --- a/examples/folio/composeApp/src/commonMain/kotlin/app/folio/feature/ledger/LedgerScreen.kt +++ b/examples/folio/composeApp/src/commonMain/kotlin/app/folio/feature/ledger/LedgerScreen.kt @@ -75,7 +75,7 @@ fun LedgerScreen(accountId: String) { val balance = balanceOf(txns) Screen( - modifier = Modifier.testTag("LedgerScreen").semantics { contentDescription = "LedgerScreen" }, + modifier = Modifier.testTag("LedgerScreen").semantics { contentDescription = "LedgerScreen:$accountId" }, header = { Header( title = account.name, @@ -92,11 +92,14 @@ fun LedgerScreen(accountId: String) { ) }, ) { - Box(Modifier.size(1.dp).semantics { contentDescription = "active_account:$accountId" }) - Box(Modifier.size(1.dp).semantics { contentDescription = "ledger_balance:$balance" }) Card { Text("BALANCE", style = Type.label, color = t.textMuted) - Text(formatCents(balance), style = Type.balance, color = t.text) + Text( + formatCents(balance), + style = Type.balance, + color = t.text, + modifier = Modifier.semantics { contentDescription = "ledger_balance_display" }, + ) } Text( "ACTIVITY", @@ -138,7 +141,7 @@ private fun TxnRow(id: String, type: TxnType, amount: Long, note: String, date: modifier = Modifier .fillMaxWidth() .padding(vertical = 14.dp) - .semantics(mergeDescendants = true) { contentDescription = "ledger_row:$id:$signed" }, + .semantics { contentDescription = "ledger_row:$id" }, verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(12.dp), ) { @@ -166,6 +169,7 @@ private fun TxnRow(id: String, type: TxnType, amount: Long, note: String, date: formatCents(signed, signed = true), style = Type.bodyStrong, color = t.text, + modifier = Modifier.semantics { contentDescription = "txn_amount" }, ) } } diff --git a/examples/folio/sanderling/spec.ts b/examples/folio/sanderling/spec.ts index 8b4c5e2..f7d2af3 100644 --- a/examples/folio/sanderling/spec.ts +++ b/examples/folio/sanderling/spec.ts @@ -20,57 +20,72 @@ interface LedgerRow { signed: number; } -function parseAccount(desc: string | undefined): Account { - const parts = (desc ?? "").split(":"); - return { id: parts[1] ?? "", balance: Number(parts[2]) }; -} - -function parseLedgerRow(desc: string | undefined): LedgerRow { - const parts = (desc ?? "").split(":"); - return { id: parts[1] ?? "", signed: Number(parts[2]) }; -} - -function parseCents(desc: string | null | undefined): number { - if (!desc) return 0; - const parts = desc.split(":"); - return Number(parts[1]) || 0; +// Parses formatCents output like "$5.00", "-$1,234.56", "+$0.50" back to integer cents. +// formatCents always uses $ prefix, . as decimal, , as thousands separator. +function parseDollarCents(text: string | undefined): number { + if (!text) return 0; + const sign = text.startsWith("-") ? -1 : 1; + const digits = text.replace(/[^0-9]/g, ""); + return digits ? sign * parseInt(digits, 10) : 0; } // Route and auth state derived from screen root nodes -const loggedIn = extract(s => s.ax.find("desc:LoginScreen") == null); +const loggedIn = extract(s => s.ax.find({ accessibilityText: "LoginScreen" }) == null); const route = extract(s => { - if (s.ax.find("desc:LoginScreen")) return "login"; - if (s.ax.find("desc:HomeScreen")) return "home"; - if (s.ax.find("desc:AddAccountScreen")) return "add-account"; - if (s.ax.find("desc:LedgerScreen")) return "ledger"; - if (s.ax.find("desc:AddTransactionScreen")) return "add-transaction"; + if (s.ax.find({ accessibilityText: "LoginScreen" })) return "login"; + if (s.ax.find({ accessibilityText: "HomeScreen" })) return "home"; + if (s.ax.find({ accessibilityText: "AddAccountScreen" })) return "add-account"; + if (s.ax.find({ accessibilityText: "LedgerScreen" })) return "ledger"; + if (s.ax.find({ accessibilityText: "AddTransactionScreen" })) return "add-transaction"; return null; }); // All element lookups scoped through their screen root -const accounts = extract(s => s.ax.findAll("desc:HomeScreen > descPrefix:account:") - .map(el => parseAccount(el.desc))); -const ledgerRows = extract(s => s.ax.findAll("desc:LedgerScreen > descPrefix:ledger_row:") - .map(el => parseLedgerRow(el.desc))); +const accounts = extract(s => + s.ax.find({ accessibilityText: "HomeScreen" })?.findAll("descPrefix:account_card:") + .map(el => ({ + id: el.desc?.split(":")[1] ?? "", + balance: parseDollarCents(el.find({ accessibilityText: "account_balance" })?.text), + })) ?? []); +const ledgerRows = extract(s => + s.ax.find({ accessibilityText: "LedgerScreen" })?.findAll("descPrefix:ledger_row:") + .map(el => ({ + id: el.desc?.split(":")[1] ?? "", + signed: parseDollarCents(el.find({ accessibilityText: "txn_amount" })?.text), + })) ?? []); const ledgerBalance = extract(s => - parseCents(s.ax.find("desc:LedgerScreen > descPrefix:ledger_balance:")?.desc)); -const activeAccountId = extract(s => - s.ax.find("desc:LedgerScreen > descPrefix:active_account:")?.desc?.split(":")[1] ?? null); + parseDollarCents( + s.ax.find({ accessibilityText: "LedgerScreen" })?.find({ accessibilityText: "ledger_balance_display" })?.text + )); +const activeAccountId = extract(s => { + const desc = s.ax.find("descPrefix:LedgerScreen:")?.desc; + return desc ? (desc.split(":")[1] ?? null) : null; +}); // focusedInput lives in the app root (not inside any screen), so unscoped const focusedInput = extract(s => s.ax.find("descPrefix:focused_input:")?.desc?.split(":")[1] ?? null); -const loginEmailField = extract(s => s.ax.find("desc:LoginScreen > desc:login_email")); -const loginPasswordField = extract(s => s.ax.find("desc:LoginScreen > desc:login_password")); -const loginSubmit = extract(s => s.ax.find("desc:LoginScreen > desc:login_submit")); -const addAccountButton = extract(s => s.ax.find("desc:HomeScreen > desc:add_account_button")); -const accountNameField = extract(s => s.ax.find("desc:AddAccountScreen > desc:account_name_field")); -const addAccountSubmit = extract(s => s.ax.find("desc:AddAccountScreen > desc:add_account_submit")); -const addTxnButton = extract(s => s.ax.find("desc:LedgerScreen > desc:add_txn_button")); -const txnAmountField = extract(s => s.ax.find("desc:AddTransactionScreen > desc:txn_amount")); -const txnSubmit = extract(s => s.ax.find("desc:AddTransactionScreen > desc:txn_submit")); -const accountCards = extract(s => s.ax.findAll("desc:HomeScreen > descPrefix:account:")); +const loginEmailField = extract(s => + s.ax.find({ accessibilityText: "LoginScreen" })?.find({ accessibilityText: "login_email" })); +const loginPasswordField = extract(s => + s.ax.find({ accessibilityText: "LoginScreen" })?.find({ accessibilityText: "login_password" })); +const loginSubmit = extract(s => + s.ax.find({ accessibilityText: "LoginScreen" })?.find({ accessibilityText: "login_submit" })); +const addAccountButton = extract(s => + s.ax.find({ accessibilityText: "HomeScreen" })?.find({ accessibilityText: "add_account_button" })); +const accountNameField = extract(s => + s.ax.find({ accessibilityText: "AddAccountScreen" })?.find({ accessibilityText: "account_name_field" })); +const addAccountSubmit = extract(s => + s.ax.find({ accessibilityText: "AddAccountScreen" })?.find({ accessibilityText: "add_account_submit" })); +const addTxnButton = extract(s => + s.ax.find({ accessibilityText: "LedgerScreen" })?.find({ accessibilityText: "add_txn_button" })); +const txnAmountField = extract(s => + s.ax.find({ accessibilityText: "AddTransactionScreen" })?.find({ accessibilityText: "txn_amount" })); +const txnSubmit = extract(s => + s.ax.find({ accessibilityText: "AddTransactionScreen" })?.find({ accessibilityText: "txn_submit" })); +const accountCards = extract(s => + s.ax.find({ accessibilityText: "HomeScreen" })?.findAll("descPrefix:account_card:") ?? []); const backButton = extract(s => s.ax.find("desc:Back")); // Property 1: every new account starts with balance === 0 diff --git a/internal/hierarchy/hierarchy.go b/internal/hierarchy/hierarchy.go index 902b7f3..78ee995 100644 --- a/internal/hierarchy/hierarchy.go +++ b/internal/hierarchy/hierarchy.go @@ -1,18 +1,24 @@ // Package hierarchy parses the TreeNode JSON produced by the Maestro sidecar // and resolves selectors against it. // -// Selector grammar (v1.0): +// Selector grammar (v2.0): // -// Single selectors (global scan): -// id: - resource-id == suffix or ends with ":id/" -// text: - exact text match -// desc: - exact content-desc match -// descPrefix: - content-desc starts with prefix +// String selectors (global scan or element-scoped): +// attribute:value - substring match; exact for "true"/"false" booleans +// id: - substring on resource-id / identifier (backward compat) +// text: - substring on text attribute +// desc: - substring on content-desc / accessibilityText +// descPrefix: - starts-with on content-desc / accessibilityText // -// Path queries (segments separated by " > "): -// > > ... - each segment is matched within the subtree of the -// previous match (any descendant, not just direct child) -// example: id:LoginScreen > desc:EmailInput +// Object selectors (multi-attribute AND, element-scoped or global): +// { attr: value, ... } - all key/value pairs must match; substring / boolean semantics +// +// Path queries (global scan only, string form): +// > > ... - each segment matched within subtree of previous match +// +// Cross-platform aliases are expanded automatically: "label" / "accessibilityLabel" +// resolve to accessibilityText; "content-desc" also checks accessibilityText and +// vice-versa; "identifier" / "accessibilityIdentifier" resolve to resource-id. package hierarchy import ( @@ -51,13 +57,14 @@ type Element struct { Package string `json:"package,omitempty"` // Screen holds the current route/screen name when set by the driver on the // root element (web platform only; empty for native platforms). - Screen string `json:"screen,omitempty"` - Clickable bool `json:"clickable,omitempty"` - Enabled bool `json:"enabled,omitempty"` - Checked bool `json:"checked,omitempty"` - Focused bool `json:"focused,omitempty"` - Selected bool `json:"selected,omitempty"` - Bounds Bounds `json:"bounds"` + Screen string `json:"screen,omitempty"` + Clickable bool `json:"clickable,omitempty"` + Enabled bool `json:"enabled,omitempty"` + Checked bool `json:"checked,omitempty"` + Focused bool `json:"focused,omitempty"` + Selected bool `json:"selected,omitempty"` + Bounds Bounds `json:"bounds"` + Attributes map[string]string `json:"attrs,omitempty"` } // Node is one node in the hierarchy tree. @@ -83,6 +90,73 @@ type treeNodeJSON struct { Selected *bool `json:"selected"` } +// Selector describes a multi-attribute AND match. +type Selector struct { + Filters []AttrFilter +} + +// AttrFilter is a single attribute predicate within a Selector. +type AttrFilter struct { + Attr string + Value string +} + +// attributeAliases maps user-written attribute names to the actual keys present +// in the TreeNode attributes map. Both directions are listed so cross-platform +// matching works regardless of which name the caller uses. +var attributeAliases = map[string][]string{ + // Android XML legacy name; web driver uses content-desc; Maestro normalises to accessibilityText + "content-desc": {"accessibilityText"}, + // iOS AXElement / UIKit names + "label": {"accessibilityText"}, + "accessibilityLabel": {"accessibilityText"}, + // accessibilityText is the canonical key; also check content-desc for Android/web + "accessibilityText": {"content-desc"}, + // resource-id canonical key; also check identifier (iOS AXElement raw field) + "resource-id": {"identifier"}, + // iOS identifier names + "identifier": {"resource-id"}, + "accessibilityIdentifier": {"resource-id"}, + // iOS AXElement raw name for hintText + "placeholderValue": {"hintText"}, + // iOS AXElement raw name for class + "elementType": {"class"}, +} + +// matchAttr returns true when the element has an attribute matching attr:value. +// Alias expansion is applied so cross-platform names resolve correctly. +// Boolean values ("true"/"false") use exact comparison; all others use substring. +// Returns false gracefully when no candidate attribute has data. +func matchAttr(element *Element, attr, value string) bool { + candidates := append([]string{attr}, attributeAliases[attr]...) + for _, key := range candidates { + attrVal, ok := element.Attributes[key] + if !ok || attrVal == "" { + continue + } + if value == "true" || value == "false" { + if attrVal == value { + return true + } + } else { + if strings.Contains(attrVal, value) { + return true + } + } + } + return false +} + +// matchSelector returns true when all filters in sel match the element (AND semantics). +func matchSelector(element *Element, sel Selector) bool { + for _, f := range sel.Filters { + if !matchAttr(element, f.Attr, f.Value) { + return false + } + } + return true +} + // Parse parses a Maestro TreeNode JSON hierarchy. func Parse(text string) (*Tree, error) { text = strings.TrimSpace(text) @@ -147,45 +221,125 @@ func elementFromNode(node *treeNodeJSON) *Element { } } + element.Attributes = make(map[string]string, len(attrs)+5) + for k, v := range attrs { + element.Attributes[k] = v + } + if node.Clickable != nil { + element.Attributes["clickable"] = strconv.FormatBool(*node.Clickable) + } + if node.Enabled != nil { + element.Attributes["enabled"] = strconv.FormatBool(*node.Enabled) + } + if node.Focused != nil { + element.Attributes["focused"] = strconv.FormatBool(*node.Focused) + } + if node.Checked != nil { + element.Attributes["checked"] = strconv.FormatBool(*node.Checked) + } + if node.Selected != nil { + element.Attributes["selected"] = strconv.FormatBool(*node.Selected) + } + return element } // Find returns the first element matching the selector, or nil. func (t *Tree) Find(selector string) *Element { + node := t.FindNode(selector) + if node == nil { + return nil + } + return &node.Element +} + +// FindAll returns every element matching the selector. +func (t *Tree) FindAll(selector string) []*Element { + nodes := t.FindAllNodes(selector) + elements := make([]*Element, len(nodes)) + for i, n := range nodes { + elements[i] = &n.Element + } + return elements +} + +// FindNode returns the first Node matching the selector, or nil. +func (t *Tree) FindNode(selector string) *Node { if strings.Contains(selector, " > ") { - return findPath(t.Root, strings.Split(selector, " > ")) + return findPathNode(t.Root, strings.Split(selector, " > ")) } kind, value, ok := parseSelector(selector) if !ok { return nil } - for _, element := range t.Elements { - if match(element, kind, value) { - return element + nodes := searchSubtree(t.Root, kind, value) + if len(nodes) == 0 { + return nil + } + return nodes[0] +} + +// FindAllNodes returns every Node matching the selector. +func (t *Tree) FindAllNodes(selector string) []*Node { + if strings.Contains(selector, " > ") { + return findPathAllNodes(t.Root, strings.Split(selector, " > ")) + } + kind, value, ok := parseSelector(selector) + if !ok { + return nil + } + return searchSubtree(t.Root, kind, value) +} + +// Find returns the first Node in this node's subtree (descendants only) matching +// the string selector. Path queries within the selector are not supported here. +func (n *Node) Find(selector string) *Node { + kind, value, ok := parseSelector(selector) + if !ok { + return nil + } + for _, child := range n.Children { + if nodes := searchSubtree(child, kind, value); len(nodes) > 0 { + return nodes[0] } } return nil } -// FindAll returns every element matching the selector. -func (t *Tree) FindAll(selector string) []*Element { - if strings.Contains(selector, " > ") { - return findPathAll(t.Root, strings.Split(selector, " > ")) - } +// FindAll returns all Nodes in this node's subtree (descendants only) matching +// the string selector. +func (n *Node) FindAll(selector string) []*Node { kind, value, ok := parseSelector(selector) if !ok { return nil } - var matches []*Element - for _, element := range t.Elements { - if match(element, kind, value) { - matches = append(matches, element) - } + var result []*Node + for _, child := range n.Children { + result = append(result, searchSubtree(child, kind, value)...) } - return matches + return result } -func findPath(root *Node, segments []string) *Element { +// FindBySelector returns the first Node in this node's subtree matching sel (AND semantics). +func (n *Node) FindBySelector(sel Selector) *Node { + for _, child := range n.Children { + if nodes := searchSubtreeBySelector(child, sel); len(nodes) > 0 { + return nodes[0] + } + } + return nil +} + +// FindAllBySelector returns all Nodes in this node's subtree matching sel (AND semantics). +func (n *Node) FindAllBySelector(sel Selector) []*Node { + var result []*Node + for _, child := range n.Children { + result = append(result, searchSubtreeBySelector(child, sel)...) + } + return result +} + +func findPathNode(root *Node, segments []string) *Node { if root == nil || len(segments) == 0 { return nil } @@ -195,16 +349,16 @@ func findPath(root *Node, segments []string) *Element { } for _, node := range searchSubtree(root, kind, value) { if len(segments) == 1 { - return &node.Element + return node } - if result := findPathDescendants(node, segments[1:]); result != nil { + if result := findPathDescendantsNode(node, segments[1:]); result != nil { return result } } return nil } -func findPathDescendants(root *Node, segments []string) *Element { +func findPathDescendantsNode(root *Node, segments []string) *Node { kind, value, ok := parseSelector(segments[0]) if !ok { return nil @@ -212,9 +366,9 @@ func findPathDescendants(root *Node, segments []string) *Element { for _, child := range root.Children { for _, node := range searchSubtree(child, kind, value) { if len(segments) == 1 { - return &node.Element + return node } - if result := findPathDescendants(node, segments[1:]); result != nil { + if result := findPathDescendantsNode(node, segments[1:]); result != nil { return result } } @@ -222,7 +376,7 @@ func findPathDescendants(root *Node, segments []string) *Element { return nil } -func findPathAll(root *Node, segments []string) []*Element { +func findPathAllNodes(root *Node, segments []string) []*Node { if root == nil || len(segments) == 0 { return nil } @@ -230,30 +384,30 @@ func findPathAll(root *Node, segments []string) []*Element { if !ok { return nil } - var result []*Element + var result []*Node for _, node := range searchSubtree(root, kind, value) { if len(segments) == 1 { - result = append(result, &node.Element) + result = append(result, node) continue } - result = append(result, findPathAllDescendants(node, segments[1:])...) + result = append(result, findPathAllDescendantsNodes(node, segments[1:])...) } return result } -func findPathAllDescendants(root *Node, segments []string) []*Element { +func findPathAllDescendantsNodes(root *Node, segments []string) []*Node { kind, value, ok := parseSelector(segments[0]) if !ok { return nil } - var result []*Element + var result []*Node for _, child := range root.Children { for _, node := range searchSubtree(child, kind, value) { if len(segments) == 1 { - result = append(result, &node.Element) + result = append(result, node) continue } - result = append(result, findPathAllDescendants(node, segments[1:])...) + result = append(result, findPathAllDescendantsNodes(node, segments[1:])...) } } return result @@ -274,6 +428,21 @@ func searchSubtree(root *Node, kind, value string) []*Node { return result } +// searchSubtreeBySelector returns all nodes under root (inclusive) matching sel. +func searchSubtreeBySelector(root *Node, sel Selector) []*Node { + if root == nil { + return nil + } + var result []*Node + if matchSelector(&root.Element, sel) { + result = append(result, root) + } + for _, child := range root.Children { + result = append(result, searchSubtreeBySelector(child, sel)...) + } + return result +} + func parseSelector(selector string) (string, string, bool) { index := strings.IndexByte(selector, ':') if index <= 0 { @@ -290,14 +459,13 @@ func match(element *Element, kind, value string) bool { } return strings.HasSuffix(element.ResourceID, ":id/"+value) case "text": - return element.Text == value + return matchAttr(element, "text", value) case "desc": - // Exact match, or iOS merged label "desc, child text". return element.Description == value || strings.HasPrefix(element.Description, value+", ") case "descPrefix": return strings.HasPrefix(element.Description, value) default: - return false + return matchAttr(element, kind, value) } } diff --git a/internal/hierarchy/hierarchy_test.go b/internal/hierarchy/hierarchy_test.go index bbbd8b1..ae896d1 100644 --- a/internal/hierarchy/hierarchy_test.go +++ b/internal/hierarchy/hierarchy_test.go @@ -307,3 +307,172 @@ func TestIOSBoundsFormat(t *testing.T) { t.Errorf("unexpected center: (%d, %d)", cx, cy) } } + +// --- full-attribute selector tests --- + +const androidAttrDump = `{ + "attributes": {"resource-id": "com.app:id/list", "bounds": "[0,0,1080,2340]"}, + "children": [ + { + "attributes": {"resource-id": "com.app:id/row1", "scrollable": "true", "bounds": "[0,0,1080,200]"}, + "children": [], + "clickable": true, + "enabled": true + }, + { + "attributes": {"resource-id": "com.app:id/row2", "scrollable": "false", "bounds": "[0,200,1080,400]"}, + "children": [], + "clickable": false, + "enabled": true + } + ] +}` + +const iosAttrDump = `{ + "attributes": {"bounds": "[0,0,390,844]"}, + "children": [ + { + "attributes": {"accessibilityText": "Close", "title": "Settings", "bounds": "[0,0,100,50]"}, + "children": [], + "enabled": true + } + ] +}` + +func TestRawResourceIDSubstringMatch(t *testing.T) { + tree, _ := Parse(androidAttrDump) + el := tree.Find("resource-id:row1") + if el == nil { + t.Fatal("expected resource-id: to match via substring") + } +} + +func TestLabelAliasMatchesAccessibilityText(t *testing.T) { + tree, _ := Parse(iosAttrDump) + el := tree.Find("label:Close") + if el == nil { + t.Fatal("expected label: to match accessibilityText via alias") + } +} + +func TestContentDescAliasOnIOS(t *testing.T) { + tree, _ := Parse(iosAttrDump) + el := tree.Find("content-desc:Close") + if el == nil { + t.Fatal("expected content-desc: to match accessibilityText via alias on iOS") + } +} + +func TestScrollableTrueMatches(t *testing.T) { + tree, _ := Parse(androidAttrDump) + el := tree.Find("scrollable:true") + if el == nil { + t.Fatal("expected scrollable:true to match") + } + if el.ResourceID != "com.app:id/row1" { + t.Fatalf("got %q, want row1", el.ResourceID) + } +} + +func TestScrollableFalseMatchesSecondRow(t *testing.T) { + tree, _ := Parse(androidAttrDump) + el := tree.Find("scrollable:false") + if el == nil { + t.Fatal("expected scrollable:false to match row2") + } + if el.ResourceID != "com.app:id/row2" { + t.Fatalf("got %q, want row2", el.ResourceID) + } +} + +func TestTitleMatchesIOSElement(t *testing.T) { + tree, _ := Parse(iosAttrDump) + el := tree.Find("title:Settings") + if el == nil { + t.Fatal("expected title:Settings to match iOS element") + } +} + +func TestTitleReturnsNilForAndroid(t *testing.T) { + tree, _ := Parse(androidAttrDump) + el := tree.Find("title:Settings") + if el != nil { + t.Fatal("expected title:Settings to return nil for Android element (graceful ignore)") + } +} + +func TestScrollableGracefulIgnoreOnIOS(t *testing.T) { + tree, _ := Parse(iosAttrDump) + el := tree.Find("scrollable:true") + if el != nil { + t.Fatal("expected scrollable:true to return nil on iOS hierarchy (graceful ignore)") + } +} + +func TestTextIsNowSubstring(t *testing.T) { + tree, _ := Parse(sampleDump) + el := tree.Find("text:Hel") + if el == nil { + t.Fatal("expected text: to match substring") + } + if el.Text != "Hello" { + t.Fatalf("got %q, want Hello", el.Text) + } +} + +func TestMultiFilterSelectorAND(t *testing.T) { + tree, _ := Parse(androidAttrDump) + sel := Selector{Filters: []AttrFilter{ + {Attr: "scrollable", Value: "true"}, + {Attr: "resource-id", Value: "row1"}, + }} + node := tree.Root.FindBySelector(sel) + if node == nil { + t.Fatal("expected AND selector to find row1 (scrollable=true AND resource-id contains row1)") + } + if node.Element.ResourceID != "com.app:id/row1" { + t.Fatalf("got %q, want row1", node.Element.ResourceID) + } +} + +func TestMultiFilterSelectorMissReturnsNil(t *testing.T) { + tree, _ := Parse(androidAttrDump) + sel := Selector{Filters: []AttrFilter{ + {Attr: "scrollable", Value: "true"}, + {Attr: "resource-id", Value: "row2"}, // row2 is not scrollable=true + }} + node := tree.Root.FindBySelector(sel) + if node != nil { + t.Fatal("expected AND selector to return nil when one filter misses") + } +} + +func TestNodeFindScopedSearch(t *testing.T) { + tree, _ := Parse(pathDump) + // A2 has a child B2 with content-desc "label_b" + // Find the A node, then search its subtree for label_b -- should find B (not B2) + aNode := tree.FindNode("id:A") + if aNode == nil { + t.Fatal("expected to find A node") + } + result := aNode.Find("desc:label_b") + if result == nil { + t.Fatal("expected Node.Find to find label_b in A's subtree") + } + if result.Element.ResourceID != "B" { + t.Fatalf("got %q, want B (not B2 from sibling A2)", result.Element.ResourceID) + } +} + +func TestNodeFindDoesNotReturnSiblings(t *testing.T) { + tree, _ := Parse(pathDump) + a2Node := tree.FindNode("id:A2") + if a2Node == nil { + t.Fatal("expected to find A2 node") + } + // B is under A, not A2 -- should not be found from A2's subtree + result := a2Node.Find("id:B") + if result != nil && result.Element.ResourceID == "B" { + t.Fatal("Node.Find should not return nodes from sibling subtrees") + } +} diff --git a/internal/verifier/marshal.go b/internal/verifier/marshal.go index 780270d..6d15f45 100644 --- a/internal/verifier/marshal.go +++ b/internal/verifier/marshal.go @@ -61,33 +61,34 @@ func stateObject(runtime *goja.Runtime, input stateInput) (*goja.Object, error) func accessibilityObject(runtime *goja.Runtime, tree *hierarchy.Tree) *goja.Object { accessibility := runtime.NewObject() - find := func(selector string) goja.Value { + find := func(call goja.FunctionCall) goja.Value { if tree == nil { return goja.Undefined() } - element := tree.Find(selector) - if element == nil { + node := findNodeFromJS(runtime, tree, call.Argument(0)) + if node == nil { return goja.Undefined() } - return elementObject(runtime, element, selector) + return nodeObject(runtime, node, selectorStringFromJS(call.Argument(0))) } - findAll := func(selector string) []goja.Value { + findAll := func(call goja.FunctionCall) goja.Value { if tree == nil { - return nil + return goja.Undefined() } - elements := tree.FindAll(selector) - result := make([]goja.Value, len(elements)) - for index, element := range elements { - result[index] = elementObject(runtime, element, selector) + nodes := findAllNodesFromJS(runtime, tree, call.Argument(0)) + array := runtime.NewArray() + for i, n := range nodes { + _ = array.Set(fmt.Sprintf("%d", i), nodeObject(runtime, n, selectorStringFromJS(call.Argument(0)))) } - return result + return array } _ = accessibility.Set("find", runtime.ToValue(find)) _ = accessibility.Set("findAll", runtime.ToValue(findAll)) return accessibility } -func elementObject(runtime *goja.Runtime, element *hierarchy.Element, selector string) goja.Value { +func nodeObject(runtime *goja.Runtime, node *hierarchy.Node, selector string) goja.Value { + element := &node.Element object := runtime.NewObject() centerX, centerY := element.Bounds.Center() _ = object.Set("id", element.ResourceID) @@ -108,9 +109,129 @@ func elementObject(runtime *goja.Runtime, element *hierarchy.Element, selector s _ = bounds.Set("right", element.Bounds.Right) _ = bounds.Set("bottom", element.Bounds.Bottom) _ = object.Set("bounds", bounds) + attrs := runtime.NewObject() + for k, v := range element.Attributes { + _ = attrs.Set(k, v) + } + _ = object.Set("attrs", attrs) + childFind := func(call goja.FunctionCall) goja.Value { + arg := call.Argument(0) + childNode := findNodeInSubtreeFromJS(runtime, node, arg) + if childNode == nil { + return goja.Undefined() + } + return nodeObject(runtime, childNode, selectorStringFromJS(arg)) + } + childFindAll := func(call goja.FunctionCall) goja.Value { + arg := call.Argument(0) + childNodes := findAllNodesInSubtreeFromJS(runtime, node, arg) + array := runtime.NewArray() + for i, n := range childNodes { + _ = array.Set(fmt.Sprintf("%d", i), nodeObject(runtime, n, selectorStringFromJS(arg))) + } + return array + } + _ = object.Set("find", runtime.ToValue(childFind)) + _ = object.Set("findAll", runtime.ToValue(childFindAll)) return object } +// findNodeFromJS dispatches a JS value (string or object) to Tree-level node lookup. +func findNodeFromJS(runtime *goja.Runtime, tree *hierarchy.Tree, arg goja.Value) *hierarchy.Node { + if goja.IsUndefined(arg) || goja.IsNull(arg) { + return nil + } + if tree == nil { + return nil + } + if s, ok := arg.Export().(string); ok { + return tree.FindNode(s) + } + sel := selectorFromJSObject(runtime, arg) + if len(sel.Filters) == 0 { + return nil + } + return tree.Root.FindBySelector(sel) +} + +// findAllNodesFromJS dispatches a JS value to Tree-level multi-node lookup. +func findAllNodesFromJS(runtime *goja.Runtime, tree *hierarchy.Tree, arg goja.Value) []*hierarchy.Node { + if goja.IsUndefined(arg) || goja.IsNull(arg) || tree == nil { + return nil + } + if s, ok := arg.Export().(string); ok { + return tree.FindAllNodes(s) + } + sel := selectorFromJSObject(runtime, arg) + if len(sel.Filters) == 0 { + return nil + } + return tree.Root.FindAllBySelector(sel) +} + +// findNodeInSubtreeFromJS dispatches a JS value to Node-level scoped lookup. +func findNodeInSubtreeFromJS(runtime *goja.Runtime, node *hierarchy.Node, arg goja.Value) *hierarchy.Node { + if goja.IsUndefined(arg) || goja.IsNull(arg) { + return nil + } + if s, ok := arg.Export().(string); ok { + return node.Find(s) + } + sel := selectorFromJSObject(runtime, arg) + if len(sel.Filters) == 0 { + return nil + } + return node.FindBySelector(sel) +} + +// findAllNodesInSubtreeFromJS dispatches a JS value to Node-level scoped multi-lookup. +func findAllNodesInSubtreeFromJS(runtime *goja.Runtime, node *hierarchy.Node, arg goja.Value) []*hierarchy.Node { + if goja.IsUndefined(arg) || goja.IsNull(arg) { + return nil + } + if s, ok := arg.Export().(string); ok { + return node.FindAll(s) + } + sel := selectorFromJSObject(runtime, arg) + if len(sel.Filters) == 0 { + return nil + } + return node.FindAllBySelector(sel) +} + +// selectorFromJSObject converts a JS object {attr: value, ...} into a Selector. +func selectorFromJSObject(runtime *goja.Runtime, arg goja.Value) hierarchy.Selector { + obj := arg.ToObject(runtime) + if obj == nil { + return hierarchy.Selector{} + } + var sel hierarchy.Selector + for _, key := range obj.Keys() { + if key == tagSelector { + continue + } + val := obj.Get(key) + if val == nil || goja.IsUndefined(val) { + continue + } + sel.Filters = append(sel.Filters, hierarchy.AttrFilter{Attr: key, Value: val.String()}) + } + return sel +} + +// selectorStringFromJS returns a string representation of the selector argument +// for tagging returned element objects (used by selectorOf to reconstruct the +// selector when the element is passed back as an action target). +func selectorStringFromJS(arg goja.Value) string { + if goja.IsUndefined(arg) || goja.IsNull(arg) { + return "" + } + if s, ok := arg.Export().(string); ok { + return s + } + return arg.String() +} + func lastActionObject(runtime *goja.Runtime, action *Action) goja.Value { if action == nil { return goja.Null() diff --git a/pkg/spec/src/types.ts b/pkg/spec/src/types.ts index 5019187..c1c1ef7 100644 --- a/pkg/spec/src/types.ts +++ b/pkg/spec/src/types.ts @@ -1,4 +1,5 @@ export type Snapshots = Record; +export type AttrSelector = Record; export interface AccessibilityElement { id?: string; @@ -13,11 +14,14 @@ export interface AccessibilityElement { bounds?: { left: number; top: number; right: number; bottom: number }; x?: number; y?: number; + attrs?: Record; + find(selector: string | AttrSelector): AccessibilityElement | undefined; + findAll(selector: string | AttrSelector): AccessibilityElement[]; } export interface AccessibilityTree { - find(selector: string): AccessibilityElement | undefined; - findAll(selector: string): AccessibilityElement[]; + find(selector: string | AttrSelector): AccessibilityElement | undefined; + findAll(selector: string | AttrSelector): AccessibilityElement[]; } export interface LogEntry {