diff --git a/companion/Sources/Snapshot.swift b/companion/Sources/Snapshot.swift index 63783ce..61e4449 100644 --- a/companion/Sources/Snapshot.swift +++ b/companion/Sources/Snapshot.swift @@ -33,27 +33,28 @@ enum Snapshot { let application = XCUIApplication(bundleIdentifier: bundleIdentifier) let root = try application.snapshot() var result: [[String: Any]] = [] - walk(root, into: &result) + walk(root, depth: 0, into: &result) return result } - private static func walk(_ node: XCUIElementSnapshot, into result: inout [[String: Any]]) { + private static func walk(_ node: XCUIElementSnapshot, depth: Int, into result: inout [[String: Any]]) { // The keyboard subtree is pruned: the legacy accessibility bridge // never exposed it, its key frames are unreliable as tap targets, and // its shift-state churn destabilizes settle hashing. if node.elementType == .keyboard { return } - result.append(serialize(node)) + result.append(serialize(node, depth: depth)) for child in node.children { - walk(child, into: &result) + walk(child, depth: depth + 1, into: &result) } } - private static func serialize(_ node: XCUIElementSnapshot) -> [String: Any] { + private static func serialize(_ node: XCUIElementSnapshot, depth: Int) -> [String: Any] { let frame = node.frame return [ "type": elementTypeName(node.elementType), + "depth": depth, "frame": [ "x": Double(frame.origin.x), "y": Double(frame.origin.y), diff --git a/internal/driver/ioscompanion/hierarchymap.go b/internal/driver/ioscompanion/hierarchymap.go index 3d5f0b5..8f1e667 100644 --- a/internal/driver/ioscompanion/hierarchymap.go +++ b/internal/driver/ioscompanion/hierarchymap.go @@ -42,6 +42,7 @@ type rawElement struct { AXLabel *string `json:"AXLabel"` AXValue *string `json:"AXValue"` Type string `json:"type"` + Depth int `json:"depth"` Enabled bool `json:"enabled"` } @@ -76,12 +77,18 @@ func MapHierarchy(dump []byte, screenWidth, screenHeight int) ([]byte, error) { if len(dump) > 0 { _ = json.Unmarshal(dump, &rawElements) } + elements := make([]rawElement, 0, len(rawElements)) for _, raw := range rawElements { var element rawElement if err := json.Unmarshal(raw, &element); err != nil { continue } - if child, ok := mapElement(&element); ok { + elements = append(elements, element) + } + + scrollable := scrollableElements(elements, screenWidth, screenHeight) + for index := range elements { + if child, ok := mapElement(&elements[index], scrollable[index]); ok { root.Children = append(root.Children, child) } } @@ -89,7 +96,84 @@ func MapHierarchy(dump []byte, screenWidth, screenHeight int) ([]byte, error) { return json.Marshal(root) } -func mapElement(element *rawElement) (treeNode, bool) { +// frameTolerance absorbs the sub-point rounding in companion frames, so a child +// that sits flush against its container's edge does not read as escaping it. +const frameTolerance = 0.5 + +// scrollableElements reports, per element, whether it is a container that clips +// content reaching past its own frame. That is the same fact Android reads off +// uiautomator's scrollable attribute and the web driver derives from overflow: +// the container can actually scroll, because there is content it is not showing. +// +// Three conditions together, because the snapshot has no clipping flag. The +// element must sit strictly inside its parent on at least one edge, which +// separates a real container from the stack of full-screen wrappers that +// inherit its overflow; some element in its subtree must lie outside it; and it +// must be on the screen, since a dismissed keyboard is reported below the screen +// and clips a much taller child without any gesture being able to reach it. +// A dump without depth (the legacy accessibility bridge) makes every element a +// root, and roots are never marked, so that path reports no scroll rather than +// a guessed one. +func scrollableElements(elements []rawElement, screenWidth, screenHeight int) []bool { + screen := rawFrame{Width: float64(screenWidth), Height: float64(screenHeight)} + scrollable := make([]bool, len(elements)) + var ancestors []int + for index, element := range elements { + for len(ancestors) > 0 && elements[ancestors[len(ancestors)-1]].Depth >= element.Depth { + ancestors = ancestors[:len(ancestors)-1] + } + if len(ancestors) > 0 && hasArea(element.Frame) && + overlaps(element.Frame, screen) && + sitsInside(element.Frame, elements[ancestors[len(ancestors)-1]].Frame) && + subtreeEscapes(elements, index) { + scrollable[index] = true + } + ancestors = append(ancestors, index) + } + return scrollable +} + +// overlaps reports whether two frames share any area. +func overlaps(frame, other rawFrame) bool { + return frame.X < other.X+other.Width && other.X < frame.X+frame.Width && + frame.Y < other.Y+other.Height && other.Y < frame.Y+frame.Height +} + +// sitsInside reports whether frame is strictly smaller than container on at +// least one edge. +func sitsInside(frame, container rawFrame) bool { + return frame.X > container.X+frameTolerance || + frame.Y > container.Y+frameTolerance || + frame.X+frame.Width < container.X+container.Width-frameTolerance || + frame.Y+frame.Height < container.Y+container.Height-frameTolerance +} + +// subtreeEscapes reports whether any descendant of the element at index is +// positioned outside its frame. Descendants are the run that follows it while +// the depth stays greater, which is the pre-order walk the companion emits. +func subtreeEscapes(elements []rawElement, index int) bool { + frame := elements[index].Frame + for next := index + 1; next < len(elements) && elements[next].Depth > elements[index].Depth; next++ { + child := elements[next].Frame + if !hasArea(child) { + continue + } + if child.X < frame.X-frameTolerance || + child.Y < frame.Y-frameTolerance || + child.X+child.Width > frame.X+frame.Width+frameTolerance || + child.Y+child.Height > frame.Y+frame.Height+frameTolerance { + return true + } + } + return false +} + +func hasArea(frame rawFrame) bool { + return finite(frame.X) && finite(frame.Y) && finite(frame.Width) && + finite(frame.Height) && frame.Width > 0 && frame.Height > 0 +} + +func mapElement(element *rawElement, scrollable bool) (treeNode, bool) { if element.Type == "" { return treeNode{}, false } @@ -106,6 +190,10 @@ func mapElement(element *rawElement) (treeNode, bool) { "class": element.Type, } + if scrollable { + attributes["scrollable"] = "true" + } + if id := stringValue(element.AXUniqueID); id != "" { attributes["identifier"] = id } diff --git a/internal/driver/ioscompanion/hierarchymap_test.go b/internal/driver/ioscompanion/hierarchymap_test.go index 1c7a938..a1b912a 100644 --- a/internal/driver/ioscompanion/hierarchymap_test.go +++ b/internal/driver/ioscompanion/hierarchymap_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "path/filepath" + "slices" "strings" "testing" @@ -331,3 +332,59 @@ func TestDumpIsCollapsed(t *testing.T) { }) } } + +func scrollableBounds(tree *hierarchy.Tree) []hierarchy.Bounds { + var bounds []hierarchy.Bounds + for _, element := range tree.FindAllNodes("scrollable:true") { + bounds = append(bounds, element.Bounds) + } + return bounds +} + +func TestScrollableMarksTheContainerThatClipsOverflowingContent(t *testing.T) { + tree := mapAndParse( + t, + readDump(t, "home-scrolling-describe.json"), + 402, + 874, + ) + got := scrollableBounds(tree) + want := []hierarchy.Bounds{{Left: 0, Top: 122, Right: 402, Bottom: 699}} + if !slices.Equal(got, want) { + t.Fatalf("scrollable containers = %+v, want %+v", got, want) + } +} + +func TestScrollableIsAbsentWhenNothingOverflows(t *testing.T) { + tree := mapAndParse(t, readDump(t, "home-fixed-describe.json"), 402, 874) + if got := scrollableBounds(tree); len(got) != 0 { + t.Fatalf( + "scrollable containers = %+v, want none on a screen that does not scroll", + got, + ) + } +} + +func TestScrollableIsAbsentWithoutTreeDepth(t *testing.T) { + tree := mapAndParse(t, readDump(t, "accounts-describe.json"), 402, 874) + if got := scrollableBounds(tree); len(got) != 0 { + t.Fatalf( + "scrollable containers = %+v, want none from a dump that carries no depth", + got, + ) + } +} + +func TestScrollableIgnoresAContainerOffTheScreen(t *testing.T) { + // The dismissed keyboard is reported below the screen, and its prediction + // bar clips a much taller child, so it satisfies every other condition. + dump := `[ + {"type":"Application","depth":0,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true}, + {"type":"Other","depth":1,"frame":{"x":0,"y":874,"width":402,"height":54},"enabled":true}, + {"type":"Other","depth":2,"frame":{"x":0,"y":274,"width":402,"height":1254},"enabled":true} + ]` + tree := mapAndParse(t, []byte(dump), 402, 874) + if got := scrollableBounds(tree); len(got) != 0 { + t.Fatalf("scrollable containers = %+v, want none off the screen", got) + } +} diff --git a/internal/driver/ioscompanion/testdata/home-fixed-describe.json b/internal/driver/ioscompanion/testdata/home-fixed-describe.json new file mode 100644 index 0000000..7111325 --- /dev/null +++ b/internal/driver/ioscompanion/testdata/home-fixed-describe.json @@ -0,0 +1,35 @@ +[ + {"type":"Application","depth":0,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":"Folio","AXValue":null,"AXUniqueId":null}, + {"type":"Window","depth":1,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":2,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":3,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":4,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":5,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":6,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":7,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":7,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"ScrollView","depth":7,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":8,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":9,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":10,"frame":{"x":0,"y":62,"width":402,"height":778},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":10,"frame":{"x":0,"y":62,"width":402,"height":778},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":"HomeScreen"}, + {"type":"Other","depth":10,"frame":{"x":0,"y":62,"width":402,"height":778},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"StaticText","depth":10,"frame":{"x":20,"y":76,"width":97.33,"height":24},"enabled":true,"AXLabel":"Accounts","AXValue":null,"AXUniqueId":null}, + {"type":"Button","depth":10,"frame":{"x":340,"y":71.33,"width":48,"height":48},"enabled":true,"AXLabel":"Log out","AXValue":null,"AXUniqueId":"LogoutButton"}, + {"type":"StaticText","depth":10,"frame":{"x":20,"y":100,"width":104,"height":14.33},"enabled":true,"AXLabel":"demo@folio.app","AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":10,"frame":{"x":0,"y":122.33,"width":402,"height":577},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":11,"frame":{"x":0,"y":122.33,"width":402,"height":577},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":11,"frame":{"x":173,"y":178.33,"width":56,"height":56},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"StaticText","depth":11,"frame":{"x":131.33,"y":248.33,"width":139.33,"height":18.0},"enabled":true,"AXLabel":"No accounts yet","AXValue":null,"AXUniqueId":null}, + {"type":"StaticText","depth":11,"frame":{"x":36,"y":272.33,"width":330,"height":28.67},"enabled":true,"AXLabel":"Create your first account to start tracking transactions.","AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":10,"frame":{"x":0,"y":676,"width":402,"height":48},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"StaticText","depth":10,"frame":{"x":20,"y":716.33,"width":105.67,"height":14.33},"enabled":true,"AXLabel":"TOTAL BALANCE","AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":10,"frame":{"x":20,"y":716.33,"width":362,"height":107.67},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"StaticText","depth":10,"frame":{"x":20,"y":730.67,"width":85.33,"height":33.33},"enabled":true,"AXLabel":"$0.00","AXValue":null,"AXUniqueId":"TotalBalance"}, + {"type":"StaticText","depth":10,"frame":{"x":307.67,"y":749.67,"width":74.33,"height":14.33},"enabled":true,"AXLabel":"0 accounts","AXValue":null,"AXUniqueId":null}, + {"type":"Button","depth":10,"frame":{"x":20,"y":777,"width":362,"height":48},"enabled":true,"AXLabel":"+ Add account","AXValue":null,"AXUniqueId":"AddAccountButton"}, + {"type":"StaticText","depth":11,"frame":{"x":140.67,"y":792,"width":120.67,"height":18},"enabled":true,"AXLabel":"+ Add account","AXValue":null,"AXUniqueId":null}, + {"type":"Window","depth":1,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":2,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":3,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null} +] diff --git a/internal/driver/ioscompanion/testdata/home-scrolling-describe.json b/internal/driver/ioscompanion/testdata/home-scrolling-describe.json new file mode 100644 index 0000000..dc82334 --- /dev/null +++ b/internal/driver/ioscompanion/testdata/home-scrolling-describe.json @@ -0,0 +1,35 @@ +[ + {"type":"Application","depth":0,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":"Folio","AXValue":null,"AXUniqueId":null}, + {"type":"Window","depth":1,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":2,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":3,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":4,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":5,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":6,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":7,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":7,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"ScrollView","depth":7,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":8,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":9,"frame":{"x":0,"y":0,"width":402,"height":874},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":10,"frame":{"x":0,"y":62,"width":402,"height":778},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":10,"frame":{"x":0,"y":62,"width":402,"height":778},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":"HomeScreen"}, + {"type":"Other","depth":10,"frame":{"x":0,"y":62,"width":402,"height":778},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"StaticText","depth":10,"frame":{"x":20,"y":76,"width":97.33,"height":24},"enabled":true,"AXLabel":"Accounts","AXValue":null,"AXUniqueId":null}, + {"type":"Button","depth":10,"frame":{"x":340,"y":71.33,"width":48,"height":48},"enabled":true,"AXLabel":"Log out","AXValue":null,"AXUniqueId":"LogoutButton"}, + {"type":"StaticText","depth":10,"frame":{"x":20,"y":100,"width":104,"height":14.33},"enabled":true,"AXLabel":"demo@folio.app","AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":10,"frame":{"x":0,"y":122.33,"width":402,"height":577},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Other","depth":11,"frame":{"x":0,"y":122.33,"width":402,"height":577},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"Button","depth":11,"frame":{"x":20,"y":704.33,"width":362,"height":19},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":"AccountCard"}, + {"type":"Button","depth":11,"frame":{"x":20,"y":130.33,"width":362,"height":72},"enabled":true,"AXLabel":"A0, Account 0, $0.00, 0 transactions","AXValue":null,"AXUniqueId":"AccountCard"}, + {"type":"StaticText","depth":12,"frame":{"x":90,"y":150.33,"width":83.67,"height":18},"enabled":true,"AXLabel":"Account 0","AXValue":null,"AXUniqueId":"AccountName"}, + {"type":"StaticText","depth":12,"frame":{"x":90,"y":168.33,"width":104,"height":14.33},"enabled":true,"AXLabel":"0 transactions","AXValue":null,"AXUniqueId":"AccountTxnCount"}, + {"type":"Button","depth":11,"frame":{"x":20,"y":212.33,"width":362,"height":72.0},"enabled":true,"AXLabel":"A1, Account 1, $0.00, 0 transactions","AXValue":null,"AXUniqueId":"AccountCard"}, + {"type":"StaticText","depth":12,"frame":{"x":90,"y":232.33,"width":83.67,"height":18},"enabled":true,"AXLabel":"Account 1","AXValue":null,"AXUniqueId":"AccountName"}, + {"type":"Button","depth":11,"frame":{"x":20,"y":786.33,"width":362,"height":72},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":"AccountCard"}, + {"type":"Button","depth":11,"frame":{"x":20,"y":1278.33,"width":362,"height":72},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":"AccountCard"}, + {"type":"Other","depth":10,"frame":{"x":0,"y":676,"width":402,"height":48},"enabled":true,"AXLabel":null,"AXValue":null,"AXUniqueId":null}, + {"type":"StaticText","depth":10,"frame":{"x":20,"y":716.33,"width":105.67,"height":14.33},"enabled":true,"AXLabel":"TOTAL BALANCE","AXValue":null,"AXUniqueId":null}, + {"type":"StaticText","depth":10,"frame":{"x":20,"y":730.67,"width":85.33,"height":33.33},"enabled":true,"AXLabel":"$0.00","AXValue":null,"AXUniqueId":"TotalBalance"}, + {"type":"Button","depth":10,"frame":{"x":20,"y":777,"width":362,"height":48},"enabled":true,"AXLabel":"+ Add account","AXValue":null,"AXUniqueId":"AddAccountButton"}, + {"type":"StaticText","depth":11,"frame":{"x":140.67,"y":792,"width":120.67,"height":18},"enabled":true,"AXLabel":"+ Add account","AXValue":null,"AXUniqueId":null} +] diff --git a/internal/hierarchy/hierarchy_test.go b/internal/hierarchy/hierarchy_test.go index 0e66e30..dfd8191 100644 --- a/internal/hierarchy/hierarchy_test.go +++ b/internal/hierarchy/hierarchy_test.go @@ -482,6 +482,11 @@ const iosAttrDump = `{ "attributes": {"accessibilityText": "Close", "title": "Settings", "bounds": "[0,0,100,50]"}, "children": [], "enabled": true + }, + { + "attributes": {"identifier": "Feed", "scrollable": "true", "bounds": "[0,120,390,700]"}, + "children": [], + "enabled": true } ] }` @@ -548,11 +553,14 @@ func TestTitleReturnsNilForAndroid(t *testing.T) { } } -func TestScrollableGracefulIgnoreOnIOS(t *testing.T) { +func TestScrollableMatchesOnIOS(t *testing.T) { tree, _ := Parse(iosAttrDump) el := tree.Find("scrollable:true") - if el != nil { - t.Fatal("expected scrollable:true to return nil on iOS hierarchy (graceful ignore)") + if el == nil { + t.Fatal("expected scrollable:true to match the iOS scroll container") + } + if el.Attributes["identifier"] != "Feed" { + t.Fatalf("got %q, want Feed", el.Attributes["identifier"]) } }