From 0f448e5e49387f707ab250d62ed39a2e370ec16d Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 5 Jun 2026 15:41:16 +0530 Subject: [PATCH] feat(sidecar): derive iOS clickable and editable from element type The XCTest hierarchy mapping dropped the element type, leaving no clickable or editable flags on iOS, so the fuzzer's tap and typing verbs never found a candidate inside the app. Map the raw accessibility tree directly and derive clickable, editable, scrollable, and class from the XCUIElementType raw value. --- .../dev/sanderling/sidecar/DriverBackend.kt | 108 +++++++++++++++++- .../sanderling/sidecar/IosHierarchyTest.kt | 101 ++++++++++++++++ 2 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 sidecar/src/test/kotlin/dev/sanderling/sidecar/IosHierarchyTest.kt diff --git a/sidecar/src/main/kotlin/dev/sanderling/sidecar/DriverBackend.kt b/sidecar/src/main/kotlin/dev/sanderling/sidecar/DriverBackend.kt index a10e58c..af8c466 100644 --- a/sidecar/src/main/kotlin/dev/sanderling/sidecar/DriverBackend.kt +++ b/sidecar/src/main/kotlin/dev/sanderling/sidecar/DriverBackend.kt @@ -629,6 +629,7 @@ private fun pngHeight(bytes: ByteArray): Int { class IosDriverBackend(private val udid: String) : DriverBackend { private lateinit var driver: maestro.drivers.IOSDriver + private lateinit var localDevice: ios.LocalIOSDevice private val reconnectLock = java.util.concurrent.locks.ReentrantLock() init { @@ -670,6 +671,7 @@ class IosDriverBackend(private val udid: String) : DriverBackend { deviceController = simctlDevice, insights = maestro.utils.NoopInsights, ) + localDevice = device driver = maestro.drivers.IOSDriver(device, maestro.utils.NoopInsights) driver.open() warmup() @@ -750,7 +752,7 @@ class IosDriverBackend(private val udid: String) : DriverBackend { override fun hierarchy(): String = withReconnect { com.fasterxml.jackson.module.kotlin.jacksonObjectMapper() - .writeValueAsString(driver.contentDescriptor(false)) + .writeValueAsString(iosAxElementToTreeNode(localDevice.viewHierarchy(false).axElement)) } override fun recentLogs(sinceUnixMillis: Long, minLevel: String): List = emptyList() @@ -784,3 +786,107 @@ private fun keyCodeToMaestro(adbKeyCode: String): maestro.KeyCode? { else -> null } } + +// XCUIElementType raw values that accept a tap. The XCTest hierarchy carries +// no clickable flag, so tappability is derived from the element type that +// XCUITest itself derives from the accessibility traits. +internal val IOS_CLICKABLE_ELEMENT_TYPES = setOf( + 9, // button + 10, // radioButton + 12, // checkBox + 14, // popUpButton + 15, // comboBox + 16, // menuButton + 17, // toolbarButton + 20, // key + 37, // segmentedControl + 40, // switch + 41, // toggle + 42, // link + 54, // menuItem + 75, // cell + 79, // stepper + 80, // tab +) + +// XCUIElementType raw values that accept text input. +internal val IOS_EDITABLE_ELEMENT_TYPES = setOf( + 45, // searchField + 49, // textField + 50, // secureTextField + 52, // textView +) + +// XCUIElementType raw values that scroll. +internal val IOS_SCROLLABLE_ELEMENT_TYPES = setOf( + 26, // table + 32, // collectionView + 46, // scrollView + 58, // webView +) + +// Checkable element types and the value convention ("1" when on) mirror the +// upstream content-descriptor mapping. +internal val IOS_CHECKABLE_ELEMENT_TYPES = setOf( + 12, // checkBox + 40, // switch + 41, // toggle +) + +private val IOS_ELEMENT_TYPE_NAMES = arrayOf( + "Any", "Other", "Application", "Group", "Window", "Sheet", "Drawer", + "Alert", "Dialog", "Button", "RadioButton", "RadioGroup", "CheckBox", + "DisclosureTriangle", "PopUpButton", "ComboBox", "MenuButton", + "ToolbarButton", "Popover", "Keyboard", "Key", "NavigationBar", "TabBar", + "TabGroup", "Toolbar", "StatusBar", "Table", "TableRow", "TableColumn", + "Outline", "OutlineRow", "Browser", "CollectionView", "Slider", + "PageIndicator", "ProgressIndicator", "ActivityIndicator", + "SegmentedControl", "Picker", "PickerWheel", "Switch", "Toggle", "Link", + "Image", "Icon", "SearchField", "ScrollView", "ScrollBar", "StaticText", + "TextField", "SecureTextField", "DatePicker", "TextView", "Menu", + "MenuItem", "MenuBar", "MenuBarItem", "Map", "WebView", "IncrementArrow", + "DecrementArrow", "Timeline", "RatingIndicator", "ValueIndicator", + "SplitGroup", "Splitter", "RelevanceIndicator", "ColorWell", "HelpTag", + "Matte", "DockItem", "Ruler", "RulerMarker", "Grid", "LevelIndicator", + "Cell", "LayoutArea", "LayoutItem", "Handle", "Stepper", "Tab", + "TouchBar", "StatusItem", +) + +internal fun iosElementTypeName(elementType: Int): String = + IOS_ELEMENT_TYPE_NAMES.getOrElse(elementType) { "Other" } + +// iosAxElementToTreeNode maps the raw XCTest accessibility tree to the +// TreeNode JSON shape the runner parses. The upstream content-descriptor +// mapping drops the element type, leaving no way to tell buttons and text +// fields apart, so fuzz verbs found no tap or typing candidates on iOS. +internal fun iosAxElementToTreeNode(element: hierarchy.AXElement): Map { + val elementType = element.elementType + val title = element.title.orEmpty() + val value = element.value.orEmpty() + val checked = elementType in IOS_CHECKABLE_ELEMENT_TYPES && value == "1" + val attributes = linkedMapOf( + "accessibilityText" to element.label.orEmpty(), + "title" to title, + "value" to value, + "text" to title.ifEmpty { value }, + "hintText" to element.placeholderValue.orEmpty(), + "resource-id" to element.identifier.orEmpty(), + "bounds" to element.frame.boundsString, + "class" to iosElementTypeName(elementType), + "enabled" to element.enabled.toString(), + "focused" to element.hasFocus.toString(), + "selected" to element.selected.toString(), + "checked" to checked.toString(), + "scrollable" to (elementType in IOS_SCROLLABLE_ELEMENT_TYPES).toString(), + ) + return linkedMapOf( + "attributes" to attributes, + "children" to element.children.map { iosAxElementToTreeNode(it) }, + "clickable" to (elementType in IOS_CLICKABLE_ELEMENT_TYPES), + "enabled" to element.enabled, + "focused" to element.hasFocus, + "checked" to checked, + "selected" to element.selected, + "editable" to (elementType in IOS_EDITABLE_ELEMENT_TYPES), + ) +} diff --git a/sidecar/src/test/kotlin/dev/sanderling/sidecar/IosHierarchyTest.kt b/sidecar/src/test/kotlin/dev/sanderling/sidecar/IosHierarchyTest.kt new file mode 100644 index 0000000..fad88eb --- /dev/null +++ b/sidecar/src/test/kotlin/dev/sanderling/sidecar/IosHierarchyTest.kt @@ -0,0 +1,101 @@ +package dev.sanderling.sidecar + +import hierarchy.AXElement +import hierarchy.AXFrame +import org.junit.Test +import kotlin.test.assertEquals + +class IosHierarchyTest { + + private fun element( + elementType: Int, + label: String = "", + identifier: String = "", + title: String = "", + value: String = "", + placeholderValue: String = "", + children: ArrayList = arrayListOf(), + ) = AXElement( + label, + elementType, + identifier, + 0, + 0L, + 0, + false, + 0, + false, + placeholderValue, + value, + AXFrame(10f, 20f, 100f, 50f), + true, + title, + children, + ) + + @Test + fun buttonIsClickableNotEditable() { + val node = iosAxElementToTreeNode(element(elementType = 9, label = "Sign in", identifier = "LoginSubmit")) + assertEquals(true, node["clickable"]) + assertEquals(false, node["editable"]) + @Suppress("UNCHECKED_CAST") + val attributes = node["attributes"] as Map + assertEquals("Button", attributes["class"]) + assertEquals("LoginSubmit", attributes["resource-id"]) + assertEquals("Sign in", attributes["accessibilityText"]) + } + + @Test + fun textFieldIsEditableNotClickable() { + val node = iosAxElementToTreeNode(element(elementType = 49, value = "demo@folio.app")) + assertEquals(false, node["clickable"]) + assertEquals(true, node["editable"]) + @Suppress("UNCHECKED_CAST") + val attributes = node["attributes"] as Map + assertEquals("TextField", attributes["class"]) + assertEquals("demo@folio.app", attributes["text"]) + assertEquals("demo@folio.app", attributes["value"]) + } + + @Test + fun titleWinsOverValueForText() { + val node = iosAxElementToTreeNode(element(elementType = 48, title = "Heading", value = "ignored")) + @Suppress("UNCHECKED_CAST") + val attributes = node["attributes"] as Map + assertEquals("Heading", attributes["text"]) + assertEquals("StaticText", attributes["class"]) + } + + @Test + fun switchOnIsChecked() { + val node = iosAxElementToTreeNode(element(elementType = 40, value = "1")) + assertEquals(true, node["checked"]) + assertEquals(true, node["clickable"]) + } + + @Test + fun scrollViewIsScrollable() { + val node = iosAxElementToTreeNode(element(elementType = 46)) + @Suppress("UNCHECKED_CAST") + val attributes = node["attributes"] as Map + assertEquals("true", attributes["scrollable"]) + assertEquals("ScrollView", attributes["class"]) + } + + @Test + fun childrenAreMappedRecursively() { + val child = element(elementType = 48, title = "Inner") + val node = iosAxElementToTreeNode(element(elementType = 1, children = arrayListOf(child))) + @Suppress("UNCHECKED_CAST") + val children = node["children"] as List> + assertEquals(1, children.size) + @Suppress("UNCHECKED_CAST") + val childAttributes = children[0]["attributes"] as Map + assertEquals("Inner", childAttributes["text"]) + } + + @Test + fun unknownElementTypeFallsBackToOther() { + assertEquals("Other", iosElementTypeName(999)) + } +}