feat(hierarchy): store the tree shape and tolerate an unreadable boolean flag

a Tree marshalled to json kept only the flat element array, so a stored tree decoded with a nil Root and resolved no selector. it now stores each element's pre-order depth and rebuilds Root from it, re-seating elements so Tree.Elements and &node.Element stay one pointer. a stored tree without depths keeps the old shape. a boolean field the producer sent as something other than a boolean now leaves the flag unset and increments UnreadableFlags rather than failing the whole dump.

internal/hierarchy/hierarchy.go

@@ -85,18 +85,148 @@ type Node struct {
 type Tree struct {
 	Root     *Node      `json:"-"`
 	Elements []*Element `json:"elements"`
+	// UnreadableFlags counts the boolean fields the producer sent as something
+	// other than a boolean. They are dropped rather than failing the dump, so
+	// the count is what keeps the drop from being silent.
+	UnreadableFlags int `json:"unreadableFlags,omitempty"`
+}
+
+// treeJSON is the stored form of a Tree. `depths` is the pre-order depth of
+// each element, which is what turns the flat array back into Root: a stored
+// tree without it (every trace written before the field existed) decodes with
+// a nil Root and resolves no selector, exactly as it did before.
+//
+// A depth per element rather than a parent index per element: the numbers are
+// one digit deep into most hierarchies where a parent index is three, and a
+// step already costs 86 KB on android.
+type treeJSON struct {
+	Elements        []*Element `json:"elements"`
+	Depths          []int      `json:"depths,omitempty"`
+	UnreadableFlags int        `json:"unreadable_flags,omitempty"`
+}
+
+func (t Tree) MarshalJSON() ([]byte, error) {
+	return json.Marshal(treeJSON{
+		Elements:        t.Elements,
+		Depths:          t.depths(),
+		UnreadableFlags: t.UnreadableFlags,
+	})
+}
+
+// depths walks Root, and yields nothing unless the walk covers exactly the
+// elements the flat array holds: a hand-built Tree whose Root and Elements
+// disagree would otherwise store a shape that rebuilds into a different tree.
+func (t Tree) depths() []int {
+	if t.Root == nil {
+		return nil
+	}
+	depths := make([]int, 0, len(t.Elements))
+	var walk func(node *Node, depth int)
+	walk = func(node *Node, depth int) {
+		depths = append(depths, depth)
+		for _, child := range node.Children {
+			walk(child, depth+1)
+		}
+	}
+	walk(t.Root, 0)
+	if len(depths) != len(t.Elements) {
+		return nil
+	}
+	return depths
+}
+
+func (t *Tree) UnmarshalJSON(data []byte) error {
+	var stored treeJSON
+	if err := json.Unmarshal(data, &stored); err != nil {
+		return err
+	}
+	t.Elements = stored.Elements
+	t.UnreadableFlags = stored.UnreadableFlags
+	t.Root = t.rebuild(stored.Depths)
+	return nil
+}
+
+// rebuild re-parents the flat pre-order array from the stored depths. Every
+// element is re-seated inside its Node so Tree.Elements and &node.Element stay
+// the same pointer, which is the identity the verifier's element scope and the
+// picker's target list are keyed on.
+func (t *Tree) rebuild(depths []int) *Node {
+	if !wellFormedDepths(depths, len(t.Elements)) {
+		return nil
+	}
+	stack := make([]*Node, 0, 32)
+	for index, depth := range depths {
+		node := &Node{Element: *t.Elements[index], tree: t}
+		t.Elements[index] = &node.Element
+		stack = stack[:depth]
+		if depth > 0 {
+			parent := stack[depth-1]
+			parent.Children = append(parent.Children, node)
+		}
+		stack = append(stack, node)
+	}
+	return stack[0]
+}
+
+// wellFormedDepths accepts only a single-rooted pre-order sequence: one root at
+// the head and no child deeper than one level below its predecessor.
+func wellFormedDepths(depths []int, elementCount int) bool {
+	if len(depths) == 0 || len(depths) != elementCount || depths[0] != 0 {
+		return false
+	}
+	for index := 1; index < len(depths); index++ {
+		if depths[index] < 1 || depths[index] > depths[index-1]+1 {
+			return false
+		}
+	}
+	return true
 }
 
 // treeNodeJSON mirrors the sidecar TreeNode JSON structure.
 type treeNodeJSON struct {
 	Attributes map[string]string `json:"attributes"`
 	Children   []treeNodeJSON    `json:"children"`
-	Clickable  *bool             `json:"clickable"`
+	Clickable  flagJSON          `json:"clickable"`
-	Enabled    *bool             `json:"enabled"`
+	Enabled    flagJSON          `json:"enabled"`
-	Focused    *bool             `json:"focused"`
+	Focused    flagJSON          `json:"focused"`
-	Checked    *bool             `json:"checked"`
+	Checked    flagJSON          `json:"checked"`
-	Selected   *bool             `json:"selected"`
+	Selected   flagJSON          `json:"selected"`
-	Editable   *bool             `json:"editable"`
+	Editable   flagJSON          `json:"editable"`
+}
+
+// flagJSON is one boolean field of a node. A value that is not a boolean
+// leaves the flag unset and marks itself unreadable rather than failing the
+// document: a dump is one observation of a whole screen, and one node's bit is
+// no reason to discard every element on it. Malformed bounds are already
+// treated this way.
+type flagJSON struct {
+	set        bool
+	value      bool
+	unreadable bool
+}
+
+func (f *flagJSON) UnmarshalJSON(data []byte) error {
+	if string(data) == "null" {
+		return nil
+	}
+	var value bool
+	if err := json.Unmarshal(data, &value); err != nil {
+		f.unreadable = true
+		return nil
+	}
+	f.set = true
+	f.value = value
+	return nil
+}
+
+func (n *treeNodeJSON) unreadableFlags() int {
+	count := 0
+	for _, flag := range []flagJSON{n.Clickable, n.Enabled, n.Focused, n.Checked, n.Selected, n.Editable} {
+		if flag.unreadable {
+			count++
+		}
+	}
+	return count
 }
 
 // Selector describes a multi-attribute AND match.
@@ -381,6 +511,7 @@ func Parse(text string) (*Tree, error) {
 }
 
 func walkNode(node *treeNodeJSON, tree *Tree) *Node {
+	tree.UnreadableFlags += node.unreadableFlags()
 	n := &Node{Element: *elementFromNode(node), tree: tree}
 	tree.Elements = append(tree.Elements, &n.Element)
 	for i := range node.Children {
@@ -421,23 +552,23 @@ func elementFromNode(node *treeNodeJSON) *Element {
 	}
 	element.Screen = attrs["sanderling-screen"]
 
-	if node.Clickable != nil {
+	if node.Clickable.set {
-		element.Clickable = *node.Clickable
+		element.Clickable = node.Clickable.value
 	}
-	if node.Enabled != nil {
+	if node.Enabled.set {
-		element.Enabled = *node.Enabled
+		element.Enabled = node.Enabled.value
 	}
-	if node.Focused != nil {
+	if node.Focused.set {
-		element.Focused = *node.Focused
+		element.Focused = node.Focused.value
 	}
-	if node.Checked != nil {
+	if node.Checked.set {
-		element.Checked = *node.Checked
+		element.Checked = node.Checked.value
 	}
-	if node.Selected != nil {
+	if node.Selected.set {
-		element.Selected = *node.Selected
+		element.Selected = node.Selected.value
 	}
-	if node.Editable != nil {
+	if node.Editable.set {
-		element.Editable = *node.Editable
+		element.Editable = node.Editable.value
 	} else {
 		element.Editable = strings.Contains(element.Class, "EditText") || attrs["hintText"] != ""
 	}
@@ -451,20 +582,20 @@ func elementFromNode(node *treeNodeJSON) *Element {
 
 	element.Attributes = make(map[string]string, len(attrs)+5)
 	maps.Copy(element.Attributes, attrs)
-	if node.Clickable != nil {
+	if node.Clickable.set {
-		element.Attributes["clickable"] = strconv.FormatBool(*node.Clickable)
+		element.Attributes["clickable"] = strconv.FormatBool(node.Clickable.value)
 	}
-	if node.Enabled != nil {
+	if node.Enabled.set {
-		element.Attributes["enabled"] = strconv.FormatBool(*node.Enabled)
+		element.Attributes["enabled"] = strconv.FormatBool(node.Enabled.value)
 	}
-	if node.Focused != nil {
+	if node.Focused.set {
-		element.Attributes["focused"] = strconv.FormatBool(*node.Focused)
+		element.Attributes["focused"] = strconv.FormatBool(node.Focused.value)
 	}
-	if node.Checked != nil {
+	if node.Checked.set {
-		element.Attributes["checked"] = strconv.FormatBool(*node.Checked)
+		element.Attributes["checked"] = strconv.FormatBool(node.Checked.value)
 	}
-	if node.Selected != nil {
+	if node.Selected.set {
-		element.Attributes["selected"] = strconv.FormatBool(*node.Selected)
+		element.Attributes["selected"] = strconv.FormatBool(node.Selected.value)
 	}
 	element.Attributes["editable"] = strconv.FormatBool(element.Editable)
 

internal/hierarchy/hierarchy_test.go

@@ -1,6 +1,7 @@
 package hierarchy
 
 import (
+	"encoding/json"
 	"slices"
 	"strings"
 	"testing"
@@ -1301,3 +1302,172 @@ func TestRawDriverAttributeIsNotUnknown(t *testing.T) {
 		t.Fatalf("got %v, want none", unknown)
 	}
 }
+
+// TestStoredTreeRebuildsRootAndResolvesSelectors is the offline half of every
+// trace: a tree that only survives as a flat pre-order array resolves nothing,
+// because every lookup walks Root.
+func TestStoredTreeRebuildsRootAndResolvesSelectors(t *testing.T) {
+	tree, err := Parse(sampleDump)
+	if err != nil {
+		t.Fatal(err)
+	}
+	stored, err := json.Marshal(tree)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var decoded Tree
+	if err := json.Unmarshal(stored, &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if decoded.Root == nil {
+		t.Fatal("Root not rebuilt from the stored tree")
+	}
+	if len(decoded.Elements) != len(tree.Elements) {
+		t.Fatalf(
+			"elements = %d, want %d",
+			len(decoded.Elements),
+			len(tree.Elements),
+		)
+	}
+	for index, element := range decoded.Elements {
+		if element.ResourceID != tree.Elements[index].ResourceID {
+			t.Fatalf(
+				"element %d = %q, want %q",
+				index,
+				element.ResourceID,
+				tree.Elements[index].ResourceID,
+			)
+		}
+	}
+	online := tree.Find("id:app:id/title")
+	offline := decoded.Find("id:app:id/title")
+	if online == nil {
+		t.Fatal("the selector does not resolve online; the fixture is wrong")
+	}
+	if offline == nil || offline.Text != online.Text {
+		t.Fatalf(
+			"selector resolves online to %+v, offline to %+v",
+			online,
+			offline,
+		)
+	}
+	if decoded.Find("id:app:id/title") != decoded.Elements[1] {
+		t.Error(
+			"the rebuilt nodes and the decoded element list are different pointers",
+		)
+	}
+	childCount := 0
+	var walk func(node *Node)
+	walk = func(node *Node) {
+		childCount++
+		for _, child := range node.Children {
+			walk(child)
+		}
+	}
+	walk(decoded.Root)
+	if childCount != len(decoded.Elements) {
+		t.Errorf(
+			"the rebuilt tree holds %d nodes, the element list %d",
+			childCount,
+			len(decoded.Elements),
+		)
+	}
+}
+
+// TestStoredTreeWithoutDepthsKeepsTheOldShape: traces written before the
+// depths field must still load, and must say "no structure" rather than
+// inventing one.
+func TestStoredTreeWithoutDepthsKeepsTheOldShape(t *testing.T) {
+	var decoded Tree
+	if err := json.Unmarshal([]byte(`{"elements":[{"resourceId":"root"},{"resourceId":"child"}]}`), &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if len(decoded.Elements) != 2 {
+		t.Fatalf("elements = %d, want 2", len(decoded.Elements))
+	}
+	if decoded.Root != nil {
+		t.Errorf(
+			"Root = %+v, want nil for a stored tree that carries no depths",
+			decoded.Root,
+		)
+	}
+}
+
+func TestStoredTreeRejectsDepthsItCannotRebuild(t *testing.T) {
+	for name, stored := range map[string]string{
+		"count mismatch": `{"elements":[{"resourceId":"a"},{"resourceId":"b"}],"depths":[0]}`,
+		"rootless":       `{"elements":[{"resourceId":"a"}],"depths":[1]}`,
+		"second root":    `{"elements":[{"resourceId":"a"},{"resourceId":"b"}],"depths":[0,0]}`,
+		"skipped level":  `{"elements":[{"resourceId":"a"},{"resourceId":"b"}],"depths":[0,2]}`,
+	} {
+		var decoded Tree
+		if err := json.Unmarshal([]byte(stored), &decoded); err != nil {
+			t.Fatalf("%s: %v", name, err)
+		}
+		if decoded.Root != nil {
+			t.Errorf("%s: Root = %+v, want nil", name, decoded.Root)
+		}
+	}
+}
+
+// TestTreeStoresNoDepthsForAnUnwalkableRoot keeps the stored shape honest:
+// a hand-built Tree whose Root and Elements disagree must not claim a
+// structure that would rebuild into a different tree.
+func TestTreeStoresNoDepthsForAnUnwalkableRoot(t *testing.T) {
+	tree := &Tree{
+		Root:     &Node{Element: Element{ResourceID: "root"}},
+		Elements: []*Element{{ResourceID: "root"}, {ResourceID: "orphan"}},
+	}
+	stored, err := json.Marshal(tree)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(string(stored), "depths") {
+		t.Errorf("stored a shape that cannot be rebuilt: %s", stored)
+	}
+}
+
+// A producer that puts a string where a flag belongs must cost that flag and
+// nothing else. Failing the document instead blanks the whole tree, and every
+// extractor then reads a screen with no elements on it.
+func TestParseUnreadableFlagKeepsTheRestOfTheTree(t *testing.T) {
+	input := `{"attributes":{"resource-id":"app:id/root","bounds":"[0,0,400,800]"},"children":[
+		{"attributes":{"resource-id":"app:id/tabs","text":"keep"},"selected":"active","children":[]},
+		{"attributes":{"resource-id":"app:id/leaf"},"clickable":true,"children":[]}
+	]}`
+	tree, err := Parse(input)
+	if err != nil {
+		t.Fatalf("Parse: %v", err)
+	}
+	if len(tree.Elements) != 3 {
+		t.Fatalf("elements = %d, want 3: one unreadable flag must not cost the tree", len(tree.Elements))
+	}
+	tabs := tree.Find("id:tabs")
+	if tabs == nil {
+		t.Fatal("the element carrying the unreadable flag was dropped")
+	}
+	if tabs.Text != "keep" {
+		t.Errorf("neighbouring field corrupted: text=%q", tabs.Text)
+	}
+	if tabs.Selected {
+		t.Error("selected must stay unset when the value the producer sent is not a boolean")
+	}
+	leaf := tree.Find("id:leaf")
+	if leaf == nil || !leaf.Clickable {
+		t.Errorf("a readable flag elsewhere in the tree must survive, got %+v", leaf)
+	}
+	if tree.UnreadableFlags != 1 {
+		t.Errorf("UnreadableFlags = %d, want 1: a dropped flag has to be countable", tree.UnreadableFlags)
+	}
+	stored, err := json.Marshal(tree)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var decoded Tree
+	if err := json.Unmarshal(stored, &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if decoded.UnreadableFlags != 1 {
+		t.Errorf("stored UnreadableFlags = %d, want 1: the trace has to carry it", decoded.UnreadableFlags)
+	}
+}