feat(exploration-reach): count the distinct structural states a stored run visited

the state is the settle path's structural hash of the recorded hierarchy, the same function the drivers wait on, so a state boundary here is the one the harness itself uses. --reference reports the observation at which two runs' hierarchies first differ. trace only: no device, no replay.

cmd/internal-tools/exploration-reach/main.go

@@ -0,0 +1,145 @@
+// Command exploration-reach counts the distinct structural states a stored
+// run visited, and compares two runs by the observation at which their
+// hierarchies first differ. Both read the trace alone: no device, no replay.
+//
+// The state is the settle path's structural hash of the recorded hierarchy,
+// the same function the drivers wait on, so a state boundary here is the state
+// boundary the harness itself uses.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"text/tabwriter"
+
+	"github.com/priyanshujain/sanderling/internal/tracecorpus"
+)
+
+func main() {
+	jsonOut := flag.Bool("json", false, "emit JSON instead of a table")
+	reference := flag.String(
+		"reference",
+		"",
+		"run directory to compare against; reports where each other run's hierarchy first differs",
+	)
+	flag.Usage = func() {
+		fmt.Fprintln(
+			os.Stderr,
+			"usage: exploration-reach [--json] [--reference RUN] <run directory> ...",
+		)
+	}
+	flag.Parse()
+	if flag.NArg() == 0 {
+		flag.Usage()
+		os.Exit(2)
+	}
+
+	reaches, err := measureAll(flag.Args())
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	if len(reaches) == 0 {
+		fmt.Fprintln(os.Stderr, "no run directory found under the given paths")
+		os.Exit(1)
+	}
+
+	report := Report{Runs: reaches, CorpusDistinct: corpusDistinct(reaches)}
+	if *reference != "" {
+		base, loadErr := load(*reference)
+		if loadErr != nil {
+			fmt.Fprintf(os.Stderr, "%s: %v\n", *reference, loadErr)
+			os.Exit(1)
+		}
+		report.Reference = base.Directory
+		for _, reach := range reaches {
+			if reach.Directory == base.Directory {
+				continue
+			}
+			report.Divergences = append(report.Divergences, diverge(base, reach))
+		}
+		report.MedianDivergence, report.Censored = medianDivergence(report.Divergences)
+	}
+
+	if *jsonOut {
+		encoder := json.NewEncoder(os.Stdout)
+		encoder.SetIndent("", "  ")
+		if err := encoder.Encode(report); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+		return
+	}
+	render(os.Stdout, report)
+}
+
+// Report is one invocation's output: reach per run and over the corpus, plus
+// the divergence rows when a reference run was named.
+type Report struct {
+	Runs             []Reach      `json:"runs"`
+	CorpusDistinct   int          `json:"corpus_distinct_states"`
+	Reference        string       `json:"reference,omitempty"`
+	Divergences      []Divergence `json:"divergences,omitempty"`
+	MedianDivergence float64      `json:"median_divergence_index,omitempty"`
+	Censored         int          `json:"never_diverged,omitempty"`
+}
+
+func load(path string) (Reach, error) {
+	run, err := tracecorpus.Load(path)
+	if err != nil {
+		return Reach{}, err
+	}
+	return measure(run), nil
+}
+
+func measureAll(paths []string) ([]Reach, error) {
+	var reaches []Reach
+	for _, path := range paths {
+		directories, err := tracecorpus.Discover(path)
+		if err != nil {
+			return nil, err
+		}
+		for _, directory := range directories {
+			reach, err := load(directory)
+			if err != nil {
+				return nil, fmt.Errorf("%s: %w", directory, err)
+			}
+			reaches = append(reaches, reach)
+		}
+	}
+	return reaches, nil
+}
+
+func render(out io.Writer, report Report) {
+	writer := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(writer, "run\tseed\tplatform\tobservations\tdistinct states")
+	for _, reach := range report.Runs {
+		fmt.Fprintf(writer, "%s\t%d\t%s\t%d\t%d\n",
+			reach.Directory, reach.Seed, reach.Platform,
+			reach.Observations, reach.Distinct)
+	}
+	writer.Flush()
+	fmt.Fprintf(out, "\n%d run(s), %d distinct structural states across the corpus\n",
+		len(report.Runs), report.CorpusDistinct)
+
+	if report.Reference == "" {
+		return
+	}
+	fmt.Fprintf(out, "\nreference: %s\n", report.Reference)
+	writer = tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(writer, "run\tseed\tfirst divergence\tobservations compared")
+	for _, divergence := range report.Divergences {
+		where := fmt.Sprintf("%d", divergence.Step)
+		if !divergence.Diverged {
+			where = fmt.Sprintf("none through %d", divergence.Step)
+		}
+		fmt.Fprintf(writer, "%s\t%d\t%s\t%d\n",
+			divergence.Directory, divergence.Seed, where, divergence.Compared)
+	}
+	writer.Flush()
+	fmt.Fprintf(out, "\nmedian first divergence %.1f over %d run(s), %d never diverged\n",
+		report.MedianDivergence, len(report.Divergences), report.Censored)
+}

cmd/internal-tools/exploration-reach/reach.go

@@ -0,0 +1,133 @@
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"sort"
+
+	"github.com/priyanshujain/sanderling/internal/driver/ioscompanion"
+	"github.com/priyanshujain/sanderling/internal/tracecorpus"
+)
+
+// observation is one hierarchy-bearing step: the index the run gave it and the
+// state it observed.
+type observation struct {
+	Step  int
+	State string
+}
+
+// Reach is what one run explored. Distinct counts the structural states its
+// observations visited, which is the measure; Observations is how many looks
+// it took to visit them.
+type Reach struct {
+	Directory    string `json:"directory"`
+	Seed         int64  `json:"seed"`
+	Platform     string `json:"platform"`
+	Arm          string `json:"arm,omitempty"`
+	Observations int    `json:"observations"`
+	Distinct     int    `json:"distinct_states"`
+	// Unobserved counts steps carrying no hierarchy, which the run-end
+	// finalize record is, so an observation count cannot be read as a step
+	// count by accident.
+	Unobserved   int `json:"steps_without_hierarchy"`
+	observations []observation
+}
+
+// measure keys each observation by a digest of its structural hash. The
+// measure is equality between hashes and nothing else, and a corpus holds
+// thousands of trees whose hashes run to tens of kilobytes each.
+func measure(run tracecorpus.Run) Reach {
+	reach := Reach{
+		Directory: run.Directory,
+		Seed:      run.Meta.Seed,
+		Platform:  run.Meta.Platform,
+		Arm:       run.Meta.Arm,
+	}
+	distinct := map[string]bool{}
+	for _, step := range run.Steps {
+		if step.Hierarchy == nil {
+			reach.Unobserved++
+			continue
+		}
+		digest := sha256.Sum256([]byte(ioscompanion.StructuralHash(step.Hierarchy)))
+		key := hex.EncodeToString(digest[:])
+		reach.observations = append(
+			reach.observations,
+			observation{Step: step.Index, State: key},
+		)
+		distinct[key] = true
+	}
+	reach.Observations = len(reach.observations)
+	reach.Distinct = len(distinct)
+	return reach
+}
+
+// corpusDistinct counts the structural states the whole corpus reached, which
+// is not the sum of the per-run counts: runs of one application revisit the
+// same screens.
+func corpusDistinct(reaches []Reach) int {
+	distinct := map[string]bool{}
+	for _, reach := range reaches {
+		for _, seen := range reach.observations {
+			distinct[seen.State] = true
+		}
+	}
+	return len(distinct)
+}
+
+// Divergence is where a replay stopped observing what the reference observed.
+// Step is the index of the first observation whose structural state differs;
+// Diverged is false when the replay matched the reference for every
+// observation the two share, in which case Step is that shared length and the
+// observation is right-censored.
+type Divergence struct {
+	Directory string `json:"directory"`
+	Seed      int64  `json:"seed"`
+	Step      int    `json:"step"`
+	Diverged  bool   `json:"diverged"`
+	Compared  int    `json:"observations_compared"`
+}
+
+func diverge(reference, replay Reach) Divergence {
+	result := Divergence{Directory: replay.Directory, Seed: replay.Seed}
+	shared := len(reference.observations)
+	if len(replay.observations) < shared {
+		shared = len(replay.observations)
+	}
+	result.Compared = shared
+	for position := 0; position < shared; position++ {
+		if reference.observations[position].State != replay.observations[position].State {
+			result.Step = reference.observations[position].Step
+			result.Diverged = true
+			return result
+		}
+	}
+	if shared > 0 {
+		result.Step = reference.observations[shared-1].Step
+	}
+	return result
+}
+
+// medianDivergence is E6's number: the median observation index at which a
+// replay first diverges from the reference. A replay that never diverged
+// enters at the last index the two share, which is where the observation is
+// censored rather than where it broke, so the count of such replays is
+// reported beside the median rather than folded into it.
+func medianDivergence(divergences []Divergence) (median float64, censored int) {
+	if len(divergences) == 0 {
+		return 0, 0
+	}
+	steps := make([]int, 0, len(divergences))
+	for _, divergence := range divergences {
+		steps = append(steps, divergence.Step)
+		if !divergence.Diverged {
+			censored++
+		}
+	}
+	sort.Ints(steps)
+	middle := len(steps) / 2
+	if len(steps)%2 == 1 {
+		return float64(steps[middle]), censored
+	}
+	return float64(steps[middle-1]+steps[middle]) / 2, censored
+}

cmd/internal-tools/exploration-reach/reach_test.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/priyanshujain/sanderling/internal/driver/ioscompanion"
+	"github.com/priyanshujain/sanderling/internal/hierarchy"
+	"github.com/priyanshujain/sanderling/internal/trace"
+	"github.com/priyanshujain/sanderling/internal/tracecorpus"
+)
+
+const (
+	home = `{"attributes": {"text": "Home", "bounds": "[0,0,10,10]"}, "children": [
+    {"attributes": {"text": "row", "bounds": "[0,0,5,5]"}, "children": []}
+  ]}`
+	homeScrolled = `{"attributes": {"text": "Home", "bounds": "[0,4,10,14]"}, "children": [
+    {"attributes": {"text": "row", "bounds": "[0,4,5,9]"}, "children": []}
+  ]}`
+	ledger = `{"attributes": {"text": "Ledger", "bounds": "[0,0,10,10]"}, "children": [
+    {"attributes": {"text": "row", "bounds": "[0,0,5,5]"}, "children": []}
+  ]}`
+	ledgerWithRow = `{"attributes": {"text": "Ledger", "bounds": "[0,0,10,10]"}, "children": [
+    {"attributes": {"text": "row", "bounds": "[0,0,5,5]"}, "children": []},
+    {"attributes": {"text": "row"}, "children": []}
+  ]}`
+)
+
+// TestReachCountsStructuresNotObservations: five observations of three
+// structures, one of them revisited and one differing only in where it sits on
+// screen, so the answer is three by construction.
+func TestReachCountsStructuresNotObservations(t *testing.T) {
+	reach := measureRun(t, 7, home, homeScrolled, ledger, home, ledgerWithRow)
+
+	if reach.Observations != 5 {
+		t.Fatalf("observations = %d, want 5", reach.Observations)
+	}
+	if reach.Distinct != 3 {
+		t.Fatalf("distinct states = %d, want 3", reach.Distinct)
+	}
+}
+
+func TestFinalizeRecordIsNoObservation(t *testing.T) {
+	directory := writeRun(t, 7, home, ledger)
+	appendFinalize(t, directory, 3)
+
+	run, err := tracecorpus.Load(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	reach := measure(run)
+	if reach.Observations != 2 || reach.Unobserved != 1 {
+		t.Fatalf("observations = %d, unobserved = %d, want 2 and 1",
+			reach.Observations, reach.Unobserved)
+	}
+}
+
+// TestStoredTreeHashesAsTheLiveTreeDid is the equivalence the whole measure
+// rests on: the hash the settle path computed from the live tree and the hash
+// this tool computes from the stored one are the same string, so a reach
+// number counts the same state boundaries the drivers wait on.
+func TestStoredTreeHashesAsTheLiveTreeDid(t *testing.T) {
+	live, err := hierarchy.Parse(home)
+	if err != nil {
+		t.Fatal(err)
+	}
+	liveHash := ioscompanion.StructuralHash(live)
+	if liveHash == "" {
+		t.Fatal("live hash is empty, so the test would pass on any stored tree")
+	}
+
+	run, err := tracecorpus.Load(writeRun(t, 7, home))
+	if err != nil {
+		t.Fatal(err)
+	}
+	stored := ioscompanion.StructuralHash(run.Steps[0].Hierarchy)
+	if stored != liveHash {
+		t.Fatalf("stored hash differs from the live one:\n live=%q\n stored=%q",
+			liveHash, stored)
+	}
+}
+
+func TestFirstDivergenceNamesTheObservationThatDiffers(t *testing.T) {
+	reference := measureRun(t, 7, home, ledger, home, ledger)
+	replay := measureRun(t, 7, home, ledger, ledgerWithRow, ledger)
+
+	divergence := diverge(reference, replay)
+	if !divergence.Diverged || divergence.Step != 3 {
+		t.Fatalf("divergence = %+v, want the third observation", divergence)
+	}
+}
+
+func TestAReplayThatMatchesIsCensoredAtTheSharedLength(t *testing.T) {
+	reference := measureRun(t, 7, home, ledger, home, ledger)
+	replay := measureRun(t, 7, home, ledger, home)
+
+	divergence := diverge(reference, replay)
+	if divergence.Diverged {
+		t.Fatalf("identical observations must not report a divergence: %+v", divergence)
+	}
+	if divergence.Step != 3 || divergence.Compared != 3 {
+		t.Fatalf("divergence = %+v, want censoring at the third observation", divergence)
+	}
+}
+
+func TestMedianDivergenceHoldsCensoredReplaysApart(t *testing.T) {
+	median, censored := medianDivergence([]Divergence{
+		{Step: 9, Diverged: true},
+		{Step: 3, Diverged: true},
+		{Step: 40, Diverged: false},
+	})
+	if median != 9 {
+		t.Fatalf("median = %v, want 9", median)
+	}
+	if censored != 1 {
+		t.Fatalf("censored = %d, want 1", censored)
+	}
+}
+
+func TestCorpusStatesAreTheUnionNotTheSum(t *testing.T) {
+	first := measureRun(t, 7, home, ledger)
+	second := measureRun(t, 11, ledger, ledgerWithRow)
+
+	if got := corpusDistinct([]Reach{first, second}); got != 3 {
+		t.Fatalf("corpus distinct = %d, want 3", got)
+	}
+}
+
+func measureRun(t *testing.T, seed int64, dumps ...string) Reach {
+	t.Helper()
+	run, err := tracecorpus.Load(writeRun(t, seed, dumps...))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return measure(run)
+}
+
+func writeRun(t *testing.T, seed int64, dumps ...string) string {
+	t.Helper()
+	directory := t.TempDir()
+	writer, err := trace.NewWriter(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.WriteMeta(trace.Meta{Seed: seed, Platform: "web"}); err != nil {
+		t.Fatal(err)
+	}
+	for index, dump := range dumps {
+		tree, err := hierarchy.Parse(dump)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := writer.WriteStep(trace.Step{Index: index + 1, Hierarchy: tree}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return directory
+}
+
+func appendFinalize(t *testing.T, directory string, index int) {
+	t.Helper()
+	writer, err := trace.NewWriter(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.WriteStep(trace.Step{
+		Index:      index,
+		Violations: []string{"someTransactionExists"},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatal(err)
+	}
+}