From 11fca22d3647d4d8d011cfd5c9041d20205009ee Mon Sep 17 00:00:00 2001 From: PJ Date: Sun, 16 Aug 2026 17:45:35 +0530 Subject: [PATCH] feat(exploration-reach): count the distinct structural states a stored run visited the state is the settle path's structural hash of the recorded hierarchy, the same function the drivers wait on, so a state boundary here is the one the harness itself uses. --reference reports the observation at which two runs' hierarchies first differ. trace only: no device, no replay. --- cmd/internal-tools/exploration-reach/main.go | 145 ++++++++++++++ cmd/internal-tools/exploration-reach/reach.go | 133 +++++++++++++ .../exploration-reach/reach_test.go | 177 ++++++++++++++++++ 3 files changed, 455 insertions(+) create mode 100644 cmd/internal-tools/exploration-reach/main.go create mode 100644 cmd/internal-tools/exploration-reach/reach.go create mode 100644 cmd/internal-tools/exploration-reach/reach_test.go diff --git a/cmd/internal-tools/exploration-reach/main.go b/cmd/internal-tools/exploration-reach/main.go new file mode 100644 index 0000000..a85c76b --- /dev/null +++ b/cmd/internal-tools/exploration-reach/main.go @@ -0,0 +1,145 @@ +// Command exploration-reach counts the distinct structural states a stored +// run visited, and compares two runs by the observation at which their +// hierarchies first differ. Both read the trace alone: no device, no replay. +// +// The state is the settle path's structural hash of the recorded hierarchy, +// the same function the drivers wait on, so a state boundary here is the state +// boundary the harness itself uses. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "os" + "text/tabwriter" + + "github.com/priyanshujain/sanderling/internal/tracecorpus" +) + +func main() { + jsonOut := flag.Bool("json", false, "emit JSON instead of a table") + reference := flag.String( + "reference", + "", + "run directory to compare against; reports where each other run's hierarchy first differs", + ) + flag.Usage = func() { + fmt.Fprintln( + os.Stderr, + "usage: exploration-reach [--json] [--reference RUN] ...", + ) + } + flag.Parse() + if flag.NArg() == 0 { + flag.Usage() + os.Exit(2) + } + + reaches, err := measureAll(flag.Args()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if len(reaches) == 0 { + fmt.Fprintln(os.Stderr, "no run directory found under the given paths") + os.Exit(1) + } + + report := Report{Runs: reaches, CorpusDistinct: corpusDistinct(reaches)} + if *reference != "" { + base, loadErr := load(*reference) + if loadErr != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", *reference, loadErr) + os.Exit(1) + } + report.Reference = base.Directory + for _, reach := range reaches { + if reach.Directory == base.Directory { + continue + } + report.Divergences = append(report.Divergences, diverge(base, reach)) + } + report.MedianDivergence, report.Censored = medianDivergence(report.Divergences) + } + + if *jsonOut { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err := encoder.Encode(report); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + return + } + render(os.Stdout, report) +} + +// Report is one invocation's output: reach per run and over the corpus, plus +// the divergence rows when a reference run was named. +type Report struct { + Runs []Reach `json:"runs"` + CorpusDistinct int `json:"corpus_distinct_states"` + Reference string `json:"reference,omitempty"` + Divergences []Divergence `json:"divergences,omitempty"` + MedianDivergence float64 `json:"median_divergence_index,omitempty"` + Censored int `json:"never_diverged,omitempty"` +} + +func load(path string) (Reach, error) { + run, err := tracecorpus.Load(path) + if err != nil { + return Reach{}, err + } + return measure(run), nil +} + +func measureAll(paths []string) ([]Reach, error) { + var reaches []Reach + for _, path := range paths { + directories, err := tracecorpus.Discover(path) + if err != nil { + return nil, err + } + for _, directory := range directories { + reach, err := load(directory) + if err != nil { + return nil, fmt.Errorf("%s: %w", directory, err) + } + reaches = append(reaches, reach) + } + } + return reaches, nil +} + +func render(out io.Writer, report Report) { + writer := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) + fmt.Fprintln(writer, "run\tseed\tplatform\tobservations\tdistinct states") + for _, reach := range report.Runs { + fmt.Fprintf(writer, "%s\t%d\t%s\t%d\t%d\n", + reach.Directory, reach.Seed, reach.Platform, + reach.Observations, reach.Distinct) + } + writer.Flush() + fmt.Fprintf(out, "\n%d run(s), %d distinct structural states across the corpus\n", + len(report.Runs), report.CorpusDistinct) + + if report.Reference == "" { + return + } + fmt.Fprintf(out, "\nreference: %s\n", report.Reference) + writer = tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) + fmt.Fprintln(writer, "run\tseed\tfirst divergence\tobservations compared") + for _, divergence := range report.Divergences { + where := fmt.Sprintf("%d", divergence.Step) + if !divergence.Diverged { + where = fmt.Sprintf("none through %d", divergence.Step) + } + fmt.Fprintf(writer, "%s\t%d\t%s\t%d\n", + divergence.Directory, divergence.Seed, where, divergence.Compared) + } + writer.Flush() + fmt.Fprintf(out, "\nmedian first divergence %.1f over %d run(s), %d never diverged\n", + report.MedianDivergence, len(report.Divergences), report.Censored) +} diff --git a/cmd/internal-tools/exploration-reach/reach.go b/cmd/internal-tools/exploration-reach/reach.go new file mode 100644 index 0000000..bb72c8c --- /dev/null +++ b/cmd/internal-tools/exploration-reach/reach.go @@ -0,0 +1,133 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "sort" + + "github.com/priyanshujain/sanderling/internal/driver/ioscompanion" + "github.com/priyanshujain/sanderling/internal/tracecorpus" +) + +// observation is one hierarchy-bearing step: the index the run gave it and the +// state it observed. +type observation struct { + Step int + State string +} + +// Reach is what one run explored. Distinct counts the structural states its +// observations visited, which is the measure; Observations is how many looks +// it took to visit them. +type Reach struct { + Directory string `json:"directory"` + Seed int64 `json:"seed"` + Platform string `json:"platform"` + Arm string `json:"arm,omitempty"` + Observations int `json:"observations"` + Distinct int `json:"distinct_states"` + // Unobserved counts steps carrying no hierarchy, which the run-end + // finalize record is, so an observation count cannot be read as a step + // count by accident. + Unobserved int `json:"steps_without_hierarchy"` + observations []observation +} + +// measure keys each observation by a digest of its structural hash. The +// measure is equality between hashes and nothing else, and a corpus holds +// thousands of trees whose hashes run to tens of kilobytes each. +func measure(run tracecorpus.Run) Reach { + reach := Reach{ + Directory: run.Directory, + Seed: run.Meta.Seed, + Platform: run.Meta.Platform, + Arm: run.Meta.Arm, + } + distinct := map[string]bool{} + for _, step := range run.Steps { + if step.Hierarchy == nil { + reach.Unobserved++ + continue + } + digest := sha256.Sum256([]byte(ioscompanion.StructuralHash(step.Hierarchy))) + key := hex.EncodeToString(digest[:]) + reach.observations = append( + reach.observations, + observation{Step: step.Index, State: key}, + ) + distinct[key] = true + } + reach.Observations = len(reach.observations) + reach.Distinct = len(distinct) + return reach +} + +// corpusDistinct counts the structural states the whole corpus reached, which +// is not the sum of the per-run counts: runs of one application revisit the +// same screens. +func corpusDistinct(reaches []Reach) int { + distinct := map[string]bool{} + for _, reach := range reaches { + for _, seen := range reach.observations { + distinct[seen.State] = true + } + } + return len(distinct) +} + +// Divergence is where a replay stopped observing what the reference observed. +// Step is the index of the first observation whose structural state differs; +// Diverged is false when the replay matched the reference for every +// observation the two share, in which case Step is that shared length and the +// observation is right-censored. +type Divergence struct { + Directory string `json:"directory"` + Seed int64 `json:"seed"` + Step int `json:"step"` + Diverged bool `json:"diverged"` + Compared int `json:"observations_compared"` +} + +func diverge(reference, replay Reach) Divergence { + result := Divergence{Directory: replay.Directory, Seed: replay.Seed} + shared := len(reference.observations) + if len(replay.observations) < shared { + shared = len(replay.observations) + } + result.Compared = shared + for position := 0; position < shared; position++ { + if reference.observations[position].State != replay.observations[position].State { + result.Step = reference.observations[position].Step + result.Diverged = true + return result + } + } + if shared > 0 { + result.Step = reference.observations[shared-1].Step + } + return result +} + +// medianDivergence is E6's number: the median observation index at which a +// replay first diverges from the reference. A replay that never diverged +// enters at the last index the two share, which is where the observation is +// censored rather than where it broke, so the count of such replays is +// reported beside the median rather than folded into it. +func medianDivergence(divergences []Divergence) (median float64, censored int) { + if len(divergences) == 0 { + return 0, 0 + } + steps := make([]int, 0, len(divergences)) + for _, divergence := range divergences { + steps = append(steps, divergence.Step) + if !divergence.Diverged { + censored++ + } + } + sort.Ints(steps) + middle := len(steps) / 2 + if len(steps)%2 == 1 { + return float64(steps[middle]), censored + } + return float64(steps[middle-1]+steps[middle]) / 2, censored +} diff --git a/cmd/internal-tools/exploration-reach/reach_test.go b/cmd/internal-tools/exploration-reach/reach_test.go new file mode 100644 index 0000000..5f56331 --- /dev/null +++ b/cmd/internal-tools/exploration-reach/reach_test.go @@ -0,0 +1,177 @@ +package main + +import ( + "testing" + + "github.com/priyanshujain/sanderling/internal/driver/ioscompanion" + "github.com/priyanshujain/sanderling/internal/hierarchy" + "github.com/priyanshujain/sanderling/internal/trace" + "github.com/priyanshujain/sanderling/internal/tracecorpus" +) + +const ( + home = `{"attributes": {"text": "Home", "bounds": "[0,0,10,10]"}, "children": [ + {"attributes": {"text": "row", "bounds": "[0,0,5,5]"}, "children": []} + ]}` + homeScrolled = `{"attributes": {"text": "Home", "bounds": "[0,4,10,14]"}, "children": [ + {"attributes": {"text": "row", "bounds": "[0,4,5,9]"}, "children": []} + ]}` + ledger = `{"attributes": {"text": "Ledger", "bounds": "[0,0,10,10]"}, "children": [ + {"attributes": {"text": "row", "bounds": "[0,0,5,5]"}, "children": []} + ]}` + ledgerWithRow = `{"attributes": {"text": "Ledger", "bounds": "[0,0,10,10]"}, "children": [ + {"attributes": {"text": "row", "bounds": "[0,0,5,5]"}, "children": []}, + {"attributes": {"text": "row"}, "children": []} + ]}` +) + +// TestReachCountsStructuresNotObservations: five observations of three +// structures, one of them revisited and one differing only in where it sits on +// screen, so the answer is three by construction. +func TestReachCountsStructuresNotObservations(t *testing.T) { + reach := measureRun(t, 7, home, homeScrolled, ledger, home, ledgerWithRow) + + if reach.Observations != 5 { + t.Fatalf("observations = %d, want 5", reach.Observations) + } + if reach.Distinct != 3 { + t.Fatalf("distinct states = %d, want 3", reach.Distinct) + } +} + +func TestFinalizeRecordIsNoObservation(t *testing.T) { + directory := writeRun(t, 7, home, ledger) + appendFinalize(t, directory, 3) + + run, err := tracecorpus.Load(directory) + if err != nil { + t.Fatal(err) + } + reach := measure(run) + if reach.Observations != 2 || reach.Unobserved != 1 { + t.Fatalf("observations = %d, unobserved = %d, want 2 and 1", + reach.Observations, reach.Unobserved) + } +} + +// TestStoredTreeHashesAsTheLiveTreeDid is the equivalence the whole measure +// rests on: the hash the settle path computed from the live tree and the hash +// this tool computes from the stored one are the same string, so a reach +// number counts the same state boundaries the drivers wait on. +func TestStoredTreeHashesAsTheLiveTreeDid(t *testing.T) { + live, err := hierarchy.Parse(home) + if err != nil { + t.Fatal(err) + } + liveHash := ioscompanion.StructuralHash(live) + if liveHash == "" { + t.Fatal("live hash is empty, so the test would pass on any stored tree") + } + + run, err := tracecorpus.Load(writeRun(t, 7, home)) + if err != nil { + t.Fatal(err) + } + stored := ioscompanion.StructuralHash(run.Steps[0].Hierarchy) + if stored != liveHash { + t.Fatalf("stored hash differs from the live one:\n live=%q\n stored=%q", + liveHash, stored) + } +} + +func TestFirstDivergenceNamesTheObservationThatDiffers(t *testing.T) { + reference := measureRun(t, 7, home, ledger, home, ledger) + replay := measureRun(t, 7, home, ledger, ledgerWithRow, ledger) + + divergence := diverge(reference, replay) + if !divergence.Diverged || divergence.Step != 3 { + t.Fatalf("divergence = %+v, want the third observation", divergence) + } +} + +func TestAReplayThatMatchesIsCensoredAtTheSharedLength(t *testing.T) { + reference := measureRun(t, 7, home, ledger, home, ledger) + replay := measureRun(t, 7, home, ledger, home) + + divergence := diverge(reference, replay) + if divergence.Diverged { + t.Fatalf("identical observations must not report a divergence: %+v", divergence) + } + if divergence.Step != 3 || divergence.Compared != 3 { + t.Fatalf("divergence = %+v, want censoring at the third observation", divergence) + } +} + +func TestMedianDivergenceHoldsCensoredReplaysApart(t *testing.T) { + median, censored := medianDivergence([]Divergence{ + {Step: 9, Diverged: true}, + {Step: 3, Diverged: true}, + {Step: 40, Diverged: false}, + }) + if median != 9 { + t.Fatalf("median = %v, want 9", median) + } + if censored != 1 { + t.Fatalf("censored = %d, want 1", censored) + } +} + +func TestCorpusStatesAreTheUnionNotTheSum(t *testing.T) { + first := measureRun(t, 7, home, ledger) + second := measureRun(t, 11, ledger, ledgerWithRow) + + if got := corpusDistinct([]Reach{first, second}); got != 3 { + t.Fatalf("corpus distinct = %d, want 3", got) + } +} + +func measureRun(t *testing.T, seed int64, dumps ...string) Reach { + t.Helper() + run, err := tracecorpus.Load(writeRun(t, seed, dumps...)) + if err != nil { + t.Fatal(err) + } + return measure(run) +} + +func writeRun(t *testing.T, seed int64, dumps ...string) string { + t.Helper() + directory := t.TempDir() + writer, err := trace.NewWriter(directory) + if err != nil { + t.Fatal(err) + } + if err := writer.WriteMeta(trace.Meta{Seed: seed, Platform: "web"}); err != nil { + t.Fatal(err) + } + for index, dump := range dumps { + tree, err := hierarchy.Parse(dump) + if err != nil { + t.Fatal(err) + } + if err := writer.WriteStep(trace.Step{Index: index + 1, Hierarchy: tree}); err != nil { + t.Fatal(err) + } + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return directory +} + +func appendFinalize(t *testing.T, directory string, index int) { + t.Helper() + writer, err := trace.NewWriter(directory) + if err != nil { + t.Fatal(err) + } + if err := writer.WriteStep(trace.Step{ + Index: index, + Violations: []string{"someTransactionExists"}, + }); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } +}