mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
feat(exploration-reach): count the distinct structural states a stored run visited
the state is the settle path's structural hash of the recorded hierarchy, the same function the drivers wait on, so a state boundary here is the one the harness itself uses. --reference reports the observation at which two runs' hierarchies first differ. trace only: no device, no replay.
This commit is contained in:
pj committed
1 parent
0d1b2cf4e8
commit
11fca22d36
3 files changed
+455
No files matched your search
@@ -0,0 +1,145 @@
|
||||
// Command exploration-reach counts the distinct structural states a stored
|
||||
// run visited, and compares two runs by the observation at which their
|
||||
// hierarchies first differ. Both read the trace alone: no device, no replay.
|
||||
//
|
||||
// The state is the settle path's structural hash of the recorded hierarchy,
|
||||
// the same function the drivers wait on, so a state boundary here is the state
|
||||
// boundary the harness itself uses.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"text/tabwriter"
|
||||
|
||||
"github.com/priyanshujain/sanderling/internal/tracecorpus"
|
||||
)
|
||||
|
||||
func main() {
|
||||
jsonOut := flag.Bool("json", false, "emit JSON instead of a table")
|
||||
reference := flag.String(
|
||||
"reference",
|
||||
"",
|
||||
"run directory to compare against; reports where each other run's hierarchy first differs",
|
||||
)
|
||||
flag.Usage = func() {
|
||||
fmt.Fprintln(
|
||||
os.Stderr,
|
||||
"usage: exploration-reach [--json] [--reference RUN] <run directory> ...",
|
||||
)
|
||||
}
|
||||
flag.Parse()
|
||||
if flag.NArg() == 0 {
|
||||
flag.Usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
reaches, err := measureAll(flag.Args())
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if len(reaches) == 0 {
|
||||
fmt.Fprintln(os.Stderr, "no run directory found under the given paths")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
report := Report{Runs: reaches, CorpusDistinct: corpusDistinct(reaches)}
|
||||
if *reference != "" {
|
||||
base, loadErr := load(*reference)
|
||||
if loadErr != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: %v\n", *reference, loadErr)
|
||||
os.Exit(1)
|
||||
}
|
||||
report.Reference = base.Directory
|
||||
for _, reach := range reaches {
|
||||
if reach.Directory == base.Directory {
|
||||
continue
|
||||
}
|
||||
report.Divergences = append(report.Divergences, diverge(base, reach))
|
||||
}
|
||||
report.MedianDivergence, report.Censored = medianDivergence(report.Divergences)
|
||||
}
|
||||
|
||||
if *jsonOut {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
if err := encoder.Encode(report); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
render(os.Stdout, report)
|
||||
}
|
||||
|
||||
// Report is one invocation's output: reach per run and over the corpus, plus
|
||||
// the divergence rows when a reference run was named.
|
||||
type Report struct {
|
||||
Runs []Reach `json:"runs"`
|
||||
CorpusDistinct int `json:"corpus_distinct_states"`
|
||||
Reference string `json:"reference,omitempty"`
|
||||
Divergences []Divergence `json:"divergences,omitempty"`
|
||||
MedianDivergence float64 `json:"median_divergence_index,omitempty"`
|
||||
Censored int `json:"never_diverged,omitempty"`
|
||||
}
|
||||
|
||||
func load(path string) (Reach, error) {
|
||||
run, err := tracecorpus.Load(path)
|
||||
if err != nil {
|
||||
return Reach{}, err
|
||||
}
|
||||
return measure(run), nil
|
||||
}
|
||||
|
||||
func measureAll(paths []string) ([]Reach, error) {
|
||||
var reaches []Reach
|
||||
for _, path := range paths {
|
||||
directories, err := tracecorpus.Discover(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, directory := range directories {
|
||||
reach, err := load(directory)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", directory, err)
|
||||
}
|
||||
reaches = append(reaches, reach)
|
||||
}
|
||||
}
|
||||
return reaches, nil
|
||||
}
|
||||
|
||||
func render(out io.Writer, report Report) {
|
||||
writer := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(writer, "run\tseed\tplatform\tobservations\tdistinct states")
|
||||
for _, reach := range report.Runs {
|
||||
fmt.Fprintf(writer, "%s\t%d\t%s\t%d\t%d\n",
|
||||
reach.Directory, reach.Seed, reach.Platform,
|
||||
reach.Observations, reach.Distinct)
|
||||
}
|
||||
writer.Flush()
|
||||
fmt.Fprintf(out, "\n%d run(s), %d distinct structural states across the corpus\n",
|
||||
len(report.Runs), report.CorpusDistinct)
|
||||
|
||||
if report.Reference == "" {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(out, "\nreference: %s\n", report.Reference)
|
||||
writer = tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(writer, "run\tseed\tfirst divergence\tobservations compared")
|
||||
for _, divergence := range report.Divergences {
|
||||
where := fmt.Sprintf("%d", divergence.Step)
|
||||
if !divergence.Diverged {
|
||||
where = fmt.Sprintf("none through %d", divergence.Step)
|
||||
}
|
||||
fmt.Fprintf(writer, "%s\t%d\t%s\t%d\n",
|
||||
divergence.Directory, divergence.Seed, where, divergence.Compared)
|
||||
}
|
||||
writer.Flush()
|
||||
fmt.Fprintf(out, "\nmedian first divergence %.1f over %d run(s), %d never diverged\n",
|
||||
report.MedianDivergence, len(report.Divergences), report.Censored)
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"sort"
|
||||
|
||||
"github.com/priyanshujain/sanderling/internal/driver/ioscompanion"
|
||||
"github.com/priyanshujain/sanderling/internal/tracecorpus"
|
||||
)
|
||||
|
||||
// observation is one hierarchy-bearing step: the index the run gave it and the
|
||||
// state it observed.
|
||||
type observation struct {
|
||||
Step int
|
||||
State string
|
||||
}
|
||||
|
||||
// Reach is what one run explored. Distinct counts the structural states its
|
||||
// observations visited, which is the measure; Observations is how many looks
|
||||
// it took to visit them.
|
||||
type Reach struct {
|
||||
Directory string `json:"directory"`
|
||||
Seed int64 `json:"seed"`
|
||||
Platform string `json:"platform"`
|
||||
Arm string `json:"arm,omitempty"`
|
||||
Observations int `json:"observations"`
|
||||
Distinct int `json:"distinct_states"`
|
||||
// Unobserved counts steps carrying no hierarchy, which the run-end
|
||||
// finalize record is, so an observation count cannot be read as a step
|
||||
// count by accident.
|
||||
Unobserved int `json:"steps_without_hierarchy"`
|
||||
observations []observation
|
||||
}
|
||||
|
||||
// measure keys each observation by a digest of its structural hash. The
|
||||
// measure is equality between hashes and nothing else, and a corpus holds
|
||||
// thousands of trees whose hashes run to tens of kilobytes each.
|
||||
func measure(run tracecorpus.Run) Reach {
|
||||
reach := Reach{
|
||||
Directory: run.Directory,
|
||||
Seed: run.Meta.Seed,
|
||||
Platform: run.Meta.Platform,
|
||||
Arm: run.Meta.Arm,
|
||||
}
|
||||
distinct := map[string]bool{}
|
||||
for _, step := range run.Steps {
|
||||
if step.Hierarchy == nil {
|
||||
reach.Unobserved++
|
||||
continue
|
||||
}
|
||||
digest := sha256.Sum256([]byte(ioscompanion.StructuralHash(step.Hierarchy)))
|
||||
key := hex.EncodeToString(digest[:])
|
||||
reach.observations = append(
|
||||
reach.observations,
|
||||
observation{Step: step.Index, State: key},
|
||||
)
|
||||
distinct[key] = true
|
||||
}
|
||||
reach.Observations = len(reach.observations)
|
||||
reach.Distinct = len(distinct)
|
||||
return reach
|
||||
}
|
||||
|
||||
// corpusDistinct counts the structural states the whole corpus reached, which
|
||||
// is not the sum of the per-run counts: runs of one application revisit the
|
||||
// same screens.
|
||||
func corpusDistinct(reaches []Reach) int {
|
||||
distinct := map[string]bool{}
|
||||
for _, reach := range reaches {
|
||||
for _, seen := range reach.observations {
|
||||
distinct[seen.State] = true
|
||||
}
|
||||
}
|
||||
return len(distinct)
|
||||
}
|
||||
|
||||
// Divergence is where a replay stopped observing what the reference observed.
|
||||
// Step is the index of the first observation whose structural state differs;
|
||||
// Diverged is false when the replay matched the reference for every
|
||||
// observation the two share, in which case Step is that shared length and the
|
||||
// observation is right-censored.
|
||||
type Divergence struct {
|
||||
Directory string `json:"directory"`
|
||||
Seed int64 `json:"seed"`
|
||||
Step int `json:"step"`
|
||||
Diverged bool `json:"diverged"`
|
||||
Compared int `json:"observations_compared"`
|
||||
}
|
||||
|
||||
func diverge(reference, replay Reach) Divergence {
|
||||
result := Divergence{Directory: replay.Directory, Seed: replay.Seed}
|
||||
shared := len(reference.observations)
|
||||
if len(replay.observations) < shared {
|
||||
shared = len(replay.observations)
|
||||
}
|
||||
result.Compared = shared
|
||||
for position := 0; position < shared; position++ {
|
||||
if reference.observations[position].State != replay.observations[position].State {
|
||||
result.Step = reference.observations[position].Step
|
||||
result.Diverged = true
|
||||
return result
|
||||
}
|
||||
}
|
||||
if shared > 0 {
|
||||
result.Step = reference.observations[shared-1].Step
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// medianDivergence is E6's number: the median observation index at which a
|
||||
// replay first diverges from the reference. A replay that never diverged
|
||||
// enters at the last index the two share, which is where the observation is
|
||||
// censored rather than where it broke, so the count of such replays is
|
||||
// reported beside the median rather than folded into it.
|
||||
func medianDivergence(divergences []Divergence) (median float64, censored int) {
|
||||
if len(divergences) == 0 {
|
||||
return 0, 0
|
||||
}
|
||||
steps := make([]int, 0, len(divergences))
|
||||
for _, divergence := range divergences {
|
||||
steps = append(steps, divergence.Step)
|
||||
if !divergence.Diverged {
|
||||
censored++
|
||||
}
|
||||
}
|
||||
sort.Ints(steps)
|
||||
middle := len(steps) / 2
|
||||
if len(steps)%2 == 1 {
|
||||
return float64(steps[middle]), censored
|
||||
}
|
||||
return float64(steps[middle-1]+steps[middle]) / 2, censored
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/priyanshujain/sanderling/internal/driver/ioscompanion"
|
||||
"github.com/priyanshujain/sanderling/internal/hierarchy"
|
||||
"github.com/priyanshujain/sanderling/internal/trace"
|
||||
"github.com/priyanshujain/sanderling/internal/tracecorpus"
|
||||
)
|
||||
|
||||
const (
|
||||
home = `{"attributes": {"text": "Home", "bounds": "[0,0,10,10]"}, "children": [
|
||||
{"attributes": {"text": "row", "bounds": "[0,0,5,5]"}, "children": []}
|
||||
]}`
|
||||
homeScrolled = `{"attributes": {"text": "Home", "bounds": "[0,4,10,14]"}, "children": [
|
||||
{"attributes": {"text": "row", "bounds": "[0,4,5,9]"}, "children": []}
|
||||
]}`
|
||||
ledger = `{"attributes": {"text": "Ledger", "bounds": "[0,0,10,10]"}, "children": [
|
||||
{"attributes": {"text": "row", "bounds": "[0,0,5,5]"}, "children": []}
|
||||
]}`
|
||||
ledgerWithRow = `{"attributes": {"text": "Ledger", "bounds": "[0,0,10,10]"}, "children": [
|
||||
{"attributes": {"text": "row", "bounds": "[0,0,5,5]"}, "children": []},
|
||||
{"attributes": {"text": "row"}, "children": []}
|
||||
]}`
|
||||
)
|
||||
|
||||
// TestReachCountsStructuresNotObservations: five observations of three
|
||||
// structures, one of them revisited and one differing only in where it sits on
|
||||
// screen, so the answer is three by construction.
|
||||
func TestReachCountsStructuresNotObservations(t *testing.T) {
|
||||
reach := measureRun(t, 7, home, homeScrolled, ledger, home, ledgerWithRow)
|
||||
|
||||
if reach.Observations != 5 {
|
||||
t.Fatalf("observations = %d, want 5", reach.Observations)
|
||||
}
|
||||
if reach.Distinct != 3 {
|
||||
t.Fatalf("distinct states = %d, want 3", reach.Distinct)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFinalizeRecordIsNoObservation(t *testing.T) {
|
||||
directory := writeRun(t, 7, home, ledger)
|
||||
appendFinalize(t, directory, 3)
|
||||
|
||||
run, err := tracecorpus.Load(directory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reach := measure(run)
|
||||
if reach.Observations != 2 || reach.Unobserved != 1 {
|
||||
t.Fatalf("observations = %d, unobserved = %d, want 2 and 1",
|
||||
reach.Observations, reach.Unobserved)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStoredTreeHashesAsTheLiveTreeDid is the equivalence the whole measure
|
||||
// rests on: the hash the settle path computed from the live tree and the hash
|
||||
// this tool computes from the stored one are the same string, so a reach
|
||||
// number counts the same state boundaries the drivers wait on.
|
||||
func TestStoredTreeHashesAsTheLiveTreeDid(t *testing.T) {
|
||||
live, err := hierarchy.Parse(home)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
liveHash := ioscompanion.StructuralHash(live)
|
||||
if liveHash == "" {
|
||||
t.Fatal("live hash is empty, so the test would pass on any stored tree")
|
||||
}
|
||||
|
||||
run, err := tracecorpus.Load(writeRun(t, 7, home))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stored := ioscompanion.StructuralHash(run.Steps[0].Hierarchy)
|
||||
if stored != liveHash {
|
||||
t.Fatalf("stored hash differs from the live one:\n live=%q\n stored=%q",
|
||||
liveHash, stored)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirstDivergenceNamesTheObservationThatDiffers(t *testing.T) {
|
||||
reference := measureRun(t, 7, home, ledger, home, ledger)
|
||||
replay := measureRun(t, 7, home, ledger, ledgerWithRow, ledger)
|
||||
|
||||
divergence := diverge(reference, replay)
|
||||
if !divergence.Diverged || divergence.Step != 3 {
|
||||
t.Fatalf("divergence = %+v, want the third observation", divergence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReplayThatMatchesIsCensoredAtTheSharedLength(t *testing.T) {
|
||||
reference := measureRun(t, 7, home, ledger, home, ledger)
|
||||
replay := measureRun(t, 7, home, ledger, home)
|
||||
|
||||
divergence := diverge(reference, replay)
|
||||
if divergence.Diverged {
|
||||
t.Fatalf("identical observations must not report a divergence: %+v", divergence)
|
||||
}
|
||||
if divergence.Step != 3 || divergence.Compared != 3 {
|
||||
t.Fatalf("divergence = %+v, want censoring at the third observation", divergence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMedianDivergenceHoldsCensoredReplaysApart(t *testing.T) {
|
||||
median, censored := medianDivergence([]Divergence{
|
||||
{Step: 9, Diverged: true},
|
||||
{Step: 3, Diverged: true},
|
||||
{Step: 40, Diverged: false},
|
||||
})
|
||||
if median != 9 {
|
||||
t.Fatalf("median = %v, want 9", median)
|
||||
}
|
||||
if censored != 1 {
|
||||
t.Fatalf("censored = %d, want 1", censored)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorpusStatesAreTheUnionNotTheSum(t *testing.T) {
|
||||
first := measureRun(t, 7, home, ledger)
|
||||
second := measureRun(t, 11, ledger, ledgerWithRow)
|
||||
|
||||
if got := corpusDistinct([]Reach{first, second}); got != 3 {
|
||||
t.Fatalf("corpus distinct = %d, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func measureRun(t *testing.T, seed int64, dumps ...string) Reach {
|
||||
t.Helper()
|
||||
run, err := tracecorpus.Load(writeRun(t, seed, dumps...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return measure(run)
|
||||
}
|
||||
|
||||
func writeRun(t *testing.T, seed int64, dumps ...string) string {
|
||||
t.Helper()
|
||||
directory := t.TempDir()
|
||||
writer, err := trace.NewWriter(directory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writer.WriteMeta(trace.Meta{Seed: seed, Platform: "web"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for index, dump := range dumps {
|
||||
tree, err := hierarchy.Parse(dump)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writer.WriteStep(trace.Step{Index: index + 1, Hierarchy: tree}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return directory
|
||||
}
|
||||
|
||||
func appendFinalize(t *testing.T, directory string, index int) {
|
||||
t.Helper()
|
||||
writer, err := trace.NewWriter(directory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writer.WriteStep(trace.Step{
|
||||
Index: index,
|
||||
Violations: []string{"someTransactionExists"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user