feat(defect-identity): count distinct defects across stored runs

a property reports at most once per run, so a run-level count is just the number of properties violated. a defect is identified across runs by the property, the action attributed as the origin of the failed obligation and the screen the witness observed.

cmd/internal-tools/defect-identity/identity.go

@@ -0,0 +1,190 @@
+package main
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/priyanshujain/sanderling/internal/trace"
+	"github.com/priyanshujain/sanderling/internal/tracecorpus"
+)
+
+// Instance is one defect as the draft identifies it across runs: the property
+// that reported, the action attributed as the origin of the failed obligation,
+// and the screen the witness observed. Two reports sharing all three are the
+// same defect seen twice; a run-level count of violated properties cannot say
+// that.
+type Instance struct {
+	Property      string   `json:"property"`
+	OriginAction  string   `json:"origin_action"`
+	WitnessScreen string   `json:"witness_screen"`
+	Runs          []string `json:"runs"`
+	Seeds         []int64  `json:"seeds"`
+	// Reports counts violations folded into this instance, which exceeds the
+	// run count only if one run reported the same property twice, and the
+	// latch says it cannot.
+	Reports int `json:"reports"`
+}
+
+// Unattributed is a violation that carries no origin, so the identity rule
+// cannot be applied to it. It is reported rather than counted, because
+// dropping it understates the defect count and guessing an origin invents one.
+type Unattributed struct {
+	Property string `json:"property"`
+	Run      string `json:"run"`
+	Step     int    `json:"step"`
+	Reason   string `json:"reason"`
+}
+
+// actionKeyMode selects how much of an action two reports must share to be the
+// same origin. The draft names the origin action and does not say which of its
+// fields identify it, so the strict reading is available beside the default.
+type actionKeyMode string
+
+const (
+	// bySelector keys an action by what it did and the name of what it did it
+	// to. Coordinates and generated text differ between two runs that took the
+	// same action against the same control.
+	bySelector actionKeyMode = "selector"
+	// byFullAction adds the text typed and the coordinates dispatched.
+	byFullAction actionKeyMode = "full"
+)
+
+type identityKey struct {
+	property string
+	action   string
+	screen   string
+}
+
+// Corpus is what one invocation read: the instances, the violations it could
+// not attribute, and the counts the report needs.
+type Corpus struct {
+	Runs          int            `json:"runs"`
+	Instances     []Instance     `json:"instances"`
+	Unattributed  []Unattributed `json:"unattributed,omitempty"`
+	UnnamedScreen int            `json:"unnamed_witness_screens,omitempty"`
+}
+
+// Singletons counts instances that appeared in exactly one run, the number the
+// evaluation reports beside the defect count.
+func (c Corpus) Singletons() int {
+	count := 0
+	for _, instance := range c.Instances {
+		if len(instance.Runs) == 1 {
+			count++
+		}
+	}
+	return count
+}
+
+func identify(runs []tracecorpus.Run, mode actionKeyMode) (Corpus, error) {
+	corpus := Corpus{Runs: len(runs)}
+	byKey := map[identityKey]*Instance{}
+	var order []identityKey
+	for _, run := range runs {
+		steps := index(run.Steps)
+		for _, step := range run.Steps {
+			for _, property := range step.Violations {
+				witness, ok := step.Witnesses[property]
+				if !ok || witness.Step == 0 {
+					corpus.Unattributed = append(corpus.Unattributed, Unattributed{
+						Property: property,
+						Run:      run.Directory,
+						Step:     step.Index,
+						Reason:   attributionGap(ok),
+					})
+					continue
+				}
+				origin, ok := steps[witness.Step]
+				if !ok {
+					return Corpus{}, fmt.Errorf(
+						"%s: %s names origin step %d, which the trace does not hold",
+						run.Directory, property, witness.Step,
+					)
+				}
+				detected, ok := steps[witness.DetectedStep]
+				if !ok {
+					return Corpus{}, fmt.Errorf(
+						"%s: %s names detection step %d, which the trace does not hold",
+						run.Directory, property, witness.DetectedStep,
+					)
+				}
+				if detected.Screen == "" {
+					corpus.UnnamedScreen++
+				}
+				key := identityKey{
+					property: property,
+					action:   actionKey(origin, mode),
+					screen:   detected.Screen,
+				}
+				instance, seen := byKey[key]
+				if !seen {
+					instance = &Instance{
+						Property:      property,
+						OriginAction:  key.action,
+						WitnessScreen: key.screen,
+					}
+					byKey[key] = instance
+					order = append(order, key)
+				}
+				instance.Reports++
+				if len(instance.Runs) == 0 ||
+					instance.Runs[len(instance.Runs)-1] != run.Directory {
+					instance.Runs = append(instance.Runs, run.Directory)
+					instance.Seeds = append(instance.Seeds, run.Meta.Seed)
+				}
+			}
+		}
+	}
+	for _, key := range order {
+		corpus.Instances = append(corpus.Instances, *byKey[key])
+	}
+	sort.SliceStable(corpus.Instances, func(i, j int) bool {
+		if corpus.Instances[i].Property != corpus.Instances[j].Property {
+			return corpus.Instances[i].Property < corpus.Instances[j].Property
+		}
+		return corpus.Instances[i].OriginAction < corpus.Instances[j].OriginAction
+	})
+	return corpus, nil
+}
+
+func attributionGap(hasWitness bool) string {
+	if !hasWitness {
+		return "no witness recorded"
+	}
+	return "witness records no origin step"
+}
+
+func index(steps []trace.Step) map[int]trace.Step {
+	byIndex := make(map[int]trace.Step, len(steps))
+	for _, step := range steps {
+		byIndex[step.Index] = step
+	}
+	return byIndex
+}
+
+// actionKey renders the action the origin step chose. The action recorded on a
+// line is the one applied after observing it, which is the alignment that
+// makes an origin index name an action at all.
+func actionKey(origin trace.Step, mode actionKeyMode) string {
+	if origin.NextAction == nil {
+		return "none"
+	}
+	if origin.ActionSkipped != "" {
+		return "none (" + origin.ActionSkipped + ")"
+	}
+	action := *origin.NextAction
+	key := action.Kind
+	switch {
+	case action.Selector != "":
+		key += " " + action.Selector
+	case action.Key != "":
+		key += " " + action.Key
+	case action.X != 0 || action.Y != 0 || action.ToX != 0 || action.ToY != 0:
+		key += fmt.Sprintf(" (%d,%d)", action.X, action.Y)
+	}
+	if mode != byFullAction {
+		return key
+	}
+	return fmt.Sprintf("%s text=%q at=(%d,%d)->(%d,%d)",
+		key, action.Text, action.X, action.Y, action.ToX, action.ToY)
+}

cmd/internal-tools/defect-identity/identity_test.go

@@ -0,0 +1,172 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/priyanshujain/sanderling/internal/trace"
+	"github.com/priyanshujain/sanderling/internal/tracecorpus"
+)
+
+// TestOneDefectSeenTwiceIsOneInstance: two runs report the same property from
+// the same origin action on the same screen, which is one defect found twice
+// and two properties violated.
+func TestOneDefectSeenTwiceIsOneInstance(t *testing.T) {
+	first := run(t, 3,
+		step(1, "/ledger", tap("id:add-txn")),
+		violating(2, "/accounts/7", tap("id:save"), "balanceMatches", 2, 2),
+	)
+	second := run(t, 5,
+		step(1, "/ledger", tap("id:add-txn")),
+		violating(2, "/accounts/7", tap("id:save"), "balanceMatches", 2, 2),
+	)
+
+	corpus := identified(t, bySelector, first, second)
+	if len(corpus.Instances) != 1 {
+		t.Fatalf("instances = %d, want 1: %+v", len(corpus.Instances), corpus.Instances)
+	}
+	instance := corpus.Instances[0]
+	if len(instance.Runs) != 2 || instance.Reports != 2 {
+		t.Fatalf("instance = %+v, want two runs reporting it", instance)
+	}
+	if corpus.Singletons() != 0 {
+		t.Fatalf("singletons = %d, want 0", corpus.Singletons())
+	}
+}
+
+func TestTheSamePropertyFromTwoOriginsIsTwoDefects(t *testing.T) {
+	first := run(t, 3, violating(1, "/ledger", tap("id:save"), "balanceMatches", 1, 1))
+	second := run(t, 5, violating(1, "/ledger", tap("id:delete"), "balanceMatches", 1, 1))
+
+	corpus := identified(t, bySelector, first, second)
+	if len(corpus.Instances) != 2 {
+		t.Fatalf("instances = %d, want 2: %+v", len(corpus.Instances), corpus.Instances)
+	}
+	if corpus.Singletons() != 2 {
+		t.Fatalf("singletons = %d, want 2", corpus.Singletons())
+	}
+}
+
+func TestTheSamePropertyOnTwoScreensIsTwoDefects(t *testing.T) {
+	first := run(t, 3, violating(1, "/ledger", tap("id:save"), "balanceMatches", 1, 1))
+	second := run(t, 5, violating(1, "/home", tap("id:save"), "balanceMatches", 1, 1))
+
+	corpus := identified(t, bySelector, first, second)
+	if len(corpus.Instances) != 2 {
+		t.Fatalf("instances = %d, want 2: %+v", len(corpus.Instances), corpus.Instances)
+	}
+}
+
+// TestADeferredViolationTakesTheActionOnItsOriginLine holds the alignment the
+// draft states: the action recorded on line k is the one applied after
+// observing k, so an obligation armed at k is attributed to that action and
+// witnessed on the screen the detection step observed.
+func TestADeferredViolationTakesTheActionOnItsOriginLine(t *testing.T) {
+	only := run(t, 3,
+		step(1, "/login", tap("id:login-submit")),
+		step(2, "/home", tap("id:open-ledger")),
+		violating(3, "/ledger", tap("id:add-txn"), "landsOnLedger", 2, 3),
+	)
+
+	corpus := identified(t, bySelector, only)
+	instance := corpus.Instances[0]
+	if instance.OriginAction != "Tap id:open-ledger" {
+		t.Fatalf("origin action = %q, want the action on the origin line", instance.OriginAction)
+	}
+	if instance.WitnessScreen != "/ledger" {
+		t.Fatalf("witness screen = %q, want the detection step's screen", instance.WitnessScreen)
+	}
+}
+
+func TestAViolationWithNoWitnessIsReportedNotCounted(t *testing.T) {
+	unattributed := trace.Step{
+		Index:      1,
+		Screen:     "/ledger",
+		Violations: []string{"balanceMatches"},
+	}
+	corpus := identified(t, bySelector, run(t, 3, unattributed))
+
+	if len(corpus.Instances) != 0 {
+		t.Fatalf("instances = %d, want 0: %+v", len(corpus.Instances), corpus.Instances)
+	}
+	if len(corpus.Unattributed) != 1 ||
+		corpus.Unattributed[0].Property != "balanceMatches" {
+		t.Fatalf("unattributed = %+v, want the one violation with no origin", corpus.Unattributed)
+	}
+}
+
+// TestTheStrictActionKeySplitsWhatTheSelectorKeyMerges quantifies the reading
+// the draft leaves open: two runs that typed different text into the same
+// field are one defect by selector and two by the whole action.
+func TestTheStrictActionKeySplitsWhatTheSelectorKeyMerges(t *testing.T) {
+	first := run(t, 3, violating(1, "/ledger", typing("id:amount", "12"), "balanceMatches", 1, 1))
+	second := run(t, 5, violating(1, "/ledger", typing("id:amount", "9000"), "balanceMatches", 1, 1))
+
+	if got := identified(t, bySelector, first, second); len(got.Instances) != 1 {
+		t.Fatalf("by selector: instances = %d, want 1", len(got.Instances))
+	}
+	if got := identified(t, byFullAction, first, second); len(got.Instances) != 2 {
+		t.Fatalf("by full action: instances = %d, want 2", len(got.Instances))
+	}
+}
+
+func identified(t *testing.T, mode actionKeyMode, runs ...tracecorpus.Run) Corpus {
+	t.Helper()
+	corpus, err := identify(runs, mode)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return corpus
+}
+
+func run(t *testing.T, seed int64, steps ...trace.Step) tracecorpus.Run {
+	t.Helper()
+	directory := t.TempDir()
+	writer, err := trace.NewWriter(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.WriteMeta(trace.Meta{Seed: seed, Platform: "web"}); err != nil {
+		t.Fatal(err)
+	}
+	for _, step := range steps {
+		if err := writer.WriteStep(step); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatal(err)
+	}
+	loaded, err := tracecorpus.Load(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return loaded
+}
+
+func step(index int, screen string, action *trace.Action) trace.Step {
+	return trace.Step{Index: index, Screen: screen, NextAction: action}
+}
+
+func violating(
+	index int,
+	screen string,
+	action *trace.Action,
+	property string,
+	origin int,
+	detected int,
+) trace.Step {
+	violated := step(index, screen, action)
+	violated.Violations = []string{property}
+	violated.Witnesses = map[string]trace.Witness{
+		property: {Reason: "predicate false", Step: origin, DetectedStep: detected},
+	}
+	return violated
+}
+
+func tap(selector string) *trace.Action {
+	return &trace.Action{Kind: "Tap", Selector: selector, X: 10, Y: 20}
+}
+
+func typing(selector string, text string) *trace.Action {
+	return &trace.Action{Kind: "InputText", Selector: selector, Text: text, X: 10, Y: 20}
+}

cmd/internal-tools/defect-identity/main.go

@@ -0,0 +1,113 @@
+// Command defect-identity counts distinct defects across stored runs. A
+// property reports at most once per run, so a run-level count is the number of
+// properties violated; a defect is identified across runs by the property, the
+// action attributed as the origin of the failed obligation, and the screen the
+// witness observed.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"text/tabwriter"
+
+	"github.com/priyanshujain/sanderling/internal/tracecorpus"
+)
+
+func main() {
+	jsonOut := flag.Bool("json", false, "emit JSON instead of a table")
+	mode := flag.String(
+		"action-key",
+		string(bySelector),
+		"how much of the origin action identifies it: selector or full",
+	)
+	flag.Usage = func() {
+		fmt.Fprintln(
+			os.Stderr,
+			"usage: defect-identity [--json] [--action-key selector|full] <run directory> ...",
+		)
+	}
+	flag.Parse()
+	if flag.NArg() == 0 {
+		flag.Usage()
+		os.Exit(2)
+	}
+	if *mode != string(bySelector) && *mode != string(byFullAction) {
+		fmt.Fprintf(os.Stderr, "unknown --action-key %q\n", *mode)
+		os.Exit(2)
+	}
+
+	runs, err := loadAll(flag.Args())
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	if len(runs) == 0 {
+		fmt.Fprintln(os.Stderr, "no run directory found under the given paths")
+		os.Exit(1)
+	}
+	corpus, err := identify(runs, actionKeyMode(*mode))
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+
+	if *jsonOut {
+		encoder := json.NewEncoder(os.Stdout)
+		encoder.SetIndent("", "  ")
+		if err := encoder.Encode(corpus); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+		return
+	}
+	render(os.Stdout, corpus)
+}
+
+func loadAll(paths []string) ([]tracecorpus.Run, error) {
+	var runs []tracecorpus.Run
+	for _, path := range paths {
+		directories, err := tracecorpus.Discover(path)
+		if err != nil {
+			return nil, err
+		}
+		for _, directory := range directories {
+			run, err := tracecorpus.Load(directory)
+			if err != nil {
+				return nil, fmt.Errorf("%s: %w", directory, err)
+			}
+			runs = append(runs, run)
+		}
+	}
+	return runs, nil
+}
+
+func render(out io.Writer, corpus Corpus) {
+	writer := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(writer, "property\torigin action\twitness screen\truns\tseeds")
+	for _, instance := range corpus.Instances {
+		screen := instance.WitnessScreen
+		if screen == "" {
+			screen = "(unnamed)"
+		}
+		fmt.Fprintf(writer, "%s\t%s\t%s\t%d\t%v\n",
+			instance.Property, instance.OriginAction, screen,
+			len(instance.Runs), instance.Seeds)
+	}
+	writer.Flush()
+
+	fmt.Fprintf(out, "\n%d distinct defect(s) over %d run(s); %d seen in exactly one run\n",
+		len(corpus.Instances), corpus.Runs, corpus.Singletons())
+	if corpus.UnnamedScreen > 0 {
+		fmt.Fprintf(out,
+			"%d violation(s) witnessed on a screen the app does not name, "+
+				"so identity rests on property and origin action alone for those\n",
+			corpus.UnnamedScreen)
+	}
+	for _, gap := range corpus.Unattributed {
+		fmt.Fprintf(out, "unattributed: %s at step %d of %s (%s)\n",
+			gap.Property, gap.Step, gap.Run, gap.Reason)
+	}
+}