diff --git a/cmd/internal-tools/defect-identity/identity.go b/cmd/internal-tools/defect-identity/identity.go new file mode 100644 index 0000000..cd138f7 --- /dev/null +++ b/cmd/internal-tools/defect-identity/identity.go @@ -0,0 +1,190 @@ +package main + +import ( + "fmt" + "sort" + + "github.com/priyanshujain/sanderling/internal/trace" + "github.com/priyanshujain/sanderling/internal/tracecorpus" +) + +// Instance is one defect as the draft identifies it across runs: the property +// that reported, the action attributed as the origin of the failed obligation, +// and the screen the witness observed. Two reports sharing all three are the +// same defect seen twice; a run-level count of violated properties cannot say +// that. +type Instance struct { + Property string `json:"property"` + OriginAction string `json:"origin_action"` + WitnessScreen string `json:"witness_screen"` + Runs []string `json:"runs"` + Seeds []int64 `json:"seeds"` + // Reports counts violations folded into this instance, which exceeds the + // run count only if one run reported the same property twice, and the + // latch says it cannot. + Reports int `json:"reports"` +} + +// Unattributed is a violation that carries no origin, so the identity rule +// cannot be applied to it. It is reported rather than counted, because +// dropping it understates the defect count and guessing an origin invents one. +type Unattributed struct { + Property string `json:"property"` + Run string `json:"run"` + Step int `json:"step"` + Reason string `json:"reason"` +} + +// actionKeyMode selects how much of an action two reports must share to be the +// same origin. The draft names the origin action and does not say which of its +// fields identify it, so the strict reading is available beside the default. +type actionKeyMode string + +const ( + // bySelector keys an action by what it did and the name of what it did it + // to. Coordinates and generated text differ between two runs that took the + // same action against the same control. + bySelector actionKeyMode = "selector" + // byFullAction adds the text typed and the coordinates dispatched. + byFullAction actionKeyMode = "full" +) + +type identityKey struct { + property string + action string + screen string +} + +// Corpus is what one invocation read: the instances, the violations it could +// not attribute, and the counts the report needs. +type Corpus struct { + Runs int `json:"runs"` + Instances []Instance `json:"instances"` + Unattributed []Unattributed `json:"unattributed,omitempty"` + UnnamedScreen int `json:"unnamed_witness_screens,omitempty"` +} + +// Singletons counts instances that appeared in exactly one run, the number the +// evaluation reports beside the defect count. +func (c Corpus) Singletons() int { + count := 0 + for _, instance := range c.Instances { + if len(instance.Runs) == 1 { + count++ + } + } + return count +} + +func identify(runs []tracecorpus.Run, mode actionKeyMode) (Corpus, error) { + corpus := Corpus{Runs: len(runs)} + byKey := map[identityKey]*Instance{} + var order []identityKey + for _, run := range runs { + steps := index(run.Steps) + for _, step := range run.Steps { + for _, property := range step.Violations { + witness, ok := step.Witnesses[property] + if !ok || witness.Step == 0 { + corpus.Unattributed = append(corpus.Unattributed, Unattributed{ + Property: property, + Run: run.Directory, + Step: step.Index, + Reason: attributionGap(ok), + }) + continue + } + origin, ok := steps[witness.Step] + if !ok { + return Corpus{}, fmt.Errorf( + "%s: %s names origin step %d, which the trace does not hold", + run.Directory, property, witness.Step, + ) + } + detected, ok := steps[witness.DetectedStep] + if !ok { + return Corpus{}, fmt.Errorf( + "%s: %s names detection step %d, which the trace does not hold", + run.Directory, property, witness.DetectedStep, + ) + } + if detected.Screen == "" { + corpus.UnnamedScreen++ + } + key := identityKey{ + property: property, + action: actionKey(origin, mode), + screen: detected.Screen, + } + instance, seen := byKey[key] + if !seen { + instance = &Instance{ + Property: property, + OriginAction: key.action, + WitnessScreen: key.screen, + } + byKey[key] = instance + order = append(order, key) + } + instance.Reports++ + if len(instance.Runs) == 0 || + instance.Runs[len(instance.Runs)-1] != run.Directory { + instance.Runs = append(instance.Runs, run.Directory) + instance.Seeds = append(instance.Seeds, run.Meta.Seed) + } + } + } + } + for _, key := range order { + corpus.Instances = append(corpus.Instances, *byKey[key]) + } + sort.SliceStable(corpus.Instances, func(i, j int) bool { + if corpus.Instances[i].Property != corpus.Instances[j].Property { + return corpus.Instances[i].Property < corpus.Instances[j].Property + } + return corpus.Instances[i].OriginAction < corpus.Instances[j].OriginAction + }) + return corpus, nil +} + +func attributionGap(hasWitness bool) string { + if !hasWitness { + return "no witness recorded" + } + return "witness records no origin step" +} + +func index(steps []trace.Step) map[int]trace.Step { + byIndex := make(map[int]trace.Step, len(steps)) + for _, step := range steps { + byIndex[step.Index] = step + } + return byIndex +} + +// actionKey renders the action the origin step chose. The action recorded on a +// line is the one applied after observing it, which is the alignment that +// makes an origin index name an action at all. +func actionKey(origin trace.Step, mode actionKeyMode) string { + if origin.NextAction == nil { + return "none" + } + if origin.ActionSkipped != "" { + return "none (" + origin.ActionSkipped + ")" + } + action := *origin.NextAction + key := action.Kind + switch { + case action.Selector != "": + key += " " + action.Selector + case action.Key != "": + key += " " + action.Key + case action.X != 0 || action.Y != 0 || action.ToX != 0 || action.ToY != 0: + key += fmt.Sprintf(" (%d,%d)", action.X, action.Y) + } + if mode != byFullAction { + return key + } + return fmt.Sprintf("%s text=%q at=(%d,%d)->(%d,%d)", + key, action.Text, action.X, action.Y, action.ToX, action.ToY) +} diff --git a/cmd/internal-tools/defect-identity/identity_test.go b/cmd/internal-tools/defect-identity/identity_test.go new file mode 100644 index 0000000..a6804ec --- /dev/null +++ b/cmd/internal-tools/defect-identity/identity_test.go @@ -0,0 +1,172 @@ +package main + +import ( + "testing" + + "github.com/priyanshujain/sanderling/internal/trace" + "github.com/priyanshujain/sanderling/internal/tracecorpus" +) + +// TestOneDefectSeenTwiceIsOneInstance: two runs report the same property from +// the same origin action on the same screen, which is one defect found twice +// and two properties violated. +func TestOneDefectSeenTwiceIsOneInstance(t *testing.T) { + first := run(t, 3, + step(1, "/ledger", tap("id:add-txn")), + violating(2, "/accounts/7", tap("id:save"), "balanceMatches", 2, 2), + ) + second := run(t, 5, + step(1, "/ledger", tap("id:add-txn")), + violating(2, "/accounts/7", tap("id:save"), "balanceMatches", 2, 2), + ) + + corpus := identified(t, bySelector, first, second) + if len(corpus.Instances) != 1 { + t.Fatalf("instances = %d, want 1: %+v", len(corpus.Instances), corpus.Instances) + } + instance := corpus.Instances[0] + if len(instance.Runs) != 2 || instance.Reports != 2 { + t.Fatalf("instance = %+v, want two runs reporting it", instance) + } + if corpus.Singletons() != 0 { + t.Fatalf("singletons = %d, want 0", corpus.Singletons()) + } +} + +func TestTheSamePropertyFromTwoOriginsIsTwoDefects(t *testing.T) { + first := run(t, 3, violating(1, "/ledger", tap("id:save"), "balanceMatches", 1, 1)) + second := run(t, 5, violating(1, "/ledger", tap("id:delete"), "balanceMatches", 1, 1)) + + corpus := identified(t, bySelector, first, second) + if len(corpus.Instances) != 2 { + t.Fatalf("instances = %d, want 2: %+v", len(corpus.Instances), corpus.Instances) + } + if corpus.Singletons() != 2 { + t.Fatalf("singletons = %d, want 2", corpus.Singletons()) + } +} + +func TestTheSamePropertyOnTwoScreensIsTwoDefects(t *testing.T) { + first := run(t, 3, violating(1, "/ledger", tap("id:save"), "balanceMatches", 1, 1)) + second := run(t, 5, violating(1, "/home", tap("id:save"), "balanceMatches", 1, 1)) + + corpus := identified(t, bySelector, first, second) + if len(corpus.Instances) != 2 { + t.Fatalf("instances = %d, want 2: %+v", len(corpus.Instances), corpus.Instances) + } +} + +// TestADeferredViolationTakesTheActionOnItsOriginLine holds the alignment the +// draft states: the action recorded on line k is the one applied after +// observing k, so an obligation armed at k is attributed to that action and +// witnessed on the screen the detection step observed. +func TestADeferredViolationTakesTheActionOnItsOriginLine(t *testing.T) { + only := run(t, 3, + step(1, "/login", tap("id:login-submit")), + step(2, "/home", tap("id:open-ledger")), + violating(3, "/ledger", tap("id:add-txn"), "landsOnLedger", 2, 3), + ) + + corpus := identified(t, bySelector, only) + instance := corpus.Instances[0] + if instance.OriginAction != "Tap id:open-ledger" { + t.Fatalf("origin action = %q, want the action on the origin line", instance.OriginAction) + } + if instance.WitnessScreen != "/ledger" { + t.Fatalf("witness screen = %q, want the detection step's screen", instance.WitnessScreen) + } +} + +func TestAViolationWithNoWitnessIsReportedNotCounted(t *testing.T) { + unattributed := trace.Step{ + Index: 1, + Screen: "/ledger", + Violations: []string{"balanceMatches"}, + } + corpus := identified(t, bySelector, run(t, 3, unattributed)) + + if len(corpus.Instances) != 0 { + t.Fatalf("instances = %d, want 0: %+v", len(corpus.Instances), corpus.Instances) + } + if len(corpus.Unattributed) != 1 || + corpus.Unattributed[0].Property != "balanceMatches" { + t.Fatalf("unattributed = %+v, want the one violation with no origin", corpus.Unattributed) + } +} + +// TestTheStrictActionKeySplitsWhatTheSelectorKeyMerges quantifies the reading +// the draft leaves open: two runs that typed different text into the same +// field are one defect by selector and two by the whole action. +func TestTheStrictActionKeySplitsWhatTheSelectorKeyMerges(t *testing.T) { + first := run(t, 3, violating(1, "/ledger", typing("id:amount", "12"), "balanceMatches", 1, 1)) + second := run(t, 5, violating(1, "/ledger", typing("id:amount", "9000"), "balanceMatches", 1, 1)) + + if got := identified(t, bySelector, first, second); len(got.Instances) != 1 { + t.Fatalf("by selector: instances = %d, want 1", len(got.Instances)) + } + if got := identified(t, byFullAction, first, second); len(got.Instances) != 2 { + t.Fatalf("by full action: instances = %d, want 2", len(got.Instances)) + } +} + +func identified(t *testing.T, mode actionKeyMode, runs ...tracecorpus.Run) Corpus { + t.Helper() + corpus, err := identify(runs, mode) + if err != nil { + t.Fatal(err) + } + return corpus +} + +func run(t *testing.T, seed int64, steps ...trace.Step) tracecorpus.Run { + t.Helper() + directory := t.TempDir() + writer, err := trace.NewWriter(directory) + if err != nil { + t.Fatal(err) + } + if err := writer.WriteMeta(trace.Meta{Seed: seed, Platform: "web"}); err != nil { + t.Fatal(err) + } + for _, step := range steps { + if err := writer.WriteStep(step); err != nil { + t.Fatal(err) + } + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + loaded, err := tracecorpus.Load(directory) + if err != nil { + t.Fatal(err) + } + return loaded +} + +func step(index int, screen string, action *trace.Action) trace.Step { + return trace.Step{Index: index, Screen: screen, NextAction: action} +} + +func violating( + index int, + screen string, + action *trace.Action, + property string, + origin int, + detected int, +) trace.Step { + violated := step(index, screen, action) + violated.Violations = []string{property} + violated.Witnesses = map[string]trace.Witness{ + property: {Reason: "predicate false", Step: origin, DetectedStep: detected}, + } + return violated +} + +func tap(selector string) *trace.Action { + return &trace.Action{Kind: "Tap", Selector: selector, X: 10, Y: 20} +} + +func typing(selector string, text string) *trace.Action { + return &trace.Action{Kind: "InputText", Selector: selector, Text: text, X: 10, Y: 20} +} diff --git a/cmd/internal-tools/defect-identity/main.go b/cmd/internal-tools/defect-identity/main.go new file mode 100644 index 0000000..a341f35 --- /dev/null +++ b/cmd/internal-tools/defect-identity/main.go @@ -0,0 +1,113 @@ +// Command defect-identity counts distinct defects across stored runs. A +// property reports at most once per run, so a run-level count is the number of +// properties violated; a defect is identified across runs by the property, the +// action attributed as the origin of the failed obligation, and the screen the +// witness observed. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "os" + "text/tabwriter" + + "github.com/priyanshujain/sanderling/internal/tracecorpus" +) + +func main() { + jsonOut := flag.Bool("json", false, "emit JSON instead of a table") + mode := flag.String( + "action-key", + string(bySelector), + "how much of the origin action identifies it: selector or full", + ) + flag.Usage = func() { + fmt.Fprintln( + os.Stderr, + "usage: defect-identity [--json] [--action-key selector|full] ...", + ) + } + flag.Parse() + if flag.NArg() == 0 { + flag.Usage() + os.Exit(2) + } + if *mode != string(bySelector) && *mode != string(byFullAction) { + fmt.Fprintf(os.Stderr, "unknown --action-key %q\n", *mode) + os.Exit(2) + } + + runs, err := loadAll(flag.Args()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if len(runs) == 0 { + fmt.Fprintln(os.Stderr, "no run directory found under the given paths") + os.Exit(1) + } + corpus, err := identify(runs, actionKeyMode(*mode)) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + if *jsonOut { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err := encoder.Encode(corpus); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + return + } + render(os.Stdout, corpus) +} + +func loadAll(paths []string) ([]tracecorpus.Run, error) { + var runs []tracecorpus.Run + for _, path := range paths { + directories, err := tracecorpus.Discover(path) + if err != nil { + return nil, err + } + for _, directory := range directories { + run, err := tracecorpus.Load(directory) + if err != nil { + return nil, fmt.Errorf("%s: %w", directory, err) + } + runs = append(runs, run) + } + } + return runs, nil +} + +func render(out io.Writer, corpus Corpus) { + writer := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) + fmt.Fprintln(writer, "property\torigin action\twitness screen\truns\tseeds") + for _, instance := range corpus.Instances { + screen := instance.WitnessScreen + if screen == "" { + screen = "(unnamed)" + } + fmt.Fprintf(writer, "%s\t%s\t%s\t%d\t%v\n", + instance.Property, instance.OriginAction, screen, + len(instance.Runs), instance.Seeds) + } + writer.Flush() + + fmt.Fprintf(out, "\n%d distinct defect(s) over %d run(s); %d seen in exactly one run\n", + len(corpus.Instances), corpus.Runs, corpus.Singletons()) + if corpus.UnnamedScreen > 0 { + fmt.Fprintf(out, + "%d violation(s) witnessed on a screen the app does not name, "+ + "so identity rests on property and origin action alone for those\n", + corpus.UnnamedScreen) + } + for _, gap := range corpus.Unattributed { + fmt.Fprintf(out, "unattributed: %s at step %d of %s (%s)\n", + gap.Property, gap.Step, gap.Run, gap.Reason) + } +}