sanderling/cmd/internal-tools/defect-identity/identity_test.go

package main

import (
	"strings"
	"testing"

	"github.com/priyanshujain/sanderling/internal/trace"
	"github.com/priyanshujain/sanderling/internal/tracecorpus"
	"github.com/priyanshujain/sanderling/internal/verifier"
)

// TestOneDefectSeenTwiceIsOneInstance: two runs report the same property from
// the same origin action on the same screen, which is one defect found twice
// and two properties violated.
func TestOneDefectSeenTwiceIsOneInstance(t *testing.T) {
	first := run(t, 3,
		step(1, "/ledger", tap("id:add-txn")),
		violating(2, "/accounts/7", tap("id:save"), "balanceMatches", 2, 2),
	)
	second := run(t, 5,
		step(1, "/ledger", tap("id:add-txn")),
		violating(2, "/accounts/7", tap("id:save"), "balanceMatches", 2, 2),
	)

	corpus := identified(t, bySelector, first, second)
	if len(corpus.Instances) != 1 {
		t.Fatalf("instances = %d, want 1: %+v", len(corpus.Instances), corpus.Instances)
	}
	instance := corpus.Instances[0]
	if len(instance.Runs) != 2 || instance.Reports != 2 {
		t.Fatalf("instance = %+v, want two runs reporting it", instance)
	}
	if corpus.Singletons() != 0 {
		t.Fatalf("singletons = %d, want 0", corpus.Singletons())
	}
}

func TestTheSamePropertyFromTwoOriginsIsTwoDefects(t *testing.T) {
	first := run(t, 3, violating(1, "/ledger", tap("id:save"), "balanceMatches", 1, 1))
	second := run(t, 5, violating(1, "/ledger", tap("id:delete"), "balanceMatches", 1, 1))

	corpus := identified(t, bySelector, first, second)
	if len(corpus.Instances) != 2 {
		t.Fatalf("instances = %d, want 2: %+v", len(corpus.Instances), corpus.Instances)
	}
	if corpus.Singletons() != 2 {
		t.Fatalf("singletons = %d, want 2", corpus.Singletons())
	}
}

func TestTheSamePropertyOnTwoScreensIsTwoDefects(t *testing.T) {
	first := run(t, 3, violating(1, "/ledger", tap("id:save"), "balanceMatches", 1, 1))
	second := run(t, 5, violating(1, "/home", tap("id:save"), "balanceMatches", 1, 1))

	corpus := identified(t, bySelector, first, second)
	if len(corpus.Instances) != 2 {
		t.Fatalf("instances = %d, want 2: %+v", len(corpus.Instances), corpus.Instances)
	}
}

// TestADeferredViolationTakesTheActionOnItsOriginLine holds the alignment the
// draft states: the action recorded on line k is the one applied after
// observing k, so an obligation armed at k is attributed to that action and
// witnessed on the screen the detection step observed.
func TestADeferredViolationTakesTheActionOnItsOriginLine(t *testing.T) {
	only := run(t, 3,
		step(1, "/login", tap("id:login-submit")),
		step(2, "/home", tap("id:open-ledger")),
		violating(3, "/ledger", tap("id:add-txn"), "landsOnLedger", 2, 3),
	)

	corpus := identified(t, bySelector, only)
	instance := corpus.Instances[0]
	if instance.OriginAction != "Tap id:open-ledger" {
		t.Fatalf("origin action = %q, want the action on the origin line", instance.OriginAction)
	}
	if instance.WitnessScreen != "/ledger" {
		t.Fatalf("witness screen = %q, want the detection step's screen", instance.WitnessScreen)
	}
}

func TestAViolationWithNoWitnessIsReportedNotCounted(t *testing.T) {
	unattributed := trace.Step{
		Index:      1,
		Screen:     "/ledger",
		Violations: []string{"balanceMatches"},
	}
	corpus := identified(t, bySelector, run(t, 3, unattributed))

	if len(corpus.Instances) != 0 {
		t.Fatalf("instances = %d, want 0: %+v", len(corpus.Instances), corpus.Instances)
	}
	if len(corpus.Unattributed) != 1 ||
		corpus.Unattributed[0].Property != "balanceMatches" {
		t.Fatalf("unattributed = %+v, want the one violation with no origin", corpus.Unattributed)
	}
}

// TestTheStrictActionKeySplitsWhatTheSelectorKeyMerges quantifies the reading
// the draft leaves open: two runs that typed different text into the same
// field are one defect by selector and two by the whole action.
func TestTheStrictActionKeySplitsWhatTheSelectorKeyMerges(t *testing.T) {
	first := run(t, 3, violating(1, "/ledger", typing("id:amount", "12"), "balanceMatches", 1, 1))
	second := run(t, 5, violating(1, "/ledger", typing("id:amount", "9000"), "balanceMatches", 1, 1))

	if got := identified(t, bySelector, first, second); len(got.Instances) != 1 {
		t.Fatalf("by selector: instances = %d, want 1", len(got.Instances))
	}
	if got := identified(t, byFullAction, first, second); len(got.Instances) != 2 {
		t.Fatalf("by full action: instances = %d, want 2", len(got.Instances))
	}
}

// TestARedactedTypedValueDegradesTheFullKeyVisibly: two runs typed different
// values into one field, both reached the trace redacted, and the whole action
// can no longer tell them apart. The pair is one row, and the report has to say
// so rather than let it read as one defect found twice.
func TestARedactedTypedValueDegradesTheFullKeyVisibly(t *testing.T) {
	first := run(t, 3, violating(1, "/login",
		typing("id:password", recordedText(t, "hunter2")), "staysSignedIn", 1, 1))
	second := run(t, 5, violating(1, "/login",
		typing("id:password", recordedText(t, "correct horse")), "staysSignedIn", 1, 1))

	corpus := identified(t, byFullAction, first, second)
	if len(corpus.Instances) != 1 {
		t.Fatalf("instances = %d, want the redacted pair to be one row: %+v",
			len(corpus.Instances), corpus.Instances)
	}
	report := rendered(corpus)
	if !strings.Contains(report, "1 identity") || !strings.Contains(report, "redacted") {
		t.Fatalf("report does not say one identity rests on a redacted value:\n%s", report)
	}
}

func TestARedactedOriginKeepsTheSelectorApart(t *testing.T) {
	first := run(t, 3, violating(1, "/login",
		typing("id:password", recordedText(t, "hunter2")), "staysSignedIn", 1, 1))
	second := run(t, 5, violating(1, "/login",
		typing("id:pin", recordedText(t, "hunter2")), "staysSignedIn", 1, 1))

	corpus := identified(t, byFullAction, first, second)
	if len(corpus.Instances) != 2 {
		t.Fatalf("instances = %d, want two fields to stay two rows: %+v",
			len(corpus.Instances), corpus.Instances)
	}
	if report := rendered(corpus); !strings.Contains(report, "2 identity") {
		t.Fatalf("report does not count both degraded identities:\n%s", report)
	}
}

func TestARedactedOriginDegradesNothingUnderTheSelectorKey(t *testing.T) {
	only := run(t, 3, violating(1, "/login",
		typing("id:password", recordedText(t, "hunter2")), "staysSignedIn", 1, 1))

	if report := rendered(identified(t, bySelector, only)); strings.Contains(report, "redacted") {
		t.Fatalf("selector key reads no text, so nothing degrades:\n%s", report)
	}
}

// recordedText renders a typed value the way the runner records it, so what the
// key sees is redaction as it really happens and not a placeholder the test
// wrote itself.
func recordedText(t *testing.T, typed string) string {
	t.Helper()
	recorded := verifier.RecordedActionText(verifier.Action{
		Kind: verifier.ActionKindInputText,
		On:   "id:password",
		Text: typed,
	}, nil)
	if recorded == typed {
		t.Fatalf("typed value %q reached the record unredacted", typed)
	}
	return recorded
}

func rendered(corpus Corpus) string {
	var report strings.Builder
	render(&report, corpus)
	return report.String()
}

func identified(t *testing.T, mode actionKeyMode, runs ...tracecorpus.Run) Corpus {
	t.Helper()
	corpus, err := identify(runs, mode)
	if err != nil {
		t.Fatal(err)
	}
	return corpus
}

func run(t *testing.T, seed int64, steps ...trace.Step) tracecorpus.Run {
	t.Helper()
	directory := t.TempDir()
	writer, err := trace.NewWriter(directory)
	if err != nil {
		t.Fatal(err)
	}
	if err := writer.WriteMeta(trace.Meta{Seed: seed, Platform: "web"}); err != nil {
		t.Fatal(err)
	}
	for _, step := range steps {
		if err := writer.WriteStep(step); err != nil {
			t.Fatal(err)
		}
	}
	if err := writer.Close(); err != nil {
		t.Fatal(err)
	}
	loaded, err := tracecorpus.Load(directory)
	if err != nil {
		t.Fatal(err)
	}
	return loaded
}

func step(index int, screen string, action *trace.Action) trace.Step {
	return trace.Step{Index: index, Screen: screen, NextAction: action}
}

func violating(
	index int,
	screen string,
	action *trace.Action,
	property string,
	origin int,
	detected int,
) trace.Step {
	violated := step(index, screen, action)
	violated.Violations = []string{property}
	violated.Witnesses = map[string]trace.Witness{
		property: {Reason: "predicate false", Step: origin, DetectedStep: detected},
	}
	return violated
}

func tap(selector string) *trace.Action {
	return &trace.Action{Kind: "Tap", Selector: selector, X: 10, Y: 20}
}

func typing(selector string, text string) *trace.Action {
	return &trace.Action{Kind: "InputText", Selector: selector, Text: text, X: 10, Y: 20}
}