sanderling/internal/runner/golden_test.go

package runner

import (
	"bufio"
	"bytes"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/priyanshujain/sanderling/internal/verifier"
)

// mustSeededVerifier builds a verifier with a fixed seed so the JS picker's PRNG
// is reproducible across runs, making the trace byte-stable.
func mustSeededVerifier(t *testing.T, spec string, seed uint64) *verifier.Verifier {
	t.Helper()
	instance, err := verifier.New(verifier.WithSeed(seed))
	if err != nil {
		t.Fatal(err)
	}
	if err := instance.Load(bundleSpec(t, spec)); err != nil {
		t.Fatal(err)
	}
	return instance
}

// goldenSeed and goldenSteps fix the run so the mock-driven trace and summary
// are byte-reproducible. Set UPDATE_GOLDEN=1 to rewrite the committed goldens;
// the checked-in files are the asserted source of truth.
const (
	goldenSeed  = 0x5eed
	goldenSteps = 4
)

// canonicalizeTrace rewrites the sole non-deterministic field, "timestamp"
// (wall-clock at step start), to a fixed zero value so equality compares only
// the run's observable shape. Every other field is a pure function of the seed,
// the spec, and the programmed mock hierarchy.
func canonicalizeTrace(t *testing.T, raw []byte) []byte {
	t.Helper()
	var out bytes.Buffer
	scanner := bufio.NewScanner(bytes.NewReader(raw))
	scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
	for scanner.Scan() {
		var line map[string]json.RawMessage
		if err := json.Unmarshal(scanner.Bytes(), &line); err != nil {
			t.Fatalf("decode trace line: %v", err)
		}
		line["timestamp"] = json.RawMessage(`"0001-01-01T00:00:00Z"`)
		encoded, err := json.Marshal(line)
		if err != nil {
			t.Fatalf("encode trace line: %v", err)
		}
		out.Write(encoded)
		out.WriteByte('\n')
	}
	if err := scanner.Err(); err != nil {
		t.Fatalf("scan trace: %v", err)
	}
	return out.Bytes()
}

func assertGolden(t *testing.T, name string, got []byte) {
	t.Helper()
	path := filepath.Join("testdata", name)
	if os.Getenv("UPDATE_GOLDEN") == "1" {
		if err := os.WriteFile(path, got, 0o644); err != nil {
			t.Fatalf("write golden %s: %v", name, err)
		}
		return
	}
	want, err := os.ReadFile(path)
	if err != nil {
		t.Fatalf("read golden %s (run with UPDATE_GOLDEN=1 to create): %v", name, err)
	}
	if !bytes.Equal(want, got) {
		t.Errorf("golden %s mismatch\n--- want ---\n%s\n--- got ---\n%s", name, want, got)
	}
}

// TestGolden_TraceStreamIsReproducible drives a fixed-seed, fixed-step run
// against the mock driver and snapshots the canonicalized trace.jsonl stream.
// A drift here means the run is no longer deterministic or the trace shape
// changed; both must be reviewed, not papered over.
func TestGolden_TraceStreamIsReproducible(t *testing.T) {
	state := newHarnessWithSpec(t, fixtureSpec)
	state.verifier = mustSeededVerifier(t, fixtureSpec, goldenSeed)
	state.mock.HierarchyJSON = `{"attributes":{"resource-id":"HomeScreen"},"children":[{"attributes":{"resource-id":"next","bounds":"[40,80,240,160]"},"children":[],"clickable":true,"enabled":true}]}`

	state.run(t, Options{IdleTimeout: 10 * time.Millisecond, MaxSteps: goldenSteps})
	raw, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl"))
	if err != nil {
		t.Fatal(err)
	}
	assertGolden(t, "trace.jsonl", canonicalizeTrace(t, raw))
}

// TestGolden_ViolationSummaryIsReproducible drives a known violation and
// snapshots the rendered summary string produced by RenderSummary (the same
// function the CLI prints). It also asserts the violated property is named, so
// a regression that drops the property from the report cannot pass silently.
func TestGolden_ViolationSummaryIsReproducible(t *testing.T) {
	state := newHarnessWithSpec(t, violationSpec)
	state.verifier = mustSeededVerifier(t, violationSpec, goldenSeed)

	summary := state.run(t, Options{IdleTimeout: 10 * time.Millisecond, MaxSteps: goldenSteps})

	var rendered bytes.Buffer
	RenderSummary(&rendered, summary, "android")

	if !strings.Contains(rendered.String(), "balanceNonNegative") {
		t.Errorf("summary must name the violated property balanceNonNegative, got:\n%s", rendered.String())
	}
	assertGolden(t, "violation-summary.txt", rendered.Bytes())
}