sanderling/internal/testrun/testrun_test.go

package testrun

import (
	"context"
	"encoding/json"
	"errors"
	"io/fs"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/priyanshujain/sanderling/internal/driver"
	"github.com/priyanshujain/sanderling/internal/runner"
	"github.com/priyanshujain/sanderling/internal/trace"
	"github.com/priyanshujain/sanderling/internal/verifier"
)

func TestResolveSeed_UsesConfiguredWhenNonZero(t *testing.T) {
	if got := resolveSeed(42); got != 42 {
		t.Fatalf("got %d, want 42", got)
	}
}

func TestResolveSeed_DerivesWhenZero(t *testing.T) {
	if got := resolveSeed(0); got == 0 {
		t.Fatal("expected a non-zero time-derived seed")
	}
}

func TestPrepareBundleInputs_MissingGojaRuntime(t *testing.T) {
	root := t.TempDir()
	specPath := filepath.Join(root, "spec.ts")
	if err := os.WriteFile(specPath, []byte(""), 0o644); err != nil {
		t.Fatal(err)
	}
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = os.Chdir(cwd) })
	if err := os.Chdir(root); err != nil {
		t.Fatal(err)
	}

	_, err = prepareBundleInputs(Options{Spec: specPath})
	if err == nil || !strings.Contains(err.Error(), "goja-runtime.ts not found") ||
		!strings.Contains(err.Error(), "checkout pkg/spec or set @sanderling/spec alias") {
		t.Fatalf("got %v, want documented goja-runtime error", err)
	}
}

func TestPrepareBundleInputs_DerivesSpecAliases(t *testing.T) {
	root := t.TempDir()
	srcDir := filepath.Join(root, "pkg", "spec", "src")
	if err := os.MkdirAll(srcDir, 0o755); err != nil {
		t.Fatal(err)
	}
	apiPath := filepath.Join(srcDir, "index.ts")
	gojaPath := filepath.Join(srcDir, "goja-runtime.ts")
	for _, p := range []string{apiPath, gojaPath} {
		if err := os.WriteFile(p, []byte("export {}"), 0o644); err != nil {
			t.Fatal(err)
		}
	}
	specPath := filepath.Join(root, "examples", "spec.ts")
	if err := os.MkdirAll(filepath.Dir(specPath), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(specPath, []byte(""), 0o644); err != nil {
		t.Fatal(err)
	}

	prep, err := prepareBundleInputs(Options{Spec: specPath})
	if err != nil {
		t.Fatal(err)
	}
	want := map[string]string{
		"@sanderling/spec":                     apiPath,
		"@sanderling/spec/defaults":            filepath.Join(srcDir, "defaults/index.ts"),
		"@sanderling/spec/defaults/properties": filepath.Join(srcDir, "defaults/properties.ts"),
	}
	for key, wantValue := range want {
		if prep.aliases[key] != wantValue {
			t.Errorf("alias %q = %q, want %q", key, prep.aliases[key], wantValue)
		}
	}
	if len(prep.aliases) != len(want) {
		t.Errorf("got %d aliases, want %d: %v", len(prep.aliases), len(want), prep.aliases)
	}
	if prep.gojaRuntimePath != gojaPath {
		t.Errorf("gojaRuntimePath = %q, want %q", prep.gojaRuntimePath, gojaPath)
	}
}

func TestResolveSpecAPIPath_FindsUpwardSibling(t *testing.T) {
	root := t.TempDir()
	apiPath := filepath.Join(root, "pkg", "spec", "src", "index.ts")
	if err := os.MkdirAll(filepath.Dir(apiPath), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(apiPath, []byte("export {}"), 0o644); err != nil {
		t.Fatal(err)
	}
	specPath := filepath.Join(root, "examples", "app", "spec.ts")
	if err := os.MkdirAll(filepath.Dir(specPath), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(specPath, []byte(""), 0o644); err != nil {
		t.Fatal(err)
	}

	got := resolveSpecAPIPath(specPath)
	if got != apiPath {
		t.Fatalf("got %q, want %q", got, apiPath)
	}
}

func TestResolveRuntimeSibling(t *testing.T) {
	const filename = "goja-runtime.ts"

	siblingRoot := t.TempDir()
	siblingAPI := filepath.Join(siblingRoot, "pkg", "spec", "src", "index.ts")
	if err := os.MkdirAll(filepath.Dir(siblingAPI), 0o755); err != nil {
		t.Fatal(err)
	}
	siblingFile := filepath.Join(filepath.Dir(siblingAPI), filename)
	if err := os.WriteFile(siblingFile, []byte(""), 0o644); err != nil {
		t.Fatal(err)
	}

	nmRoot := t.TempDir()
	nmFile := filepath.Join(nmRoot, "node_modules", "@sanderling", "spec", "src", filename)
	if err := os.MkdirAll(filepath.Dir(nmFile), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(nmFile, []byte(""), 0o644); err != nil {
		t.Fatal(err)
	}
	nmSpec := filepath.Join(nmRoot, "examples", "deep", "spec.ts")
	if err := os.MkdirAll(filepath.Dir(nmSpec), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(nmSpec, []byte(""), 0o644); err != nil {
		t.Fatal(err)
	}

	missingSpec := filepath.Join(t.TempDir(), "spec.ts")
	if err := os.WriteFile(missingSpec, []byte(""), 0o644); err != nil {
		t.Fatal(err)
	}

	tests := []struct {
		name        string
		specAPIPath string
		userSpec    string
		want        string
	}{
		{"sibling next to spec-API wins", siblingAPI, missingSpec, siblingFile},
		{"node_modules fallback upward", "", nmSpec, nmFile},
		{"neither reachable returns empty", "", missingSpec, ""},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := resolveRuntimeSibling(tt.specAPIPath, tt.userSpec, filename); got != tt.want {
				t.Fatalf("got %q, want %q", got, tt.want)
			}
		})
	}
}

func TestResolveSpecAPIPath_ReturnsEmptyWhenMissing(t *testing.T) {
	root := t.TempDir()
	specPath := filepath.Join(root, "spec.ts")
	if err := os.WriteFile(specPath, []byte(""), 0o644); err != nil {
		t.Fatal(err)
	}

	cwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = os.Chdir(cwd) })
	if err := os.Chdir(root); err != nil {
		t.Fatal(err)
	}

	got := resolveSpecAPIPath(specPath)
	if got != "" {
		t.Fatalf("got %q, want empty (no sanderling source tree reachable)", got)
	}
}

func TestBuildRunMeta_RecordsArmMembership(t *testing.T) {
	options := Options{
		Spec:        "spec.ts",
		BundleID:    "com.example",
		Platform:    "android",
		Duration:    3 * time.Minute,
		MaxSteps:    300,
		Arm:         "llm-visible-text",
		Generator:   "llm",
		LabelSource: verifier.LabelSourceVisibleText,
	}
	meta := buildRunMeta(options, "deadbeef", 7, "farm-01",
		verifier.LLMConfig{Model: "claude-sonnet-5", Instructions: "exercise the outbox"}, true)

	if meta.Arm != "llm-visible-text" || meta.Generator != "llm" {
		t.Errorf("arm membership: got arm=%q generator=%q", meta.Arm, meta.Generator)
	}
	if meta.LabelSource != verifier.LabelSourceVisibleText {
		t.Errorf("label source: got %q, want %q", meta.LabelSource, verifier.LabelSourceVisibleText)
	}
	if meta.Model != "claude-sonnet-5" || meta.Instructions != "exercise the outbox" {
		t.Errorf("llm config: got model=%q instructions=%q", meta.Model, meta.Instructions)
	}
	if meta.MaxSteps != 300 || meta.DurationMillis != 180000 {
		t.Errorf("budget: got maxSteps=%d durationMillis=%d", meta.MaxSteps, meta.DurationMillis)
	}
	if meta.Host != "farm-01" || meta.Seed != 7 {
		t.Errorf("host and seed: got host=%q seed=%d", meta.Host, meta.Seed)
	}
}

// The device a run drove has to survive in the run's own artifact. Held only in
// the campaign's runs.jsonl, a trace read on its own cannot say which emulator,
// or which API level, produced it.
func TestBuildRunMeta_RecordsTheDeviceInMetaJSON(t *testing.T) {
	directory := t.TempDir()
	writer, err := trace.NewWriter(directory)
	if err != nil {
		t.Fatal(err)
	}
	defer writer.Close()

	options := Options{Platform: "android", Generator: "seeded", Duration: time.Minute, Device: "emulator-5556"}
	if err := writer.WriteMeta(buildRunMeta(options, "deadbeef", 3, "farm-01", verifier.LLMConfig{}, false)); err != nil {
		t.Fatal(err)
	}

	body, err := os.ReadFile(filepath.Join(directory, "meta.json"))
	if err != nil {
		t.Fatal(err)
	}
	var stored trace.Meta
	if err := json.Unmarshal(body, &stored); err != nil {
		t.Fatalf("meta.json is not valid JSON: %v\n%s", err, body)
	}
	if stored.Device != "emulator-5556" {
		t.Errorf("device in meta.json: got %q, want emulator-5556\n%s", stored.Device, body)
	}
}

func TestBuildRunMeta_OmitsModelWhenSeededPickerRuns(t *testing.T) {
	options := Options{Platform: "android", Generator: "seeded", Duration: time.Minute}
	meta := buildRunMeta(options, "deadbeef", 1, "farm-01",
		verifier.LLMConfig{Model: "claude-sonnet-5", Instructions: "hunt bugs"}, true)

	if meta.Model != "" || meta.Instructions != "" {
		t.Errorf("a seeded run must not be labelled with a model it never called: model=%q instructions=%q",
			meta.Model, meta.Instructions)
	}
}

// TestBuildRunMeta_RecordsLabelSourceForASeededRun is the deliberate difference
// from Model and Instructions above. The seeded picker never reads a label, but
// the run still belongs to a labelling cell, and the pair of seeded runs across
// the two cells is the manipulation check. Omitting it here would leave those
// two runs indistinguishable in the artifact.
func TestBuildRunMeta_RecordsLabelSourceForASeededRun(t *testing.T) {
	options := Options{
		Platform:    "android",
		Generator:   "seeded",
		Duration:    time.Minute,
		LabelSource: verifier.LabelSourceResourceID,
	}
	meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false)

	if meta.LabelSource != verifier.LabelSourceResourceID {
		t.Errorf("label source: got %q, want %q", meta.LabelSource, verifier.LabelSourceResourceID)
	}
}

func TestBuildRunMeta_OmitsModelWhenSpecDeclaresNoLLMGenerator(t *testing.T) {
	options := Options{Platform: "android", Generator: "llm", Duration: time.Minute}
	meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false)

	if meta.Model != "" {
		t.Errorf("model recorded without a spec-declared llm generator: %q", meta.Model)
	}
}

// TestRunOutcome_ReportsViolationsOnlyUnderTheFlag pins the CI contract: the
// typed error is what makes `sanderling test` exit 2, and it must appear only
// when the caller asked for it. A run that finds violations without the flag
// stays a successful run, which is what every existing invocation expects.
func TestRunOutcome_ReportsViolationsOnlyUnderTheFlag(t *testing.T) {
	violated := runner.Summary{
		Steps:             7,
		DispatchedActions: 7,
		Violations:        []runner.ViolationRecord{{StepIndex: 3, Properties: []string{"balanceMoves"}}},
	}
	clean := runner.Summary{Steps: 7, DispatchedActions: 7}

	if err := runOutcome(Options{}, violated); err != nil {
		t.Errorf("without --exit-on-violation a violated run must succeed, got %v", err)
	}
	if err := runOutcome(Options{ExitOnViolation: true}, clean); err != nil {
		t.Errorf("a clean run must succeed under --exit-on-violation, got %v", err)
	}

	err := runOutcome(Options{ExitOnViolation: true}, violated)
	var violations ViolationsError
	if !errors.As(err, &violations) {
		t.Fatalf("expected a ViolationsError, got %v", err)
	}
	if violations.Count != 1 {
		t.Errorf("count: got %d, want 1", violations.Count)
	}
}

// A step the verifier skipped was judged by nothing, so a run whose every step
// was skipped holds no verdict at all: "no violations" there is the absence of
// an answer rather than a clean one. Reporting it as a successful run is the
// green and vacuous outcome structuralShape's own design notes call worse than
// the composition it catches, and the runner's hold is what makes a fully
// skipped run reachable.
func TestRunOutcome_ARunThatJudgedNothingIsNotASuccess(t *testing.T) {
	nothingJudged := runner.Summary{Steps: 6, SkippedVerification: 6}
	err := runOutcome(Options{}, nothingJudged)
	var vacuous VacuousRunError
	if !errors.As(err, &vacuous) {
		t.Fatalf("a run that judged none of its 6 steps came back %v, want a VacuousRunError", err)
	}
	if vacuous.Steps != 6 {
		t.Errorf("steps: got %d, want 6", vacuous.Steps)
	}

	// A screen that composes now and then costs a run steps, not its verdict. A
	// check that fired here would turn every healthy android run red.
	mostlyJudged := runner.Summary{Steps: 6, SkippedVerification: 5, DispatchedActions: 1}
	if err := runOutcome(Options{}, mostlyJudged); err != nil {
		t.Errorf("a run that judged one of its 6 steps must succeed, got %v", err)
	}
}

// A run that dispatched no action at all observed one screen for its whole life
// and never drove the app, so its empty violation list is what an unplugged
// instrument reports. The provider rate-limiting a model picker is the way this
// happens in practice: every step ends with the picker handing back nothing.
func TestRunOutcome_ARunThatDroveNothingIsNotASuccess(t *testing.T) {
	droveNothing := runner.Summary{
		Steps:          200,
		SkippedActions: map[string]int{"no_action_produced": 200},
	}
	err := runOutcome(Options{}, droveNothing)
	var dead NoActionsDispatchedError
	if !errors.As(err, &dead) {
		t.Fatalf("a run that dispatched none of its 200 steps' actions came back %v, "+
			"want a NoActionsDispatchedError", err)
	}
	if dead.Steps != 200 {
		t.Errorf("steps: got %d, want 200", dead.Steps)
	}
	if !strings.Contains(dead.Error(), "no_action_produced") {
		t.Errorf("the error never names why nothing was dispatched: %v", dead)
	}

	// One action is exploration, however little. A check that fired here would
	// be red on any run whose screen offers the generator nothing for a while.
	droveOnce := runner.Summary{Steps: 200, DispatchedActions: 1}
	if err := runOutcome(Options{}, droveOnce); err != nil {
		t.Errorf("a run that dispatched one action must succeed, got %v", err)
	}

	// The sweeps that measure what a spec extracts and where a generator reaches
	// ask for a run that judges nothing by name, and "the generator reached
	// nothing here" is their measurement rather than their failure.
	if err := runOutcome(Options{AllowNoProperties: true}, droveNothing); err != nil {
		t.Errorf("the property-free opt-out no longer carries a run through, got %v", err)
	}

	// A run cut short before it took a step never got going, which the deadline
	// and the step count already say.
	if err := runOutcome(Options{}, runner.Summary{}); err != nil {
		t.Errorf("a run with no steps at all must not report as dead-on-arrival, got %v", err)
	}

	// CI reads exit 2 as "the run found the bug". A run whose first screen
	// already violated must keep reporting that, or the found bug is downgraded
	// to a broken harness.
	foundOnTheFirstScreen := droveNothing
	foundOnTheFirstScreen.Violations = []runner.ViolationRecord{
		{StepIndex: 1, Properties: []string{"balanceNonNegative"}},
	}
	err = runOutcome(Options{ExitOnViolation: true}, foundOnTheFirstScreen)
	var violations ViolationsError
	if !errors.As(err, &violations) {
		t.Errorf("a run that found a violation came back %v, want a ViolationsError", err)
	}
}

// wedgedLaunchDriver never returns from Launch, standing in for a driver whose
// device-side session is stuck.
type wedgedLaunchDriver struct {
	driver.DeviceDriver
	release chan struct{}
}

func (w *wedgedLaunchDriver) Launch(ctx context.Context, _ string, _ bool, _ map[string]string) error {
	select {
	case <-ctx.Done():
		return ctx.Err()
	case <-w.release:
		return nil
	}
}

// TestLaunchAppBoundsWedgedDriver proves the pre-run launch carries a deadline.
// It runs before the runner starts, so --duration does not cover it and
// Execute's root context has no deadline: unbounded, a wedged driver hangs the
// run forever with no trace directory and no error.
func TestLaunchAppBoundsWedgedDriver(t *testing.T) {
	previous := launchTimeout
	launchTimeout = 100 * time.Millisecond
	defer func() { launchTimeout = previous }()

	wedged := &wedgedLaunchDriver{release: make(chan struct{})}
	defer close(wedged.release)

	done := make(chan error, 1)
	go func() { done <- launchApp(context.Background(), wedged, Options{BundleID: "com.example.app"}) }()

	select {
	case err := <-done:
		if !errors.Is(err, context.DeadlineExceeded) {
			t.Fatalf("err = %v, want a deadline-exceeded error", err)
		}
	case <-time.After(10 * time.Second):
		t.Fatal("launchApp never returned: the pre-run launch is unbounded, so a wedged driver hangs the run forever")
	}
}

// repoFile walks up from the test's working directory and returns the absolute
// path of rel inside the sanderling checkout.
func repoFile(t *testing.T, rel string) string {
	t.Helper()
	directory, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	for {
		candidate := filepath.Join(directory, rel)
		if _, err := os.Stat(candidate); err == nil {
			return candidate
		}
		parent := filepath.Dir(directory)
		if parent == directory {
			t.Fatalf("%s not found above the test directory", rel)
		}
		directory = parent
	}
}

// publishedFiles returns the "files" entries of pkg/spec/package.json, the
// exact set npm ships in the @sanderling/spec tarball.
func publishedFiles(t *testing.T) []string {
	t.Helper()
	raw, err := os.ReadFile(repoFile(t, "pkg/spec/package.json"))
	if err != nil {
		t.Fatal(err)
	}
	var manifest struct {
		Files []string `json:"files"`
	}
	if err := json.Unmarshal(raw, &manifest); err != nil {
		t.Fatal(err)
	}
	return manifest.Files
}

// installPublishedPackage reproduces what `npm install @sanderling/spec`
// unpacks into node_modules: only the paths package.json publishes.
func installPublishedPackage(t *testing.T, dest string) {
	t.Helper()
	specDir := filepath.Dir(repoFile(t, "pkg/spec/package.json"))
	for _, entry := range publishedFiles(t) {
		source := filepath.Join(specDir, entry)
		if _, err := os.Stat(source); err != nil {
			continue
		}
		copyTree(t, source, filepath.Join(dest, entry))
	}
}

func copyTree(t *testing.T, source, dest string) {
	t.Helper()
	err := filepath.WalkDir(source, func(path string, entry fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		relative, err := filepath.Rel(source, path)
		if err != nil {
			return err
		}
		target := filepath.Join(dest, relative)
		if entry.IsDir() {
			return os.MkdirAll(target, 0o755)
		}
		data, err := os.ReadFile(path)
		if err != nil {
			return err
		}
		if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
			return err
		}
		return os.WriteFile(target, data, 0o644)
	})
	if err != nil {
		t.Fatal(err)
	}
}

// TestResolveRuntimeSibling_PublishedPackageShipsTheRuntimes pins npm's "files"
// list against the resolver that consumes it. The tarball shipped dist/ alone
// while the node_modules fallback looks for src/goja-runtime.ts, so every
// `npm install @sanderling/spec` user hit "goja-runtime.ts not found".
func TestResolveRuntimeSibling_PublishedPackageShipsTheRuntimes(t *testing.T) {
	root := t.TempDir()
	installPublishedPackage(t, filepath.Join(root, "node_modules", "@sanderling", "spec"))
	specPath := filepath.Join(root, "spec.ts")
	if err := os.WriteFile(specPath, []byte(""), 0o644); err != nil {
		t.Fatal(err)
	}

	for _, filename := range []string{"goja-runtime.ts", "web-runtime.ts"} {
		if resolveRuntimeSibling("", specPath, filename) == "" {
			t.Errorf("%s unreachable from a published install; package.json publishes %v",
				filename, publishedFiles(t))
		}
	}
}

// TestPrepareBundleInputs_InstalledPackageSharesOneModuleGraph pins the
// downstream case: with no sanderling checkout above the spec, the aliases and
// the runtime entry must name the SAME installed copy. An unset alias let
// esbuild resolve @sanderling/spec to dist/ while the runtime came from src/,
// which loads sampler-rng.ts twice; from(), strings(), integers() and emails()
// then read an rng the picker never set and collapse to a fixed default.
func TestPrepareBundleInputs_InstalledPackageSharesOneModuleGraph(t *testing.T) {
	root := t.TempDir()
	installed := filepath.Join(root, "node_modules", "@sanderling", "spec")
	installPublishedPackage(t, installed)
	specPath := filepath.Join(root, "sanderling", "spec.ts")
	if err := os.MkdirAll(filepath.Dir(specPath), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(specPath, []byte(""), 0o644); err != nil {
		t.Fatal(err)
	}

	cwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = os.Chdir(cwd) })
	if err := os.Chdir(root); err != nil {
		t.Fatal(err)
	}

	prep, err := prepareBundleInputs(Options{Spec: specPath})
	if err != nil {
		t.Fatal(err)
	}
	source := filepath.Join(installed, "src")
	want := map[string]string{
		"@sanderling/spec":                     filepath.Join(source, "index.ts"),
		"@sanderling/spec/defaults":            filepath.Join(source, "defaults/index.ts"),
		"@sanderling/spec/defaults/properties": filepath.Join(source, "defaults/properties.ts"),
	}
	for key, wantValue := range want {
		if prep.aliases[key] != wantValue {
			t.Errorf("alias %q = %q, want %q", key, prep.aliases[key], wantValue)
		}
	}
	if got := prep.gojaRuntimePath; got != filepath.Join(source, "goja-runtime.ts") {
		t.Errorf("gojaRuntimePath = %q, want it beside the aliased index.ts", got)
	}
	if got := resolveWebRuntimePath(prep.specAPIPath, specPath); got != filepath.Join(source, "web-runtime.ts") {
		t.Errorf("webRuntimePath = %q, want it beside the aliased index.ts", got)
	}
}