sanderling/internal/testrun/testrun.go

// Package testrun wires together the device, bundler, runner, and verifier into a single test pipeline.
package testrun

import (
	"context"
	"fmt"
	"io"
	"maps"
	"os"
	"path/filepath"
	"slices"
	"strconv"
	"strings"
	"time"

	"github.com/priyanshujain/sanderling/internal/android"
	"github.com/priyanshujain/sanderling/internal/bundler"
	"github.com/priyanshujain/sanderling/internal/driver"
	"github.com/priyanshujain/sanderling/internal/runner"
	"github.com/priyanshujain/sanderling/internal/trace"
	"github.com/priyanshujain/sanderling/internal/verifier"
)

const sidecarStartupTimeout = 30 * time.Second

// launchTimeout bounds the pre-run app launch. It happens before the runner
// starts, so --duration does not cover it, and Execute's context is the bare
// signal-aware root with no deadline of its own: a driver wedged here would
// hang the run forever having printed nothing and written no trace. Generous
// enough to sit above every driver's own launch bound (the iOS clear-state path
// reinstalls the app first) so a driver-level error is what a user usually
// sees, and this stays the backstop. A variable so the timeout test can shrink
// it.
var launchTimeout = 3 * time.Minute

// launchApp starts the app under test under a bounded context.
func launchApp(ctx context.Context, activeDriver driver.DeviceDriver, options Options) error {
	launchCtx, cancel := context.WithTimeout(ctx, launchTimeout)
	defer cancel()
	if err := activeDriver.Launch(launchCtx, options.BundleID, options.ClearData, nil); err != nil {
		return fmt.Errorf("launch app: %w", err)
	}
	return nil
}

// Options are the parameters for a single test pipeline run.
type Options struct {
	Spec           string
	BundleID       string
	Platform       string
	AVD            string
	Device         string
	IosDevice      string
	IosAppPath     string
	AndroidAppPath string
	Duration       time.Duration
	MaxSteps       int
	Seed           int64
	Output         string
	ClearData      bool
	// Arm labels the experiment cell this run belongs to and is recorded in
	// meta.json so a directory of runs can be attributed to a cell.
	Arm string
	// ExitOnViolation stops the run at the first violation and reports the
	// recorded violations as a ViolationsError, so a caller (CI) can tell
	// "the run found the bug" from "the run finished clean".
	ExitOnViolation bool
	// AllowNoProperties lets a run proceed against a spec that registers no
	// properties. The extraction and portability sweeps pass it: they measure
	// what a spec can read and where the generator reaches, and they report no
	// detection count. Every other run without it is a false green.
	AllowNoProperties bool
	// Generator selects the action picker: "llm" or the default seeded picker.
	Generator string
	// LabelSource selects how candidates are named to the model picker, and is
	// recorded in meta.json as part of the run's cell.
	LabelSource string

	// iosUDID, iosIsSimulator, and iosCoreDeviceID are filled by Execute after
	// resolving the iOS target, then read by buildDriver to choose the simulator
	// companion or the physical-device driver. On the device path iosUDID is the
	// hardware UDID and iosCoreDeviceID is the CoreDevice id.
	iosUDID         string
	iosIsSimulator  bool
	iosCoreDeviceID string
}

// Execute runs the full test pipeline: bundle, launch app, verify properties.
// buildRunMeta assembles the run's meta.json. Model and Instructions are
// recorded only when the LLM picker is the one that will actually run, so a
// spec that declares generator = llm() but is run under the seeded picker does
// not label its trace with a model it never called.
// runDevice reads whichever flag named the hardware for this platform. An ios
// run is selected with --ios-device and leaves --device empty.
func runDevice(options Options) string {
	if options.Platform == "ios" {
		return options.IosDevice
	}
	return options.Device
}

func buildRunMeta(options Options, bundleSHA256 string, seed int64, host string, llmConfig verifier.LLMConfig, hasLLMConfig bool) trace.Meta {
	meta := trace.Meta{
		Seed:              seed,
		SpecPath:          options.Spec,
		BundleSHA256:      bundleSHA256,
		Platform:          options.Platform,
		BundleID:          options.BundleID,
		StartedAt:         time.Now().UTC(),
		SanderlingVersion: "0.0.1",
		Arm:               options.Arm,
		Generator:         options.Generator,
		LabelSource:       options.LabelSource,
		MaxSteps:          options.MaxSteps,
		DurationMillis:    options.Duration.Milliseconds(),
		Host:              host,
		Device:            runDevice(options),
	}
	if options.Generator == "llm" && hasLLMConfig {
		meta.Model = llmConfig.Model
		meta.Instructions = llmConfig.Instructions
	}
	return meta
}

func Execute(ctx context.Context, options Options, stdout io.Writer) error {
	switch options.Platform {
	case "android":
		if err := android.EnsureDevice(ctx, options.Device, options.AVD, stdout); err != nil {
			return err
		}
		if err := android.PrepareDevice(ctx, options.Device, stdout); err != nil {
			return err
		}
		// Switch to 3-button navigation for the run so fuzzer swipes cannot
		// trigger the gesture-nav home/back and fling the app off screen;
		// restore the original mode when the run ends.
		restoreNav := android.ForceThreeButtonNav(ctx, options.Device, stdout)
		defer restoreNav()
	case "ios":
		resolved, err := resolveIOSTarget(ctx, options, stdout)
		if err != nil {
			return err
		}
		options = resolved
	}
	prep, err := prepareBundleInputs(options)
	if err != nil {
		return err
	}
	aliases := prep.aliases
	seed := prep.seed
	defines := prep.defines
	specAPIPath := prep.specAPIPath
	bundle, err := bundler.Bundle(bundler.Options{
		EntryFile:   options.Spec,
		RuntimeFile: prep.gojaRuntimePath,
		Defines:     defines,
		Aliases:     aliases,
	})
	if err != nil {
		return fmt.Errorf("bundle spec: %w", err)
	}
	fmt.Fprintf(stdout, "bundled spec: %d bytes (sha256=%s)\n", len(bundle.JavaScript), bundle.SHA256[:12])

	var webBundle bundler.Result
	if options.Platform == "web" {
		runtimePath := resolveWebRuntimePath(specAPIPath, options.Spec)
		if runtimePath == "" {
			return fmt.Errorf("web-runtime.ts not found near %s; checkout pkg/spec or set @sanderling/spec alias", options.Spec)
		}
		webBundle, err = bundler.BundleWeb(bundler.WebOptions{
			EntryFile:      options.Spec,
			WebRuntimeFile: runtimePath,
			Defines:        defines,
			Aliases:        aliases,
		})
		if err != nil {
			return fmt.Errorf("bundle web spec: %w", err)
		}
		fmt.Fprintf(stdout, "bundled web spec: %d bytes (sha256=%s)\n", len(webBundle.JavaScript), webBundle.SHA256[:12])
	}

	activeDriver, cleanup, err := buildDriver(ctx, options, stdout)
	if err != nil {
		return err
	}
	defer cleanup()

	if err := launchApp(ctx, activeDriver, options); err != nil {
		return err
	}

	if web, ok := activeDriver.(driver.WebDriver); ok && len(webBundle.JavaScript) > 0 {
		if err := web.InstallBundle(ctx, webBundle.JavaScript); err != nil {
			return fmt.Errorf("install web bundle: %w", err)
		}
	}

	verifierInstance, err := verifier.New(
		verifier.WithSeed(uint64(seed)),
		verifier.WithPlatform(options.Platform),
		verifier.WithAppPackage(options.BundleID),
	)
	if err != nil {
		return fmt.Errorf("verifier: %w", err)
	}
	if err := verifierInstance.Load(string(bundle.JavaScript)); err != nil {
		return fmt.Errorf("load spec: %w", err)
	}
	if !options.AllowNoProperties && len(verifierInstance.PropertyNames()) == 0 {
		return NoPropertiesError{Spec: options.Spec}
	}
	fmt.Fprintln(stdout, "spec loaded into verifier")

	runDirectory := filepath.Join(options.Output, time.Now().UTC().Format("20060102-150405"))
	traceWriter, err := trace.NewWriter(runDirectory)
	if err != nil {
		return fmt.Errorf("trace writer: %w", err)
	}
	defer traceWriter.Close()
	hostname, _ := os.Hostname()
	llmConfig, hasLLMConfig := verifierInstance.LLMConfig()
	meta := buildRunMeta(options, bundle.SHA256, seed, hostname, llmConfig, hasLLMConfig)
	if err := traceWriter.WriteMeta(meta); err != nil {
		return fmt.Errorf("trace meta: %w", err)
	}
	defer func() {
		endedAt := time.Now().UTC()
		meta.EndedAt = &endedAt
		_ = traceWriter.WriteMeta(meta)
	}()
	fmt.Fprintf(stdout, "trace dir: %s\n", runDirectory)

	if options.MaxSteps > 0 {
		fmt.Fprintf(stdout, "running for %s or %d steps, whichever comes first (seed=%d)\n", options.Duration, options.MaxSteps, seed)
	} else {
		fmt.Fprintf(stdout, "running for %s (seed=%d)\n", options.Duration, seed)
	}
	summary, err := runner.Run(ctx, runner.Options{
		Duration:        options.Duration,
		MaxSteps:        options.MaxSteps,
		IdleTimeout:     1 * time.Second,
		BundleID:        options.BundleID,
		Driver:          activeDriver,
		Verifier:        verifierInstance,
		TraceWriter:     traceWriter,
		Logger:          newProgressLogger(stdout),
		Generator:       options.Generator,
		LabelSource:     options.LabelSource,
		StopOnViolation: options.ExitOnViolation,
	})

	terminateCtx, terminateCancel := context.WithTimeout(context.Background(), 5*time.Second)
	_ = activeDriver.Terminate(terminateCtx)
	terminateCancel()

	if err != nil {
		return fmt.Errorf("runner: %w", err)
	}

	fmt.Fprintf(stdout, "\nelapsed: %s\n", summary.EndTime.Sub(summary.StartTime).Round(time.Millisecond))
	runner.RenderSummary(stdout, summary, options.Platform)
	return runOutcome(options, summary)
}

// runOutcome turns a finished run into the pipeline's result. Without
// --exit-on-violation a run that found violations is still a successful run
// (the summary reports them), which is the behaviour every existing caller
// depends on.
//
// A run none of whose steps reached the verifier fails whatever the flags say,
// because it holds no verdict to report. The threshold is every step and not a
// fraction of them: a screen that composes now and then costs a healthy android
// run a step or two, and a check that fired on those would be red on every run.
//
// A run whose generator dispatched no action fails on the same grounds, and the
// threshold is zero for the same reason: a generator with nothing to offer on
// some screens is ordinary, one with nothing to offer on every screen of a whole
// run drove nothing. The count is the generator's alone because a spec's setup
// drives the app before the generator is consulted, so a login that ran leaves
// dispatched actions behind whatever the generator then did.
// --exit-on-violation keeps precedence over it so a run that found something
// still exits on its evidence, and the property-free opt-out exempts the sweeps,
// whose measurement is where a generator reaches and for which "nowhere on this
// build" is a result rather than a broken run.
func runOutcome(options Options, summary runner.Summary) error {
	if summary.Steps > 0 && summary.SkippedVerification == summary.Steps {
		return VacuousRunError{Steps: summary.Steps}
	}
	if options.ExitOnViolation && len(summary.Violations) > 0 {
		return ViolationsError{Count: len(summary.Violations)}
	}
	if !options.AllowNoProperties && summary.Steps > 0 && summary.GeneratorActions == 0 {
		return NoGeneratorActionsError{
			Steps:          summary.Steps,
			SkippedActions: summary.SkippedActions,
		}
	}
	return nil
}

// ViolationsError reports a run that recorded violations under
// --exit-on-violation. It is deliberately distinct from every other error the
// pipeline returns: those mean the harness broke, this one means the run did
// its job and found something.
type ViolationsError struct {
	Count int
}

func (e ViolationsError) Error() string {
	return fmt.Sprintf("%d violation record(s)", e.Count)
}

// BundleSpec produces the goja bundle a run of this spec loaded, seeded as that
// run was. An offline replay of the run's trace has to load the same JavaScript
// the run did, and the seed is one of the bundle's defines, so it is part of
// the bundle's identity.
func BundleSpec(specPath string, seed int64) (bundler.Result, error) {
	inputs, err := prepareBundleInputs(Options{Spec: specPath, Seed: seed})
	if err != nil {
		return bundler.Result{}, err
	}
	return bundler.Bundle(bundler.Options{
		EntryFile:   specPath,
		RuntimeFile: inputs.gojaRuntimePath,
		Defines:     inputs.defines,
		Aliases:     inputs.aliases,
	})
}

// VacuousRunError reports a run in which no step reached the verifier, so no
// property ever judged anything. It is not a clean run and it is not a found
// bug: it is a run that produced no evidence either way, and the absence of
// violations in it says nothing about the app. It stays untyped to the CLI's
// violation path on purpose, so it exits 1 as a broken run rather than 2.
type VacuousRunError struct {
	Steps int
}

func (e VacuousRunError) Error() string {
	return fmt.Sprintf(
		"%d step(s) ran and none of them reached the verifier: the screen was "+
			"still moving every time it was read, so no property judged this run",
		e.Steps)
}

// NoPropertiesError reports a spec that bundled and loaded cleanly and holds no
// properties. Nothing is broken: the run would drive the app, fill a trace and
// report no violations having judged nothing, and that green says as much about
// the app as an unplugged meter says about a wire. It stays untyped to the CLI's
// violation path like VacuousRunError, so it exits 1 as a run that cannot
// produce a verdict rather than 2.
type NoPropertiesError struct {
	Spec string
}

func (e NoPropertiesError) Error() string {
	return fmt.Sprintf(
		"%s bundled and loaded into the verifier cleanly and registers no properties: "+
			"nothing is wrong with the spec and nothing is wrong with the run, but this run "+
			"would check nothing and report no violations. Pass --allow-no-properties for a "+
			"run that measures extraction or exploration instead of judging the app",
		e.Spec)
}

// NoGeneratorActionsError reports a run not one of whose steps was driven by the
// action generator. Setup can put the app in position, but only the generator
// explores it, so every screen this run judged was one setup left it on and its
// empty violation list says as much about the app as a spec with no properties
// would: the run observed, judged the same state over and over, and exercised
// nothing. It stays untyped to the CLI's violation path like VacuousRunError, so
// it exits 1 as a run that holds no verdict rather than 2.
type NoGeneratorActionsError struct {
	Steps int
	// SkippedActions is the runner's per-reason count of actions that never
	// reached the app, which is where the cause is: a picker with no candidate
	// reads differently from one whose every model call failed.
	SkippedActions map[string]int
}

func (e NoGeneratorActionsError) Error() string {
	return fmt.Sprintf(
		"%d step(s) ran and the action generator drove the app in none of them: whatever "+
			"the spec's setup did to get the app into position, nothing explored it from "+
			"there, so the run judged one screen over and over and its violation count "+
			"says nothing about the rest of the app%s",
		e.Steps, skipReasonSuffix(e.SkippedActions))
}

// skipReasonSuffix renders the skip-reason tally as a trailing clause, empty
// when the run recorded none.
func skipReasonSuffix(skipped map[string]int) string {
	if len(skipped) == 0 {
		return ""
	}
	reasons := make([]string, 0, len(skipped))
	for _, reason := range slices.Sorted(maps.Keys(skipped)) {
		reasons = append(reasons, fmt.Sprintf("%s %d", reason, skipped[reason]))
	}
	return ". Actions that never reached the app: " + strings.Join(reasons, ", ")
}

// bundleInputs holds the pre-driver assembly: alias map, seed, esbuild defines,
// and the resolved spec-API/goja-runtime paths the bundler consumes.
type bundleInputs struct {
	aliases         map[string]string
	seed            int64
	defines         map[string]string
	specAPIPath     string
	gojaRuntimePath string
}

// prepareBundleInputs builds the alias map, defines, seed, and resolves the
// goja runtime path. It is the pure (no driver/JVM) front half of Execute,
// returning the documented error when the runtime entry cannot be located.
func prepareBundleInputs(options Options) (bundleInputs, error) {
	aliases := map[string]string{}
	specAPIPath := resolveSpecAPIPath(options.Spec)
	if specAPIPath != "" {
		aliases["@sanderling/spec"] = specAPIPath
		base := filepath.Dir(specAPIPath)
		aliases["@sanderling/spec/defaults"] = filepath.Join(base, "defaults/index.ts")
		aliases["@sanderling/spec/defaults/properties"] = filepath.Join(base, "defaults/properties.ts")
	}
	seed := resolveSeed(options.Seed)
	defines := map[string]string{
		"SANDERLING_TEST_PHONE": os.Getenv("SANDERLING_TEST_PHONE"),
		"SANDERLING_TEST_OTP":   os.Getenv("SANDERLING_TEST_OTP"),
		"SANDERLING_SEED":       strconv.FormatInt(seed, 10),
	}
	gojaRuntimePath := resolveGojaRuntimePath(specAPIPath, options.Spec)
	if gojaRuntimePath == "" {
		return bundleInputs{}, fmt.Errorf("goja-runtime.ts not found near %s; checkout pkg/spec or set @sanderling/spec alias", options.Spec)
	}
	return bundleInputs{
		aliases:         aliases,
		seed:            seed,
		defines:         defines,
		specAPIPath:     specAPIPath,
		gojaRuntimePath: gojaRuntimePath,
	}, nil
}

// resolveSeed returns the configured seed, or a time-derived one when unset.
// The same value seeds both the goja PRNG and the web bundle's SANDERLING_SEED
// define, so a single run is reproducible across both runtimes.
func resolveSeed(configured int64) int64 {
	if configured != 0 {
		return configured
	}
	return time.Now().UnixNano()
}

// resolveWebRuntimePath returns the path to pkg/spec/src/web-runtime.ts.
func resolveWebRuntimePath(specAPIPath, userSpecPath string) string {
	return resolveRuntimeSibling(specAPIPath, userSpecPath, "web-runtime.ts")
}

// resolveGojaRuntimePath returns the path to pkg/spec/src/goja-runtime.ts, the
// native verifier's runtime entry that installs __sanderlingNextAction__.
func resolveGojaRuntimePath(specAPIPath, userSpecPath string) string {
	return resolveRuntimeSibling(specAPIPath, userSpecPath, "goja-runtime.ts")
}

// resolveRuntimeSibling finds a runtime-entry file that sits beside the spec-API
// index.ts. Tries the spec-API checkout first (so monorepo development works
// without publishing the package), then falls back to a node_modules path.
func resolveRuntimeSibling(specAPIPath, userSpecPath, filename string) string {
	if specAPIPath != "" {
		candidate := filepath.Join(filepath.Dir(specAPIPath), filename)
		if _, err := os.Stat(candidate); err == nil {
			return candidate
		}
	}
	if absoluteSpec, err := filepath.Abs(userSpecPath); err == nil {
		directory := filepath.Dir(absoluteSpec)
		for {
			candidate := filepath.Join(directory, "node_modules", "@sanderling", "spec", "src", filename)
			if _, err := os.Stat(candidate); err == nil {
				return candidate
			}
			parent := filepath.Dir(directory)
			if parent == directory {
				break
			}
			directory = parent
		}
	}
	return ""
}

// resolveSpecAPIPath returns the path to the spec API's index.ts: a sanderling
// source checkout first, searched upward from the spec file and the cwd, then
// an installed node_modules/@sanderling/spec. Aliasing the installed copy is
// what keeps the spec and the runtime entry on one module graph; resolving the
// bare specifier through package.json "exports" would load dist/ alongside the
// runtime's src/ and give sampler-rng.ts two instances.
func resolveSpecAPIPath(specPath string) string {
	var checkout, installed []string
	if absoluteSpec, err := filepath.Abs(specPath); err == nil {
		directory := filepath.Dir(absoluteSpec)
		for {
			checkout = append(checkout, filepath.Join(directory, "pkg/spec/src/index.ts"))
			installed = append(installed,
				filepath.Join(directory, "node_modules/@sanderling/spec/src/index.ts"))
			parent := filepath.Dir(directory)
			if parent == directory {
				break
			}
			directory = parent
		}
	}
	if cwd, err := os.Getwd(); err == nil {
		checkout = append(checkout, filepath.Join(cwd, "pkg/spec/src/index.ts"))
	}
	for _, candidate := range append(checkout, installed...) {
		if _, err := os.Stat(candidate); err == nil {
			return candidate
		}
	}
	return ""
}