feat(campaign): sweep seeds for one experiment cell

campaign.json lists the seeds a sweep intended to run and is written before
the first run, so a host that dropped runs shows up as missing seeds rather
than as a smaller sample. Seed 0 is rejected: sanderling test reads it as
"derive a seed from the clock", which is why conformance/gates.sh controls
nothing today.

Each run contributes one runs.jsonl line carrying steps to first violation by
origin step, the step that armed the failed obligation, so the survival
analysis never reopens a trace.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX

cmd/internal-tools/campaign/campaign.go

@@ -0,0 +1,219 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+// commandExecutor runs one sanderling invocation and returns its exit code.
+// A non-nil error means the process could not be run at all, which is a
+// different failure from a run that started and exited non-zero.
+type commandExecutor func(ctx context.Context, binary string, arguments []string, output io.Writer) (int, error)
+
+func executeCommand(ctx context.Context, binary string, arguments []string, output io.Writer) (int, error) {
+	command := exec.CommandContext(ctx, binary, arguments...)
+	command.Stdout = output
+	command.Stderr = output
+	err := command.Run()
+	if err == nil {
+		return 0, nil
+	}
+	var exitError *exec.ExitError
+	if errors.As(err, &exitError) {
+		return exitError.ExitCode(), nil
+	}
+	return -1, err
+}
+
+// runRecord is one line of runs.jsonl.
+type runRecord struct {
+	Seed           int64     `json:"seed"`
+	Device         string    `json:"device,omitempty"`
+	ExitCode       int       `json:"exit_code"`
+	LaunchError    string    `json:"launch_error,omitempty"`
+	StartedAt      time.Time `json:"started_at"`
+	DurationMillis int64     `json:"duration_millis"`
+	RunDirectory   string    `json:"run_directory,omitempty"`
+	TraceError     string    `json:"trace_error,omitempty"`
+	traceSummary
+}
+
+type campaign struct {
+	configuration config
+	executor      commandExecutor
+	stdout        io.Writer
+	records       io.Writer
+	mutex         sync.Mutex
+	failures      int
+	unreadable    int
+}
+
+func runCampaign(ctx context.Context, configuration config, executor commandExecutor, stdout io.Writer) error {
+	if _, err := os.Stat(filepath.Join(configuration.outputDirectory, manifestFileName)); err == nil {
+		return fmt.Errorf("%s already exists in %s: pick a fresh --output so two campaigns do not share a directory",
+			manifestFileName, configuration.outputDirectory)
+	}
+	if err := os.MkdirAll(configuration.outputDirectory, 0o755); err != nil {
+		return fmt.Errorf("create campaign dir: %w", err)
+	}
+	binaryPath := configuration.sanderlingPath
+	if resolved, err := exec.LookPath(binaryPath); err == nil {
+		if absolute, err := filepath.Abs(resolved); err == nil {
+			binaryPath = absolute
+		}
+	}
+	version, err := probeVersion(ctx, configuration.sanderlingPath, executor)
+	if err != nil {
+		return err
+	}
+	host, _ := os.Hostname()
+	if err := writeManifest(configuration.outputDirectory, buildManifest(configuration, host, binaryPath, version, time.Now().UTC())); err != nil {
+		return fmt.Errorf("write %s: %w", manifestFileName, err)
+	}
+
+	recordsFile, err := os.OpenFile(filepath.Join(configuration.outputDirectory, recordsFileName),
+		os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
+	if err != nil {
+		return fmt.Errorf("open %s: %w", recordsFileName, err)
+	}
+	defer recordsFile.Close()
+
+	sweep := &campaign{configuration: configuration, executor: executor, stdout: stdout, records: recordsFile}
+	fmt.Fprintf(stdout, "campaign %s: %d seeds, %d worker(s), %s\n",
+		configuration.arm, len(configuration.seeds), len(workerDevices(configuration.devices)), configuration.outputDirectory)
+	sweep.sweep(ctx)
+
+	fmt.Fprintf(stdout, "campaign complete: %d of %d runs failed, %d produced an unreadable trace\n",
+		sweep.failures, len(configuration.seeds), sweep.unreadable)
+	if sweep.failures > 0 {
+		return fmt.Errorf("%d of %d runs failed", sweep.failures, len(configuration.seeds))
+	}
+	if sweep.unreadable > 0 {
+		// A run that exits 0 and leaves a trace the analysis cannot read is a
+		// lost cell, not a successful campaign, and an unattended sweep has to
+		// say so rather than reporting no failures.
+		return fmt.Errorf("%d of %d runs produced an unreadable trace", sweep.unreadable, len(configuration.seeds))
+	}
+	return nil
+}
+
+func probeVersion(ctx context.Context, binary string, executor commandExecutor) (string, error) {
+	var output bytes.Buffer
+	code, err := executor(ctx, binary, []string{"version"}, &output)
+	if err != nil {
+		return "", fmt.Errorf("run %s version: %w", binary, err)
+	}
+	if code != 0 {
+		return "", fmt.Errorf("%s version exited %d: %s", binary, code, strings.TrimSpace(output.String()))
+	}
+	return strings.TrimSpace(output.String()), nil
+}
+
+// workerDevices returns one entry per concurrent worker. With no --devices
+// there is a single worker and no device to name.
+func workerDevices(devices []string) []string {
+	if len(devices) == 0 {
+		return []string{""}
+	}
+	return devices
+}
+
+func (c *campaign) sweep(ctx context.Context) {
+	queue := make(chan int64, len(c.configuration.seeds))
+	for _, seed := range c.configuration.seeds {
+		queue <- seed
+	}
+	close(queue)
+
+	var waitGroup sync.WaitGroup
+	for _, device := range workerDevices(c.configuration.devices) {
+		waitGroup.Add(1)
+		go func(device string) {
+			defer waitGroup.Done()
+			for seed := range queue {
+				if ctx.Err() != nil {
+					return
+				}
+				c.report(c.runSeed(ctx, seed, device))
+			}
+		}(device)
+	}
+	waitGroup.Wait()
+}
+
+func (c *campaign) runSeed(ctx context.Context, seed int64, device string) runRecord {
+	seedText := strconv.FormatInt(seed, 10)
+	directory := seedDirectory(c.configuration, seedText)
+	record := runRecord{Seed: seed, Device: device, StartedAt: time.Now().UTC()}
+
+	if err := os.MkdirAll(directory, 0o755); err != nil {
+		record.ExitCode = -1
+		record.LaunchError = err.Error()
+		return record
+	}
+	logFile, err := os.Create(filepath.Join(directory, "sanderling.log"))
+	if err != nil {
+		record.ExitCode = -1
+		record.LaunchError = err.Error()
+		return record
+	}
+	defer logFile.Close()
+
+	start := time.Now()
+	exitCode, runErr := c.executor(ctx, c.configuration.sanderlingPath, runArguments(c.configuration, seedText, device), logFile)
+	record.DurationMillis = time.Since(start).Milliseconds()
+	record.ExitCode = exitCode
+	if runErr != nil {
+		record.LaunchError = runErr.Error()
+	}
+
+	name, summary, err := summarizeRun(directory)
+	if name != "" {
+		record.RunDirectory = filepath.Join(filepath.Base(directory), name)
+	}
+	if err != nil {
+		record.TraceError = err.Error()
+		return record
+	}
+	record.traceSummary = summary
+	return record
+}
+
+func (c *campaign) report(record runRecord) {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	if record.ExitCode != 0 {
+		c.failures++
+	} else if record.TraceError != "" {
+		c.unreadable++
+	}
+	if err := json.NewEncoder(c.records).Encode(record); err != nil {
+		fmt.Fprintf(c.stdout, "warning: seed %d record: %v\n", record.Seed, err)
+	}
+	fmt.Fprintf(c.stdout, "seed=%d device=%q outcome=%s steps=%d exit=%d duration=%s\n",
+		record.Seed, record.Device, outcome(record), record.Steps, record.ExitCode,
+		time.Duration(record.DurationMillis)*time.Millisecond)
+}
+
+func outcome(record runRecord) string {
+	switch {
+	case record.ExitCode != 0:
+		return "failed"
+	case record.FirstViolationOriginStep != nil:
+		return fmt.Sprintf("violation@%d(%s)", *record.FirstViolationOriginStep,
+			strings.Join(record.FirstViolationProperties, ","))
+	default:
+		return "clean"
+	}
+}

cmd/internal-tools/campaign/campaign_test.go

@@ -0,0 +1,317 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/priyanshujain/sanderling/internal/trace"
+)
+
+func testConfiguration(t *testing.T, outputDirectory string, extra ...string) config {
+	t.Helper()
+	arguments := append(baseArguments(), "--output", outputDirectory)
+	configuration, err := parseArguments(append(arguments, extra...), io.Discard)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return configuration
+}
+
+// versionAnswering wraps a test executor so every fake answers `sanderling
+// version`, which the campaign probes before it writes the manifest.
+func versionAnswering(executor commandExecutor) commandExecutor {
+	return func(ctx context.Context, binary string, arguments []string, output io.Writer) (int, error) {
+		if len(arguments) > 0 && arguments[0] == "version" {
+			fmt.Fprintln(output, "stub-version")
+			return 0, nil
+		}
+		return executor(ctx, binary, arguments, output)
+	}
+}
+
+func readRecords(t *testing.T, campaignDirectory string) []runRecord {
+	t.Helper()
+	file, err := os.Open(filepath.Join(campaignDirectory, recordsFileName))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+	var records []runRecord
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		var record runRecord
+		if err := json.Unmarshal(scanner.Bytes(), &record); err != nil {
+			t.Fatalf("decode %q: %v", scanner.Text(), err)
+		}
+		records = append(records, record)
+	}
+	if err := scanner.Err(); err != nil {
+		t.Fatal(err)
+	}
+	return records
+}
+
+func writeFakeRun(t *testing.T, arguments []string, steps []trace.Step) {
+	t.Helper()
+	writeRunDirectory(t, argumentValue(arguments, "--output"), "20260101-000000", steps)
+}
+
+func TestRunCampaign_WritesManifestBeforeAnyRun(t *testing.T) {
+	directory := t.TempDir()
+	configuration := testConfiguration(t, directory)
+	var stdout bytes.Buffer
+
+	executor := versionAnswering(func(_ context.Context, _ string, arguments []string, _ io.Writer) (int, error) {
+		if _, err := os.Stat(filepath.Join(directory, manifestFileName)); err != nil {
+			t.Errorf("manifest missing when the first run started: %v", err)
+		}
+		writeFakeRun(t, arguments, []trace.Step{observedStep(1)})
+		return 0, nil
+	})
+	if err := runCampaign(context.Background(), configuration, executor, &stdout); err != nil {
+		t.Fatal(err)
+	}
+
+	body, err := os.ReadFile(filepath.Join(directory, manifestFileName))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var recorded manifest
+	if err := json.Unmarshal(body, &recorded); err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Equal(recorded.Seeds, []int64{1, 2, 3}) {
+		t.Errorf("intended seeds: got %v", recorded.Seeds)
+	}
+	if recorded.SanderlingVersion != "stub-version" {
+		t.Errorf("version: got %q", recorded.SanderlingVersion)
+	}
+	if recorded.Host == "" {
+		t.Error("host was not recorded")
+	}
+}
+
+func TestRunCampaign_RecordsPerRunSummary(t *testing.T) {
+	directory := t.TempDir()
+	configuration := testConfiguration(t, directory, "--seeds", "1-3")
+	var stdout bytes.Buffer
+
+	executor := versionAnswering(func(_ context.Context, _ string, arguments []string, output io.Writer) (int, error) {
+		fmt.Fprintln(output, "stub run log")
+		steps := []trace.Step{observedStep(1), observedStep(2), observedStep(3)}
+		if argumentValue(arguments, "--seed") == "2" {
+			violating := observedStep(3)
+			violating.Violations = []string{"listNeverEmpty"}
+			violating.Witnesses = map[string]trace.Witness{
+				"listNeverEmpty": {Reason: "list emptied", Step: 2, DetectedStep: 3},
+			}
+			steps[2] = violating
+		}
+		writeFakeRun(t, arguments, steps)
+		return 0, nil
+	})
+	if err := runCampaign(context.Background(), configuration, executor, &stdout); err != nil {
+		t.Fatal(err)
+	}
+
+	records := readRecords(t, directory)
+	if len(records) != 3 {
+		t.Fatalf("records: got %d, want 3", len(records))
+	}
+	for _, record := range records {
+		if record.Steps != 3 {
+			t.Errorf("seed %d steps: got %d, want 3", record.Seed, record.Steps)
+		}
+		if record.RunDirectory != fmt.Sprintf("seed-%d/20260101-000000", record.Seed) {
+			t.Errorf("seed %d run directory: got %q", record.Seed, record.RunDirectory)
+		}
+		if record.TraceError != "" {
+			t.Errorf("seed %d trace error: %s", record.Seed, record.TraceError)
+		}
+		violated := record.FirstViolationOriginStep != nil
+		if violated != (record.Seed == 2) {
+			t.Errorf("seed %d violation: got %v", record.Seed, record.FirstViolationOriginStep)
+		}
+		if record.Seed != 2 {
+			continue
+		}
+		if *record.FirstViolationOriginStep != 2 || *record.FirstViolationDetectedStep != 3 {
+			t.Errorf("seed 2 violation steps: origin %d detected %d",
+				*record.FirstViolationOriginStep, *record.FirstViolationDetectedStep)
+		}
+		if !slices.Equal(record.ViolatedProperties, []string{"listNeverEmpty"}) {
+			t.Errorf("seed 2 properties: got %v", record.ViolatedProperties)
+		}
+	}
+
+	log, err := os.ReadFile(filepath.Join(directory, "seed-1", "sanderling.log"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(log), "stub run log") {
+		t.Errorf("per-run log: got %q", log)
+	}
+	if !strings.Contains(stdout.String(), "seed=2") {
+		t.Errorf("progress output missing seed 2: %q", stdout.String())
+	}
+}
+
+func TestRunCampaign_DistributesSeedsAcrossDeviceWorkers(t *testing.T) {
+	directory := t.TempDir()
+	configuration := testConfiguration(t, directory, "--seeds", "1-9", "--devices", "device-a,device-b,device-c")
+
+	var mutex sync.Mutex
+	assignments := map[int64]string{}
+	var inFlight atomic.Int32
+	var releaseOnce sync.Once
+	var timedOut atomic.Bool
+	release := make(chan struct{})
+
+	executor := versionAnswering(func(_ context.Context, _ string, arguments []string, _ io.Writer) (int, error) {
+		if inFlight.Add(1) >= 3 {
+			releaseOnce.Do(func() { close(release) })
+		}
+		select {
+		case <-release:
+		case <-time.After(5 * time.Second):
+			timedOut.Store(true)
+		}
+		seed, err := strconv.ParseInt(argumentValue(arguments, "--seed"), 10, 64)
+		if err != nil {
+			t.Errorf("seed argument: %v", err)
+		}
+		mutex.Lock()
+		if previous, seen := assignments[seed]; seen {
+			t.Errorf("seed %d ran twice (%s then %s)", seed, previous, argumentValue(arguments, "--device"))
+		}
+		assignments[seed] = argumentValue(arguments, "--device")
+		mutex.Unlock()
+		writeFakeRun(t, arguments, []trace.Step{observedStep(1)})
+		return 0, nil
+	})
+	if err := runCampaign(context.Background(), configuration, executor, io.Discard); err != nil {
+		t.Fatal(err)
+	}
+	if timedOut.Load() {
+		t.Fatal("three device workers never ran concurrently")
+	}
+	if len(assignments) != 9 {
+		t.Fatalf("seeds run: got %d, want 9", len(assignments))
+	}
+	used := map[string]int{}
+	for seed, device := range assignments {
+		if !slices.Contains(configuration.devices, device) {
+			t.Errorf("seed %d ran on unknown device %q", seed, device)
+		}
+		used[device]++
+	}
+	if len(used) != 3 {
+		t.Errorf("devices used: got %v, want all three", used)
+	}
+	if got := len(readRecords(t, directory)); got != 9 {
+		t.Errorf("records: got %d, want 9", got)
+	}
+}
+
+func TestRunCampaign_ContinuesAfterFailingRun(t *testing.T) {
+	directory := t.TempDir()
+	configuration := testConfiguration(t, directory, "--seeds", "1-3")
+
+	executor := versionAnswering(func(_ context.Context, _ string, arguments []string, output io.Writer) (int, error) {
+		if argumentValue(arguments, "--seed") == "2" {
+			fmt.Fprintln(output, "error: device offline")
+			return 1, nil
+		}
+		writeFakeRun(t, arguments, []trace.Step{observedStep(1)})
+		return 0, nil
+	})
+	err := runCampaign(context.Background(), configuration, executor, io.Discard)
+	if err == nil || !strings.Contains(err.Error(), "1 of 3 runs failed") {
+		t.Fatalf("expected a failure summary, got %v", err)
+	}
+
+	records := readRecords(t, directory)
+	if len(records) != 3 {
+		t.Fatalf("a failing run must not abort the campaign: got %d records", len(records))
+	}
+	for _, record := range records {
+		if record.Seed == 2 {
+			if record.ExitCode != 1 {
+				t.Errorf("seed 2 exit code: got %d, want 1", record.ExitCode)
+			}
+			if record.TraceError == "" {
+				t.Error("seed 2 produced no trace; that should be recorded")
+			}
+			continue
+		}
+		if record.ExitCode != 0 {
+			t.Errorf("seed %d exit code: got %d", record.Seed, record.ExitCode)
+		}
+	}
+}
+
+func TestRunCampaign_RefusesToReuseACampaignDirectory(t *testing.T) {
+	directory := t.TempDir()
+	configuration := testConfiguration(t, directory)
+	if err := os.WriteFile(filepath.Join(directory, manifestFileName), []byte("{}"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	executor := versionAnswering(func(context.Context, string, []string, io.Writer) (int, error) {
+		t.Error("no run should start in a directory that already holds a campaign")
+		return 0, nil
+	})
+	if err := runCampaign(context.Background(), configuration, executor, io.Discard); err == nil {
+		t.Fatal("expected a refusal to reuse the campaign directory")
+	}
+}
+
+func TestRunCampaign_AbortsWhenVersionProbeFails(t *testing.T) {
+	directory := t.TempDir()
+	configuration := testConfiguration(t, directory)
+	executor := func(_ context.Context, _ string, arguments []string, output io.Writer) (int, error) {
+		if arguments[0] != "version" {
+			t.Error("a run started despite an unusable binary")
+		}
+		fmt.Fprintln(output, "no such command")
+		return 2, nil
+	}
+	if err := runCampaign(context.Background(), configuration, executor, io.Discard); err == nil {
+		t.Fatal("expected the campaign to abort before writing a manifest it cannot attribute")
+	}
+	if _, err := os.Stat(filepath.Join(directory, manifestFileName)); err == nil {
+		t.Error("manifest was written despite an unusable binary")
+	}
+}
+
+func TestRunCampaign_UnreadableTraceIsNotASuccessfulCampaign(t *testing.T) {
+	campaignDirectory := filepath.Join(t.TempDir(), "cell")
+	configuration := testConfiguration(t, campaignDirectory, "--seeds", "1-2")
+
+	executor := versionAnswering(func(context.Context, string, []string, io.Writer) (int, error) {
+		return 0, nil
+	})
+
+	var stdout bytes.Buffer
+	err := runCampaign(context.Background(), configuration, executor, &stdout)
+	if err == nil || !strings.Contains(err.Error(), "2 of 2 runs produced an unreadable trace") {
+		t.Fatalf("a campaign whose runs left no readable trace must not report success: %v", err)
+	}
+	for _, record := range readRecords(t, campaignDirectory) {
+		if record.TraceError == "" {
+			t.Errorf("seed %d: expected a trace error, got none", record.Seed)
+		}
+	}
+}

cmd/internal-tools/campaign/end_to_end_test.go

@@ -0,0 +1,144 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+)
+
+// stubSanderling answers `version`, writes a run directory shaped like the one
+// `sanderling test` produces, violates on seed 5 and fails on seed 7.
+const stubSanderling = `#!/bin/sh
+if [ "$1" = "version" ]; then
+  echo "stub 9.9.9"
+  exit 0
+fi
+seed=""
+output=""
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --seed) seed="$2"; shift 2 ;;
+    --output) output="$2"; shift 2 ;;
+    *) shift ;;
+  esac
+done
+echo "stub run seed=$seed"
+run="$output/20260101-000000"
+mkdir -p "$run"
+echo "{\"seed\":$seed,\"platform\":\"web\"}" > "$run/meta.json"
+{
+  echo '{"step":1,"hierarchy":{"elements":[]}}'
+  echo '{"step":2,"hierarchy":{"elements":[]}}'
+  if [ "$seed" = "5" ]; then
+    echo '{"step":3,"hierarchy":{"elements":[]},"violations":["cartTotalMatches"],"witnesses":{"cartTotalMatches":{"reason":"total drifted","step":2,"detected_step":3}}}'
+  else
+    echo '{"step":3,"hierarchy":{"elements":[]}}'
+  fi
+} > "$run/trace.jsonl"
+if [ "$seed" = "7" ]; then
+  echo "driver crashed" >&2
+  exit 3
+fi
+`
+
+func TestRun_EndToEndAgainstStubBinary(t *testing.T) {
+	directory := t.TempDir()
+	binaryPath := filepath.Join(directory, "stub-sanderling")
+	if err := os.WriteFile(binaryPath, []byte(stubSanderling), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	campaignDirectory := filepath.Join(directory, "campaign")
+	var stdout bytes.Buffer
+
+	err := run([]string{
+		"--spec", "/specs/folio.ts",
+		"--bundle-id", "app.folio",
+		"--platform", "web",
+		"--arm", "seeded-web",
+		"--generator", "seeded",
+		"--max-steps", "50",
+		"--duration", "30s",
+		"--seeds", "4-5,7",
+		"--devices", "worker-a,worker-b",
+		"--sanderling", binaryPath,
+		"--output", campaignDirectory,
+		"--", "--clear-data=false",
+	}, &stdout, io.Discard)
+	if err == nil || !strings.Contains(err.Error(), "1 of 3 runs failed") {
+		t.Fatalf("expected the failing seed to be reported, got %v", err)
+	}
+
+	body, err := os.ReadFile(filepath.Join(campaignDirectory, manifestFileName))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var recorded manifest
+	if err := json.Unmarshal(body, &recorded); err != nil {
+		t.Fatal(err)
+	}
+	if recorded.SanderlingVersion != "stub 9.9.9" {
+		t.Errorf("version: got %q", recorded.SanderlingVersion)
+	}
+	if !slices.Equal(recorded.Seeds, []int64{4, 5, 7}) {
+		t.Errorf("intended seeds: got %v", recorded.Seeds)
+	}
+	if slices.Contains(recorded.ArgumentTemplate, "--device") {
+		t.Errorf("web template should carry no device flag: %v", recorded.ArgumentTemplate)
+	}
+	if recorded.ArgumentTemplate[len(recorded.ArgumentTemplate)-1] != "--clear-data=false" {
+		t.Errorf("template lost the passthrough flag: %v", recorded.ArgumentTemplate)
+	}
+
+	records := readRecords(t, campaignDirectory)
+	if len(records) != 3 {
+		t.Fatalf("records: got %d, want 3", len(records))
+	}
+	bySeed := map[int64]runRecord{}
+	for _, record := range records {
+		bySeed[record.Seed] = record
+	}
+	for _, seed := range []int64{4, 5, 7} {
+		record, ok := bySeed[seed]
+		if !ok {
+			t.Fatalf("seed %d is missing from %s", seed, recordsFileName)
+		}
+		if record.Steps != 3 {
+			t.Errorf("seed %d steps: got %d, want 3", seed, record.Steps)
+		}
+		if record.RunDirectory == "" {
+			t.Errorf("seed %d has no run directory", seed)
+		}
+		if !slices.Contains([]string{"worker-a", "worker-b"}, record.Device) {
+			t.Errorf("seed %d device: got %q", seed, record.Device)
+		}
+	}
+	violating := bySeed[5]
+	if violating.FirstViolationOriginStep == nil || *violating.FirstViolationOriginStep != 2 {
+		t.Fatalf("seed 5 origin step: got %v, want 2", violating.FirstViolationOriginStep)
+	}
+	if *violating.FirstViolationDetectedStep != 3 || violating.FirstViolationReason != "total drifted" {
+		t.Errorf("seed 5 violation: %+v", violating)
+	}
+	if bySeed[4].FirstViolationOriginStep != nil {
+		t.Errorf("seed 4 should be censored at the budget: %+v", bySeed[4])
+	}
+	if bySeed[7].ExitCode != 3 {
+		t.Errorf("seed 7 exit code: got %d, want 3", bySeed[7].ExitCode)
+	}
+
+	log, err := os.ReadFile(filepath.Join(campaignDirectory, "seed-7", "sanderling.log"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(log), "driver crashed") {
+		t.Errorf("seed 7 log lost the stderr line: %q", log)
+	}
+	if !strings.Contains(stdout.String(), "outcome=violation@2") {
+		t.Errorf("progress output: %q", stdout.String())
+	}
+}

cmd/internal-tools/campaign/main.go

@@ -0,0 +1,190 @@
+// Command campaign sweeps a list of seeds for one experiment cell, writing a
+// directory an analysis pipeline can read without re-parsing raw traces.
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+)
+
+type config struct {
+	specPath        string
+	bundleID        string
+	platform        string
+	arm             string
+	generator       string
+	maxSteps        int
+	duration        time.Duration
+	seeds           []int64
+	devices         []string
+	sanderlingPath  string
+	outputDirectory string
+	extraArguments  []string
+}
+
+const usage = `campaign sweeps seeds for one experiment cell of a sanderling evaluation.
+
+Usage:
+  campaign --spec <path> --bundle-id <id> --platform <android|ios|web>
+           --arm <label> --generator <seeded|llm> --max-steps <n>
+           --seeds <spec> --output <dir> [flags] [-- <sanderling test flags>]
+
+Everything after a bare -- is appended verbatim to every sanderling test call.
+`
+
+func parseArguments(arguments []string, stderr io.Writer) (config, error) {
+	flagSet := flag.NewFlagSet("campaign", flag.ContinueOnError)
+	flagSet.SetOutput(stderr)
+	flagSet.Usage = func() {
+		fmt.Fprint(stderr, usage)
+		flagSet.PrintDefaults()
+	}
+	var configuration config
+	var seedSpecification string
+	var deviceList string
+	flagSet.StringVar(&configuration.specPath, "spec", "", "path to the TypeScript spec (required)")
+	flagSet.StringVar(&configuration.bundleID, "bundle-id", "", "target app bundle ID (required)")
+	flagSet.StringVar(&configuration.platform, "platform", "android", "target platform: android, ios, web")
+	flagSet.StringVar(&configuration.arm, "arm", "", "experiment cell label recorded on every run (required)")
+	flagSet.StringVar(&configuration.generator, "generator", "seeded", "action generator: seeded or llm")
+	flagSet.IntVar(&configuration.maxSteps, "max-steps", 0, "per-run step budget (required, must be positive)")
+	flagSet.DurationVar(&configuration.duration, "duration", 5*time.Minute, "per-run wall-clock ceiling")
+	flagSet.StringVar(&seedSpecification, "seeds", "", "seeds to run: ranges and lists, e.g. 1-10,20,30-32 (required)")
+	flagSet.StringVar(&deviceList, "devices", "", "comma-separated device identifiers; one concurrent worker per device (on web these are worker labels, no device flag is passed)")
+	flagSet.StringVar(&configuration.sanderlingPath, "sanderling", "sanderling", "sanderling binary to invoke")
+	flagSet.StringVar(&configuration.outputDirectory, "output", "", "campaign directory to create (required)")
+	if err := flagSet.Parse(arguments); err != nil {
+		return config{}, err
+	}
+	configuration.extraArguments = flagSet.Args()
+
+	for name, value := range map[string]string{
+		"--spec":      configuration.specPath,
+		"--bundle-id": configuration.bundleID,
+		"--arm":       configuration.arm,
+		"--seeds":     seedSpecification,
+		"--output":    configuration.outputDirectory,
+	} {
+		if value == "" {
+			return config{}, fmt.Errorf("%s is required", name)
+		}
+	}
+	switch configuration.platform {
+	case "android", "ios", "web":
+	default:
+		return config{}, fmt.Errorf("unsupported platform: %q (android, ios, web)", configuration.platform)
+	}
+	switch configuration.generator {
+	case "seeded", "llm":
+	default:
+		return config{}, fmt.Errorf("unsupported generator: %q (seeded, llm)", configuration.generator)
+	}
+	if configuration.maxSteps <= 0 {
+		// Steps to first violation is right-censored at the budget, so a
+		// campaign without one has nothing to censor its clean runs at.
+		return config{}, fmt.Errorf("--max-steps must be positive: every run needs the same step budget")
+	}
+	if configuration.duration <= 0 {
+		return config{}, fmt.Errorf("--duration must be positive: %s", configuration.duration)
+	}
+	seeds, err := parseSeeds(seedSpecification)
+	if err != nil {
+		return config{}, fmt.Errorf("--seeds: %w", err)
+	}
+	configuration.seeds = seeds
+	devices, err := parseDevices(deviceList)
+	if err != nil {
+		return config{}, fmt.Errorf("--devices: %w", err)
+	}
+	configuration.devices = devices
+	return configuration, nil
+}
+
+func parseDevices(list string) ([]string, error) {
+	if strings.TrimSpace(list) == "" {
+		return nil, nil
+	}
+	var devices []string
+	seen := map[string]bool{}
+	for _, part := range strings.Split(list, ",") {
+		device := strings.TrimSpace(part)
+		if device == "" {
+			return nil, fmt.Errorf("empty device in %q", list)
+		}
+		if seen[device] {
+			return nil, fmt.Errorf("duplicate device %q", device)
+		}
+		seen[device] = true
+		devices = append(devices, device)
+	}
+	return devices, nil
+}
+
+// deviceFlag names the `sanderling test` flag that selects a target on this
+// platform. Web has no device, so its workers only bound concurrency.
+func deviceFlag(platform string) string {
+	switch platform {
+	case "android":
+		return "--device"
+	case "ios":
+		return "--ios-device"
+	default:
+		return ""
+	}
+}
+
+func seedDirectory(configuration config, seed string) string {
+	return filepath.Join(configuration.outputDirectory, "seed-"+seed)
+}
+
+// runArguments builds one `sanderling test` invocation. seed and device are
+// passed as strings so the same code produces both a real command and the
+// placeholder template recorded in campaign.json.
+func runArguments(configuration config, seed, device string) []string {
+	arguments := []string{
+		"test",
+		"--spec", configuration.specPath,
+		"--bundle-id", configuration.bundleID,
+		"--platform", configuration.platform,
+		"--arm", configuration.arm,
+		"--generator", configuration.generator,
+		"--max-steps", strconv.Itoa(configuration.maxSteps),
+		"--duration", configuration.duration.String(),
+		"--seed", seed,
+		"--output", seedDirectory(configuration, seed),
+	}
+	if flagName := deviceFlag(configuration.platform); flagName != "" && device != "" {
+		arguments = append(arguments, flagName, device)
+	}
+	return append(arguments, configuration.extraArguments...)
+}
+
+func run(arguments []string, stdout, stderr io.Writer) error {
+	configuration, err := parseArguments(arguments, stderr)
+	if err != nil {
+		return err
+	}
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer cancel()
+	return runCampaign(ctx, configuration, executeCommand, stdout)
+}
+
+func main() {
+	if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return
+		}
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+}

cmd/internal-tools/campaign/main_test.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"io"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+)
+
+func baseArguments() []string {
+	return []string{
+		"--spec", "/specs/folio.ts",
+		"--bundle-id", "app.folio",
+		"--platform", "android",
+		"--arm", "seeded-baseline",
+		"--generator", "seeded",
+		"--max-steps", "300",
+		"--seeds", "1-3",
+		"--output", "/campaigns/a",
+	}
+}
+
+func TestParseArguments_AcceptsFullInvocation(t *testing.T) {
+	configuration, err := parseArguments(append(baseArguments(), "--devices", "emulator-5554,emulator-5556"), io.Discard)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if configuration.arm != "seeded-baseline" || configuration.maxSteps != 300 {
+		t.Errorf("arm/max-steps: got %q/%d", configuration.arm, configuration.maxSteps)
+	}
+	if !slices.Equal(configuration.seeds, []int64{1, 2, 3}) {
+		t.Errorf("seeds: got %v", configuration.seeds)
+	}
+	if !slices.Equal(configuration.devices, []string{"emulator-5554", "emulator-5556"}) {
+		t.Errorf("devices: got %v", configuration.devices)
+	}
+	if configuration.duration != 5*time.Minute {
+		t.Errorf("duration default: got %s", configuration.duration)
+	}
+	if configuration.sanderlingPath != "sanderling" {
+		t.Errorf("sanderling default: got %q", configuration.sanderlingPath)
+	}
+}
+
+func TestParseArguments_Rejections(t *testing.T) {
+	cases := []struct {
+		name      string
+		arguments []string
+		want      string
+	}{
+		{"missing spec", []string{"--bundle-id", "a", "--arm", "b", "--seeds", "1", "--output", "o"}, "--spec is required"},
+		{"missing arm", []string{"--spec", "s", "--bundle-id", "a", "--seeds", "1", "--output", "o", "--max-steps", "10"}, "--arm is required"},
+		{"missing output", []string{"--spec", "s", "--bundle-id", "a", "--arm", "b", "--seeds", "1", "--max-steps", "10"}, "--output is required"},
+		{"bad platform", append(baseArguments(), "--platform", "windows"), "unsupported platform"},
+		{"bad generator", append(baseArguments(), "--generator", "vibes"), "unsupported generator"},
+		{"zero max steps", append(baseArguments(), "--max-steps", "0"), "--max-steps must be positive"},
+		{"seed zero", append(baseArguments(), "--seeds", "0-2"), "not reproducible"},
+		{"duplicate device", append(baseArguments(), "--devices", "a,a"), "duplicate device"},
+	}
+	for _, testCase := range cases {
+		_, err := parseArguments(testCase.arguments, io.Discard)
+		if err == nil {
+			t.Errorf("%s: expected error", testCase.name)
+			continue
+		}
+		if !strings.Contains(err.Error(), testCase.want) {
+			t.Errorf("%s: got %q, want it to contain %q", testCase.name, err, testCase.want)
+		}
+	}
+}
+
+func TestRunArguments_PlatformDeviceFlagAndPassthrough(t *testing.T) {
+	cases := []struct {
+		platform string
+		want     []string
+	}{
+		{"android", []string{"--device", "target-1"}},
+		{"ios", []string{"--ios-device", "target-1"}},
+		{"web", nil},
+	}
+	for _, testCase := range cases {
+		arguments := append(baseArguments(), "--platform", testCase.platform, "--", "--clear-data=false")
+		configuration, err := parseArguments(arguments, io.Discard)
+		if err != nil {
+			t.Fatalf("%s: %v", testCase.platform, err)
+		}
+		got := runArguments(configuration, "7", "target-1")
+		if got[0] != "test" {
+			t.Errorf("%s: first argument is %q, want test", testCase.platform, got[0])
+		}
+		for _, pair := range [][2]string{
+			{"--seed", "7"},
+			{"--arm", "seeded-baseline"},
+			{"--max-steps", "300"},
+			{"--generator", "seeded"},
+			{"--output", "/campaigns/a/seed-7"},
+		} {
+			if value := argumentValue(got, pair[0]); value != pair[1] {
+				t.Errorf("%s: %s = %q, want %q", testCase.platform, pair[0], value, pair[1])
+			}
+		}
+		if testCase.want != nil {
+			if value := argumentValue(got, testCase.want[0]); value != testCase.want[1] {
+				t.Errorf("%s: %s = %q, want %q", testCase.platform, testCase.want[0], value, testCase.want[1])
+			}
+		} else if slices.Contains(got, "--device") || slices.Contains(got, "--ios-device") {
+			t.Errorf("web: unexpected device flag in %v", got)
+		}
+		if got[len(got)-1] != "--clear-data=false" {
+			t.Errorf("%s: passthrough argument lost: %v", testCase.platform, got)
+		}
+	}
+}
+
+func argumentValue(arguments []string, name string) string {
+	index := slices.Index(arguments, name)
+	if index < 0 || index+1 >= len(arguments) {
+		return ""
+	}
+	return arguments[index+1]
+}

cmd/internal-tools/campaign/manifest.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+const (
+	manifestFileName  = "campaign.json"
+	recordsFileName   = "runs.jsonl"
+	seedPlaceholder   = "{seed}"
+	devicePlaceholder = "{device}"
+)
+
+// manifest is campaign.json: what the campaign INTENDED to run, written before
+// the first run so a host that dropped runs shows up as missing seeds rather
+// than as a smaller sample.
+type manifest struct {
+	Arm               string    `json:"arm"`
+	Generator         string    `json:"generator"`
+	Platform          string    `json:"platform"`
+	SpecPath          string    `json:"spec_path"`
+	BundleID          string    `json:"bundle_id"`
+	MaxSteps          int       `json:"max_steps"`
+	DurationMillis    int64     `json:"duration_millis"`
+	Seeds             []int64   `json:"seeds"`
+	Devices           []string  `json:"devices"`
+	Host              string    `json:"host"`
+	SanderlingPath    string    `json:"sanderling_path"`
+	SanderlingVersion string    `json:"sanderling_version"`
+	StartedAt         time.Time `json:"started_at"`
+	ArgumentTemplate  []string  `json:"argument_template"`
+}
+
+func buildManifest(configuration config, host, binaryPath, version string, startedAt time.Time) manifest {
+	devices := configuration.devices
+	if devices == nil {
+		devices = []string{}
+	}
+	return manifest{
+		Arm:               configuration.arm,
+		Generator:         configuration.generator,
+		Platform:          configuration.platform,
+		SpecPath:          configuration.specPath,
+		BundleID:          configuration.bundleID,
+		MaxSteps:          configuration.maxSteps,
+		DurationMillis:    configuration.duration.Milliseconds(),
+		Seeds:             configuration.seeds,
+		Devices:           devices,
+		Host:              host,
+		SanderlingPath:    binaryPath,
+		SanderlingVersion: version,
+		StartedAt:         startedAt,
+		ArgumentTemplate:  runArguments(configuration, seedPlaceholder, devicePlaceholder),
+	}
+}
+
+func writeManifest(directory string, value manifest) error {
+	body, err := json.MarshalIndent(value, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal manifest: %w", err)
+	}
+	return os.WriteFile(filepath.Join(directory, manifestFileName), append(body, '\n'), 0o644)
+}

cmd/internal-tools/campaign/manifest_test.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+	"time"
+)
+
+func TestBuildManifest_RecordsIntendedRunsAndTemplate(t *testing.T) {
+	configuration, err := parseArguments(append(baseArguments(), "--devices", "a,b", "--duration", "90s"), io.Discard)
+	if err != nil {
+		t.Fatal(err)
+	}
+	startedAt := time.Date(2026, 8, 12, 9, 0, 0, 0, time.UTC)
+	value := buildManifest(configuration, "worker-7", "/usr/local/bin/sanderling", "1.4.2", startedAt)
+
+	if !slices.Equal(value.Seeds, []int64{1, 2, 3}) {
+		t.Errorf("seeds: got %v", value.Seeds)
+	}
+	if value.Host != "worker-7" || value.SanderlingVersion != "1.4.2" || value.SanderlingPath != "/usr/local/bin/sanderling" {
+		t.Errorf("provenance: %+v", value)
+	}
+	if value.DurationMillis != 90_000 || value.MaxSteps != 300 {
+		t.Errorf("budget: duration=%d max_steps=%d", value.DurationMillis, value.MaxSteps)
+	}
+	if !value.StartedAt.Equal(startedAt) {
+		t.Errorf("started_at: got %s", value.StartedAt)
+	}
+	if got := argumentValue(value.ArgumentTemplate, "--seed"); got != seedPlaceholder {
+		t.Errorf("template seed: got %q, want %q", got, seedPlaceholder)
+	}
+	if got := argumentValue(value.ArgumentTemplate, "--device"); got != devicePlaceholder {
+		t.Errorf("template device: got %q, want %q", got, devicePlaceholder)
+	}
+	if got := argumentValue(value.ArgumentTemplate, "--output"); got != "/campaigns/a/seed-"+seedPlaceholder {
+		t.Errorf("template output: got %q", got)
+	}
+}
+
+func TestBuildManifest_EmptyDeviceListSerializesAsArray(t *testing.T) {
+	configuration, err := parseArguments(baseArguments(), io.Discard)
+	if err != nil {
+		t.Fatal(err)
+	}
+	directory := t.TempDir()
+	if err := writeManifest(directory, buildManifest(configuration, "host", "sanderling", "dev", time.Now())); err != nil {
+		t.Fatal(err)
+	}
+	body, err := os.ReadFile(filepath.Join(directory, manifestFileName))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var decoded struct {
+		Devices []string `json:"devices"`
+		Arm     string   `json:"arm"`
+	}
+	if err := json.Unmarshal(body, &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if decoded.Devices == nil || len(decoded.Devices) != 0 {
+		t.Errorf("devices: got %v, want []", decoded.Devices)
+	}
+	if decoded.Arm != "seeded-baseline" {
+		t.Errorf("arm: got %q", decoded.Arm)
+	}
+}

cmd/internal-tools/campaign/seeds.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// parseSeeds expands a seed specification such as "1-10,20,30-32" into the
+// explicit seed list a campaign intends to run.
+func parseSeeds(specification string) ([]int64, error) {
+	trimmed := strings.TrimSpace(specification)
+	if trimmed == "" {
+		return nil, fmt.Errorf("empty seed spec")
+	}
+	var seeds []int64
+	seen := map[int64]bool{}
+	for _, part := range strings.Split(trimmed, ",") {
+		part = strings.TrimSpace(part)
+		if part == "" {
+			return nil, fmt.Errorf("empty seed in %q", specification)
+		}
+		expanded, err := expandSeedPart(part)
+		if err != nil {
+			return nil, err
+		}
+		for _, seed := range expanded {
+			if seen[seed] {
+				return nil, fmt.Errorf("duplicate seed %d in %q", seed, specification)
+			}
+			seen[seed] = true
+			seeds = append(seeds, seed)
+		}
+	}
+	return seeds, nil
+}
+
+func expandSeedPart(part string) ([]int64, error) {
+	start, end, isRange := strings.Cut(part, "-")
+	if !isRange {
+		seed, err := parseSeed(part)
+		if err != nil {
+			return nil, err
+		}
+		return []int64{seed}, nil
+	}
+	first, err := parseSeed(strings.TrimSpace(start))
+	if err != nil {
+		return nil, fmt.Errorf("seed range %q: %w", part, err)
+	}
+	last, err := parseSeed(strings.TrimSpace(end))
+	if err != nil {
+		return nil, fmt.Errorf("seed range %q: %w", part, err)
+	}
+	if first > last {
+		return nil, fmt.Errorf("seed range %q: start %d is above end %d", part, first, last)
+	}
+	seeds := make([]int64, 0, last-first+1)
+	for seed := first; seed <= last; seed++ {
+		seeds = append(seeds, seed)
+	}
+	return seeds, nil
+}
+
+func parseSeed(text string) (int64, error) {
+	seed, err := strconv.ParseInt(text, 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("invalid seed %q: want a positive integer", text)
+	}
+	if seed == 0 {
+		// `sanderling test` reads --seed 0 as "derive a seed from the clock",
+		// so a campaign listing seed 0 records a run nobody can reproduce.
+		return 0, fmt.Errorf("seed 0 is not reproducible: sanderling test derives a random seed when --seed is 0, so list explicit non-zero seeds")
+	}
+	if seed < 0 {
+		return 0, fmt.Errorf("invalid seed %d: want a positive integer", seed)
+	}
+	return seed, nil
+}

cmd/internal-tools/campaign/seeds_test.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"slices"
+	"strings"
+	"testing"
+)
+
+func TestParseSeeds_RangesAndLists(t *testing.T) {
+	cases := []struct {
+		specification string
+		want          []int64
+	}{
+		{"1-5", []int64{1, 2, 3, 4, 5}},
+		{"1,5,9", []int64{1, 5, 9}},
+		{"1-3,20,30-32", []int64{1, 2, 3, 20, 30, 31, 32}},
+		{" 7 , 8 ", []int64{7, 8}},
+		{"4-4", []int64{4}},
+	}
+	for _, testCase := range cases {
+		got, err := parseSeeds(testCase.specification)
+		if err != nil {
+			t.Fatalf("%q: %v", testCase.specification, err)
+		}
+		if !slices.Equal(got, testCase.want) {
+			t.Errorf("%q: got %v, want %v", testCase.specification, got, testCase.want)
+		}
+	}
+}
+
+func TestParseSeeds_RejectsSeedZero(t *testing.T) {
+	for _, specification := range []string{"0", "1,0,2", "0-3"} {
+		_, err := parseSeeds(specification)
+		if err == nil {
+			t.Fatalf("%q: expected rejection of seed 0", specification)
+		}
+		if !strings.Contains(err.Error(), "not reproducible") {
+			t.Errorf("%q: error should explain why seed 0 is rejected: %v", specification, err)
+		}
+	}
+}
+
+func TestParseSeeds_RejectsMalformed(t *testing.T) {
+	for _, specification := range []string{"", "   ", "abc", "1,,2", "5-1", "1-", "-5", "1-2-3", "1.5", "2,2"} {
+		if seeds, err := parseSeeds(specification); err == nil {
+			t.Errorf("%q: expected error, got %v", specification, seeds)
+		}
+	}
+}

cmd/internal-tools/campaign/summary.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"maps"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"github.com/priyanshujain/sanderling/internal/trace"
+)
+
+// Hierarchy dumps make trace lines large; match the replay server's ceiling.
+const maxTraceLineBytes = 16 * 1024 * 1024
+
+// traceSummary is everything the analysis needs from one run, so it never has
+// to open trace.jsonl again.
+type traceSummary struct {
+	Steps                      int      `json:"steps"`
+	FirstViolationOriginStep   *int     `json:"first_violation_origin_step"`
+	FirstViolationDetectedStep *int     `json:"first_violation_detected_step"`
+	FirstViolationProperties   []string `json:"first_violation_properties,omitempty"`
+	FirstViolationReason       string   `json:"first_violation_reason,omitempty"`
+	FirstViolationIsError      bool     `json:"first_violation_is_error,omitempty"`
+	ViolatedProperties         []string `json:"violated_properties,omitempty"`
+}
+
+type traceLine struct {
+	Index int `json:"step"`
+	// Hierarchy is read only for its presence: the run-end finalize line is the
+	// one line carrying violations without an observed hierarchy.
+	Hierarchy  json.RawMessage          `json:"hierarchy"`
+	Violations []string                 `json:"violations"`
+	Witnesses  map[string]trace.Witness `json:"witnesses"`
+}
+
+// findRunDirectory returns the run directory `sanderling test` created inside
+// seedDirectory. Names are UTC timestamps, so the last one sorted is the newest.
+func findRunDirectory(seedDirectory string) (string, error) {
+	entries, err := os.ReadDir(seedDirectory)
+	if err != nil {
+		return "", fmt.Errorf("read seed dir: %w", err)
+	}
+	names := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+		if _, err := os.Stat(filepath.Join(seedDirectory, entry.Name(), "meta.json")); err != nil {
+			continue
+		}
+		names = append(names, entry.Name())
+	}
+	if len(names) == 0 {
+		return "", fmt.Errorf("no run directory with meta.json under %s", seedDirectory)
+	}
+	slices.Sort(names)
+	return names[len(names)-1], nil
+}
+
+// summarizeRun locates the run under seedDirectory and reduces its trace to the
+// fields the analysis reads. The returned path is relative to seedDirectory.
+func summarizeRun(seedDirectory string) (string, traceSummary, error) {
+	name, err := findRunDirectory(seedDirectory)
+	if err != nil {
+		return "", traceSummary{}, err
+	}
+	summary, err := summarizeTrace(filepath.Join(seedDirectory, name, "trace.jsonl"))
+	if err != nil {
+		return name, traceSummary{}, err
+	}
+	return name, summary, nil
+}
+
+func summarizeTrace(tracePath string) (traceSummary, error) {
+	file, err := os.Open(tracePath)
+	if err != nil {
+		return traceSummary{}, fmt.Errorf("open trace: %w", err)
+	}
+	defer file.Close()
+
+	var summary traceSummary
+	violated := map[string]bool{}
+	scanner := bufio.NewScanner(file)
+	scanner.Buffer(make([]byte, 0, 64*1024), maxTraceLineBytes)
+	lineNumber := 0
+	for scanner.Scan() {
+		lineNumber++
+		raw := strings.TrimSpace(scanner.Text())
+		if raw == "" {
+			continue
+		}
+		var line traceLine
+		if err := json.Unmarshal([]byte(raw), &line); err != nil {
+			return traceSummary{}, fmt.Errorf("trace line %d: %w", lineNumber, err)
+		}
+		// The finalize line is synthetic: it reports obligations that never
+		// discharged, at an index one past the last step actually executed.
+		synthetic := len(line.Violations) > 0 && len(line.Hierarchy) == 0
+		if !synthetic && line.Index > summary.Steps {
+			summary.Steps = line.Index
+		}
+		for _, property := range line.Violations {
+			violated[property] = true
+			recordViolation(&summary, line, property)
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return traceSummary{}, fmt.Errorf("read trace: %w", err)
+	}
+	if len(violated) > 0 {
+		summary.ViolatedProperties = slices.Sorted(maps.Keys(violated))
+	}
+	slices.Sort(summary.FirstViolationProperties)
+	return summary, nil
+}
+
+// recordViolation folds one violated property into the first-violation fields.
+// The origin step (the step that armed the failed obligation) orders the event,
+// because that is the step count the survival analysis measures.
+func recordViolation(summary *traceSummary, line traceLine, property string) {
+	origin, detected := line.Index, line.Index
+	witness := line.Witnesses[property]
+	if witness.Step > 0 {
+		origin = witness.Step
+	}
+	if witness.DetectedStep > 0 {
+		detected = witness.DetectedStep
+	}
+	switch {
+	case summary.FirstViolationOriginStep == nil || origin < *summary.FirstViolationOriginStep:
+		summary.FirstViolationOriginStep = &origin
+		summary.FirstViolationDetectedStep = &detected
+		summary.FirstViolationProperties = []string{property}
+		summary.FirstViolationReason = witness.Reason
+		summary.FirstViolationIsError = witness.IsError
+	case origin == *summary.FirstViolationOriginStep:
+		summary.FirstViolationProperties = append(summary.FirstViolationProperties, property)
+		if detected < *summary.FirstViolationDetectedStep {
+			summary.FirstViolationDetectedStep = &detected
+			summary.FirstViolationReason = witness.Reason
+			summary.FirstViolationIsError = witness.IsError
+		}
+	}
+}

cmd/internal-tools/campaign/summary_test.go

@@ -0,0 +1,200 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+	"time"
+
+	"github.com/priyanshujain/sanderling/internal/hierarchy"
+	"github.com/priyanshujain/sanderling/internal/trace"
+)
+
+func observedStep(index int) trace.Step {
+	return trace.Step{
+		Index:     index,
+		Timestamp: time.Date(2026, 8, 12, 9, 0, index, 0, time.UTC),
+		Screen:    "Home",
+		Hierarchy: &hierarchy.Tree{Elements: []*hierarchy.Element{{ResourceID: "root"}}},
+	}
+}
+
+func writeRunDirectory(t *testing.T, seedDirectory, name string, steps []trace.Step) string {
+	t.Helper()
+	directory := filepath.Join(seedDirectory, name)
+	if err := os.MkdirAll(directory, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	meta, err := json.Marshal(trace.Meta{Seed: 11, Platform: "web", Arm: "seeded-baseline"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(directory, "meta.json"), meta, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	var buffer bytes.Buffer
+	encoder := json.NewEncoder(&buffer)
+	for _, step := range steps {
+		if err := encoder.Encode(step); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := os.WriteFile(filepath.Join(directory, "trace.jsonl"), buffer.Bytes(), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return directory
+}
+
+func TestSummarizeRun_CleanRunIsCensored(t *testing.T) {
+	seedDirectory := t.TempDir()
+	writeRunDirectory(t, seedDirectory, "20260812-090000", []trace.Step{
+		observedStep(1), observedStep(2), observedStep(3), observedStep(4), observedStep(5),
+	})
+
+	name, summary, err := summarizeRun(seedDirectory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if name != "20260812-090000" {
+		t.Errorf("run directory: got %q", name)
+	}
+	if summary.Steps != 5 {
+		t.Errorf("steps: got %d, want 5", summary.Steps)
+	}
+	if summary.FirstViolationOriginStep != nil || summary.FirstViolationDetectedStep != nil {
+		t.Errorf("clean run reported a violation: %+v", summary)
+	}
+	if len(summary.ViolatedProperties) != 0 {
+		t.Errorf("violated properties: got %v", summary.ViolatedProperties)
+	}
+}
+
+func TestSummarizeRun_UsesWitnessOriginNotDetectionStep(t *testing.T) {
+	seedDirectory := t.TempDir()
+	violating := observedStep(9)
+	violating.Violations = []string{"balanceNeverNegative"}
+	violating.Witnesses = map[string]trace.Witness{
+		"balanceNeverNegative": {Reason: "balance went negative", Step: 4, DetectedStep: 9},
+	}
+	writeRunDirectory(t, seedDirectory, "20260812-090000", []trace.Step{
+		observedStep(1), observedStep(2), violating, observedStep(10),
+	})
+
+	_, summary, err := summarizeRun(seedDirectory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if summary.FirstViolationOriginStep == nil || *summary.FirstViolationOriginStep != 4 {
+		t.Fatalf("origin step: got %v, want 4", summary.FirstViolationOriginStep)
+	}
+	if summary.FirstViolationDetectedStep == nil || *summary.FirstViolationDetectedStep != 9 {
+		t.Fatalf("detected step: got %v, want 9", summary.FirstViolationDetectedStep)
+	}
+	if summary.FirstViolationReason != "balance went negative" {
+		t.Errorf("reason: got %q", summary.FirstViolationReason)
+	}
+	if !slices.Equal(summary.FirstViolationProperties, []string{"balanceNeverNegative"}) {
+		t.Errorf("first violation properties: got %v", summary.FirstViolationProperties)
+	}
+}
+
+func TestSummarizeRun_EarliestOriginWinsOverEarliestDetection(t *testing.T) {
+	seedDirectory := t.TempDir()
+	early := observedStep(3)
+	early.Violations = []string{"detectedFirst"}
+	early.Witnesses = map[string]trace.Witness{"detectedFirst": {Step: 3, DetectedStep: 3}}
+	late := observedStep(8)
+	late.Violations = []string{"armedFirst"}
+	late.Witnesses = map[string]trace.Witness{"armedFirst": {Step: 1, DetectedStep: 8, IsError: true}}
+	writeRunDirectory(t, seedDirectory, "20260812-090000", []trace.Step{observedStep(1), early, late})
+
+	_, summary, err := summarizeRun(seedDirectory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if summary.FirstViolationOriginStep == nil || *summary.FirstViolationOriginStep != 1 {
+		t.Fatalf("origin step: got %v, want 1", summary.FirstViolationOriginStep)
+	}
+	if !summary.FirstViolationIsError {
+		t.Error("is_error should come from the earliest-origin violation")
+	}
+	if !slices.Equal(summary.ViolatedProperties, []string{"armedFirst", "detectedFirst"}) {
+		t.Errorf("violated properties: got %v", summary.ViolatedProperties)
+	}
+}
+
+func TestSummarizeRun_FinalizeLineIsNotAStep(t *testing.T) {
+	seedDirectory := t.TempDir()
+	finalize := trace.Step{
+		Index:      4,
+		Timestamp:  time.Now().UTC(),
+		Violations: []string{"eventuallySettles"},
+		Witnesses:  map[string]trace.Witness{"eventuallySettles": {Step: 2, DetectedStep: 4}},
+	}
+	writeRunDirectory(t, seedDirectory, "20260812-090000", []trace.Step{
+		observedStep(1), observedStep(2), observedStep(3), finalize,
+	})
+
+	_, summary, err := summarizeRun(seedDirectory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if summary.Steps != 3 {
+		t.Errorf("steps: got %d, want 3 (the finalize line is synthetic)", summary.Steps)
+	}
+	if summary.FirstViolationOriginStep == nil || *summary.FirstViolationOriginStep != 2 {
+		t.Fatalf("origin step: got %v, want 2", summary.FirstViolationOriginStep)
+	}
+}
+
+func TestSummarizeRun_FallsBackToStepIndexWithoutWitness(t *testing.T) {
+	seedDirectory := t.TempDir()
+	violating := observedStep(6)
+	violating.Violations = []string{"noWitness"}
+	writeRunDirectory(t, seedDirectory, "20260812-090000", []trace.Step{observedStep(5), violating})
+
+	_, summary, err := summarizeRun(seedDirectory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if summary.FirstViolationOriginStep == nil || *summary.FirstViolationOriginStep != 6 {
+		t.Fatalf("origin step: got %v, want 6", summary.FirstViolationOriginStep)
+	}
+	if summary.FirstViolationDetectedStep == nil || *summary.FirstViolationDetectedStep != 6 {
+		t.Fatalf("detected step: got %v, want 6", summary.FirstViolationDetectedStep)
+	}
+}
+
+func TestSummarizeRun_PicksNewestRunDirectory(t *testing.T) {
+	seedDirectory := t.TempDir()
+	writeRunDirectory(t, seedDirectory, "20260812-090000", []trace.Step{observedStep(1)})
+	writeRunDirectory(t, seedDirectory, "20260812-093000", []trace.Step{observedStep(1), observedStep(2)})
+
+	name, summary, err := summarizeRun(seedDirectory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if name != "20260812-093000" || summary.Steps != 2 {
+		t.Errorf("got %q with %d steps", name, summary.Steps)
+	}
+}
+
+func TestSummarizeRun_MissingRunDirectory(t *testing.T) {
+	if _, _, err := summarizeRun(t.TempDir()); err == nil {
+		t.Fatal("expected an error when no run directory was produced")
+	}
+}
+
+func TestSummarizeTrace_MalformedLine(t *testing.T) {
+	seedDirectory := t.TempDir()
+	directory := writeRunDirectory(t, seedDirectory, "20260812-090000", []trace.Step{observedStep(1)})
+	if err := os.WriteFile(filepath.Join(directory, "trace.jsonl"), []byte("{\"step\":1}\n{not json\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := summarizeRun(seedDirectory); err == nil {
+		t.Fatal("expected an error for a malformed trace line")
+	}
+}