feat(campaign): sweep seeds for one experiment cell

campaign.json lists the seeds a sweep intended to run and is written before
the first run, so a host that dropped runs shows up as missing seeds rather
than as a smaller sample. Seed 0 is rejected: sanderling test reads it as
"derive a seed from the clock", which is why conformance/gates.sh controls
nothing today.

Each run contributes one runs.jsonl line carrying steps to first violation by
origin step, the step that armed the failed obligation, so the survival
analysis never reopens a trace.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX

cmd/internal-tools/campaign/campaign_test.go

@@ -0,0 +1,317 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/priyanshujain/sanderling/internal/trace"
+)
+
+func testConfiguration(t *testing.T, outputDirectory string, extra ...string) config {
+	t.Helper()
+	arguments := append(baseArguments(), "--output", outputDirectory)
+	configuration, err := parseArguments(append(arguments, extra...), io.Discard)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return configuration
+}
+
+// versionAnswering wraps a test executor so every fake answers `sanderling
+// version`, which the campaign probes before it writes the manifest.
+func versionAnswering(executor commandExecutor) commandExecutor {
+	return func(ctx context.Context, binary string, arguments []string, output io.Writer) (int, error) {
+		if len(arguments) > 0 && arguments[0] == "version" {
+			fmt.Fprintln(output, "stub-version")
+			return 0, nil
+		}
+		return executor(ctx, binary, arguments, output)
+	}
+}
+
+func readRecords(t *testing.T, campaignDirectory string) []runRecord {
+	t.Helper()
+	file, err := os.Open(filepath.Join(campaignDirectory, recordsFileName))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+	var records []runRecord
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		var record runRecord
+		if err := json.Unmarshal(scanner.Bytes(), &record); err != nil {
+			t.Fatalf("decode %q: %v", scanner.Text(), err)
+		}
+		records = append(records, record)
+	}
+	if err := scanner.Err(); err != nil {
+		t.Fatal(err)
+	}
+	return records
+}
+
+func writeFakeRun(t *testing.T, arguments []string, steps []trace.Step) {
+	t.Helper()
+	writeRunDirectory(t, argumentValue(arguments, "--output"), "20260101-000000", steps)
+}
+
+func TestRunCampaign_WritesManifestBeforeAnyRun(t *testing.T) {
+	directory := t.TempDir()
+	configuration := testConfiguration(t, directory)
+	var stdout bytes.Buffer
+
+	executor := versionAnswering(func(_ context.Context, _ string, arguments []string, _ io.Writer) (int, error) {
+		if _, err := os.Stat(filepath.Join(directory, manifestFileName)); err != nil {
+			t.Errorf("manifest missing when the first run started: %v", err)
+		}
+		writeFakeRun(t, arguments, []trace.Step{observedStep(1)})
+		return 0, nil
+	})
+	if err := runCampaign(context.Background(), configuration, executor, &stdout); err != nil {
+		t.Fatal(err)
+	}
+
+	body, err := os.ReadFile(filepath.Join(directory, manifestFileName))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var recorded manifest
+	if err := json.Unmarshal(body, &recorded); err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Equal(recorded.Seeds, []int64{1, 2, 3}) {
+		t.Errorf("intended seeds: got %v", recorded.Seeds)
+	}
+	if recorded.SanderlingVersion != "stub-version" {
+		t.Errorf("version: got %q", recorded.SanderlingVersion)
+	}
+	if recorded.Host == "" {
+		t.Error("host was not recorded")
+	}
+}
+
+func TestRunCampaign_RecordsPerRunSummary(t *testing.T) {
+	directory := t.TempDir()
+	configuration := testConfiguration(t, directory, "--seeds", "1-3")
+	var stdout bytes.Buffer
+
+	executor := versionAnswering(func(_ context.Context, _ string, arguments []string, output io.Writer) (int, error) {
+		fmt.Fprintln(output, "stub run log")
+		steps := []trace.Step{observedStep(1), observedStep(2), observedStep(3)}
+		if argumentValue(arguments, "--seed") == "2" {
+			violating := observedStep(3)
+			violating.Violations = []string{"listNeverEmpty"}
+			violating.Witnesses = map[string]trace.Witness{
+				"listNeverEmpty": {Reason: "list emptied", Step: 2, DetectedStep: 3},
+			}
+			steps[2] = violating
+		}
+		writeFakeRun(t, arguments, steps)
+		return 0, nil
+	})
+	if err := runCampaign(context.Background(), configuration, executor, &stdout); err != nil {
+		t.Fatal(err)
+	}
+
+	records := readRecords(t, directory)
+	if len(records) != 3 {
+		t.Fatalf("records: got %d, want 3", len(records))
+	}
+	for _, record := range records {
+		if record.Steps != 3 {
+			t.Errorf("seed %d steps: got %d, want 3", record.Seed, record.Steps)
+		}
+		if record.RunDirectory != fmt.Sprintf("seed-%d/20260101-000000", record.Seed) {
+			t.Errorf("seed %d run directory: got %q", record.Seed, record.RunDirectory)
+		}
+		if record.TraceError != "" {
+			t.Errorf("seed %d trace error: %s", record.Seed, record.TraceError)
+		}
+		violated := record.FirstViolationOriginStep != nil
+		if violated != (record.Seed == 2) {
+			t.Errorf("seed %d violation: got %v", record.Seed, record.FirstViolationOriginStep)
+		}
+		if record.Seed != 2 {
+			continue
+		}
+		if *record.FirstViolationOriginStep != 2 || *record.FirstViolationDetectedStep != 3 {
+			t.Errorf("seed 2 violation steps: origin %d detected %d",
+				*record.FirstViolationOriginStep, *record.FirstViolationDetectedStep)
+		}
+		if !slices.Equal(record.ViolatedProperties, []string{"listNeverEmpty"}) {
+			t.Errorf("seed 2 properties: got %v", record.ViolatedProperties)
+		}
+	}
+
+	log, err := os.ReadFile(filepath.Join(directory, "seed-1", "sanderling.log"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(log), "stub run log") {
+		t.Errorf("per-run log: got %q", log)
+	}
+	if !strings.Contains(stdout.String(), "seed=2") {
+		t.Errorf("progress output missing seed 2: %q", stdout.String())
+	}
+}
+
+func TestRunCampaign_DistributesSeedsAcrossDeviceWorkers(t *testing.T) {
+	directory := t.TempDir()
+	configuration := testConfiguration(t, directory, "--seeds", "1-9", "--devices", "device-a,device-b,device-c")
+
+	var mutex sync.Mutex
+	assignments := map[int64]string{}
+	var inFlight atomic.Int32
+	var releaseOnce sync.Once
+	var timedOut atomic.Bool
+	release := make(chan struct{})
+
+	executor := versionAnswering(func(_ context.Context, _ string, arguments []string, _ io.Writer) (int, error) {
+		if inFlight.Add(1) >= 3 {
+			releaseOnce.Do(func() { close(release) })
+		}
+		select {
+		case <-release:
+		case <-time.After(5 * time.Second):
+			timedOut.Store(true)
+		}
+		seed, err := strconv.ParseInt(argumentValue(arguments, "--seed"), 10, 64)
+		if err != nil {
+			t.Errorf("seed argument: %v", err)
+		}
+		mutex.Lock()
+		if previous, seen := assignments[seed]; seen {
+			t.Errorf("seed %d ran twice (%s then %s)", seed, previous, argumentValue(arguments, "--device"))
+		}
+		assignments[seed] = argumentValue(arguments, "--device")
+		mutex.Unlock()
+		writeFakeRun(t, arguments, []trace.Step{observedStep(1)})
+		return 0, nil
+	})
+	if err := runCampaign(context.Background(), configuration, executor, io.Discard); err != nil {
+		t.Fatal(err)
+	}
+	if timedOut.Load() {
+		t.Fatal("three device workers never ran concurrently")
+	}
+	if len(assignments) != 9 {
+		t.Fatalf("seeds run: got %d, want 9", len(assignments))
+	}
+	used := map[string]int{}
+	for seed, device := range assignments {
+		if !slices.Contains(configuration.devices, device) {
+			t.Errorf("seed %d ran on unknown device %q", seed, device)
+		}
+		used[device]++
+	}
+	if len(used) != 3 {
+		t.Errorf("devices used: got %v, want all three", used)
+	}
+	if got := len(readRecords(t, directory)); got != 9 {
+		t.Errorf("records: got %d, want 9", got)
+	}
+}
+
+func TestRunCampaign_ContinuesAfterFailingRun(t *testing.T) {
+	directory := t.TempDir()
+	configuration := testConfiguration(t, directory, "--seeds", "1-3")
+
+	executor := versionAnswering(func(_ context.Context, _ string, arguments []string, output io.Writer) (int, error) {
+		if argumentValue(arguments, "--seed") == "2" {
+			fmt.Fprintln(output, "error: device offline")
+			return 1, nil
+		}
+		writeFakeRun(t, arguments, []trace.Step{observedStep(1)})
+		return 0, nil
+	})
+	err := runCampaign(context.Background(), configuration, executor, io.Discard)
+	if err == nil || !strings.Contains(err.Error(), "1 of 3 runs failed") {
+		t.Fatalf("expected a failure summary, got %v", err)
+	}
+
+	records := readRecords(t, directory)
+	if len(records) != 3 {
+		t.Fatalf("a failing run must not abort the campaign: got %d records", len(records))
+	}
+	for _, record := range records {
+		if record.Seed == 2 {
+			if record.ExitCode != 1 {
+				t.Errorf("seed 2 exit code: got %d, want 1", record.ExitCode)
+			}
+			if record.TraceError == "" {
+				t.Error("seed 2 produced no trace; that should be recorded")
+			}
+			continue
+		}
+		if record.ExitCode != 0 {
+			t.Errorf("seed %d exit code: got %d", record.Seed, record.ExitCode)
+		}
+	}
+}
+
+func TestRunCampaign_RefusesToReuseACampaignDirectory(t *testing.T) {
+	directory := t.TempDir()
+	configuration := testConfiguration(t, directory)
+	if err := os.WriteFile(filepath.Join(directory, manifestFileName), []byte("{}"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	executor := versionAnswering(func(context.Context, string, []string, io.Writer) (int, error) {
+		t.Error("no run should start in a directory that already holds a campaign")
+		return 0, nil
+	})
+	if err := runCampaign(context.Background(), configuration, executor, io.Discard); err == nil {
+		t.Fatal("expected a refusal to reuse the campaign directory")
+	}
+}
+
+func TestRunCampaign_AbortsWhenVersionProbeFails(t *testing.T) {
+	directory := t.TempDir()
+	configuration := testConfiguration(t, directory)
+	executor := func(_ context.Context, _ string, arguments []string, output io.Writer) (int, error) {
+		if arguments[0] != "version" {
+			t.Error("a run started despite an unusable binary")
+		}
+		fmt.Fprintln(output, "no such command")
+		return 2, nil
+	}
+	if err := runCampaign(context.Background(), configuration, executor, io.Discard); err == nil {
+		t.Fatal("expected the campaign to abort before writing a manifest it cannot attribute")
+	}
+	if _, err := os.Stat(filepath.Join(directory, manifestFileName)); err == nil {
+		t.Error("manifest was written despite an unusable binary")
+	}
+}
+
+func TestRunCampaign_UnreadableTraceIsNotASuccessfulCampaign(t *testing.T) {
+	campaignDirectory := filepath.Join(t.TempDir(), "cell")
+	configuration := testConfiguration(t, campaignDirectory, "--seeds", "1-2")
+
+	executor := versionAnswering(func(context.Context, string, []string, io.Writer) (int, error) {
+		return 0, nil
+	})
+
+	var stdout bytes.Buffer
+	err := runCampaign(context.Background(), configuration, executor, &stdout)
+	if err == nil || !strings.Contains(err.Error(), "2 of 2 runs produced an unreadable trace") {
+		t.Fatalf("a campaign whose runs left no readable trace must not report success: %v", err)
+	}
+	for _, record := range readRecords(t, campaignDirectory) {
+		if record.TraceError == "" {
+			t.Errorf("seed %d: expected a trace error, got none", record.Seed)
+		}
+	}
+}