feat(campaign): kill a run that outlives --run-timeout

A wedged run holds its worker for the rest of the sweep, and on an unattended
host nothing else will send it a signal. Defaults to three times --duration and
must exceed it. A killed run is recorded as timed_out rather than as a generic
failure, so the analysis can tell a lost cell from a real crash.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX

cmd/internal-tools/campaign/campaign.go

@@ -42,6 +42,7 @@ type runRecord struct {
 	Device         string    `json:"device,omitempty"`
 	ExitCode       int       `json:"exit_code"`
 	LaunchError    string    `json:"launch_error,omitempty"`
+	TimedOut       bool      `json:"timed_out,omitempty"`
 	StartedAt      time.Time `json:"started_at"`
 	DurationMillis int64     `json:"duration_millis"`
 	RunDirectory   string    `json:"run_directory,omitempty"`
@@ -170,8 +171,13 @@ func (c *campaign) runSeed(ctx context.Context, seed int64, device string) runRe
 	}
 	defer logFile.Close()
 
+	runCtx, cancelRun := context.WithTimeout(ctx, c.configuration.runTimeout)
+	defer cancelRun()
 	start := time.Now()
-	exitCode, runErr := c.executor(ctx, c.configuration.sanderlingPath, runArguments(c.configuration, seedText, device), logFile)
+	exitCode, runErr := c.executor(runCtx, c.configuration.sanderlingPath, runArguments(c.configuration, seedText, device), logFile)
+	if runCtx.Err() != nil && ctx.Err() == nil {
+		record.TimedOut = true
+	}
 	record.DurationMillis = time.Since(start).Milliseconds()
 	record.ExitCode = exitCode
 	if runErr != nil {
@@ -208,6 +214,8 @@ func (c *campaign) report(record runRecord) {
 
 func outcome(record runRecord) string {
 	switch {
+	case record.TimedOut:
+		return "timed out"
 	case record.ExitCode != 0:
 		return "failed"
 	case record.FirstViolationOriginStep != nil:

cmd/internal-tools/campaign/campaign_test.go

@@ -315,3 +315,48 @@ func TestRunCampaign_UnreadableTraceIsNotASuccessfulCampaign(t *testing.T) {
 		}
 	}
 }
+
+func TestRunCampaign_KillsAndRecordsAWedgedRun(t *testing.T) {
+	campaignDirectory := filepath.Join(t.TempDir(), "cell")
+	configuration := testConfiguration(t, campaignDirectory,
+		"--seeds", "1", "--duration", "50ms", "--run-timeout", "300ms")
+
+	executor := versionAnswering(func(ctx context.Context, _ string, _ []string, _ io.Writer) (int, error) {
+		<-ctx.Done()
+		return -1, ctx.Err()
+	})
+
+	var stdout bytes.Buffer
+	if err := runCampaign(context.Background(), configuration, executor, &stdout); err == nil {
+		t.Fatal("a campaign whose only run was killed must not report success")
+	}
+	records := readRecords(t, campaignDirectory)
+	if len(records) != 1 {
+		t.Fatalf("want one record, got %d", len(records))
+	}
+	if !records[0].TimedOut {
+		t.Error("a run killed by the run timeout must be recorded as timed out")
+	}
+	if !strings.Contains(stdout.String(), "timed out") {
+		t.Errorf("the progress line must name the outcome:\n%s", stdout.String())
+	}
+}
+
+func TestParseArguments_RunTimeoutMustExceedDuration(t *testing.T) {
+	_, err := parseArguments(append(baseArguments(),
+		"--output", t.TempDir(), "--duration", "5m", "--run-timeout", "1m"), io.Discard)
+	if err == nil {
+		t.Fatal("a run timeout below the duration would kill every run before it finished")
+	}
+}
+
+func TestParseArguments_RunTimeoutDefaultsToThreeTimesDuration(t *testing.T) {
+	configuration, err := parseArguments(append(baseArguments(),
+		"--output", t.TempDir(), "--duration", "4m"), io.Discard)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if configuration.runTimeout != 12*time.Minute {
+		t.Errorf("run timeout default: got %s, want 12m", configuration.runTimeout)
+	}
+}

cmd/internal-tools/campaign/main.go

@@ -29,6 +29,7 @@ type config struct {
 	devices         []string
 	sanderlingPath  string
 	outputDirectory string
+	runTimeout      time.Duration
 	extraArguments  []string
 }
 
@@ -61,6 +62,7 @@ func parseArguments(arguments []string, stderr io.Writer) (config, error) {
 	flagSet.DurationVar(&configuration.duration, "duration", 5*time.Minute, "per-run wall-clock ceiling")
 	flagSet.StringVar(&seedSpecification, "seeds", "", "seeds to run: ranges and lists, e.g. 1-10,20,30-32 (required)")
 	flagSet.StringVar(&deviceList, "devices", "", "comma-separated device identifiers; one concurrent worker per device (on web these are worker labels, no device flag is passed)")
+	flagSet.DurationVar(&configuration.runTimeout, "run-timeout", 0, "kill a run that outlives this (default: three times --duration). A wedged run holds its worker for the rest of the sweep, and nothing else will send it a signal on an unattended host")
 	flagSet.StringVar(&configuration.sanderlingPath, "sanderling", "sanderling", "sanderling binary to invoke")
 	flagSet.StringVar(&configuration.outputDirectory, "output", "", "campaign directory to create (required)")
 	if err := flagSet.Parse(arguments); err != nil {
@@ -97,6 +99,16 @@ func parseArguments(arguments []string, stderr io.Writer) (config, error) {
 	if configuration.duration <= 0 {
 		return config{}, fmt.Errorf("--duration must be positive: %s", configuration.duration)
 	}
+	if configuration.runTimeout < 0 {
+		return config{}, fmt.Errorf("--run-timeout must not be negative: %s", configuration.runTimeout)
+	}
+	if configuration.runTimeout == 0 {
+		configuration.runTimeout = 3 * configuration.duration
+	}
+	if configuration.runTimeout <= configuration.duration {
+		return config{}, fmt.Errorf("--run-timeout %s must exceed --duration %s, or every run is killed before it finishes",
+			configuration.runTimeout, configuration.duration)
+	}
 	seeds, err := parseSeeds(seedSpecification)
 	if err != nil {
 		return config{}, fmt.Errorf("--seeds: %w", err)

cmd/internal-tools/campaign/manifest.go

@@ -26,6 +26,7 @@ type manifest struct {
 	BundleID          string    `json:"bundle_id"`
 	MaxSteps          int       `json:"max_steps"`
 	DurationMillis    int64     `json:"duration_millis"`
+	RunTimeoutMillis  int64     `json:"run_timeout_millis"`
 	Seeds             []int64   `json:"seeds"`
 	Devices           []string  `json:"devices"`
 	Host              string    `json:"host"`
@@ -48,6 +49,7 @@ func buildManifest(configuration config, host, binaryPath, version string, start
 		BundleID:          configuration.bundleID,
 		MaxSteps:          configuration.maxSteps,
 		DurationMillis:    configuration.duration.Milliseconds(),
+		RunTimeoutMillis:  configuration.runTimeout.Milliseconds(),
 		Seeds:             configuration.seeds,
 		Devices:           devices,
 		Host:              host,