From 024913ceebd419e46f6cf0e94bf14f0042f76930 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 12 Aug 2026 21:39:09 +0530 Subject: [PATCH] feat(campaign): kill a run that outlives --run-timeout A wedged run holds its worker for the rest of the sweep, and on an unattended host nothing else will send it a signal. Defaults to three times --duration and must exceed it. A killed run is recorded as timed_out rather than as a generic failure, so the analysis can tell a lost cell from a real crash. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX --- cmd/internal-tools/campaign/campaign.go | 10 ++++- cmd/internal-tools/campaign/campaign_test.go | 45 ++++++++++++++++++++ cmd/internal-tools/campaign/main.go | 12 ++++++ cmd/internal-tools/campaign/manifest.go | 2 + 4 files changed, 68 insertions(+), 1 deletion(-) diff --git a/cmd/internal-tools/campaign/campaign.go b/cmd/internal-tools/campaign/campaign.go index 42cc8ca..a610e1e 100644 --- a/cmd/internal-tools/campaign/campaign.go +++ b/cmd/internal-tools/campaign/campaign.go @@ -42,6 +42,7 @@ type runRecord struct { Device string `json:"device,omitempty"` ExitCode int `json:"exit_code"` LaunchError string `json:"launch_error,omitempty"` + TimedOut bool `json:"timed_out,omitempty"` StartedAt time.Time `json:"started_at"` DurationMillis int64 `json:"duration_millis"` RunDirectory string `json:"run_directory,omitempty"` @@ -170,8 +171,13 @@ func (c *campaign) runSeed(ctx context.Context, seed int64, device string) runRe } defer logFile.Close() + runCtx, cancelRun := context.WithTimeout(ctx, c.configuration.runTimeout) + defer cancelRun() start := time.Now() - exitCode, runErr := c.executor(ctx, c.configuration.sanderlingPath, runArguments(c.configuration, seedText, device), logFile) + exitCode, runErr := c.executor(runCtx, c.configuration.sanderlingPath, runArguments(c.configuration, seedText, device), logFile) + if runCtx.Err() != nil && ctx.Err() == nil { + record.TimedOut = true + } record.DurationMillis = time.Since(start).Milliseconds() record.ExitCode = exitCode if runErr != nil { @@ -208,6 +214,8 @@ func (c *campaign) report(record runRecord) { func outcome(record runRecord) string { switch { + case record.TimedOut: + return "timed out" case record.ExitCode != 0: return "failed" case record.FirstViolationOriginStep != nil: diff --git a/cmd/internal-tools/campaign/campaign_test.go b/cmd/internal-tools/campaign/campaign_test.go index 59cc4f1..1a1b224 100644 --- a/cmd/internal-tools/campaign/campaign_test.go +++ b/cmd/internal-tools/campaign/campaign_test.go @@ -315,3 +315,48 @@ func TestRunCampaign_UnreadableTraceIsNotASuccessfulCampaign(t *testing.T) { } } } + +func TestRunCampaign_KillsAndRecordsAWedgedRun(t *testing.T) { + campaignDirectory := filepath.Join(t.TempDir(), "cell") + configuration := testConfiguration(t, campaignDirectory, + "--seeds", "1", "--duration", "50ms", "--run-timeout", "300ms") + + executor := versionAnswering(func(ctx context.Context, _ string, _ []string, _ io.Writer) (int, error) { + <-ctx.Done() + return -1, ctx.Err() + }) + + var stdout bytes.Buffer + if err := runCampaign(context.Background(), configuration, executor, &stdout); err == nil { + t.Fatal("a campaign whose only run was killed must not report success") + } + records := readRecords(t, campaignDirectory) + if len(records) != 1 { + t.Fatalf("want one record, got %d", len(records)) + } + if !records[0].TimedOut { + t.Error("a run killed by the run timeout must be recorded as timed out") + } + if !strings.Contains(stdout.String(), "timed out") { + t.Errorf("the progress line must name the outcome:\n%s", stdout.String()) + } +} + +func TestParseArguments_RunTimeoutMustExceedDuration(t *testing.T) { + _, err := parseArguments(append(baseArguments(), + "--output", t.TempDir(), "--duration", "5m", "--run-timeout", "1m"), io.Discard) + if err == nil { + t.Fatal("a run timeout below the duration would kill every run before it finished") + } +} + +func TestParseArguments_RunTimeoutDefaultsToThreeTimesDuration(t *testing.T) { + configuration, err := parseArguments(append(baseArguments(), + "--output", t.TempDir(), "--duration", "4m"), io.Discard) + if err != nil { + t.Fatal(err) + } + if configuration.runTimeout != 12*time.Minute { + t.Errorf("run timeout default: got %s, want 12m", configuration.runTimeout) + } +} diff --git a/cmd/internal-tools/campaign/main.go b/cmd/internal-tools/campaign/main.go index 8586bfd..c2a3f48 100644 --- a/cmd/internal-tools/campaign/main.go +++ b/cmd/internal-tools/campaign/main.go @@ -29,6 +29,7 @@ type config struct { devices []string sanderlingPath string outputDirectory string + runTimeout time.Duration extraArguments []string } @@ -61,6 +62,7 @@ func parseArguments(arguments []string, stderr io.Writer) (config, error) { flagSet.DurationVar(&configuration.duration, "duration", 5*time.Minute, "per-run wall-clock ceiling") flagSet.StringVar(&seedSpecification, "seeds", "", "seeds to run: ranges and lists, e.g. 1-10,20,30-32 (required)") flagSet.StringVar(&deviceList, "devices", "", "comma-separated device identifiers; one concurrent worker per device (on web these are worker labels, no device flag is passed)") + flagSet.DurationVar(&configuration.runTimeout, "run-timeout", 0, "kill a run that outlives this (default: three times --duration). A wedged run holds its worker for the rest of the sweep, and nothing else will send it a signal on an unattended host") flagSet.StringVar(&configuration.sanderlingPath, "sanderling", "sanderling", "sanderling binary to invoke") flagSet.StringVar(&configuration.outputDirectory, "output", "", "campaign directory to create (required)") if err := flagSet.Parse(arguments); err != nil { @@ -97,6 +99,16 @@ func parseArguments(arguments []string, stderr io.Writer) (config, error) { if configuration.duration <= 0 { return config{}, fmt.Errorf("--duration must be positive: %s", configuration.duration) } + if configuration.runTimeout < 0 { + return config{}, fmt.Errorf("--run-timeout must not be negative: %s", configuration.runTimeout) + } + if configuration.runTimeout == 0 { + configuration.runTimeout = 3 * configuration.duration + } + if configuration.runTimeout <= configuration.duration { + return config{}, fmt.Errorf("--run-timeout %s must exceed --duration %s, or every run is killed before it finishes", + configuration.runTimeout, configuration.duration) + } seeds, err := parseSeeds(seedSpecification) if err != nil { return config{}, fmt.Errorf("--seeds: %w", err) diff --git a/cmd/internal-tools/campaign/manifest.go b/cmd/internal-tools/campaign/manifest.go index 81c72ce..244b453 100644 --- a/cmd/internal-tools/campaign/manifest.go +++ b/cmd/internal-tools/campaign/manifest.go @@ -26,6 +26,7 @@ type manifest struct { BundleID string `json:"bundle_id"` MaxSteps int `json:"max_steps"` DurationMillis int64 `json:"duration_millis"` + RunTimeoutMillis int64 `json:"run_timeout_millis"` Seeds []int64 `json:"seeds"` Devices []string `json:"devices"` Host string `json:"host"` @@ -48,6 +49,7 @@ func buildManifest(configuration config, host, binaryPath, version string, start BundleID: configuration.bundleID, MaxSteps: configuration.maxSteps, DurationMillis: configuration.duration.Milliseconds(), + RunTimeoutMillis: configuration.runTimeout.Milliseconds(), Seeds: configuration.seeds, Devices: devices, Host: host,