feat(campaign): kill a run that outlives --run-timeout

A wedged run holds its worker for the rest of the sweep, and on an unattended
host nothing else will send it a signal. Defaults to three times --duration and
must exceed it. A killed run is recorded as timed_out rather than as a generic
failure, so the analysis can tell a lost cell from a real crash.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
pj committed 2026-08-12 21:39:09 +05:30
1 parent 808d607eac
commit 024913ceeb
4 files changed
+68 -1

No files matched your search

@@ -315,3 +315,48 @@ func TestRunCampaign_UnreadableTraceIsNotASuccessfulCampaign(t *testing.T) {
}
}
}
func TestRunCampaign_KillsAndRecordsAWedgedRun(t *testing.T) {
campaignDirectory := filepath.Join(t.TempDir(), "cell")
configuration := testConfiguration(t, campaignDirectory,
"--seeds", "1", "--duration", "50ms", "--run-timeout", "300ms")
executor := versionAnswering(func(ctx context.Context, _ string, _ []string, _ io.Writer) (int, error) {
<-ctx.Done()
return -1, ctx.Err()
})
var stdout bytes.Buffer
if err := runCampaign(context.Background(), configuration, executor, &stdout); err == nil {
t.Fatal("a campaign whose only run was killed must not report success")
}
records := readRecords(t, campaignDirectory)
if len(records) != 1 {
t.Fatalf("want one record, got %d", len(records))
}
if !records[0].TimedOut {
t.Error("a run killed by the run timeout must be recorded as timed out")
}
if !strings.Contains(stdout.String(), "timed out") {
t.Errorf("the progress line must name the outcome:\n%s", stdout.String())
}
}
func TestParseArguments_RunTimeoutMustExceedDuration(t *testing.T) {
_, err := parseArguments(append(baseArguments(),
"--output", t.TempDir(), "--duration", "5m", "--run-timeout", "1m"), io.Discard)
if err == nil {
t.Fatal("a run timeout below the duration would kill every run before it finished")
}
}
func TestParseArguments_RunTimeoutDefaultsToThreeTimesDuration(t *testing.T) {
configuration, err := parseArguments(append(baseArguments(),
"--output", t.TempDir(), "--duration", "4m"), io.Discard)
if err != nil {
t.Fatal(err)
}
if configuration.runTimeout != 12*time.Minute {
t.Errorf("run timeout default: got %s, want 12m", configuration.runTimeout)
}
}