feat(campaign): kill a run that outlives --run-timeout

A wedged run holds its worker for the rest of the sweep, and on an unattended
host nothing else will send it a signal. Defaults to three times --duration and
must exceed it. A killed run is recorded as timed_out rather than as a generic
failure, so the analysis can tell a lost cell from a real crash.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
pj committed 2026-08-12 21:39:09 +05:30
1 parent 808d607eac
commit 024913ceeb
4 files changed
+68 -1

No files matched your search

+9 -1
View File
@@ -42,6 +42,7 @@ type runRecord struct {
Device string `json:"device,omitempty"`
ExitCode int `json:"exit_code"`
LaunchError string `json:"launch_error,omitempty"`
TimedOut bool `json:"timed_out,omitempty"`
StartedAt time.Time `json:"started_at"`
DurationMillis int64 `json:"duration_millis"`
RunDirectory string `json:"run_directory,omitempty"`
@@ -170,8 +171,13 @@ func (c *campaign) runSeed(ctx context.Context, seed int64, device string) runRe
}
defer logFile.Close()
runCtx, cancelRun := context.WithTimeout(ctx, c.configuration.runTimeout)
defer cancelRun()
start := time.Now()
exitCode, runErr := c.executor(ctx, c.configuration.sanderlingPath, runArguments(c.configuration, seedText, device), logFile)
exitCode, runErr := c.executor(runCtx, c.configuration.sanderlingPath, runArguments(c.configuration, seedText, device), logFile)
if runCtx.Err() != nil && ctx.Err() == nil {
record.TimedOut = true
}
record.DurationMillis = time.Since(start).Milliseconds()
record.ExitCode = exitCode
if runErr != nil {
@@ -208,6 +214,8 @@ func (c *campaign) report(record runRecord) {
func outcome(record runRecord) string {
switch {
case record.TimedOut:
return "timed out"
case record.ExitCode != 0:
return "failed"
case record.FirstViolationOriginStep != nil: