mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 12:07:09 +00:00
fix(campaign): signal a timed-out run so it reaps its sidecar
CommandContext kills outright, so a run stopped by --run-timeout never ran its own shutdown and left a sidecar holding a port and a quarter gigabyte, reparented to init and deaf to SIGTERM. The timeout exists for unattended hosts, which is exactly where nobody is watching to reap what it leaves. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
9a2bbbf769
commit
dd17f831ef
2 files changed
+42
No files matched your search
@@ -13,6 +13,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -25,6 +26,12 @@ func executeCommand(ctx context.Context, binary string, arguments []string, outp
|
||||
command := exec.CommandContext(ctx, binary, arguments...)
|
||||
command.Stdout = output
|
||||
command.Stderr = output
|
||||
// SIGTERM rather than the default kill: a run killed outright never runs its
|
||||
// own shutdown, and its sidecar survives holding a port and a quarter
|
||||
// gigabyte. The run timeout exists for unattended hosts, which is exactly
|
||||
// where nobody is watching to reap what it leaves.
|
||||
command.Cancel = func() error { return command.Process.Signal(syscall.SIGTERM) }
|
||||
command.WaitDelay = runShutdownGrace
|
||||
err := command.Run()
|
||||
if err == nil {
|
||||
return 0, nil
|
||||
@@ -33,9 +40,17 @@ func executeCommand(ctx context.Context, binary string, arguments []string, outp
|
||||
if errors.As(err, &exitError) {
|
||||
return exitError.ExitCode(), nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return -1, nil
|
||||
}
|
||||
return -1, err
|
||||
}
|
||||
|
||||
// runShutdownGrace bounds how long a signalled run gets to stop its sidecar
|
||||
// before it is killed. It exceeds the sidecar's own 15s shutdown grace, or the
|
||||
// escalation would land while the run was still doing what it was asked.
|
||||
const runShutdownGrace = 30 * time.Second
|
||||
|
||||
// runRecord is one line of runs.jsonl.
|
||||
type runRecord struct {
|
||||
Seed int64 `json:"seed"`
|
||||
|
||||
Reference in new issue
Block a user