diff --git a/cmd/internal-tools/campaign/campaign.go b/cmd/internal-tools/campaign/campaign.go index a610e1e..e022732 100644 --- a/cmd/internal-tools/campaign/campaign.go +++ b/cmd/internal-tools/campaign/campaign.go @@ -13,6 +13,7 @@ import ( "strconv" "strings" "sync" + "syscall" "time" ) @@ -25,6 +26,12 @@ func executeCommand(ctx context.Context, binary string, arguments []string, outp command := exec.CommandContext(ctx, binary, arguments...) command.Stdout = output command.Stderr = output + // SIGTERM rather than the default kill: a run killed outright never runs its + // own shutdown, and its sidecar survives holding a port and a quarter + // gigabyte. The run timeout exists for unattended hosts, which is exactly + // where nobody is watching to reap what it leaves. + command.Cancel = func() error { return command.Process.Signal(syscall.SIGTERM) } + command.WaitDelay = runShutdownGrace err := command.Run() if err == nil { return 0, nil @@ -33,9 +40,17 @@ func executeCommand(ctx context.Context, binary string, arguments []string, outp if errors.As(err, &exitError) { return exitError.ExitCode(), nil } + if ctx.Err() != nil { + return -1, nil + } return -1, err } +// runShutdownGrace bounds how long a signalled run gets to stop its sidecar +// before it is killed. It exceeds the sidecar's own 15s shutdown grace, or the +// escalation would land while the run was still doing what it was asked. +const runShutdownGrace = 30 * time.Second + // runRecord is one line of runs.jsonl. type runRecord struct { Seed int64 `json:"seed"` diff --git a/cmd/internal-tools/campaign/end_to_end_test.go b/cmd/internal-tools/campaign/end_to_end_test.go index 61c8297..317fbab 100644 --- a/cmd/internal-tools/campaign/end_to_end_test.go +++ b/cmd/internal-tools/campaign/end_to_end_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "encoding/json" "io" "os" @@ -9,6 +10,7 @@ import ( "slices" "strings" "testing" + "time" ) // stubSanderling answers `version`, writes a run directory shaped like the one @@ -142,3 +144,28 @@ func TestRun_EndToEndAgainstStubBinary(t *testing.T) { t.Errorf("progress output: %q", stdout.String()) } } + +func TestExecuteCommand_RunTimeoutSignalsSoTheRunReapsItsChildren(t *testing.T) { + directory := t.TempDir() + marker := filepath.Join(directory, "child-reaped") + script := filepath.Join(directory, "wedged") + body := "#!/bin/sh\n" + + "sleep 300 &\n" + + "child=$!\n" + + "trap 'kill $child; echo reaped > " + marker + "; exit 143' TERM\n" + + "wait $child\n" + if err := os.WriteFile(script, []byte(body), 0o755); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := executeCommand(ctx, script, nil, io.Discard); err != nil { + t.Fatalf("execute: %v", err) + } + + if _, err := os.Stat(marker); err != nil { + t.Fatal("the run timeout killed the run outright, so it never reaped its own children: " + + "an unattended sweep leaks one sidecar per wedged run") + } +}