fix(campaign): record both clocks a run was measured on

Duration came from the monotonic clock, which does not advance while a host
sleeps: one calibration run under-reported by about 15 minutes. A run now
carries monotonic_millis for how long it worked and wall_clock_millis for how
much time passed, which is what makes a sleep visible at all.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
pj committed 2026-08-14 17:43:12 +05:30
1 parent 95a474339e
commit c63a2e4897
2 files changed
+86 -15

No files matched your search

+41 -15
View File
@@ -51,25 +51,48 @@ func executeCommand(ctx context.Context, binary string, arguments []string, outp
// escalation would land while the run was still doing what it was asked.
const runShutdownGrace = 30 * time.Second
// runRecord is one line of runs.jsonl.
// runRecord is one line of runs.jsonl. MonotonicMillis is how long the run
// worked and WallClockMillis is how much time passed; they answer different
// questions and differ by however long the host slept mid-run, which an
// unattended overnight sweep is exactly where to expect.
type runRecord struct {
Seed int64 `json:"seed"`
Device string `json:"device,omitempty"`
ExitCode int `json:"exit_code"`
LaunchError string `json:"launch_error,omitempty"`
TimedOut bool `json:"timed_out,omitempty"`
StartedAt time.Time `json:"started_at"`
DurationMillis int64 `json:"duration_millis"`
RunDirectory string `json:"run_directory,omitempty"`
TraceError string `json:"trace_error,omitempty"`
Seed int64 `json:"seed"`
Device string `json:"device,omitempty"`
ExitCode int `json:"exit_code"`
LaunchError string `json:"launch_error,omitempty"`
TimedOut bool `json:"timed_out,omitempty"`
StartedAt time.Time `json:"started_at"`
MonotonicMillis int64 `json:"monotonic_millis"`
WallClockMillis int64 `json:"wall_clock_millis"`
RunDirectory string `json:"run_directory,omitempty"`
TraceError string `json:"trace_error,omitempty"`
traceSummary
}
// clocks reads the two measures a run is timed on. The monotonic clock does
// not advance while the host is asleep, so on its own it reports a run that
// slept through a quarter of an hour as a quarter of an hour shorter than it
// was; the wall clock advances but can be stepped by the host.
type clocks struct {
monotonicNow func() time.Time
wallClockNow func() time.Time
}
func systemClocks() clocks {
return clocks{
monotonicNow: time.Now,
// Round(0) drops the monotonic reading time.Now carries, so
// subtracting two of these readings uses the wall clock.
wallClockNow: func() time.Time { return time.Now().Round(0) },
}
}
type campaign struct {
configuration config
executor commandExecutor
stdout io.Writer
records io.Writer
clocks clocks
mutex sync.Mutex
failures int
unreadable int
@@ -105,7 +128,7 @@ func runCampaign(ctx context.Context, configuration config, executor commandExec
}
defer recordsFile.Close()
sweep := &campaign{configuration: configuration, executor: executor, stdout: stdout, records: recordsFile}
sweep := &campaign{configuration: configuration, executor: executor, stdout: stdout, records: recordsFile, clocks: systemClocks()}
fmt.Fprintf(stdout, "campaign %s: %d seeds, %d worker(s), %s\n",
configuration.arm, len(configuration.seeds), len(workerDevices(configuration.devices)), configuration.outputDirectory)
sweep.sweep(ctx)
@@ -188,12 +211,14 @@ func (c *campaign) runSeed(ctx context.Context, seed int64, device string) runRe
runCtx, cancelRun := context.WithTimeout(ctx, c.configuration.runTimeout)
defer cancelRun()
start := time.Now()
monotonicStart := c.clocks.monotonicNow()
wallClockStart := c.clocks.wallClockNow()
exitCode, runErr := c.executor(runCtx, c.configuration.sanderlingPath, runArguments(c.configuration, seedText, device), logFile)
if runCtx.Err() != nil && ctx.Err() == nil {
record.TimedOut = true
}
record.DurationMillis = time.Since(start).Milliseconds()
record.MonotonicMillis = c.clocks.monotonicNow().Sub(monotonicStart).Milliseconds()
record.WallClockMillis = c.clocks.wallClockNow().Sub(wallClockStart).Milliseconds()
record.ExitCode = exitCode
if runErr != nil {
record.LaunchError = runErr.Error()
@@ -222,9 +247,10 @@ func (c *campaign) report(record runRecord) {
if err := json.NewEncoder(c.records).Encode(record); err != nil {
fmt.Fprintf(c.stdout, "warning: seed %d record: %v\n", record.Seed, err)
}
fmt.Fprintf(c.stdout, "seed=%d device=%q outcome=%s steps=%d exit=%d duration=%s\n",
fmt.Fprintf(c.stdout, "seed=%d device=%q outcome=%s steps=%d exit=%d monotonic=%s wall_clock=%s\n",
record.Seed, record.Device, outcome(record), record.Steps, record.ExitCode,
time.Duration(record.DurationMillis)*time.Millisecond)
time.Duration(record.MonotonicMillis)*time.Millisecond,
time.Duration(record.WallClockMillis)*time.Millisecond)
}
func outcome(record runRecord) string {