From abacd5c5e1d253e8548e07ab7e073a7b6fb4e8d4 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 1 Sep 2026 00:01:26 +0530 Subject: [PATCH] Bound sample wall clock; a request timeout does not stop a hang One baseline sample sat on a single model request for 2h15m with an established connection, 0.1% CPU and no file activity since the init solver wrote func.py. The per-request timeout and max_retries did not bound it, and it blocked the rest of the run behind it. time_limit is the control that actually applies, per sample. Samples that finish take 8 to 15 minutes, so 30 is generous. An unbounded straggler costs more than the sample does. Claude-Session: https://claude.ai/code/session_01Cq98H7sNoSJdL3W98f18bu --- scripts/baseline.py | 7 +++++++ scripts/pilot.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/scripts/baseline.py b/scripts/baseline.py index e325a63..ab997a9 100644 --- a/scripts/baseline.py +++ b/scripts/baseline.py @@ -35,6 +35,12 @@ CONCURRENCY = int(os.environ.get("MBB_CONCURRENCY", "12")) # holding a slot for the length of the run, not to cut short slow thinking. REQUEST_TIMEOUT = int(os.environ.get("MBB_TIMEOUT", "900")) +# Per-sample wall clock. One baseline sample hung on a single model request for +# 2h15m with no file activity and never came back; the request timeout did not +# bound it. Samples that finish take 8 to 15 minutes, so 30 is generous, and an +# unbounded straggler blocking a whole run is worth more than the sample. +SAMPLE_TIME_LIMIT = int(os.environ.get("MBB_SAMPLE_LIMIT", "1800")) + def credits_used() -> float | None: """Total credits spent on the key so far, or None if the endpoint is unavailable.""" @@ -71,6 +77,7 @@ def run_split(split: str) -> dict: fail_on_error=False, timeout=REQUEST_TIMEOUT, max_retries=3, + time_limit=SAMPLE_TIME_LIMIT, ) after = credits_used() log = logs[0] diff --git a/scripts/pilot.py b/scripts/pilot.py index 0241be0..794d756 100644 --- a/scripts/pilot.py +++ b/scripts/pilot.py @@ -27,6 +27,12 @@ MODEL = os.environ.get("MBB_MODEL", "openrouter/z-ai/glm-5.3-flash") LIMIT = int(os.environ.get("MBB_LIMIT", "30")) CONCURRENCY = int(os.environ.get("MBB_CONCURRENCY", "12")) REQUEST_TIMEOUT = int(os.environ.get("MBB_TIMEOUT", "900")) + +# Per-sample wall clock. One baseline sample hung on a single model request for +# 2h15m with no file activity and never came back; the request timeout did not +# bound it. Samples that finish take 8 to 15 minutes, so 30 is generous, and an +# unbounded straggler blocking a whole run is worth more than the sample. +SAMPLE_TIME_LIMIT = int(os.environ.get("MBB_SAMPLE_LIMIT", "1800")) OUT = Path("results") @@ -88,6 +94,7 @@ if __name__ == "__main__": fail_on_error=False, timeout=REQUEST_TIMEOUT, max_retries=3, + time_limit=SAMPLE_TIME_LIMIT, )[0] after = credits_used()