fix(runner): a source that was asked and handed nothing says so

NextAction returning ErrNoAction left the step with no skip reason, so a
run whose every model call failed on transport, a non-2xx, an empty
choices array or an echo mismatch printed no violations and exited 0.
Only llm-calls.jsonl knew it had never touched the app.

The reason now travels the path the other five already take, so it
reaches the trace, the summary, and the campaign's dispatched-action
exclusion. A held step never asks and keeps carrying nothing.
This commit is contained in:
pj committed 2026-08-18 14:16:42 +05:30
1 parent 1da0c3e118
commit 0646611a7f
5 files changed
+195 -11

No files matched your search

+66
View File
@@ -752,6 +752,72 @@ func TestLLMSourceSkipsOnHTTPError(t *testing.T) {
}
}
// TestRunner_EveryModelCallFailingIsNotACleanRun drives the whole loop against a
// provider that answers every call with a rate limit. The picker hands back no
// action on every step, so the run observes the app, judges the same launch
// screen over and over and never touches it. Only llm-calls.jsonl used to know:
// the trace, the summary and the exit status were a healthy run's.
func TestRunner_EveryModelCallFailingIsNotACleanRun(t *testing.T) {
fake := newFakeOpenRouter(t)
fake.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusTooManyRequests)
})
t.Setenv("OPENROUTER_API_KEY", "test-key")
t.Setenv("OPENROUTER_BASE_URL", fake.server.URL)
state := newHarnessWithSpec(t, llmFixtureSpec)
state.mock.HierarchyJSON = llmTreeJSON
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
summary, err := Run(ctx, Options{
Duration: 30 * time.Second,
IdleTimeout: 20 * time.Millisecond,
MaxSteps: 3,
Driver: state.mock,
Verifier: state.verifier,
TraceWriter: state.writer,
Generator: "llm",
LabelSource: verifier.LabelSourceVisibleText,
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
})
if err != nil {
t.Fatalf("Run: %v", err)
}
calls := readLLMCalls(t, state.writer.Directory())
if len(calls) != 3 {
t.Fatalf("recorded %d model calls, want 3", len(calls))
}
for _, call := range calls {
if call.Outcome != trace.LLMOutcomeRequestFailed {
t.Fatalf("call at step %d ended %q, want %q: the fixture must fail every call",
call.Step, call.Outcome, trace.LLMOutcomeRequestFailed)
}
}
if summary.DispatchedActions != 0 {
t.Fatalf("DispatchedActions = %d, want 0: every model call failed",
summary.DispatchedActions)
}
if got := summary.SkippedActions[string(actionSkippedNoActionProduced)]; got != 3 {
t.Errorf("summary counted %d step(s) as %q, want 3: %v",
got, actionSkippedNoActionProduced, summary.SkippedActions)
}
for _, line := range readTraceLines(t, state.writer.Directory()) {
if line.ActionSkipped != string(actionSkippedNoActionProduced) {
t.Errorf("step %d action_skipped = %q, want %q",
line.Step, line.ActionSkipped, actionSkippedNoActionProduced)
}
}
for _, action := range state.mock.Actions() {
switch action.Kind {
case mockdriver.ActionTap, mockdriver.ActionTapSelector, mockdriver.ActionInputText:
t.Errorf("the run drove the app with %v after every model call failed", action)
}
}
}
// llmSetupFixtureSpec drives the first action from setup, so the model is never
// consulted for that step.
const llmSetupFixtureSpec = `