feat(runner): stamp llm ranked list and chosen rank on trace

This commit is contained in:
pj committed 2026-06-18 11:48:51 -04:00
1 parent 84adc0f177
commit 1916bd9eea
2 files changed
+45 -15

No files matched your search

+12
View File
@@ -300,6 +300,18 @@ func TestLLMSourceFirstValidIndexWins(t *testing.T) {
if action.Kind != verifier.ActionKindTap {
t.Errorf("action = %+v, want Tap from the first valid index", action)
}
// The trace records the full ranked list and the 1-based rank that won, so
// the reasoning (which describes the top pick) can be reconciled with the
// executed action when an earlier pick was skipped.
traceAction := traceActionFor(action, nil)
stampActionSource(traceAction, source)
if !slices.Equal(traceAction.LLMRanked, []int{-1, tapIndex}) {
t.Errorf("trace ranked = %v, want [-1 %d]", traceAction.LLMRanked, tapIndex)
}
if traceAction.LLMChosenRank != 2 {
t.Errorf("trace chosen rank = %d, want 2 (the second pick won)", traceAction.LLMChosenRank)
}
}
func TestLLMSourceSkipsOnHTTPError(t *testing.T) {