mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
feat(runner): stamp llm ranked list and chosen rank on trace
This commit is contained in:
1 parent
84adc0f177
commit
1916bd9eea
2 files changed
+45
-15
No files matched your search
@@ -300,6 +300,18 @@ func TestLLMSourceFirstValidIndexWins(t *testing.T) {
|
||||
if action.Kind != verifier.ActionKindTap {
|
||||
t.Errorf("action = %+v, want Tap from the first valid index", action)
|
||||
}
|
||||
|
||||
// The trace records the full ranked list and the 1-based rank that won, so
|
||||
// the reasoning (which describes the top pick) can be reconciled with the
|
||||
// executed action when an earlier pick was skipped.
|
||||
traceAction := traceActionFor(action, nil)
|
||||
stampActionSource(traceAction, source)
|
||||
if !slices.Equal(traceAction.LLMRanked, []int{-1, tapIndex}) {
|
||||
t.Errorf("trace ranked = %v, want [-1 %d]", traceAction.LLMRanked, tapIndex)
|
||||
}
|
||||
if traceAction.LLMChosenRank != 2 {
|
||||
t.Errorf("trace chosen rank = %d, want 2 (the second pick won)", traceAction.LLMChosenRank)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMSourceSkipsOnHTTPError(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user