mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-03 11:37:09 +00:00
fix(runner): no silent generator fallback, and llm on web
--generator llm against a spec declaring no generator = llm(...) logged a warning and ran the seeded picker. For a comparison campaign that is silent arm corruption: the run completes, the directory looks correct, and the wrong policy drove it. It is now fatal. pickSources also returned the V8 source for both action and extractor on web before it looked at the generator, so the llm policy was unreachable there. The two axes are now independent: the driver picks the extractor source, the flag picks the action source, and llmSource composes with either because the runner populates the candidate list and screenshot on every platform. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
028534cadc
commit
bee430cf72
2 files changed
+132
-26
No files matched your search
@@ -16,6 +16,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/priyanshujain/sanderling/internal/driver"
|
||||
mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
|
||||
"github.com/priyanshujain/sanderling/internal/hierarchy"
|
||||
"github.com/priyanshujain/sanderling/internal/llmclient"
|
||||
"github.com/priyanshujain/sanderling/internal/verifier"
|
||||
@@ -276,6 +278,100 @@ func TestPickSourcesSeededByDefault(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// seededFixtureSpec declares no generator = llm(...), so --generator llm has
|
||||
// nothing to build a picker from.
|
||||
const seededFixtureSpec = `
|
||||
import { always, taps, typing, weighted } from "@sanderling/spec";
|
||||
globalThis.properties = { ok: always(() => true) };
|
||||
globalThis.actions = weighted([1, taps], [1, typing]);
|
||||
`
|
||||
|
||||
func newSeededVerifier(t *testing.T) *verifier.Verifier {
|
||||
t.Helper()
|
||||
verifierInstance, err := verifier.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifierInstance.Load(bundleSpec(t, seededFixtureSpec)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := verifierInstance.LLMConfig(); ok {
|
||||
t.Fatal("seeded fixture spec must not register an llm action backend")
|
||||
}
|
||||
return verifierInstance
|
||||
}
|
||||
|
||||
// TestPickSourcesRejectsLLMWithoutSpecConfig pins the abort. Falling back to the
|
||||
// seeded picker here completes the run, writes a well-formed output directory,
|
||||
// and records it under the requested arm, so a campaign cell silently reports
|
||||
// the wrong policy's numbers.
|
||||
func TestPickSourcesRejectsLLMWithoutSpecConfig(t *testing.T) {
|
||||
for name, activeDriver := range map[string]driver.DeviceDriver{
|
||||
"native": nil,
|
||||
"web": &webMockDriver{Driver: mockdriver.New()},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
action, extractor, err := pickSources(Options{
|
||||
Driver: activeDriver,
|
||||
Verifier: newSeededVerifier(t),
|
||||
Generator: "llm",
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("pickSources = (%T, %T), want an error", action, extractor)
|
||||
}
|
||||
if action != nil || extractor != nil {
|
||||
t.Errorf("sources = (%v, %v), want both nil alongside the error", action, extractor)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "generator = llm(...)") {
|
||||
t.Errorf("error = %q, want it to name the missing spec declaration", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPickSourcesOnWebComposesLLMWithWebExtractors covers the second half of the
|
||||
// same claim: the picker is chosen by --generator and the extractor source by
|
||||
// the driver, so the llm policy runs on web instead of being silently replaced
|
||||
// by the V8 picker.
|
||||
func TestPickSourcesOnWebComposesLLMWithWebExtractors(t *testing.T) {
|
||||
fake := newFakeOpenRouter(t)
|
||||
_, verifierInstance := newLLMSource(t, fake)
|
||||
action, extractor, err := pickSources(Options{
|
||||
Driver: &webMockDriver{Driver: mockdriver.New()},
|
||||
Verifier: verifierInstance,
|
||||
Generator: "llm",
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("pickSources: %v", err)
|
||||
}
|
||||
if _, ok := action.(*llmSource); !ok {
|
||||
t.Errorf("action source = %T, want *llmSource on web with --generator llm", action)
|
||||
}
|
||||
if _, ok := extractor.(webSource); !ok {
|
||||
t.Errorf("extractor source = %T, want webSource so overrides still come from V8", extractor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPickSourcesOnWebSeededKeepsBothOnV8(t *testing.T) {
|
||||
action, extractor, err := pickSources(Options{
|
||||
Driver: &webMockDriver{Driver: mockdriver.New()},
|
||||
Verifier: newSeededVerifier(t),
|
||||
Generator: "seeded",
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("pickSources: %v", err)
|
||||
}
|
||||
if _, ok := action.(webSource); !ok {
|
||||
t.Errorf("action source = %T, want webSource for the seeded web path", action)
|
||||
}
|
||||
if _, ok := extractor.(webSource); !ok {
|
||||
t.Errorf("extractor source = %T, want webSource for the seeded web path", extractor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMSourceDrivesExecutedActions(t *testing.T) {
|
||||
fake := newFakeOpenRouter(t)
|
||||
source, verifierInstance := newLLMSource(t, fake)
|
||||
|
||||
Reference in new issue
Block a user