mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
feat(verifier): read llm config off globalThis.generator
This commit is contained in:
1 parent
aa67d7ddcf
commit
dcf7c4a02a
3 files changed
+13
-11
No files matched your search
@@ -31,7 +31,7 @@ var llmInputCorpus = []string{
|
|||||||
const llmFixtureSpec = `
|
const llmFixtureSpec = `
|
||||||
import { llm, always } from "@sanderling/spec";
|
import { llm, always } from "@sanderling/spec";
|
||||||
globalThis.properties = { ok: always(() => true) };
|
globalThis.properties = { ok: always(() => true) };
|
||||||
globalThis.actions = llm({ model: "test/model" });
|
globalThis.generator = llm({ model: "test/model" });
|
||||||
`
|
`
|
||||||
|
|
||||||
const llmTreeJSON = `{
|
const llmTreeJSON = `{
|
||||||
|
|||||||
@@ -8,8 +8,10 @@ import (
|
|||||||
"github.com/priyanshujain/sanderling/internal/hierarchy"
|
"github.com/priyanshujain/sanderling/internal/hierarchy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// LLMConfig is the spec-declared configuration for the LLM action backend,
|
// LLMConfig is the spec-declared configuration for the LLM action generator,
|
||||||
// read off globalThis.actions when the spec assigned `actions = llm({...})`.
|
// read off globalThis.generator when the spec assigned `generator = llm({...})`.
|
||||||
|
// It is orthogonal to globalThis.actions (the weighted tree the LLM picks from);
|
||||||
|
// only the picker differs.
|
||||||
type LLMConfig struct {
|
type LLMConfig struct {
|
||||||
Model string
|
Model string
|
||||||
// Instructions is optional spec-level guidance appended to the prompt to
|
// Instructions is optional spec-level guidance appended to the prompt to
|
||||||
@@ -17,15 +19,15 @@ type LLMConfig struct {
|
|||||||
Instructions string
|
Instructions string
|
||||||
}
|
}
|
||||||
|
|
||||||
// LLMConfig reports the LLM action backend config when the spec selected it
|
// LLMConfig reports the LLM action-generator config when the spec declared one
|
||||||
// (globalThis.actions.kind === "llm"). The second return is false for every
|
// (globalThis.generator.kind === "llm"). The second return is false for every
|
||||||
// other spec, so the runner falls back to the seeded picker.
|
// other spec, so the runner falls back to the seeded picker.
|
||||||
func (v *Verifier) LLMConfig() (LLMConfig, bool) {
|
func (v *Verifier) LLMConfig() (LLMConfig, bool) {
|
||||||
actions := v.runtime.GlobalObject().Get("actions")
|
generator := v.runtime.GlobalObject().Get("generator")
|
||||||
if actions == nil || goja.IsUndefined(actions) || goja.IsNull(actions) {
|
if generator == nil || goja.IsUndefined(generator) || goja.IsNull(generator) {
|
||||||
return LLMConfig{}, false
|
return LLMConfig{}, false
|
||||||
}
|
}
|
||||||
object := actions.ToObject(v.runtime)
|
object := generator.ToObject(v.runtime)
|
||||||
if object == nil {
|
if object == nil {
|
||||||
return LLMConfig{}, false
|
return LLMConfig{}, false
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -88,7 +88,7 @@ func TestAllCandidatesNilTree(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestLLMConfigDetectsMarker(t *testing.T) {
|
func TestLLMConfigDetectsMarker(t *testing.T) {
|
||||||
v := newLoadedVerifier(t, `globalThis.actions = { kind: "llm", config: { model: "vendor/model" } };`)
|
v := newLoadedVerifier(t, `globalThis.generator = { kind: "llm", config: { model: "vendor/model" } };`)
|
||||||
config, ok := v.LLMConfig()
|
config, ok := v.LLMConfig()
|
||||||
if !ok {
|
if !ok {
|
||||||
t.Fatal("LLMConfig not detected for llm marker")
|
t.Fatal("LLMConfig not detected for llm marker")
|
||||||
@@ -102,7 +102,7 @@ func TestLLMConfigDetectsMarker(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestLLMConfigReadsInstructions(t *testing.T) {
|
func TestLLMConfigReadsInstructions(t *testing.T) {
|
||||||
v := newLoadedVerifier(t, `globalThis.actions = { kind: "llm", config: { model: "m", instructions: "find bugs" } };`)
|
v := newLoadedVerifier(t, `globalThis.generator = { kind: "llm", config: { model: "m", instructions: "find bugs" } };`)
|
||||||
config, ok := v.LLMConfig()
|
config, ok := v.LLMConfig()
|
||||||
if !ok {
|
if !ok {
|
||||||
t.Fatal("LLMConfig not detected")
|
t.Fatal("LLMConfig not detected")
|
||||||
@@ -115,7 +115,7 @@ func TestLLMConfigReadsInstructions(t *testing.T) {
|
|||||||
func TestLLMConfigAbsentForSeededSpec(t *testing.T) {
|
func TestLLMConfigAbsentForSeededSpec(t *testing.T) {
|
||||||
v := newLoadedVerifier(t, `globalThis.actions = { kind: "builtin", verb: "taps" };`)
|
v := newLoadedVerifier(t, `globalThis.actions = { kind: "builtin", verb: "taps" };`)
|
||||||
if _, ok := v.LLMConfig(); ok {
|
if _, ok := v.LLMConfig(); ok {
|
||||||
t.Error("LLMConfig should be false for a non-llm actions root")
|
t.Error("LLMConfig should be false when no generator is declared")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user