diff --git a/examples/folio/README.md b/examples/folio/README.md index 2b03740..7587a6c 100644 --- a/examples/folio/README.md +++ b/examples/folio/README.md @@ -81,19 +81,21 @@ Traces land in `./sanderling/runs//`. ## Run with the LLM action backend `sanderling/spec-llm.ts` reuses the same properties and login setup but swaps the -seeded fuzzer for `llm({ model })`: an OpenRouter vision model chooses which of -the already-enumerated candidates to act on each step, from the screenshot. +seeded fuzzer for `llm({ model })`: a vision model chooses which of the +already-enumerated candidates to act on each step, from the screenshot. ```sh -export OPENROUTER_API_KEY=sk-or-... +export OPENROUTER_API_KEY=sk-or-... # or OPENAI_API_KEY=sk-... for OpenAI direct sanderling test --spec sanderling/spec-llm.ts --bundle-id app.folio --duration 2m ``` -The model must support image input **and** strict `json_schema` structured -outputs (e.g. `openai/gpt-4o`); edit the `model` string in `spec-llm.ts` to pick -another. Each step is one multimodal call, so keep the duration / step budget -modest. The trace records the model's reasoning and `source: "llm"` on each -chosen action, so the replay UI shows why each pick was made. +OpenRouter wins when both keys are set. With a plain OpenAI key, drop the vendor +prefix from the model id in `spec-llm.ts` (`gpt-5.4-nano`, not +`openai/gpt-5.4-nano`). The model must support image input **and** strict +`json_schema` structured outputs. Each step is one multimodal call, so keep the +duration / step budget modest. The trace records the model's reasoning and +`source: "llm"` on each chosen action, so the replay UI shows why each pick was +made. ## Run a sanderling test (iOS) diff --git a/examples/folio/sanderling/spec-llm.ts b/examples/folio/sanderling/spec-llm.ts index e5f33de..f5c4861 100644 --- a/examples/folio/sanderling/spec-llm.ts +++ b/examples/folio/sanderling/spec-llm.ts @@ -2,17 +2,19 @@ // // The properties and the login `setup` are reused verbatim; only the action // generator changes. Instead of the seeded fuzzer drawing a random candidate, -// `llm({ model })` hands selection to an OpenRouter vision model: each step it -// sees the screenshot plus the candidate list the system already enumerates and -// returns which candidate to act on. The candidate set, the typed input values, -// action execution, and the trace are all identical to the seeded run. +// `llm({ model })` hands selection to a vision model: each step it sees the +// screenshot plus the candidate list the system already enumerates and returns +// which candidate to act on. The candidate set, the typed input values, action +// execution, and the trace are all identical to the seeded run. // // Requirements: -// - OPENROUTER_API_KEY in the environment. +// - OPENROUTER_API_KEY (OpenRouter) or OPENAI_API_KEY (OpenAI) in the +// environment; OpenRouter wins when both are set. With a plain OpenAI key, +// drop the vendor prefix from the model id ("gpt-5.4-nano"). // - A model that supports image input AND strict json_schema structured -// outputs (e.g. "openai/gpt-4o"). A model lacking either fails clearly. +// outputs. A model lacking either fails clearly. import { llm } from "@sanderling/spec"; export { properties, setup } from "./spec"; -export const actionsRoot = llm({ model: "openai/gpt-4o" }); +export const actionsRoot = llm({ model: "openai/gpt-5.4-nano" }); diff --git a/pkg/spec/src/actions.ts b/pkg/spec/src/actions.ts index a254606..6a69bcf 100644 --- a/pkg/spec/src/actions.ts +++ b/pkg/spec/src/actions.ts @@ -35,10 +35,11 @@ export function actions(generator: () => Action[]): GeneratorNode { } // llm selects the LLM action backend: instead of the seeded picker drawing a -// random candidate, Go drives an OpenRouter model that chooses which candidate -// to act on from the screenshot + the candidate list. The returned marker is -// inert on the JS picker (pick.ts walks it to null); Go reads its config.model -// off globalThis.actions. API key comes from OPENROUTER_API_KEY. +// random candidate, Go drives an OpenAI-compatible model that chooses which +// candidate to act on from the screenshot + the candidate list. The returned +// marker is inert on the JS picker (pick.ts walks it to null); Go reads its +// config.model off globalThis.actions. API key comes from OPENROUTER_API_KEY +// (OpenRouter) or OPENAI_API_KEY (OpenAI); OpenRouter wins when both are set. export function llm(config: { model: string }): GeneratorNode { return { kind: "llm", config }; }