feat(folio): drive spec.ts under --generator llm; drop spec-llm.ts

This commit is contained in:
pj committed 2026-07-13 07:27:51 +05:30
1 parent 98483926a6
commit 9845b89173
4 files changed
+48 -38

No files matched your search

-29
View File
@@ -1,29 +0,0 @@
// LLM action-backend variant of spec.ts.
//
// The properties and the login `setup` are reused verbatim; only the action
// generator changes. Instead of the seeded fuzzer drawing a random candidate,
// `llm({ model })` hands selection to a vision model: each step it sees the
// screenshot plus the candidate list the system already enumerates and returns
// which candidate to act on. The candidate set, the typed input values, action
// execution, and the trace are all identical to the seeded run.
//
// Requirements:
// - OPENROUTER_API_KEY (OpenRouter) or OPENAI_API_KEY (OpenAI) in the
// environment; OpenRouter wins when both are set. With a plain OpenAI key,
// drop the vendor prefix from the model id ("gpt-5.4-nano").
// - A model that supports image input AND strict json_schema structured
// outputs. A model lacking either fails clearly.
import { llm } from "@sanderling/spec";
export { properties, setup } from "./spec";
// instructions only describe WHAT the app is — its purpose and features. They
// say nothing about HOW to test it: no bug, no technique, no "try to break it"
// (the base prompt already carries the bug-finding goal). The model figures out
// how to test entirely on its own. If we encoded the answer here, a "pass"
// would prove nothing and the feature would be worse than useless.
export const actionsRoot = llm({
model: "gpt-5.4-nano",
instructions:
"Folio is a personal-finance ledger app. After signing in, the home screen lists accounts, each with a balance. You can create accounts, open an account to see its ledger, and add transactions; each transaction has an amount and changes that account's balance and the overall total.",
});
+13
View File
@@ -6,6 +6,7 @@ import {
extract,
from,
integers,
llm,
next,
weighted,
whenRoute,
@@ -187,3 +188,15 @@ export const actionsRoot = weighted(
[5, doubleTaps],
[25, defaultActions],
);
// The LLM generator is orthogonal to actionsRoot: with `--generator llm` a model
// picks from the SAME weighted candidate set above, reading the screenshot and a
// numbered, weight-annotated list; the default `--generator seeded` ignores it.
// instructions describe only WHAT the app is, never HOW to test it — the model
// figures out how to surface bugs on its own. With a plain OpenAI key, drop the
// vendor prefix from the model id.
export const generator = llm({
model: "gpt-5.4-nano",
instructions:
"Folio is a personal-finance ledger app. After signing in, the home screen lists accounts, each with a balance. You can create accounts, open an account to see its ledger, and add transactions; each transaction has an amount and changes that account's balance and the overall total.",
});