diff --git a/.gitignore b/.gitignore index 1857ba2..390b12e 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,7 @@ examples/folio-web/.vite/ # coding agent files .claude/ .claude/* +.playwright-mcp/ # Personal research notes /research/ diff --git a/CLAUDE.md b/CLAUDE.md index b73800f..0299e96 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,6 +15,19 @@ - No slashes in branch names (e.g., use `fix-something` not `fix/something`). +## PR Rules + + - Simple PR title, few-line description. Never write a wall of text — nobody reads it. + - Everything lowercase in PR titles, descriptions, and comments. + +### PR Description Rules + + - Plain text only. No markdown, no headings, no bullets, no bold, no code blocks, no emoji. + - A few lines, that's it. Don't write an essay. + - Super casual, like you're telling a teammate over chat. Lowercase is fine. + - Don't polish it. A few typos and loose grammar are fine and preferred over something that reads like a template. + - Keep the facts right even though the tone is casual. Casual is about the voice, not about being vague or wrong. + ## Git Commit Rules - Commit after every small, atomic change. Each commit should touch 1-3 files max. diff --git a/cmd/sanderling/main.go b/cmd/sanderling/main.go index db63e4b..9ba778e 100644 --- a/cmd/sanderling/main.go +++ b/cmd/sanderling/main.go @@ -30,6 +30,7 @@ type testOptions struct { seed int64 output string clearData bool + generator string } const topUsage = `sanderling is a property-based UI fuzzer for mobile apps. @@ -62,6 +63,7 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) { flagSet.Int64Var(&options.seed, "seed", 0, "RNG seed (0 = random)") flagSet.StringVar(&options.output, "output", "./runs", "output directory for traces") flagSet.BoolVar(&options.clearData, "clear-data", true, "clear app data before launching so each run starts from a fresh install; pass --clear-data=false to resume prior state") + flagSet.StringVar(&options.generator, "generator", "seeded", "action generator: seeded (weighted random) or llm (model picks from the same candidate set; requires generator = llm() in the spec)") if err := flagSet.Parse(args); err != nil { return testOptions{}, err } @@ -76,6 +78,11 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) { default: return testOptions{}, fmt.Errorf("unsupported platform: %q (android, ios, web)", options.platform) } + switch options.generator { + case "seeded", "llm": + default: + return testOptions{}, fmt.Errorf("unsupported generator: %q (seeded, llm)", options.generator) + } return options, nil } diff --git a/cmd/sanderling/main_test.go b/cmd/sanderling/main_test.go index fa9ebc0..7367551 100644 --- a/cmd/sanderling/main_test.go +++ b/cmd/sanderling/main_test.go @@ -93,6 +93,41 @@ func TestParseTestArgs_AVDIsOptional(t *testing.T) { } } +func TestParseTestArgs_GeneratorDefaultsToSeeded(t *testing.T) { + options, err := parseTestArgs([]string{"--spec", "s.ts", "--bundle-id", "com.example"}, io.Discard) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if options.generator != "seeded" { + t.Fatalf("generator default: got %q, want seeded", options.generator) + } +} + +func TestParseTestArgs_AcceptsLLMGenerator(t *testing.T) { + options, err := parseTestArgs([]string{ + "--spec", "s.ts", + "--bundle-id", "com.example", + "--generator", "llm", + }, io.Discard) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if options.generator != "llm" { + t.Fatalf("generator: got %q, want llm", options.generator) + } +} + +func TestParseTestArgs_RejectsUnknownGenerator(t *testing.T) { + _, err := parseTestArgs([]string{ + "--spec", "s.ts", + "--bundle-id", "com.example", + "--generator", "oracle", + }, io.Discard) + if err == nil || !strings.Contains(err.Error(), "unsupported generator") { + t.Fatalf("expected unsupported-generator error, got %v", err) + } +} + func TestParseTestArgs_RejectsUnknownPlatform(t *testing.T) { _, err := parseTestArgs([]string{ "--spec", "s.ts", diff --git a/cmd/sanderling/test_run.go b/cmd/sanderling/test_run.go index b1c4764..477392f 100644 --- a/cmd/sanderling/test_run.go +++ b/cmd/sanderling/test_run.go @@ -21,5 +21,6 @@ func runTestPipeline(ctx context.Context, options testOptions, stdout io.Writer) Seed: options.seed, Output: options.output, ClearData: options.clearData, + Generator: options.generator, }, stdout) } diff --git a/docs/manual/cli.md b/docs/manual/cli.md index ce48e73..060675b 100644 --- a/docs/manual/cli.md +++ b/docs/manual/cli.md @@ -23,6 +23,7 @@ Run a spec against an app for a fixed duration. | `--ios-app-path` | optional (ios) | Path to the `.app` bundle for clear-state reinstall (simulator via `simctl`, device via `devicectl`). | | `--duration` | `5m` | Total test duration (`30s`, `5m`, `2h`, `1d`). | | `--seed` | `0` | PRNG seed. `0` uses a random seed and records it in `meta.json`. | +| `--generator` | `seeded` | Who picks each action: `seeded` (the run's PRNG) or `llm` (a vision model). See [the LLM generator](../spec-language/#llm-generator). | | `--output` | `./runs` | Output directory for traces. | | `--clear-data` | `true` | Clear app data before launching so the run starts from a fresh install. Pass `--clear-data=false` to resume prior state. | diff --git a/docs/manual/spec-language.md b/docs/manual/spec-language.md index aeff4cc..b8a5908 100644 --- a/docs/manual/spec-language.md +++ b/docs/manual/spec-language.md @@ -8,14 +8,15 @@ Lookup reference for everything importable from `@sanderling/spec`. For a worked ## Module structure -A spec is a TypeScript module evaluated by the Go runner each step. It exports `properties` and `actionsRoot`, plus an optional `setup`: +A spec is a TypeScript module evaluated by the Go runner each step. It exports `properties` and `actionsRoot`, plus an optional `setup` and `generator`: ```ts import { ... } from "@sanderling/spec"; export const properties = { ... }; export const actionsRoot = weighted(...); -export const setup = login; // optional +export const setup = login; // optional +export const generator = llm(...); // optional, see below ``` `setup` is an `ActionGenerator` the runner consults before `actionsRoot` each step. While it returns actions, they run; when it returns an empty list, the runner falls through to `actionsRoot`. Use it for preconditions like login and onboarding. If the app later regresses across the precondition (a logout mid-run), `setup` re-engages on its own. @@ -266,6 +267,23 @@ InputText({ into: nameField, text: names.generate() }) InputText({ into: amountField, text: String(amounts.generate()) }) ``` +## LLM generator + +By default the run's PRNG picks each action. `--generator llm` swaps out the picker for a vision model and nothing else: same spec, same `actionsRoot`, same weights, same actions. Add the export and pick a model. + +```ts +export const generator = llm({ + model: "gpt-5.4-nano", + instructions: "Folio is a personal-finance ledger app. The home screen lists accounts with balances; you can open an account and add transactions.", +}); +``` + +Set `OPENROUTER_API_KEY` or `OPENAI_API_KEY` (OpenRouter wins if both are set). With a plain OpenAI key, drop the vendor prefix from the model id. The model needs image input and strict `json_schema` structured output. + +Each step it gets a screenshot plus a numbered list of the concrete actions your tree yields right now, each tagged with its weight, and picks one number. `instructions` are appended to the prompt: say what the app is, not how to test it — the model works that part out. Everything else is unchanged. Setup actions still run first, typing still falls back to the edge-case corpus when the model supplies no text, and the trace records the reasoning, the chosen number, and `source: "llm"` so the replay UI can show why each pick happened. + +It is one model call per step, so keep `--duration` modest. + ## Defaults ```ts diff --git a/examples/folio/README.md b/examples/folio/README.md index 503c8a0..8e539ce 100644 --- a/examples/folio/README.md +++ b/examples/folio/README.md @@ -78,6 +78,27 @@ DURATION=5m Traces land in `./sanderling/runs//`. +## Run with the LLM action generator + +The same `sanderling/spec.ts` runs under either generator: `--generator seeded` +(the default weighted fuzzer) or `--generator llm`, where a vision model picks +from the SAME weighted candidate set — reading the screenshot plus a numbered, +weight-annotated list of concrete actions — and returns one number. The spec's +`generator = llm({ model, instructions })` export configures it. + +```sh +export OPENROUTER_API_KEY=sk-or-... # or OPENAI_API_KEY=sk-... for OpenAI direct +just test-llm # or: sanderling test --generator llm --spec sanderling/spec.ts --bundle-id app.folio +``` + +OpenRouter wins when both keys are set. With a plain OpenAI key, drop the vendor +prefix from the model id in `spec.ts` (`gpt-5.4-nano`, not +`openai/gpt-5.4-nano`). The model must support image input **and** strict +`json_schema` structured outputs. Each step is one multimodal call, so keep the +duration / step budget modest. The trace records the model's reasoning, the +chosen number, and `source: "llm"` on each action, so the replay UI shows why +each pick was made. + ## Run a sanderling test (iOS) ```sh diff --git a/examples/folio/app/shared/src/commonMain/kotlin/app/folio/feature/ledger/AddTransactionScreen.kt b/examples/folio/app/shared/src/commonMain/kotlin/app/folio/feature/ledger/AddTransactionScreen.kt index e319cec..10e0cc4 100644 --- a/examples/folio/app/shared/src/commonMain/kotlin/app/folio/feature/ledger/AddTransactionScreen.kt +++ b/examples/folio/app/shared/src/commonMain/kotlin/app/folio/feature/ledger/AddTransactionScreen.kt @@ -4,6 +4,7 @@ import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding +import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue @@ -15,6 +16,7 @@ import androidx.compose.ui.unit.dp import androidx.lifecycle.viewmodel.compose.viewModel import app.folio.core.data.TxnType import app.folio.di.LocalAppGraph +import app.folio.util.formatCents import app.folio.ui.BackHandler import app.folio.ui.component.AppButton import app.folio.ui.component.BackButton @@ -26,6 +28,7 @@ import app.folio.ui.component.Header import app.folio.ui.component.Screen import app.folio.ui.component.Segmented import app.folio.ui.component.TextInput +import app.folio.ui.theme.LocalTokens import app.folio.ui.theme.Type @Composable @@ -65,6 +68,14 @@ fun AddTransactionScreen(state: AddTransactionUiState, onEvent: (AddTransactionE Header( title = "Add transaction", subtitle = account.name, + detail = { + Text( + "Balance: ${formatCents(state.balanceCents)}", + style = Type.caption, + color = LocalTokens.current.textMuted, + modifier = Modifier.testTag("TxnCurrentBalance"), + ) + }, left = { BackButton { onEvent(AddTransactionEvent.Back) } }, ) }, diff --git a/examples/folio/app/shared/src/commonMain/kotlin/app/folio/feature/ledger/AddTransactionViewModel.kt b/examples/folio/app/shared/src/commonMain/kotlin/app/folio/feature/ledger/AddTransactionViewModel.kt index 60bc0aa..fbb160d 100644 --- a/examples/folio/app/shared/src/commonMain/kotlin/app/folio/feature/ledger/AddTransactionViewModel.kt +++ b/examples/folio/app/shared/src/commonMain/kotlin/app/folio/feature/ledger/AddTransactionViewModel.kt @@ -7,6 +7,7 @@ import app.folio.core.data.Repository import app.folio.core.data.TxnType import app.folio.navigation.Navigator import app.folio.navigation.Route +import app.folio.util.balanceOf import app.folio.util.parseCents import dev.zacsweers.metro.Assisted import dev.zacsweers.metro.AssistedFactory @@ -24,6 +25,7 @@ private val AMOUNT_REGEX = Regex("""^\d*(\.\d{0,2})?$""") data class AddTransactionUiState( val account: Account? = null, + val balanceCents: Long = 0, val type: TxnType = TxnType.credit, val amount: String = "", val note: String = "", @@ -50,8 +52,13 @@ class AddTransactionViewModel( val state: StateFlow = combine( form, repository.accounts.map { accounts -> accounts.firstOrNull { it.id == accountId } }, - ) { f, account -> f.copy(account = account) } - .stateIn(viewModelScope, SharingStarted.Eagerly, AddTransactionUiState()) + repository.transactions, + ) { f, account, transactions -> + f.copy( + account = account, + balanceCents = balanceOf(transactions.filter { it.accountId == accountId }), + ) + }.stateIn(viewModelScope, SharingStarted.Eagerly, AddTransactionUiState()) fun onEvent(event: AddTransactionEvent) { when (event) { diff --git a/examples/folio/app/ui-components/src/commonMain/kotlin/app/folio/ui/component/Header.kt b/examples/folio/app/ui-components/src/commonMain/kotlin/app/folio/ui/component/Header.kt index 58af281..1105be8 100644 --- a/examples/folio/app/ui-components/src/commonMain/kotlin/app/folio/ui/component/Header.kt +++ b/examples/folio/app/ui-components/src/commonMain/kotlin/app/folio/ui/component/Header.kt @@ -19,6 +19,7 @@ import app.folio.ui.theme.Type fun Header( title: String, subtitle: String? = null, + detail: @Composable (() -> Unit)? = null, left: @Composable (() -> Unit)? = null, right: @Composable (() -> Unit)? = null, ) { @@ -42,6 +43,7 @@ fun Header( if (subtitle != null) { Text(subtitle, style = Type.caption, color = t.textMuted) } + if (detail != null) detail() } if (right != null) right() } diff --git a/examples/folio/justfile b/examples/folio/justfile index df14b86..4328766 100644 --- a/examples/folio/justfile +++ b/examples/folio/justfile @@ -176,6 +176,30 @@ test: install --seed "{{seed}}" \ --output "{{output}}" +# Run 'sanderling test' with the LLM action generator instead of the seeded +# fuzzer. Needs OPENROUTER_API_KEY (or OPENAI_API_KEY) in the environment; the +# model is configured by generator = llm({...}) in spec.ts. +test-llm: install + #!/usr/bin/env bash + set -euo pipefail + avd_flag=() + if [[ -n "{{avd}}" ]]; then + avd_flag=(--avd "{{avd}}") + fi + if [[ -n "{{android_device}}" ]]; then + avd_flag+=(--device "{{android_device}}") + fi + apk="{{justfile_directory()}}/app/androidApp/build/outputs/apk/debug/androidApp-debug.apk" + "{{sanderling}}" test \ + --spec "{{justfile_directory()}}/sanderling/spec.ts" \ + --bundle-id app.folio \ + --generator llm \ + "${avd_flag[@]}" \ + --android-app-path "$apk" \ + --duration "{{duration}}" \ + --seed "{{seed}}" \ + --output "{{output}}" + # Serve the wasmJs web app from a webpack dev server with COOP/COEP headers. web: #!/usr/bin/env bash diff --git a/examples/folio/sanderling/spec.ts b/examples/folio/sanderling/spec.ts index 649238a..4316613 100644 --- a/examples/folio/sanderling/spec.ts +++ b/examples/folio/sanderling/spec.ts @@ -6,6 +6,7 @@ import { extract, from, integers, + llm, next, weighted, whenRoute, @@ -187,3 +188,15 @@ export const actionsRoot = weighted( [5, doubleTaps], [25, defaultActions], ); + +// The LLM generator is orthogonal to actionsRoot: with `--generator llm` a model +// picks from the SAME weighted candidate set above, reading the screenshot and a +// numbered, weight-annotated list; the default `--generator seeded` ignores it. +// instructions describe only WHAT the app is, never HOW to test it — the model +// figures out how to surface bugs on its own. With a plain OpenAI key, drop the +// vendor prefix from the model id. +export const generator = llm({ + model: "gpt-5.4-nano", + instructions: + "Folio is a personal-finance ledger app. After signing in, the home screen lists accounts, each with a balance. You can create accounts, open an account to see its ledger, and add transactions; each transaction has an amount and changes that account's balance and the overall total.", +}); diff --git a/internal/bundler/bundler.go b/internal/bundler/bundler.go index 043c125..6b05a03 100644 --- a/internal/bundler/bundler.go +++ b/internal/bundler/bundler.go @@ -110,6 +110,7 @@ func Bundle(options Options) (Result, error) { func registrationEntry(specAbs string) string { return fmt.Sprintf(`import * as __spec from %q; if (__spec.actionsRoot !== undefined) globalThis.actions = __spec.actionsRoot; +if (__spec.generator !== undefined) globalThis.generator = __spec.generator; if (__spec.properties !== undefined) globalThis.properties = __spec.properties; if (__spec.setup !== undefined) globalThis.setup = __spec.setup; `, specAbs) diff --git a/internal/bundler/bundler_test.go b/internal/bundler/bundler_test.go index 56221f9..07b69e4 100644 --- a/internal/bundler/bundler_test.go +++ b/internal/bundler/bundler_test.go @@ -156,6 +156,7 @@ func TestBundle_RegistersNamedExportsOnGlobalThis(t *testing.T) { spec := ` export const properties = "PROPS_MARKER"; export const actionsRoot = "ACTIONS_MARKER"; + export const generator = "GENERATOR_MARKER"; export const setup = "SETUP_MARKER"; ` if err := os.WriteFile(specPath, []byte(spec), 0o600); err != nil { @@ -166,7 +167,7 @@ func TestBundle_RegistersNamedExportsOnGlobalThis(t *testing.T) { t.Fatal(err) } body := string(result.JavaScript) - for _, marker := range []string{"PROPS_MARKER", "ACTIONS_MARKER", "SETUP_MARKER"} { + for _, marker := range []string{"PROPS_MARKER", "ACTIONS_MARKER", "GENERATOR_MARKER", "SETUP_MARKER"} { if !strings.Contains(body, marker) { t.Errorf("named export %q not registered in bundle:\n%s", marker, body) } @@ -174,6 +175,9 @@ func TestBundle_RegistersNamedExportsOnGlobalThis(t *testing.T) { if !strings.Contains(body, "globalThis.actions") { t.Errorf("trailer should assign globalThis.actions:\n%s", body) } + if !strings.Contains(body, "globalThis.generator") { + t.Errorf("trailer should assign globalThis.generator:\n%s", body) + } } func TestBundle_RegistersWithoutSetupExport(t *testing.T) { diff --git a/internal/hierarchy/hierarchy.go b/internal/hierarchy/hierarchy.go index 9972cb4..c54a052 100644 --- a/internal/hierarchy/hierarchy.go +++ b/internal/hierarchy/hierarchy.go @@ -272,6 +272,25 @@ func elementFromNode(node *treeNodeJSON) *Element { return element } +// Transitional reports more than one resource id ending in "Screen": the marker +// of a Compose NavHost mid cross-fade, where the source and destination route +// composables are both alive in a collapsed, mid-animation layout. +func (t *Tree) Transitional() bool { + if t == nil { + return false + } + screens := 0 + for _, element := range t.Elements { + if strings.HasSuffix(element.ResourceID, "Screen") { + screens++ + if screens > 1 { + return true + } + } + } + return false +} + // Find returns the first element matching the selector, or nil. func (t *Tree) Find(selector string) *Element { node := t.FindNode(selector) diff --git a/internal/hierarchy/hierarchy_test.go b/internal/hierarchy/hierarchy_test.go index 97b3286..91abd4c 100644 --- a/internal/hierarchy/hierarchy_test.go +++ b/internal/hierarchy/hierarchy_test.go @@ -1006,3 +1006,32 @@ func TestParseBoundsRejectsBadInput(t *testing.T) { } } } + +// Bug class: a NavHost cross-fade carries two route-level *Screen ids at once; +// both the runner's re-fetch guard and the LLM's candidate enumeration depend on +// spotting it, and neither must flag a settled single-screen tree. +func TestTreeTransitional(t *testing.T) { + multi, err := Parse(`{"attributes":{"resource-id":"root"},"children":[ + {"attributes":{"resource-id":"AddAccountScreen"},"children":[]}, + {"attributes":{"resource-id":"HomeScreen"},"children":[]} + ]}`) + if err != nil { + t.Fatal(err) + } + if !multi.Transitional() { + t.Error("expected multi-screen tree to be flagged as transitional") + } + + single, err := Parse(`{"attributes":{"resource-id":"HomeScreen"},"children":[]}`) + if err != nil { + t.Fatal(err) + } + if single.Transitional() { + t.Error("single-screen tree must not be flagged as transitional") + } + + var nilTree *Tree + if nilTree.Transitional() { + t.Error("nil tree must not be flagged as transitional") + } +} diff --git a/internal/llmclient/client.go b/internal/llmclient/client.go new file mode 100644 index 0000000..7b21b84 --- /dev/null +++ b/internal/llmclient/client.go @@ -0,0 +1,169 @@ +// Package llmclient is a minimal client for the OpenAI-compatible +// chat-completions API, covering only what the LLM action backend needs: a +// single multimodal (text + one image) request per step with strict +// json_schema structured output. No streaming, tools, or other extras. +// +// The provider comes from the environment: OPENROUTER_API_KEY routes to +// OpenRouter, OPENAI_API_KEY to OpenAI; OpenRouter wins when both are set. +// Both speak the same wire format, so there is no provider-specific code. +package llmclient + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "time" +) + +const ( + openRouterBaseURL = "https://openrouter.ai/api/v1" + openAIBaseURL = "https://api.openai.com/v1" +) + +// requestTimeout bounds a single chat-completion round-trip. Vision + strict +// structured output is slower than a plain text call, so this is generous; the +// runner also passes a context the caller can cancel. +const requestTimeout = 60 * time.Second + +// Client talks to an OpenAI-compatible chat-completions endpoint. +type Client struct { + httpClient *http.Client + apiKey string + baseURL string +} + +// New builds a Client from the environment. OPENROUTER_API_KEY selects +// OpenRouter, OPENAI_API_KEY selects OpenAI; OpenRouter wins when both are +// set. OPENROUTER_BASE_URL / OPENAI_BASE_URL override the chosen provider's +// endpoint (tests, local OpenAI-compatible servers). +func New() (*Client, error) { + apiKey := os.Getenv("OPENROUTER_API_KEY") + baseURL := openRouterBaseURL + override := os.Getenv("OPENROUTER_BASE_URL") + if apiKey == "" { + apiKey = os.Getenv("OPENAI_API_KEY") + baseURL = openAIBaseURL + override = os.Getenv("OPENAI_BASE_URL") + } + if apiKey == "" { + return nil, errors.New("llmclient: neither OPENROUTER_API_KEY nor OPENAI_API_KEY is set") + } + if override != "" { + baseURL = override + } + return &Client{ + httpClient: &http.Client{Timeout: requestTimeout}, + apiKey: apiKey, + baseURL: baseURL, + }, nil +} + +// Request is a chat-completions request body. Only the fields the action +// backend sets are modeled. +type Request struct { + Model string `json:"model"` + Messages []Message `json:"messages"` + ResponseFormat *ResponseFormat `json:"response_format,omitempty"` +} + +// Message is one chat message. Content is always the array form (a list of +// parts), which OpenRouter accepts for every role. +type Message struct { + Role string `json:"role"` + Content []ContentPart `json:"content"` +} + +// ContentPart is one piece of a message: either a text run or an image given +// as a data URL. +type ContentPart struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL *ImageURL `json:"image_url,omitempty"` +} + +// ImageURL carries an image as a (typically data:) URL. +type ImageURL struct { + URL string `json:"url"` +} + +// TextPart builds a text content part. +func TextPart(text string) ContentPart { + return ContentPart{Type: "text", Text: text} +} + +// ImagePart builds an image content part from a data URL. +func ImagePart(dataURL string) ContentPart { + return ContentPart{Type: "image_url", ImageURL: &ImageURL{URL: dataURL}} +} + +// ResponseFormat pins the model to a strict JSON schema. +type ResponseFormat struct { + Type string `json:"type"` + JSONSchema JSONSchema `json:"json_schema"` +} + +// JSONSchema is the strict structured-output schema. Schema is raw JSON so the +// caller controls property ORDER: OpenAI emits fields in schema order, and a +// reasoning-first schema must not be re-sorted alphabetically (as a Go map +// would be). +type JSONSchema struct { + Name string `json:"name"` + Strict bool `json:"strict"` + Schema json.RawMessage `json:"schema"` +} + +// Response is the slice of a chat-completions response we read. +type Response struct { + Choices []Choice `json:"choices"` +} + +// Choice is one completion choice. +type Choice struct { + Message ResponseMessage `json:"message"` +} + +// ResponseMessage carries the assistant's content. With json_schema output the +// content is a JSON string matching the schema. +type ResponseMessage struct { + Content string `json:"content"` +} + +// ChatCompletion POSTs req to /chat/completions and decodes the response. A +// non-2xx status is returned as an error carrying the response body. +func (c *Client) ChatCompletion(ctx context.Context, req Request) (*Response, error) { + body, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("llmclient: marshal request: %w", err) + } + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/chat/completions", bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("llmclient: build request: %w", err) + } + httpReq.Header.Set("Authorization", "Bearer "+c.apiKey) + httpReq.Header.Set("Content-Type", "application/json") + + resp, err := c.httpClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("llmclient: request failed: %w", err) + } + defer resp.Body.Close() + + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("llmclient: read response: %w", err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("llmclient: status %d: %s", resp.StatusCode, string(responseBody)) + } + + var out Response + if err := json.Unmarshal(responseBody, &out); err != nil { + return nil, fmt.Errorf("llmclient: decode response: %w", err) + } + return &out, nil +} diff --git a/internal/llmclient/client_test.go b/internal/llmclient/client_test.go new file mode 100644 index 0000000..b4bac25 --- /dev/null +++ b/internal/llmclient/client_test.go @@ -0,0 +1,177 @@ +package llmclient + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestChatCompletionRequestShapeAndParse(t *testing.T) { + var captured map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/chat/completions" { + t.Errorf("path = %q, want /chat/completions", r.URL.Path) + } + if got := r.Header.Get("Authorization"); got != "Bearer test-key" { + t.Errorf("Authorization = %q, want Bearer test-key", got) + } + body, _ := io.ReadAll(r.Body) + if err := json.Unmarshal(body, &captured); err != nil { + t.Fatalf("unmarshal request: %v", err) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"reasoning\":\"tap login\",\"ranked\":[2,0]}"}}]}`)) + })) + defer server.Close() + + t.Setenv("OPENROUTER_API_KEY", "test-key") + t.Setenv("OPENROUTER_BASE_URL", server.URL) + client, err := New() + if err != nil { + t.Fatalf("New: %v", err) + } + + resp, err := client.ChatCompletion(context.Background(), Request{ + Model: "vendor/model", + Messages: []Message{ + {Role: "system", Content: []ContentPart{TextPart("system")}}, + {Role: "user", Content: []ContentPart{ + TextPart("candidates"), + ImagePart("data:image/png;base64,AAAA"), + }}, + }, + ResponseFormat: &ResponseFormat{ + Type: "json_schema", + JSONSchema: JSONSchema{ + Name: "ranked_actions", + Strict: true, + Schema: json.RawMessage(`{"type":"object"}`), + }, + }, + }) + if err != nil { + t.Fatalf("ChatCompletion: %v", err) + } + + // Request carried the model. + if captured["model"] != "vendor/model" { + t.Errorf("model = %v, want vendor/model", captured["model"]) + } + // Request carried an image_url content part. + messages := captured["messages"].([]any) + user := messages[1].(map[string]any) + parts := user["content"].([]any) + foundImage := false + for _, part := range parts { + if part.(map[string]any)["type"] == "image_url" { + foundImage = true + image := part.(map[string]any)["image_url"].(map[string]any) + if !strings.HasPrefix(image["url"].(string), "data:image/png;base64,") { + t.Errorf("image url = %v, want data URL", image["url"]) + } + } + } + if !foundImage { + t.Error("request carried no image_url content part") + } + // Request carried the strict json_schema response_format. + rf := captured["response_format"].(map[string]any) + if rf["type"] != "json_schema" { + t.Errorf("response_format.type = %v, want json_schema", rf["type"]) + } + if schema := rf["json_schema"].(map[string]any); schema["strict"] != true { + t.Errorf("json_schema.strict = %v, want true", schema["strict"]) + } + + // Response parsed into the ranked-index content. + if len(resp.Choices) != 1 { + t.Fatalf("choices = %d, want 1", len(resp.Choices)) + } + var content struct { + Reasoning string `json:"reasoning"` + Ranked []int `json:"ranked"` + } + if err := json.Unmarshal([]byte(resp.Choices[0].Message.Content), &content); err != nil { + t.Fatalf("unmarshal content: %v", err) + } + if content.Reasoning != "tap login" { + t.Errorf("reasoning = %q, want tap login", content.Reasoning) + } + if len(content.Ranked) != 2 || content.Ranked[0] != 2 || content.Ranked[1] != 0 { + t.Errorf("ranked = %v, want [2 0]", content.Ranked) + } +} + +func TestNewRequiresAPIKey(t *testing.T) { + t.Setenv("OPENROUTER_API_KEY", "") + t.Setenv("OPENAI_API_KEY", "") + if _, err := New(); err == nil { + t.Fatal("expected error when neither API key is set") + } +} + +func TestNewFallsBackToOpenAIKey(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer openai-key" { + t.Errorf("Authorization = %q, want Bearer openai-key", got) + } + _, _ = w.Write([]byte(`{"choices":[]}`)) + })) + defer server.Close() + + t.Setenv("OPENROUTER_API_KEY", "") + t.Setenv("OPENAI_API_KEY", "openai-key") + t.Setenv("OPENAI_BASE_URL", server.URL) + client, err := New() + if err != nil { + t.Fatalf("New: %v", err) + } + if _, err := client.ChatCompletion(context.Background(), Request{Model: "m"}); err != nil { + t.Fatalf("ChatCompletion: %v", err) + } +} + +func TestNewPrefersOpenRouterOverOpenAI(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer router-key" { + t.Errorf("Authorization = %q, want Bearer router-key (OpenRouter must win)", got) + } + _, _ = w.Write([]byte(`{"choices":[]}`)) + })) + defer server.Close() + + t.Setenv("OPENROUTER_API_KEY", "router-key") + t.Setenv("OPENROUTER_BASE_URL", server.URL) + t.Setenv("OPENAI_API_KEY", "openai-key") + t.Setenv("OPENAI_BASE_URL", "http://127.0.0.1:1") // unreachable; must not be used + client, err := New() + if err != nil { + t.Fatalf("New: %v", err) + } + if _, err := client.ChatCompletion(context.Background(), Request{Model: "m"}); err != nil { + t.Fatalf("ChatCompletion: %v", err) + } +} + +func TestChatCompletionSurfacesHTTPError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"error":"rate limited"}`)) + })) + defer server.Close() + + t.Setenv("OPENROUTER_API_KEY", "test-key") + t.Setenv("OPENROUTER_BASE_URL", server.URL) + client, err := New() + if err != nil { + t.Fatalf("New: %v", err) + } + _, err = client.ChatCompletion(context.Background(), Request{Model: "m"}) + if err == nil || !strings.Contains(err.Error(), "429") { + t.Fatalf("expected 429 error, got %v", err) + } +} diff --git a/internal/runner/llm_source.go b/internal/runner/llm_source.go new file mode 100644 index 0000000..e7f1afb --- /dev/null +++ b/internal/runner/llm_source.go @@ -0,0 +1,450 @@ +package runner + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "image" + "image/color" + "image/png" + "log/slog" + "regexp" + "strings" + + "github.com/priyanshujain/sanderling/internal/llmclient" + "github.com/priyanshujain/sanderling/internal/trace" + "github.com/priyanshujain/sanderling/internal/verifier" +) + +const ( + // llmMaxImageEdge downscales the screenshot's long edge to bound the payload + // while keeping the UI legible. + llmMaxImageEdge = 1024 + // llmHistorySize is how many recent actions (and the screen each led to) the + // prompt carries as context. + llmHistorySize = 5 +) + +// llmSystemPrompt frames the selection task: a short, generic bug-hunting +// instruction. Each candidate is already a concrete, correctly-labeled action +// with a weight hinting the spec's testing priority; the model reads the +// screenshot, picks ONE number, and echoes that action so a mismatch can be +// caught. The app-specific description (spec instructions) is appended. +const llmSystemPrompt = "You are exercising a UI to find bugs. Each turn you get a screenshot and a numbered list of concrete actions, each with a weight hinting how much the test author wants it exercised (higher = more). " + + "Pick the ONE action most likely to make progress or expose a defect. Bugs often hide in repeated or rapid actions, so once a screen works, deliberately stress it — for example submitting the same form twice in a row to check it is not applied more than once — rather than only advancing. " + + "Respond with your reasoning, the chosen number, and chosen_action copied verbatim from that line. For a typing action, also provide the text to enter." + +// llmSource selects each step's action with an OpenAI-compatible vision model +// instead of the seeded random pick. It replaces ONLY the pick: the candidate list, the input +// values, and action execution are all reused unchanged. The spec's JS setup +// still runs first each tick (setup precedence), and the LLM drives once setup +// yields nothing. +type llmSource struct { + verifier *verifier.Verifier + client *llmclient.Client + model string + // instructions is optional spec-level guidance appended to the system prompt + // to steer the model's bug-hunting (empty when unset). + instructions string + logger *slog.Logger + history *actionHistory + + // lastSource/lastReasoning describe the most recent NextAction so the runner + // can stamp the trace. lastSource is "llm" only when the LLM (not setup) + // chose the action; lastReasoning is the model's rationale. lastChoice is the + // 1-based number it picked and lastChosenAction the description it echoed, so + // the trace shows what the model believed it was doing. + lastSource string + lastReasoning string + lastChoice int + lastChosenAction string +} + +// llmSelection is the outcome of one LLM selection call. +type llmSelection struct { + action verifier.Action + reasoning string + choice int + chosenAction string +} + +// NextAction returns the step's action. Setup precedence is preserved by +// running the JS path first (the llm marker is inert there, so a null result +// means setup yielded nothing); the LLM selection then takes over. +func (s *llmSource) NextAction(ctx context.Context) (verifier.Action, error) { + s.lastSource = "" + s.lastReasoning = "" + s.lastChoice = 0 + s.lastChosenAction = "" + s.history.completeLast(s.verifier.CurrentScreen()) + + // Setup precedence only: the LLM replaces the seeded action root, so we run + // setup (e.g. login) first but never the weighted picker. + action, err := s.verifier.SetupAction() + if err == nil { + s.history.add(describeAction(action)) + return action, nil + } + if !errors.Is(err, verifier.ErrNoAction) { + return verifier.Action{}, err + } + + selection, ok := s.selectViaLLM(ctx) + if !ok { + // Any failure (HTTP error, unusable output, invalid choice, echo + // mismatch) skips the step; the next step re-observes and tries again. + return verifier.Action{}, verifier.ErrNoAction + } + s.lastSource = "llm" + s.lastReasoning = selection.reasoning + s.lastChoice = selection.choice + s.lastChosenAction = selection.chosenAction + s.history.add(describeAction(selection.action)) + return selection.action, nil +} + +// selectViaLLM runs one multimodal call and maps the chosen number to an action. +// It returns ok=false on any error/empty/invalid output, logging the cause; the +// caller turns that into a skipped step. +func (s *llmSource) selectViaLLM(ctx context.Context) (llmSelection, bool) { + candidates := s.verifier.Candidates() + if len(candidates) == 0 { + return llmSelection{}, false + } + + response, err := s.client.ChatCompletion(ctx, s.buildRequest(candidates)) + if err != nil { + s.logger.Warn("llm action selection failed", "err", err) + return llmSelection{}, false + } + if len(response.Choices) == 0 { + s.logger.Warn("llm returned no choices") + return llmSelection{}, false + } + + output, err := parseChoice(response.Choices[0].Message.Content) + if err != nil { + s.logger.Warn("llm output unusable", "err", err) + return llmSelection{}, false + } + // choice is 1-based into the numbered list. + if output.Choice < 1 || output.Choice > len(candidates) { + s.logger.Warn("llm choice out of range", "choice", output.Choice, "candidates", len(candidates)) + return llmSelection{}, false + } + candidate := candidates[output.Choice-1] + // Strict skip: the echoed action must match the numbered entry, so a model + // that reasoned about one target but named a number for another cannot act. + // Models copy the whole rendered line including its trailing "(w34)" weight + // annotation, so strip that before comparing to the (weight-free) description. + if stripWeightSuffix(output.ChosenAction) != candidate.Description { + s.logger.Warn("llm chosen_action mismatch; skipping", + "choice", output.Choice, "echoed", output.ChosenAction, "candidate", candidate.Description) + return llmSelection{}, false + } + action, err := s.actionForCandidate(candidate, output.Text) + if err != nil { + s.logger.Warn("building action from candidate failed", "choice", output.Choice, "err", err) + return llmSelection{}, false + } + return llmSelection{ + action: action, + reasoning: output.Reasoning, + choice: output.Choice, + chosenAction: candidate.Description, + }, true +} + +// actionForCandidate turns a chosen candidate into the executable action. The +// candidate already carries a ready action; only builtin typing needs the +// model's value spliced in (authored InputText keeps its sampled value, and any +// other kind runs verbatim). +func (s *llmSource) actionForCandidate(candidate verifier.ActionCandidate, text string) (verifier.Action, error) { + action := candidate.Action + if candidate.Kind == verifier.ActionKindInputText && candidate.LLMText { + if strings.TrimSpace(text) == "" { + // The model omitted a value; fall back to the shared corpus sampler + // so typing still exercises an edge-case string. + sampled, err := s.verifier.SampleInput() + if err != nil { + return verifier.Action{}, err + } + text = sampled + } + action.Text = text + } + return action, nil +} + +// buildRequest assembles the one-shot multimodal request: a system frame, the +// numbered candidate list plus recent-action memory, and the downscaled +// screenshot. The strict json_schema response format pins the ranked output. +func (s *llmSource) buildRequest(candidates []verifier.ActionCandidate) llmclient.Request { + userParts := []llmclient.ContentPart{llmclient.TextPart(s.userPrompt(candidates))} + if screenshot := s.verifier.Screenshot(); len(screenshot) > 0 { + if dataURL, ok := screenshotDataURL(screenshot, llmMaxImageEdge); ok { + userParts = append(userParts, llmclient.ImagePart(dataURL)) + } + } + return llmclient.Request{ + Model: s.model, + Messages: []llmclient.Message{ + {Role: "system", Content: []llmclient.ContentPart{llmclient.TextPart(s.systemPrompt())}}, + {Role: "user", Content: userParts}, + }, + ResponseFormat: choiceResponseFormat(len(candidates)), + } +} + +// systemPrompt is the base framing plus any spec-level instructions, appended as +// extra guidance so a spec can steer the model's bug-hunting without losing the +// candidate-kind semantics the base prompt establishes. +func (s *llmSource) systemPrompt() string { + if strings.TrimSpace(s.instructions) == "" { + return llmSystemPrompt + } + return llmSystemPrompt + "\n\n" + s.instructions +} + +// userPrompt renders the numbered candidate list (with weights) and the +// recent-action memory. +func (s *llmSource) userPrompt(candidates []verifier.ActionCandidate) string { + var builder strings.Builder + builder.WriteString("Actions available on the current screen:\n") + for _, candidate := range candidates { + fmt.Fprintf(&builder, "%d. %s", candidate.Index, candidate.Description) + if candidate.Weighted { + fmt.Fprintf(&builder, " (w%d)", candidate.Weight) + } + builder.WriteByte('\n') + } + if recent := s.history.recent(); len(recent) > 0 { + builder.WriteString("\nYour recent actions (oldest first) and the screen each led to:\n") + for _, entry := range recent { + screen := entry.screen + if screen == "" { + screen = "(current screen)" + } + fmt.Fprintf(&builder, "- %s -> %s\n", entry.action, screen) + } + } + builder.WriteString("\nPick one action by its number.") + return builder.String() +} + +// choiceResponseFormat is the strict structured-output schema. Field order is +// pinned via raw JSON with reasoning FIRST, so the model reasons before it +// commits to a number (a materially better ordering than answer-first). text is +// required by strict mode but empty for non-typing actions. +func choiceResponseFormat(candidateCount int) *llmclient.ResponseFormat { + schema := fmt.Sprintf(`{ + "type": "object", + "properties": { + "reasoning": {"type": "string", "description": "One short sentence on what you are trying to do and why this action."}, + "choice": {"type": "integer", "minimum": 1, "maximum": %d, "description": "The number of the chosen action."}, + "chosen_action": {"type": "string", "description": "The chosen action's text, copied verbatim from its numbered line."}, + "text": {"type": "string", "description": "For a typing action, the text to enter; otherwise an empty string."} + }, + "required": ["reasoning", "choice", "chosen_action", "text"], + "additionalProperties": false +}`, candidateCount) + return &llmclient.ResponseFormat{ + Type: "json_schema", + JSONSchema: llmclient.JSONSchema{ + Name: "action_choice", + Strict: true, + Schema: json.RawMessage(schema), + }, + } +} + +// choiceOutput is the model's structured response, reasoning first. +type choiceOutput struct { + Reasoning string `json:"reasoning"` + Choice int `json:"choice"` + ChosenAction string `json:"chosen_action"` + Text string `json:"text"` +} + +// weightSuffix matches the trailing " (w34)" annotation appended to each +// numbered line, which models copy verbatim into chosen_action. +var weightSuffix = regexp.MustCompile(`\s*\(w\d+\)$`) + +// stripWeightSuffix trims surrounding whitespace and a trailing weight +// annotation from the model's echoed action so it can be compared to the +// weight-free candidate description. +func stripWeightSuffix(echo string) string { + return strings.TrimSpace(weightSuffix.ReplaceAllString(strings.TrimSpace(echo), "")) +} + +// parseChoice decodes the model's JSON content into the structured choice. +func parseChoice(content string) (choiceOutput, error) { + content = strings.TrimSpace(content) + if content == "" { + return choiceOutput{}, errors.New("empty content") + } + var out choiceOutput + if err := json.Unmarshal([]byte(content), &out); err != nil { + return choiceOutput{}, err + } + if out.Choice == 0 { + return choiceOutput{}, errors.New("no choice") + } + return out, nil +} + +// describeAction renders a short action summary for the recent-action memory. +func describeAction(action verifier.Action) string { + switch action.Kind { + case verifier.ActionKindInputText: + return fmt.Sprintf("InputText %s = %q", actionTarget(action), action.Text) + case verifier.ActionKindScroll: + return fmt.Sprintf("Scroll %s %s", action.Direction, action.On) + case verifier.ActionKindSwipe: + // Coordinates make a repeated identical swipe recognizable in the + // prompt's recent-action memory. + return fmt.Sprintf("Swipe from (%d,%d)", action.FromX, action.FromY) + case verifier.ActionKindPressKey: + return "PressKey " + action.Key + case verifier.ActionKindWait: + return "Wait" + default: + return fmt.Sprintf("%s %s", action.Kind, actionTarget(action)) + } +} + +func actionTarget(action verifier.Action) string { + if action.On != "" { + return action.On + } + return fmt.Sprintf("(%d,%d)", action.X, action.Y) +} + +// historyEntry records one performed action and the screen it led to (filled on +// the following step, once that screen is observed). +type historyEntry struct { + action string + screen string +} + +// actionHistory is a bounded ring of recent actions for the prompt. +type actionHistory struct { + entries []historyEntry + size int +} + +func newActionHistory(size int) *actionHistory { + return &actionHistory{size: size} +} + +// completeLast fills the most recent action's led-to screen with the +// just-observed screen, if it was still pending. +func (h *actionHistory) completeLast(screen string) { + if n := len(h.entries); n > 0 && h.entries[n-1].screen == "" { + h.entries[n-1].screen = screen + } +} + +// add appends an action (its led-to screen pending) and trims to size. +func (h *actionHistory) add(action string) { + h.entries = append(h.entries, historyEntry{action: action}) + if len(h.entries) > h.size { + h.entries = h.entries[len(h.entries)-h.size:] + } +} + +func (h *actionHistory) recent() []historyEntry { + return h.entries +} + +// stampActionSource records the backend that chose an action on the trace. +// Only an LLM-selected action (not a setup action the JS path produced) carries +// source="llm" and the model's reasoning. +func stampActionSource(traceAction *trace.Action, source ActionSource) { + if traceAction == nil { + return + } + llm, ok := source.(*llmSource) + if !ok || llm.lastSource == "" { + return + } + traceAction.Source = llm.lastSource + traceAction.LLMReasoning = llm.lastReasoning + traceAction.LLMChoice = llm.lastChoice + traceAction.LLMChosenAction = llm.lastChosenAction +} + +// screenshotDataURL downscales the PNG and encodes it as a data URL for the +// image content part. +func screenshotDataURL(pngBytes []byte, maxEdge int) (string, bool) { + scaled := downscalePNG(pngBytes, maxEdge) + if len(scaled) == 0 { + return "", false + } + return "data:image/png;base64," + base64.StdEncoding.EncodeToString(scaled), true +} + +// downscalePNG shrinks the image so its long edge is at most maxEdge, returning +// the original bytes when it is already small enough and nil on decode failure. +func downscalePNG(pngBytes []byte, maxEdge int) []byte { + source, err := png.Decode(bytes.NewReader(pngBytes)) + if err != nil { + return nil + } + bounds := source.Bounds() + width, height := bounds.Dx(), bounds.Dy() + if width <= 0 || height <= 0 { + return nil + } + longEdge := max(width, height) + if longEdge <= maxEdge { + return pngBytes + } + scale := float64(maxEdge) / float64(longEdge) + newWidth := max(1, int(float64(width)*scale)) + newHeight := max(1, int(float64(height)*scale)) + + var buffer bytes.Buffer + if err := png.Encode(&buffer, boxDownscale(source, newWidth, newHeight)); err != nil { + return nil + } + return buffer.Bytes() +} + +// boxDownscale averages each destination pixel over its source box, a cheap +// dependency-free downscale that keeps text legible enough for the model. +func boxDownscale(source image.Image, newWidth, newHeight int) image.Image { + bounds := source.Bounds() + width, height := bounds.Dx(), bounds.Dy() + dest := image.NewRGBA(image.Rect(0, 0, newWidth, newHeight)) + for dy := range newHeight { + sy0 := dy * height / newHeight + sy1 := max((dy+1)*height/newHeight, sy0+1) + for dx := range newWidth { + sx0 := dx * width / newWidth + sx1 := max((dx+1)*width/newWidth, sx0+1) + var r, g, b, a, count uint64 + for sy := sy0; sy < sy1; sy++ { + for sx := sx0; sx < sx1; sx++ { + pr, pg, pb, pa := source.At(bounds.Min.X+sx, bounds.Min.Y+sy).RGBA() + r += uint64(pr) + g += uint64(pg) + b += uint64(pb) + a += uint64(pa) + count++ + } + } + count = max(1, count) + dest.Set(dx, dy, color.RGBA64{ + R: uint16(r / count), + G: uint16(g / count), + B: uint16(b / count), + A: uint16(a / count), + }) + } + } + return dest +} diff --git a/internal/runner/llm_source_test.go b/internal/runner/llm_source_test.go new file mode 100644 index 0000000..057f583 --- /dev/null +++ b/internal/runner/llm_source_test.go @@ -0,0 +1,468 @@ +package runner + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "image" + "image/png" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "slices" + "strconv" + "strings" + "testing" + + "github.com/priyanshujain/sanderling/internal/hierarchy" + "github.com/priyanshujain/sanderling/internal/llmclient" + "github.com/priyanshujain/sanderling/internal/verifier" +) + +// llmInputCorpus mirrors pkg/spec/src/corpus.ts INPUT_CORPUS so an InputText +// value drawn by the shared sampler can be asserted to come from the pool. +var llmInputCorpus = []string{ + "", "a", strings.Repeat("a", 4096), "🙂🔥💸", " ", "\t\n", "-1", + "999999999999999999999", "0.0000001", "1e10", "'; DROP TABLE--", + "", "../../etc/passwd", "%s%n", "NaN", +} + +const llmFixtureSpec = ` +import { llm, always, taps, typing, weighted } from "@sanderling/spec"; +globalThis.properties = { ok: always(() => true) }; +globalThis.actions = weighted([1, taps], [1, typing]); +globalThis.generator = llm({ model: "test/model" }); +` + +const llmTreeJSON = `{ + "attributes": {"bounds": "[0,0,400,800]"}, + "children": [ + {"attributes": {"resource-id": "Submit", "text": "Submit", "bounds": "[0,0,400,100]"}, "clickable": true, "enabled": true, "children": []}, + {"attributes": {"resource-id": "Name", "class": "EditText", "bounds": "[0,100,400,200]"}, "enabled": true, "children": []} + ] +}` + +func TestActionForCandidatePassesNonTypingThrough(t *testing.T) { + source := &llmSource{} + candidate := verifier.ActionCandidate{ + Kind: verifier.ActionKindTap, + Action: verifier.Action{Kind: verifier.ActionKindTap, On: "id:Submit", X: 10, Y: 20}, + } + action, err := source.actionForCandidate(candidate, "ignored") + if err != nil { + t.Fatalf("actionForCandidate: %v", err) + } + if action.Kind != verifier.ActionKindTap || action.On != "id:Submit" || action.X != 10 || action.Y != 20 { + t.Errorf("tap = %+v, want the candidate action verbatim", action) + } + if action.Text != "" { + t.Errorf("non-typing action must not carry text, got %q", action.Text) + } +} + +func TestActionForCandidateUsesModelText(t *testing.T) { + source := &llmSource{} + candidate := verifier.ActionCandidate{ + Kind: verifier.ActionKindInputText, + LLMText: true, + Action: verifier.Action{Kind: verifier.ActionKindInputText, On: "id:Name"}, + } + action, err := source.actionForCandidate(candidate, "Priya") + if err != nil { + t.Fatalf("actionForCandidate: %v", err) + } + if action.Text != "Priya" { + t.Errorf("text = %q, want the model-supplied value", action.Text) + } +} + +func TestActionForCandidateAuthoredTypingKeepsSampledValue(t *testing.T) { + source := &llmSource{} + candidate := verifier.ActionCandidate{ + Kind: verifier.ActionKindInputText, + LLMText: false, // authored InputText: replay the spec's sampled value + Action: verifier.Action{Kind: verifier.ActionKindInputText, On: "id:Amount", Text: "42"}, + } + action, err := source.actionForCandidate(candidate, "ignored") + if err != nil { + t.Fatalf("actionForCandidate: %v", err) + } + if action.Text != "42" { + t.Errorf("text = %q, want the authored value 42", action.Text) + } +} + +func TestActionForCandidateFallsBackToSampler(t *testing.T) { + fake := newFakeOpenRouter(t) + source, _ := newLLMSource(t, fake) + candidate := verifier.ActionCandidate{ + Kind: verifier.ActionKindInputText, + LLMText: true, + Action: verifier.Action{Kind: verifier.ActionKindInputText, On: "id:Name"}, + } + action, err := source.actionForCandidate(candidate, " ") + if err != nil { + t.Fatalf("actionForCandidate: %v", err) + } + if !slices.Contains(llmInputCorpus, action.Text) { + t.Errorf("empty model text should fall back to the corpus, got %q", action.Text) + } +} + +func TestParseChoice(t *testing.T) { + out, err := parseChoice(`{"reasoning":"go home","choice":3,"chosen_action":"Tap \"Home\"","text":""}`) + if err != nil { + t.Fatalf("parseChoice: %v", err) + } + if out.Reasoning != "go home" || out.Choice != 3 || out.ChosenAction != `Tap "Home"` { + t.Errorf("parseChoice = %+v", out) + } + + if _, err := parseChoice(""); err == nil { + t.Error("expected error for empty content") + } + if _, err := parseChoice(`{"reasoning":"x","choice":0,"chosen_action":"","text":""}`); err == nil { + t.Error("expected error for a zero choice") + } + if _, err := parseChoice(`not json`); err == nil { + t.Error("expected error for malformed JSON") + } +} + +func TestSystemPromptAppendsInstructions(t *testing.T) { + if got := (&llmSource{}).systemPrompt(); got != llmSystemPrompt { + t.Error("empty instructions should yield the base prompt unchanged") + } + withInstr := (&llmSource{instructions: "hunt for double submits"}).systemPrompt() + if !strings.Contains(withInstr, llmSystemPrompt) { + t.Error("system prompt must retain the base framing") + } + if !strings.Contains(withInstr, "hunt for double submits") { + t.Error("system prompt must include the spec instructions") + } +} + +// fakeOpenRouter is a configurable in-process OpenRouter server. Set ranked / +// reasoning before each call; it echoes them as a json_schema content body. +type fakeOpenRouter struct { + server *httptest.Server + choice int + chosenAction string + text string + reasoning string + lastRequest map[string]any +} + +func newFakeOpenRouter(t *testing.T) *fakeOpenRouter { + t.Helper() + fake := &fakeOpenRouter{reasoning: "because"} + fake.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(body, &fake.lastRequest) + content, _ := json.Marshal(map[string]any{ + "reasoning": fake.reasoning, + "choice": fake.choice, + "chosen_action": fake.chosenAction, + "text": fake.text, + }) + response, _ := json.Marshal(llmclient.Response{ + Choices: []llmclient.Choice{{Message: llmclient.ResponseMessage{Content: string(content)}}}, + }) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(response) + })) + t.Cleanup(fake.server.Close) + return fake +} + +func newLLMSource(t *testing.T, fake *fakeOpenRouter) (*llmSource, *verifier.Verifier) { + t.Helper() + t.Setenv("OPENROUTER_API_KEY", "test-key") + t.Setenv("OPENROUTER_BASE_URL", fake.server.URL) + client, err := llmclient.New() + if err != nil { + t.Fatalf("llmclient.New: %v", err) + } + + verifierInstance, err := verifier.New() + if err != nil { + t.Fatal(err) + } + if err := verifierInstance.Load(bundleSpec(t, llmFixtureSpec)); err != nil { + t.Fatal(err) + } + if _, ok := verifierInstance.LLMConfig(); !ok { + t.Fatal("llm fixture spec did not register the llm action backend") + } + + source := &llmSource{ + verifier: verifierInstance, + client: client, + model: "test/model", + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + history: newActionHistory(llmHistorySize), + } + return source, verifierInstance +} + +func pushLLMSnapshot(t *testing.T, v *verifier.Verifier) { + t.Helper() + tree, err := hierarchy.Parse(llmTreeJSON) + if err != nil { + t.Fatal(err) + } + if err := v.PushSnapshot(verifier.SnapshotInput{Tree: tree, ScreenshotPNG: tinyPNG(t)}); err != nil { + t.Fatalf("PushSnapshot: %v", err) + } +} + +func candidateByKind(t *testing.T, candidates []verifier.ActionCandidate, kind verifier.ActionKind) verifier.ActionCandidate { + t.Helper() + for _, candidate := range candidates { + if candidate.Kind == kind { + return candidate + } + } + t.Fatalf("no candidate of kind %q in %v", kind, candidates) + return verifier.ActionCandidate{} +} + +func TestPickSourcesSelectsLLMWhenRequested(t *testing.T) { + fake := newFakeOpenRouter(t) + _, verifierInstance := newLLMSource(t, fake) + action, _, err := pickSources(Options{ + Verifier: verifierInstance, + Generator: "llm", + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + }) + if err != nil { + t.Fatalf("pickSources: %v", err) + } + if _, ok := action.(*llmSource); !ok { + t.Errorf("action source = %T, want *llmSource for --generator llm", action) + } +} + +func TestPickSourcesSeededByDefault(t *testing.T) { + fake := newFakeOpenRouter(t) + _, verifierInstance := newLLMSource(t, fake) + action, _, err := pickSources(Options{ + Verifier: verifierInstance, + Generator: "seeded", + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + }) + if err != nil { + t.Fatalf("pickSources: %v", err) + } + // Even with a generator = llm() config present, the seeded flag wins. + if _, ok := action.(gojaSource); !ok { + t.Errorf("action source = %T, want gojaSource for --generator seeded", action) + } +} + +func TestLLMSourceDrivesExecutedActions(t *testing.T) { + fake := newFakeOpenRouter(t) + source, verifierInstance := newLLMSource(t, fake) + pushLLMSnapshot(t, verifierInstance) + candidates := verifierInstance.Candidates() + + // Step 1: the model picks the Tap on Submit by its number, echoing its + // description. + tap := candidateByKind(t, candidates, verifier.ActionKindTap) + fake.choice = tap.Index + fake.chosenAction = tap.Description + fake.reasoning = "tap submit" + fake.text = "" + action, err := source.NextAction(context.Background()) + if err != nil { + t.Fatalf("NextAction: %v", err) + } + if action.Kind != verifier.ActionKindTap || action.On != "id:Submit" { + t.Errorf("step 1 action = %+v, want Tap on id:Submit", action) + } + if source.lastSource != "llm" || source.lastReasoning != "tap submit" { + t.Errorf("source state = %q/%q, want llm/tap submit", source.lastSource, source.lastReasoning) + } + + // The request carried the model and a screenshot image part. + if fake.lastRequest["model"] != "test/model" { + t.Errorf("request model = %v", fake.lastRequest["model"]) + } + if !requestHasImage(fake.lastRequest) { + t.Error("request carried no screenshot image part") + } + + // Step 2: the model picks the typing candidate and supplies the value. + pushLLMSnapshot(t, verifierInstance) + typing := candidateByKind(t, candidates, verifier.ActionKindInputText) + fake.choice = typing.Index + fake.chosenAction = typing.Description + fake.reasoning = "type a name" + fake.text = "Priya" + action, err = source.NextAction(context.Background()) + if err != nil { + t.Fatalf("NextAction: %v", err) + } + if action.Kind != verifier.ActionKindInputText || action.On != "id:Name" { + t.Errorf("step 2 action = %+v, want InputText on id:Name", action) + } + if action.Text != "Priya" { + t.Errorf("InputText text = %q, want the model-supplied Priya", action.Text) + } + + // The trace records source=llm, the reasoning, the choice, and the echo. + traceAction := traceActionFor(action, nil) + stampActionSource(traceAction, source) + if traceAction.Source != "llm" || traceAction.LLMReasoning != "type a name" { + t.Errorf("trace action = %+v, want source=llm reasoning=type a name", traceAction) + } + if traceAction.LLMChoice != typing.Index || traceAction.LLMChosenAction != typing.Description { + t.Errorf("trace choice = %d/%q, want %d/%q", traceAction.LLMChoice, traceAction.LLMChosenAction, typing.Index, typing.Description) + } +} + +func TestLLMSourceSkipsOnOutOfRangeChoice(t *testing.T) { + fake := newFakeOpenRouter(t) + source, verifierInstance := newLLMSource(t, fake) + pushLLMSnapshot(t, verifierInstance) + + fake.choice = 9999 + fake.chosenAction = "whatever" + _, err := source.NextAction(context.Background()) + if !errors.Is(err, verifier.ErrNoAction) { + t.Fatalf("NextAction err = %v, want ErrNoAction for an out-of-range choice", err) + } + if source.lastSource != "" { + t.Errorf("lastSource = %q, want empty after a skipped step", source.lastSource) + } +} + +func TestLLMSourceAcceptsEchoWithWeightSuffix(t *testing.T) { + // Real models copy the whole numbered line, including its trailing "(w34)" + // weight annotation. That must still count as a match, not a strict skip. + fake := newFakeOpenRouter(t) + source, verifierInstance := newLLMSource(t, fake) + pushLLMSnapshot(t, verifierInstance) + candidates := verifierInstance.Candidates() + + tap := candidateByKind(t, candidates, verifier.ActionKindTap) + fake.choice = tap.Index + fake.chosenAction = tap.Description + " (w" + strconv.Itoa(tap.Weight) + ")" + action, err := source.NextAction(context.Background()) + if err != nil { + t.Fatalf("NextAction: %v", err) + } + if action.Kind != verifier.ActionKindTap { + t.Errorf("action = %+v, want Tap; the weight-suffixed echo was wrongly rejected", action) + } + if source.lastSource != "llm" { + t.Error("weight-suffixed echo should be accepted, not strict-skipped") + } +} + +func TestStripWeightSuffix(t *testing.T) { + cases := map[string]string{ + `Tap "+ Add account" (w34)`: `Tap "+ Add account"`, + `Tap "Sign in"`: `Tap "Sign in"`, + `Scroll down (w7)`: `Scroll down`, + ` Tap "x" (w1) `: `Tap "x"`, + } + for in, want := range cases { + if got := stripWeightSuffix(in); got != want { + t.Errorf("stripWeightSuffix(%q) = %q, want %q", in, got, want) + } + } +} + +func TestLLMSourceStrictSkipsOnEchoMismatch(t *testing.T) { + fake := newFakeOpenRouter(t) + source, verifierInstance := newLLMSource(t, fake) + pushLLMSnapshot(t, verifierInstance) + candidates := verifierInstance.Candidates() + + // A valid number, but the echoed action disagrees with that numbered entry: + // the model reasoned about one control and picked another's number. + tap := candidateByKind(t, candidates, verifier.ActionKindTap) + fake.choice = tap.Index + fake.chosenAction = "Tap \"Something Else\"" + _, err := source.NextAction(context.Background()) + if !errors.Is(err, verifier.ErrNoAction) { + t.Fatalf("NextAction err = %v, want ErrNoAction on chosen_action mismatch", err) + } + if source.lastSource != "" { + t.Errorf("lastSource = %q, want empty after a strict skip", source.lastSource) + } +} + +func TestLLMSourceSkipsOnHTTPError(t *testing.T) { + fake := newFakeOpenRouter(t) + // Replace the handler with one that always errors. + fake.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }) + source, verifierInstance := newLLMSource(t, fake) + pushLLMSnapshot(t, verifierInstance) + + fake.choice = 1 + _, err := source.NextAction(context.Background()) + if !errors.Is(err, verifier.ErrNoAction) { + t.Fatalf("NextAction err = %v, want ErrNoAction on HTTP failure", err) + } +} + +func TestDownscalePNGShrinksLongEdge(t *testing.T) { + large := image.NewRGBA(image.Rect(0, 0, 2048, 1024)) + var buffer bytes.Buffer + if err := png.Encode(&buffer, large); err != nil { + t.Fatal(err) + } + scaled := downscalePNG(buffer.Bytes(), 1024) + decoded, err := png.Decode(bytes.NewReader(scaled)) + if err != nil { + t.Fatalf("decode scaled: %v", err) + } + if decoded.Bounds().Dx() != 1024 { + t.Errorf("scaled width = %d, want 1024", decoded.Bounds().Dx()) + } + if decoded.Bounds().Dy() != 512 { + t.Errorf("scaled height = %d, want 512", decoded.Bounds().Dy()) + } +} + +func TestDownscalePNGKeepsSmallImage(t *testing.T) { + original := tinyPNG(t) + if got := downscalePNG(original, 1024); !bytes.Equal(got, original) { + t.Error("a sub-maxEdge image should be returned unchanged") + } +} + +func requestHasImage(request map[string]any) bool { + messages, ok := request["messages"].([]any) + if !ok { + return false + } + for _, message := range messages { + parts, ok := message.(map[string]any)["content"].([]any) + if !ok { + continue + } + for _, part := range parts { + if part.(map[string]any)["type"] == "image_url" { + return true + } + } + } + return false +} + +func tinyPNG(t *testing.T) []byte { + t.Helper() + img := image.NewRGBA(image.Rect(0, 0, 8, 8)) + var buffer bytes.Buffer + if err := png.Encode(&buffer, img); err != nil { + t.Fatal(err) + } + return buffer.Bytes() +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c5274c1..66462aa 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -36,6 +36,10 @@ type Options struct { Verifier *verifier.Verifier TraceWriter *trace.Writer Logger *slog.Logger + // Generator selects the action picker: "llm" drives selection with the + // spec's generator = llm({...}) config; anything else (the default) uses the + // seeded weighted picker. Both draw from the same actionsRoot candidate set. + Generator string } type Summary struct { @@ -74,7 +78,10 @@ func Run(ctx context.Context, options Options) (Summary, error) { // Pick the action and extractor sources once from the driver's // capabilities so the step loop runs one uniform path with no per-step // driver type assertion. - actionSource, extractorSource := pickSources(options) + actionSource, extractorSource, err := pickSources(options) + if err != nil { + return Summary{}, err + } summary := Summary{StartTime: time.Now()} deadline := summary.StartTime.Add(options.Duration) @@ -105,6 +112,7 @@ func Run(ctx context.Context, options Options) (Summary, error) { var tree *hierarchy.Tree var hierarchyErr error var transitional bool + var screenshotPNG []byte var metrics *trace.Metrics var logs []verifier.LogEntry @@ -118,7 +126,7 @@ func Run(ctx context.Context, options Options) (Summary, error) { // screenshot describe the same frame, then re-fetches the pair // while the tree still looks transitional. g.Go(func() error { - tree, transitional, hierarchyErr = fetchSyncedState(gctx, options, logger, si) + tree, screenshotPNG, transitional, hierarchyErr = fetchSyncedState(gctx, options, logger, si) return nil }) g.Go(func() error { @@ -183,12 +191,13 @@ func Run(ctx context.Context, options Options) (Summary, error) { skippedVerification := false if !transitional { if err := options.Verifier.PushSnapshot(verifier.SnapshotInput{ - Tree: tree, - LastAction: lastAction, - StepTime: stepStart, - StepIndex: stepIndex, - RunStart: summary.StartTime, - Logs: logs, + Tree: tree, + ScreenshotPNG: screenshotPNG, + LastAction: lastAction, + StepTime: stepStart, + StepIndex: stepIndex, + RunStart: summary.StartTime, + Logs: logs, }); err != nil { return summary, fmt.Errorf("step %d push: %w", stepIndex, err) } @@ -215,6 +224,7 @@ func Run(ctx context.Context, options Options) (Summary, error) { var traceAction *trace.Action if nextErr == nil { traceAction = traceActionFor(nextAction, tree) + stampActionSource(traceAction, actionSource) } else if !errors.Is(nextErr, verifier.ErrNoAction) { return summary, fmt.Errorf("step %d next action: %w", stepIndex, nextErr) } @@ -855,7 +865,7 @@ const ( // on a still-transitional tree. Callers use it to skip the verifier for // that step so the previous/current extractor advance does not absorb // transient state. -func fetchSyncedState(ctx context.Context, options Options, logger *slog.Logger, stepIndex int) (tree *hierarchy.Tree, transitional bool, err error) { +func fetchSyncedState(ctx context.Context, options Options, logger *slog.Logger, stepIndex int) (tree *hierarchy.Tree, png []byte, transitional bool, err error) { var pngBytes []byte var previousJSON string retryLoop: @@ -868,7 +878,7 @@ retryLoop: tree, err = hierarchy.Parse(hierarchyJSON) pngBytes = image.PNG } - if err != nil || !isTransitionalHierarchy(tree) { + if err != nil || !tree.Transitional() { break } // A tree unchanged since the previous attempt is a settled state @@ -896,28 +906,7 @@ retryLoop: logger.Warn("screenshot write failed", "step", stepIndex, "err", writeErr) } } - return tree, transitional, err -} - -// isTransitionalHierarchy returns true when the tree carries more than one -// resource-id ending in "Screen" - the marker of a Compose NavHost mid -// cross-fade where both source and destination route composables are alive. -// Mirrors the sidecar's stabilitySnapshot heuristic so runner-side rejection -// stays consistent with the settle poll. -func isTransitionalHierarchy(tree *hierarchy.Tree) bool { - if tree == nil { - return false - } - screens := 0 - for _, element := range tree.Elements { - if strings.HasSuffix(element.ResourceID, "Screen") { - screens++ - if screens > 1 { - return true - } - } - } - return false + return tree, pngBytes, transitional, err } func traceActionFor(action verifier.Action, tree *hierarchy.Tree) *trace.Action { diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index db7c7fa..31fecaa 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -1118,34 +1118,6 @@ func TestRunner_OneScreenshotPerStep(t *testing.T) { } } -// TestIsTransitionalHierarchy_DetectsMultipleScreens covers the runner-side -// guard that re-fetches when the hierarchy still carries two route-level -// *Screen ids - the NavHost cross-fade signature. -func TestIsTransitionalHierarchy_DetectsMultipleScreens(t *testing.T) { - multi, err := hierarchy.Parse(`{"attributes":{"resource-id":"root"},"children":[ - {"attributes":{"resource-id":"AddAccountScreen"},"children":[]}, - {"attributes":{"resource-id":"HomeScreen"},"children":[]} - ]}`) - if err != nil { - t.Fatal(err) - } - if !isTransitionalHierarchy(multi) { - t.Error("expected multi-screen tree to be flagged as transitional") - } - - single, err := hierarchy.Parse(`{"attributes":{"resource-id":"HomeScreen"},"children":[]}`) - if err != nil { - t.Fatal(err) - } - if isTransitionalHierarchy(single) { - t.Error("single-screen tree must not be flagged as transitional") - } - - if isTransitionalHierarchy(nil) { - t.Error("nil tree must not be flagged as transitional") - } -} - // TestRunner_StableTransitionalTreeIsVerified feeds a driver whose hierarchy // constantly carries two route-level *Screen ids but never changes between // retry attempts. Such a tree is a settled state that merely matches the diff --git a/internal/runner/source.go b/internal/runner/source.go index f2a7367..570c5ff 100644 --- a/internal/runner/source.go +++ b/internal/runner/source.go @@ -4,8 +4,10 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "github.com/priyanshujain/sanderling/internal/driver" + "github.com/priyanshujain/sanderling/internal/llmclient" "github.com/priyanshujain/sanderling/internal/verifier" ) @@ -61,12 +63,40 @@ func (s webSource) ExtractorOverrides(ctx context.Context) (map[int]json.RawMess } // pickSources selects the runtime's action and extractor sources ONCE at setup -// from the driver's capabilities, so the step loop never type-asserts. -func pickSources(options Options) (ActionSource, ExtractorSource) { +// from the driver's capabilities, the --generator flag, and the spec, so the +// step loop never type-asserts. With --generator llm and a spec-declared +// generator = llm({...}) it constructs the chat-completions client and returns +// an llmSource for selection while extractor overrides still come from the goja +// path. Otherwise the seeded goja picker drives. +func pickSources(options Options) (ActionSource, ExtractorSource, error) { if web, ok := options.Driver.(driver.WebDriver); ok { source := webSource{web: web} - return source, source + return source, source, nil + } + logger := options.Logger + if logger == nil { + logger = slog.Default() + } + if options.Generator == "llm" { + config, ok := options.Verifier.LLMConfig() + if !ok { + logger.Warn("--generator llm requested but spec declares no generator = llm(); using the seeded picker") + } else { + client, err := llmclient.New() + if err != nil { + return nil, nil, fmt.Errorf("llm action generator: %w", err) + } + action := &llmSource{ + verifier: options.Verifier, + client: client, + model: config.Model, + instructions: config.Instructions, + logger: logger, + history: newActionHistory(llmHistorySize), + } + return action, gojaSource{verifier: options.Verifier}, nil + } } source := gojaSource{verifier: options.Verifier} - return source, source + return source, source, nil } diff --git a/internal/testrun/testrun.go b/internal/testrun/testrun.go index cda97a1..b214eb7 100644 --- a/internal/testrun/testrun.go +++ b/internal/testrun/testrun.go @@ -34,6 +34,8 @@ type Options struct { Seed int64 Output string ClearData bool + // Generator selects the action picker: "llm" or the default seeded picker. + Generator string // iosUDID, iosIsSimulator, and iosCoreDeviceID are filled by Execute after // resolving the iOS target, then read by buildDriver to choose the simulator @@ -166,6 +168,7 @@ func Execute(ctx context.Context, options Options, stdout io.Writer) error { Verifier: verifierInstance, TraceWriter: traceWriter, Logger: newProgressLogger(stdout), + Generator: options.Generator, }) terminateCtx, terminateCancel := context.WithTimeout(context.Background(), 5*time.Second) diff --git a/internal/trace/writer.go b/internal/trace/writer.go index 970d8be..ae91d8a 100644 --- a/internal/trace/writer.go +++ b/internal/trace/writer.go @@ -82,6 +82,18 @@ type Action struct { Selector string `json:"selector,omitempty"` ResolvedBounds *BoundsRecord `json:"resolved_bounds,omitempty"` TapPoint *PointRecord `json:"tap_point,omitempty"` + // Source names the backend that chose this action: "llm" when the LLM + // action backend selected it, empty for the seeded picker. LLMReasoning is + // the model's short rationale, shown by the replay UI to explain the pick. + Source string `json:"source,omitempty"` + LLMReasoning string `json:"llm_reasoning,omitempty"` + // LLMChoice is the 1-based number the model picked from the candidate list; + // LLMChosenAction is the action description it echoed for that number. The + // runner strict-skips when the echo disagrees with the numbered entry, so on + // a recorded action the two always agree — the replay UI shows them to + // confirm the reasoning matched the executed action. + LLMChoice int `json:"llm_choice,omitempty"` + LLMChosenAction string `json:"llm_chosen_action,omitempty"` } type BoundsRecord struct { diff --git a/internal/verifier/llm.go b/internal/verifier/llm.go new file mode 100644 index 0000000..607dff7 --- /dev/null +++ b/internal/verifier/llm.go @@ -0,0 +1,684 @@ +package verifier + +import ( + "errors" + "fmt" + "math" + "strconv" + "strings" + + "github.com/dop251/goja" + + "github.com/priyanshujain/sanderling/internal/hierarchy" +) + +// LLMConfig is the spec-declared configuration for the LLM action generator, +// read off globalThis.generator when the spec assigned `generator = llm({...})`. +// It is orthogonal to globalThis.actions (the weighted tree the LLM picks from); +// only the picker differs. +type LLMConfig struct { + Model string + // Instructions is optional spec-level guidance appended to the prompt to + // steer the model toward bug-hunting (empty when unset). + Instructions string +} + +// LLMConfig reports the LLM action-generator config when the spec declared one +// (globalThis.generator.kind === "llm"). The second return is false for every +// other spec, so the runner falls back to the seeded picker. +func (v *Verifier) LLMConfig() (LLMConfig, bool) { + generator := v.runtime.GlobalObject().Get("generator") + if generator == nil || goja.IsUndefined(generator) || goja.IsNull(generator) { + return LLMConfig{}, false + } + object := generator.ToObject(v.runtime) + if object == nil { + return LLMConfig{}, false + } + kind := object.Get("kind") + if kind == nil || kind.String() != "llm" { + return LLMConfig{}, false + } + config := object.Get("config") + if config == nil || goja.IsUndefined(config) || goja.IsNull(config) { + return LLMConfig{}, false + } + configObject := config.ToObject(v.runtime) + if configObject == nil { + return LLMConfig{}, false + } + model := "" + if value := configObject.Get("model"); value != nil && !goja.IsUndefined(value) { + model = value.String() + } + instructions := "" + if value := configObject.Get("instructions"); value != nil && !goja.IsUndefined(value) && !goja.IsNull(value) { + instructions = value.String() + } + return LLMConfig{Model: model, Instructions: instructions}, true +} + +// Screenshot returns the most recent step's screenshot PNG (set by +// PushSnapshot), or nil if none was captured. +func (v *Verifier) Screenshot() []byte { + return v.lastScreenshot +} + +// CurrentScreen returns the screen id of the most recent snapshot's first +// element, matching the runner's own screen labeling. Empty when no tree is +// loaded. +func (v *Verifier) CurrentScreen() string { + if v.lastTree == nil || len(v.lastTree.Elements) == 0 { + return "" + } + return v.lastTree.Elements[0].Screen +} + +// SampleInput draws one InputText value from the shared corpus via the bundled +// __sanderlingSampleInput__. It errors when the bundle did not install the +// callable (a raw-JS fixture) so the caller can skip typing rather than send an +// empty string. +func (v *Verifier) SampleInput() (string, error) { + if v.sampleInputFn == nil { + return "", errors.New("verifier: input sampler not available") + } + value, err := v.sampleInputFn(goja.Undefined()) + if err != nil { + return "", err + } + if value == nil || goja.IsUndefined(value) || goja.IsNull(value) { + return "", nil + } + return value.String(), nil +} + +// ActionCandidate is one selectable action the LLM generator may choose from, +// enumerated by collect-walking the spec's weighted actionsRoot (the same tree +// the seeded picker draws). Each candidate is a concrete, ready-to-execute +// action carrying a plainly-worded Description (numbered and echoed for +// strict-skip) plus its effective Weight so the model sees the spec's testing +// priorities. +type ActionCandidate struct { + // Index is the candidate's 1-based position in the numbered list the model + // picks a number from. + Index int + // Kind is the resulting action kind. + Kind ActionKind + // Description is the rendered action shown to the model and echoed back as + // chosen_action, e.g. `Tap "Add credit"`. Dedup keys on it, so it is unique. + Description string + // Label is the visible-text target label (empty for gestures). + Label string + // Weight is the effective selection weight as a percentage (1..100), + // meaningful only when Weighted is true (the tree used `weighted`). + Weight int + Weighted bool + // InputType hints a typing field's expected input (e.g. "number", or the + // field's hint); empty when unknown or not a typing candidate. + InputType string + // Direction is up/down/left/right for gesture (Scroll) candidates, else "". + Direction string + // LLMText is true for builtin typing, where the model supplies the value; + // false for authored InputText, whose sampled Action.Text is replayed as-is. + LLMText bool + // Action is the concrete action executed when this candidate is chosen. For + // builtin typing it carries no text until the model's value is filled in. + Action Action + + // prob is the internal accumulated selection probability, summed across + // dedup, then rounded into Weight. Not exposed in the prompt directly. + prob float64 +} + +// verbActionKind maps a picker verb to the action kind it dispatches. +func verbActionKind(verb string) ActionKind { + switch verb { + case "taps": + return ActionKindTap + case "doubleTaps": + return ActionKindDoubleTap + case "longPresses": + return ActionKindLongPress + case "typing": + return ActionKindInputText + case "scrolls": + return ActionKindScroll + case "swipes": + return ActionKindSwipe + default: + return "" + } +} + +// maxLabelRunes caps a visible-text label so joined descendant text stays short +// enough to render on one numbered line. +const maxLabelRunes = 40 + +// gestureDirections are the directional scrolls emitted per scrollable +// container. Vertical only: most mobile lists scroll up/down, and keeping the +// set tiny is the whole point of folding per-element swipes away. +var gestureDirections = []string{"down", "up"} + +// Candidates enumerates every action the spec's weighted actionsRoot yields at +// the current step, each tagged with a plainly-worded description and its +// effective weight, for the LLM generator to pick one number from. It walks the +// SAME tree the seeded picker draws: weighted branches recurse (accumulating the +// selection probability), authored actions()/whenRoute leaves are called once +// for their concrete actions, and builtin verbs enumerate per applicable +// element. Disabled controls are dropped, per-element gestures fold into a few +// directional scrolls over scrollable containers, and identical descriptions +// dedup (summing weight). +func (v *Verifier) Candidates() []ActionCandidate { + if v.lastTree == nil { + return nil + } + // A cross-fade frame's layout is mid-animation, often in a collapsed + // coordinate space, so acting on it taps garbage (e.g. the soft keyboard). + // Skip it so the LLM re-observes a settled frame next step. + if v.lastTree.Transitional() { + return nil + } + root := v.runtime.GlobalObject().Get("actions") + if root == nil || goja.IsUndefined(root) || goja.IsNull(root) { + return nil + } + nodeIndex := buildNodeIndex(v.lastTree) + var raw []ActionCandidate + v.collectNode(root, 1.0, false, nodeIndex, &raw) + return finalizeCandidates(raw) +} + +// collectNode dispatches one GeneratorNode of the action tree. prob is the +// accumulated probability the seeded picker reaches this node; weighted records +// whether any weighted node lies on the path (so weights are shown only when the +// spec actually declared them). +func (v *Verifier) collectNode(node goja.Value, prob float64, weighted bool, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) { + object := node.ToObject(v.runtime) + if object == nil { + return + } + kind := object.Get("kind") + if kind == nil || goja.IsUndefined(kind) { + return + } + switch kind.String() { + case "weighted": + v.collectWeighted(object, prob, nodeIndex, out) + case "actions": + v.collectActions(object, prob, weighted, nodeIndex, out) + case "builtin": + verb := object.Get("verb") + if verb != nil && !goja.IsUndefined(verb) { + v.collectBuiltin(verb.String(), prob, weighted, nodeIndex, out) + } + case "llm": + // The llm marker is the generator, not part of the candidate tree. + } +} + +// collectWeighted recurses each branch, splitting the incoming probability by +// the branch weight over the sibling total (matching the seeded picker's single +// weighted draw). +func (v *Verifier) collectWeighted(object *goja.Object, prob float64, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) { + branches := object.Get("branches") + if branches == nil { + return + } + array := branches.ToObject(v.runtime) + if array == nil { + return + } + length := int(array.Get("length").ToInteger()) + weights := make([]float64, length) + children := make([]goja.Value, length) + total := 0.0 + for i := range length { + entry := array.Get(strconv.Itoa(i)) + pair := entry.ToObject(v.runtime) + if pair == nil { + continue + } + weight := pair.Get("0").ToFloat() + if weight < 0 || math.IsNaN(weight) { + weight = 0 + } + weights[i] = weight + children[i] = pair.Get("1") + total += weight + } + if total <= 0 { + return + } + for i := range length { + if children[i] == nil { + continue + } + v.collectNode(children[i], prob*weights[i]/total, true, nodeIndex, out) + } +} + +// collectActions calls an authored leaf's generator once (safe: it reads state +// and, off-route, returns []), turning each concrete descriptor into a +// candidate. It runs OUTSIDE the picker's rng scope, so from(...).generate() +// draws nothing and no seed advances. +func (v *Verifier) collectActions(object *goja.Object, prob float64, weighted bool, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) { + generate, ok := goja.AssertFunction(object.Get("generate")) + if !ok { + return + } + result, err := generate(goja.Undefined()) + if err != nil { + return + } + array := result.ToObject(v.runtime) + if array == nil { + return + } + length := int(array.Get("length").ToInteger()) + for i := range length { + candidate, ok := v.candidateFromDescriptor(array.Get(strconv.Itoa(i)), nodeIndex) + if !ok { + continue + } + candidate.prob = prob + candidate.Weighted = weighted + *out = append(*out, candidate) + } +} + +// candidateFromDescriptor lowers one authored ActionDescriptor (as a goja +// object) into a ready-to-run candidate, resolving the target's coordinates, +// selector, and visible-text label. Actions on a disabled control are dropped. +func (v *Verifier) candidateFromDescriptor(value goja.Value, nodeIndex map[*hierarchy.Element]*hierarchy.Node) (ActionCandidate, bool) { + object := value.ToObject(v.runtime) + if object == nil { + return ActionCandidate{}, false + } + kindValue := object.Get("kind") + if kindValue == nil || goja.IsUndefined(kindValue) { + return ActionCandidate{}, false + } + kind := ActionKind(kindValue.String()) + switch kind { + case ActionKindTap, ActionKindDoubleTap, ActionKindLongPress: + target := v.resolveTarget(object.Get("on"), nodeIndex) + if target.disabled { + return ActionCandidate{}, false + } + return ActionCandidate{ + Kind: kind, + Label: target.label, + Action: Action{Kind: kind, On: target.selector, X: target.x, Y: target.y}, + }, true + case ActionKindInputText: + target := v.resolveTarget(object.Get("into"), nodeIndex) + if target.disabled { + return ActionCandidate{}, false + } + text := stringField(object, "text") + return ActionCandidate{ + Kind: kind, + Label: target.label, + InputType: target.inputType, + Action: Action{Kind: kind, On: target.selector, X: target.x, Y: target.y, Text: text}, + }, true + case ActionKindScroll: + target := v.resolveTarget(object.Get("in"), nodeIndex) + direction := stringField(object, "direction") + if direction == "" { + direction = "down" + } + return ActionCandidate{ + Kind: kind, + Direction: direction, + Action: Action{Kind: kind, On: target.selector, Direction: direction}, + }, true + case ActionKindPressKey: + return ActionCandidate{ + Kind: kind, + Action: Action{Kind: kind, Key: stringField(object, "key")}, + }, true + case ActionKindWait: + return ActionCandidate{Kind: kind, Action: Action{Kind: kind}}, true + default: + return ActionCandidate{}, false + } +} + +// resolvedTarget is the geometry, selector, label, and input hint a target +// (ax element, selector string, or bare point) resolves to. +type resolvedTarget struct { + x, y int + selector string + label string + inputType string + disabled bool +} + +// resolveTarget reads an authored action's target. Ax element handles carry +// x/y/__sanderlingSelector plus their own text; a bare selector string resolves +// against the current tree; a point carries geometry only. +func (v *Verifier) resolveTarget(value goja.Value, nodeIndex map[*hierarchy.Element]*hierarchy.Node) resolvedTarget { + if value == nil || goja.IsUndefined(value) || goja.IsNull(value) { + return resolvedTarget{} + } + if selector, ok := value.Export().(string); ok { + return v.targetFromSelector(selector, nodeIndex) + } + object := value.ToObject(v.runtime) + if object == nil { + return resolvedTarget{} + } + selector := stringField(object, tagSelector) + if selector == "" { + selector = stringField(object, "selector") + } + target := resolvedTarget{ + x: int(object.Get("x").ToInteger()), + y: int(object.Get("y").ToInteger()), + selector: selector, + } + if element := v.findBySelector(selector); element != nil { + target.label = visibleLabel(element, nodeIndex) + target.inputType = inputTypeHint(element) + target.disabled = !element.Enabled && hasEnabled(element) + } + if target.label == "" { + target.label = truncateLabel(stringField(object, "text")) + } + return target +} + +// targetFromSelector resolves a bare selector-string target against the tree. +func (v *Verifier) targetFromSelector(selector string, nodeIndex map[*hierarchy.Element]*hierarchy.Node) resolvedTarget { + target := resolvedTarget{selector: selector} + element := v.findBySelector(selector) + if element == nil { + return target + } + target.x, target.y = element.Bounds.Center() + target.label = visibleLabel(element, nodeIndex) + target.inputType = inputTypeHint(element) + target.disabled = !element.Enabled && hasEnabled(element) + return target +} + +func (v *Verifier) findBySelector(selector string) *hierarchy.Element { + if selector == "" || v.lastTree == nil { + return nil + } + return v.lastTree.Find(selector) +} + +// collectBuiltin enumerates a builtin verb over the current tree: tap-family and +// typing emit one candidate per applicable element; scrolls/swipes fold into +// directional gestures over scrollable containers. +func (v *Verifier) collectBuiltin(verb string, prob float64, weighted bool, nodeIndex map[*hierarchy.Element]*hierarchy.Node, out *[]ActionCandidate) { + switch verb { + case "taps", "doubleTaps", "longPresses": + kind := verbActionKind(verb) + for _, element := range v.elementsForVerb(verb) { + x, y := element.Bounds.Center() + *out = append(*out, ActionCandidate{ + Kind: kind, + Label: visibleLabel(element, nodeIndex), + Action: Action{Kind: kind, On: selectorForElement(v.lastTree, element), X: x, Y: y}, + prob: prob, + Weighted: weighted, + }) + } + case "typing": + for _, element := range v.elementsForVerb(verb) { + x, y := element.Bounds.Center() + *out = append(*out, ActionCandidate{ + Kind: ActionKindInputText, + Label: visibleLabel(element, nodeIndex), + InputType: inputTypeHint(element), + LLMText: true, + Action: Action{Kind: ActionKindInputText, On: selectorForElement(v.lastTree, element), X: x, Y: y}, + prob: prob, + Weighted: weighted, + }) + } + case "scrolls", "swipes": + v.collectGestures(prob, weighted, out) + } +} + +// collectGestures emits directional scrolls scoped to each scrollable container, +// never per element and never element-labeled. Folding both scrolls and swipes +// here is what removes the flood of mislabeled `Swipe "X"` gestures. +func (v *Verifier) collectGestures(prob float64, weighted bool, out *[]ActionCandidate) { + scope := v.scopedElements() + for _, element := range v.lastTree.Elements { + if !scope[element] { + continue + } + if element.Attributes["scrollable"] != "true" { + continue + } + if element.Bounds.Width() <= 0 || element.Bounds.Height() <= 0 { + continue + } + selector := selectorForElement(v.lastTree, element) + for _, direction := range gestureDirections { + action := Action{Kind: ActionKindScroll, On: selector, Direction: direction} + if selector == "" { + action.FromX, action.FromY, action.ToX, action.ToY = scrollGeometry(element.Bounds, direction) + } + *out = append(*out, ActionCandidate{ + Kind: ActionKindScroll, + Direction: direction, + Action: action, + prob: prob, + Weighted: weighted, + }) + } + } +} + +// scrollGeometry lowers a directional scroll to swipe endpoints over the given +// container bounds, matching the runner's own derivation, used only when the +// container has no resolving selector. +func scrollGeometry(bounds hierarchy.Bounds, direction string) (fromX, fromY, toX, toY int) { + cx, cy := bounds.Center() + fromX, fromY, toX, toY = cx, cy, cx, cy + switch direction { + case "down": + toY = cy - 4*bounds.Height()/10 + case "up": + toY = cy + 4*bounds.Height()/10 + case "left": + toX = cx + 4*bounds.Width()/10 + case "right": + toX = cx - 4*bounds.Width()/10 + } + return fromX, fromY, max(0, toX), max(0, toY) +} + +// elementsForVerb returns the in-scope elements a builtin verb applies to, in +// tree order (the seeded picker's enumeration order), reusing verbAccepts. +func (v *Verifier) elementsForVerb(verb string) []*hierarchy.Element { + scope := v.scopedElements() + var elements []*hierarchy.Element + for _, element := range v.lastTree.Elements { + if scope[element] && verbAccepts(verb, element) { + elements = append(elements, element) + } + } + return elements +} + +// finalizeCandidates renders each candidate's description, dedups identical +// descriptions (summing weight), numbers the survivors 1..N, and rounds the +// accumulated probability into a percentage Weight. +func finalizeCandidates(raw []ActionCandidate) []ActionCandidate { + seen := make(map[string]int, len(raw)) + result := make([]ActionCandidate, 0, len(raw)) + for _, candidate := range raw { + candidate.Description = describeCandidate(candidate) + if index, ok := seen[candidate.Description]; ok { + result[index].prob += candidate.prob + result[index].Weighted = result[index].Weighted || candidate.Weighted + continue + } + seen[candidate.Description] = len(result) + result = append(result, candidate) + } + for i := range result { + result[i].Index = i + 1 + if result[i].Weighted { + result[i].Weight = max(1, int(math.Round(result[i].prob*100))) + } + } + return result +} + +// describeCandidate renders the plain, echo-friendly description shown in the +// numbered list. It is the dedup key, so it must be stable and unique per +// distinct action. +func describeCandidate(candidate ActionCandidate) string { + switch candidate.Kind { + case ActionKindTap: + return fmt.Sprintf("Tap %q", candidate.Label) + case ActionKindDoubleTap: + return fmt.Sprintf("Double-tap %q", candidate.Label) + case ActionKindLongPress: + return fmt.Sprintf("Long-press %q", candidate.Label) + case ActionKindInputText: + if candidate.LLMText { + if candidate.InputType != "" { + return fmt.Sprintf("Type into %q (%s)", candidate.Label, candidate.InputType) + } + return fmt.Sprintf("Type into %q", candidate.Label) + } + return fmt.Sprintf("Type %q into %q", candidate.Action.Text, candidate.Label) + case ActionKindScroll: + return "Scroll " + candidate.Direction + case ActionKindPressKey: + return "Press " + candidate.Action.Key + case ActionKindWait: + return "Wait" + default: + return string(candidate.Kind) + } +} + +// buildNodeIndex maps each Element pointer to its Node so descendant text can be +// borrowed for a control whose own text is empty. +func buildNodeIndex(tree *hierarchy.Tree) map[*hierarchy.Element]*hierarchy.Node { + index := make(map[*hierarchy.Element]*hierarchy.Node) + if tree == nil || tree.Root == nil { + return index + } + var walk func(node *hierarchy.Node) + walk = func(node *hierarchy.Node) { + index[&node.Element] = node + for _, child := range node.Children { + walk(child) + } + } + walk(tree.Root) + return index +} + +// visibleLabel names a control by what a user would read: its own text, then +// description, then a field hint, then text borrowed from its descendants (the +// case that fixes empty-text Compose buttons whose word lives on a child), then +// its class as a last resort. +func visibleLabel(element *hierarchy.Element, nodeIndex map[*hierarchy.Element]*hierarchy.Node) string { + // An editable field's own text is the transient typed value ("1"); its hint + // names its purpose ("Amount") and stays stable, so prefer the hint there. + if element.Editable { + if hint := element.Attributes["hintText"]; hint != "" { + return truncateLabel(hint) + } + } + if element.Text != "" { + return truncateLabel(element.Text) + } + if element.Description != "" { + return truncateLabel(element.Description) + } + if hint := element.Attributes["hintText"]; hint != "" { + return truncateLabel(hint) + } + if node := nodeIndex[element]; node != nil { + if text := descendantText(node); text != "" { + return truncateLabel(text) + } + } + if element.Class != "" { + return element.Class + } + if element.ResourceID != "" { + return element.ResourceID + } + return "control" +} + +// descendantText joins the visible text of a node's descendants in tree order, +// so a clickable wrapper borrows the label of the Text child it contains. +func descendantText(node *hierarchy.Node) string { + var parts []string + var walk func(node *hierarchy.Node) + walk = func(node *hierarchy.Node) { + for _, child := range node.Children { + switch { + case child.Element.Text != "": + parts = append(parts, child.Element.Text) + case child.Element.Description != "": + parts = append(parts, child.Element.Description) + } + walk(child) + } + } + walk(node) + return strings.Join(parts, " ") +} + +// truncateLabel trims and shortens a label to one line's worth of runes. +func truncateLabel(text string) string { + text = strings.TrimSpace(strings.ReplaceAll(text, "\n", " ")) + runes := []rune(text) + if len(runes) <= maxLabelRunes { + return text + } + return strings.TrimSpace(string(runes[:maxLabelRunes])) + "…" +} + +// inputTypeHint reports a typing field's expected input as a short word the +// model can use to synthesize a value, or "" when nothing distinguishes it. +func inputTypeHint(element *hierarchy.Element) string { + haystack := strings.ToLower(element.Class + " " + + element.Attributes["inputType"] + " " + element.Attributes["hintText"]) + switch { + case strings.Contains(haystack, "number") || strings.Contains(haystack, "amount") || strings.Contains(haystack, "numeric"): + return "number" + case strings.Contains(haystack, "email"): + return "email" + case strings.Contains(haystack, "password"): + return "password" + case strings.Contains(haystack, "phone"): + return "phone" + default: + return "" + } +} + +// hasEnabled reports whether the source tree carried an explicit enabled flag +// for the element, so a missing flag is not mistaken for "disabled". +func hasEnabled(element *hierarchy.Element) bool { + _, ok := element.Attributes["enabled"] + return ok +} + +// stringField reads a string property off a goja object, returning "" when +// absent, null, or undefined. +func stringField(object *goja.Object, key string) string { + value := object.Get(key) + if value == nil || goja.IsUndefined(value) || goja.IsNull(value) { + return "" + } + return value.String() +} diff --git a/internal/verifier/llm_test.go b/internal/verifier/llm_test.go new file mode 100644 index 0000000..f689449 --- /dev/null +++ b/internal/verifier/llm_test.go @@ -0,0 +1,349 @@ +package verifier + +import ( + "strings" + "testing" + + "github.com/priyanshujain/sanderling/internal/hierarchy" +) + +// enumTreeJSON exercises every labeling path: a clickable wrapper whose own text +// is empty but whose child Text reads "Add credit" (descendant borrowing), an +// editable field labeled by its hint, a text-labeled button, a DISABLED button, +// and a scrollable list (the only valid gesture origin). +const enumTreeJSON = `{ + "attributes": {"bounds": "[0,0,1080,2400]"}, + "children": [ + {"attributes": {"resource-id": "AddCredit", "bounds": "[0,100,1080,200]"}, "clickable": true, "enabled": true, "children": [ + {"attributes": {"text": "Add credit", "bounds": "[0,100,540,200]"}, "children": []} + ]}, + {"attributes": {"resource-id": "Amount", "class": "EditText", "hintText": "Amount", "bounds": "[0,300,1080,400]"}, "enabled": true, "children": []}, + {"attributes": {"resource-id": "SignIn", "text": "Sign in", "bounds": "[0,450,1080,550]"}, "clickable": true, "enabled": true, "children": []}, + {"attributes": {"resource-id": "Off", "text": "Off", "bounds": "[0,600,1080,700]"}, "clickable": true, "enabled": false, "children": []}, + {"attributes": {"resource-id": "List", "scrollable": "true", "bounds": "[0,800,1080,2000]"}, "children": []} + ] +}` + +// enumVerifier loads a spec whose actions root is the given plain-object graph +// and stages the given tree, so Candidates walks a controlled action tree. +func enumVerifier(t *testing.T, actionsJS, treeJSON string) *Verifier { + t.Helper() + v := newLoadedVerifier(t, "globalThis.actions = "+actionsJS+";") + tree, err := hierarchy.Parse(treeJSON) + if err != nil { + t.Fatalf("parse tree: %v", err) + } + v.lastTree = tree + return v +} + +func findCandidate(candidates []ActionCandidate, description string) (ActionCandidate, bool) { + for _, candidate := range candidates { + if candidate.Description == description { + return candidate, true + } + } + return ActionCandidate{}, false +} + +func hasCandidate(candidates []ActionCandidate, description string) bool { + _, ok := findCandidate(candidates, description) + return ok +} + +func TestCandidatesLabelsControlsByVisibleText(t *testing.T) { + v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", enumTreeJSON) + candidates := v.Candidates() + + // The empty-text clickable wrapper is labeled by its child Text, NOT its + // resource-id. + if !hasCandidate(candidates, `Tap "Add credit"`) { + t.Errorf("want Tap \"Add credit\" (descendant text), got %v", descriptions(candidates)) + } + // The plain text button is labeled by its own text. + if !hasCandidate(candidates, `Tap "Sign in"`) { + t.Errorf("want Tap \"Sign in\", got %v", descriptions(candidates)) + } + // Descriptions are never the opaque resource-id. + if hasCandidate(candidates, `Tap "AddCredit"`) { + t.Error("labeled a control by its resource-id instead of visible text") + } + // Indices are dense and 1-based. + for i, candidate := range candidates { + if candidate.Index != i+1 { + t.Errorf("candidate %d has Index %d, want %d", i, candidate.Index, i+1) + } + } +} + +func TestCandidatesDropsDisabledControls(t *testing.T) { + v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", enumTreeJSON) + for _, candidate := range v.Candidates() { + if strings.Contains(candidate.Description, "Off") { + t.Errorf("disabled control surfaced as %q", candidate.Description) + } + } +} + +func TestCandidatesTypingExposesInputType(t *testing.T) { + v := enumVerifier(t, "{kind:'builtin', verb:'typing'}", enumTreeJSON) + candidates := v.Candidates() + candidate, ok := findCandidate(candidates, `Type into "Amount" (number)`) + if !ok { + t.Fatalf("want typing candidate with input type, got %v", descriptions(candidates)) + } + if !candidate.LLMText { + t.Error("builtin typing must flag LLMText so the model supplies the value") + } + if candidate.InputType != "number" { + t.Errorf("InputType = %q, want number", candidate.InputType) + } +} + +func TestCandidatesLabelsEditableFieldByHintNotTypedValue(t *testing.T) { + // A field already showing "99" must still be labeled by its purpose (the + // hint), not by its transient content, so the description stays stable. + tree := `{ + "attributes": {"bounds": "[0,0,400,800]"}, + "children": [ + {"attributes": {"resource-id": "Amt", "class": "EditText", "hintText": "Amount", "text": "99", "bounds": "[0,0,400,100]"}, "enabled": true, "children": []} + ] + }` + v := enumVerifier(t, "{kind:'builtin', verb:'typing'}", tree) + candidates := v.Candidates() + if hasCandidate(candidates, `Type into "99" (number)`) || hasCandidate(candidates, `Type into "99"`) { + t.Errorf("editable field labeled by its typed value: %v", descriptions(candidates)) + } + if !hasCandidate(candidates, `Type into "Amount" (number)`) { + t.Errorf("want the field labeled by its hint, got %v", descriptions(candidates)) + } +} + +func TestCandidatesFoldsGesturesIntoDirectionalScrolls(t *testing.T) { + v := enumVerifier(t, + "{kind:'weighted', branches:[[1,{kind:'builtin',verb:'scrolls'}],[1,{kind:'builtin',verb:'swipes'}]]}", + enumTreeJSON) + candidates := v.Candidates() + + // Gestures are directional and scoped to the one scrollable container: no + // per-element, element-labeled Swipe entries. + for _, candidate := range candidates { + if strings.HasPrefix(candidate.Description, "Swipe") { + t.Errorf("gesture kept as element-labeled swipe: %q", candidate.Description) + } + } + if !hasCandidate(candidates, "Scroll down") || !hasCandidate(candidates, "Scroll up") { + t.Errorf("want directional scrolls, got %v", descriptions(candidates)) + } + // scrolls and swipes fold into the SAME directional entries: one each. + if got := count(candidates, "Scroll down"); got != 1 { + t.Errorf("Scroll down appears %d times, want 1 (folded)", got) + } +} + +func TestCandidatesWeightsCombineAcrossPaths(t *testing.T) { + // A single clickable reached through two equal branches: its weight sums to + // the full distribution. + oneClickable := `{ + "attributes": {"bounds": "[0,0,400,800]"}, + "children": [ + {"attributes": {"resource-id": "SignIn", "text": "Sign in", "bounds": "[0,0,400,100]"}, "clickable": true, "enabled": true, "children": []} + ] + }` + v := enumVerifier(t, + "{kind:'weighted', branches:[[1,{kind:'builtin',verb:'taps'}],[1,{kind:'builtin',verb:'taps'}]]}", + oneClickable) + candidates := v.Candidates() + if len(candidates) != 1 { + t.Fatalf("want one deduped candidate, got %v", descriptions(candidates)) + } + candidate := candidates[0] + if !candidate.Weighted { + t.Fatal("candidate under a weighted tree must be Weighted") + } + if candidate.Weight != 100 { + t.Errorf("summed weight = %d, want 100", candidate.Weight) + } +} + +func TestCandidatesWeightReflectsBranchShare(t *testing.T) { + v := enumVerifier(t, + "{kind:'weighted', branches:[[1,{kind:'builtin',verb:'taps'}],[3,{kind:'builtin',verb:'typing'}]]}", + enumTreeJSON) + candidates := v.Candidates() + tap, ok := findCandidate(candidates, `Tap "Sign in"`) + if !ok { + t.Fatalf("missing tap candidate: %v", descriptions(candidates)) + } + if tap.Weight != 25 { + t.Errorf("tap weight = %d, want 25 (1/4 share)", tap.Weight) + } + typing, ok := findCandidate(candidates, `Type into "Amount" (number)`) + if !ok { + t.Fatalf("missing typing candidate: %v", descriptions(candidates)) + } + if typing.Weight != 75 { + t.Errorf("typing weight = %d, want 75 (3/4 share)", typing.Weight) + } +} + +func TestCandidatesUnweightedTreeShowsNoWeight(t *testing.T) { + v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", enumTreeJSON) + for _, candidate := range v.Candidates() { + if candidate.Weighted || candidate.Weight != 0 { + t.Errorf("%q carries a weight despite no weighted node", candidate.Description) + } + } +} + +func TestCandidatesCallsAuthoredLeafOnce(t *testing.T) { + actions := `{kind:'actions', generate: () => [ + {kind:'Tap', on:'id:SignIn'}, + {kind:'Tap', on:'id:Off'}, + {kind:'InputText', into:'id:Amount', text:'42'} + ]}` + v := enumVerifier(t, actions, enumTreeJSON) + candidates := v.Candidates() + + // Authored Tap resolves its selector to the visible-text label. + if !hasCandidate(candidates, `Tap "Sign in"`) { + t.Errorf("authored tap missing: %v", descriptions(candidates)) + } + // A disabled authored target is dropped. + for _, candidate := range candidates { + if strings.Contains(candidate.Description, "Off") { + t.Errorf("authored action on disabled control surfaced: %q", candidate.Description) + } + } + // Authored InputText replays its own sampled value (LLM does not supply it). + authored, ok := findCandidate(candidates, `Type "42" into "Amount"`) + if !ok { + t.Fatalf("authored typing missing: %v", descriptions(candidates)) + } + if authored.LLMText { + t.Error("authored InputText must not request an LLM-supplied value") + } + if authored.Action.Text != "42" { + t.Errorf("authored text = %q, want 42", authored.Action.Text) + } +} + +func TestCandidatesOffRouteLeafYieldsNothing(t *testing.T) { + v := enumVerifier(t, "{kind:'actions', generate: () => []}", enumTreeJSON) + if got := v.Candidates(); len(got) != 0 { + t.Errorf("off-route leaf should yield no candidates, got %v", descriptions(got)) + } +} + +func TestCandidatesSkipsCrossFadeFrames(t *testing.T) { + // Two route *Screen tags alive at once is a NavHost cross-fade: its layout is + // mid-animation (collapsed coordinate space), so the LLM must NOT act on it. + crossFade := `{ + "attributes": {"bounds": "[0,0,320,640]"}, + "children": [ + {"attributes": {"resource-id": "LedgerScreen", "bounds": "[0,0,320,640]"}, "children": [ + {"attributes": {"resource-id": "TxnSubmit", "text": "Add credit", "bounds": "[20,332,300,380]"}, "clickable": true, "enabled": true, "children": []} + ]}, + {"attributes": {"resource-id": "AddTransactionScreen", "bounds": "[0,0,320,640]"}, "children": []} + ] + }` + v := enumVerifier(t, "{kind:'builtin', verb:'taps'}", crossFade) + if got := v.Candidates(); got != nil { + t.Errorf("cross-fade frame should yield no candidates, got %v", descriptions(got)) + } +} + +func TestCandidatesNilWithoutTreeOrActions(t *testing.T) { + withActions := newLoadedVerifier(t, "globalThis.actions = {kind:'builtin', verb:'taps'};") + if got := withActions.Candidates(); got != nil { + t.Errorf("Candidates with no tree = %v, want nil", got) + } + noActions := newLoadedVerifier(t, "globalThis.properties = {};") + tree, _ := hierarchy.Parse(enumTreeJSON) + noActions.lastTree = tree + if got := noActions.Candidates(); got != nil { + t.Errorf("Candidates with no actions root = %v, want nil", got) + } +} + +func descriptions(candidates []ActionCandidate) []string { + out := make([]string, len(candidates)) + for i, candidate := range candidates { + out[i] = candidate.Description + } + return out +} + +func count(candidates []ActionCandidate, description string) int { + n := 0 + for _, candidate := range candidates { + if candidate.Description == description { + n++ + } + } + return n +} + +func TestLLMConfigDetectsMarker(t *testing.T) { + v := newLoadedVerifier(t, `globalThis.generator = { kind: "llm", config: { model: "vendor/model" } };`) + config, ok := v.LLMConfig() + if !ok { + t.Fatal("LLMConfig not detected for llm marker") + } + if config.Model != "vendor/model" { + t.Errorf("model = %q, want vendor/model", config.Model) + } + if config.Instructions != "" { + t.Errorf("instructions = %q, want empty when unset", config.Instructions) + } +} + +func TestLLMConfigReadsInstructions(t *testing.T) { + v := newLoadedVerifier(t, `globalThis.generator = { kind: "llm", config: { model: "m", instructions: "find bugs" } };`) + config, ok := v.LLMConfig() + if !ok { + t.Fatal("LLMConfig not detected") + } + if config.Instructions != "find bugs" { + t.Errorf("instructions = %q, want %q", config.Instructions, "find bugs") + } +} + +func TestLLMConfigAbsentForSeededSpec(t *testing.T) { + v := newLoadedVerifier(t, `globalThis.actions = { kind: "builtin", verb: "taps" };`) + if _, ok := v.LLMConfig(); ok { + t.Error("LLMConfig should be false when no generator is declared") + } +} + +func TestSampleInputErrorsWithoutBundle(t *testing.T) { + v := newLoadedVerifier(t, `globalThis.actions = { kind: "llm", config: { model: "m" } };`) + if _, err := v.SampleInput(); err == nil { + t.Error("expected SampleInput to error when the sampler is not installed") + } +} + +func TestSampleInputDrawsFromCorpus(t *testing.T) { + v := newLoadedVerifier(t, `globalThis.__sanderlingSampleInput__ = () => "sampled";`) + got, err := v.SampleInput() + if err != nil { + t.Fatalf("SampleInput: %v", err) + } + if got != "sampled" { + t.Errorf("SampleInput = %q, want sampled", got) + } +} + +func newLoadedVerifier(t *testing.T, source string) *Verifier { + t.Helper() + v, err := New() + if err != nil { + t.Fatalf("New: %v", err) + } + if err := v.Load(source); err != nil { + t.Fatalf("Load: %v", err) + } + return v +} + + diff --git a/internal/verifier/setup_action_test.go b/internal/verifier/setup_action_test.go new file mode 100644 index 0000000..d0bee1a --- /dev/null +++ b/internal/verifier/setup_action_test.go @@ -0,0 +1,95 @@ +package verifier + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/priyanshujain/sanderling/internal/bundler" + "github.com/priyanshujain/sanderling/internal/hierarchy" +) + +// bundleInlineSpec bundles an inline spec through the real @sanderling/spec API +// and goja runtime entry, so the bundle installs __sanderlingSetupAction__ the +// way the CLI does. +func bundleInlineSpec(t *testing.T, source string) string { + t.Helper() + dir := t.TempDir() + specPath := filepath.Join(dir, "spec.ts") + if err := os.WriteFile(specPath, []byte(source), 0o600); err != nil { + t.Fatal(err) + } + abs := func(rel string) string { + path, err := filepath.Abs(rel) + if err != nil { + t.Fatal(err) + } + return path + } + bundle, err := bundler.Bundle(bundler.Options{ + EntryFile: specPath, + RuntimeFile: abs("../../pkg/spec/src/goja-runtime.ts"), + Aliases: map[string]string{ + "@sanderling/spec": abs("../../pkg/spec/src/index.ts"), + "@sanderling/spec/defaults": abs("../../pkg/spec/src/defaults/index.ts"), + "@sanderling/spec/defaults/properties": abs("../../pkg/spec/src/defaults/properties.ts"), + }, + }) + if err != nil { + t.Fatalf("bundle: %v", err) + } + return string(bundle.JavaScript) +} + +func loadBundled(t *testing.T, source, treeJSON string) *Verifier { + t.Helper() + v, err := New() + if err != nil { + t.Fatal(err) + } + if err := v.Load(bundleInlineSpec(t, source)); err != nil { + t.Fatalf("load: %v", err) + } + tree, err := hierarchy.Parse(treeJSON) + if err != nil { + t.Fatal(err) + } + if err := v.PushSnapshot(SnapshotInput{Tree: tree}); err != nil { + t.Fatalf("push snapshot: %v", err) + } + return v +} + +func TestSetupActionWalksSetupOnly(t *testing.T) { + spec := ` +import { Tap, actions, taps } from "@sanderling/spec"; +export const setup = actions(() => [Tap({ on: "id:SignIn" })]); +export const actionsRoot = taps; +` + v := loadBundled(t, spec, enumTreeJSON) + action, err := v.SetupAction() + if err != nil { + t.Fatalf("SetupAction: %v", err) + } + if action.Kind != ActionKindTap || action.On != "id:SignIn" { + t.Errorf("SetupAction = %+v, want Tap on id:SignIn from setup", action) + } +} + +func TestSetupActionIgnoresActionsRoot(t *testing.T) { + // No setup, but a live actionsRoot that NextAction would happily draw from. + spec := ` +import { taps } from "@sanderling/spec"; +export const actionsRoot = taps; +` + v := loadBundled(t, spec, enumTreeJSON) + if _, err := v.SetupAction(); !errors.Is(err, ErrNoAction) { + t.Fatalf("SetupAction err = %v, want ErrNoAction when no setup is declared", err) + } + // Sanity: the seeded root would have produced an action, proving SetupAction + // deliberately skips it. + if _, err := v.NextAction(); err != nil { + t.Fatalf("NextAction should draw from actionsRoot: %v", err) + } +} diff --git a/internal/verifier/worker.go b/internal/verifier/worker.go index bcbb28b..7284033 100644 --- a/internal/verifier/worker.go +++ b/internal/verifier/worker.go @@ -28,6 +28,17 @@ type Verifier struct { // the shared picker (pick.ts) over the shared Pcg. nextActionFn goja.Callable + // setupActionFn is the bundle-installed __sanderlingSetupAction__, which + // walks ONLY the setup generator. The LLM action generator runs it for setup + // precedence (e.g. login) without triggering the seeded action root. + setupActionFn goja.Callable + + // sampleInputFn is the bundle-installed __sanderlingSampleInput__, which + // draws one value from the shared INPUT_CORPUS. The LLM action backend uses + // it to fill InputText values, reusing the exact corpus draw rather than + // reimplementing the corpus on the Go side. + sampleInputFn goja.Callable + evaluators map[string]*ltl.Evaluator priorVerdicts map[string]ltl.Verdict @@ -35,6 +46,7 @@ type Verifier struct { witnesses map[string]Witness lastTree *hierarchy.Tree + lastScreenshot []byte scopeCache map[*hierarchy.Element]bool scopeCacheTree *hierarchy.Tree lastAction *Action @@ -151,6 +163,20 @@ func (v *Verifier) Load(source string) error { v.nextActionFn = callable } } + if fn := v.runtime.GlobalObject().Get("__sanderlingSetupAction__"); fn != nil { + if callable, ok := goja.AssertFunction(fn); ok { + v.setupActionFn = callable + } + } + + // __sanderlingSampleInput__ draws an InputText value from the shared corpus. + // The LLM action backend uses it; a raw-JS fixture without the runtime entry + // leaves it nil and SampleInput reports an error. + if fn := v.runtime.GlobalObject().Get("__sanderlingSampleInput__"); fn != nil { + if callable, ok := goja.AssertFunction(fn); ok { + v.sampleInputFn = callable + } + } return nil } @@ -258,6 +284,7 @@ func (v *Verifier) buildFormulaNode(index int) (ltl.Formula, error) { // allowed and yields an empty ax scope. func (v *Verifier) PushSnapshot(input SnapshotInput) error { v.lastTree = input.Tree + v.lastScreenshot = input.ScreenshotPNG v.scopeCache = nil v.lastAction = input.LastAction v.lastLogs = input.Logs @@ -386,9 +413,13 @@ func (v *Verifier) OverrideExtractorValues(overrides map[int]json.RawMessage) (s // other than Snapshots are optional; callers that only have snapshots can // populate Snapshots alone and leave the rest zero. type SnapshotInput struct { - Snapshots Snapshots - Tree *hierarchy.Tree - LastAction *Action + Snapshots Snapshots + Tree *hierarchy.Tree + // ScreenshotPNG is the step's screenshot, captured alongside Tree. The LLM + // action backend reads it via Screenshot() to select a candidate; other + // callers may leave it nil. + ScreenshotPNG []byte + LastAction *Action StepTime time.Time // StepIndex is the runner's step number for this snapshot. Evaluators label // observations with it so violation witnesses carry runner step numbers even @@ -575,6 +606,28 @@ func (v *Verifier) NextAction() (Action, error) { return DecodeAction(raw) } +// SetupAction walks ONLY the setup generator (globalThis.setup), returning its +// action or ErrNoAction. The LLM action generator runs this for setup +// precedence (e.g. login) without triggering the seeded action root, which it +// replaces entirely. Mirrors NextAction's decode. +func (v *Verifier) SetupAction() (Action, error) { + if v.setupActionFn == nil { + return Action{}, ErrNoAction + } + value, err := v.setupActionFn(goja.Undefined()) + if err != nil { + return Action{}, fmt.Errorf("setup action: %w", err) + } + if value == nil || goja.IsNull(value) || goja.IsUndefined(value) { + return Action{}, ErrNoAction + } + raw, err := json.Marshal(value.Export()) + if err != nil { + return Action{}, fmt.Errorf("marshal action: %w", err) + } + return DecodeAction(raw) +} + var ErrNoAction = errors.New("verifier: no action available") func (v *Verifier) formulaThunk(index int) func() (bool, error) { diff --git a/pkg/spec/src/action-tree.ts b/pkg/spec/src/action-tree.ts index 910b149..d34b73c 100644 --- a/pkg/spec/src/action-tree.ts +++ b/pkg/spec/src/action-tree.ts @@ -57,10 +57,13 @@ export type ActionDescriptor = // weighted: probabilistic choice over child nodes, scanned ascending. // actions: author callback returning a list to pick uniformly from. // builtin: host-backed leaf identified by a verb. +// llm: marker selecting the LLM action backend; inert on the JS picker +// (walk returns null), Go reads config.model off globalThis.actions. export type GeneratorNode = | { kind: "weighted"; branches: ReadonlyArray } | { kind: "actions"; generate: () => ActionDescriptor[] } - | { kind: "builtin"; verb: BuiltinVerb }; + | { kind: "builtin"; verb: BuiltinVerb } + | { kind: "llm"; config: { model: string; instructions?: string } }; // Candidate is one host-enumerated target for a builtin verb. The host // resolves geometry (and a native selector) so no element handle crosses into diff --git a/pkg/spec/src/actions.ts b/pkg/spec/src/actions.ts index 5e11445..317fe16 100644 --- a/pkg/spec/src/actions.ts +++ b/pkg/spec/src/actions.ts @@ -34,6 +34,18 @@ export function actions(generator: () => Action[]): GeneratorNode { return { kind: "actions", generate: generator as () => ActionDescriptor[] }; } +// llm selects the LLM action backend: instead of the seeded picker drawing a +// random candidate, Go drives an OpenAI-compatible model that picks one +// candidate from the screenshot + the numbered candidate list. Assign it to the +// `generator` export; Go reads the config off globalThis.generator and the +// marker is inert on the JS picker (pick.ts walks it to null). API key comes +// from OPENROUTER_API_KEY (OpenRouter) or OPENAI_API_KEY (OpenAI); OpenRouter +// wins when both are set. Optional `instructions` are appended to the model's +// prompt to describe the app under test. +export function llm(config: { model: string; instructions?: string }): GeneratorNode { + return { kind: "llm", config }; +} + export function whenRoute( routeExtractor: { readonly current: string | null }, routes: string | readonly string[], diff --git a/pkg/spec/src/index.ts b/pkg/spec/src/index.ts index 809f4bf..4ac8c31 100644 --- a/pkg/spec/src/index.ts +++ b/pkg/spec/src/index.ts @@ -42,6 +42,7 @@ export { actions, doubleTaps, from, + llm, longPresses, pressKeys, scrolls, diff --git a/pkg/spec/src/pick.ts b/pkg/spec/src/pick.ts index 97c7b4f..f2019e9 100644 --- a/pkg/spec/src/pick.ts +++ b/pkg/spec/src/pick.ts @@ -86,6 +86,12 @@ export function walk( } case "builtin": return walkBuiltin(node.verb, rng, host); + case "llm": + // The LLM backend is driven by Go (it reads config.model off + // globalThis.actions and selects via OpenRouter). On the JS picker the + // marker is inert, so the goja NextAction reports no action and the Go + // llmSource takes over. + return null; } } diff --git a/pkg/spec/src/runtime-entry.ts b/pkg/spec/src/runtime-entry.ts index bc5fc83..95b0293 100644 --- a/pkg/spec/src/runtime-entry.ts +++ b/pkg/spec/src/runtime-entry.ts @@ -9,6 +9,7 @@ import { Pcg } from "./pcg.ts"; import { nextAction, walk } from "./pick.ts"; +import { INPUT_CORPUS } from "./corpus.ts"; import type { ActionDescriptor, GeneratorNode, Host } from "./action-tree.ts"; import type { Point } from "./types.ts"; @@ -130,7 +131,24 @@ export function installRuntime( const resolveRoot = typeof root === "function" ? root : () => root; const resolveSetup = () => (globalThis as { setup?: GeneratorNode }).setup ?? null; + // The LLM action backend (Go) types InputText values by drawing from the same + // edge-case corpus the seeded `typing` builtin uses. Expose that draw here so + // Go reuses the exact sampler rather than reimplementing the corpus. + defineLockedGlobal( + "__sanderlingSampleInput__", + () => INPUT_CORPUS[rng.intN(INPUT_CORPUS.length)] ?? "", + ); defineLockedGlobal("__sanderlingExtractors__", () => evaluateExtractors()); + // __sanderlingSetupAction__ walks ONLY the setup generator once, for the LLM + // action generator (Go), which drives selection itself and must not run the + // seeded action root — but still wants setup's precondition steps (e.g. login) + // to run first. Returns null when setup is unset or yields nothing. + defineLockedGlobal("__sanderlingSetupAction__", () => { + resolveRoot(); + const setup = resolveSetup(); + if (!setup) return null; + return serializeAction(walk(setup, rng, host)); + }); defineLockedGlobal("__sanderlingNextAction__", () => { // resolveRoot runs first: on web it also resets the per-tick candidate // cache, which setup's walk below must see fresh. diff --git a/replay-ui/src/__tests__/device-space.test.ts b/replay-ui/src/__tests__/device-space.test.ts index db3f453..0d6f927 100644 --- a/replay-ui/src/__tests__/device-space.test.ts +++ b/replay-ui/src/__tests__/device-space.test.ts @@ -44,4 +44,18 @@ describe("deviceSpaceOf", () => { }; expect(deviceSpaceOf(hierarchy)).toEqual({ width: 402, height: 874 }); }); + + it("uses the screen extent, not a short status-bar node listed first", () => { + // Regression: a 320x24 status bar precedes the 320x640 screen. Picking the + // first positive-bounds element gave a 320/24 aspect ratio, squashing the + // screenshot overlay into a grey horizontal band. + const hierarchy: Hierarchy = { + elements: [ + { bounds: { left: 0, top: 0, right: 320, bottom: 24 } }, + { bounds: { left: 0, top: 0, right: 320, bottom: 640 } }, + { bounds: { left: 20, top: 271, right: 286, bottom: 319 } }, + ], + }; + expect(deviceSpaceOf(hierarchy)).toEqual({ width: 320, height: 640 }); + }); }); diff --git a/replay-ui/src/__tests__/run-history.test.ts b/replay-ui/src/__tests__/run-history.test.ts index e9bc1d3..1a2dc85 100644 --- a/replay-ui/src/__tests__/run-history.test.ts +++ b/replay-ui/src/__tests__/run-history.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from "bun:test"; import { buildRunHistory, collectPropertyNames, + relocateViolationsToCause, sortLanes, statusForProperty, } from "../lib/run-history"; @@ -52,7 +53,67 @@ describe("sortLanes", () => { }); }); +// Bug class: a next/eventually violation records on the DETECTION step but is +// caused earlier; leaving it on the detection step lights up an unrelated +// action in the Violations tab while the timeline dot sits on the cause step. +describe("relocateViolationsToCause", () => { + it("moves a deferred violation to the step its witness blames", () => { + const steps = [ + step({ step: 290, residuals: { p: { op: "predicate", name: "p3" } } }), + step({ + step: 291, + violations: ["p"], + witnesses: { p: { step: 290, reason: "predicate false" } }, + residuals: { p: { op: "false" } }, + }), + step({ step: 292 }), + ]; + const moved = relocateViolationsToCause(steps); + + expect(moved[0]?.violations).toEqual(["p"]); + expect(moved[0]?.witnesses?.p?.reason).toBe("predicate false"); + expect(moved[1]?.violations).toEqual([]); + expect(moved[1]?.witnesses?.p).toBeUndefined(); + // originals are cloned, never mutated + expect(steps[1]?.violations).toEqual(["p"]); + expect(steps[1]?.witnesses?.p?.step).toBe(290); + }); + + it("leaves a violation without a witness on its detection step", () => { + const steps = [step({ step: 5, violations: ["p"], residuals: { p: { op: "false" } } })]; + expect(relocateViolationsToCause(steps)).toBe(steps); + }); + + it("keeps the violation put when the blamed step is absent from the trace", () => { + const steps = [step({ step: 9, violations: ["p"], witnesses: { p: { step: 3 } } })]; + expect(relocateViolationsToCause(steps)[0]?.violations).toEqual(["p"]); + }); +}); + describe("buildRunHistory", () => { + it("anchors a deferred violation's lane cell to the cause step, not detection", () => { + const run = { + id: "run-2", + steps: [summary({ index: 290, has_violations: true }), summary({ index: 291 })], + } as unknown as Run; + const responses = [ + step({ step: 290, residuals: { p: { op: "predicate" } } }), + step({ + step: 291, + violations: ["p"], + witnesses: { p: { step: 290 } }, + residuals: { p: { op: "false" } }, + }), + ]; + + const history = buildRunHistory(run, responses); + + expect(history.lanes[0].statuses).toEqual(["violated", "pending"]); + expect(history.steps[0]?.violations).toEqual(["p"]); + expect(history.steps[1]?.violations).toEqual([]); + expect(history.firstViolationStep).toBe(290); + }); + it("aligns lane statuses, metrics samples, and first-violation index by position", () => { const run = { id: "run-1", diff --git a/replay-ui/src/lib/device-space.ts b/replay-ui/src/lib/device-space.ts index 5c89828..cfd90c6 100644 --- a/replay-ui/src/lib/device-space.ts +++ b/replay-ui/src/lib/device-space.ts @@ -3,18 +3,23 @@ import type { Hierarchy } from "../types"; // Tap points and resolved bounds in the trace share the hierarchy root's // coordinate space (iOS points, Android pixels, web CSS px). Screenshots may // be scaled (iOS 3x, web DPR>1), so the overlay viewBox must come from the -// root bounds, not the image's natural pixel size. Elements are in pre-order; -// the first one with positive extent is the root window (iOS prepends a -// synthetic zero-bounds node, so plain elements[0] is not enough). +// device bounds, not the image's natural pixel size. +// +// Use the maximum extent across all elements (mirroring the runner's +// screenBounds), NOT the first positive-bounds element: the first element is +// often a status-bar/decor node a few px tall (e.g. 320x24 on Android), which +// would give a wildly wrong aspect ratio and squash the overlay into a band. export function deviceSpaceOf( hierarchy?: Hierarchy, ): { width: number; height: number } | undefined { + let width = 0; + let height = 0; for (const element of hierarchy?.elements ?? []) { - const width = element.bounds.right; - const height = element.bounds.bottom; - if (width > 0 && height > 0) { - return { width, height }; - } + if (element.bounds.right > width) width = element.bounds.right; + if (element.bounds.bottom > height) height = element.bounds.bottom; + } + if (width > 0 && height > 0) { + return { width, height }; } return undefined; } diff --git a/replay-ui/src/lib/run-history.ts b/replay-ui/src/lib/run-history.ts index 5103100..09e925f 100644 --- a/replay-ui/src/lib/run-history.ts +++ b/replay-ui/src/lib/run-history.ts @@ -14,6 +14,55 @@ export interface RunHistory { steps: (Step | null)[]; } +// A next/eventually obligation is evaluated one or more steps AFTER the action +// that armed it, so the checker records the violation on the DETECTION step +// while its witness names the CAUSE step. The timeline dot (the backend's +// markViolations) already sits on the cause step; mirror that here so the +// Violations tab and property lanes light up on the same step — the guilty +// action — not the unrelated action that happened to be running when the +// obligation resolved. Steps are cloned, never mutated in place. +export function relocateViolationsToCause( + steps: (Step | null)[], +): (Step | null)[] { + const byIndex = new Map(); + for (const s of steps) if (s) byIndex.set(s.step, s); + + const clones = new Map(); + const clone = (s: Step): Step => { + let c = clones.get(s.step); + if (!c) { + c = { + ...s, + violations: [...(s.violations ?? [])], + witnesses: { ...(s.witnesses ?? {}) }, + }; + clones.set(s.step, c); + } + return c; + }; + + for (const s of steps) { + for (const name of s?.violations ?? []) { + const cause = s?.witnesses?.[name]?.step; + if (cause === undefined || cause === s?.step) continue; + const target = byIndex.get(cause); + if (!target) continue; + const from = clone(s as Step); + const to = clone(target); + from.violations = (from.violations ?? []).filter((n) => n !== name); + const witness = s?.witnesses?.[name]; + if (witness) { + delete from.witnesses?.[name]; + (to.witnesses ??= {})[name] = witness; + } + if (!(to.violations ?? []).includes(name)) (to.violations ??= []).push(name); + } + } + + if (clones.size === 0) return steps; + return steps.map((s) => (s ? clones.get(s.step) ?? s : s)); +} + export function collectPropertyNames(steps: (Step | null)[]): string[] { const names = new Set(); for (const step of steps) { @@ -47,10 +96,11 @@ export function buildRunHistory( run: Run, responses: (Step | null)[], ): RunHistory { - const propertyNames = collectPropertyNames(responses); + const steps = relocateViolationsToCause(responses); + const propertyNames = collectPropertyNames(steps); const lanes: PropertyLane[] = propertyNames.map((name) => ({ name, - statuses: responses.map((step) => statusForProperty(name, step)), + statuses: steps.map((step) => statusForProperty(name, step)), })); const firstViolationStep = run.steps.find((entry) => entry.has_violations)?.index; const firstExceptionStep = run.steps.find((entry) => entry.has_exceptions)?.index; @@ -63,7 +113,7 @@ export function buildRunHistory( const metricsSamples: MetricsSample[] = run.steps.map((entry, position) => ({ stepIndex: entry.index, timestamp: entry.timestamp, - metrics: responses[position]?.metrics, + metrics: steps[position]?.metrics, })); return { names: propertyNames, @@ -73,6 +123,6 @@ export function buildRunHistory( exceptionStepIndices, violationStepIndices, metricsSamples, - steps: responses, + steps, }; } diff --git a/replay-ui/src/routes/RunDetail.tsx b/replay-ui/src/routes/RunDetail.tsx index 7529cee..c22f9b5 100644 --- a/replay-ui/src/routes/RunDetail.tsx +++ b/replay-ui/src/routes/RunDetail.tsx @@ -113,12 +113,13 @@ export default function RunDetail() { return
loading run...
; } - const violationsBefore = currentStep?.violations ?? []; - const violationsAfter = nextStep?.violations ?? violationsBefore; + // A violation belongs to the action that caused it (relocateViolationsToCause + // already anchored it to that step), so show this step's own violations in + // both panels rather than bleeding the next step's record into "after". + const stepViolations = currentStep?.violations ?? []; + const stepWitnesses = currentStep?.witnesses; const residualsBefore = currentStep?.residuals; const residualsAfter = nextStep?.residuals ?? residualsBefore; - const witnessesBefore = currentStep?.witnesses; - const witnessesAfter = nextStep?.witnesses ?? witnessesBefore; const exceptionsForStep = currentStep?.exceptions; const beforeSpace = deviceSpaceOf(currentStep?.hierarchy); const afterSpace = deviceSpaceOf(nextStep?.hierarchy ?? currentStep?.hierarchy); @@ -157,9 +158,9 @@ export default function RunDetail() { content: ( 0 ? ( + stepViolations.length > 0 ? ( - {violationsBefore.length} + {stepViolations.length} ) : undefined, content: ( 0 ? ( + stepViolations.length > 0 ? ( - {violationsAfter.length} + {stepViolations.length} ) : undefined, content: (