Files
sanderling/cmd/sanderling/main_test.go
T
pj 7343085614 llm action-selection backend (#68)
* feat(spec): add llm() action-backend marker

* feat(spec): make llm marker inert on the JS picker

* feat(spec): expose __sanderlingSampleInput__ corpus draw

* feat(openrouter): minimal chat-completions client

* test(openrouter): cover request shape, parse, and errors

* feat(verifier): thread screenshot + capture corpus sampler

* feat(verifier): LLM accessors — candidates, config, sampler

* test(verifier): cover AllCandidates, LLMConfig, SampleInput

* feat(trace): record action Source and LLMReasoning

* feat(runner): thread step screenshot into PushSnapshot

* feat(runner): llmSource selects actions via OpenRouter

* feat(runner): wire llmSource selection and trace stamping

* test(runner): cover llmSource selection, mapping, downscale

* docs(folio): add llm action-backend example spec

* docs(folio): document the LLM action backend run

* feat(llmclient): support OPENAI_API_KEY, openrouter wins

* refactor(runner): rename openrouter package to llmclient

* docs: both api keys, example model gpt-5.4-nano

* docs: add pr style rules to claude.md

* fix(runner): explain action kinds in llm prompt to stop swipe loops

* feat(trace): record llm ranked list and chosen rank

* feat(runner): stamp llm ranked list and chosen rank on trace

* fix(runner): tap by selector to survive layout shift after observe

* revert(runner): drop selector-first tap; broke path/testTag selectors

* feat(spec): llm() accepts optional instructions

* feat(verifier): read llm instructions off config

* feat(runner): append spec instructions to llm system prompt

* docs(folio): describe app in llm spec instructions

* feat(bundler): map generator export to globalThis.generator

* feat(verifier): read llm config off globalThis.generator

* feat(runner): gate llm source on --generator flag

* feat(cmd): add --generator llm|seeded flag

* test: cover --generator flag parsing and pickSources gating

* feat(verifier): enumerate llm candidates by walking actionsRoot

collect-walk the weighted action tree: recurse weighted branches
accumulating selection probability, call authored leaves once for
concrete actions, enumerate builtins per element. label controls by
visible text (borrowing descendant text), fold gestures into directional
scrolls over scrollable containers, drop disabled, dedup descriptions.

* test(verifier): cover candidate enumeration walk

* feat(verifier): add SetupAction to walk setup without the seeded root

* test(verifier): cover SetupAction setup-only precedence

* refactor(llmclient): make JSONSchema.Schema raw json for pinned field order

* feat(trace): record llm choice number and chosen_action echo

* feat(runner): llm picks one number from weighted candidates

drop the seeded-root call for a setup-only precedence path, render a
numbered weighted candidate list, pin a reasoning-first choice schema,
strict-skip when chosen_action does not echo the numbered entry, and let
the model supply typed values (corpus fallback when empty).

* test(runner): cover choice schema, strict-skip, and setup precedence

* refactor(verifier): drop the superseded AllCandidates enumeration

* feat(folio): drive spec.ts under --generator llm; drop spec-llm.ts

* fix(verifier): label editable fields by hint, not the typed value

an editable field's own text is its transient content; prefer the hint
so the field is named by purpose and the label stays stable.

* test(runner): cover weight-suffixed echo and stripWeightSuffix

* fix(runner): accept chosen_action echo that carries the weight suffix

real runs showed the model copies the whole numbered line including the
trailing (w34) weight annotation, so strict-skip rejected ~91% of picks
and the llm was paralyzed. strip the weight suffix before comparing. also
nudge the prompt to stress-test repeated submissions (idempotency).

* fix(verifier): skip llm enumeration on cross-fade frames

a navhost mid-transition carries >1 route *Screen in a collapsed
coordinate space; acting on it taps garbage (soft keyboard). real runs
showed the llm acting on 44% of steps being such frames. skip them so the
llm re-observes a settled frame next step.

* feat(folio): show current balance on the add-transaction screen

renders the account's balance (testTag TxnCurrentBalance) below the
account name, above the credit/debit toggle, so before/after screenshots
carry comparison data.

* fix(replay): derive device space from screen extent, not first node

the first positive-bounds element is often a short status-bar node
(320x24 on android); using it gave a 320/24 aspect ratio that squashed
the screenshot overlay into a grey horizontal band. use the max extent
across elements (like the runner's screenBounds) instead.

* fix(folio): show balance as a compact one-line label

per review: one line, account-name-sized, e.g. "Balance: $0.00"
instead of a large balance card.

* fix(folio): move balance into the header, one compact line under the account name

* fix(replay): attribute deferred violations to the causing step, not detection

* fix(replay): show a step's own violations in both panels, no next-step bleed

* refactor(hierarchy): one Tree.Transitional, drop the duplicated cross-fade check

* chore: ignore .playwright-mcp scratch output

* docs: document the llm generator and --generator flag

* docs(spec): correct the llm() comment; config reads off globalThis.generator

* docs: add pr description rules
2026-07-31 21:12:00 +05:30

262 lines
7.4 KiB
Go

package main
import (
"bytes"
"io"
"strings"
"testing"
"time"
)
func TestParseTestArgs_Defaults(t *testing.T) {
options, err := parseTestArgs([]string{
"--spec", "s.ts",
"--bundle-id", "com.example",
"--avd", "Pixel_5_API_33",
}, io.Discard)
if err != nil {
t.Fatal(err)
}
if options.spec != "s.ts" || options.bundleID != "com.example" {
t.Errorf("unexpected options: %+v", options)
}
if options.platform != "android" {
t.Errorf("platform default: got %q, want android", options.platform)
}
if options.duration != 5*time.Minute {
t.Errorf("duration default: got %v, want 5m", options.duration)
}
if options.output != "./runs" {
t.Errorf("output default: got %q, want ./runs", options.output)
}
if options.seed != 0 {
t.Errorf("seed default: got %d, want 0", options.seed)
}
if !options.clearData {
t.Errorf("clearData default: got false, want true")
}
}
func TestParseTestArgs_ClearDataDisabled(t *testing.T) {
options, err := parseTestArgs([]string{
"--spec", "s.ts",
"--bundle-id", "com.example",
"--clear-data=false",
}, io.Discard)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if options.clearData {
t.Errorf("expected clearData=false, got true")
}
}
func TestParseTestArgs_AllFlags(t *testing.T) {
options, err := parseTestArgs([]string{
"--spec", "s.ts",
"--bundle-id", "com.example",
"--platform", "android",
"--avd", "Pixel_5_API_33",
"--duration", "10m",
"--seed", "42",
"--output", "./out",
}, io.Discard)
if err != nil {
t.Fatal(err)
}
if options.avd != "Pixel_5_API_33" || options.duration != 10*time.Minute || options.seed != 42 || options.output != "./out" {
t.Errorf("unexpected options: %+v", options)
}
}
func TestParseTestArgs_RequiresSpec(t *testing.T) {
_, err := parseTestArgs([]string{"--bundle-id", "com.example", "--avd", "x"}, io.Discard)
if err == nil || !strings.Contains(err.Error(), "--spec") {
t.Fatalf("expected missing --spec error, got %v", err)
}
}
func TestParseTestArgs_RequiresBundleID(t *testing.T) {
_, err := parseTestArgs([]string{"--spec", "s.ts", "--avd", "x"}, io.Discard)
if err == nil || !strings.Contains(err.Error(), "--bundle-id") {
t.Fatalf("expected missing --bundle-id error, got %v", err)
}
}
func TestParseTestArgs_AVDIsOptional(t *testing.T) {
options, err := parseTestArgs([]string{"--spec", "s.ts", "--bundle-id", "com.example"}, io.Discard)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if options.avd != "" {
t.Fatalf("avd default: got %q, want empty", options.avd)
}
}
func TestParseTestArgs_GeneratorDefaultsToSeeded(t *testing.T) {
options, err := parseTestArgs([]string{"--spec", "s.ts", "--bundle-id", "com.example"}, io.Discard)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if options.generator != "seeded" {
t.Fatalf("generator default: got %q, want seeded", options.generator)
}
}
func TestParseTestArgs_AcceptsLLMGenerator(t *testing.T) {
options, err := parseTestArgs([]string{
"--spec", "s.ts",
"--bundle-id", "com.example",
"--generator", "llm",
}, io.Discard)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if options.generator != "llm" {
t.Fatalf("generator: got %q, want llm", options.generator)
}
}
func TestParseTestArgs_RejectsUnknownGenerator(t *testing.T) {
_, err := parseTestArgs([]string{
"--spec", "s.ts",
"--bundle-id", "com.example",
"--generator", "oracle",
}, io.Discard)
if err == nil || !strings.Contains(err.Error(), "unsupported generator") {
t.Fatalf("expected unsupported-generator error, got %v", err)
}
}
func TestParseTestArgs_RejectsUnknownPlatform(t *testing.T) {
_, err := parseTestArgs([]string{
"--spec", "s.ts",
"--bundle-id", "com.example",
"--platform", "fuchsia",
"--avd", "x",
}, io.Discard)
if err == nil || !strings.Contains(err.Error(), "unsupported platform") {
t.Fatalf("expected unsupported-platform error, got %v", err)
}
}
func TestParseTestArgs_AcceptsWebPlatform(t *testing.T) {
options, err := parseTestArgs([]string{
"--spec", "s.ts",
"--bundle-id", "http://localhost:3000",
"--platform", "web",
}, io.Discard)
if err != nil {
t.Fatalf("unexpected error for web platform: %v", err)
}
if options.platform != "web" {
t.Errorf("expected platform=web, got %q", options.platform)
}
}
func TestRun_HelpPrintsUsage(t *testing.T) {
var stdout bytes.Buffer
if err := run([]string{"sanderling"}, &stdout, io.Discard); err != nil {
t.Fatal(err)
}
if !strings.Contains(stdout.String(), "sanderling <command>") {
t.Errorf("usage missing, got: %q", stdout.String())
}
}
func TestRun_VersionPrintsVersion(t *testing.T) {
prev := Version
Version = "1.2.3-test"
defer func() { Version = prev }()
for _, arg := range []string{"version", "--version", "-v"} {
var stdout bytes.Buffer
if err := run([]string{"sanderling", arg}, &stdout, io.Discard); err != nil {
t.Fatalf("%s: %v", arg, err)
}
if strings.TrimSpace(stdout.String()) != "1.2.3-test" {
t.Errorf("%s: got %q, want 1.2.3-test", arg, stdout.String())
}
}
}
func TestRun_UnknownCommand(t *testing.T) {
err := run([]string{"sanderling", "wat"}, io.Discard, io.Discard)
if err == nil || !strings.Contains(err.Error(), "unknown command") {
t.Fatalf("expected unknown-command error, got %v", err)
}
}
func TestParseTestArgs_AcceptsIosPlatform(t *testing.T) {
options, err := parseTestArgs([]string{
"--spec", "s.ts",
"--bundle-id", "com.example.app",
"--platform", "ios",
}, io.Discard)
if err != nil {
t.Fatalf("unexpected error for ios platform: %v", err)
}
if options.platform != "ios" {
t.Errorf("expected platform=ios, got %q", options.platform)
}
}
func TestParseTestArgs_IosDeviceFlag(t *testing.T) {
options, err := parseTestArgs([]string{
"--spec", "s.ts",
"--bundle-id", "com.example.app",
"--platform", "ios",
"--ios-device", "iPhone 15 Pro",
}, io.Discard)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if options.iosDevice != "iPhone 15 Pro" {
t.Errorf("expected iosDevice=iPhone 15 Pro, got %q", options.iosDevice)
}
}
func TestParseTestArgs_IosAppPathFlag(t *testing.T) {
options, err := parseTestArgs([]string{
"--spec", "s.ts",
"--bundle-id", "com.example.app",
"--platform", "ios",
"--ios-app-path", "/tmp/build/iosApp.app",
}, io.Discard)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if options.iosAppPath != "/tmp/build/iosApp.app" {
t.Errorf("expected iosAppPath=/tmp/build/iosApp.app, got %q", options.iosAppPath)
}
}
func TestParseTestArgs_IosAppPathOptional(t *testing.T) {
options, err := parseTestArgs([]string{"--spec", "s.ts", "--bundle-id", "com.example.app"}, io.Discard)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if options.iosAppPath != "" {
t.Errorf("iosAppPath default: got %q, want empty", options.iosAppPath)
}
}
func TestRun_TestSubcommand_PipelineErrors(t *testing.T) {
// Web skips host-dependent device boot and bundles first, so a missing spec
// deterministically surfaces a bundle-resolution error. This proves the
// flag wiring reaches the bundler rather than panicking on the way.
err := run([]string{
"sanderling", "test",
"--spec", "definitely-missing-spec.ts",
"--bundle-id", "http://localhost:3000",
"--platform", "web",
}, io.Discard, io.Discard)
if err == nil {
t.Fatal("expected error, got nil")
}
message := err.Error()
if !strings.HasPrefix(message, "bundle spec:") || !strings.Contains(message, "definitely-missing-spec.ts") {
t.Errorf("expected bundle-resolution error for the missing spec, got %v", err)
}
}