From d0578dbaaa8e2508ced63e35f91b3141023e8986 Mon Sep 17 00:00:00 2001 From: pjay Date: Sat, 18 Apr 2026 17:14:31 +0700 Subject: [PATCH] fix(runner): warn on malformed screen snapshot (#13) * fix(runner): warn on malformed screen snapshot screenFromSnapshot swallowed json.Unmarshal errors, so a non-string screen value silently became "" in the step log and trace while the verifier still saw the raw JSON. Return the error and warn at the call site, matching the hierarchy warning pattern. * docs: clarify --avd is optional for uatu test The CLI accepts --avd as an empty-string default (cmd/uatu/main.go:49) and only requires it when no device is connected and multiple AVDs exist (cmd/uatu/android_env.go:63). Docs and examples that showed it as required or always-passed were misleading. --- docs/manual/cli.md | 2 +- docs/manual/getting-started.md | 3 ++- docs/manual/runs.md | 2 +- examples/sample-app/README.md | 2 +- internal/runner/runner.go | 18 ++++++++++++------ internal/runner/runner_test.go | 32 ++++++++++++++++++++++++++++++++ 6 files changed, 49 insertions(+), 10 deletions(-) diff --git a/docs/manual/cli.md b/docs/manual/cli.md index bc03a45..1e607ff 100644 --- a/docs/manual/cli.md +++ b/docs/manual/cli.md @@ -18,7 +18,7 @@ Run a spec against an app for a fixed duration. | `--bundle-id` | required | Target app bundle ID (Android: applicationId). | | `--launcher-activity` | resolved | Optional `/` to launch. Overrides default resolution. | | `--platform` | `android` | Target platform. Only `android` in the current alpha. | -| `--avd` | required (android) | Android AVD name. | +| `--avd` | optional (android) | Android AVD name to boot if no device is connected. Required only when no device is connected and multiple AVDs exist. | | `--duration` | `5m` | Total test duration (`30s`, `5m`, `2h`, `1d`). | | `--seed` | `0` | PRNG seed. `0` uses a random seed and records it in `meta.json`. | | `--output` | `./runs` | Output directory for traces. | diff --git a/docs/manual/getting-started.md b/docs/manual/getting-started.md index 2249df0..898f937 100644 --- a/docs/manual/getting-started.md +++ b/docs/manual/getting-started.md @@ -59,10 +59,11 @@ uatu test \ --spec spec.ts \ --bundle-id dev.uatu.sample \ --platform android \ - --avd Pixel_7 \ --duration 2m ``` +Pass `--avd ` only when no device is connected and you have multiple AVDs; otherwise uatu uses the connected device or boots the single AVD it finds. + When the run ends, the trace lands in `runs//`: ``` diff --git a/docs/manual/runs.md b/docs/manual/runs.md index 91542d1..1f0441c 100644 --- a/docs/manual/runs.md +++ b/docs/manual/runs.md @@ -11,7 +11,7 @@ A run is not analogous to a unit test. A closer framing is: boot a fuzzer for an ## Lifecycle ``` -uatu test --spec spec.ts --bundle-id com.example.app --avd Pixel_7 --duration 30m +uatu test --spec spec.ts --bundle-id com.example.app --duration 30m │ ├── uninstall and reinstall the app (clean slate, every run) ├── boot the sidecar, connect the agent socket diff --git a/examples/sample-app/README.md b/examples/sample-app/README.md index eed65af..9f58a77 100644 --- a/examples/sample-app/README.md +++ b/examples/sample-app/README.md @@ -47,4 +47,4 @@ Traces land in `./runs//`. registers snapshot extractors (`app_state`, `click_count`) - `spec.ts` imports `@uatu/spec` (see `package.json`), reads those snapshots, asserts properties on them, and weights the actions the fuzzer picks from -- `just test` invokes `uatu test` against the installed APK on the named AVD +- `just test` invokes `uatu test` against the installed APK on the connected device (or the AVD named via `AVD=`) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 2ce99b3..4da598a 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -79,8 +79,12 @@ func Run(ctx context.Context, options Options) (Summary, error) { if err := options.Verifier.PushSnapshot(verifier.Snapshots(snapshot.Snapshots), tree); err != nil { return summary, fmt.Errorf("step %d push: %w", stepIndex, err) } + screen, screenErr := screenFromSnapshot(snapshot.Snapshots) + if screenErr != nil { + fmt.Printf("warning: step %d screen: %v\n", stepIndex, screenErr) + } fmt.Printf("step %d: screen=%q hierarchy=%d nodes\n", - stepIndex, screenFromSnapshot(snapshot.Snapshots), treeSize) + stepIndex, screen, treeSize) verdicts := options.Verifier.EvaluateProperties() violations := violationNames(verdicts) @@ -95,7 +99,7 @@ func Run(ctx context.Context, options Options) (Summary, error) { step := trace.Step{ Index: stepIndex, Timestamp: stepStart, - Screen: screenFromSnapshot(snapshot.Snapshots), + Screen: screen, Snapshots: snapshot.Snapshots, Action: traceAction, Violations: violations, @@ -175,14 +179,16 @@ func violationNames(verdicts map[string]ltl.Verdict) []string { return names } -func screenFromSnapshot(snapshots map[string]json.RawMessage) string { +func screenFromSnapshot(snapshots map[string]json.RawMessage) (string, error) { raw, ok := snapshots["screen"] if !ok { - return "" + return "", nil } var screen string - _ = json.Unmarshal(raw, &screen) - return screen + if err := json.Unmarshal(raw, &screen); err != nil { + return "", err + } + return screen, nil } func applyAction(ctx context.Context, drv driver.Driver, action verifier.Action, tree *hierarchy.Tree) error { diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 98e6b91..e5cf9d5 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -235,6 +235,38 @@ func TestRunner_RecordsScreenFieldFromSnapshot(t *testing.T) { } } +func TestScreenFromSnapshot(t *testing.T) { + t.Run("string value returns screen", func(t *testing.T) { + snapshots := map[string]json.RawMessage{"screen": json.RawMessage(`"home"`)} + screen, err := screenFromSnapshot(snapshots) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if screen != "home" { + t.Errorf("screen = %q, want %q", screen, "home") + } + }) + t.Run("missing key returns empty with no error", func(t *testing.T) { + screen, err := screenFromSnapshot(map[string]json.RawMessage{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if screen != "" { + t.Errorf("screen = %q, want empty", screen) + } + }) + t.Run("non-string value returns error", func(t *testing.T) { + snapshots := map[string]json.RawMessage{"screen": json.RawMessage(`{"nested":1}`)} + screen, err := screenFromSnapshot(snapshots) + if err == nil { + t.Fatalf("expected error for non-string screen, got nil") + } + if screen != "" { + t.Errorf("screen = %q, want empty on error", screen) + } + }) +} + func TestApplyAction_InputTextSurfacesFocusTapError(t *testing.T) { t.Run("selector focus tap fails", func(t *testing.T) { driverMock := mockdriver.New()