fix(runner): warn on malformed screen snapshot (#13)

* fix(runner): warn on malformed screen snapshot

screenFromSnapshot swallowed json.Unmarshal errors, so a non-string
screen value silently became "" in the step log and trace while the
verifier still saw the raw JSON. Return the error and warn at the
call site, matching the hierarchy warning pattern.

* docs: clarify --avd is optional for uatu test

The CLI accepts --avd as an empty-string default (cmd/uatu/main.go:49)
and only requires it when no device is connected and multiple AVDs
exist (cmd/uatu/android_env.go:63). Docs and examples that showed it
as required or always-passed were misleading.
This commit is contained in:
pj authored and GitHub committed 2026-04-18 17:14:31 +07:00
1 parent 16e55086d8
commit d0578dbaaa
6 files changed
+49 -10

No files matched your search

+1 -1
View File
@@ -18,7 +18,7 @@ Run a spec against an app for a fixed duration.
| `--bundle-id` | required | Target app bundle ID (Android: applicationId). | | `--bundle-id` | required | Target app bundle ID (Android: applicationId). |
| `--launcher-activity` | resolved | Optional `<pkg>/<activity>` to launch. Overrides default resolution. | | `--launcher-activity` | resolved | Optional `<pkg>/<activity>` to launch. Overrides default resolution. |
| `--platform` | `android` | Target platform. Only `android` in the current alpha. | | `--platform` | `android` | Target platform. Only `android` in the current alpha. |
| `--avd` | required (android) | Android AVD name. | | `--avd` | optional (android) | Android AVD name to boot if no device is connected. Required only when no device is connected and multiple AVDs exist. |
| `--duration` | `5m` | Total test duration (`30s`, `5m`, `2h`, `1d`). | | `--duration` | `5m` | Total test duration (`30s`, `5m`, `2h`, `1d`). |
| `--seed` | `0` | PRNG seed. `0` uses a random seed and records it in `meta.json`. | | `--seed` | `0` | PRNG seed. `0` uses a random seed and records it in `meta.json`. |
| `--output` | `./runs` | Output directory for traces. | | `--output` | `./runs` | Output directory for traces. |
+2 -1
View File
@@ -59,10 +59,11 @@ uatu test \
--spec spec.ts \ --spec spec.ts \
--bundle-id dev.uatu.sample \ --bundle-id dev.uatu.sample \
--platform android \ --platform android \
--avd Pixel_7 \
--duration 2m --duration 2m
``` ```
Pass `--avd <name>` only when no device is connected and you have multiple AVDs; otherwise uatu uses the connected device or boots the single AVD it finds.
When the run ends, the trace lands in `runs/<timestamp>/`: When the run ends, the trace lands in `runs/<timestamp>/`:
``` ```
+1 -1
View File
@@ -11,7 +11,7 @@ A run is not analogous to a unit test. A closer framing is: boot a fuzzer for an
## Lifecycle ## Lifecycle
``` ```
uatu test --spec spec.ts --bundle-id com.example.app --avd Pixel_7 --duration 30m uatu test --spec spec.ts --bundle-id com.example.app --duration 30m
│ │
├── uninstall and reinstall the app (clean slate, every run) ├── uninstall and reinstall the app (clean slate, every run)
├── boot the sidecar, connect the agent socket ├── boot the sidecar, connect the agent socket
+1 -1
View File
@@ -47,4 +47,4 @@ Traces land in `./runs/<timestamp>/`.
registers snapshot extractors (`app_state`, `click_count`) registers snapshot extractors (`app_state`, `click_count`)
- `spec.ts` imports `@uatu/spec` (see `package.json`), reads those snapshots, - `spec.ts` imports `@uatu/spec` (see `package.json`), reads those snapshots,
asserts properties on them, and weights the actions the fuzzer picks from asserts properties on them, and weights the actions the fuzzer picks from
- `just test` invokes `uatu test` against the installed APK on the named AVD - `just test` invokes `uatu test` against the installed APK on the connected device (or the AVD named via `AVD=`)
+12 -6
View File
@@ -79,8 +79,12 @@ func Run(ctx context.Context, options Options) (Summary, error) {
if err := options.Verifier.PushSnapshot(verifier.Snapshots(snapshot.Snapshots), tree); err != nil { if err := options.Verifier.PushSnapshot(verifier.Snapshots(snapshot.Snapshots), tree); err != nil {
return summary, fmt.Errorf("step %d push: %w", stepIndex, err) return summary, fmt.Errorf("step %d push: %w", stepIndex, err)
} }
screen, screenErr := screenFromSnapshot(snapshot.Snapshots)
if screenErr != nil {
fmt.Printf("warning: step %d screen: %v\n", stepIndex, screenErr)
}
fmt.Printf("step %d: screen=%q hierarchy=%d nodes\n", fmt.Printf("step %d: screen=%q hierarchy=%d nodes\n",
stepIndex, screenFromSnapshot(snapshot.Snapshots), treeSize) stepIndex, screen, treeSize)
verdicts := options.Verifier.EvaluateProperties() verdicts := options.Verifier.EvaluateProperties()
violations := violationNames(verdicts) violations := violationNames(verdicts)
@@ -95,7 +99,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
step := trace.Step{ step := trace.Step{
Index: stepIndex, Index: stepIndex,
Timestamp: stepStart, Timestamp: stepStart,
Screen: screenFromSnapshot(snapshot.Snapshots), Screen: screen,
Snapshots: snapshot.Snapshots, Snapshots: snapshot.Snapshots,
Action: traceAction, Action: traceAction,
Violations: violations, Violations: violations,
@@ -175,14 +179,16 @@ func violationNames(verdicts map[string]ltl.Verdict) []string {
return names return names
} }
func screenFromSnapshot(snapshots map[string]json.RawMessage) string { func screenFromSnapshot(snapshots map[string]json.RawMessage) (string, error) {
raw, ok := snapshots["screen"] raw, ok := snapshots["screen"]
if !ok { if !ok {
return "" return "", nil
} }
var screen string var screen string
_ = json.Unmarshal(raw, &screen) if err := json.Unmarshal(raw, &screen); err != nil {
return screen return "", err
}
return screen, nil
} }
func applyAction(ctx context.Context, drv driver.Driver, action verifier.Action, tree *hierarchy.Tree) error { func applyAction(ctx context.Context, drv driver.Driver, action verifier.Action, tree *hierarchy.Tree) error {
+32
View File
@@ -235,6 +235,38 @@ func TestRunner_RecordsScreenFieldFromSnapshot(t *testing.T) {
} }
} }
func TestScreenFromSnapshot(t *testing.T) {
t.Run("string value returns screen", func(t *testing.T) {
snapshots := map[string]json.RawMessage{"screen": json.RawMessage(`"home"`)}
screen, err := screenFromSnapshot(snapshots)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if screen != "home" {
t.Errorf("screen = %q, want %q", screen, "home")
}
})
t.Run("missing key returns empty with no error", func(t *testing.T) {
screen, err := screenFromSnapshot(map[string]json.RawMessage{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if screen != "" {
t.Errorf("screen = %q, want empty", screen)
}
})
t.Run("non-string value returns error", func(t *testing.T) {
snapshots := map[string]json.RawMessage{"screen": json.RawMessage(`{"nested":1}`)}
screen, err := screenFromSnapshot(snapshots)
if err == nil {
t.Fatalf("expected error for non-string screen, got nil")
}
if screen != "" {
t.Errorf("screen = %q, want empty on error", screen)
}
})
}
func TestApplyAction_InputTextSurfacesFocusTapError(t *testing.T) { func TestApplyAction_InputTextSurfacesFocusTapError(t *testing.T) {
t.Run("selector focus tap fails", func(t *testing.T) { t.Run("selector focus tap fails", func(t *testing.T) {
driverMock := mockdriver.New() driverMock := mockdriver.New()