mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 11:07:10 +00:00
fix(runner): warn on malformed screen snapshot (#13)
* fix(runner): warn on malformed screen snapshot screenFromSnapshot swallowed json.Unmarshal errors, so a non-string screen value silently became "" in the step log and trace while the verifier still saw the raw JSON. Return the error and warn at the call site, matching the hierarchy warning pattern. * docs: clarify --avd is optional for uatu test The CLI accepts --avd as an empty-string default (cmd/uatu/main.go:49) and only requires it when no device is connected and multiple AVDs exist (cmd/uatu/android_env.go:63). Docs and examples that showed it as required or always-passed were misleading.
This commit is contained in:
6 files changed
+49
-10
No files matched your search
+1
-1
@@ -18,7 +18,7 @@ Run a spec against an app for a fixed duration.
|
||||
| `--bundle-id` | required | Target app bundle ID (Android: applicationId). |
|
||||
| `--launcher-activity` | resolved | Optional `<pkg>/<activity>` to launch. Overrides default resolution. |
|
||||
| `--platform` | `android` | Target platform. Only `android` in the current alpha. |
|
||||
| `--avd` | required (android) | Android AVD name. |
|
||||
| `--avd` | optional (android) | Android AVD name to boot if no device is connected. Required only when no device is connected and multiple AVDs exist. |
|
||||
| `--duration` | `5m` | Total test duration (`30s`, `5m`, `2h`, `1d`). |
|
||||
| `--seed` | `0` | PRNG seed. `0` uses a random seed and records it in `meta.json`. |
|
||||
| `--output` | `./runs` | Output directory for traces. |
|
||||
|
||||
@@ -59,10 +59,11 @@ uatu test \
|
||||
--spec spec.ts \
|
||||
--bundle-id dev.uatu.sample \
|
||||
--platform android \
|
||||
--avd Pixel_7 \
|
||||
--duration 2m
|
||||
```
|
||||
|
||||
Pass `--avd <name>` only when no device is connected and you have multiple AVDs; otherwise uatu uses the connected device or boots the single AVD it finds.
|
||||
|
||||
When the run ends, the trace lands in `runs/<timestamp>/`:
|
||||
|
||||
```
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ A run is not analogous to a unit test. A closer framing is: boot a fuzzer for an
|
||||
## Lifecycle
|
||||
|
||||
```
|
||||
uatu test --spec spec.ts --bundle-id com.example.app --avd Pixel_7 --duration 30m
|
||||
uatu test --spec spec.ts --bundle-id com.example.app --duration 30m
|
||||
│
|
||||
├── uninstall and reinstall the app (clean slate, every run)
|
||||
├── boot the sidecar, connect the agent socket
|
||||
|
||||
@@ -47,4 +47,4 @@ Traces land in `./runs/<timestamp>/`.
|
||||
registers snapshot extractors (`app_state`, `click_count`)
|
||||
- `spec.ts` imports `@uatu/spec` (see `package.json`), reads those snapshots,
|
||||
asserts properties on them, and weights the actions the fuzzer picks from
|
||||
- `just test` invokes `uatu test` against the installed APK on the named AVD
|
||||
- `just test` invokes `uatu test` against the installed APK on the connected device (or the AVD named via `AVD=`)
|
||||
@@ -79,8 +79,12 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
||||
if err := options.Verifier.PushSnapshot(verifier.Snapshots(snapshot.Snapshots), tree); err != nil {
|
||||
return summary, fmt.Errorf("step %d push: %w", stepIndex, err)
|
||||
}
|
||||
screen, screenErr := screenFromSnapshot(snapshot.Snapshots)
|
||||
if screenErr != nil {
|
||||
fmt.Printf("warning: step %d screen: %v\n", stepIndex, screenErr)
|
||||
}
|
||||
fmt.Printf("step %d: screen=%q hierarchy=%d nodes\n",
|
||||
stepIndex, screenFromSnapshot(snapshot.Snapshots), treeSize)
|
||||
stepIndex, screen, treeSize)
|
||||
verdicts := options.Verifier.EvaluateProperties()
|
||||
violations := violationNames(verdicts)
|
||||
|
||||
@@ -95,7 +99,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
||||
step := trace.Step{
|
||||
Index: stepIndex,
|
||||
Timestamp: stepStart,
|
||||
Screen: screenFromSnapshot(snapshot.Snapshots),
|
||||
Screen: screen,
|
||||
Snapshots: snapshot.Snapshots,
|
||||
Action: traceAction,
|
||||
Violations: violations,
|
||||
@@ -175,14 +179,16 @@ func violationNames(verdicts map[string]ltl.Verdict) []string {
|
||||
return names
|
||||
}
|
||||
|
||||
func screenFromSnapshot(snapshots map[string]json.RawMessage) string {
|
||||
func screenFromSnapshot(snapshots map[string]json.RawMessage) (string, error) {
|
||||
raw, ok := snapshots["screen"]
|
||||
if !ok {
|
||||
return ""
|
||||
return "", nil
|
||||
}
|
||||
var screen string
|
||||
_ = json.Unmarshal(raw, &screen)
|
||||
return screen
|
||||
if err := json.Unmarshal(raw, &screen); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return screen, nil
|
||||
}
|
||||
|
||||
func applyAction(ctx context.Context, drv driver.Driver, action verifier.Action, tree *hierarchy.Tree) error {
|
||||
|
||||
@@ -235,6 +235,38 @@ func TestRunner_RecordsScreenFieldFromSnapshot(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestScreenFromSnapshot(t *testing.T) {
|
||||
t.Run("string value returns screen", func(t *testing.T) {
|
||||
snapshots := map[string]json.RawMessage{"screen": json.RawMessage(`"home"`)}
|
||||
screen, err := screenFromSnapshot(snapshots)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if screen != "home" {
|
||||
t.Errorf("screen = %q, want %q", screen, "home")
|
||||
}
|
||||
})
|
||||
t.Run("missing key returns empty with no error", func(t *testing.T) {
|
||||
screen, err := screenFromSnapshot(map[string]json.RawMessage{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if screen != "" {
|
||||
t.Errorf("screen = %q, want empty", screen)
|
||||
}
|
||||
})
|
||||
t.Run("non-string value returns error", func(t *testing.T) {
|
||||
snapshots := map[string]json.RawMessage{"screen": json.RawMessage(`{"nested":1}`)}
|
||||
screen, err := screenFromSnapshot(snapshots)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for non-string screen, got nil")
|
||||
}
|
||||
if screen != "" {
|
||||
t.Errorf("screen = %q, want empty on error", screen)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestApplyAction_InputTextSurfacesFocusTapError(t *testing.T) {
|
||||
t.Run("selector focus tap fails", func(t *testing.T) {
|
||||
driverMock := mockdriver.New()
|
||||
|
||||
Reference in new issue
Block a user