fix(runner,sample-app): surface silent errors and demo a failing property (#15)

* fix(runner): surface non-deadline WaitForIdle errors

Previously the WaitForIdle return value was discarded entirely, hiding
real driver failures (gRPC transport errors, sidecar crashes) behind
the expected deadline-exceeded case. Log non-deadline errors so they
are visible without changing control flow.

* chore(sample-app): drop unused uptime_millis extractor

Registered in SampleApplication but never consumed by spec.ts.

* fix(sample-app): drop trivial appIsRunning property

app_state was hardcoded to 'running' so the property was a tautology
that could never fail. Removing both the extractor and the property
is the simplest fix; demo-grade properties that can fail land next.

* feat(sample-app): add Reset button that zeroes clickCount

Pairs with the next commit's tap-reset action so the fuzzer can
violate clickCountNeverDecreases and demonstrate uatu actually
finding a property violation.

* feat(sample-app): add tap-reset action to exercise Reset button

Weighted at 10/122, fuzzer reaches it within a short run. Pairs with
the Reset button to demonstrate uatu detecting the
clickCountNeverDecreases violation.

* fix(runner): filter WaitForIdle errors via context state, not errors.Is

errors.Is(err, context.DeadlineExceeded) misses gRPC's wrapped
status.DeadlineExceeded, so every step under the maestro driver
logged a spurious warning. Check idleCtx.Err() instead — captures
both deadline-fired and parent-canceled cases regardless of how the
driver wraps them.

* chore(sample-app): tune action weights so demo violates in ~30s

Prior weights left tap-reset rare enough that short demo runs missed
the violation by chance. Bumped to 30/107, with typeUsername reduced
since username noise doesn't help exercise clickCount.

* refactor(runner): route warnings through slog

Adds Options.Logger (defaults to slog.Default()) and converts the
three warning sites that were using fmt.Printf. Progress line stays
on Printf since it's user-facing UI, not a log. Makes the warnings
testable via a capturing handler.

* test(runner): assert WaitForIdle driver errors are logged

Captures slog output via TextHandler into a buffer and asserts the
warning message + injected error text appear when the mock driver
returns a non-context error from WaitForIdle. Guards against a
regression of the silent-error swallow.
This commit is contained in:
pj authored and GitHub committed 2026-04-18 18:48:25 +07:00
1 parent 00f66ba48d
commit 69002b07b1
5 files changed
+67 -13

No files matched your search

@@ -47,6 +47,16 @@ class MainActivity : Activity() {
}
layout.addView(button)
val resetButton = Button(this).apply {
text = "Reset"
textSize = 18f
setOnClickListener {
clickCount = 0
label.text = "Clicks: $clickCount"
}
}
layout.addView(resetButton)
val usernameLabel = TextView(this).apply {
text = "Username: "
textSize = 20f
@@ -7,11 +7,7 @@ class SampleApplication : Application() {
override fun onCreate() {
super.onCreate()
Uatu.start(this)
Uatu.extract("app_state") { "running" }
Uatu.extract("click_count") { MainActivity.clickCount }
Uatu.extract("username") { MainActivity.username }
Uatu.extract("uptime_millis") { System.currentTimeMillis() - startedAt }
}
private val startedAt: Long = System.currentTimeMillis()
}
+8 -6
View File
@@ -11,9 +11,6 @@ import {
// ── Snapshot extractors (fed by SampleApplication.kt) ──────────
// See ./android/src/main/kotlin/dev/uatu/sample/SampleApplication.kt
const appState = extract<string>(
(state) => (state.snapshots.app_state as string) ?? "",
);
const clickCount = extract<number>(
(state) => (state.snapshots.click_count as number) ?? 0,
);
@@ -23,11 +20,11 @@ const username = extract<string>(
// ── UI elements ────────────────────────────────────────────────
const clickButton = extract((state) => state.ax.find("text:Click me"));
const resetButton = extract((state) => state.ax.find("text:Reset"));
const usernameField = extract((state) => state.ax.find("desc:username_field"));
// ── Properties ─────────────────────────────────────────────────
export const properties = {
appIsRunning: always(() => appState.current === "running"),
clickCountNonNegative: always(() => clickCount.current >= 0),
clickCountNeverDecreases: always(() => {
const previous = clickCount.previous;
@@ -50,10 +47,15 @@ const typeUsername = actions(() => {
: [];
});
const tapReset = actions(() => {
return resetButton.current ? [Tap({ on: resetButton.current })] : [];
});
export const actionsRoot = weighted(
[50, tapClickMe],
[50, typeUsername],
[10, taps],
[20, typeUsername],
[30, tapReset],
[5, taps],
[2, swipes],
);
+12 -3
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"log/slog"
"time"
"github.com/priyanshujain/uatu/internal/agent"
@@ -24,6 +25,7 @@ type Options struct {
Driver driver.Driver
Verifier *verifier.Verifier
TraceWriter *trace.Writer
Logger *slog.Logger
}
type Summary struct {
@@ -46,6 +48,10 @@ func Run(ctx context.Context, options Options) (Summary, error) {
if err := validate(options); err != nil {
return Summary{}, err
}
logger := options.Logger
if logger == nil {
logger = slog.Default()
}
summary := Summary{StartTime: time.Now()}
deadline := summary.StartTime.Add(options.Duration)
@@ -64,7 +70,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
// a hierarchy that matches the snapshots captured a moment later.
tree, hierarchyErr := fetchHierarchy(ctx, options.Driver)
if hierarchyErr != nil {
fmt.Printf("warning: step %d hierarchy: %v\n", stepIndex, hierarchyErr)
logger.Warn("hierarchy fetch failed", "step", stepIndex, "err", hierarchyErr)
}
treeSize := 0
if tree != nil {
@@ -81,7 +87,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
}
screen, screenErr := screenFromSnapshot(snapshot.Snapshots)
if screenErr != nil {
fmt.Printf("warning: step %d screen: %v\n", stepIndex, screenErr)
logger.Warn("screen snapshot decode failed", "step", stepIndex, "err", screenErr)
}
fmt.Printf("step %d: screen=%q hierarchy=%d nodes\n",
stepIndex, screen, treeSize)
@@ -126,7 +132,10 @@ func Run(ctx context.Context, options Options) (Summary, error) {
}
idleCtx, idleCancel := context.WithTimeout(ctx, options.IdleTimeout)
_ = options.Driver.WaitForIdle(idleCtx, options.IdleTimeout)
idleErr := options.Driver.WaitForIdle(idleCtx, options.IdleTimeout)
if idleErr != nil && idleCtx.Err() == nil {
logger.Warn("wait_for_idle failed", "step", stepIndex, "err", idleErr)
}
idleCancel()
}
+37
View File
@@ -1,9 +1,11 @@
package runner
import (
"bytes"
"context"
"encoding/json"
"errors"
"log/slog"
"net"
"os"
"path/filepath"
@@ -267,6 +269,41 @@ func TestScreenFromSnapshot(t *testing.T) {
})
}
func TestRunner_LogsWaitForIdleDriverErrors(t *testing.T) {
snapshots := []map[string]json.RawMessage{
{"balance": json.RawMessage(`100`)},
}
state := newHarness(t, snapshots)
state.startSDK(t)
state.acceptConnection(t)
state.mock.Failures[mockdriver.ActionWaitForIdle] = errors.New("sidecar lost gRPC stream")
var logBuf bytes.Buffer
logger := slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if _, err := Run(ctx, Options{
Duration: 100 * time.Millisecond,
SnapshotTimeout: 2 * time.Second,
IdleTimeout: 50 * time.Millisecond,
Connection: state.conn,
Driver: state.mock,
Verifier: state.verifier,
TraceWriter: state.writer,
Logger: logger,
}); err != nil {
t.Fatalf("Run: %v", err)
}
output := logBuf.String()
if !strings.Contains(output, "wait_for_idle failed") {
t.Errorf("expected wait_for_idle warning, got: %q", output)
}
if !strings.Contains(output, "sidecar lost gRPC stream") {
t.Errorf("expected driver error message in warning, got: %q", output)
}
}
func TestApplyAction_InputTextSurfacesFocusTapError(t *testing.T) {
t.Run("selector focus tap fails", func(t *testing.T) {
driverMock := mockdriver.New()