mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
fix(runner,sample-app): surface silent errors and demo a failing property (#15)
* fix(runner): surface non-deadline WaitForIdle errors Previously the WaitForIdle return value was discarded entirely, hiding real driver failures (gRPC transport errors, sidecar crashes) behind the expected deadline-exceeded case. Log non-deadline errors so they are visible without changing control flow. * chore(sample-app): drop unused uptime_millis extractor Registered in SampleApplication but never consumed by spec.ts. * fix(sample-app): drop trivial appIsRunning property app_state was hardcoded to 'running' so the property was a tautology that could never fail. Removing both the extractor and the property is the simplest fix; demo-grade properties that can fail land next. * feat(sample-app): add Reset button that zeroes clickCount Pairs with the next commit's tap-reset action so the fuzzer can violate clickCountNeverDecreases and demonstrate uatu actually finding a property violation. * feat(sample-app): add tap-reset action to exercise Reset button Weighted at 10/122, fuzzer reaches it within a short run. Pairs with the Reset button to demonstrate uatu detecting the clickCountNeverDecreases violation. * fix(runner): filter WaitForIdle errors via context state, not errors.Is errors.Is(err, context.DeadlineExceeded) misses gRPC's wrapped status.DeadlineExceeded, so every step under the maestro driver logged a spurious warning. Check idleCtx.Err() instead — captures both deadline-fired and parent-canceled cases regardless of how the driver wraps them. * chore(sample-app): tune action weights so demo violates in ~30s Prior weights left tap-reset rare enough that short demo runs missed the violation by chance. Bumped to 30/107, with typeUsername reduced since username noise doesn't help exercise clickCount. * refactor(runner): route warnings through slog Adds Options.Logger (defaults to slog.Default()) and converts the three warning sites that were using fmt.Printf. Progress line stays on Printf since it's user-facing UI, not a log. Makes the warnings testable via a capturing handler. * test(runner): assert WaitForIdle driver errors are logged Captures slog output via TextHandler into a buffer and asserts the warning message + injected error text appear when the mock driver returns a non-context error from WaitForIdle. Guards against a regression of the silent-error swallow.
This commit is contained in:
5 files changed
+67
-13
No files matched your search
@@ -47,6 +47,16 @@ class MainActivity : Activity() {
|
||||
}
|
||||
layout.addView(button)
|
||||
|
||||
val resetButton = Button(this).apply {
|
||||
text = "Reset"
|
||||
textSize = 18f
|
||||
setOnClickListener {
|
||||
clickCount = 0
|
||||
label.text = "Clicks: $clickCount"
|
||||
}
|
||||
}
|
||||
layout.addView(resetButton)
|
||||
|
||||
val usernameLabel = TextView(this).apply {
|
||||
text = "Username: "
|
||||
textSize = 20f
|
||||
|
||||
@@ -7,11 +7,7 @@ class SampleApplication : Application() {
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
Uatu.start(this)
|
||||
Uatu.extract("app_state") { "running" }
|
||||
Uatu.extract("click_count") { MainActivity.clickCount }
|
||||
Uatu.extract("username") { MainActivity.username }
|
||||
Uatu.extract("uptime_millis") { System.currentTimeMillis() - startedAt }
|
||||
}
|
||||
|
||||
private val startedAt: Long = System.currentTimeMillis()
|
||||
}
|
||||
@@ -11,9 +11,6 @@ import {
|
||||
|
||||
// ── Snapshot extractors (fed by SampleApplication.kt) ──────────
|
||||
// See ./android/src/main/kotlin/dev/uatu/sample/SampleApplication.kt
|
||||
const appState = extract<string>(
|
||||
(state) => (state.snapshots.app_state as string) ?? "",
|
||||
);
|
||||
const clickCount = extract<number>(
|
||||
(state) => (state.snapshots.click_count as number) ?? 0,
|
||||
);
|
||||
@@ -23,11 +20,11 @@ const username = extract<string>(
|
||||
|
||||
// ── UI elements ────────────────────────────────────────────────
|
||||
const clickButton = extract((state) => state.ax.find("text:Click me"));
|
||||
const resetButton = extract((state) => state.ax.find("text:Reset"));
|
||||
const usernameField = extract((state) => state.ax.find("desc:username_field"));
|
||||
|
||||
// ── Properties ─────────────────────────────────────────────────
|
||||
export const properties = {
|
||||
appIsRunning: always(() => appState.current === "running"),
|
||||
clickCountNonNegative: always(() => clickCount.current >= 0),
|
||||
clickCountNeverDecreases: always(() => {
|
||||
const previous = clickCount.previous;
|
||||
@@ -50,10 +47,15 @@ const typeUsername = actions(() => {
|
||||
: [];
|
||||
});
|
||||
|
||||
const tapReset = actions(() => {
|
||||
return resetButton.current ? [Tap({ on: resetButton.current })] : [];
|
||||
});
|
||||
|
||||
export const actionsRoot = weighted(
|
||||
[50, tapClickMe],
|
||||
[50, typeUsername],
|
||||
[10, taps],
|
||||
[20, typeUsername],
|
||||
[30, tapReset],
|
||||
[5, taps],
|
||||
[2, swipes],
|
||||
);
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/priyanshujain/uatu/internal/agent"
|
||||
@@ -24,6 +25,7 @@ type Options struct {
|
||||
Driver driver.Driver
|
||||
Verifier *verifier.Verifier
|
||||
TraceWriter *trace.Writer
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
type Summary struct {
|
||||
@@ -46,6 +48,10 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
||||
if err := validate(options); err != nil {
|
||||
return Summary{}, err
|
||||
}
|
||||
logger := options.Logger
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
|
||||
summary := Summary{StartTime: time.Now()}
|
||||
deadline := summary.StartTime.Add(options.Duration)
|
||||
@@ -64,7 +70,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
||||
// a hierarchy that matches the snapshots captured a moment later.
|
||||
tree, hierarchyErr := fetchHierarchy(ctx, options.Driver)
|
||||
if hierarchyErr != nil {
|
||||
fmt.Printf("warning: step %d hierarchy: %v\n", stepIndex, hierarchyErr)
|
||||
logger.Warn("hierarchy fetch failed", "step", stepIndex, "err", hierarchyErr)
|
||||
}
|
||||
treeSize := 0
|
||||
if tree != nil {
|
||||
@@ -81,7 +87,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
||||
}
|
||||
screen, screenErr := screenFromSnapshot(snapshot.Snapshots)
|
||||
if screenErr != nil {
|
||||
fmt.Printf("warning: step %d screen: %v\n", stepIndex, screenErr)
|
||||
logger.Warn("screen snapshot decode failed", "step", stepIndex, "err", screenErr)
|
||||
}
|
||||
fmt.Printf("step %d: screen=%q hierarchy=%d nodes\n",
|
||||
stepIndex, screen, treeSize)
|
||||
@@ -126,7 +132,10 @@ func Run(ctx context.Context, options Options) (Summary, error) {
|
||||
}
|
||||
|
||||
idleCtx, idleCancel := context.WithTimeout(ctx, options.IdleTimeout)
|
||||
_ = options.Driver.WaitForIdle(idleCtx, options.IdleTimeout)
|
||||
idleErr := options.Driver.WaitForIdle(idleCtx, options.IdleTimeout)
|
||||
if idleErr != nil && idleCtx.Err() == nil {
|
||||
logger.Warn("wait_for_idle failed", "step", stepIndex, "err", idleErr)
|
||||
}
|
||||
idleCancel()
|
||||
}
|
||||
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -267,6 +269,41 @@ func TestScreenFromSnapshot(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunner_LogsWaitForIdleDriverErrors(t *testing.T) {
|
||||
snapshots := []map[string]json.RawMessage{
|
||||
{"balance": json.RawMessage(`100`)},
|
||||
}
|
||||
state := newHarness(t, snapshots)
|
||||
state.startSDK(t)
|
||||
state.acceptConnection(t)
|
||||
state.mock.Failures[mockdriver.ActionWaitForIdle] = errors.New("sidecar lost gRPC stream")
|
||||
|
||||
var logBuf bytes.Buffer
|
||||
logger := slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if _, err := Run(ctx, Options{
|
||||
Duration: 100 * time.Millisecond,
|
||||
SnapshotTimeout: 2 * time.Second,
|
||||
IdleTimeout: 50 * time.Millisecond,
|
||||
Connection: state.conn,
|
||||
Driver: state.mock,
|
||||
Verifier: state.verifier,
|
||||
TraceWriter: state.writer,
|
||||
Logger: logger,
|
||||
}); err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
output := logBuf.String()
|
||||
if !strings.Contains(output, "wait_for_idle failed") {
|
||||
t.Errorf("expected wait_for_idle warning, got: %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "sidecar lost gRPC stream") {
|
||||
t.Errorf("expected driver error message in warning, got: %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyAction_InputTextSurfacesFocusTapError(t *testing.T) {
|
||||
t.Run("selector focus tap fails", func(t *testing.T) {
|
||||
driverMock := mockdriver.New()
|
||||
|
||||
Reference in new issue
Block a user