From 577993b5402392b5af1a5ccd8b9c89b0285b0faa Mon Sep 17 00:00:00 2001 From: PJ Date: Sat, 22 Aug 2026 21:39:22 +0530 Subject: [PATCH] test(runner): retire the scaffolding the package grew by copying Four consolidations, no assertion changed and the same 217 tests passing: harness gains run and tryRun, retiring 68 copies of context.WithTimeout plus Run(ctx, Options{}) plus an error check. The seven sites whose error check said something specific keep their own wording. mock.FailurePlan retires six wrapper drivers that were the same seven-line method with a different error constant. The three that need more than an error on call N stay. traceStepLine absorbs the 15 local trace-line declarations and readTraceLines is now the only reader. No local copy disagreed with internal/trace about a JSON tag. Three raw readers stay on purpose: one decodes a real trace.Step, one snapshots bytes, one asserts on raw JSON text. One embeddable base carries the five V8-path methods the five web fakes each redeclared. --- internal/runner/composition_reread_test.go | 99 +- .../runner/element_extractor_trace_test.go | 43 +- internal/runner/error_surface_trace_test.go | 37 +- .../runner/foreground_gate_budget_test.go | 18 +- .../foreground_guard_last_action_test.go | 18 +- internal/runner/golden_test.go | 29 +- internal/runner/llm_source_test.go | 39 +- internal/runner/navigation_trace_test.go | 39 +- internal/runner/redaction_test.go | 16 +- internal/runner/runner_test.go | 945 ++++-------------- internal/runner/uncertain_last_action_test.go | 15 +- internal/runner/web_carrier_test.go | 75 +- internal/runner/web_extractor_trace_test.go | 72 +- internal/runner/web_last_action_test.go | 70 +- 14 files changed, 233 insertions(+), 1282 deletions(-) diff --git a/internal/runner/composition_reread_test.go b/internal/runner/composition_reread_test.go index 942e602..85d1240 100644 --- a/internal/runner/composition_reread_test.go +++ b/internal/runner/composition_reread_test.go @@ -1,20 +1,16 @@ package runner import ( - "bytes" "context" - "encoding/json" "fmt" "os" "path/filepath" "strings" "sync/atomic" "testing" - "time" "github.com/priyanshujain/sanderling/internal/driver" mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock" - "github.com/priyanshujain/sanderling/internal/trace" ) // homeWithRows is one settled route whose list holds rows. A row arriving @@ -68,19 +64,7 @@ func TestRunner_AStepWhoseTreeChangedBetweenReadsIsNotVerified(t *testing.T) { state := newHarnessWithSpec(t, violationSpec) device := &composesLateDriver{Driver: state.mock, composingReads: composingReads} - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 3, - Driver: device, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{MaxSteps: 3, Driver: device}) if summary.Steps != 3 { t.Fatalf("steps = %d, want 3", summary.Steps) } @@ -107,7 +91,7 @@ func TestRunner_AStepWhoseTreeChangedBetweenReadsIsNotVerified(t *testing.T) { } // Skipped is not lost: the step is still recorded, screenshot and all, // so the run can be replayed over the frame nothing judged. - steps := traceSteps(t, directory) + steps := readTraceLines(t, directory) if len(steps) != 3 { t.Fatalf("trace holds %d step(s), want 3", len(steps)) } @@ -201,19 +185,7 @@ func TestRunner_ASkippedStepDoesNotSwallowTheActionBeforeIt(t *testing.T) { composingRead: composingRead, } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 3, - Driver: device, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{MaxSteps: 3, Driver: device}) if summary.Steps != 3 { t.Fatalf("steps = %d, want 3", summary.Steps) } @@ -277,19 +249,7 @@ func TestRunner_ARunOfSkippedStepsReportsEveryActionItApplied(t *testing.T) { composingThrough: 5, } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 6, - Driver: device, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{MaxSteps: 6, Driver: device}) if summary.Steps != 6 { t.Fatalf("steps = %d, want 6", summary.Steps) } @@ -341,19 +301,7 @@ func TestRunner_AScreenThatNeverSettlesActsOnNothingAndSaysSo(t *testing.T) { state := newHarnessWithSpec(t, specWithFolioPredicates(t)) device := &submitsOnTapDriver{Driver: state.mock, commitsPerTap: 1, everyRead: true} - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 5, - Driver: device, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{MaxSteps: 5, Driver: device}) if summary.Steps != 5 { t.Fatalf("steps = %d, want 5; the run stalled instead of finishing its budget", summary.Steps) @@ -442,19 +390,7 @@ func TestRunner_OnlyAChangeOfShapeCostsAStepItsVerdict(t *testing.T) { rereadTree: testCase.reread, } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 3, - Driver: device, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{MaxSteps: 3, Driver: device}) if summary.Steps != 3 { t.Fatalf("steps = %d, want 3", summary.Steps) } @@ -484,26 +420,3 @@ func TestRunner_OnlyAChangeOfShapeCostsAStepItsVerdict(t *testing.T) { }) } } - -type traceLine struct { - Step int `json:"step"` - Violations []string `json:"violations"` - ExtractorChanges map[string]trace.ExtractorChange `json:"extractor_changes"` -} - -func traceSteps(t *testing.T, directory string) []traceLine { - t.Helper() - body, err := os.ReadFile(filepath.Join(directory, "trace.jsonl")) - if err != nil { - t.Fatal(err) - } - var steps []traceLine - for _, raw := range bytes.Split(bytes.TrimSpace(body), []byte("\n")) { - var line traceLine - if err := json.Unmarshal(raw, &line); err != nil { - t.Fatalf("decode trace line: %v", err) - } - steps = append(steps, line) - } - return steps -} diff --git a/internal/runner/element_extractor_trace_test.go b/internal/runner/element_extractor_trace_test.go index 4eb7611..2bb5dc7 100644 --- a/internal/runner/element_extractor_trace_test.go +++ b/internal/runner/element_extractor_trace_test.go @@ -1,16 +1,9 @@ package runner import ( - "bufio" - "context" "encoding/json" "fmt" - "os" - "path/filepath" "testing" - "time" - - "github.com/priyanshujain/sanderling/internal/trace" ) // elementExtractorSpec reads a live ax element, the shape every field and @@ -47,43 +40,14 @@ func TestRunner_TraceRecordsElementValuedExtractors(t *testing.T) { state := newHarnessWithSpec(t, elementExtractorSpec) state.mock.HierarchyJSON = amountFieldTreeJSON - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 2, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{MaxSteps: 2}) if !containsProperty(summary.Violations, "noAmountField") { t.Fatalf("noAmountField did not violate, so the element never reached a predicate: %v", summary.Violations) } - file, err := os.Open(filepath.Join(state.writer.Directory(), "trace.jsonl")) - if err != nil { - t.Fatal(err) - } - defer file.Close() - - type traceLine struct { - Step int `json:"step"` - ExtractorChanges map[string]trace.ExtractorChange `json:"extractor_changes"` - Witnesses map[string]trace.Witness `json:"witnesses"` - } changes, witnesses := 0, 0 - scanner := bufio.NewScanner(file) - scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) - for scanner.Scan() { - var line traceLine - if err := json.Unmarshal(scanner.Bytes(), &line); err != nil { - t.Fatalf("trace line decode: %v", err) - } + for _, line := range readTraceLines(t, state.writer.Directory()) { if change, ok := line.ExtractorChanges["amountField"]; ok { changes++ assertAmountField(t, fmt.Sprintf("step %d extractor_changes", line.Step), change.Curr) @@ -94,9 +58,6 @@ func TestRunner_TraceRecordsElementValuedExtractors(t *testing.T) { witness.Extractors["amountField"]) } } - if err := scanner.Err(); err != nil { - t.Fatalf("scan trace: %v", err) - } if changes == 0 { t.Error("amountField never appears in extractor_changes; the element the run read is not in the trace") } diff --git a/internal/runner/error_surface_trace_test.go b/internal/runner/error_surface_trace_test.go index 61991d0..b4321ce 100644 --- a/internal/runner/error_surface_trace_test.go +++ b/internal/runner/error_surface_trace_test.go @@ -2,10 +2,6 @@ package runner import ( "context" - "encoding/json" - "os" - "path/filepath" - "strings" "testing" "time" @@ -43,34 +39,13 @@ func TestRunner_LogsAndExceptionsLandInTheTrace(t *testing.T) { Message: "FATAL EXCEPTION: main", }} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if _, err := Run(ctx, Options{ - Duration: 100 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 1, - Driver: &throwingDriver{Driver: state.mock}, - Verifier: state.verifier, - TraceWriter: state.writer, - }); err != nil { - t.Fatalf("Run: %v", err) - } + state.run(t, Options{ + Duration: 100 * time.Millisecond, + MaxSteps: 1, + Driver: &throwingDriver{Driver: state.mock}, + }) - body, err := os.ReadFile( - filepath.Join(state.writer.Directory(), "trace.jsonl"), - ) - if err != nil { - t.Fatal(err) - } - line := strings.SplitN(strings.TrimSpace(string(body)), "\n", 2)[0] - var stored struct { - TraceVersion int `json:"trace_version"` - Logs []trace.LogEntry `json:"logs"` - Exceptions []trace.Exception `json:"exceptions"` - } - if err := json.Unmarshal([]byte(line), &stored); err != nil { - t.Fatalf("decode trace line: %v\n%s", err, line) - } + stored := readTraceLines(t, state.writer.Directory())[0] if stored.TraceVersion != trace.TraceVersion { t.Errorf( "trace_version = %d, want %d; an old trace could not be told apart", diff --git a/internal/runner/foreground_gate_budget_test.go b/internal/runner/foreground_gate_budget_test.go index 1b23fab..4dfc289 100644 --- a/internal/runner/foreground_gate_budget_test.go +++ b/internal/runner/foreground_gate_budget_test.go @@ -96,16 +96,11 @@ func TestRun_AppNeverReachesForegroundEndsTheRun(t *testing.T) { state := newHarness(t) device := &neverDrawsDriver{Driver: state.mock} - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, + summary, err := state.tryRun(t, Options{ IdleTimeout: time.Millisecond, MaxSteps: 3, BundleID: guardedBundleID, Driver: device, - Verifier: state.verifier, - TraceWriter: state.writer, }) var notReached ForegroundNotReachedError @@ -165,19 +160,12 @@ func TestRun_StepsOutsideTheAppAreRecordedInTheTrace(t *testing.T) { state := newHarness(t) device := &leavesForegroundForeverDriver{Driver: state.mock} - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - if _, err := Run(ctx, Options{ - Duration: time.Hour, + state.run(t, Options{ IdleTimeout: time.Millisecond, MaxSteps: 2, BundleID: guardedBundleID, Driver: device, - Verifier: state.verifier, - TraceWriter: state.writer, - }); err != nil { - t.Fatalf("Run: %v", err) - } + }) records := preconditionRecords(t, state.writer.Directory()) if len(records) == 0 { diff --git a/internal/runner/foreground_guard_last_action_test.go b/internal/runner/foreground_guard_last_action_test.go index 426f672..85ed671 100644 --- a/internal/runner/foreground_guard_last_action_test.go +++ b/internal/runner/foreground_guard_last_action_test.go @@ -6,7 +6,6 @@ import ( "path/filepath" "sync/atomic" "testing" - "time" "github.com/priyanshujain/sanderling/internal/driver" mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock" @@ -146,20 +145,7 @@ func runTwoSubmitSteps( commitsPerTap int64, ) []ViolationRecord { t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 2, - BundleID: guardedBundleID, - Driver: device, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{MaxSteps: 2, BundleID: guardedBundleID, Driver: device}) if summary.Steps != 2 { t.Fatalf("steps = %d, want 2; the run never reached the step that judges the pair", summary.Steps) @@ -320,7 +306,7 @@ func runReportingTheGuard(t *testing.T, device committingDevice, state *harness) if violations := runTwoSubmitSteps(t, state, device, 1); len(violations) != 0 { t.Errorf("the spec was told the action ran untouched by any guard: %v", violations) } - steps := traceSteps(t, state.writer.Directory()) + steps := readTraceLines(t, state.writer.Directory()) if len(steps) != 2 { t.Fatalf("trace holds %d step(s), want 2", len(steps)) } diff --git a/internal/runner/golden_test.go b/internal/runner/golden_test.go index 156858c..91b0e2c 100644 --- a/internal/runner/golden_test.go +++ b/internal/runner/golden_test.go @@ -3,7 +3,6 @@ package runner import ( "bufio" "bytes" - "context" "encoding/json" "os" "path/filepath" @@ -91,19 +90,7 @@ func TestGolden_TraceStreamIsReproducible(t *testing.T) { state.verifier = mustSeededVerifier(t, fixtureSpec, goldenSeed) state.mock.HierarchyJSON = `{"attributes":{"resource-id":"HomeScreen"},"children":[{"attributes":{"resource-id":"next","bounds":"[40,80,240,160]"},"children":[],"clickable":true,"enabled":true}]}` - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - _, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 10 * time.Millisecond, - MaxSteps: goldenSteps, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + state.run(t, Options{IdleTimeout: 10 * time.Millisecond, MaxSteps: goldenSteps}) raw, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl")) if err != nil { t.Fatal(err) @@ -119,19 +106,7 @@ func TestGolden_ViolationSummaryIsReproducible(t *testing.T) { state := newHarnessWithSpec(t, violationSpec) state.verifier = mustSeededVerifier(t, violationSpec, goldenSeed) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 10 * time.Millisecond, - MaxSteps: goldenSteps, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{IdleTimeout: 10 * time.Millisecond, MaxSteps: goldenSteps}) var rendered bytes.Buffer RenderSummary(&rendered, summary, "android") diff --git a/internal/runner/llm_source_test.go b/internal/runner/llm_source_test.go index f22121f..26f6ff3 100644 --- a/internal/runner/llm_source_test.go +++ b/internal/runner/llm_source_test.go @@ -493,7 +493,7 @@ func newSeededVerifier(t *testing.T) *verifier.Verifier { func TestPickSourcesRejectsLLMWithoutSpecConfig(t *testing.T) { for name, activeDriver := range map[string]driver.DeviceDriver{ "native": nil, - "web": &webMockDriver{Driver: mockdriver.New()}, + "web": &webMockDriver{webDriverBase: webDriverBase{Driver: mockdriver.New()}}, } { t.Run(name, func(t *testing.T) { action, extractor, err := pickSources(Options{ @@ -523,7 +523,7 @@ func TestPickSourcesOnWebComposesLLMWithWebExtractors(t *testing.T) { fake := newFakeOpenRouter(t) _, verifierInstance := newLLMSource(t, fake) action, extractor, err := pickSources(Options{ - Driver: &webMockDriver{Driver: mockdriver.New()}, + Driver: &webMockDriver{webDriverBase: webDriverBase{Driver: mockdriver.New()}}, Verifier: verifierInstance, Generator: "llm", Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), @@ -541,7 +541,7 @@ func TestPickSourcesOnWebComposesLLMWithWebExtractors(t *testing.T) { func TestPickSourcesOnWebSeededKeepsBothOnV8(t *testing.T) { action, extractor, err := pickSources(Options{ - Driver: &webMockDriver{Driver: mockdriver.New()}, + Driver: &webMockDriver{webDriverBase: webDriverBase{Driver: mockdriver.New()}}, Verifier: newSeededVerifier(t), Generator: "seeded", Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), @@ -768,22 +768,13 @@ func TestRunner_EveryModelCallFailingIsNotACleanRun(t *testing.T) { state := newHarnessWithSpec(t, llmFixtureSpec) state.mock.HierarchyJSON = llmTreeJSON - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ + summary := state.run(t, Options{ Duration: 30 * time.Second, - IdleTimeout: 20 * time.Millisecond, MaxSteps: 3, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, Generator: "llm", LabelSource: verifier.LabelSourceVisibleText, Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), }) - if err != nil { - t.Fatalf("Run: %v", err) - } calls := readLLMCalls(t, state.writer.Directory()) if len(calls) != 3 { @@ -845,22 +836,13 @@ func TestRunner_SetupActionsAreNotTheGeneratorDrivingTheApp(t *testing.T) { state := newHarnessWithSpec(t, llmLoginSetupSpec) state.mock.HierarchyJSON = llmTreeJSON - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ + summary := state.run(t, Options{ Duration: 30 * time.Second, - IdleTimeout: 20 * time.Millisecond, MaxSteps: 4, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, Generator: "llm", LabelSource: verifier.LabelSourceVisibleText, Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), }) - if err != nil { - t.Fatalf("Run: %v", err) - } wantOutcomes := []string{ trace.LLMOutcomeSetupAction, trace.LLMOutcomeSetupAction, @@ -933,22 +915,13 @@ func TestRunner_SetupAndGeneratorBothDrivingIsAHealthyRun(t *testing.T) { fake.choice = tap.Index fake.chosenAction = tap.Description - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ + summary := state.run(t, Options{ Duration: 30 * time.Second, - IdleTimeout: 20 * time.Millisecond, MaxSteps: 4, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, Generator: "llm", LabelSource: verifier.LabelSourceVisibleText, Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), }) - if err != nil { - t.Fatalf("Run: %v", err) - } if summary.DispatchedActions != 4 { t.Errorf("DispatchedActions = %d, want 4: every step drove the app", diff --git a/internal/runner/navigation_trace_test.go b/internal/runner/navigation_trace_test.go index ebac4ae..5ae1bd8 100644 --- a/internal/runner/navigation_trace_test.go +++ b/internal/runner/navigation_trace_test.go @@ -1,17 +1,12 @@ package runner import ( - "bufio" "context" - "encoding/json" - "os" - "path/filepath" "testing" "time" "github.com/priyanshujain/sanderling/internal/driver" mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock" - "github.com/priyanshujain/sanderling/internal/trace" ) // navigatingDriver reports one navigation per step, the way a page that submits @@ -33,46 +28,20 @@ func TestRunner_TheTraceRecordsThatThePageNavigated(t *testing.T) { const url = "http://127.0.0.1/index.html?" navigating := &navigatingDriver{Driver: state.mock, url: url} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if _, err := Run(ctx, Options{ - Duration: 100 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 3, - Driver: navigating, - Verifier: state.verifier, - TraceWriter: state.writer, - }); err != nil { - t.Fatalf("Run: %v", err) - } + state.run(t, Options{Duration: 100 * time.Millisecond, MaxSteps: 3, Driver: navigating}) if err := state.writer.Close(); err != nil { t.Fatalf("close trace: %v", err) } - file, err := os.Open(filepath.Join(state.writer.Directory(), "trace.jsonl")) - if err != nil { - t.Fatal(err) - } - defer file.Close() - recorded := 0 - scanner := bufio.NewScanner(file) - scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) - for scanner.Scan() { - var step struct { - Index int `json:"step"` - Navigations []trace.Navigation `json:"navigations"` - } - if err := json.Unmarshal(scanner.Bytes(), &step); err != nil { - t.Fatalf("trace line decode: %v", err) - } + for _, step := range readTraceLines(t, state.writer.Directory()) { for _, navigation := range step.Navigations { recorded++ if navigation.URL != url { - t.Errorf("step %d records navigation to %q, want %q", step.Index, navigation.URL, url) + t.Errorf("step %d records navigation to %q, want %q", step.Step, navigation.URL, url) } if navigation.UnixMillis == 0 { - t.Errorf("step %d records a navigation with no timestamp", step.Index) + t.Errorf("step %d records a navigation with no timestamp", step.Step) } } } diff --git a/internal/runner/redaction_test.go b/internal/runner/redaction_test.go index 6b11cbe..fd3402b 100644 --- a/internal/runner/redaction_test.go +++ b/internal/runner/redaction_test.go @@ -5,7 +5,6 @@ import ( "fmt" "strings" "testing" - "time" mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock" @@ -160,20 +159,9 @@ func TestTheTraceNeverCarriesATypedSecretThroughALastActionExtractor(t *testing. state := newHarnessWithSpec(t, fmt.Sprintf(lastActionExtractorSpec, testCase.selector)) state.mock.HierarchyJSON = testCase.treeJSON - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - if _, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 2, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }); err != nil { - t.Fatalf("Run: %v", err) - } + state.run(t, Options{MaxSteps: 2}) - steps := traceSteps(t, state.writer.Directory()) + steps := readTraceLines(t, state.writer.Directory()) if len(steps) < 2 { t.Fatalf("trace holds %d step(s), want the step that reports the action back", len(steps)) } diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index dd3a1d3..1daf5ff 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -1,7 +1,6 @@ package runner import ( - "bufio" "bytes" "context" "encoding/json" @@ -178,21 +177,72 @@ func newHarnessWithSpec(t *testing.T, spec string) *harness { return state } -func TestRunner_HappyPathStepsAndTraces(t *testing.T) { - state := newHarness(t) +// harnessRunTimeout bounds every harness run. It is the backstop for a runner +// that stopped making progress, never the budget a test asserts on: what ends a +// run is its own Duration or MaxSteps. +const harnessRunTimeout = 60 * time.Second - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) +// tryRun fills in the wiring every run shares and hands back whatever Run +// decided, so a test that expects a failure keeps its own words for it. +func (h *harness) tryRun(t *testing.T, options Options) (Summary, error) { + t.Helper() + if options.Duration == 0 { + options.Duration = time.Hour + } + if options.IdleTimeout == 0 { + options.IdleTimeout = 20 * time.Millisecond + } + if options.Driver == nil { + options.Driver = h.mock + } + if options.Verifier == nil { + options.Verifier = h.verifier + } + if options.TraceWriter == nil { + options.TraceWriter = h.writer + } + ctx, cancel := context.WithTimeout(context.Background(), harnessRunTimeout) defer cancel() - summary, err := Run(ctx, Options{ - Duration: 100 * time.Millisecond, - IdleTimeout: 50 * time.Millisecond, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) + return Run(ctx, options) +} + +func (h *harness) run(t *testing.T, options Options) Summary { + t.Helper() + summary, err := h.tryRun(t, options) if err != nil { t.Fatalf("Run: %v", err) } + return summary +} + +// webDriverBase presents the mock device driver as a web target: the five calls +// the runner's V8 path makes, each answering the way a page carrying the current +// runtime does. A fake embeds it and overrides only the one it exists to change. +type webDriverBase struct { + *mockdriver.Driver +} + +func (d *webDriverBase) InstallBundle(context.Context, []byte) error { return nil } + +func (d *webDriverBase) EvaluateExtractors(context.Context) (map[int]json.RawMessage, error) { + return nil, nil +} + +func (d *webDriverBase) NextActionFromV8(context.Context) (json.RawMessage, error) { + return nil, nil +} + +func (d *webDriverBase) SetLastAction(context.Context, json.RawMessage) error { return nil } + +func (d *webDriverBase) SetLogs(context.Context, json.RawMessage) error { return nil } + +func TestRunner_HappyPathStepsAndTraces(t *testing.T) { + state := newHarness(t) + + summary := state.run(t, Options{ + Duration: 100 * time.Millisecond, + IdleTimeout: 50 * time.Millisecond, + }) if summary.Steps == 0 { t.Errorf("expected at least one step, got 0") } @@ -217,18 +267,7 @@ func TestRunner_HappyPathStepsAndTraces(t *testing.T) { func TestRunner_SeededRunRecordsNoModelCalls(t *testing.T) { state := newHarness(t) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if _, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 10 * time.Millisecond, - MaxSteps: 3, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }); err != nil { - t.Fatalf("Run: %v", err) - } + state.run(t, Options{IdleTimeout: 10 * time.Millisecond, MaxSteps: 3}) if _, err := os.Stat(filepath.Join(state.directory, trace.LLMCallFileName)); !os.IsNotExist(err) { t.Errorf("stat %s = %v, want no model-call file for a seeded run", trace.LLMCallFileName, err) } @@ -252,20 +291,11 @@ func TestRunner_SeededSetupActionsAreNotTheGeneratorDrivingTheApp(t *testing.T) state := newHarnessWithSpec(t, seededLoginSetupSpec) state.mock.HierarchyJSON = llmTreeJSON - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 30 * time.Second, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 4, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + summary := state.run(t, Options{ + Duration: 30 * time.Second, + MaxSteps: 4, + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), }) - if err != nil { - t.Fatalf("Run: %v", err) - } if summary.DispatchedActions != 4 { t.Errorf("DispatchedActions = %d, want 4: every step drove the app", summary.DispatchedActions) @@ -294,20 +324,12 @@ func TestRunner_MaxStepsStopsAfterExactlyNSteps(t *testing.T) { state := newHarness(t) const maxSteps = 3 - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() // A long duration ensures MaxSteps, not the deadline, ends the run. - summary, err := Run(ctx, Options{ + summary := state.run(t, Options{ Duration: time.Hour, IdleTimeout: 10 * time.Millisecond, MaxSteps: maxSteps, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, }) - if err != nil { - t.Fatalf("Run: %v", err) - } if summary.Steps != maxSteps { t.Errorf("expected exactly %d steps, got %d", maxSteps, summary.Steps) } @@ -369,20 +391,11 @@ func TestRunner_ReadsTheDriverCannotMakeAreNotPassedChecks(t *testing.T) { state.mock.Failures[mockdriver.ActionMetrics] = mockdriver.FailurePlan{Err: driver.ErrNotSupported} state.mock.Failures[mockdriver.ActionHealth] = mockdriver.FailurePlan{Err: driver.ErrNotSupported} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 5 * time.Second, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 3, - BundleID: "com.fixture", - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, + summary := state.run(t, Options{ + Duration: 5 * time.Second, + MaxSteps: 3, + BundleID: "com.fixture", }) - if err != nil { - t.Fatalf("Run: %v", err) - } if summary.Steps != 3 { t.Fatalf("Steps = %d, want 3", summary.Steps) } @@ -412,20 +425,11 @@ func TestRunner_ReadsTheDriverCannotMakeAreNotPassedChecks(t *testing.T) { func TestRunner_ADriverThatAnswersEveryReadReportsNoneMissing(t *testing.T) { state := newHarness(t) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 5 * time.Second, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 2, - BundleID: "com.fixture", - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, + summary := state.run(t, Options{ + Duration: 5 * time.Second, + MaxSteps: 2, + BundleID: "com.fixture", }) - if err != nil { - t.Fatalf("Run: %v", err) - } if len(summary.UnsupportedReads) != 0 { t.Errorf("UnsupportedReads = %v, want none", summary.UnsupportedReads) } @@ -439,18 +443,10 @@ func TestRunner_ADriverThatAnswersEveryReadReportsNoneMissing(t *testing.T) { func TestRunner_ViolationSurfacesInSummary(t *testing.T) { state := newHarnessWithSpec(t, violationSpec) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ + summary := state.run(t, Options{ Duration: 100 * time.Millisecond, IdleTimeout: 50 * time.Millisecond, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, }) - if err != nil { - t.Fatalf("Run: %v", err) - } if len(summary.Violations) == 0 { t.Errorf("expected at least one violation, got %v", summary.Violations) } @@ -466,18 +462,7 @@ func TestRunner_ViolationSurfacesOnlyOnOnsetStep(t *testing.T) { // lines, not on every subsequent step. state := newHarnessWithSpec(t, violationSpec) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 200 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{Duration: 200 * time.Millisecond}) if summary.Steps < 2 { t.Fatalf("need at least 2 steps to prove onset-only behavior, got %d", summary.Steps) } @@ -494,24 +479,8 @@ func TestRunner_ViolationSurfacesOnlyOnOnsetStep(t *testing.T) { summary.Violations[0].Properties) } - file, err := os.Open(filepath.Join(state.writer.Directory(), "trace.jsonl")) - if err != nil { - t.Fatal(err) - } - defer file.Close() - - type traceLine struct { - Step int `json:"step"` - Violations []string `json:"violations"` - } linesWithViolations := 0 - scanner := bufio.NewScanner(file) - scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) - for scanner.Scan() { - var line traceLine - if err := json.Unmarshal(scanner.Bytes(), &line); err != nil { - t.Fatalf("trace line decode: %v", err) - } + for _, line := range readTraceLines(t, state.writer.Directory()) { if len(line.Violations) == 0 { continue } @@ -521,9 +490,6 @@ func TestRunner_ViolationSurfacesOnlyOnOnsetStep(t *testing.T) { line.Step, line.Violations) } } - if err := scanner.Err(); err != nil { - t.Fatalf("scan trace: %v", err) - } if linesWithViolations != 1 { t.Errorf("expected exactly 1 trace line with violations, got %d", linesWithViolations) } @@ -545,19 +511,7 @@ globalThis.actions = actions(() => []); ` state := newHarnessWithSpec(t, nextViolationSpec) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 5, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{MaxSteps: 5}) if len(summary.Violations) != 1 { t.Fatalf("expected exactly one ViolationRecord, got %d: %v", len(summary.Violations), summary.Violations) @@ -570,25 +524,8 @@ globalThis.actions = actions(() => []); t.Errorf("properties: got %v, want [nextHolds]", summary.Violations[0].Properties) } - file, err := os.Open(filepath.Join(state.writer.Directory(), "trace.jsonl")) - if err != nil { - t.Fatal(err) - } - defer file.Close() - - type traceLine struct { - Step int `json:"step"` - Violations []string `json:"violations"` - Witnesses map[string]trace.Witness `json:"witnesses"` - } found := false - scanner := bufio.NewScanner(file) - scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) - for scanner.Scan() { - var line traceLine - if err := json.Unmarshal(scanner.Bytes(), &line); err != nil { - t.Fatalf("trace line decode: %v", err) - } + for _, line := range readTraceLines(t, state.writer.Directory()) { if len(line.Violations) == 0 { continue } @@ -609,9 +546,6 @@ globalThis.actions = actions(() => []); t.Errorf("witness detected step: got %d, want 3", witness.DetectedStep) } } - if err := scanner.Err(); err != nil { - t.Fatalf("scan trace: %v", err) - } if !found { t.Error("no trace line carried the violation") } @@ -630,19 +564,7 @@ globalThis.actions = actions(() => []); ` state := newHarnessWithSpec(t, spec) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 3, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{MaxSteps: 3}) if len(summary.Violations) != 0 { t.Errorf("expected no violations, got %v", summary.Violations) } @@ -662,42 +584,14 @@ globalThis.actions = actions(() => []); ` state := newHarnessWithSpec(t, spec) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 3, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{MaxSteps: 3}) if !containsProperty(summary.Violations, "neverFires") { t.Fatalf("expected neverFires in violations: %v", summary.Violations) } - file, err := os.Open(filepath.Join(state.writer.Directory(), "trace.jsonl")) - if err != nil { - t.Fatal(err) - } - defer file.Close() - - type traceLine struct { - Step int `json:"step"` - Violations []string `json:"violations"` - } seen := map[int]bool{} finalizeStep := 0 - scanner := bufio.NewScanner(file) - scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) - for scanner.Scan() { - var line traceLine - if err := json.Unmarshal(scanner.Bytes(), &line); err != nil { - t.Fatalf("trace line decode: %v", err) - } + for _, line := range readTraceLines(t, state.writer.Directory()) { if seen[line.Step] { t.Errorf("duplicate step index %d in trace", line.Step) } @@ -706,9 +600,6 @@ globalThis.actions = actions(() => []); finalizeStep = line.Step } } - if err := scanner.Err(); err != nil { - t.Fatalf("scan trace: %v", err) - } if finalizeStep != summary.Steps+1 { t.Errorf("finalize record step = %d, want %d (steps+1)", finalizeStep, summary.Steps+1) } @@ -727,19 +618,11 @@ globalThis.actions = actions(() => [Tap({ on: "id:next" })]); var buffer bytes.Buffer logger := slog.New(slog.NewTextHandler(&buffer, &slog.HandlerOptions{Level: slog.LevelWarn})) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ + summary := state.run(t, Options{ Duration: 100 * time.Millisecond, IdleTimeout: 50 * time.Millisecond, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, Logger: logger, }) - if err != nil { - t.Fatalf("Run: %v", err) - } if !containsProperty(summary.Violations, "broken") { t.Errorf("expected broken in violations: %v", summary.Violations) } @@ -770,17 +653,7 @@ func TestRunner_StampsHierarchyResolvedBoundsAndResiduals(t *testing.T) { state := newHarness(t) state.mock.HierarchyJSON = `{"attributes":{"resource-id":"com.fixture:id/next","bounds":"[40,80,240,160]"},"children":[],"clickable":true,"enabled":true}` - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if _, err := Run(ctx, Options{ - Duration: 100 * time.Millisecond, - IdleTimeout: 50 * time.Millisecond, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }); err != nil { - t.Fatalf("Run: %v", err) - } + state.run(t, Options{Duration: 100 * time.Millisecond, IdleTimeout: 50 * time.Millisecond}) body, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl")) if err != nil { t.Fatal(err) @@ -882,18 +755,11 @@ func TestRunner_LogsWaitForIdleDriverErrors(t *testing.T) { var logBuf bytes.Buffer logger := slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if _, err := Run(ctx, Options{ + state.run(t, Options{ Duration: 100 * time.Millisecond, IdleTimeout: 50 * time.Millisecond, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, Logger: logger, - }); err != nil { - t.Fatalf("Run: %v", err) - } + }) output := logBuf.String() if !strings.Contains(output, "wait_for_idle failed") { t.Errorf("expected wait_for_idle warning, got: %q", output) @@ -1519,19 +1385,7 @@ func TestApplyAction_NonDispatchPathsReportWhy(t *testing.T) { func TestRunner_RecordsWhyAChosenActionNeverRan(t *testing.T) { state := newHarnessWithSpec(t, zeroWaitSpec) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 5 * time.Second, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 2, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{Duration: 5 * time.Second, MaxSteps: 2}) if summary.Steps != 2 { t.Fatalf("Steps = %d, want 2", summary.Steps) } @@ -1560,18 +1414,7 @@ func TestRunner_RecordsWhyAChosenActionNeverRan(t *testing.T) { func TestRunner_DispatchedActionRecordsNoSkipReason(t *testing.T) { state := newHarness(t) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if _, err := Run(ctx, Options{ - Duration: 5 * time.Second, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 2, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }); err != nil { - t.Fatalf("Run: %v", err) - } + state.run(t, Options{Duration: 5 * time.Second, MaxSteps: 2}) if !containsAction(state.mock.Actions(), mockdriver.ActionTapSelector, "id:next") { t.Fatalf("fixture tap never dispatched, got %v", state.mock.Actions()) } @@ -1591,19 +1434,7 @@ func TestRunner_DispatchedActionRecordsNoSkipReason(t *testing.T) { func TestRunner_ASourceAskedAndHandedNothingSaysSo(t *testing.T) { state := newHarnessWithSpec(t, noActionSpec) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 5 * time.Second, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 3, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{Duration: 5 * time.Second, MaxSteps: 3}) if summary.Steps != 3 { t.Fatalf("Steps = %d, want 3", summary.Steps) } @@ -1641,19 +1472,7 @@ func TestRunner_AHeldStepIsNotRecordedAsASourceThatDeclined(t *testing.T) { state := newHarness(t) state.mock.Failures[mockdriver.ActionSnapshot] = mockdriver.FailurePlan{Err: errors.New("adb: device offline")} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 5 * time.Second, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 3, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{Duration: 5 * time.Second, MaxSteps: 3}) if summary.SkippedVerification != 3 { t.Fatalf("SkippedVerification = %d, want 3", summary.SkippedVerification) } @@ -1672,12 +1491,19 @@ func TestRunner_AHeldStepIsNotRecordedAsASourceThatDeclined(t *testing.T) { } type traceStepLine struct { - Step int `json:"step"` - NextAction *trace.Action `json:"next_action"` - ActionSkipped string `json:"action_skipped"` - Transitional bool `json:"transitional"` - ObservationError string `json:"observation_error"` - SkippedVerification bool `json:"skipped_verification"` + Step int `json:"step"` + TraceVersion int `json:"trace_version"` + NextAction *trace.Action `json:"next_action"` + ActionSkipped string `json:"action_skipped"` + Transitional bool `json:"transitional"` + ObservationError string `json:"observation_error"` + SkippedVerification bool `json:"skipped_verification"` + Violations []string `json:"violations"` + ExtractorChanges map[string]trace.ExtractorChange `json:"extractor_changes"` + Witnesses map[string]trace.Witness `json:"witnesses"` + Navigations []trace.Navigation `json:"navigations"` + Logs []trace.LogEntry `json:"logs"` + Exceptions []trace.Exception `json:"exceptions"` } func readTraceLines(t *testing.T, directory string) []traceStepLine { @@ -1704,19 +1530,11 @@ func TestRunner_ParallelFetchCallsAllDriverMethods(t *testing.T) { {UnixMillis: 1000, Level: "E", Tag: "test", Message: "boom"}, } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := Run(ctx, Options{ + state.run(t, Options{ Duration: 100 * time.Millisecond, IdleTimeout: 50 * time.Millisecond, BundleID: "com.fixture", - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, }) - if err != nil { - t.Fatalf("Run: %v", err) - } actions := state.mock.Actions() var hasSnapshot, hasMetrics, hasLogs bool @@ -1749,18 +1567,7 @@ func TestRunner_UsesAtomicSnapshot(t *testing.T) { state := newHarness(t) state.mock.ImageData = driver.Image{PNG: []byte("png"), Width: 1, Height: 1} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 100 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{Duration: 100 * time.Millisecond}) if summary.Steps == 0 { t.Fatal("expected at least one step") } @@ -1801,18 +1608,10 @@ func TestRunner_OneScreenshotPerStep(t *testing.T) { state := newHarness(t) state.mock.ImageData = driver.Image{PNG: []byte("fakepng"), Width: 100, Height: 200} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ + summary := state.run(t, Options{ Duration: 200 * time.Millisecond, IdleTimeout: 50 * time.Millisecond, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, }) - if err != nil { - t.Fatalf("Run: %v", err) - } if summary.Steps < 2 { t.Fatalf("need at least 2 steps for screenshot test, got %d", summary.Steps) } @@ -1849,18 +1648,7 @@ func TestRunner_StableTransitionalTreeIsVerified(t *testing.T) { ]}` state.mock.ImageData = driver.Image{PNG: []byte("fakepng"), Width: 100, Height: 200} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 200 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{Duration: 200 * time.Millisecond}) if summary.Steps == 0 { t.Fatal("expected at least one step") } @@ -1868,19 +1656,7 @@ func TestRunner_StableTransitionalTreeIsVerified(t *testing.T) { t.Fatalf("expected verifier to run on a stable two-screen tree, got %v", summary.Violations) } - type traceLine struct { - Step int `json:"step"` - Transitional bool `json:"transitional"` - } - body, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl")) - if err != nil { - t.Fatal(err) - } - for _, raw := range bytes.Split(bytes.TrimSpace(body), []byte("\n")) { - var line traceLine - if err := json.Unmarshal(raw, &line); err != nil { - t.Fatalf("decode trace line: %v", err) - } + for _, line := range readTraceLines(t, state.writer.Directory()) { if line.Transitional { t.Errorf("step %d: stable tree must not be marked transitional", line.Step) } @@ -1918,18 +1694,7 @@ func TestRunner_GenuineCrossFadeStillRetried(t *testing.T) { state := newHarnessWithSpec(t, violationSpec) wrapped := &snapshotCrossFade{Driver: state.mock} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 200 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - Driver: wrapped, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{Duration: 200 * time.Millisecond, Driver: wrapped}) if summary.Steps == 0 { t.Fatal("expected at least one step") } @@ -1937,21 +1702,8 @@ func TestRunner_GenuineCrossFadeStillRetried(t *testing.T) { t.Fatalf("verifier must be skipped on cross-fade steps; got %v", summary.Violations) } - type traceLine struct { - Step int `json:"step"` - Transitional bool `json:"transitional"` - Violations []string `json:"violations"` - } - body, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl")) - if err != nil { - t.Fatal(err) - } lines := 0 - for _, raw := range bytes.Split(bytes.TrimSpace(body), []byte("\n")) { - var line traceLine - if err := json.Unmarshal(raw, &line); err != nil { - t.Fatalf("decode trace line: %v", err) - } + for _, line := range readTraceLines(t, state.writer.Directory()) { lines++ if !line.Transitional { t.Errorf("step %d: expected transitional=true on every step, got false", line.Step) @@ -1972,57 +1724,18 @@ func TestRunner_CleanTreeStillVerified(t *testing.T) { state := newHarnessWithSpec(t, violationSpec) state.mock.HierarchyJSON = `{"attributes":{"resource-id":"HomeScreen"},"children":[]}` - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 200 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{Duration: 200 * time.Millisecond}) if !containsProperty(summary.Violations, "balanceNonNegative") { t.Fatalf("expected verifier to surface balanceNonNegative on a clean tree, got %v", summary.Violations) } - type traceLine struct { - Step int `json:"step"` - Transitional bool `json:"transitional"` - } - body, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl")) - if err != nil { - t.Fatal(err) - } - for _, raw := range bytes.Split(bytes.TrimSpace(body), []byte("\n")) { - var line traceLine - if err := json.Unmarshal(raw, &line); err != nil { - t.Fatalf("decode trace line: %v", err) - } + for _, line := range readTraceLines(t, state.writer.Directory()) { if line.Transitional { t.Errorf("step %d: clean tree must not be marked transitional", line.Step) } } } -// snapshotFailFirst wraps a mock driver so the first Snapshot call returns an -// error (mimicking a sidecar timeout while fetching view hierarchy), then -// delegates every subsequent call back to the mock. -type snapshotFailFirst struct { - *mockdriver.Driver - calls int -} - -func (d *snapshotFailFirst) Snapshot(ctx context.Context) (string, driver.Image, error) { - d.calls++ - if d.calls == 1 { - return "", driver.Image{}, errors.New("Timeout while fetching view hierarchy") - } - return d.Driver.Snapshot(ctx) -} - // TestRunner_NilHierarchyMarksTransitional verifies that when the sidecar's // hierarchy fetch fails (nil tree), the runner marks the step transitional and // skips the verifier instead of pushing a nil tree that would crash the spec. @@ -2030,20 +1743,12 @@ func (d *snapshotFailFirst) Snapshot(ctx context.Context) (string, driver.Image, func TestRunner_NilHierarchyMarksTransitional(t *testing.T) { state := newHarnessWithSpec(t, violationSpec) state.mock.HierarchyJSON = `{"attributes":{"resource-id":"HomeScreen"},"children":[]}` - wrapped := &snapshotFailFirst{Driver: state.mock} - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 200 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - Driver: wrapped, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) + state.mock.Failures[mockdriver.ActionSnapshot] = mockdriver.FailurePlan{ + Err: errors.New("Timeout while fetching view hierarchy"), + OnCalls: []int{1}, } + + summary := state.run(t, Options{Duration: 200 * time.Millisecond}) if summary.Steps < 2 { t.Fatalf("need at least 2 steps to verify the first is skipped and the second runs, got %d", summary.Steps) } @@ -2056,19 +1761,7 @@ func TestRunner_NilHierarchyMarksTransitional(t *testing.T) { t.Errorf("onset step: got %d, want 2 (step 1 verifier skipped due to nil tree)", summary.Violations[0].StepIndex) } - type traceLine struct { - Step int `json:"step"` - Transitional bool `json:"transitional"` - Violations []string `json:"violations"` - } - body, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl")) - if err != nil { - t.Fatal(err) - } - var first traceLine - if err := json.Unmarshal(bytes.SplitN(bytes.TrimSpace(body), []byte("\n"), 2)[0], &first); err != nil { - t.Fatalf("decode first trace line: %v", err) - } + first := readTraceLines(t, state.writer.Directory())[0] if first.Step != 1 { t.Fatalf("first trace line step: got %d, want 1", first.Step) } @@ -2080,42 +1773,23 @@ func TestRunner_NilHierarchyMarksTransitional(t *testing.T) { } } -// tapSelectorFailFirst wraps a mock driver so the first TapSelector call -// returns a gRPC DeadlineExceeded error (mimicking a sidecar-side RPC hang), -// then delegates every subsequent call back to the mock. -type tapSelectorFailFirst struct { - *mockdriver.Driver - calls int -} - -func (d *tapSelectorFailFirst) TapSelector(ctx context.Context, selector string) error { - d.calls++ - if d.calls == 1 { - return status.Error(codes.DeadlineExceeded, "boom") - } - return d.Driver.TapSelector(ctx, selector) -} - // TestRunner_TransientApplyErrorMarksTransitional verifies that a transient // gRPC error from applyAction (e.g. sidecar RPC deadline) does not kill the // run: the step is marked transitional, the verifier is skipped for it, and // the loop continues with the next step running cleanly. func TestRunner_TransientApplyErrorMarksTransitional(t *testing.T) { state := newHarness(t) - wrapped := &tapSelectorFailFirst{Driver: state.mock} + state.mock.Failures[mockdriver.ActionTapSelector] = mockdriver.FailurePlan{ + Err: status.Error(codes.DeadlineExceeded, "boom"), + OnCalls: []int{1}, + } var logBuf bytes.Buffer logger := slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 300 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - Driver: wrapped, - Verifier: state.verifier, - TraceWriter: state.writer, - Logger: logger, + summary, err := state.tryRun(t, Options{ + Duration: 300 * time.Millisecond, + Logger: logger, }) if err != nil { t.Fatalf("Run must not return on transient apply error, got %v", err) @@ -2130,36 +1804,15 @@ func TestRunner_TransientApplyErrorMarksTransitional(t *testing.T) { t.Errorf("expected apply-error WARN log, got %q", logBuf.String()) } - type traceLine struct { - Step int `json:"step"` - Transitional bool `json:"transitional"` - Violations []string `json:"violations"` - } - body, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl")) - if err != nil { - t.Fatal(err) - } - lines := bytes.Split(bytes.TrimSpace(body), []byte("\n")) - var first, second traceLine - if err := json.Unmarshal(lines[0], &first); err != nil { - t.Fatalf("decode first trace line: %v", err) - } - if err := json.Unmarshal(lines[1], &second); err != nil { - t.Fatalf("decode second trace line: %v", err) - } + lines := readTraceLines(t, state.writer.Directory()) + first, second := lines[0], lines[1] if first.Step != 1 || !first.Transitional { t.Errorf("step 1 must be transitional after transient apply error, got step=%d transitional=%v", first.Step, first.Transitional) } // The step still records a next_action it never dispatched, so the reason // has to be on the line or an executed-action count includes it. - var firstSkip struct { - ActionSkipped string `json:"action_skipped"` - } - if err := json.Unmarshal(lines[0], &firstSkip); err != nil { - t.Fatalf("decode first trace line: %v", err) - } - if firstSkip.ActionSkipped != string(actionSkippedApplyError) { - t.Errorf("step 1 action_skipped = %q, want %q", firstSkip.ActionSkipped, actionSkippedApplyError) + if first.ActionSkipped != string(actionSkippedApplyError) { + t.Errorf("step 1 action_skipped = %q, want %q", first.ActionSkipped, actionSkippedApplyError) } if len(first.Violations) != 0 { t.Errorf("transient apply step must have no violations, got %v", first.Violations) @@ -2169,39 +1822,18 @@ func TestRunner_TransientApplyErrorMarksTransitional(t *testing.T) { } } -// internalApplyErrorFailFirst wraps a mock driver so the first InputText call -// fails with the bare Internal error the iOS runner's input handler emits -// when it chokes (HTTP 500 with an empty body), then recovers. -type internalApplyErrorFailFirst struct { - *mockdriver.Driver - calls int -} - -func (d *internalApplyErrorFailFirst) TapSelector(ctx context.Context, selector string) error { - d.calls++ - if d.calls == 1 { - return status.Error(codes.Internal, "UnknownFailure(errorResponse=Request for inputText failed, code: 500, body: )") - } - return d.Driver.TapSelector(ctx, selector) -} - // TestRunner_InternalApplyErrorMarksTransitional pins the policy that a // one-off device-side failure (e.g. the iOS input handler's bare 500) is // absorbed as a transitional step instead of killing the run. Persistent // failure is covered by the consecutive-failure cap. func TestRunner_InternalApplyErrorMarksTransitional(t *testing.T) { state := newHarness(t) - wrapped := &internalApplyErrorFailFirst{Driver: state.mock} + state.mock.Failures[mockdriver.ActionTapSelector] = mockdriver.FailurePlan{ + Err: status.Error(codes.Internal, "UnknownFailure(errorResponse=Request for inputText failed, code: 500, body: )"), + OnCalls: []int{1}, + } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 300 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - Driver: wrapped, - Verifier: state.verifier, - TraceWriter: state.writer, - }) + summary, err := state.tryRun(t, Options{Duration: 300 * time.Millisecond}) if err != nil { t.Fatalf("Run must not return on a one-off internal apply error, got %v", err) } @@ -2253,33 +1885,16 @@ func TestIsWDADrop_Classification(t *testing.T) { } } -// tapSelectorAlwaysUnavailable wraps a mock driver so every TapSelector call -// fails with a transient Unavailable error, mimicking a device whose channel -// never recovers between steps. -type tapSelectorAlwaysUnavailable struct { - *mockdriver.Driver -} - -func (d *tapSelectorAlwaysUnavailable) TapSelector(ctx context.Context, selector string) error { - return status.Error(codes.Unavailable, "connection dropped mid-action; the action may have applied") -} - // TestRunner_ConsecutiveTransientApplyFailuresAbort verifies the run fails // fast once transient apply errors form an unbroken streak instead of burning // the whole budget on a wedged device. func TestRunner_ConsecutiveTransientApplyFailuresAbort(t *testing.T) { state := newHarness(t) - wrapped := &tapSelectorAlwaysUnavailable{Driver: state.mock} + state.mock.Failures[mockdriver.ActionTapSelector] = mockdriver.FailurePlan{ + Err: status.Error(codes.Unavailable, "connection dropped mid-action; the action may have applied"), + } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 5 * time.Second, - IdleTimeout: 20 * time.Millisecond, - Driver: wrapped, - Verifier: state.verifier, - TraceWriter: state.writer, - }) + summary, err := state.tryRun(t, Options{Duration: 5 * time.Second}) if err == nil { t.Fatal("Run must abort after consecutive transient apply failures") } @@ -2301,18 +1916,7 @@ import { actions, Wait } from "@sanderling/spec"; globalThis.actions = actions(() => [Wait({ durationMillis: 5 })]); ` state := newHarnessWithSpec(t, waitSpec) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := Run(ctx, Options{ - Duration: 150 * time.Millisecond, - IdleTimeout: 50 * time.Millisecond, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + state.run(t, Options{Duration: 150 * time.Millisecond, IdleTimeout: 50 * time.Millisecond}) for _, action := range state.mock.Actions() { if action.Kind == mockdriver.ActionWaitForIdle { t.Fatalf("Wait action must skip WaitForIdle, got: %v", action) @@ -2379,19 +1983,7 @@ func TestRunner_RelaunchesWhenAppLeavesForeground(t *testing.T) { // relaunch. state.mock.ForegroundResults = []string{"app.folio", "com.android.chrome"} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := Run(ctx, Options{ - Duration: 100 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - BundleID: "app.folio", - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + state.run(t, Options{Duration: 100 * time.Millisecond, BundleID: "app.folio"}) relaunches := 0 for _, a := range state.mock.Actions() { if a.Kind == mockdriver.ActionLaunch && a.BundleID == "app.folio" && !a.ClearState { @@ -2407,19 +1999,7 @@ func TestRunner_NoRelaunchWhenAppInForeground(t *testing.T) { state := newHarness(t) state.mock.ForegroundResults = []string{"app.folio"} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := Run(ctx, Options{ - Duration: 100 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - BundleID: "app.folio", - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + state.run(t, Options{Duration: 100 * time.Millisecond, BundleID: "app.folio"}) for _, a := range state.mock.Actions() { if a.Kind == mockdriver.ActionLaunch { t.Fatalf("expected no relaunch while app in foreground, got %v", a) @@ -2436,19 +2016,7 @@ func TestRunner_WaitsForForegroundBeforeFirstAction(t *testing.T) { // First the device shows a system setup screen, then the app is on top. state.mock.ForegroundResults = []string{"com.google.android.setupwizard", "app.folio"} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := Run(ctx, Options{ - Duration: 100 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - BundleID: "app.folio", - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + state.run(t, Options{Duration: 100 * time.Millisecond, BundleID: "app.folio"}) actions := state.mock.Actions() firstLaunch, firstTap := -1, -1 @@ -2487,19 +2055,7 @@ func TestRunner_WaitsForWindowDrawnBeforeFirstAction(t *testing.T) { state.mock.ForegroundResults = []string{"app.folio"} state.mock.FocusedWindowResults = []string{"com.android.settings", "com.android.settings", "app.folio"} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := Run(ctx, Options{ - Duration: 100 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - BundleID: "app.folio", - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + state.run(t, Options{Duration: 100 * time.Millisecond, BundleID: "app.folio"}) // The gate must have polled the focused window until it named the app, // i.e. at least the three queued results were consumed. @@ -2692,38 +2248,16 @@ func TestRunner_SkipsActionWhenOverlayStealsFocusAtApplyTime(t *testing.T) { state.mock.ForegroundResults = []string{"app.folio"} state.mock.FocusedWindowResults = []string{"app.folio", "com.android.systemui"} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 100 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - BundleID: "app.folio", - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{Duration: 100 * time.Millisecond, BundleID: "app.folio"}) if summary.Steps == 0 { t.Fatal("expected the loop to run steps") } if containsAction(state.mock.Actions(), mockdriver.ActionTapSelector, "id:next") { t.Error("apply-time guard failed: a tap fired while a system overlay held focus") } - body, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl")) - if err != nil { - t.Fatal(err) - } var skipped bool - for _, line := range bytes.Split(bytes.TrimSpace(body), []byte("\n")) { - var step struct { - ActionSkipped string `json:"action_skipped"` - } - if err := json.Unmarshal(line, &step); err != nil { - t.Fatalf("decode trace line: %v", err) - } - skipped = skipped || step.ActionSkipped == string(actionSkippedForeground) + for _, line := range readTraceLines(t, state.writer.Directory()) { + skipped = skipped || line.ActionSkipped == string(actionSkippedForeground) } if !skipped { t.Errorf("no step recorded action_skipped=%q, so the undispatched action looks executed", actionSkippedForeground) @@ -2746,20 +2280,7 @@ globalThis.actions = actions(() => []); ` state := newHarnessWithSpec(t, thirdStepViolationSpec) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 8, - StopOnViolation: true, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{MaxSteps: 8, StopOnViolation: true}) if !containsProperty(summary.Violations, "staysUnderThree") { t.Fatalf("expected staysUnderThree to fire, got %v", summary.Violations) } @@ -2781,19 +2302,7 @@ globalThis.actions = actions(() => []); func TestRunner_WithoutStopOnViolationRunsTheWholeBudget(t *testing.T) { state := newHarnessWithSpec(t, violationSpec) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 4, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{MaxSteps: 4}) if summary.Steps != 4 { t.Errorf("steps: got %d, want 4; a violation must not shorten a default run", summary.Steps) } @@ -2803,26 +2312,10 @@ func TestRunner_WithoutStopOnViolationRunsTheWholeBudget(t *testing.T) { // assert on what the run actually wrote rather than on the summary alone. func traceStepIndices(t *testing.T, directory string) []int { t.Helper() - file, err := os.Open(filepath.Join(directory, "trace.jsonl")) - if err != nil { - t.Fatal(err) - } - defer file.Close() var steps []int - scanner := bufio.NewScanner(file) - scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) - for scanner.Scan() { - var line struct { - Step int `json:"step"` - } - if err := json.Unmarshal(scanner.Bytes(), &line); err != nil { - t.Fatalf("trace line decode: %v", err) - } + for _, line := range readTraceLines(t, directory) { steps = append(steps, line.Step) } - if err := scanner.Err(); err != nil { - t.Fatalf("scan trace: %v", err) - } return steps } @@ -2890,41 +2383,19 @@ func TestRunner_SiblingTapsReachTheDriverAtTheirOwnCoordinates(t *testing.T) { } } -// tapReachesNoElement wraps a mock driver so every tap reports what the chrome -// driver reports when the action's point holds no element: nothing was -// dispatched, so the app cannot have responded. -type tapReachesNoElement struct { - *mockdriver.Driver -} - -func (d *tapReachesNoElement) TapSelector( - _ context.Context, - selector string, -) error { - return fmt.Errorf("%w: %s", driver.ErrGestureUndelivered, selector) -} - -func (d *tapReachesNoElement) Tap(_ context.Context, x, y int) error { - return fmt.Errorf("%w: (%d,%d)", driver.ErrGestureUndelivered, x, y) -} - // TestRunner_UndeliveredGestureIsRecordedNotSilentlyClean is the runner half of // the silent-actuation bug: a run whose every tap reached nothing used to look // exactly like a run that exercised the app and found no violations. The step // now names the reason, and the run says how many actions did nothing. func TestRunner_UndeliveredGestureIsRecordedNotSilentlyClean(t *testing.T) { state := newHarness(t) - wrapped := &tapReachesNoElement{Driver: state.mock} + state.mock.Failures[mockdriver.ActionTapSelector] = mockdriver.FailurePlan{ + Err: fmt.Errorf("%w: id:next", driver.ErrGestureUndelivered), + } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 10 * time.Second, - MaxSteps: maxConsecutiveApplyFailures + 2, - IdleTimeout: 20 * time.Millisecond, - Driver: wrapped, - Verifier: state.verifier, - TraceWriter: state.writer, + summary, err := state.tryRun(t, Options{ + Duration: 10 * time.Second, + MaxSteps: maxConsecutiveApplyFailures + 2, }) if err != nil { t.Fatalf( @@ -2956,22 +2427,7 @@ func TestRunner_UndeliveredGestureIsRecordedNotSilentlyClean(t *testing.T) { ) } - type traceLine struct { - Step int `json:"step"` - ActionSkipped string `json:"action_skipped"` - Transitional bool `json:"transitional"` - } - body, err := os.ReadFile( - filepath.Join(state.writer.Directory(), "trace.jsonl"), - ) - if err != nil { - t.Fatal(err) - } - for _, raw := range bytes.Split(bytes.TrimSpace(body), []byte("\n")) { - var line traceLine - if err := json.Unmarshal(raw, &line); err != nil { - t.Fatalf("decode trace line: %v", err) - } + for _, line := range readTraceLines(t, state.writer.Directory()) { if line.ActionSkipped != "gesture_undelivered" { t.Errorf("step %d: action_skipped = %q, want %q", line.Step, line.ActionSkipped, "gesture_undelivered") @@ -2986,16 +2442,6 @@ func TestRunner_UndeliveredGestureIsRecordedNotSilentlyClean(t *testing.T) { } } -// selectorMatchesNothing wraps a mock driver so every by-selector tap reports -// what the drivers report when the selector names no element on the screen. -type selectorMatchesNothing struct { - *mockdriver.Driver -} - -func (d *selectorMatchesNothing) TapSelector(_ context.Context, selector string) error { - return fmt.Errorf("%w: %q", driver.ErrSelectorMatchedNothing, selector) -} - // TestRunner_SelectorThatMatchesNothingIsRecordedApartFromAnUndeliveredGesture // keeps the two silent paths distinguishable. A selector that named no element // is a resolution failure, so the step records unresolved_selector, keeps the @@ -3003,17 +2449,13 @@ func (d *selectorMatchesNothing) TapSelector(_ context.Context, selector string) // budget that a wedged device is meant to exhaust. func TestRunner_SelectorThatMatchesNothingIsRecordedApartFromAnUndeliveredGesture(t *testing.T) { state := newHarnessWithSpec(t, absentSelectorSpec) - wrapped := &selectorMatchesNothing{Driver: state.mock} + state.mock.Failures[mockdriver.ActionTapSelector] = mockdriver.FailurePlan{ + Err: fmt.Errorf("%w: %q", driver.ErrSelectorMatchedNothing, "id:absent"), + } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 10 * time.Second, - MaxSteps: maxConsecutiveApplyFailures + 2, - IdleTimeout: 20 * time.Millisecond, - Driver: wrapped, - Verifier: state.verifier, - TraceWriter: state.writer, + summary, err := state.tryRun(t, Options{ + Duration: 10 * time.Second, + MaxSteps: maxConsecutiveApplyFailures + 2, }) if err != nil { t.Fatalf("a selector that matched nothing is not a device fault: %v", err) @@ -3034,20 +2476,7 @@ func TestRunner_SelectorThatMatchesNothingIsRecordedApartFromAnUndeliveredGestur t.Errorf("the summary must name the actions that found no target, got:\n%s", rendered.String()) } - type traceLine struct { - Step int `json:"step"` - ActionSkipped string `json:"action_skipped"` - Transitional bool `json:"transitional"` - } - body, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl")) - if err != nil { - t.Fatal(err) - } - for _, raw := range bytes.Split(bytes.TrimSpace(body), []byte("\n")) { - var line traceLine - if err := json.Unmarshal(raw, &line); err != nil { - t.Fatalf("decode trace line: %v", err) - } + for _, line := range readTraceLines(t, state.writer.Directory()) { if line.ActionSkipped != "unresolved_selector" { t.Errorf("step %d: action_skipped = %q, want %q", line.Step, line.ActionSkipped, "unresolved_selector") @@ -3086,8 +2515,11 @@ func TestApplyAction_TapAboveTheViewportReachesTheDriver(t *testing.T) { // element under is still a failure, reported as ErrGestureUndelivered rather // than as an action the runner declined to try. func TestApplyAction_TapAboveTheViewportKeepsTheUndeliveredReport(t *testing.T) { - drv := &tapReachesNoElement{Driver: mockdriver.New()} - skipped, err := applyAction(context.Background(), drv, verifier.Action{ + mock := mockdriver.New() + mock.Failures[mockdriver.ActionTap] = mockdriver.FailurePlan{ + Err: fmt.Errorf("%w: (622,-208)", driver.ErrGestureUndelivered), + } + skipped, err := applyAction(context.Background(), mock, verifier.Action{ Kind: verifier.ActionKindTap, X: 622, Y: -208, @@ -3107,19 +2539,7 @@ func TestApplyAction_TapAboveTheViewportKeepsTheUndeliveredReport(t *testing.T) func TestRenderSummary_NamesTheActionsThatNeverReachedTheApp(t *testing.T) { state := newHarnessWithSpec(t, zeroWaitSpec) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 5 * time.Second, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 3, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{Duration: 5 * time.Second, MaxSteps: 3}) reason := string(actionSkippedZeroDurationWait) if summary.SkippedActions[reason] != 3 { t.Errorf("SkippedActions[%s] = %d, want 3", reason, summary.SkippedActions[reason]) @@ -3168,16 +2588,7 @@ func TestRunner_AnActionThatNeverReturnsFailsTheStepNotTheRun(t *testing.T) { t.Cleanup(func() { applyTimeout = previousApplyTimeout }) wrapped := &wedgedTapSelector{Driver: state.mock} - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 3, - Driver: wrapped, - Verifier: state.verifier, - TraceWriter: state.writer, - }) + summary, err := state.tryRun(t, Options{MaxSteps: 3, Driver: wrapped}) if err != nil { t.Fatalf("Run must outlive an action that never returns, got %v", err) } @@ -3230,19 +2641,7 @@ func TestRunner_AFailedObservationIsNotAnObservationOfAnEmptyScreen(t *testing.T state.mock.HierarchyJSON = `{"attributes":{"resource-id":"HomeScreen"},"children":[]}` wrapped := &snapshotFailThenEmpty{Driver: state.mock} - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 3, - Driver: wrapped, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{MaxSteps: 3, Driver: wrapped}) if summary.Steps != 3 { t.Fatalf("Steps = %d, want 3", summary.Steps) } diff --git a/internal/runner/uncertain_last_action_test.go b/internal/runner/uncertain_last_action_test.go index cd122f5..47ed1e7 100644 --- a/internal/runner/uncertain_last_action_test.go +++ b/internal/runner/uncertain_last_action_test.go @@ -7,7 +7,6 @@ import ( "path/filepath" "sync/atomic" "testing" - "time" "github.com/priyanshujain/sanderling/internal/driver" mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock" @@ -106,19 +105,7 @@ func TestRunner_ApplyErrorAfterDispatchDoesNotConvictTheSubmitCountingProperty(t state := newHarnessWithSpec(t, spec) device := &dispatchThenFailDriver{Driver: state.mock, commitsPerTap: commitsPerTap} - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 2, - Driver: device, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{MaxSteps: 2, Driver: device}) if summary.Steps != 2 { t.Fatalf("steps = %d, want 2; the run never reached the step that judges the pair", summary.Steps) } diff --git a/internal/runner/web_carrier_test.go b/internal/runner/web_carrier_test.go index e47a8fb..a575ed7 100644 --- a/internal/runner/web_carrier_test.go +++ b/internal/runner/web_carrier_test.go @@ -6,15 +6,11 @@ import ( "encoding/json" "errors" "fmt" - "os" - "path/filepath" "strconv" "testing" "time" "github.com/priyanshujain/sanderling/internal/driver" - mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock" - "github.com/priyanshujain/sanderling/internal/trace" ) // carrierSpec registers one extractor whose value the page supplies. It stands @@ -33,7 +29,7 @@ globalThis.actions = actions(() => []); // whose page-side extractor advances a counter on every evaluation - exactly // what a spec-authored carrier does in V8. type carrierWebDriver struct { - *mockdriver.Driver + webDriverBase transitional bool snapshots int reads int @@ -53,8 +49,6 @@ func (d *carrierWebDriver) Snapshot(ctx context.Context) (string, driver.Image, ]}`, d.snapshots), image, err } -func (d *carrierWebDriver) InstallBundle(context.Context, []byte) error { return nil } - // A web target says so. The runner's per-step hierarchy reread is android-only, // and a fake claiming android would take a path no chrome run takes. func (d *carrierWebDriver) Health(context.Context) (driver.Health, error) { @@ -73,10 +67,6 @@ func (d *carrierWebDriver) NextActionFromV8(context.Context) (json.RawMessage, e return json.RawMessage(`{"kind":"Tap","x":5,"y":5}`), nil } -func (d *carrierWebDriver) SetLastAction(context.Context, json.RawMessage) error { return nil } - -func (d *carrierWebDriver) SetLogs(context.Context, json.RawMessage) error { return nil } - // TestRunner_TransitionalStepNeverAdvancesThePageCarrier pins the ordering the // web path depends on. The page-side extractors must run only on steps the // verifier accepts: their getters advance spec state every time they evaluate, @@ -86,41 +76,16 @@ func (d *carrierWebDriver) SetLogs(context.Context, json.RawMessage) error { ret // convicts an app that did nothing wrong. func TestRunner_TransitionalStepNeverAdvancesThePageCarrier(t *testing.T) { state := newHarnessWithSpec(t, carrierSpec) - web := &carrierWebDriver{Driver: state.mock} + web := &carrierWebDriver{webDriverBase: webDriverBase{Driver: state.mock}} - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 30 * time.Second, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 5, - Driver: web, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{Duration: 30 * time.Second, MaxSteps: 5, Driver: web}) if summary.Steps != 5 { t.Fatalf("steps = %d, want 5", summary.Steps) } - type traceLine struct { - Step int `json:"step"` - Transitional bool `json:"transitional"` - ExtractorChanges map[string]trace.ExtractorChange `json:"extractor_changes"` - } - body, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl")) - if err != nil { - t.Fatal(err) - } verified, transitional := 0, 0 previous := 0 - for _, raw := range bytes.Split(bytes.TrimSpace(body), []byte("\n")) { - var line traceLine - if err := json.Unmarshal(raw, &line); err != nil { - t.Fatalf("decode trace line: %v", err) - } + for _, line := range readTraceLines(t, state.writer.Directory()) { if line.Transitional { transitional++ continue @@ -157,11 +122,9 @@ func TestRunner_TransitionalStepNeverAdvancesThePageCarrier(t *testing.T) { // lastAction: an older published @sanderling/spec runtime, a bundle that never // installed, a tab that navigated away from it. type installFailsWebDriver struct { - *mockdriver.Driver + webDriverBase } -func (d *installFailsWebDriver) InstallBundle(context.Context, []byte) error { return nil } - func (d *installFailsWebDriver) EvaluateExtractors(context.Context) (map[int]json.RawMessage, error) { return map[int]json.RawMessage{0: json.RawMessage(`1`)}, nil } @@ -174,8 +137,6 @@ func (d *installFailsWebDriver) SetLastAction(context.Context, json.RawMessage) return errors.New("__sanderlingSetLastAction__ is not a function") } -func (d *installFailsWebDriver) SetLogs(context.Context, json.RawMessage) error { return nil } - // TestRunner_LastActionInstallFailureFailsTheRun covers the other half of the // same trust boundary. A run that cannot install lastAction in the page cannot // apply the page's extractor values either, so the step keeps goja's @@ -184,18 +145,9 @@ func (d *installFailsWebDriver) SetLogs(context.Context, json.RawMessage) error // green run reporting a violation nobody can reproduce. func TestRunner_LastActionInstallFailureFailsTheRun(t *testing.T) { state := newHarnessWithSpec(t, carrierSpec) - web := &installFailsWebDriver{Driver: state.mock} + web := &installFailsWebDriver{webDriverBase: webDriverBase{Driver: state.mock}} - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _, err := Run(ctx, Options{ - Duration: 2 * time.Second, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 3, - Driver: web, - Verifier: state.verifier, - TraceWriter: state.writer, - }) + _, err := state.tryRun(t, Options{Duration: 2 * time.Second, MaxSteps: 3, Driver: web}) if err == nil { t.Fatal("Run succeeded with a page that cannot take lastAction; the run " + "reported green while its extractor values came from two engines") @@ -229,19 +181,10 @@ func (d *logInstallFailsWebDriver) SetLogs(context.Context, json.RawMessage) err func TestRunner_LogInstallFailureFailsTheRun(t *testing.T) { state := newHarnessWithSpec(t, carrierSpec) web := &logInstallFailsWebDriver{ - installFailsWebDriver: &installFailsWebDriver{Driver: state.mock}, + installFailsWebDriver: &installFailsWebDriver{webDriverBase: webDriverBase{Driver: state.mock}}, } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _, err := Run(ctx, Options{ - Duration: 2 * time.Second, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 3, - Driver: web, - Verifier: state.verifier, - TraceWriter: state.writer, - }) + _, err := state.tryRun(t, Options{Duration: 2 * time.Second, MaxSteps: 3, Driver: web}) if err == nil { t.Fatal("Run succeeded with a page that cannot take the step's logs; " + "every property reading the log stream ran against an empty array") diff --git a/internal/runner/web_extractor_trace_test.go b/internal/runner/web_extractor_trace_test.go index d2c54e4..7adab05 100644 --- a/internal/runner/web_extractor_trace_test.go +++ b/internal/runner/web_extractor_trace_test.go @@ -1,17 +1,11 @@ package runner import ( - "bufio" "context" "encoding/json" - "os" - "path/filepath" "strings" "testing" "time" - - mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock" - "github.com/priyanshujain/sanderling/internal/trace" ) // engineDisagreementSpec makes the two runtimes disagree on purpose: the @@ -31,24 +25,14 @@ globalThis.actions = actions(() => []); // takes the V8 path, where extractor values come from the page rather than // from goja. type webMockDriver struct { - *mockdriver.Driver + webDriverBase overrides map[int]json.RawMessage } -func (d *webMockDriver) InstallBundle(context.Context, []byte) error { return nil } - func (d *webMockDriver) EvaluateExtractors(context.Context) (map[int]json.RawMessage, error) { return d.overrides, nil } -func (d *webMockDriver) NextActionFromV8(context.Context) (json.RawMessage, error) { - return nil, nil -} - -func (d *webMockDriver) SetLastAction(context.Context, json.RawMessage) error { return nil } - -func (d *webMockDriver) SetLogs(context.Context, json.RawMessage) error { return nil } - // TestRunner_TraceRecordsTheValueTheVerdictUsed fails if the trace and the // verdict disagree about an extractor. A witness is only an explanation of a // violation if it holds the state the violated property was evaluated against. @@ -56,46 +40,18 @@ func TestRunner_TraceRecordsTheValueTheVerdictUsed(t *testing.T) { state := newHarnessWithSpec(t, engineDisagreementSpec) const pageValue = `"v8"` web := &webMockDriver{ - Driver: state.mock, - overrides: map[int]json.RawMessage{0: json.RawMessage(pageValue)}, + webDriverBase: webDriverBase{Driver: state.mock}, + overrides: map[int]json.RawMessage{0: json.RawMessage(pageValue)}, } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - summary, err := Run(ctx, Options{ - Duration: 100 * time.Millisecond, - IdleTimeout: 20 * time.Millisecond, - Driver: web, - Verifier: state.verifier, - TraceWriter: state.writer, - }) - if err != nil { - t.Fatalf("Run: %v", err) - } + summary := state.run(t, Options{Duration: 100 * time.Millisecond, Driver: web}) if !containsProperty(summary.Violations, "ranInGoja") { t.Fatalf("ranInGoja did not violate, so the page value never reached the verdict: %v", summary.Violations) } - file, err := os.Open(filepath.Join(state.writer.Directory(), "trace.jsonl")) - if err != nil { - t.Fatal(err) - } - defer file.Close() - - type traceLine struct { - Step int `json:"step"` - ExtractorChanges map[string]trace.ExtractorChange `json:"extractor_changes"` - Witnesses map[string]trace.Witness `json:"witnesses"` - } changes, witnesses := 0, 0 - scanner := bufio.NewScanner(file) - scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) - for scanner.Scan() { - var line traceLine - if err := json.Unmarshal(scanner.Bytes(), &line); err != nil { - t.Fatalf("trace line decode: %v", err) - } + for _, line := range readTraceLines(t, state.writer.Directory()) { if change, ok := line.ExtractorChanges["engine"]; ok { changes++ if got := string(change.Curr); got != pageValue { @@ -111,9 +67,6 @@ func TestRunner_TraceRecordsTheValueTheVerdictUsed(t *testing.T) { } } } - if err := scanner.Err(); err != nil { - t.Fatalf("scan trace: %v", err) - } if changes == 0 { t.Error("no extractor change reached the trace; nothing was compared") } @@ -143,20 +96,11 @@ globalThis.actions = actions(() => []); func TestRunner_PartialExtractorTableIsFatal(t *testing.T) { state := newHarnessWithSpec(t, splitTableSpec) web := &webMockDriver{ - Driver: state.mock, - overrides: map[int]json.RawMessage{0: json.RawMessage(`"v8"`)}, + webDriverBase: webDriverBase{Driver: state.mock}, + overrides: map[int]json.RawMessage{0: json.RawMessage(`"v8"`)}, } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 2, - Driver: web, - Verifier: state.verifier, - TraceWriter: state.writer, - }) + _, err := state.tryRun(t, Options{MaxSteps: 2, Driver: web}) if err == nil { t.Fatal("the run completed on a page that reported 1 of 2 extractors; " + "the second extractor silently kept goja's value") diff --git a/internal/runner/web_last_action_test.go b/internal/runner/web_last_action_test.go index 02d9617..2857d24 100644 --- a/internal/runner/web_last_action_test.go +++ b/internal/runner/web_last_action_test.go @@ -8,7 +8,6 @@ import ( "log/slog" "strings" "testing" - "time" "github.com/priyanshujain/sanderling/internal/driver" mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock" @@ -29,17 +28,11 @@ globalThis.properties = {}; // tappingWebDriver is a web target whose V8 picker always taps one named // control, so the runner has a real applied action to report on the next step. type tappingWebDriver struct { - *mockdriver.Driver + webDriverBase installed []string installedLogs []string } -func (d *tappingWebDriver) InstallBundle(context.Context, []byte) error { return nil } - -func (d *tappingWebDriver) EvaluateExtractors(context.Context) (map[int]json.RawMessage, error) { - return nil, nil -} - func (d *tappingWebDriver) NextActionFromV8(context.Context) (json.RawMessage, error) { return json.RawMessage(`{"kind":"Tap","x":12,"y":34,"selector":"id:TxnSubmit"}`), nil } @@ -56,20 +49,9 @@ func (d *tappingWebDriver) SetLogs(_ context.Context, encoded json.RawMessage) e func TestRunner_WebInstallsLastActionInThePage(t *testing.T) { state := newHarnessWithSpec(t, lastActionSpec) - web := &tappingWebDriver{Driver: state.mock} + web := &tappingWebDriver{webDriverBase: webDriverBase{Driver: state.mock}} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if _, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 3, - Driver: web, - Verifier: state.verifier, - TraceWriter: state.writer, - }); err != nil { - t.Fatalf("Run: %v", err) - } + state.run(t, Options{MaxSteps: 3, Driver: web}) if len(web.installed) < 2 { t.Fatalf("the page was handed lastAction %d time(s); the web path never installed it", @@ -97,20 +79,9 @@ func TestRunner_WebInstallsTheStepsLogsInThePage(t *testing.T) { state.mock.LogEntries = []driver.LogEntry{ {UnixMillis: 1700000000123, Level: "E", Tag: "console", Message: "boom from the page"}, } - web := &tappingWebDriver{Driver: state.mock} + web := &tappingWebDriver{webDriverBase: webDriverBase{Driver: state.mock}} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if _, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 2, - Driver: web, - Verifier: state.verifier, - TraceWriter: state.writer, - }); err != nil { - t.Fatalf("Run: %v", err) - } + state.run(t, Options{MaxSteps: 2, Driver: web}) if len(web.installedLogs) == 0 { t.Fatal("the page was never handed the step's logs; every property reading " + @@ -135,19 +106,7 @@ func TestRunner_ReportsALogFetchItCouldNotMake(t *testing.T) { var buffer bytes.Buffer logger := slog.New(slog.NewTextHandler(&buffer, &slog.HandlerOptions{Level: slog.LevelWarn})) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if _, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 2, - Driver: state.mock, - Verifier: state.verifier, - TraceWriter: state.writer, - Logger: logger, - }); err != nil { - t.Fatalf("Run: %v", err) - } + state.run(t, Options{MaxSteps: 2, Logger: logger}) if !strings.Contains(buffer.String(), "adb: device offline") { t.Errorf("the run never reported the failed log fetch, so noLogcatErrors "+ @@ -170,21 +129,12 @@ func (d *failingTapWebDriver) Tap(context.Context, int, int) error { // decides for itself, which it cannot do if the page is handed a bare null. func TestRunner_WebInstallsAnUnconfirmedActionWithItsFateUnknown(t *testing.T) { state := newHarnessWithSpec(t, lastActionSpec) - web := &failingTapWebDriver{tappingWebDriver: &tappingWebDriver{Driver: state.mock}} - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if _, err := Run(ctx, Options{ - Duration: time.Hour, - IdleTimeout: 20 * time.Millisecond, - MaxSteps: 2, - Driver: web, - Verifier: state.verifier, - TraceWriter: state.writer, - }); err != nil { - t.Fatalf("Run: %v", err) + web := &failingTapWebDriver{ + tappingWebDriver: &tappingWebDriver{webDriverBase: webDriverBase{Driver: state.mock}}, } + state.run(t, Options{MaxSteps: 2, Driver: web}) + if len(web.installed) < 2 { t.Fatalf("the page was handed lastAction %d time(s); the web path never installed it", len(web.installed))