diff --git a/internal/runner/runner.go b/internal/runner/runner.go index de98cb6..529db61 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -220,7 +220,7 @@ func Run(ctx context.Context, options Options) (Summary, error) { } logger.Info("step", "index", stepIndex, "screen", screen, "nodes", treeSize) - nextAction, nextErr := actionSource.NextAction(ctx) + nextAction, nextErr := actionSource.NextAction(ctx, stepIndex) var traceAction *trace.Action if nextErr == nil { traceAction = traceActionFor(nextAction, tree) @@ -235,6 +235,7 @@ func Run(ctx context.Context, options Options) (Summary, error) { } applySkipped := false + actionSkipped := "" if nextErr == nil && !appIsForeground(ctx, options) { // The app left the foreground between observe and apply (a prior // action's gesture settling late, or an async navigation). The @@ -244,6 +245,7 @@ func Run(ctx context.Context, options Options) (Summary, error) { logger.Warn("app not in foreground at action time; skipping (relaunch next step)", "step", stepIndex, "action", nextAction.Kind) applySkipped = true + actionSkipped = actionSkippedForeground lastAction = nil } else if nextErr == nil { if err := applyAction(ctx, options.Driver, nextAction, tree); err != nil { @@ -266,6 +268,7 @@ func Run(ctx context.Context, options Options) (Summary, error) { logger.Warn("apply error; marking step transitional", "step", stepIndex, "err", err) transitional = true applySkipped = true + actionSkipped = actionSkippedApplyError lastAction = nil } else { consecutiveApplyFailures = 0 @@ -287,6 +290,7 @@ func Run(ctx context.Context, options Options) (Summary, error) { Metrics: metrics, ExtractorChanges: extractorChanges, Transitional: transitional, + ActionSkipped: actionSkipped, SkippedVerification: skippedVerification, Witnesses: witnesses, } @@ -1073,6 +1077,14 @@ func encodeResiduals(residuals map[string]ltl.Formula) (map[string]json.RawMessa return encoded, firstErr } +// Reasons a chosen action was never dispatched, recorded on the step so a count +// of executed actions is not inflated by the next_action of a step that acted on +// nothing. +const ( + actionSkippedForeground = "app_left_foreground" + actionSkippedApplyError = "apply_error" +) + // maxConsecutiveApplyFailures bounds how many transient apply failures in a // row the run tolerates before aborting. One or two absorb a runner restart; // an unbroken streak means the device is wedged and the rest of the budget diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 0e884f4..35a9847 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -45,9 +45,10 @@ globalThis.actions = actions(() => []); ` type harness struct { - mock *mockdriver.Driver - verifier *verifier.Verifier - writer *trace.Writer + mock *mockdriver.Driver + verifier *verifier.Verifier + writer *trace.Writer + directory string } func newHarness(t *testing.T) *harness { @@ -103,9 +104,10 @@ func newHarnessWithSpec(t *testing.T, spec string) *harness { t.Fatal(err) } state := &harness{ - mock: mockdriver.New(), - verifier: verifierInstance, - writer: writer, + mock: mockdriver.New(), + verifier: verifierInstance, + writer: writer, + directory: directory, } t.Cleanup(func() { _ = writer.Close() }) return state @@ -144,6 +146,29 @@ func TestRunner_HappyPathStepsAndTraces(t *testing.T) { } } +// TestRunner_SeededRunRecordsNoModelCalls keeps the arms distinguishable: the +// seeded picker consults nothing, so its run directory must carry no model-call +// output at all rather than a file of empty records. +func TestRunner_SeededRunRecordsNoModelCalls(t *testing.T) { + state := newHarness(t) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := Run(ctx, Options{ + Duration: time.Hour, + IdleTimeout: 10 * time.Millisecond, + MaxSteps: 3, + Driver: state.mock, + Verifier: state.verifier, + TraceWriter: state.writer, + }); err != nil { + t.Fatalf("Run: %v", err) + } + if _, err := os.Stat(filepath.Join(state.directory, trace.LLMCallFileName)); !os.IsNotExist(err) { + t.Errorf("stat %s = %v, want no model-call file for a seeded run", trace.LLMCallFileName, err) + } +} + func TestRunner_MaxStepsStopsAfterExactlyNSteps(t *testing.T) { state := newHarness(t) @@ -1437,6 +1462,17 @@ func TestRunner_TransientApplyErrorMarksTransitional(t *testing.T) { if first.Step != 1 || !first.Transitional { t.Errorf("step 1 must be transitional after transient apply error, got step=%d transitional=%v", first.Step, first.Transitional) } + // The step still records a next_action it never dispatched, so the reason + // has to be on the line or an executed-action count includes it. + var firstSkip struct { + ActionSkipped string `json:"action_skipped"` + } + if err := json.Unmarshal(lines[0], &firstSkip); err != nil { + t.Fatalf("decode first trace line: %v", err) + } + if firstSkip.ActionSkipped != actionSkippedApplyError { + t.Errorf("step 1 action_skipped = %q, want %q", firstSkip.ActionSkipped, actionSkippedApplyError) + } if len(first.Violations) != 0 { t.Errorf("transient apply step must have no violations, got %v", first.Violations) } @@ -1975,4 +2011,21 @@ func TestRunner_SkipsActionWhenOverlayStealsFocusAtApplyTime(t *testing.T) { if containsAction(state.mock.Actions(), mockdriver.ActionTapSelector, "id:next") { t.Error("apply-time guard failed: a tap fired while a system overlay held focus") } + body, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl")) + if err != nil { + t.Fatal(err) + } + var skipped bool + for _, line := range bytes.Split(bytes.TrimSpace(body), []byte("\n")) { + var step struct { + ActionSkipped string `json:"action_skipped"` + } + if err := json.Unmarshal(line, &step); err != nil { + t.Fatalf("decode trace line: %v", err) + } + skipped = skipped || step.ActionSkipped == actionSkippedForeground + } + if !skipped { + t.Errorf("no step recorded action_skipped=%q, so the undispatched action looks executed", actionSkippedForeground) + } }