fix(testrun): the refusal asks whether the generator drove, not whether anything did

A dead provider against folio exited 0 on a real emulator: the login
setup dispatched three actions before the generator was consulted, so
DispatchedActions was 3 and the gate never fired while the generator
drove the app zero times across 83 steps. Any spec with a login setup
was immune, which is the normal case.

Summary counts generator actions separately and the refusal reads that.
NoActionsDispatchedError becomes NoGeneratorActionsError, because a run
that dispatched three login taps was lying in the old name.
This commit is contained in:
pj committed 2026-08-18 16:29:03 +05:30
1 parent 5a36b583ea
commit 58168a6a3e
6 files changed
+274 -29

No files matched your search

+50 -8
View File
@@ -299,9 +299,10 @@ func TestRunOutcome_ReportsViolationsOnlyUnderTheFlag(t *testing.T) {
violated := runner.Summary{
Steps: 7,
DispatchedActions: 7,
GeneratorActions: 7,
Violations: []runner.ViolationRecord{{StepIndex: 3, Properties: []string{"balanceMoves"}}},
}
clean := runner.Summary{Steps: 7, DispatchedActions: 7}
clean := runner.Summary{Steps: 7, DispatchedActions: 7, GeneratorActions: 7}
if err := runOutcome(Options{}, violated); err != nil {
t.Errorf("without --exit-on-violation a violated run must succeed, got %v", err)
@@ -339,7 +340,12 @@ func TestRunOutcome_ARunThatJudgedNothingIsNotASuccess(t *testing.T) {
// A screen that composes now and then costs a run steps, not its verdict. A
// check that fired here would turn every healthy android run red.
mostlyJudged := runner.Summary{Steps: 6, SkippedVerification: 5, DispatchedActions: 1}
mostlyJudged := runner.Summary{
Steps: 6,
SkippedVerification: 5,
DispatchedActions: 1,
GeneratorActions: 1,
}
if err := runOutcome(Options{}, mostlyJudged); err != nil {
t.Errorf("a run that judged one of its 6 steps must succeed, got %v", err)
}
@@ -355,10 +361,10 @@ func TestRunOutcome_ARunThatDroveNothingIsNotASuccess(t *testing.T) {
SkippedActions: map[string]int{"no_action_produced": 200},
}
err := runOutcome(Options{}, droveNothing)
var dead NoActionsDispatchedError
var dead NoGeneratorActionsError
if !errors.As(err, &dead) {
t.Fatalf("a run that dispatched none of its 200 steps' actions came back %v, "+
"want a NoActionsDispatchedError", err)
"want a NoGeneratorActionsError", err)
}
if dead.Steps != 200 {
t.Errorf("steps: got %d, want 200", dead.Steps)
@@ -367,11 +373,12 @@ func TestRunOutcome_ARunThatDroveNothingIsNotASuccess(t *testing.T) {
t.Errorf("the error never names why nothing was dispatched: %v", dead)
}
// One action is exploration, however little. A check that fired here would
// be red on any run whose screen offers the generator nothing for a while.
droveOnce := runner.Summary{Steps: 200, DispatchedActions: 1}
// One generator action is exploration, however little. A check that fired
// here would be red on any run whose screen offers the generator nothing for
// a while.
droveOnce := runner.Summary{Steps: 200, DispatchedActions: 1, GeneratorActions: 1}
if err := runOutcome(Options{}, droveOnce); err != nil {
t.Errorf("a run that dispatched one action must succeed, got %v", err)
t.Errorf("a run whose generator dispatched one action must succeed, got %v", err)
}
// The sweeps that measure what a spec extracts and where a generator reaches
@@ -401,6 +408,41 @@ func TestRunOutcome_ARunThatDroveNothingIsNotASuccess(t *testing.T) {
}
}
// A spec whose setup logs in drives the app before the generator is ever
// consulted, so a run whose every model call failed still reached the driver a
// few times. Those actions are the harness getting into position: counting them
// as the run driving the app passed 86 steps of folio that explored nothing.
func TestRunOutcome_SetupActionsDoNotCarryARunWhoseGeneratorDroveNothing(t *testing.T) {
loginThenNothing := runner.Summary{
Steps: 86,
DispatchedActions: 3,
GeneratorActions: 0,
SkippedActions: map[string]int{"no_action_produced": 83},
}
err := runOutcome(Options{}, loginThenNothing)
var dead NoGeneratorActionsError
if !errors.As(err, &dead) {
t.Fatalf("a run whose 3 dispatched actions all came from setup came back %v, "+
"want a NoGeneratorActionsError", err)
}
if dead.Steps != 86 {
t.Errorf("steps: got %d, want 86", dead.Steps)
}
if !strings.Contains(dead.Error(), "no_action_produced") {
t.Errorf("the error never names why the generator dispatched nothing: %v", dead)
}
// The same run under --exit-on-violation still reports its evidence: CI
// reads exit 2 as "the run found the bug", and a setup-only run that found
// one must not be downgraded to a broken harness.
found := loginThenNothing
found.Violations = []runner.ViolationRecord{{StepIndex: 2, Properties: []string{"balanceMoves"}}}
var violations ViolationsError
if err := runOutcome(Options{ExitOnViolation: true}, found); !errors.As(err, &violations) {
t.Errorf("a setup-only run that found a violation came back %v, want a ViolationsError", err)
}
}
// wedgedLaunchDriver never returns from Launch, standing in for a driver whose
// device-side session is stuck.
type wedgedLaunchDriver struct {