diff --git a/internal/testrun/testrun.go b/internal/testrun/testrun.go index bd5cf5f..adcbc0a 100644 --- a/internal/testrun/testrun.go +++ b/internal/testrun/testrun.go @@ -66,10 +66,15 @@ type Options struct { // "the run found the bug" from "the run finished clean". ExitOnViolation bool // AllowNoProperties lets a run proceed against a spec that registers no - // properties. The extraction and portability sweeps pass it: they measure - // what a spec can read and where the generator reaches, and they report no - // detection count. Every other run without it is a false green. + // properties. The extraction sweeps pass it: they measure what a spec can + // read and report no detection count. Every other run without it is a false + // green. AllowNoProperties bool + // AllowNoGeneratorActions lets a run finish having never been driven by its + // action generator. The exploration-reach sweeps pass it, because "the + // generator reached nothing on this build" is their measurement rather than + // a broken run. + AllowNoGeneratorActions bool // Generator selects the action picker: "llm" or the default seeded picker. Generator string // LabelSource selects how candidates are named to the model picker, and is @@ -293,7 +298,7 @@ func runOutcome(options Options, summary runner.Summary) error { if options.ExitOnViolation && len(summary.Violations) > 0 { return ViolationsError{Count: len(summary.Violations)} } - if !options.AllowNoProperties && len(summary.Violations) == 0 && + if !options.AllowNoGeneratorActions && len(summary.Violations) == 0 && summary.Steps > 0 && summary.GeneratorActions == 0 { return NoGeneratorActionsError{ Steps: summary.Steps, @@ -363,7 +368,7 @@ func (e NoPropertiesError) Error() string { "%s bundled and loaded into the verifier cleanly and registers no properties: "+ "nothing is wrong with the spec and nothing is wrong with the run, but this run "+ "would check nothing and report no violations. Pass --allow-no-properties for a "+ - "run that measures extraction or exploration instead of judging the app", + "run that measures what the spec extracts instead of judging the app", e.Spec) } @@ -387,7 +392,8 @@ func (e NoGeneratorActionsError) Error() string { "%d step(s) ran and the action generator drove the app in none of them: whatever "+ "the spec's setup did to get the app into position, nothing explored it from "+ "there, so the run judged one screen over and over and its violation count "+ - "says nothing about the rest of the app%s", + "says nothing about the rest of the app%s. Pass --allow-no-generator-actions "+ + "for a run that measures where a generator reaches instead of judging the app", e.Steps, skipReasonSuffix(e.SkippedActions)) } diff --git a/internal/testrun/testrun_test.go b/internal/testrun/testrun_test.go index f8cda07..c83f846 100644 --- a/internal/testrun/testrun_test.go +++ b/internal/testrun/testrun_test.go @@ -411,11 +411,11 @@ func TestRunOutcome_ARunThatDroveNothingIsNotASuccess(t *testing.T) { t.Errorf("a run whose generator dispatched one action must succeed, got %v", err) } - // The sweeps that measure what a spec extracts and where a generator reaches - // ask for a run that judges nothing by name, and "the generator reached - // nothing here" is their measurement rather than their failure. - if err := runOutcome(Options{AllowNoProperties: true}, droveNothing); err != nil { - t.Errorf("the property-free opt-out no longer carries a run through, got %v", err) + // The sweeps that measure where a generator reaches ask for a run that + // explores nothing by name, and "the generator reached nothing here" is + // their measurement rather than their failure. + if err := runOutcome(Options{AllowNoGeneratorActions: true}, droveNothing); err != nil { + t.Errorf("the dead-run opt-out no longer carries a run through, got %v", err) } // A run cut short before it took a step never got going, which the deadline @@ -473,6 +473,25 @@ func TestRunOutcome_SetupActionsDoNotCarryARunWhoseGeneratorDroveNothing(t *test } } +// Two refusals, two flags. A sweep that runs a property-free spec asked to +// judge nothing, not to explore nothing, and a run with properties that needs +// the dead-run exemption must be able to say so without claiming a waiver it +// does not want. +func TestRunOutcome_TheDeadRunRefusalHasItsOwnOptOut(t *testing.T) { + droveNothing := runner.Summary{ + Steps: 200, + SkippedActions: map[string]int{"no_action_produced": 200}, + } + + var dead NoGeneratorActionsError + if err := runOutcome(Options{AllowNoProperties: true}, droveNothing); !errors.As(err, &dead) { + t.Errorf("--allow-no-properties waived the dead-run refusal, which it does not name: got %v", err) + } + if err := runOutcome(Options{AllowNoGeneratorActions: true}, droveNothing); err != nil { + t.Errorf("--allow-no-generator-actions did not carry a run that drove nothing through, got %v", err) + } +} + // A recorded violation is a verdict, and a run that reached one is not a dead // run whatever drove the app to it. Campaigns are where this bites: they never // pass --exit-on-violation, so refusing such a run writes exit_code 1 into the