fix(testrun): the dead-run refusal gets its own opt-out

--allow-no-properties was waiving two unrelated refusals, so a sweep passing
it for the property-free reason silently lost a detector it never asked to
disable, and a run with properties could only get the dead-run exemption by
claiming one it did not want.
This commit is contained in:
pj committed 2026-08-18 20:10:29 +05:30
1 parent aa49c3bc34
commit 93614a48b9
2 files changed
+36 -11

No files matched your search

+12 -6
View File
@@ -66,10 +66,15 @@ type Options struct {
// "the run found the bug" from "the run finished clean".
ExitOnViolation bool
// AllowNoProperties lets a run proceed against a spec that registers no
// properties. The extraction and portability sweeps pass it: they measure
// what a spec can read and where the generator reaches, and they report no
// detection count. Every other run without it is a false green.
// properties. The extraction sweeps pass it: they measure what a spec can
// read and report no detection count. Every other run without it is a false
// green.
AllowNoProperties bool
// AllowNoGeneratorActions lets a run finish having never been driven by its
// action generator. The exploration-reach sweeps pass it, because "the
// generator reached nothing on this build" is their measurement rather than
// a broken run.
AllowNoGeneratorActions bool
// Generator selects the action picker: "llm" or the default seeded picker.
Generator string
// LabelSource selects how candidates are named to the model picker, and is
@@ -293,7 +298,7 @@ func runOutcome(options Options, summary runner.Summary) error {
if options.ExitOnViolation && len(summary.Violations) > 0 {
return ViolationsError{Count: len(summary.Violations)}
}
if !options.AllowNoProperties && len(summary.Violations) == 0 &&
if !options.AllowNoGeneratorActions && len(summary.Violations) == 0 &&
summary.Steps > 0 && summary.GeneratorActions == 0 {
return NoGeneratorActionsError{
Steps: summary.Steps,
@@ -363,7 +368,7 @@ func (e NoPropertiesError) Error() string {
"%s bundled and loaded into the verifier cleanly and registers no properties: "+
"nothing is wrong with the spec and nothing is wrong with the run, but this run "+
"would check nothing and report no violations. Pass --allow-no-properties for a "+
"run that measures extraction or exploration instead of judging the app",
"run that measures what the spec extracts instead of judging the app",
e.Spec)
}
@@ -387,7 +392,8 @@ func (e NoGeneratorActionsError) Error() string {
"%d step(s) ran and the action generator drove the app in none of them: whatever "+
"the spec's setup did to get the app into position, nothing explored it from "+
"there, so the run judged one screen over and over and its violation count "+
"says nothing about the rest of the app%s",
"says nothing about the rest of the app%s. Pass --allow-no-generator-actions "+
"for a run that measures where a generator reaches instead of judging the app",
e.Steps, skipReasonSuffix(e.SkippedActions))
}
+24 -5
View File
@@ -411,11 +411,11 @@ func TestRunOutcome_ARunThatDroveNothingIsNotASuccess(t *testing.T) {
t.Errorf("a run whose generator dispatched one action must succeed, got %v", err)
}
// The sweeps that measure what a spec extracts and where a generator reaches
// ask for a run that judges nothing by name, and "the generator reached
// nothing here" is their measurement rather than their failure.
if err := runOutcome(Options{AllowNoProperties: true}, droveNothing); err != nil {
t.Errorf("the property-free opt-out no longer carries a run through, got %v", err)
// The sweeps that measure where a generator reaches ask for a run that
// explores nothing by name, and "the generator reached nothing here" is
// their measurement rather than their failure.
if err := runOutcome(Options{AllowNoGeneratorActions: true}, droveNothing); err != nil {
t.Errorf("the dead-run opt-out no longer carries a run through, got %v", err)
}
// A run cut short before it took a step never got going, which the deadline
@@ -473,6 +473,25 @@ func TestRunOutcome_SetupActionsDoNotCarryARunWhoseGeneratorDroveNothing(t *test
}
}
// Two refusals, two flags. A sweep that runs a property-free spec asked to
// judge nothing, not to explore nothing, and a run with properties that needs
// the dead-run exemption must be able to say so without claiming a waiver it
// does not want.
func TestRunOutcome_TheDeadRunRefusalHasItsOwnOptOut(t *testing.T) {
droveNothing := runner.Summary{
Steps: 200,
SkippedActions: map[string]int{"no_action_produced": 200},
}
var dead NoGeneratorActionsError
if err := runOutcome(Options{AllowNoProperties: true}, droveNothing); !errors.As(err, &dead) {
t.Errorf("--allow-no-properties waived the dead-run refusal, which it does not name: got %v", err)
}
if err := runOutcome(Options{AllowNoGeneratorActions: true}, droveNothing); err != nil {
t.Errorf("--allow-no-generator-actions did not carry a run that drove nothing through, got %v", err)
}
}
// A recorded violation is a verdict, and a run that reached one is not a dead
// run whatever drove the app to it. Campaigns are where this bites: they never
// pass --exit-on-violation, so refusing such a run writes exit_code 1 into the