mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-03 19:47:10 +00:00
fix(testrun): a run that judged nothing is not a green run
every step skipped means no property ever evaluated, so no violations is the absence of a verdict rather than a clean one. the hold makes that reachable now, so the run says it instead of exiting 0.
This commit is contained in:
1 parent
8e37be2afd
commit
33c585de7b
2 files changed
+49
No files matched your search
@@ -242,7 +242,15 @@ func Execute(ctx context.Context, options Options, stdout io.Writer) error {
|
||||
// --exit-on-violation a run that found violations is still a successful run
|
||||
// (the summary reports them), which is the behaviour every existing caller
|
||||
// depends on.
|
||||
//
|
||||
// A run none of whose steps reached the verifier fails whatever the flags say,
|
||||
// because it holds no verdict to report. The threshold is every step and not a
|
||||
// fraction of them: a screen that composes now and then costs a healthy android
|
||||
// run a step or two, and a check that fired on those would be red on every run.
|
||||
func runOutcome(options Options, summary runner.Summary) error {
|
||||
if summary.Steps > 0 && summary.SkippedVerification == summary.Steps {
|
||||
return VacuousRunError{Steps: summary.Steps}
|
||||
}
|
||||
if options.ExitOnViolation && len(summary.Violations) > 0 {
|
||||
return ViolationsError{Count: len(summary.Violations)}
|
||||
}
|
||||
@@ -261,6 +269,22 @@ func (e ViolationsError) Error() string {
|
||||
return fmt.Sprintf("%d violation record(s)", e.Count)
|
||||
}
|
||||
|
||||
// VacuousRunError reports a run in which no step reached the verifier, so no
|
||||
// property ever judged anything. It is not a clean run and it is not a found
|
||||
// bug: it is a run that produced no evidence either way, and the absence of
|
||||
// violations in it says nothing about the app. It stays untyped to the CLI's
|
||||
// violation path on purpose, so it exits 1 as a broken run rather than 2.
|
||||
type VacuousRunError struct {
|
||||
Steps int
|
||||
}
|
||||
|
||||
func (e VacuousRunError) Error() string {
|
||||
return fmt.Sprintf(
|
||||
"%d step(s) ran and none of them reached the verifier: the screen was "+
|
||||
"still moving every time it was read, so no property judged this run",
|
||||
e.Steps)
|
||||
}
|
||||
|
||||
// bundleInputs holds the pre-driver assembly: alias map, seed, esbuild defines,
|
||||
// and the resolved spec-API/goja-runtime paths the bundler consumes.
|
||||
type bundleInputs struct {
|
||||
|
||||
Reference in new issue
Block a user