mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 11:07:10 +00:00
fix(testrun): a run that judged nothing is not a green run
every step skipped means no property ever evaluated, so no violations is the absence of a verdict rather than a clean one. the hold makes that reachable now, so the run says it instead of exiting 0.
This commit is contained in:
1 parent
8e37be2afd
commit
33c585de7b
2 files changed
+49
No files matched your search
@@ -242,7 +242,15 @@ func Execute(ctx context.Context, options Options, stdout io.Writer) error {
|
||||
// --exit-on-violation a run that found violations is still a successful run
|
||||
// (the summary reports them), which is the behaviour every existing caller
|
||||
// depends on.
|
||||
//
|
||||
// A run none of whose steps reached the verifier fails whatever the flags say,
|
||||
// because it holds no verdict to report. The threshold is every step and not a
|
||||
// fraction of them: a screen that composes now and then costs a healthy android
|
||||
// run a step or two, and a check that fired on those would be red on every run.
|
||||
func runOutcome(options Options, summary runner.Summary) error {
|
||||
if summary.Steps > 0 && summary.SkippedVerification == summary.Steps {
|
||||
return VacuousRunError{Steps: summary.Steps}
|
||||
}
|
||||
if options.ExitOnViolation && len(summary.Violations) > 0 {
|
||||
return ViolationsError{Count: len(summary.Violations)}
|
||||
}
|
||||
@@ -261,6 +269,22 @@ func (e ViolationsError) Error() string {
|
||||
return fmt.Sprintf("%d violation record(s)", e.Count)
|
||||
}
|
||||
|
||||
// VacuousRunError reports a run in which no step reached the verifier, so no
|
||||
// property ever judged anything. It is not a clean run and it is not a found
|
||||
// bug: it is a run that produced no evidence either way, and the absence of
|
||||
// violations in it says nothing about the app. It stays untyped to the CLI's
|
||||
// violation path on purpose, so it exits 1 as a broken run rather than 2.
|
||||
type VacuousRunError struct {
|
||||
Steps int
|
||||
}
|
||||
|
||||
func (e VacuousRunError) Error() string {
|
||||
return fmt.Sprintf(
|
||||
"%d step(s) ran and none of them reached the verifier: the screen was "+
|
||||
"still moving every time it was read, so no property judged this run",
|
||||
e.Steps)
|
||||
}
|
||||
|
||||
// bundleInputs holds the pre-driver assembly: alias map, seed, esbuild defines,
|
||||
// and the resolved spec-API/goja-runtime paths the bundler consumes.
|
||||
type bundleInputs struct {
|
||||
|
||||
@@ -266,6 +266,31 @@ func TestRunOutcome_ReportsViolationsOnlyUnderTheFlag(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A step the verifier skipped was judged by nothing, so a run whose every step
|
||||
// was skipped holds no verdict at all: "no violations" there is the absence of
|
||||
// an answer rather than a clean one. Reporting it as a successful run is the
|
||||
// green and vacuous outcome structuralShape's own design notes call worse than
|
||||
// the composition it catches, and the runner's hold is what makes a fully
|
||||
// skipped run reachable.
|
||||
func TestRunOutcome_ARunThatJudgedNothingIsNotASuccess(t *testing.T) {
|
||||
nothingJudged := runner.Summary{Steps: 6, SkippedVerification: 6}
|
||||
err := runOutcome(Options{}, nothingJudged)
|
||||
var vacuous VacuousRunError
|
||||
if !errors.As(err, &vacuous) {
|
||||
t.Fatalf("a run that judged none of its 6 steps came back %v, want a VacuousRunError", err)
|
||||
}
|
||||
if vacuous.Steps != 6 {
|
||||
t.Errorf("steps: got %d, want 6", vacuous.Steps)
|
||||
}
|
||||
|
||||
// A screen that composes now and then costs a run steps, not its verdict. A
|
||||
// check that fired here would turn every healthy android run red.
|
||||
mostlyJudged := runner.Summary{Steps: 6, SkippedVerification: 5}
|
||||
if err := runOutcome(Options{}, mostlyJudged); err != nil {
|
||||
t.Errorf("a run that judged one of its 6 steps must succeed, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// wedgedLaunchDriver never returns from Launch, standing in for a driver whose
|
||||
// device-side session is stuck.
|
||||
type wedgedLaunchDriver struct {
|
||||
|
||||
Reference in new issue
Block a user