diff --git a/cmd/sanderling/main.go b/cmd/sanderling/main.go index c3036cd..1298ecd 100644 --- a/cmd/sanderling/main.go +++ b/cmd/sanderling/main.go @@ -11,6 +11,8 @@ import ( "os/signal" "syscall" "time" + + "github.com/priyanshujain/sanderling/internal/testrun" ) // Version is stamped at build time via goreleaser ldflags. @@ -18,21 +20,22 @@ import ( var Version = "dev" type testOptions struct { - spec string - bundleID string - platform string - avd string - device string - iosDevice string - iosAppPath string - androidAppPath string - duration time.Duration - maxSteps int - arm string - seed int64 - output string - clearData bool - generator string + spec string + bundleID string + platform string + avd string + device string + iosDevice string + iosAppPath string + androidAppPath string + duration time.Duration + maxSteps int + arm string + seed int64 + output string + clearData bool + generator string + exitOnViolation bool } const topUsage = `sanderling is a property-based UI fuzzer for mobile apps. @@ -68,6 +71,7 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) { flagSet.BoolVar(&options.clearData, "clear-data", true, "clear app data before launching so each run starts from a fresh install; pass --clear-data=false to resume prior state") flagSet.StringVar(&options.arm, "arm", "", "experiment cell label, recorded in meta.json so a directory of runs can be attributed to a cell") flagSet.StringVar(&options.generator, "generator", "seeded", "action generator: seeded (weighted random) or llm (model picks from the same candidate set; requires generator = llm() in the spec)") + flagSet.BoolVar(&options.exitOnViolation, "exit-on-violation", false, "stop the run at the first property violation and exit 2, so CI can tell a found bug (2) from a broken harness (1)") if err := flagSet.Parse(args); err != nil { return testOptions{}, err } @@ -141,13 +145,25 @@ func run(args []string, stdout, stderr io.Writer) error { } func main() { - if err := run(os.Args, os.Stdout, os.Stderr); err != nil { - // flag.ErrHelp means -h/--help was requested; flag already printed - // usage to stderr, so exit 0 rather than treating it as a failure. - if errors.Is(err, flag.ErrHelp) { - return - } - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) + if code := exitCode(run(os.Args, os.Stdout, os.Stderr), os.Stderr); code != 0 { + os.Exit(code) } } + +// exitCode maps a command result to the process status and reports it on +// stderr. 2 means the run did its job and found violations under +// --exit-on-violation; 1 stays "something went wrong", so CI can tell a found +// bug from a broken harness. flag.ErrHelp means -h/--help was requested and +// flag already printed usage, so it exits 0 rather than reading as a failure. +func exitCode(err error, stderr io.Writer) int { + if err == nil || errors.Is(err, flag.ErrHelp) { + return 0 + } + var violations testrun.ViolationsError + if errors.As(err, &violations) { + fmt.Fprintf(stderr, "violations: %d\n", violations.Count) + return 2 + } + fmt.Fprintf(stderr, "error: %v\n", err) + return 1 +} diff --git a/cmd/sanderling/main_test.go b/cmd/sanderling/main_test.go index 8061f7e..0793581 100644 --- a/cmd/sanderling/main_test.go +++ b/cmd/sanderling/main_test.go @@ -2,10 +2,14 @@ package main import ( "bytes" + "errors" + "flag" "io" "strings" "testing" "time" + + "github.com/priyanshujain/sanderling/internal/testrun" ) func TestParseTestArgs_Defaults(t *testing.T) { @@ -311,3 +315,58 @@ func TestParseTestArgs_ArmLabel(t *testing.T) { t.Errorf("arm: got %q, want seeded-identifier", options.arm) } } + +func TestParseTestArgs_ExitOnViolationDefaultsOff(t *testing.T) { + options, err := parseTestArgs([]string{ + "--spec", "s.ts", + "--bundle-id", "com.example", + }, io.Discard) + if err != nil { + t.Fatal(err) + } + if options.exitOnViolation { + t.Error("exitOnViolation default: got true, want false") + } +} + +func TestParseTestArgs_ExitOnViolation(t *testing.T) { + options, err := parseTestArgs([]string{ + "--spec", "s.ts", + "--bundle-id", "com.example", + "--exit-on-violation", + }, io.Discard) + if err != nil { + t.Fatal(err) + } + if !options.exitOnViolation { + t.Error("expected exitOnViolation=true") + } +} + +// TestExitCode_SeparatesFoundBugsFromBrokenHarnesses pins the three statuses CI +// reads: 0 clean, 2 the run found violations, 1 everything else. A workflow +// that asserts "the known bug is still found" is only meaningful while 2 and 1 +// stay distinct. +func TestExitCode_SeparatesFoundBugsFromBrokenHarnesses(t *testing.T) { + for _, testCase := range []struct { + name string + err error + want int + says string + }{ + {"clean run", nil, 0, ""}, + {"help", flag.ErrHelp, 0, ""}, + {"violations found", testrun.ViolationsError{Count: 2}, 2, "violations: 2"}, + {"broken harness", errors.New("launch app: no device"), 1, "error: launch app"}, + } { + t.Run(testCase.name, func(t *testing.T) { + var stderr bytes.Buffer + if got := exitCode(testCase.err, &stderr); got != testCase.want { + t.Errorf("exit code: got %d, want %d", got, testCase.want) + } + if testCase.says != "" && !strings.Contains(stderr.String(), testCase.says) { + t.Errorf("stderr %q does not mention %q", stderr.String(), testCase.says) + } + }) + } +} diff --git a/cmd/sanderling/test_run.go b/cmd/sanderling/test_run.go index 06ff8f6..14ea54f 100644 --- a/cmd/sanderling/test_run.go +++ b/cmd/sanderling/test_run.go @@ -9,20 +9,21 @@ import ( func runTestPipeline(ctx context.Context, options testOptions, stdout io.Writer) error { return testrun.Execute(ctx, testrun.Options{ - Spec: options.spec, - BundleID: options.bundleID, - Platform: options.platform, - AVD: options.avd, - Device: options.device, - IosDevice: options.iosDevice, - IosAppPath: options.iosAppPath, - AndroidAppPath: options.androidAppPath, - Duration: options.duration, - MaxSteps: options.maxSteps, - Seed: options.seed, - Output: options.output, - ClearData: options.clearData, - Generator: options.generator, - Arm: options.arm, + Spec: options.spec, + BundleID: options.bundleID, + Platform: options.platform, + AVD: options.avd, + Device: options.device, + IosDevice: options.iosDevice, + IosAppPath: options.iosAppPath, + AndroidAppPath: options.androidAppPath, + Duration: options.duration, + MaxSteps: options.maxSteps, + Seed: options.seed, + Output: options.output, + ClearData: options.clearData, + Generator: options.generator, + Arm: options.arm, + ExitOnViolation: options.exitOnViolation, }, stdout) }