feat(cli): add --exit-on-violation and exit 2 when it fires

cmd/sanderling/main.go

@@ -11,6 +11,8 @@ import (
 	"os/signal"
 	"syscall"
 	"time"
+
+	"github.com/priyanshujain/sanderling/internal/testrun"
 )
 
 // Version is stamped at build time via goreleaser ldflags.
@@ -18,21 +20,22 @@ import (
 var Version = "dev"
 
 type testOptions struct {
-	spec           string
-	bundleID       string
-	platform       string
-	avd            string
-	device         string
-	iosDevice      string
-	iosAppPath     string
-	androidAppPath string
-	duration       time.Duration
-	maxSteps       int
-	arm            string
-	seed           int64
-	output         string
-	clearData      bool
-	generator      string
+	spec            string
+	bundleID        string
+	platform        string
+	avd             string
+	device          string
+	iosDevice       string
+	iosAppPath      string
+	androidAppPath  string
+	duration        time.Duration
+	maxSteps        int
+	arm             string
+	seed            int64
+	output          string
+	clearData       bool
+	generator       string
+	exitOnViolation bool
 }
 
 const topUsage = `sanderling is a property-based UI fuzzer for mobile apps.
@@ -68,6 +71,7 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) {
 	flagSet.BoolVar(&options.clearData, "clear-data", true, "clear app data before launching so each run starts from a fresh install; pass --clear-data=false to resume prior state")
 	flagSet.StringVar(&options.arm, "arm", "", "experiment cell label, recorded in meta.json so a directory of runs can be attributed to a cell")
 	flagSet.StringVar(&options.generator, "generator", "seeded", "action generator: seeded (weighted random) or llm (model picks from the same candidate set; requires generator = llm() in the spec)")
+	flagSet.BoolVar(&options.exitOnViolation, "exit-on-violation", false, "stop the run at the first property violation and exit 2, so CI can tell a found bug (2) from a broken harness (1)")
 	if err := flagSet.Parse(args); err != nil {
 		return testOptions{}, err
 	}
@@ -141,13 +145,25 @@ func run(args []string, stdout, stderr io.Writer) error {
 }
 
 func main() {
-	if err := run(os.Args, os.Stdout, os.Stderr); err != nil {
-		// flag.ErrHelp means -h/--help was requested; flag already printed
-		// usage to stderr, so exit 0 rather than treating it as a failure.
-		if errors.Is(err, flag.ErrHelp) {
-			return
-		}
-		fmt.Fprintf(os.Stderr, "error: %v\n", err)
-		os.Exit(1)
+	if code := exitCode(run(os.Args, os.Stdout, os.Stderr), os.Stderr); code != 0 {
+		os.Exit(code)
 	}
 }
+
+// exitCode maps a command result to the process status and reports it on
+// stderr. 2 means the run did its job and found violations under
+// --exit-on-violation; 1 stays "something went wrong", so CI can tell a found
+// bug from a broken harness. flag.ErrHelp means -h/--help was requested and
+// flag already printed usage, so it exits 0 rather than reading as a failure.
+func exitCode(err error, stderr io.Writer) int {
+	if err == nil || errors.Is(err, flag.ErrHelp) {
+		return 0
+	}
+	var violations testrun.ViolationsError
+	if errors.As(err, &violations) {
+		fmt.Fprintf(stderr, "violations: %d\n", violations.Count)
+		return 2
+	}
+	fmt.Fprintf(stderr, "error: %v\n", err)
+	return 1
+}

cmd/sanderling/main_test.go

@@ -2,10 +2,14 @@ package main
 
 import (
 	"bytes"
+	"errors"
+	"flag"
 	"io"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/priyanshujain/sanderling/internal/testrun"
 )
 
 func TestParseTestArgs_Defaults(t *testing.T) {
@@ -311,3 +315,58 @@ func TestParseTestArgs_ArmLabel(t *testing.T) {
 		t.Errorf("arm: got %q, want seeded-identifier", options.arm)
 	}
 }
+
+func TestParseTestArgs_ExitOnViolationDefaultsOff(t *testing.T) {
+	options, err := parseTestArgs([]string{
+		"--spec", "s.ts",
+		"--bundle-id", "com.example",
+	}, io.Discard)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if options.exitOnViolation {
+		t.Error("exitOnViolation default: got true, want false")
+	}
+}
+
+func TestParseTestArgs_ExitOnViolation(t *testing.T) {
+	options, err := parseTestArgs([]string{
+		"--spec", "s.ts",
+		"--bundle-id", "com.example",
+		"--exit-on-violation",
+	}, io.Discard)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !options.exitOnViolation {
+		t.Error("expected exitOnViolation=true")
+	}
+}
+
+// TestExitCode_SeparatesFoundBugsFromBrokenHarnesses pins the three statuses CI
+// reads: 0 clean, 2 the run found violations, 1 everything else. A workflow
+// that asserts "the known bug is still found" is only meaningful while 2 and 1
+// stay distinct.
+func TestExitCode_SeparatesFoundBugsFromBrokenHarnesses(t *testing.T) {
+	for _, testCase := range []struct {
+		name string
+		err  error
+		want int
+		says string
+	}{
+		{"clean run", nil, 0, ""},
+		{"help", flag.ErrHelp, 0, ""},
+		{"violations found", testrun.ViolationsError{Count: 2}, 2, "violations: 2"},
+		{"broken harness", errors.New("launch app: no device"), 1, "error: launch app"},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			var stderr bytes.Buffer
+			if got := exitCode(testCase.err, &stderr); got != testCase.want {
+				t.Errorf("exit code: got %d, want %d", got, testCase.want)
+			}
+			if testCase.says != "" && !strings.Contains(stderr.String(), testCase.says) {
+				t.Errorf("stderr %q does not mention %q", stderr.String(), testCase.says)
+			}
+		})
+	}
+}

cmd/sanderling/test_run.go

@@ -9,20 +9,21 @@ import (
 
 func runTestPipeline(ctx context.Context, options testOptions, stdout io.Writer) error {
 	return testrun.Execute(ctx, testrun.Options{
-		Spec:           options.spec,
-		BundleID:       options.bundleID,
-		Platform:       options.platform,
-		AVD:            options.avd,
-		Device:         options.device,
-		IosDevice:      options.iosDevice,
-		IosAppPath:     options.iosAppPath,
-		AndroidAppPath: options.androidAppPath,
-		Duration:       options.duration,
-		MaxSteps:       options.maxSteps,
-		Seed:           options.seed,
-		Output:         options.output,
-		ClearData:      options.clearData,
-		Generator:      options.generator,
-		Arm:            options.arm,
+		Spec:            options.spec,
+		BundleID:        options.bundleID,
+		Platform:        options.platform,
+		AVD:             options.avd,
+		Device:          options.device,
+		IosDevice:       options.iosDevice,
+		IosAppPath:      options.iosAppPath,
+		AndroidAppPath:  options.androidAppPath,
+		Duration:        options.duration,
+		MaxSteps:        options.maxSteps,
+		Seed:            options.seed,
+		Output:          options.output,
+		ClearData:       options.clearData,
+		Generator:       options.generator,
+		Arm:             options.arm,
+		ExitOnViolation: options.exitOnViolation,
 	}, stdout)
 }