feat(cli): add --exit-on-violation and exit 2 when it fires

cmd/sanderling/main.go

@@ -11,6 +11,8 @@ import (
 	"os/signal"
 	"syscall"
 	"time"
+
+	"github.com/priyanshujain/sanderling/internal/testrun"
 )
 
 // Version is stamped at build time via goreleaser ldflags.
@@ -33,6 +35,7 @@ type testOptions struct {
 	output          string
 	clearData       bool
 	generator       string
+	exitOnViolation bool
 }
 
 const topUsage = `sanderling is a property-based UI fuzzer for mobile apps.
@@ -68,6 +71,7 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) {
 	flagSet.BoolVar(&options.clearData, "clear-data", true, "clear app data before launching so each run starts from a fresh install; pass --clear-data=false to resume prior state")
 	flagSet.StringVar(&options.arm, "arm", "", "experiment cell label, recorded in meta.json so a directory of runs can be attributed to a cell")
 	flagSet.StringVar(&options.generator, "generator", "seeded", "action generator: seeded (weighted random) or llm (model picks from the same candidate set; requires generator = llm() in the spec)")
+	flagSet.BoolVar(&options.exitOnViolation, "exit-on-violation", false, "stop the run at the first property violation and exit 2, so CI can tell a found bug (2) from a broken harness (1)")
 	if err := flagSet.Parse(args); err != nil {
 		return testOptions{}, err
 	}
@@ -141,13 +145,25 @@ func run(args []string, stdout, stderr io.Writer) error {
 }
 
 func main() {
-	if err := run(os.Args, os.Stdout, os.Stderr); err != nil {
-		// flag.ErrHelp means -h/--help was requested; flag already printed
-		// usage to stderr, so exit 0 rather than treating it as a failure.
-		if errors.Is(err, flag.ErrHelp) {
-			return
-		}
-		fmt.Fprintf(os.Stderr, "error: %v\n", err)
-		os.Exit(1)
+	if code := exitCode(run(os.Args, os.Stdout, os.Stderr), os.Stderr); code != 0 {
+		os.Exit(code)
 	}
 }
+
+// exitCode maps a command result to the process status and reports it on
+// stderr. 2 means the run did its job and found violations under
+// --exit-on-violation; 1 stays "something went wrong", so CI can tell a found
+// bug from a broken harness. flag.ErrHelp means -h/--help was requested and
+// flag already printed usage, so it exits 0 rather than reading as a failure.
+func exitCode(err error, stderr io.Writer) int {
+	if err == nil || errors.Is(err, flag.ErrHelp) {
+		return 0
+	}
+	var violations testrun.ViolationsError
+	if errors.As(err, &violations) {
+		fmt.Fprintf(stderr, "violations: %d\n", violations.Count)
+		return 2
+	}
+	fmt.Fprintf(stderr, "error: %v\n", err)
+	return 1
+}

cmd/sanderling/main_test.go

@@ -2,10 +2,14 @@ package main
 
 import (
 	"bytes"
+	"errors"
+	"flag"
 	"io"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/priyanshujain/sanderling/internal/testrun"
 )
 
 func TestParseTestArgs_Defaults(t *testing.T) {
@@ -311,3 +315,58 @@ func TestParseTestArgs_ArmLabel(t *testing.T) {
 		t.Errorf("arm: got %q, want seeded-identifier", options.arm)
 	}
 }
+
+func TestParseTestArgs_ExitOnViolationDefaultsOff(t *testing.T) {
+	options, err := parseTestArgs([]string{
+		"--spec", "s.ts",
+		"--bundle-id", "com.example",
+	}, io.Discard)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if options.exitOnViolation {
+		t.Error("exitOnViolation default: got true, want false")
+	}
+}
+
+func TestParseTestArgs_ExitOnViolation(t *testing.T) {
+	options, err := parseTestArgs([]string{
+		"--spec", "s.ts",
+		"--bundle-id", "com.example",
+		"--exit-on-violation",
+	}, io.Discard)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !options.exitOnViolation {
+		t.Error("expected exitOnViolation=true")
+	}
+}
+
+// TestExitCode_SeparatesFoundBugsFromBrokenHarnesses pins the three statuses CI
+// reads: 0 clean, 2 the run found violations, 1 everything else. A workflow
+// that asserts "the known bug is still found" is only meaningful while 2 and 1
+// stay distinct.
+func TestExitCode_SeparatesFoundBugsFromBrokenHarnesses(t *testing.T) {
+	for _, testCase := range []struct {
+		name string
+		err  error
+		want int
+		says string
+	}{
+		{"clean run", nil, 0, ""},
+		{"help", flag.ErrHelp, 0, ""},
+		{"violations found", testrun.ViolationsError{Count: 2}, 2, "violations: 2"},
+		{"broken harness", errors.New("launch app: no device"), 1, "error: launch app"},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			var stderr bytes.Buffer
+			if got := exitCode(testCase.err, &stderr); got != testCase.want {
+				t.Errorf("exit code: got %d, want %d", got, testCase.want)
+			}
+			if testCase.says != "" && !strings.Contains(stderr.String(), testCase.says) {
+				t.Errorf("stderr %q does not mention %q", stderr.String(), testCase.says)
+			}
+		})
+	}
+}

cmd/sanderling/test_run.go

@@ -24,5 +24,6 @@ func runTestPipeline(ctx context.Context, options testOptions, stdout io.Writer)
 		ClearData:       options.clearData,
 		Generator:       options.generator,
 		Arm:             options.arm,
+		ExitOnViolation: options.exitOnViolation,
 	}, stdout)
 }