From 1e8db8524d6b918534d79e66b31a01da89f9e823 Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 17 Aug 2026 23:27:25 +0530 Subject: [PATCH] feat(cli): --allow-no-properties opts a run out of the refusal --- cmd/sanderling/main.go | 36 +++++++++++++++++++----------------- cmd/sanderling/main_test.go | 25 +++++++++++++++++++++++++ cmd/sanderling/test_run.go | 35 ++++++++++++++++++----------------- 3 files changed, 62 insertions(+), 34 deletions(-) diff --git a/cmd/sanderling/main.go b/cmd/sanderling/main.go index 75548a2..140ba9e 100644 --- a/cmd/sanderling/main.go +++ b/cmd/sanderling/main.go @@ -20,23 +20,24 @@ import ( var Version = "dev" type testOptions struct { - spec string - bundleID string - platform string - avd string - device string - iosDevice string - iosAppPath string - androidAppPath string - duration time.Duration - maxSteps int - arm string - seed int64 - output string - clearData bool - generator string - labelSource string - exitOnViolation bool + spec string + bundleID string + platform string + avd string + device string + iosDevice string + iosAppPath string + androidAppPath string + duration time.Duration + maxSteps int + arm string + seed int64 + output string + clearData bool + generator string + labelSource string + exitOnViolation bool + allowNoProperties bool } const topUsage = `sanderling is a property-based UI fuzzer for mobile apps. @@ -74,6 +75,7 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) { flagSet.StringVar(&options.generator, "generator", "seeded", "action generator: seeded (weighted random) or llm (model picks from the same candidate set; requires generator = llm() in the spec)") flagSet.StringVar(&options.labelSource, "label-source", "visible-text", "how candidates are named to the llm generator: visible-text (what a user reads) or resource-id (the identifier the app assigned). The seeded generator picks by index and ignores this") flagSet.BoolVar(&options.exitOnViolation, "exit-on-violation", false, "stop the run at the first property violation and exit 2, so CI can tell a found bug (2) from a broken harness (1)") + flagSet.BoolVar(&options.allowNoProperties, "allow-no-properties", false, "run a spec that registers no properties. Such a run judges nothing and can only report no violations, so it is refused by default; pass this when the run measures what the spec extracts or where the generator reaches") if err := flagSet.Parse(args); err != nil { return testOptions{}, err } diff --git a/cmd/sanderling/main_test.go b/cmd/sanderling/main_test.go index 31a81a8..50d6354 100644 --- a/cmd/sanderling/main_test.go +++ b/cmd/sanderling/main_test.go @@ -401,6 +401,30 @@ func TestParseTestArgs_ExitOnViolation(t *testing.T) { } } +// TestParseTestArgs_AllowNoPropertiesReachesThePipeline pins the opt-out the +// extraction and portability sweeps pass. Dropped here, the guard is either +// unreachable or permanent: a sweep that deliberately judges nothing cannot ask +// for it, and every other run keeps the false green the guard exists to stop. +func TestParseTestArgs_AllowNoPropertiesReachesThePipeline(t *testing.T) { + base := []string{"--spec", "s.ts", "--bundle-id", "com.example"} + + options, err := parseTestArgs(base, io.Discard) + if err != nil { + t.Fatal(err) + } + if pipelineOptions(options).AllowNoProperties { + t.Error("allowNoProperties default: got true, want false") + } + + options, err = parseTestArgs(append(base, "--allow-no-properties"), io.Discard) + if err != nil { + t.Fatal(err) + } + if !pipelineOptions(options).AllowNoProperties { + t.Error("--allow-no-properties never reached the pipeline options") + } +} + // TestExitCode_SeparatesFoundBugsFromBrokenHarnesses pins the three statuses CI // reads: 0 clean, 2 the run found violations, 1 everything else. A workflow // that asserts "the known bug is still found" is only meaningful while 2 and 1 @@ -416,6 +440,7 @@ func TestExitCode_SeparatesFoundBugsFromBrokenHarnesses(t *testing.T) { {"help", flag.ErrHelp, 0, ""}, {"violations found", testrun.ViolationsError{Count: 2}, 2, "violations: 2"}, {"broken harness", errors.New("launch app: no device"), 1, "error: launch app"}, + {"spec judges nothing", testrun.NoPropertiesError{Spec: "s.ts"}, 1, "registers no properties"}, } { t.Run(testCase.name, func(t *testing.T) { var stderr bytes.Buffer diff --git a/cmd/sanderling/test_run.go b/cmd/sanderling/test_run.go index 499aa04..45597b9 100644 --- a/cmd/sanderling/test_run.go +++ b/cmd/sanderling/test_run.go @@ -16,22 +16,23 @@ func runTestPipeline(ctx context.Context, options testOptions, stdout io.Writer) // and records another, which is worth being able to test on its own. func pipelineOptions(options testOptions) testrun.Options { return testrun.Options{ - Spec: options.spec, - BundleID: options.bundleID, - Platform: options.platform, - AVD: options.avd, - Device: options.device, - IosDevice: options.iosDevice, - IosAppPath: options.iosAppPath, - AndroidAppPath: options.androidAppPath, - Duration: options.duration, - MaxSteps: options.maxSteps, - Seed: options.seed, - Output: options.output, - ClearData: options.clearData, - Generator: options.generator, - LabelSource: options.labelSource, - Arm: options.arm, - ExitOnViolation: options.exitOnViolation, + Spec: options.spec, + AllowNoProperties: options.allowNoProperties, + BundleID: options.bundleID, + Platform: options.platform, + AVD: options.avd, + Device: options.device, + IosDevice: options.iosDevice, + IosAppPath: options.iosAppPath, + AndroidAppPath: options.androidAppPath, + Duration: options.duration, + MaxSteps: options.maxSteps, + Seed: options.seed, + Output: options.output, + ClearData: options.clearData, + Generator: options.generator, + LabelSource: options.labelSource, + Arm: options.arm, + ExitOnViolation: options.exitOnViolation, } }