diff --git a/cmd/sanderling/main.go b/cmd/sanderling/main.go index 140ba9e..6032b66 100644 --- a/cmd/sanderling/main.go +++ b/cmd/sanderling/main.go @@ -20,24 +20,25 @@ import ( var Version = "dev" type testOptions struct { - spec string - bundleID string - platform string - avd string - device string - iosDevice string - iosAppPath string - androidAppPath string - duration time.Duration - maxSteps int - arm string - seed int64 - output string - clearData bool - generator string - labelSource string - exitOnViolation bool - allowNoProperties bool + spec string + bundleID string + platform string + avd string + device string + iosDevice string + iosAppPath string + androidAppPath string + duration time.Duration + maxSteps int + arm string + seed int64 + output string + clearData bool + generator string + labelSource string + exitOnViolation bool + allowNoProperties bool + allowNoGeneratorActions bool } const topUsage = `sanderling is a property-based UI fuzzer for mobile apps. @@ -75,7 +76,8 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) { flagSet.StringVar(&options.generator, "generator", "seeded", "action generator: seeded (weighted random) or llm (model picks from the same candidate set; requires generator = llm() in the spec)") flagSet.StringVar(&options.labelSource, "label-source", "visible-text", "how candidates are named to the llm generator: visible-text (what a user reads) or resource-id (the identifier the app assigned). The seeded generator picks by index and ignores this") flagSet.BoolVar(&options.exitOnViolation, "exit-on-violation", false, "stop the run at the first property violation and exit 2, so CI can tell a found bug (2) from a broken harness (1)") - flagSet.BoolVar(&options.allowNoProperties, "allow-no-properties", false, "run a spec that registers no properties. Such a run judges nothing and can only report no violations, so it is refused by default; pass this when the run measures what the spec extracts or where the generator reaches") + flagSet.BoolVar(&options.allowNoProperties, "allow-no-properties", false, "run a spec that registers no properties. Such a run judges nothing and can only report no violations, so it is refused by default; pass this when the run measures what the spec extracts") + flagSet.BoolVar(&options.allowNoGeneratorActions, "allow-no-generator-actions", false, "finish a run the action generator never drove. Such a run judged whatever screen the spec's setup left it on and explored nothing, so it is refused by default; pass this when the run measures where the generator reaches and reaching nothing is the measurement") if err := flagSet.Parse(args); err != nil { return testOptions{}, err } diff --git a/cmd/sanderling/main_test.go b/cmd/sanderling/main_test.go index 50d6354..25e70b3 100644 --- a/cmd/sanderling/main_test.go +++ b/cmd/sanderling/main_test.go @@ -425,6 +425,38 @@ func TestParseTestArgs_AllowNoPropertiesReachesThePipeline(t *testing.T) { } } +// The exploration sweeps ask for a run the generator never drove, and they ask +// for that alone. Riding on --allow-no-properties made one flag name two +// unrelated waivers, so a sweep that wanted the property-free one silently lost +// the dead-run detector as well. +func TestParseTestArgs_AllowNoGeneratorActionsIsItsOwnFlag(t *testing.T) { + base := []string{"--spec", "s.ts", "--bundle-id", "com.example"} + + options, err := parseTestArgs(base, io.Discard) + if err != nil { + t.Fatal(err) + } + if pipelineOptions(options).AllowNoGeneratorActions { + t.Error("allowNoGeneratorActions default: got true, want false") + } + + options, err = parseTestArgs(append(base, "--allow-no-generator-actions"), io.Discard) + if err != nil { + t.Fatal(err) + } + if !pipelineOptions(options).AllowNoGeneratorActions { + t.Error("--allow-no-generator-actions never reached the pipeline options") + } + + options, err = parseTestArgs(append(base, "--allow-no-properties"), io.Discard) + if err != nil { + t.Fatal(err) + } + if pipelineOptions(options).AllowNoGeneratorActions { + t.Error("--allow-no-properties still waives the dead-run refusal it does not name") + } +} + // TestExitCode_SeparatesFoundBugsFromBrokenHarnesses pins the three statuses CI // reads: 0 clean, 2 the run found violations, 1 everything else. A workflow // that asserts "the known bug is still found" is only meaningful while 2 and 1 diff --git a/cmd/sanderling/test_run.go b/cmd/sanderling/test_run.go index 45597b9..47778a7 100644 --- a/cmd/sanderling/test_run.go +++ b/cmd/sanderling/test_run.go @@ -16,23 +16,24 @@ func runTestPipeline(ctx context.Context, options testOptions, stdout io.Writer) // and records another, which is worth being able to test on its own. func pipelineOptions(options testOptions) testrun.Options { return testrun.Options{ - Spec: options.spec, - AllowNoProperties: options.allowNoProperties, - BundleID: options.bundleID, - Platform: options.platform, - AVD: options.avd, - Device: options.device, - IosDevice: options.iosDevice, - IosAppPath: options.iosAppPath, - AndroidAppPath: options.androidAppPath, - Duration: options.duration, - MaxSteps: options.maxSteps, - Seed: options.seed, - Output: options.output, - ClearData: options.clearData, - Generator: options.generator, - LabelSource: options.labelSource, - Arm: options.arm, - ExitOnViolation: options.exitOnViolation, + Spec: options.spec, + AllowNoProperties: options.allowNoProperties, + AllowNoGeneratorActions: options.allowNoGeneratorActions, + BundleID: options.bundleID, + Platform: options.platform, + AVD: options.avd, + Device: options.device, + IosDevice: options.iosDevice, + IosAppPath: options.iosAppPath, + AndroidAppPath: options.androidAppPath, + Duration: options.duration, + MaxSteps: options.maxSteps, + Seed: options.seed, + Output: options.output, + ClearData: options.clearData, + Generator: options.generator, + LabelSource: options.labelSource, + Arm: options.arm, + ExitOnViolation: options.exitOnViolation, } }