feat(cli): --allow-no-properties opts a run out of the refusal

This commit is contained in:
pj committed 2026-08-17 23:27:25 +05:30
1 parent 7bafed8af4
commit 1e8db8524d
3 files changed
+28

No files matched your search

+2
View File
@@ -37,6 +37,7 @@ type testOptions struct {
generator string
labelSource string
exitOnViolation bool
allowNoProperties bool
}
const topUsage = `sanderling is a property-based UI fuzzer for mobile apps.
@@ -74,6 +75,7 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) {
flagSet.StringVar(&options.generator, "generator", "seeded", "action generator: seeded (weighted random) or llm (model picks from the same candidate set; requires generator = llm() in the spec)")
flagSet.StringVar(&options.labelSource, "label-source", "visible-text", "how candidates are named to the llm generator: visible-text (what a user reads) or resource-id (the identifier the app assigned). The seeded generator picks by index and ignores this")
flagSet.BoolVar(&options.exitOnViolation, "exit-on-violation", false, "stop the run at the first property violation and exit 2, so CI can tell a found bug (2) from a broken harness (1)")
flagSet.BoolVar(&options.allowNoProperties, "allow-no-properties", false, "run a spec that registers no properties. Such a run judges nothing and can only report no violations, so it is refused by default; pass this when the run measures what the spec extracts or where the generator reaches")
if err := flagSet.Parse(args); err != nil {
return testOptions{}, err
}
+25
View File
@@ -401,6 +401,30 @@ func TestParseTestArgs_ExitOnViolation(t *testing.T) {
}
}
// TestParseTestArgs_AllowNoPropertiesReachesThePipeline pins the opt-out the
// extraction and portability sweeps pass. Dropped here, the guard is either
// unreachable or permanent: a sweep that deliberately judges nothing cannot ask
// for it, and every other run keeps the false green the guard exists to stop.
func TestParseTestArgs_AllowNoPropertiesReachesThePipeline(t *testing.T) {
base := []string{"--spec", "s.ts", "--bundle-id", "com.example"}
options, err := parseTestArgs(base, io.Discard)
if err != nil {
t.Fatal(err)
}
if pipelineOptions(options).AllowNoProperties {
t.Error("allowNoProperties default: got true, want false")
}
options, err = parseTestArgs(append(base, "--allow-no-properties"), io.Discard)
if err != nil {
t.Fatal(err)
}
if !pipelineOptions(options).AllowNoProperties {
t.Error("--allow-no-properties never reached the pipeline options")
}
}
// TestExitCode_SeparatesFoundBugsFromBrokenHarnesses pins the three statuses CI
// reads: 0 clean, 2 the run found violations, 1 everything else. A workflow
// that asserts "the known bug is still found" is only meaningful while 2 and 1
@@ -416,6 +440,7 @@ func TestExitCode_SeparatesFoundBugsFromBrokenHarnesses(t *testing.T) {
{"help", flag.ErrHelp, 0, ""},
{"violations found", testrun.ViolationsError{Count: 2}, 2, "violations: 2"},
{"broken harness", errors.New("launch app: no device"), 1, "error: launch app"},
{"spec judges nothing", testrun.NoPropertiesError{Spec: "s.ts"}, 1, "registers no properties"},
} {
t.Run(testCase.name, func(t *testing.T) {
var stderr bytes.Buffer
+1
View File
@@ -17,6 +17,7 @@ func runTestPipeline(ctx context.Context, options testOptions, stdout io.Writer)
func pipelineOptions(options testOptions) testrun.Options {
return testrun.Options{
Spec: options.spec,
AllowNoProperties: options.allowNoProperties,
BundleID: options.bundleID,
Platform: options.platform,
AVD: options.avd,