mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
feat(cli): --allow-no-properties opts a run out of the refusal
This commit is contained in:
1 parent
7bafed8af4
commit
1e8db8524d
3 files changed
+28
No files matched your search
@@ -37,6 +37,7 @@ type testOptions struct {
|
|||||||
generator string
|
generator string
|
||||||
labelSource string
|
labelSource string
|
||||||
exitOnViolation bool
|
exitOnViolation bool
|
||||||
|
allowNoProperties bool
|
||||||
}
|
}
|
||||||
|
|
||||||
const topUsage = `sanderling is a property-based UI fuzzer for mobile apps.
|
const topUsage = `sanderling is a property-based UI fuzzer for mobile apps.
|
||||||
@@ -74,6 +75,7 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) {
|
|||||||
flagSet.StringVar(&options.generator, "generator", "seeded", "action generator: seeded (weighted random) or llm (model picks from the same candidate set; requires generator = llm() in the spec)")
|
flagSet.StringVar(&options.generator, "generator", "seeded", "action generator: seeded (weighted random) or llm (model picks from the same candidate set; requires generator = llm() in the spec)")
|
||||||
flagSet.StringVar(&options.labelSource, "label-source", "visible-text", "how candidates are named to the llm generator: visible-text (what a user reads) or resource-id (the identifier the app assigned). The seeded generator picks by index and ignores this")
|
flagSet.StringVar(&options.labelSource, "label-source", "visible-text", "how candidates are named to the llm generator: visible-text (what a user reads) or resource-id (the identifier the app assigned). The seeded generator picks by index and ignores this")
|
||||||
flagSet.BoolVar(&options.exitOnViolation, "exit-on-violation", false, "stop the run at the first property violation and exit 2, so CI can tell a found bug (2) from a broken harness (1)")
|
flagSet.BoolVar(&options.exitOnViolation, "exit-on-violation", false, "stop the run at the first property violation and exit 2, so CI can tell a found bug (2) from a broken harness (1)")
|
||||||
|
flagSet.BoolVar(&options.allowNoProperties, "allow-no-properties", false, "run a spec that registers no properties. Such a run judges nothing and can only report no violations, so it is refused by default; pass this when the run measures what the spec extracts or where the generator reaches")
|
||||||
if err := flagSet.Parse(args); err != nil {
|
if err := flagSet.Parse(args); err != nil {
|
||||||
return testOptions{}, err
|
return testOptions{}, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -401,6 +401,30 @@ func TestParseTestArgs_ExitOnViolation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestParseTestArgs_AllowNoPropertiesReachesThePipeline pins the opt-out the
|
||||||
|
// extraction and portability sweeps pass. Dropped here, the guard is either
|
||||||
|
// unreachable or permanent: a sweep that deliberately judges nothing cannot ask
|
||||||
|
// for it, and every other run keeps the false green the guard exists to stop.
|
||||||
|
func TestParseTestArgs_AllowNoPropertiesReachesThePipeline(t *testing.T) {
|
||||||
|
base := []string{"--spec", "s.ts", "--bundle-id", "com.example"}
|
||||||
|
|
||||||
|
options, err := parseTestArgs(base, io.Discard)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if pipelineOptions(options).AllowNoProperties {
|
||||||
|
t.Error("allowNoProperties default: got true, want false")
|
||||||
|
}
|
||||||
|
|
||||||
|
options, err = parseTestArgs(append(base, "--allow-no-properties"), io.Discard)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !pipelineOptions(options).AllowNoProperties {
|
||||||
|
t.Error("--allow-no-properties never reached the pipeline options")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestExitCode_SeparatesFoundBugsFromBrokenHarnesses pins the three statuses CI
|
// TestExitCode_SeparatesFoundBugsFromBrokenHarnesses pins the three statuses CI
|
||||||
// reads: 0 clean, 2 the run found violations, 1 everything else. A workflow
|
// reads: 0 clean, 2 the run found violations, 1 everything else. A workflow
|
||||||
// that asserts "the known bug is still found" is only meaningful while 2 and 1
|
// that asserts "the known bug is still found" is only meaningful while 2 and 1
|
||||||
@@ -416,6 +440,7 @@ func TestExitCode_SeparatesFoundBugsFromBrokenHarnesses(t *testing.T) {
|
|||||||
{"help", flag.ErrHelp, 0, ""},
|
{"help", flag.ErrHelp, 0, ""},
|
||||||
{"violations found", testrun.ViolationsError{Count: 2}, 2, "violations: 2"},
|
{"violations found", testrun.ViolationsError{Count: 2}, 2, "violations: 2"},
|
||||||
{"broken harness", errors.New("launch app: no device"), 1, "error: launch app"},
|
{"broken harness", errors.New("launch app: no device"), 1, "error: launch app"},
|
||||||
|
{"spec judges nothing", testrun.NoPropertiesError{Spec: "s.ts"}, 1, "registers no properties"},
|
||||||
} {
|
} {
|
||||||
t.Run(testCase.name, func(t *testing.T) {
|
t.Run(testCase.name, func(t *testing.T) {
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ func runTestPipeline(ctx context.Context, options testOptions, stdout io.Writer)
|
|||||||
func pipelineOptions(options testOptions) testrun.Options {
|
func pipelineOptions(options testOptions) testrun.Options {
|
||||||
return testrun.Options{
|
return testrun.Options{
|
||||||
Spec: options.spec,
|
Spec: options.spec,
|
||||||
|
AllowNoProperties: options.allowNoProperties,
|
||||||
BundleID: options.bundleID,
|
BundleID: options.bundleID,
|
||||||
Platform: options.platform,
|
Platform: options.platform,
|
||||||
AVD: options.avd,
|
AVD: options.avd,
|
||||||
|
|||||||
Reference in new issue
Block a user