mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 20:17:09 +00:00
feat(cli): --allow-no-generator-actions
The flag the dead-run refusal names, wired through to the pipeline. The property-free flag goes back to meaning what it says.
This commit is contained in:
1 parent
93614a48b9
commit
74cae01a5c
3 files changed
+36
-1
No files matched your search
@@ -38,6 +38,7 @@ type testOptions struct {
|
|||||||
labelSource string
|
labelSource string
|
||||||
exitOnViolation bool
|
exitOnViolation bool
|
||||||
allowNoProperties bool
|
allowNoProperties bool
|
||||||
|
allowNoGeneratorActions bool
|
||||||
}
|
}
|
||||||
|
|
||||||
const topUsage = `sanderling is a property-based UI fuzzer for mobile apps.
|
const topUsage = `sanderling is a property-based UI fuzzer for mobile apps.
|
||||||
@@ -75,7 +76,8 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) {
|
|||||||
flagSet.StringVar(&options.generator, "generator", "seeded", "action generator: seeded (weighted random) or llm (model picks from the same candidate set; requires generator = llm() in the spec)")
|
flagSet.StringVar(&options.generator, "generator", "seeded", "action generator: seeded (weighted random) or llm (model picks from the same candidate set; requires generator = llm() in the spec)")
|
||||||
flagSet.StringVar(&options.labelSource, "label-source", "visible-text", "how candidates are named to the llm generator: visible-text (what a user reads) or resource-id (the identifier the app assigned). The seeded generator picks by index and ignores this")
|
flagSet.StringVar(&options.labelSource, "label-source", "visible-text", "how candidates are named to the llm generator: visible-text (what a user reads) or resource-id (the identifier the app assigned). The seeded generator picks by index and ignores this")
|
||||||
flagSet.BoolVar(&options.exitOnViolation, "exit-on-violation", false, "stop the run at the first property violation and exit 2, so CI can tell a found bug (2) from a broken harness (1)")
|
flagSet.BoolVar(&options.exitOnViolation, "exit-on-violation", false, "stop the run at the first property violation and exit 2, so CI can tell a found bug (2) from a broken harness (1)")
|
||||||
flagSet.BoolVar(&options.allowNoProperties, "allow-no-properties", false, "run a spec that registers no properties. Such a run judges nothing and can only report no violations, so it is refused by default; pass this when the run measures what the spec extracts or where the generator reaches")
|
flagSet.BoolVar(&options.allowNoProperties, "allow-no-properties", false, "run a spec that registers no properties. Such a run judges nothing and can only report no violations, so it is refused by default; pass this when the run measures what the spec extracts")
|
||||||
|
flagSet.BoolVar(&options.allowNoGeneratorActions, "allow-no-generator-actions", false, "finish a run the action generator never drove. Such a run judged whatever screen the spec's setup left it on and explored nothing, so it is refused by default; pass this when the run measures where the generator reaches and reaching nothing is the measurement")
|
||||||
if err := flagSet.Parse(args); err != nil {
|
if err := flagSet.Parse(args); err != nil {
|
||||||
return testOptions{}, err
|
return testOptions{}, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -425,6 +425,38 @@ func TestParseTestArgs_AllowNoPropertiesReachesThePipeline(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The exploration sweeps ask for a run the generator never drove, and they ask
|
||||||
|
// for that alone. Riding on --allow-no-properties made one flag name two
|
||||||
|
// unrelated waivers, so a sweep that wanted the property-free one silently lost
|
||||||
|
// the dead-run detector as well.
|
||||||
|
func TestParseTestArgs_AllowNoGeneratorActionsIsItsOwnFlag(t *testing.T) {
|
||||||
|
base := []string{"--spec", "s.ts", "--bundle-id", "com.example"}
|
||||||
|
|
||||||
|
options, err := parseTestArgs(base, io.Discard)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if pipelineOptions(options).AllowNoGeneratorActions {
|
||||||
|
t.Error("allowNoGeneratorActions default: got true, want false")
|
||||||
|
}
|
||||||
|
|
||||||
|
options, err = parseTestArgs(append(base, "--allow-no-generator-actions"), io.Discard)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !pipelineOptions(options).AllowNoGeneratorActions {
|
||||||
|
t.Error("--allow-no-generator-actions never reached the pipeline options")
|
||||||
|
}
|
||||||
|
|
||||||
|
options, err = parseTestArgs(append(base, "--allow-no-properties"), io.Discard)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if pipelineOptions(options).AllowNoGeneratorActions {
|
||||||
|
t.Error("--allow-no-properties still waives the dead-run refusal it does not name")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestExitCode_SeparatesFoundBugsFromBrokenHarnesses pins the three statuses CI
|
// TestExitCode_SeparatesFoundBugsFromBrokenHarnesses pins the three statuses CI
|
||||||
// reads: 0 clean, 2 the run found violations, 1 everything else. A workflow
|
// reads: 0 clean, 2 the run found violations, 1 everything else. A workflow
|
||||||
// that asserts "the known bug is still found" is only meaningful while 2 and 1
|
// that asserts "the known bug is still found" is only meaningful while 2 and 1
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ func pipelineOptions(options testOptions) testrun.Options {
|
|||||||
return testrun.Options{
|
return testrun.Options{
|
||||||
Spec: options.spec,
|
Spec: options.spec,
|
||||||
AllowNoProperties: options.allowNoProperties,
|
AllowNoProperties: options.allowNoProperties,
|
||||||
|
AllowNoGeneratorActions: options.allowNoGeneratorActions,
|
||||||
BundleID: options.bundleID,
|
BundleID: options.bundleID,
|
||||||
Platform: options.platform,
|
Platform: options.platform,
|
||||||
AVD: options.avd,
|
AVD: options.avd,
|
||||||
|
|||||||
Reference in new issue
Block a user