From caa4b16866203abba5c604b5826011cc2a14cc83 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 12 Aug 2026 23:19:24 +0530 Subject: [PATCH] feat(cli): add --label-source Unknown values are rejected at parse time rather than falling back to the default, matching the generator check: a campaign that completes with the wrong arm and a correct-looking output directory is worse than one that fails. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX --- cmd/sanderling/main.go | 10 +++++++ cmd/sanderling/main_test.go | 59 +++++++++++++++++++++++++++++++++++++ cmd/sanderling/test_run.go | 12 ++++++-- 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/cmd/sanderling/main.go b/cmd/sanderling/main.go index c3036cd..a38f577 100644 --- a/cmd/sanderling/main.go +++ b/cmd/sanderling/main.go @@ -33,6 +33,7 @@ type testOptions struct { output string clearData bool generator string + labelSource string } const topUsage = `sanderling is a property-based UI fuzzer for mobile apps. @@ -68,6 +69,7 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) { flagSet.BoolVar(&options.clearData, "clear-data", true, "clear app data before launching so each run starts from a fresh install; pass --clear-data=false to resume prior state") flagSet.StringVar(&options.arm, "arm", "", "experiment cell label, recorded in meta.json so a directory of runs can be attributed to a cell") flagSet.StringVar(&options.generator, "generator", "seeded", "action generator: seeded (weighted random) or llm (model picks from the same candidate set; requires generator = llm() in the spec)") + flagSet.StringVar(&options.labelSource, "label-source", "visible-text", "how candidates are named to the llm generator: visible-text (what a user reads) or resource-id (the identifier the app assigned). The seeded generator picks by index and ignores this") if err := flagSet.Parse(args); err != nil { return testOptions{}, err } @@ -90,6 +92,14 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) { default: return testOptions{}, fmt.Errorf("unsupported generator: %q (seeded, llm)", options.generator) } + // Rejected here rather than defaulted, because a campaign that finishes with + // the wrong labelling and a plausible output directory is worse than one + // that never starts. + switch options.labelSource { + case "visible-text", "resource-id": + default: + return testOptions{}, fmt.Errorf("unsupported label source: %q (visible-text, resource-id)", options.labelSource) + } return options, nil } diff --git a/cmd/sanderling/main_test.go b/cmd/sanderling/main_test.go index 8061f7e..cc12aff 100644 --- a/cmd/sanderling/main_test.go +++ b/cmd/sanderling/main_test.go @@ -6,6 +6,8 @@ import ( "strings" "testing" "time" + + "github.com/priyanshujain/sanderling/internal/verifier" ) func TestParseTestArgs_Defaults(t *testing.T) { @@ -128,6 +130,63 @@ func TestParseTestArgs_RejectsUnknownGenerator(t *testing.T) { } } +// The label-source cases compare against the verifier's own constants, because +// the flag and the code that reads it are the two halves of one contract: a +// rename on either side would otherwise leave every run silently labelled by +// the default channel while meta.json claimed the other one. +func TestParseTestArgs_LabelSourceDefaultsToVisibleText(t *testing.T) { + options, err := parseTestArgs([]string{"--spec", "s.ts", "--bundle-id", "com.example"}, io.Discard) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if options.labelSource != verifier.LabelSourceVisibleText { + t.Fatalf("label source default: got %q, want %q", options.labelSource, verifier.LabelSourceVisibleText) + } +} + +func TestParseTestArgs_AcceptsResourceIDLabelSource(t *testing.T) { + options, err := parseTestArgs([]string{ + "--spec", "s.ts", + "--bundle-id", "com.example", + "--label-source", verifier.LabelSourceResourceID, + }, io.Discard) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if options.labelSource != verifier.LabelSourceResourceID { + t.Fatalf("label source: got %q, want %q", options.labelSource, verifier.LabelSourceResourceID) + } +} + +func TestParseTestArgs_RejectsUnknownLabelSource(t *testing.T) { + _, err := parseTestArgs([]string{ + "--spec", "s.ts", + "--bundle-id", "com.example", + "--label-source", "resource_id", + }, io.Discard) + if err == nil || !strings.Contains(err.Error(), "unsupported label source") { + t.Fatalf("expected unsupported-label-source error, got %v", err) + } +} + +func TestPipelineOptionsCarriesTheExperimentCell(t *testing.T) { + options, err := parseTestArgs([]string{ + "--spec", "s.ts", + "--bundle-id", "com.example", + "--generator", "llm", + "--label-source", verifier.LabelSourceResourceID, + "--arm", "llm-resource-id", + }, io.Discard) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + pipeline := pipelineOptions(options) + if pipeline.Generator != "llm" || pipeline.LabelSource != verifier.LabelSourceResourceID || pipeline.Arm != "llm-resource-id" { + t.Errorf("cell lost between the flags and the pipeline: generator=%q labelSource=%q arm=%q", + pipeline.Generator, pipeline.LabelSource, pipeline.Arm) + } +} + func TestParseTestArgs_RejectsUnknownPlatform(t *testing.T) { _, err := parseTestArgs([]string{ "--spec", "s.ts", diff --git a/cmd/sanderling/test_run.go b/cmd/sanderling/test_run.go index 06ff8f6..00c5f21 100644 --- a/cmd/sanderling/test_run.go +++ b/cmd/sanderling/test_run.go @@ -8,7 +8,14 @@ import ( ) func runTestPipeline(ctx context.Context, options testOptions, stdout io.Writer) error { - return testrun.Execute(ctx, testrun.Options{ + return testrun.Execute(ctx, pipelineOptions(options), stdout) +} + +// pipelineOptions maps the parsed flags onto the pipeline's options. A field +// dropped on the way through here is a run that executes one experiment cell +// and records another, which is worth being able to test on its own. +func pipelineOptions(options testOptions) testrun.Options { + return testrun.Options{ Spec: options.spec, BundleID: options.bundleID, Platform: options.platform, @@ -23,6 +30,7 @@ func runTestPipeline(ctx context.Context, options testOptions, stdout io.Writer) Output: options.output, ClearData: options.clearData, Generator: options.generator, + LabelSource: options.labelSource, Arm: options.arm, - }, stdout) + } }