diff --git a/cmd/internal-tools/campaign/campaign_test.go b/cmd/internal-tools/campaign/campaign_test.go index 9d044f6..349b756 100644 --- a/cmd/internal-tools/campaign/campaign_test.go +++ b/cmd/internal-tools/campaign/campaign_test.go @@ -343,6 +343,28 @@ func TestRunCampaign_ContinuesAfterFailingRun(t *testing.T) { } } +func TestRunCampaign_GivesEveryRunTheCellsLabelSource(t *testing.T) { + directory := t.TempDir() + configuration := testConfiguration(t, directory, "--seeds", "1-3", "--label-source", "resource-id") + + var mutex sync.Mutex + var dispatched []string + executor := versionAnswering(func(_ context.Context, _ string, arguments []string, _ io.Writer) (int, error) { + mutex.Lock() + dispatched = append(dispatched, argumentValue(arguments, "--label-source")) + mutex.Unlock() + writeFakeRun(t, arguments, []trace.Step{observedStep(1)}) + return 0, nil + }) + if err := runCampaign(context.Background(), configuration, executor, io.Discard); err != nil { + t.Fatal(err) + } + + if !slices.Equal(dispatched, []string{"resource-id", "resource-id", "resource-id"}) { + t.Errorf("--label-source reaching sanderling: got %v, want resource-id on every run", dispatched) + } +} + func TestRunCampaign_RefusesToReuseACampaignDirectory(t *testing.T) { directory := t.TempDir() configuration := testConfiguration(t, directory) diff --git a/cmd/internal-tools/campaign/main.go b/cmd/internal-tools/campaign/main.go index c2a3f48..851dab6 100644 --- a/cmd/internal-tools/campaign/main.go +++ b/cmd/internal-tools/campaign/main.go @@ -23,6 +23,7 @@ type config struct { platform string arm string generator string + labelSource string maxSteps int duration time.Duration seeds []int64 @@ -58,6 +59,7 @@ func parseArguments(arguments []string, stderr io.Writer) (config, error) { flagSet.StringVar(&configuration.platform, "platform", "android", "target platform: android, ios, web") flagSet.StringVar(&configuration.arm, "arm", "", "experiment cell label recorded on every run (required)") flagSet.StringVar(&configuration.generator, "generator", "seeded", "action generator: seeded or llm") + flagSet.StringVar(&configuration.labelSource, "label-source", "visible-text", "how candidates are named to the llm generator: visible-text (what a user reads) or resource-id (the identifier the app assigned). The seeded generator picks by index and ignores this") flagSet.IntVar(&configuration.maxSteps, "max-steps", 0, "per-run step budget (required, must be positive)") flagSet.DurationVar(&configuration.duration, "duration", 5*time.Minute, "per-run wall-clock ceiling") flagSet.StringVar(&seedSpecification, "seeds", "", "seeds to run: ranges and lists, e.g. 1-10,20,30-32 (required)") @@ -91,6 +93,13 @@ func parseArguments(arguments []string, stderr io.Writer) (config, error) { default: return config{}, fmt.Errorf("unsupported generator: %q (seeded, llm)", configuration.generator) } + // Rejected before the first run, because a sweep that discovers the bad + // value on run 1 of 40 has already spent the device time of a whole cell. + switch configuration.labelSource { + case "visible-text", "resource-id": + default: + return config{}, fmt.Errorf("unsupported label source: %q (visible-text, resource-id)", configuration.labelSource) + } if configuration.maxSteps <= 0 { // Steps to first violation is right-censored at the budget, so a // campaign without one has nothing to censor its clean runs at. @@ -170,6 +179,7 @@ func runArguments(configuration config, seed, device string) []string { "--platform", configuration.platform, "--arm", configuration.arm, "--generator", configuration.generator, + "--label-source", configuration.labelSource, "--max-steps", strconv.Itoa(configuration.maxSteps), "--duration", configuration.duration.String(), "--seed", seed, diff --git a/cmd/internal-tools/campaign/main_test.go b/cmd/internal-tools/campaign/main_test.go index a205d56..7ba8798 100644 --- a/cmd/internal-tools/campaign/main_test.go +++ b/cmd/internal-tools/campaign/main_test.go @@ -54,6 +54,7 @@ func TestParseArguments_Rejections(t *testing.T) { {"missing output", []string{"--spec", "s", "--bundle-id", "a", "--arm", "b", "--seeds", "1", "--max-steps", "10"}, "--output is required"}, {"bad platform", append(baseArguments(), "--platform", "windows"), "unsupported platform"}, {"bad generator", append(baseArguments(), "--generator", "vibes"), "unsupported generator"}, + {"bad label source", append(baseArguments(), "--label-source", "resource_id"), `unsupported label source: "resource_id"`}, {"zero max steps", append(baseArguments(), "--max-steps", "0"), "--max-steps must be positive"}, {"seed zero", append(baseArguments(), "--seeds", "0-2"), "not reproducible"}, {"duplicate device", append(baseArguments(), "--devices", "a,a"), "duplicate device"}, @@ -113,6 +114,16 @@ func TestRunArguments_PlatformDeviceFlagAndPassthrough(t *testing.T) { } } +func TestRunArguments_LabelSourceDefaultsToVisibleText(t *testing.T) { + configuration, err := parseArguments(baseArguments(), io.Discard) + if err != nil { + t.Fatal(err) + } + if got := argumentValue(runArguments(configuration, "7", ""), "--label-source"); got != "visible-text" { + t.Errorf("--label-source = %q, want visible-text", got) + } +} + func argumentValue(arguments []string, name string) string { index := slices.Index(arguments, name) if index < 0 || index+1 >= len(arguments) {