feat(campaign): make the label source a cell dimension

A 2x2 of policy against labelling needs the runner to express both factors.
It could only express the policy, so half the factorial had to go through
--extra, where the manifest would not record what was actually run.

Rejected at parse rather than on dispatch: a sweep that finds the bad value
on run 1 of 40 has already spent a cell's worth of device time.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX

cmd/internal-tools/campaign/campaign_test.go

@@ -343,6 +343,28 @@ func TestRunCampaign_ContinuesAfterFailingRun(t *testing.T) {
 	}
 }
 
+func TestRunCampaign_GivesEveryRunTheCellsLabelSource(t *testing.T) {
+	directory := t.TempDir()
+	configuration := testConfiguration(t, directory, "--seeds", "1-3", "--label-source", "resource-id")
+
+	var mutex sync.Mutex
+	var dispatched []string
+	executor := versionAnswering(func(_ context.Context, _ string, arguments []string, _ io.Writer) (int, error) {
+		mutex.Lock()
+		dispatched = append(dispatched, argumentValue(arguments, "--label-source"))
+		mutex.Unlock()
+		writeFakeRun(t, arguments, []trace.Step{observedStep(1)})
+		return 0, nil
+	})
+	if err := runCampaign(context.Background(), configuration, executor, io.Discard); err != nil {
+		t.Fatal(err)
+	}
+
+	if !slices.Equal(dispatched, []string{"resource-id", "resource-id", "resource-id"}) {
+		t.Errorf("--label-source reaching sanderling: got %v, want resource-id on every run", dispatched)
+	}
+}
+
 func TestRunCampaign_RefusesToReuseACampaignDirectory(t *testing.T) {
 	directory := t.TempDir()
 	configuration := testConfiguration(t, directory)

cmd/internal-tools/campaign/main.go

@@ -23,6 +23,7 @@ type config struct {
 	platform        string
 	arm             string
 	generator       string
+	labelSource     string
 	maxSteps        int
 	duration        time.Duration
 	seeds           []int64
@@ -58,6 +59,7 @@ func parseArguments(arguments []string, stderr io.Writer) (config, error) {
 	flagSet.StringVar(&configuration.platform, "platform", "android", "target platform: android, ios, web")
 	flagSet.StringVar(&configuration.arm, "arm", "", "experiment cell label recorded on every run (required)")
 	flagSet.StringVar(&configuration.generator, "generator", "seeded", "action generator: seeded or llm")
+	flagSet.StringVar(&configuration.labelSource, "label-source", "visible-text", "how candidates are named to the llm generator: visible-text (what a user reads) or resource-id (the identifier the app assigned). The seeded generator picks by index and ignores this")
 	flagSet.IntVar(&configuration.maxSteps, "max-steps", 0, "per-run step budget (required, must be positive)")
 	flagSet.DurationVar(&configuration.duration, "duration", 5*time.Minute, "per-run wall-clock ceiling")
 	flagSet.StringVar(&seedSpecification, "seeds", "", "seeds to run: ranges and lists, e.g. 1-10,20,30-32 (required)")
@@ -91,6 +93,13 @@ func parseArguments(arguments []string, stderr io.Writer) (config, error) {
 	default:
 		return config{}, fmt.Errorf("unsupported generator: %q (seeded, llm)", configuration.generator)
 	}
+	// Rejected before the first run, because a sweep that discovers the bad
+	// value on run 1 of 40 has already spent the device time of a whole cell.
+	switch configuration.labelSource {
+	case "visible-text", "resource-id":
+	default:
+		return config{}, fmt.Errorf("unsupported label source: %q (visible-text, resource-id)", configuration.labelSource)
+	}
 	if configuration.maxSteps <= 0 {
 		// Steps to first violation is right-censored at the budget, so a
 		// campaign without one has nothing to censor its clean runs at.
@@ -170,6 +179,7 @@ func runArguments(configuration config, seed, device string) []string {
 		"--platform", configuration.platform,
 		"--arm", configuration.arm,
 		"--generator", configuration.generator,
+		"--label-source", configuration.labelSource,
 		"--max-steps", strconv.Itoa(configuration.maxSteps),
 		"--duration", configuration.duration.String(),
 		"--seed", seed,

cmd/internal-tools/campaign/main_test.go

@@ -54,6 +54,7 @@ func TestParseArguments_Rejections(t *testing.T) {
 		{"missing output", []string{"--spec", "s", "--bundle-id", "a", "--arm", "b", "--seeds", "1", "--max-steps", "10"}, "--output is required"},
 		{"bad platform", append(baseArguments(), "--platform", "windows"), "unsupported platform"},
 		{"bad generator", append(baseArguments(), "--generator", "vibes"), "unsupported generator"},
+		{"bad label source", append(baseArguments(), "--label-source", "resource_id"), `unsupported label source: "resource_id"`},
 		{"zero max steps", append(baseArguments(), "--max-steps", "0"), "--max-steps must be positive"},
 		{"seed zero", append(baseArguments(), "--seeds", "0-2"), "not reproducible"},
 		{"duplicate device", append(baseArguments(), "--devices", "a,a"), "duplicate device"},
@@ -113,6 +114,16 @@ func TestRunArguments_PlatformDeviceFlagAndPassthrough(t *testing.T) {
 	}
 }
 
+func TestRunArguments_LabelSourceDefaultsToVisibleText(t *testing.T) {
+	configuration, err := parseArguments(baseArguments(), io.Discard)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := argumentValue(runArguments(configuration, "7", ""), "--label-source"); got != "visible-text" {
+		t.Errorf("--label-source = %q, want visible-text", got)
+	}
+}
+
 func argumentValue(arguments []string, name string) string {
 	index := slices.Index(arguments, name)
 	if index < 0 || index+1 >= len(arguments) {