mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 12:07:09 +00:00
feat(campaign): make the label source a cell dimension
A 2x2 of policy against labelling needs the runner to express both factors. It could only express the policy, so half the factorial had to go through --extra, where the manifest would not record what was actually run. Rejected at parse rather than on dispatch: a sweep that finds the bad value on run 1 of 40 has already spent a cell's worth of device time. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
d987526e47
commit
ff2de344e9
3 files changed
+43
No files matched your search
@@ -23,6 +23,7 @@ type config struct {
|
||||
platform string
|
||||
arm string
|
||||
generator string
|
||||
labelSource string
|
||||
maxSteps int
|
||||
duration time.Duration
|
||||
seeds []int64
|
||||
@@ -58,6 +59,7 @@ func parseArguments(arguments []string, stderr io.Writer) (config, error) {
|
||||
flagSet.StringVar(&configuration.platform, "platform", "android", "target platform: android, ios, web")
|
||||
flagSet.StringVar(&configuration.arm, "arm", "", "experiment cell label recorded on every run (required)")
|
||||
flagSet.StringVar(&configuration.generator, "generator", "seeded", "action generator: seeded or llm")
|
||||
flagSet.StringVar(&configuration.labelSource, "label-source", "visible-text", "how candidates are named to the llm generator: visible-text (what a user reads) or resource-id (the identifier the app assigned). The seeded generator picks by index and ignores this")
|
||||
flagSet.IntVar(&configuration.maxSteps, "max-steps", 0, "per-run step budget (required, must be positive)")
|
||||
flagSet.DurationVar(&configuration.duration, "duration", 5*time.Minute, "per-run wall-clock ceiling")
|
||||
flagSet.StringVar(&seedSpecification, "seeds", "", "seeds to run: ranges and lists, e.g. 1-10,20,30-32 (required)")
|
||||
@@ -91,6 +93,13 @@ func parseArguments(arguments []string, stderr io.Writer) (config, error) {
|
||||
default:
|
||||
return config{}, fmt.Errorf("unsupported generator: %q (seeded, llm)", configuration.generator)
|
||||
}
|
||||
// Rejected before the first run, because a sweep that discovers the bad
|
||||
// value on run 1 of 40 has already spent the device time of a whole cell.
|
||||
switch configuration.labelSource {
|
||||
case "visible-text", "resource-id":
|
||||
default:
|
||||
return config{}, fmt.Errorf("unsupported label source: %q (visible-text, resource-id)", configuration.labelSource)
|
||||
}
|
||||
if configuration.maxSteps <= 0 {
|
||||
// Steps to first violation is right-censored at the budget, so a
|
||||
// campaign without one has nothing to censor its clean runs at.
|
||||
@@ -170,6 +179,7 @@ func runArguments(configuration config, seed, device string) []string {
|
||||
"--platform", configuration.platform,
|
||||
"--arm", configuration.arm,
|
||||
"--generator", configuration.generator,
|
||||
"--label-source", configuration.labelSource,
|
||||
"--max-steps", strconv.Itoa(configuration.maxSteps),
|
||||
"--duration", configuration.duration.String(),
|
||||
"--seed", seed,
|
||||
|
||||
Reference in new issue
Block a user