feat(cli): add --arm and populate run meta from it

Model and instructions are recorded only when the LLM picker is the one that
will actually run, so a spec declaring generator = llm() that is run under the
seeded picker does not label its trace with a model it never called.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
pj committed 2026-08-12 18:14:07 +05:30
1 parent 54e33643b6
commit 9f9a01f773
5 files changed
+99 -9

No files matched your search

+2
View File
@@ -28,6 +28,7 @@ type testOptions struct {
androidAppPath string
duration time.Duration
maxSteps int
arm string
seed int64
output string
clearData bool
@@ -65,6 +66,7 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) {
flagSet.Int64Var(&options.seed, "seed", 0, "RNG seed (0 = random)")
flagSet.StringVar(&options.output, "output", "./runs", "output directory for traces")
flagSet.BoolVar(&options.clearData, "clear-data", true, "clear app data before launching so each run starts from a fresh install; pass --clear-data=false to resume prior state")
flagSet.StringVar(&options.arm, "arm", "", "experiment cell label, recorded in meta.json so a directory of runs can be attributed to a cell")
flagSet.StringVar(&options.generator, "generator", "seeded", "action generator: seeded (weighted random) or llm (model picks from the same candidate set; requires generator = llm() in the spec)")
if err := flagSet.Parse(args); err != nil {
return testOptions{}, err
+14
View File
@@ -297,3 +297,17 @@ func TestParseTestArgs_RejectsNegativeMaxSteps(t *testing.T) {
t.Fatal("expected an error for a negative --max-steps")
}
}
func TestParseTestArgs_ArmLabel(t *testing.T) {
options, err := parseTestArgs([]string{
"--spec", "s.ts",
"--bundle-id", "com.example",
"--arm", "seeded-identifier",
}, io.Discard)
if err != nil {
t.Fatal(err)
}
if options.arm != "seeded-identifier" {
t.Errorf("arm: got %q, want seeded-identifier", options.arm)
}
}
+1
View File
@@ -23,5 +23,6 @@ func runTestPipeline(ctx context.Context, options testOptions, stdout io.Writer)
Output: options.output,
ClearData: options.clearData,
Generator: options.generator,
Arm: options.arm,
}, stdout)
}