mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 11:07:10 +00:00
feat(cli): add --arm and populate run meta from it
Model and instructions are recorded only when the LLM picker is the one that will actually run, so a spec declaring generator = llm() that is run under the seeded picker does not label its trace with a model it never called. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
54e33643b6
commit
9f9a01f773
5 files changed
+99
-9
No files matched your search
@@ -28,6 +28,7 @@ type testOptions struct {
|
||||
androidAppPath string
|
||||
duration time.Duration
|
||||
maxSteps int
|
||||
arm string
|
||||
seed int64
|
||||
output string
|
||||
clearData bool
|
||||
@@ -65,6 +66,7 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) {
|
||||
flagSet.Int64Var(&options.seed, "seed", 0, "RNG seed (0 = random)")
|
||||
flagSet.StringVar(&options.output, "output", "./runs", "output directory for traces")
|
||||
flagSet.BoolVar(&options.clearData, "clear-data", true, "clear app data before launching so each run starts from a fresh install; pass --clear-data=false to resume prior state")
|
||||
flagSet.StringVar(&options.arm, "arm", "", "experiment cell label, recorded in meta.json so a directory of runs can be attributed to a cell")
|
||||
flagSet.StringVar(&options.generator, "generator", "seeded", "action generator: seeded (weighted random) or llm (model picks from the same candidate set; requires generator = llm() in the spec)")
|
||||
if err := flagSet.Parse(args); err != nil {
|
||||
return testOptions{}, err
|
||||
|
||||
@@ -297,3 +297,17 @@ func TestParseTestArgs_RejectsNegativeMaxSteps(t *testing.T) {
|
||||
t.Fatal("expected an error for a negative --max-steps")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTestArgs_ArmLabel(t *testing.T) {
|
||||
options, err := parseTestArgs([]string{
|
||||
"--spec", "s.ts",
|
||||
"--bundle-id", "com.example",
|
||||
"--arm", "seeded-identifier",
|
||||
}, io.Discard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if options.arm != "seeded-identifier" {
|
||||
t.Errorf("arm: got %q, want seeded-identifier", options.arm)
|
||||
}
|
||||
}
|
||||
@@ -23,5 +23,6 @@ func runTestPipeline(ctx context.Context, options testOptions, stdout io.Writer)
|
||||
Output: options.output,
|
||||
ClearData: options.clearData,
|
||||
Generator: options.generator,
|
||||
Arm: options.arm,
|
||||
}, stdout)
|
||||
}
|
||||
Reference in new issue
Block a user