From 9f9a01f7737f467c7c7d53b86b1a23558cae0c31 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 12 Aug 2026 18:14:07 +0530 Subject: [PATCH] feat(cli): add --arm and populate run meta from it Model and instructions are recorded only when the LLM picker is the one that will actually run, so a spec declaring generator = llm() that is run under the seeded picker does not label its trace with a model it never called. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX --- cmd/sanderling/main.go | 2 ++ cmd/sanderling/main_test.go | 14 +++++++++ cmd/sanderling/test_run.go | 1 + internal/testrun/testrun.go | 41 ++++++++++++++++++++------ internal/testrun/testrun_test.go | 50 ++++++++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 9 deletions(-) diff --git a/cmd/sanderling/main.go b/cmd/sanderling/main.go index b57dcd1..c3036cd 100644 --- a/cmd/sanderling/main.go +++ b/cmd/sanderling/main.go @@ -28,6 +28,7 @@ type testOptions struct { androidAppPath string duration time.Duration maxSteps int + arm string seed int64 output string clearData bool @@ -65,6 +66,7 @@ func parseTestArgs(args []string, stderr io.Writer) (testOptions, error) { flagSet.Int64Var(&options.seed, "seed", 0, "RNG seed (0 = random)") flagSet.StringVar(&options.output, "output", "./runs", "output directory for traces") flagSet.BoolVar(&options.clearData, "clear-data", true, "clear app data before launching so each run starts from a fresh install; pass --clear-data=false to resume prior state") + flagSet.StringVar(&options.arm, "arm", "", "experiment cell label, recorded in meta.json so a directory of runs can be attributed to a cell") flagSet.StringVar(&options.generator, "generator", "seeded", "action generator: seeded (weighted random) or llm (model picks from the same candidate set; requires generator = llm() in the spec)") if err := flagSet.Parse(args); err != nil { return testOptions{}, err diff --git a/cmd/sanderling/main_test.go b/cmd/sanderling/main_test.go index 3e1cb85..8061f7e 100644 --- a/cmd/sanderling/main_test.go +++ b/cmd/sanderling/main_test.go @@ -297,3 +297,17 @@ func TestParseTestArgs_RejectsNegativeMaxSteps(t *testing.T) { t.Fatal("expected an error for a negative --max-steps") } } + +func TestParseTestArgs_ArmLabel(t *testing.T) { + options, err := parseTestArgs([]string{ + "--spec", "s.ts", + "--bundle-id", "com.example", + "--arm", "seeded-identifier", + }, io.Discard) + if err != nil { + t.Fatal(err) + } + if options.arm != "seeded-identifier" { + t.Errorf("arm: got %q, want seeded-identifier", options.arm) + } +} diff --git a/cmd/sanderling/test_run.go b/cmd/sanderling/test_run.go index 8b31ab0..06ff8f6 100644 --- a/cmd/sanderling/test_run.go +++ b/cmd/sanderling/test_run.go @@ -23,5 +23,6 @@ func runTestPipeline(ctx context.Context, options testOptions, stdout io.Writer) Output: options.output, ClearData: options.clearData, Generator: options.generator, + Arm: options.arm, }, stdout) } diff --git a/internal/testrun/testrun.go b/internal/testrun/testrun.go index c5727e6..ce6f188 100644 --- a/internal/testrun/testrun.go +++ b/internal/testrun/testrun.go @@ -35,6 +35,9 @@ type Options struct { Seed int64 Output string ClearData bool + // Arm labels the experiment cell this run belongs to and is recorded in + // meta.json so a directory of runs can be attributed to a cell. + Arm string // Generator selects the action picker: "llm" or the default seeded picker. Generator string @@ -48,6 +51,32 @@ type Options struct { } // Execute runs the full test pipeline: bundle, launch app, verify properties. +// buildRunMeta assembles the run's meta.json. Model and Instructions are +// recorded only when the LLM picker is the one that will actually run, so a +// spec that declares generator = llm() but is run under the seeded picker does +// not label its trace with a model it never called. +func buildRunMeta(options Options, bundleSHA256 string, seed int64, host string, llmConfig verifier.LLMConfig, hasLLMConfig bool) trace.Meta { + meta := trace.Meta{ + Seed: seed, + SpecPath: options.Spec, + BundleSHA256: bundleSHA256, + Platform: options.Platform, + BundleID: options.BundleID, + StartedAt: time.Now().UTC(), + SanderlingVersion: "0.0.1", + Arm: options.Arm, + Generator: options.Generator, + MaxSteps: options.MaxSteps, + DurationMillis: options.Duration.Milliseconds(), + Host: host, + } + if options.Generator == "llm" && hasLLMConfig { + meta.Model = llmConfig.Model + meta.Instructions = llmConfig.Instructions + } + return meta +} + func Execute(ctx context.Context, options Options, stdout io.Writer) error { switch options.Platform { case "android": @@ -141,15 +170,9 @@ func Execute(ctx context.Context, options Options, stdout io.Writer) error { return fmt.Errorf("trace writer: %w", err) } defer traceWriter.Close() - meta := trace.Meta{ - Seed: seed, - SpecPath: options.Spec, - BundleSHA256: bundle.SHA256, - Platform: options.Platform, - BundleID: options.BundleID, - StartedAt: time.Now().UTC(), - SanderlingVersion: "0.0.1", - } + hostname, _ := os.Hostname() + llmConfig, hasLLMConfig := verifierInstance.LLMConfig() + meta := buildRunMeta(options, bundle.SHA256, seed, hostname, llmConfig, hasLLMConfig) if err := traceWriter.WriteMeta(meta); err != nil { return fmt.Errorf("trace meta: %w", err) } diff --git a/internal/testrun/testrun_test.go b/internal/testrun/testrun_test.go index 6ab171c..2df82c4 100644 --- a/internal/testrun/testrun_test.go +++ b/internal/testrun/testrun_test.go @@ -5,6 +5,9 @@ import ( "path/filepath" "strings" "testing" + "time" + + "github.com/priyanshujain/sanderling/internal/verifier" ) func TestResolveSeed_UsesConfiguredWhenNonZero(t *testing.T) { @@ -181,3 +184,50 @@ func TestResolveSpecAPIPath_ReturnsEmptyWhenMissing(t *testing.T) { t.Fatalf("got %q, want empty (no sanderling source tree reachable)", got) } } + +func TestBuildRunMeta_RecordsArmMembership(t *testing.T) { + options := Options{ + Spec: "spec.ts", + BundleID: "com.example", + Platform: "android", + Duration: 3 * time.Minute, + MaxSteps: 300, + Arm: "llm-visible-text", + Generator: "llm", + } + meta := buildRunMeta(options, "deadbeef", 7, "farm-01", + verifier.LLMConfig{Model: "claude-sonnet-5", Instructions: "exercise the outbox"}, true) + + if meta.Arm != "llm-visible-text" || meta.Generator != "llm" { + t.Errorf("arm membership: got arm=%q generator=%q", meta.Arm, meta.Generator) + } + if meta.Model != "claude-sonnet-5" || meta.Instructions != "exercise the outbox" { + t.Errorf("llm config: got model=%q instructions=%q", meta.Model, meta.Instructions) + } + if meta.MaxSteps != 300 || meta.DurationMillis != 180000 { + t.Errorf("budget: got maxSteps=%d durationMillis=%d", meta.MaxSteps, meta.DurationMillis) + } + if meta.Host != "farm-01" || meta.Seed != 7 { + t.Errorf("host and seed: got host=%q seed=%d", meta.Host, meta.Seed) + } +} + +func TestBuildRunMeta_OmitsModelWhenSeededPickerRuns(t *testing.T) { + options := Options{Platform: "android", Generator: "seeded", Duration: time.Minute} + meta := buildRunMeta(options, "deadbeef", 1, "farm-01", + verifier.LLMConfig{Model: "claude-sonnet-5", Instructions: "hunt bugs"}, true) + + if meta.Model != "" || meta.Instructions != "" { + t.Errorf("a seeded run must not be labelled with a model it never called: model=%q instructions=%q", + meta.Model, meta.Instructions) + } +} + +func TestBuildRunMeta_OmitsModelWhenSpecDeclaresNoLLMGenerator(t *testing.T) { + options := Options{Platform: "android", Generator: "llm", Duration: time.Minute} + meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false) + + if meta.Model != "" { + t.Errorf("model recorded without a spec-declared llm generator: %q", meta.Model) + } +}