feat(cli): add --arm and populate run meta from it

Model and instructions are recorded only when the LLM picker is the one that
will actually run, so a spec declaring generator = llm() that is run under the
seeded picker does not label its trace with a model it never called.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
pj committed 2026-08-12 18:14:07 +05:30
1 parent 54e33643b6
commit 9f9a01f773
5 files changed
+99 -9

No files matched your search

+32 -9
View File
@@ -35,6 +35,9 @@ type Options struct {
Seed int64
Output string
ClearData bool
// Arm labels the experiment cell this run belongs to and is recorded in
// meta.json so a directory of runs can be attributed to a cell.
Arm string
// Generator selects the action picker: "llm" or the default seeded picker.
Generator string
@@ -48,6 +51,32 @@ type Options struct {
}
// Execute runs the full test pipeline: bundle, launch app, verify properties.
// buildRunMeta assembles the run's meta.json. Model and Instructions are
// recorded only when the LLM picker is the one that will actually run, so a
// spec that declares generator = llm() but is run under the seeded picker does
// not label its trace with a model it never called.
func buildRunMeta(options Options, bundleSHA256 string, seed int64, host string, llmConfig verifier.LLMConfig, hasLLMConfig bool) trace.Meta {
meta := trace.Meta{
Seed: seed,
SpecPath: options.Spec,
BundleSHA256: bundleSHA256,
Platform: options.Platform,
BundleID: options.BundleID,
StartedAt: time.Now().UTC(),
SanderlingVersion: "0.0.1",
Arm: options.Arm,
Generator: options.Generator,
MaxSteps: options.MaxSteps,
DurationMillis: options.Duration.Milliseconds(),
Host: host,
}
if options.Generator == "llm" && hasLLMConfig {
meta.Model = llmConfig.Model
meta.Instructions = llmConfig.Instructions
}
return meta
}
func Execute(ctx context.Context, options Options, stdout io.Writer) error {
switch options.Platform {
case "android":
@@ -141,15 +170,9 @@ func Execute(ctx context.Context, options Options, stdout io.Writer) error {
return fmt.Errorf("trace writer: %w", err)
}
defer traceWriter.Close()
meta := trace.Meta{
Seed: seed,
SpecPath: options.Spec,
BundleSHA256: bundle.SHA256,
Platform: options.Platform,
BundleID: options.BundleID,
StartedAt: time.Now().UTC(),
SanderlingVersion: "0.0.1",
}
hostname, _ := os.Hostname()
llmConfig, hasLLMConfig := verifierInstance.LLMConfig()
meta := buildRunMeta(options, bundle.SHA256, seed, hostname, llmConfig, hasLLMConfig)
if err := traceWriter.WriteMeta(meta); err != nil {
return fmt.Errorf("trace meta: %w", err)
}
+50
View File
@@ -5,6 +5,9 @@ import (
"path/filepath"
"strings"
"testing"
"time"
"github.com/priyanshujain/sanderling/internal/verifier"
)
func TestResolveSeed_UsesConfiguredWhenNonZero(t *testing.T) {
@@ -181,3 +184,50 @@ func TestResolveSpecAPIPath_ReturnsEmptyWhenMissing(t *testing.T) {
t.Fatalf("got %q, want empty (no sanderling source tree reachable)", got)
}
}
func TestBuildRunMeta_RecordsArmMembership(t *testing.T) {
options := Options{
Spec: "spec.ts",
BundleID: "com.example",
Platform: "android",
Duration: 3 * time.Minute,
MaxSteps: 300,
Arm: "llm-visible-text",
Generator: "llm",
}
meta := buildRunMeta(options, "deadbeef", 7, "farm-01",
verifier.LLMConfig{Model: "claude-sonnet-5", Instructions: "exercise the outbox"}, true)
if meta.Arm != "llm-visible-text" || meta.Generator != "llm" {
t.Errorf("arm membership: got arm=%q generator=%q", meta.Arm, meta.Generator)
}
if meta.Model != "claude-sonnet-5" || meta.Instructions != "exercise the outbox" {
t.Errorf("llm config: got model=%q instructions=%q", meta.Model, meta.Instructions)
}
if meta.MaxSteps != 300 || meta.DurationMillis != 180000 {
t.Errorf("budget: got maxSteps=%d durationMillis=%d", meta.MaxSteps, meta.DurationMillis)
}
if meta.Host != "farm-01" || meta.Seed != 7 {
t.Errorf("host and seed: got host=%q seed=%d", meta.Host, meta.Seed)
}
}
func TestBuildRunMeta_OmitsModelWhenSeededPickerRuns(t *testing.T) {
options := Options{Platform: "android", Generator: "seeded", Duration: time.Minute}
meta := buildRunMeta(options, "deadbeef", 1, "farm-01",
verifier.LLMConfig{Model: "claude-sonnet-5", Instructions: "hunt bugs"}, true)
if meta.Model != "" || meta.Instructions != "" {
t.Errorf("a seeded run must not be labelled with a model it never called: model=%q instructions=%q",
meta.Model, meta.Instructions)
}
}
func TestBuildRunMeta_OmitsModelWhenSpecDeclaresNoLLMGenerator(t *testing.T) {
options := Options{Platform: "android", Generator: "llm", Duration: time.Minute}
meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false)
if meta.Model != "" {
t.Errorf("model recorded without a spec-declared llm generator: %q", meta.Model)
}
}