feat(cli): add --arm and populate run meta from it

Model and instructions are recorded only when the LLM picker is the one that
will actually run, so a spec declaring generator = llm() that is run under the
seeded picker does not label its trace with a model it never called.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
pj committed 2026-08-12 18:14:07 +05:30
1 parent 54e33643b6
commit 9f9a01f773
5 files changed
+99 -9

No files matched your search

+32 -9
View File
@@ -35,6 +35,9 @@ type Options struct {
Seed int64
Output string
ClearData bool
// Arm labels the experiment cell this run belongs to and is recorded in
// meta.json so a directory of runs can be attributed to a cell.
Arm string
// Generator selects the action picker: "llm" or the default seeded picker.
Generator string
@@ -48,6 +51,32 @@ type Options struct {
}
// Execute runs the full test pipeline: bundle, launch app, verify properties.
// buildRunMeta assembles the run's meta.json. Model and Instructions are
// recorded only when the LLM picker is the one that will actually run, so a
// spec that declares generator = llm() but is run under the seeded picker does
// not label its trace with a model it never called.
func buildRunMeta(options Options, bundleSHA256 string, seed int64, host string, llmConfig verifier.LLMConfig, hasLLMConfig bool) trace.Meta {
meta := trace.Meta{
Seed: seed,
SpecPath: options.Spec,
BundleSHA256: bundleSHA256,
Platform: options.Platform,
BundleID: options.BundleID,
StartedAt: time.Now().UTC(),
SanderlingVersion: "0.0.1",
Arm: options.Arm,
Generator: options.Generator,
MaxSteps: options.MaxSteps,
DurationMillis: options.Duration.Milliseconds(),
Host: host,
}
if options.Generator == "llm" && hasLLMConfig {
meta.Model = llmConfig.Model
meta.Instructions = llmConfig.Instructions
}
return meta
}
func Execute(ctx context.Context, options Options, stdout io.Writer) error {
switch options.Platform {
case "android":
@@ -141,15 +170,9 @@ func Execute(ctx context.Context, options Options, stdout io.Writer) error {
return fmt.Errorf("trace writer: %w", err)
}
defer traceWriter.Close()
meta := trace.Meta{
Seed: seed,
SpecPath: options.Spec,
BundleSHA256: bundle.SHA256,
Platform: options.Platform,
BundleID: options.BundleID,
StartedAt: time.Now().UTC(),
SanderlingVersion: "0.0.1",
}
hostname, _ := os.Hostname()
llmConfig, hasLLMConfig := verifierInstance.LLMConfig()
meta := buildRunMeta(options, bundle.SHA256, seed, hostname, llmConfig, hasLLMConfig)
if err := traceWriter.WriteMeta(meta); err != nil {
return fmt.Errorf("trace meta: %w", err)
}