feat(trace): record arm membership and host in meta.json

meta.json recorded the seed but not which picker ran, how it was configured,
what budget it was given, or which machine produced it. A directory of runs
cannot be attributed to an experiment cell without those, which makes any
factorial computed from such a directory unanalysable after the fact.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
pj committed 2026-08-12 18:12:28 +05:30
1 parent 8388ac5d2a
commit 54e33643b6
2 files changed
+81

No files matched your search

+18
View File
@@ -126,6 +126,24 @@ type Meta struct {
StartedAt time.Time `json:"started_at"`
EndedAt *time.Time `json:"ended_at,omitempty"`
SanderlingVersion string `json:"sanderling_version"`
// Arm labels the experiment cell this run belongs to, set from --arm. A
// directory of runs cannot be attributed to a cell after the fact without
// it, which makes any factorial computed from such a directory unanalysable.
Arm string `json:"arm,omitempty"`
// Generator, Model and Instructions record which picker ran and how it was
// configured. Two runs that differ in any of these are different arms.
Generator string `json:"generator,omitempty"`
Model string `json:"model,omitempty"`
Instructions string `json:"instructions,omitempty"`
// MaxSteps and DurationMillis are the budget the run was given, which has
// to be identical across arms for a comparison to mean anything.
MaxSteps int `json:"max_steps,omitempty"`
DurationMillis int64 `json:"duration_millis,omitempty"`
// Host is the machine that produced the run. Campaigns are split across
// several hosts, so a per-host effect has to be detectable rather than
// invisible.
Host string `json:"host,omitempty"`
}
type Writer struct {
+63
View File
@@ -429,3 +429,66 @@ func readLines(t *testing.T, path string) []string {
}
return lines
}
func TestWriteMeta_ArmMembershipRoundTrip(t *testing.T) {
directory := t.TempDir()
writer, err := NewWriter(directory)
if err != nil {
t.Fatal(err)
}
defer writer.Close()
meta := Meta{
Seed: 7,
SpecPath: "spec.ts",
BundleSHA256: "deadbeef",
Platform: "android",
BundleID: "com.example",
StartedAt: time.Date(2026, 8, 12, 10, 0, 0, 0, time.UTC),
SanderlingVersion: "0.0.1",
Arm: "llm-visible-text",
Generator: "llm",
Model: "claude-sonnet-5",
Instructions: "exercise the outbox",
MaxSteps: 300,
DurationMillis: 180000,
Host: "emulator-farm-01",
}
if err := writer.WriteMeta(meta); err != nil {
t.Fatal(err)
}
body, err := os.ReadFile(filepath.Join(directory, "meta.json"))
if err != nil {
t.Fatal(err)
}
var got Meta
if err := json.Unmarshal(body, &got); err != nil {
t.Fatalf("meta.json is not valid JSON: %v\n%s", err, body)
}
if got != meta {
t.Errorf("meta round-trip mismatch:\n got: %+v\nwant: %+v", got, meta)
}
}
func TestWriteMeta_OmitsArmMembershipWhenUnset(t *testing.T) {
directory := t.TempDir()
writer, err := NewWriter(directory)
if err != nil {
t.Fatal(err)
}
defer writer.Close()
if err := writer.WriteMeta(Meta{Seed: 1, Platform: "web"}); err != nil {
t.Fatal(err)
}
body, err := os.ReadFile(filepath.Join(directory, "meta.json"))
if err != nil {
t.Fatal(err)
}
for _, key := range []string{"arm", "generator", "model", "instructions", "max_steps", "duration_millis", "host"} {
if strings.Contains(string(body), `"`+key+`"`) {
t.Errorf("meta.json carries %q when unset:\n%s", key, body)
}
}
}