mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 11:07:10 +00:00
feat(trace): record arm membership and host in meta.json
meta.json recorded the seed but not which picker ran, how it was configured, what budget it was given, or which machine produced it. A directory of runs cannot be attributed to an experiment cell without those, which makes any factorial computed from such a directory unanalysable after the fact. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
8388ac5d2a
commit
54e33643b6
2 files changed
+81
No files matched your search
@@ -126,6 +126,24 @@ type Meta struct {
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
EndedAt *time.Time `json:"ended_at,omitempty"`
|
||||
SanderlingVersion string `json:"sanderling_version"`
|
||||
|
||||
// Arm labels the experiment cell this run belongs to, set from --arm. A
|
||||
// directory of runs cannot be attributed to a cell after the fact without
|
||||
// it, which makes any factorial computed from such a directory unanalysable.
|
||||
Arm string `json:"arm,omitempty"`
|
||||
// Generator, Model and Instructions record which picker ran and how it was
|
||||
// configured. Two runs that differ in any of these are different arms.
|
||||
Generator string `json:"generator,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Instructions string `json:"instructions,omitempty"`
|
||||
// MaxSteps and DurationMillis are the budget the run was given, which has
|
||||
// to be identical across arms for a comparison to mean anything.
|
||||
MaxSteps int `json:"max_steps,omitempty"`
|
||||
DurationMillis int64 `json:"duration_millis,omitempty"`
|
||||
// Host is the machine that produced the run. Campaigns are split across
|
||||
// several hosts, so a per-host effect has to be detectable rather than
|
||||
// invisible.
|
||||
Host string `json:"host,omitempty"`
|
||||
}
|
||||
|
||||
type Writer struct {
|
||||
|
||||
@@ -429,3 +429,66 @@ func readLines(t *testing.T, path string) []string {
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func TestWriteMeta_ArmMembershipRoundTrip(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
writer, err := NewWriter(directory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer writer.Close()
|
||||
|
||||
meta := Meta{
|
||||
Seed: 7,
|
||||
SpecPath: "spec.ts",
|
||||
BundleSHA256: "deadbeef",
|
||||
Platform: "android",
|
||||
BundleID: "com.example",
|
||||
StartedAt: time.Date(2026, 8, 12, 10, 0, 0, 0, time.UTC),
|
||||
SanderlingVersion: "0.0.1",
|
||||
Arm: "llm-visible-text",
|
||||
Generator: "llm",
|
||||
Model: "claude-sonnet-5",
|
||||
Instructions: "exercise the outbox",
|
||||
MaxSteps: 300,
|
||||
DurationMillis: 180000,
|
||||
Host: "emulator-farm-01",
|
||||
}
|
||||
if err := writer.WriteMeta(meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
body, err := os.ReadFile(filepath.Join(directory, "meta.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got Meta
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatalf("meta.json is not valid JSON: %v\n%s", err, body)
|
||||
}
|
||||
if got != meta {
|
||||
t.Errorf("meta round-trip mismatch:\n got: %+v\nwant: %+v", got, meta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteMeta_OmitsArmMembershipWhenUnset(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
writer, err := NewWriter(directory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer writer.Close()
|
||||
|
||||
if err := writer.WriteMeta(Meta{Seed: 1, Platform: "web"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := os.ReadFile(filepath.Join(directory, "meta.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, key := range []string{"arm", "generator", "model", "instructions", "max_steps", "duration_millis", "host"} {
|
||||
if strings.Contains(string(body), `"`+key+`"`) {
|
||||
t.Errorf("meta.json carries %q when unset:\n%s", key, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user