mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 12:07:09 +00:00
feat(trace): record the device a run executed on
meta.json carried the host but not the device, so a trace could not say what hardware produced it without the campaign manifest beside it. An experiment splitting cells across api levels could only join them through that manifest. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
a7f26e27c4
commit
7b6a346443
4 files changed
+38
-1
No files matched your search
@@ -102,6 +102,7 @@ func buildRunMeta(options Options, bundleSHA256 string, seed int64, host string,
|
|||||||
MaxSteps: options.MaxSteps,
|
MaxSteps: options.MaxSteps,
|
||||||
DurationMillis: options.Duration.Milliseconds(),
|
DurationMillis: options.Duration.Milliseconds(),
|
||||||
Host: host,
|
Host: host,
|
||||||
|
Device: options.Device,
|
||||||
}
|
}
|
||||||
if options.Generator == "llm" && hasLLMConfig {
|
if options.Generator == "llm" && hasLLMConfig {
|
||||||
meta.Model = llmConfig.Model
|
meta.Model = llmConfig.Model
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
|
|
||||||
"github.com/priyanshujain/sanderling/internal/driver"
|
"github.com/priyanshujain/sanderling/internal/driver"
|
||||||
"github.com/priyanshujain/sanderling/internal/runner"
|
"github.com/priyanshujain/sanderling/internal/runner"
|
||||||
|
"github.com/priyanshujain/sanderling/internal/trace"
|
||||||
"github.com/priyanshujain/sanderling/internal/verifier"
|
"github.com/priyanshujain/sanderling/internal/verifier"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -222,6 +223,35 @@ func TestBuildRunMeta_RecordsArmMembership(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The device a run drove has to survive in the run's own artifact. Held only in
|
||||||
|
// the campaign's runs.jsonl, a trace read on its own cannot say which emulator,
|
||||||
|
// or which API level, produced it.
|
||||||
|
func TestBuildRunMeta_RecordsTheDeviceInMetaJSON(t *testing.T) {
|
||||||
|
directory := t.TempDir()
|
||||||
|
writer, err := trace.NewWriter(directory)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer writer.Close()
|
||||||
|
|
||||||
|
options := Options{Platform: "android", Generator: "seeded", Duration: time.Minute, Device: "emulator-5556"}
|
||||||
|
if err := writer.WriteMeta(buildRunMeta(options, "deadbeef", 3, "farm-01", verifier.LLMConfig{}, false)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := os.ReadFile(filepath.Join(directory, "meta.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var stored trace.Meta
|
||||||
|
if err := json.Unmarshal(body, &stored); err != nil {
|
||||||
|
t.Fatalf("meta.json is not valid JSON: %v\n%s", err, body)
|
||||||
|
}
|
||||||
|
if stored.Device != "emulator-5556" {
|
||||||
|
t.Errorf("device in meta.json: got %q, want emulator-5556\n%s", stored.Device, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBuildRunMeta_OmitsModelWhenSeededPickerRuns(t *testing.T) {
|
func TestBuildRunMeta_OmitsModelWhenSeededPickerRuns(t *testing.T) {
|
||||||
options := Options{Platform: "android", Generator: "seeded", Duration: time.Minute}
|
options := Options{Platform: "android", Generator: "seeded", Duration: time.Minute}
|
||||||
meta := buildRunMeta(options, "deadbeef", 1, "farm-01",
|
meta := buildRunMeta(options, "deadbeef", 1, "farm-01",
|
||||||
|
|||||||
@@ -197,6 +197,11 @@ type Meta struct {
|
|||||||
// several hosts, so a per-host effect has to be detectable rather than
|
// several hosts, so a per-host effect has to be detectable rather than
|
||||||
// invisible.
|
// invisible.
|
||||||
Host string `json:"host,omitempty"`
|
Host string `json:"host,omitempty"`
|
||||||
|
// Device is the target the run drove, from --device. One host drives
|
||||||
|
// several emulators at different API levels, so without it a trace on its
|
||||||
|
// own cannot say what produced it and a per-device split can only be
|
||||||
|
// recovered by joining against the campaign manifest.
|
||||||
|
Device string `json:"device,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Writer struct {
|
type Writer struct {
|
||||||
|
|||||||
@@ -463,6 +463,7 @@ func TestWriteMeta_ArmMembershipRoundTrip(t *testing.T) {
|
|||||||
MaxSteps: 300,
|
MaxSteps: 300,
|
||||||
DurationMillis: 180000,
|
DurationMillis: 180000,
|
||||||
Host: "emulator-farm-01",
|
Host: "emulator-farm-01",
|
||||||
|
Device: "emulator-5556",
|
||||||
}
|
}
|
||||||
if err := writer.WriteMeta(meta); err != nil {
|
if err := writer.WriteMeta(meta); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -496,7 +497,7 @@ func TestWriteMeta_OmitsArmMembershipWhenUnset(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
for _, key := range []string{"arm", "generator", "label_source", "model", "instructions", "max_steps", "duration_millis", "host"} {
|
for _, key := range []string{"arm", "generator", "label_source", "model", "instructions", "max_steps", "duration_millis", "host", "device"} {
|
||||||
if strings.Contains(string(body), `"`+key+`"`) {
|
if strings.Contains(string(body), `"`+key+`"`) {
|
||||||
t.Errorf("meta.json carries %q when unset:\n%s", key, body)
|
t.Errorf("meta.json carries %q when unset:\n%s", key, body)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user