diff --git a/internal/testrun/testrun.go b/internal/testrun/testrun.go index 64d6c73..57ad6c6 100644 --- a/internal/testrun/testrun.go +++ b/internal/testrun/testrun.go @@ -90,6 +90,15 @@ type Options struct { // recorded only when the LLM picker is the one that will actually run, so a // spec that declares generator = llm() but is run under the seeded picker does // not label its trace with a model it never called. +// runDevice reads whichever flag named the hardware for this platform. An ios +// run is selected with --ios-device and leaves --device empty. +func runDevice(options Options) string { + if options.Platform == "ios" { + return options.IosDevice + } + return options.Device +} + func buildRunMeta(options Options, bundleSHA256 string, seed int64, host string, llmConfig verifier.LLMConfig, hasLLMConfig bool) trace.Meta { meta := trace.Meta{ Seed: seed, @@ -105,7 +114,7 @@ func buildRunMeta(options Options, bundleSHA256 string, seed int64, host string, MaxSteps: options.MaxSteps, DurationMillis: options.Duration.Milliseconds(), Host: host, - Device: options.Device, + Device: runDevice(options), } if options.Generator == "llm" && hasLLMConfig { meta.Model = llmConfig.Model diff --git a/internal/testrun/testrun_test.go b/internal/testrun/testrun_test.go index 5a87309..6aa125c 100644 --- a/internal/testrun/testrun_test.go +++ b/internal/testrun/testrun_test.go @@ -282,6 +282,36 @@ func TestBuildRunMeta_RecordsLabelSourceForASeededRun(t *testing.T) { } } +// An ios run names its simulator with --ios-device, so reading the android +// --device flag left every ios trace unable to say what it executed on. +func TestBuildRunMeta_NamesTheIosSimulatorItRanOn(t *testing.T) { + options := Options{ + Platform: "ios", + Generator: "seeded", + Duration: time.Minute, + IosDevice: "iPhone 17 Pro", + } + meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false) + + if meta.Device != "iPhone 17 Pro" { + t.Errorf("device: got %q, want the ios simulator the run named", meta.Device) + } +} + +func TestBuildRunMeta_NamesTheAndroidDeviceItRanOn(t *testing.T) { + options := Options{ + Platform: "android", + Generator: "seeded", + Duration: time.Minute, + Device: "emulator-5554", + } + meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false) + + if meta.Device != "emulator-5554" { + t.Errorf("device: got %q, want the android device the run named", meta.Device) + } +} + func TestBuildRunMeta_OmitsModelWhenSpecDeclaresNoLLMGenerator(t *testing.T) { options := Options{Platform: "android", Generator: "llm", Duration: time.Minute} meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false)