fix(testrun): an ios run records the simulator it executed on

Device was read from --device, which only an android run sets, so every
ios meta.json left the field empty and the trace could not say what
hardware produced it.
This commit is contained in:
pj committed 2026-08-18 16:38:00 +05:30
1 parent 6cac99cc32
commit b278fc869d
2 files changed
+40 -1

No files matched your search

+10 -1
View File
@@ -90,6 +90,15 @@ type Options struct {
// recorded only when the LLM picker is the one that will actually run, so a
// spec that declares generator = llm() but is run under the seeded picker does
// not label its trace with a model it never called.
// runDevice reads whichever flag named the hardware for this platform. An ios
// run is selected with --ios-device and leaves --device empty.
func runDevice(options Options) string {
if options.Platform == "ios" {
return options.IosDevice
}
return options.Device
}
func buildRunMeta(options Options, bundleSHA256 string, seed int64, host string, llmConfig verifier.LLMConfig, hasLLMConfig bool) trace.Meta {
meta := trace.Meta{
Seed: seed,
@@ -105,7 +114,7 @@ func buildRunMeta(options Options, bundleSHA256 string, seed int64, host string,
MaxSteps: options.MaxSteps,
DurationMillis: options.Duration.Milliseconds(),
Host: host,
Device: options.Device,
Device: runDevice(options),
}
if options.Generator == "llm" && hasLLMConfig {
meta.Model = llmConfig.Model
+30
View File
@@ -282,6 +282,36 @@ func TestBuildRunMeta_RecordsLabelSourceForASeededRun(t *testing.T) {
}
}
// An ios run names its simulator with --ios-device, so reading the android
// --device flag left every ios trace unable to say what it executed on.
func TestBuildRunMeta_NamesTheIosSimulatorItRanOn(t *testing.T) {
options := Options{
Platform: "ios",
Generator: "seeded",
Duration: time.Minute,
IosDevice: "iPhone 17 Pro",
}
meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false)
if meta.Device != "iPhone 17 Pro" {
t.Errorf("device: got %q, want the ios simulator the run named", meta.Device)
}
}
func TestBuildRunMeta_NamesTheAndroidDeviceItRanOn(t *testing.T) {
options := Options{
Platform: "android",
Generator: "seeded",
Duration: time.Minute,
Device: "emulator-5554",
}
meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false)
if meta.Device != "emulator-5554" {
t.Errorf("device: got %q, want the android device the run named", meta.Device)
}
}
func TestBuildRunMeta_OmitsModelWhenSpecDeclaresNoLLMGenerator(t *testing.T) {
options := Options{Platform: "android", Generator: "llm", Duration: time.Minute}
meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false)