mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
fix(testrun): an ios run records the simulator it executed on
Device was read from --device, which only an android run sets, so every ios meta.json left the field empty and the trace could not say what hardware produced it.
This commit is contained in:
1 parent
6cac99cc32
commit
b278fc869d
2 files changed
+40
-1
No files matched your search
@@ -90,6 +90,15 @@ type Options struct {
|
||||
// recorded only when the LLM picker is the one that will actually run, so a
|
||||
// spec that declares generator = llm() but is run under the seeded picker does
|
||||
// not label its trace with a model it never called.
|
||||
// runDevice reads whichever flag named the hardware for this platform. An ios
|
||||
// run is selected with --ios-device and leaves --device empty.
|
||||
func runDevice(options Options) string {
|
||||
if options.Platform == "ios" {
|
||||
return options.IosDevice
|
||||
}
|
||||
return options.Device
|
||||
}
|
||||
|
||||
func buildRunMeta(options Options, bundleSHA256 string, seed int64, host string, llmConfig verifier.LLMConfig, hasLLMConfig bool) trace.Meta {
|
||||
meta := trace.Meta{
|
||||
Seed: seed,
|
||||
@@ -105,7 +114,7 @@ func buildRunMeta(options Options, bundleSHA256 string, seed int64, host string,
|
||||
MaxSteps: options.MaxSteps,
|
||||
DurationMillis: options.Duration.Milliseconds(),
|
||||
Host: host,
|
||||
Device: options.Device,
|
||||
Device: runDevice(options),
|
||||
}
|
||||
if options.Generator == "llm" && hasLLMConfig {
|
||||
meta.Model = llmConfig.Model
|
||||
|
||||
@@ -282,6 +282,36 @@ func TestBuildRunMeta_RecordsLabelSourceForASeededRun(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// An ios run names its simulator with --ios-device, so reading the android
|
||||
// --device flag left every ios trace unable to say what it executed on.
|
||||
func TestBuildRunMeta_NamesTheIosSimulatorItRanOn(t *testing.T) {
|
||||
options := Options{
|
||||
Platform: "ios",
|
||||
Generator: "seeded",
|
||||
Duration: time.Minute,
|
||||
IosDevice: "iPhone 17 Pro",
|
||||
}
|
||||
meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false)
|
||||
|
||||
if meta.Device != "iPhone 17 Pro" {
|
||||
t.Errorf("device: got %q, want the ios simulator the run named", meta.Device)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRunMeta_NamesTheAndroidDeviceItRanOn(t *testing.T) {
|
||||
options := Options{
|
||||
Platform: "android",
|
||||
Generator: "seeded",
|
||||
Duration: time.Minute,
|
||||
Device: "emulator-5554",
|
||||
}
|
||||
meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false)
|
||||
|
||||
if meta.Device != "emulator-5554" {
|
||||
t.Errorf("device: got %q, want the android device the run named", meta.Device)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRunMeta_OmitsModelWhenSpecDeclaresNoLLMGenerator(t *testing.T) {
|
||||
options := Options{Platform: "android", Generator: "llm", Duration: time.Minute}
|
||||
meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false)
|
||||
|
||||
Reference in new issue
Block a user