diff --git a/internal/testrun/testrun.go b/internal/testrun/testrun.go index ce6f188..57cc027 100644 --- a/internal/testrun/testrun.go +++ b/internal/testrun/testrun.go @@ -40,6 +40,9 @@ type Options struct { Arm string // Generator selects the action picker: "llm" or the default seeded picker. Generator string + // LabelSource selects how candidates are named to the model picker, and is + // recorded in meta.json as part of the run's cell. + LabelSource string // iosUDID, iosIsSimulator, and iosCoreDeviceID are filled by Execute after // resolving the iOS target, then read by buildDriver to choose the simulator @@ -66,6 +69,7 @@ func buildRunMeta(options Options, bundleSHA256 string, seed int64, host string, SanderlingVersion: "0.0.1", Arm: options.Arm, Generator: options.Generator, + LabelSource: options.LabelSource, MaxSteps: options.MaxSteps, DurationMillis: options.Duration.Milliseconds(), Host: host, @@ -198,6 +202,7 @@ func Execute(ctx context.Context, options Options, stdout io.Writer) error { TraceWriter: traceWriter, Logger: newProgressLogger(stdout), Generator: options.Generator, + LabelSource: options.LabelSource, }) terminateCtx, terminateCancel := context.WithTimeout(context.Background(), 5*time.Second) diff --git a/internal/testrun/testrun_test.go b/internal/testrun/testrun_test.go index 2df82c4..6be7777 100644 --- a/internal/testrun/testrun_test.go +++ b/internal/testrun/testrun_test.go @@ -187,13 +187,14 @@ func TestResolveSpecAPIPath_ReturnsEmptyWhenMissing(t *testing.T) { func TestBuildRunMeta_RecordsArmMembership(t *testing.T) { options := Options{ - Spec: "spec.ts", - BundleID: "com.example", - Platform: "android", - Duration: 3 * time.Minute, - MaxSteps: 300, - Arm: "llm-visible-text", - Generator: "llm", + Spec: "spec.ts", + BundleID: "com.example", + Platform: "android", + Duration: 3 * time.Minute, + MaxSteps: 300, + Arm: "llm-visible-text", + Generator: "llm", + LabelSource: verifier.LabelSourceVisibleText, } meta := buildRunMeta(options, "deadbeef", 7, "farm-01", verifier.LLMConfig{Model: "claude-sonnet-5", Instructions: "exercise the outbox"}, true) @@ -201,6 +202,9 @@ func TestBuildRunMeta_RecordsArmMembership(t *testing.T) { if meta.Arm != "llm-visible-text" || meta.Generator != "llm" { t.Errorf("arm membership: got arm=%q generator=%q", meta.Arm, meta.Generator) } + if meta.LabelSource != verifier.LabelSourceVisibleText { + t.Errorf("label source: got %q, want %q", meta.LabelSource, verifier.LabelSourceVisibleText) + } if meta.Model != "claude-sonnet-5" || meta.Instructions != "exercise the outbox" { t.Errorf("llm config: got model=%q instructions=%q", meta.Model, meta.Instructions) } @@ -223,6 +227,25 @@ func TestBuildRunMeta_OmitsModelWhenSeededPickerRuns(t *testing.T) { } } +// TestBuildRunMeta_RecordsLabelSourceForASeededRun is the deliberate difference +// from Model and Instructions above. The seeded picker never reads a label, but +// the run still belongs to a labelling cell, and the pair of seeded runs across +// the two cells is the manipulation check. Omitting it here would leave those +// two runs indistinguishable in the artifact. +func TestBuildRunMeta_RecordsLabelSourceForASeededRun(t *testing.T) { + options := Options{ + Platform: "android", + Generator: "seeded", + Duration: time.Minute, + LabelSource: verifier.LabelSourceResourceID, + } + meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false) + + if meta.LabelSource != verifier.LabelSourceResourceID { + t.Errorf("label source: got %q, want %q", meta.LabelSource, verifier.LabelSourceResourceID) + } +} + func TestBuildRunMeta_OmitsModelWhenSpecDeclaresNoLLMGenerator(t *testing.T) { options := Options{Platform: "android", Generator: "llm", Duration: time.Minute} meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false) diff --git a/internal/trace/writer.go b/internal/trace/writer.go index 30e6ce2..d2f6b3f 100644 --- a/internal/trace/writer.go +++ b/internal/trace/writer.go @@ -140,6 +140,12 @@ type Meta struct { Generator string `json:"generator,omitempty"` Model string `json:"model,omitempty"` Instructions string `json:"instructions,omitempty"` + // LabelSource records how candidates were named to the picker. It is written + // for a seeded run too, even though that picker selects by index and never + // reads a label: it is the cell the run was assigned to, and the pair of + // seeded runs across the two label modes is the manipulation check that says + // how much of any difference is just application nondeterminism. + LabelSource string `json:"label_source,omitempty"` // MaxSteps and DurationMillis are the budget the run was given, which has // to be identical across arms for a comparison to mean anything. MaxSteps int `json:"max_steps,omitempty"` diff --git a/internal/trace/writer_test.go b/internal/trace/writer_test.go index 1d50b70..119c339 100644 --- a/internal/trace/writer_test.go +++ b/internal/trace/writer_test.go @@ -448,6 +448,7 @@ func TestWriteMeta_ArmMembershipRoundTrip(t *testing.T) { SanderlingVersion: "0.0.1", Arm: "llm-visible-text", Generator: "llm", + LabelSource: "visible-text", Model: "claude-sonnet-5", Instructions: "exercise the outbox", MaxSteps: 300, @@ -486,7 +487,7 @@ func TestWriteMeta_OmitsArmMembershipWhenUnset(t *testing.T) { if err != nil { t.Fatal(err) } - for _, key := range []string{"arm", "generator", "model", "instructions", "max_steps", "duration_millis", "host"} { + for _, key := range []string{"arm", "generator", "label_source", "model", "instructions", "max_steps", "duration_millis", "host"} { if strings.Contains(string(body), `"`+key+`"`) { t.Errorf("meta.json carries %q when unset:\n%s", key, body) }