feat(trace): record the label source as arm membership

Recorded for seeded runs too, unlike model and instructions. Without it the two
seeded cells are indistinguishable in the artifact and the manipulation check
cannot be grouped.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
pj committed 2026-08-12 23:19:24 +05:30
1 parent de00412f67
commit ba8c4810cf
4 files changed
+43 -8

No files matched your search

+30 -7
View File
@@ -187,13 +187,14 @@ func TestResolveSpecAPIPath_ReturnsEmptyWhenMissing(t *testing.T) {
func TestBuildRunMeta_RecordsArmMembership(t *testing.T) {
options := Options{
Spec: "spec.ts",
BundleID: "com.example",
Platform: "android",
Duration: 3 * time.Minute,
MaxSteps: 300,
Arm: "llm-visible-text",
Generator: "llm",
Spec: "spec.ts",
BundleID: "com.example",
Platform: "android",
Duration: 3 * time.Minute,
MaxSteps: 300,
Arm: "llm-visible-text",
Generator: "llm",
LabelSource: verifier.LabelSourceVisibleText,
}
meta := buildRunMeta(options, "deadbeef", 7, "farm-01",
verifier.LLMConfig{Model: "claude-sonnet-5", Instructions: "exercise the outbox"}, true)
@@ -201,6 +202,9 @@ func TestBuildRunMeta_RecordsArmMembership(t *testing.T) {
if meta.Arm != "llm-visible-text" || meta.Generator != "llm" {
t.Errorf("arm membership: got arm=%q generator=%q", meta.Arm, meta.Generator)
}
if meta.LabelSource != verifier.LabelSourceVisibleText {
t.Errorf("label source: got %q, want %q", meta.LabelSource, verifier.LabelSourceVisibleText)
}
if meta.Model != "claude-sonnet-5" || meta.Instructions != "exercise the outbox" {
t.Errorf("llm config: got model=%q instructions=%q", meta.Model, meta.Instructions)
}
@@ -223,6 +227,25 @@ func TestBuildRunMeta_OmitsModelWhenSeededPickerRuns(t *testing.T) {
}
}
// TestBuildRunMeta_RecordsLabelSourceForASeededRun is the deliberate difference
// from Model and Instructions above. The seeded picker never reads a label, but
// the run still belongs to a labelling cell, and the pair of seeded runs across
// the two cells is the manipulation check. Omitting it here would leave those
// two runs indistinguishable in the artifact.
func TestBuildRunMeta_RecordsLabelSourceForASeededRun(t *testing.T) {
options := Options{
Platform: "android",
Generator: "seeded",
Duration: time.Minute,
LabelSource: verifier.LabelSourceResourceID,
}
meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false)
if meta.LabelSource != verifier.LabelSourceResourceID {
t.Errorf("label source: got %q, want %q", meta.LabelSource, verifier.LabelSourceResourceID)
}
}
func TestBuildRunMeta_OmitsModelWhenSpecDeclaresNoLLMGenerator(t *testing.T) {
options := Options{Platform: "android", Generator: "llm", Duration: time.Minute}
meta := buildRunMeta(options, "deadbeef", 1, "farm-01", verifier.LLMConfig{}, false)