feat(trace): record the label source as arm membership

Recorded for seeded runs too, unlike model and instructions. Without it the two
seeded cells are indistinguishable in the artifact and the manipulation check
cannot be grouped.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
pj committed 2026-08-12 23:19:24 +05:30
1 parent de00412f67
commit ba8c4810cf
4 files changed
+43 -8

No files matched your search

+6
View File
@@ -140,6 +140,12 @@ type Meta struct {
Generator string `json:"generator,omitempty"`
Model string `json:"model,omitempty"`
Instructions string `json:"instructions,omitempty"`
// LabelSource records how candidates were named to the picker. It is written
// for a seeded run too, even though that picker selects by index and never
// reads a label: it is the cell the run was assigned to, and the pair of
// seeded runs across the two label modes is the manipulation check that says
// how much of any difference is just application nondeterminism.
LabelSource string `json:"label_source,omitempty"`
// MaxSteps and DurationMillis are the budget the run was given, which has
// to be identical across arms for a comparison to mean anything.
MaxSteps int `json:"max_steps,omitempty"`
+2 -1
View File
@@ -448,6 +448,7 @@ func TestWriteMeta_ArmMembershipRoundTrip(t *testing.T) {
SanderlingVersion: "0.0.1",
Arm: "llm-visible-text",
Generator: "llm",
LabelSource: "visible-text",
Model: "claude-sonnet-5",
Instructions: "exercise the outbox",
MaxSteps: 300,
@@ -486,7 +487,7 @@ func TestWriteMeta_OmitsArmMembershipWhenUnset(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, key := range []string{"arm", "generator", "model", "instructions", "max_steps", "duration_millis", "host"} {
for _, key := range []string{"arm", "generator", "label_source", "model", "instructions", "max_steps", "duration_millis", "host"} {
if strings.Contains(string(body), `"`+key+`"`) {
t.Errorf("meta.json carries %q when unset:\n%s", key, body)
}