fix(trace): an action names the generator that produced it

The setup exclusion landed for the model arm only, because only a model
pick stamped a source. A seeded run returned setup's action through the
same entry with no marker, so its denominator still counted the login
while the model arm's did not, and the two are compared.

serializeAction names setup and seeded on the wire, so both arms are
counted by one rule. An already-recorded trace names nothing and keeps
exactly the count it was reported with; unattributed_actions counts those
steps so the old denominator cannot pass as the new one. TraceVersion is
deliberately unbumped: oracle-reduction refuses a differing version, and
a bump would make all 169 recorded runs unreplayable.

cmd/internal-tools/analyze/analysis.go

@@ -225,9 +225,10 @@ func summarize(current arm) armSummary {
 		}
 		summary.Usable++
 		summary.TotalSteps += item.Steps
-		// Steps and actions differ by the steps that chose no action and the
-		// steps whose action was never dispatched. Only dispatched actions
-		// exercised the app, so only they belong in a per-action rate.
+		// Steps and actions differ by the steps that chose no action, the steps
+		// whose action was never dispatched, and the steps the spec's setup
+		// drove into position. Only what the action generator dispatched
+		// explored the app, so only that belongs in a per-action rate.
 		summary.TotalActions += item.Actions
 		summary.TotalRunHours += float64(item.MonotonicMillis) / float64(time.Hour/time.Millisecond)
 		if item.ClampedToBudget {

cmd/internal-tools/campaign/summary.go

@@ -27,7 +27,12 @@ type traceSummary struct {
 	// counting any of them as an action inflates the denominator of every
 	// per-action rate. The inflation is policy-dependent, so it does not cancel
 	// between arms.
-	Actions                    int      `json:"actions"`
+	Actions int `json:"actions"`
+	// UnattributedActions counts the dispatched steps whose action names no
+	// producer, which only a trace recorded before actions carried one can do.
+	// Such a run's Actions is the count it was already reported with rather than
+	// a setup-excluding one, and this is what says so.
+	UnattributedActions        int      `json:"unattributed_actions,omitempty"`
 	FirstViolationOriginStep   *int     `json:"first_violation_origin_step"`
 	FirstViolationDetectedStep *int     `json:"first_violation_detected_step"`
 	FirstViolationProperties   []string `json:"first_violation_properties,omitempty"`
@@ -65,21 +70,32 @@ type actionLine struct {
 // generator's behalf, which is the exposure a per-action rate divides by. A
 // spec's setup puts the app into its starting position before the generator is
 // consulted, so its login taps are dispatched actions that explored nothing.
+// Both arms are read the same way, off the source the action names, because a
+// denominator that excludes setup on one arm and includes it on the other makes
+// the two rates incomparable.
 //
-// Only a model run separates the two: the LLM backend stamps source="llm" on
-// what it chose and leaves setup's actions unstamped. The seeded picker resolves
-// setup precedence inside the one JS call it makes and stamps nothing at all, so
-// its setup steps are indistinguishable from its generator steps in the trace
-// and are counted; reading their absent source as setup would report every
-// seeded run as having explored nothing.
+// An action naming no source at all is one recorded before the distinction
+// existed and cannot be attributed now, so each arm keeps the count it was
+// already reported with: everything the seeded picker dispatched, and only what
+// the model stamped. summarizeTrace counts those steps separately so a
+// pre-source run cannot pass its denominator off as a setup-excluding one.
 func generatorDispatched(line traceLine, generator string) bool {
 	if line.ActionSkipped != "" || line.NextAction == nil {
 		return false
 	}
-	if generator != "llm" {
+	switch line.NextAction.Source {
+	case "":
+		return generator != trace.ActionSourceModel
+	case trace.ActionSourceSetup:
+		return false
+	default:
 		return true
 	}
-	return line.NextAction.Source == "llm"
+}
+
+// actionUnattributed reports a dispatched step whose action names no producer.
+func actionUnattributed(line traceLine) bool {
+	return line.ActionSkipped == "" && line.NextAction != nil && line.NextAction.Source == ""
 }
 
 // findRunDirectory returns the run directory `sanderling test` created inside
@@ -125,9 +141,10 @@ func summarizeRun(seedDirectory string) (string, traceSummary, error) {
 	return name, summary, nil
 }
 
-// runGenerator reads which picker drove the run. The trace on its own cannot
-// say whether an unstamped action came from the seeded picker or from the
-// spec's setup under the model picker, and the two count differently.
+// runGenerator reads which picker drove the run. A trace recorded before
+// actions named their source cannot say whether an unstamped action came from
+// the seeded picker or from the spec's setup under the model picker, and the
+// two count differently.
 func runGenerator(runDirectory string) (string, error) {
 	body, err := os.ReadFile(filepath.Join(runDirectory, "meta.json"))
 	if err != nil {
@@ -171,6 +188,9 @@ func summarizeTrace(tracePath, generator string) (traceSummary, error) {
 		if !synthetic && generatorDispatched(line, generator) {
 			summary.Actions++
 		}
+		if !synthetic && actionUnattributed(line) {
+			summary.UnattributedActions++
+		}
 		if line.PreconditionFailure != "" {
 			summary.PreconditionFailures++
 		}

cmd/internal-tools/campaign/summary_test.go

@@ -40,13 +40,20 @@ func setupStep(index int) trace.Step {
 		Kind:     "InputText",
 		Selector: "testTag:LoginScreen > testTag:LoginEmail",
 		Text:     "[email protected]",
+		Source:   trace.ActionSourceSetup,
 	}
 	return step
 }
 
+func seededStep(index int) trace.Step {
+	step := actingStep(index)
+	step.NextAction.Source = trace.ActionSourceSeeded
+	return step
+}
+
 func modelStep(index int) trace.Step {
 	step := actingStep(index)
-	step.NextAction.Source = "llm"
+	step.NextAction.Source = trace.ActionSourceModel
 	return step
 }
 
@@ -189,7 +196,7 @@ func TestSummarizeRun_ModelRunWithoutSetupCountsEveryDispatchedStep(t *testing.T
 	}
 }
 
-func TestSummarizeRun_SeededRunCountsSetupBecauseItsTraceCannotNameIt(t *testing.T) {
+func TestSummarizeRun_SeededRunLeavesTheSetupsLoginOutOfTheActionCount(t *testing.T) {
 	seedDirectory := t.TempDir()
 	writeRunDirectoryWithMeta(t, seedDirectory, "20260812-090000",
 		trace.Meta{Seed: 11, Platform: "web", Arm: "seeded-baseline", Generator: "seeded"},
@@ -197,17 +204,73 @@ func TestSummarizeRun_SeededRunCountsSetupBecauseItsTraceCannotNameIt(t *testing
 			setupStep(1),
 			setupStep(2),
 			setupStep(3),
-			actingStep(4),
-			actingStep(5),
+			seededStep(4),
+			seededStep(5),
 		})
 
 	_, summary, err := summarizeRun(seedDirectory)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if summary.Actions != 5 {
-		t.Errorf("actions: got %d, want 5: the seeded picker stamps no source, so excluding "+
-			"unstamped steps would count its whole run as setup", summary.Actions)
+	if summary.Actions != 2 {
+		t.Errorf("actions: got %d, want 2 (three login steps were setup's), which is the same "+
+			"rule the model arm is counted by", summary.Actions)
+	}
+	if summary.UnattributedActions != 0 {
+		t.Errorf("unattributed actions: got %d, want 0: every step named its source",
+			summary.UnattributedActions)
+	}
+}
+
+func TestSummarizeRun_SeededRunWithoutSetupCountsEveryDispatchedStep(t *testing.T) {
+	seedDirectory := t.TempDir()
+	writeRunDirectoryWithMeta(t, seedDirectory, "20260812-090000",
+		trace.Meta{Seed: 11, Platform: "web", Arm: "seeded-baseline", Generator: "seeded"},
+		[]trace.Step{seededStep(1), seededStep(2), seededStep(3), seededStep(4)})
+
+	_, summary, err := summarizeRun(seedDirectory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if summary.Actions != 4 {
+		t.Errorf("actions: got %d, want 4", summary.Actions)
+	}
+}
+
+// Traces recorded before actions named their source cannot be re-attributed
+// after the fact, so each arm keeps the count it was already reported with: the
+// seeded arm counts every dispatched step, the model arm counts only what the
+// model stamped. UnattributedActions is how such a run says so rather than
+// passing its old denominator off as a setup-excluding one.
+func TestSummarizeRun_TraceWithoutSourcesKeepsTheCountItWasReportedWith(t *testing.T) {
+	for _, testCase := range []struct {
+		generator           string
+		actions             int
+		unattributedActions int
+	}{
+		{generator: "seeded", actions: 5, unattributedActions: 5},
+		{generator: "llm", actions: 0, unattributedActions: 5},
+	} {
+		t.Run(testCase.generator, func(t *testing.T) {
+			seedDirectory := t.TempDir()
+			writeRunDirectoryWithMeta(t, seedDirectory, "20260812-090000",
+				trace.Meta{Seed: 11, Platform: "web", Generator: testCase.generator},
+				[]trace.Step{
+					actingStep(1), actingStep(2), actingStep(3), actingStep(4), actingStep(5),
+				})
+
+			_, summary, err := summarizeRun(seedDirectory)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if summary.Actions != testCase.actions {
+				t.Errorf("actions: got %d, want %d", summary.Actions, testCase.actions)
+			}
+			if summary.UnattributedActions != testCase.unattributedActions {
+				t.Errorf("unattributed actions: got %d, want %d",
+					summary.UnattributedActions, testCase.unattributedActions)
+			}
+		})
 	}
 }