fix(trace): an action names the generator that produced it

The setup exclusion landed for the model arm only, because only a model
pick stamped a source. A seeded run returned setup's action through the
same entry with no marker, so its denominator still counted the login
while the model arm's did not, and the two are compared.

serializeAction names setup and seeded on the wire, so both arms are
counted by one rule. An already-recorded trace names nothing and keeps
exactly the count it was reported with; unattributed_actions counts those
steps so the old denominator cannot pass as the new one. TraceVersion is
deliberately unbumped: oracle-reduction refuses a differing version, and
a bump would make all 169 recorded runs unreplayable.

internal/runner/llm_source.go

@@ -65,14 +65,11 @@ type llmSource struct {
 	// can stamp the trace. lastSource is "llm" only when the LLM (not setup)
 	// chose the action; lastReasoning is the model's rationale. lastChoice is the
 	// 1-based number it picked and lastChosenAction the description it echoed, so
-	// the trace shows what the model believed it was doing. lastFromSetup says
-	// the spec's setup produced the action, which is the app being put in
-	// position rather than the generator exploring it.
+	// the trace shows what the model believed it was doing.
 	lastSource       string
 	lastReasoning    string
 	lastChoice       int
 	lastChosenAction string
-	lastFromSetup    bool
 }
 
 // llmSelection is the outcome of one LLM selection call.
@@ -99,14 +96,12 @@ func (s *llmSource) NextAction(ctx context.Context, stepIndex int) (verifier.Act
 	s.lastReasoning = ""
 	s.lastChoice = 0
 	s.lastChosenAction = ""
-	s.lastFromSetup = false
 	s.history.completeLast(s.verifier.CurrentScreen())
 
 	// Setup precedence only: the LLM replaces the seeded action root, so we run
 	// setup (e.g. login) first but never the weighted picker.
 	action, err := s.verifier.SetupAction()
 	if err == nil {
-		s.lastFromSetup = true
 		s.record(stepIndex, trace.LLMCall{Outcome: trace.LLMOutcomeSetupAction})
 		s.history.add(describeAction(action, s.verifier.Tree()))
 		return action, nil
@@ -501,9 +496,10 @@ func (h *actionHistory) recent() []historyEntry {
 	return h.entries
 }
 
-// stampActionSource records the backend that chose an action on the trace.
-// Only an LLM-selected action (not a setup action the JS path produced) carries
-// source="llm" and the model's reasoning.
+// stampActionSource names the model as the producer of an action it chose. The
+// spec's two generators name themselves on the wire (runtime-entry.ts) and
+// traceActionFor has already carried that over; a model pick is built here from
+// the candidate list, so nothing on the wire could have named it.
 func stampActionSource(traceAction *trace.Action, source ActionSource) {
 	if traceAction == nil {
 		return
@@ -518,14 +514,13 @@ func stampActionSource(traceAction *trace.Action, source ActionSource) {
 	traceAction.LLMChosenAction = llm.lastChosenAction
 }
 
-// generatorChoseAction reports whether the action the source just returned came
-// from the generator rather than from the spec's setup driving the app into
-// position. Only the model source can tell them apart: the seeded picker
-// resolves setup precedence inside the one JS call it makes, so everything it
-// returns counts as the generator's.
-func generatorChoseAction(source ActionSource) bool {
-	llm, ok := source.(*llmSource)
-	return !ok || !llm.lastFromSetup
+// generatorChoseAction reports whether this step drove the app on the action
+// generator's behalf rather than on setup's, which is the exposure a per-action
+// rate divides by. Both policies are read the same way, off the stamped trace
+// action: an unstamped one is a source the runner cannot name, and counting it
+// as setup would report a run that explored as having explored nothing.
+func generatorChoseAction(traceAction *trace.Action) bool {
+	return traceAction != nil && traceAction.Source != trace.ActionSourceSetup
 }
 
 // screenshotDataURL downscales the PNG and encodes it as a data URL for the

internal/runner/llm_source_test.go

@@ -968,13 +968,13 @@ func TestRunner_SetupAndGeneratorBothDrivingIsAHealthyRun(t *testing.T) {
 		}
 	}
 	for _, line := range lines[:2] {
-		if line.NextAction.Source != "" {
-			t.Errorf("step %d action source = %q, want none: setup chose it",
+		if line.NextAction.Source != trace.ActionSourceSetup {
+			t.Errorf("step %d action source = %q, want setup: setup chose it",
 				line.Step, line.NextAction.Source)
 		}
 	}
 	for _, line := range lines[2:] {
-		if line.NextAction.Source != "llm" {
+		if line.NextAction.Source != trace.ActionSourceModel {
 			t.Errorf("step %d action source = %q, want llm", line.Step, line.NextAction.Source)
 		}
 	}

internal/runner/runner.go

@@ -82,9 +82,8 @@ type Summary struct {
 	// GeneratorActions counts the dispatched actions the generator chose. The
 	// spec's setup drives the app into its starting position before the
 	// generator is consulted, so a run at zero here explored nothing however
-	// many actions its login fired. Only the model generator separates the two:
-	// the seeded picker resolves setup precedence inside the one JS call it
-	// makes, so everything it returns counts as the generator's.
+	// many actions its login fired. Both generators separate the two the same
+	// way, by the producer each action names on the trace.
 	GeneratorActions int
 }
 
@@ -506,7 +505,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
 		}
 		if nextErr == nil && !applySkipped {
 			summary.DispatchedActions++
-			if generatorChoseAction(actionSource) {
+			if generatorChoseAction(traceAction) {
 				summary.GeneratorActions++
 			}
 		}
@@ -1580,7 +1579,12 @@ func driverIsAndroid(ctx context.Context, options Options, logger *slog.Logger)
 }
 
 func traceActionFor(action verifier.Action, tree *hierarchy.Tree) *trace.Action {
-	traceAction := &trace.Action{Kind: string(action.Kind), X: action.X, Y: action.Y}
+	traceAction := &trace.Action{
+		Kind:   string(action.Kind),
+		X:      action.X,
+		Y:      action.Y,
+		Source: action.Source,
+	}
 	switch action.Kind {
 	case verifier.ActionKindTap, verifier.ActionKindDoubleTap, verifier.ActionKindLongPress:
 		traceAction.Selector = action.On

internal/runner/runner_test.go

@@ -223,6 +223,62 @@ func TestRunner_SeededRunRecordsNoModelCalls(t *testing.T) {
 	}
 }
 
+// seededLoginSetupSpec drives the first two steps from setup, the way a
+// login-fronted spec does, and leaves the rest to the seeded action root.
+const seededLoginSetupSpec = `
+import { always, actions, taps, typing, weighted, Tap } from "@sanderling/spec";
+globalThis.properties = { ok: always(() => true) };
+let setupTapsLeft = 2;
+globalThis.setup = actions(() => (setupTapsLeft-- > 0 ? [Tap({ on: "id:Submit" })] : []));
+globalThis.actions = weighted([1, taps], [1, typing]);
+`
+
+// TestRunner_SeededSetupActionsAreNotTheGeneratorDrivingTheApp: a seeded run's
+// login steps used to be indistinguishable from its exploration, so the arm
+// divided its defect rate by every action it dispatched while the model arm
+// divided by the ones its policy chose. The two rates were then compared.
+func TestRunner_SeededSetupActionsAreNotTheGeneratorDrivingTheApp(t *testing.T) {
+	state := newHarnessWithSpec(t, seededLoginSetupSpec)
+	state.mock.HierarchyJSON = llmTreeJSON
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	summary, err := Run(ctx, Options{
+		Duration:    30 * time.Second,
+		IdleTimeout: 20 * time.Millisecond,
+		MaxSteps:    4,
+		Driver:      state.mock,
+		Verifier:    state.verifier,
+		TraceWriter: state.writer,
+		Logger:      slog.New(slog.NewTextHandler(io.Discard, nil)),
+	})
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if summary.DispatchedActions != 4 {
+		t.Errorf("DispatchedActions = %d, want 4: every step drove the app",
+			summary.DispatchedActions)
+	}
+	if summary.GeneratorActions != 2 {
+		t.Errorf("GeneratorActions = %d, want 2: the picker drove the two steps setup left it",
+			summary.GeneratorActions)
+	}
+	lines := readTraceLines(t, state.writer.Directory())
+	if len(lines) != 4 {
+		t.Fatalf("wrote %d trace lines, want 4", len(lines))
+	}
+	for _, line := range lines[:2] {
+		if line.NextAction == nil || line.NextAction.Source != trace.ActionSourceSetup {
+			t.Errorf("step %d action = %+v, want one named setup", line.Step, line.NextAction)
+		}
+	}
+	for _, line := range lines[2:] {
+		if line.NextAction == nil || line.NextAction.Source != trace.ActionSourceSeeded {
+			t.Errorf("step %d action = %+v, want one named seeded", line.Step, line.NextAction)
+		}
+	}
+}
+
 func TestRunner_MaxStepsStopsAfterExactlyNSteps(t *testing.T) {
 	state := newHarness(t)
 

internal/runner/testdata/trace.jsonl

@@ -1,4 +1,4 @@
-{"extractor_changes":{"extractor_0":{"prev":null,"curr":0}},"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":1,"timestamp":"0001-01-01T00:00:00Z","trace_version":1}
-{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":2,"timestamp":"0001-01-01T00:00:00Z","trace_version":1}
-{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":3,"timestamp":"0001-01-01T00:00:00Z","trace_version":1}
-{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120}},"residuals":{"balanceNonNegative":{"op":"true"}},"step":4,"timestamp":"0001-01-01T00:00:00Z","trace_version":1}
+{"extractor_changes":{"extractor_0":{"prev":null,"curr":0}},"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120},"source":"seeded"},"residuals":{"balanceNonNegative":{"op":"true"}},"step":1,"timestamp":"0001-01-01T00:00:00Z","trace_version":1}
+{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120},"source":"seeded"},"residuals":{"balanceNonNegative":{"op":"true"}},"step":2,"timestamp":"0001-01-01T00:00:00Z","trace_version":1}
+{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120},"source":"seeded"},"residuals":{"balanceNonNegative":{"op":"true"}},"step":3,"timestamp":"0001-01-01T00:00:00Z","trace_version":1}
+{"hierarchy":{"elements":[{"resourceId":"HomeScreen","bounds":{"left":0,"top":0,"right":0,"bottom":0},"attrs":{"editable":"false","resource-id":"HomeScreen"}},{"resourceId":"next","clickable":true,"enabled":true,"bounds":{"left":40,"top":80,"right":240,"bottom":160},"attrs":{"bounds":"[40,80,240,160]","clickable":"true","editable":"false","enabled":"true","resource-id":"next"}}],"depths":[0,1]},"next_action":{"kind":"Tap","selector":"id:next","resolved_bounds":{"x":40,"y":80,"width":200,"height":80},"tap_point":{"x":140,"y":120},"source":"seeded"},"residuals":{"balanceNonNegative":{"op":"true"}},"step":4,"timestamp":"0001-01-01T00:00:00Z","trace_version":1}

internal/trace/writer.go

@@ -109,6 +109,16 @@ type Metrics struct {
 	TotalMemoryBytes int64   `json:"total_memory_bytes,omitempty"`
 }
 
+// The three producers an action can come from. The spec's setup drives the app
+// into position and explores nothing, so a per-action rate divides by the other
+// two; an action recorded before this distinction existed carries none of them
+// and cannot be attributed after the fact.
+const (
+	ActionSourceSetup  = "setup"
+	ActionSourceSeeded = "seeded"
+	ActionSourceModel  = "llm"
+)
+
 type Action struct {
 	Kind           string        `json:"kind"`
 	X              int           `json:"x,omitempty"`
@@ -123,8 +133,8 @@ type Action struct {
 	Selector       string        `json:"selector,omitempty"`
 	ResolvedBounds *BoundsRecord `json:"resolved_bounds,omitempty"`
 	TapPoint       *PointRecord  `json:"tap_point,omitempty"`
-	// Source names the backend that chose this action: "llm" when the LLM
-	// action backend selected it, empty for the seeded picker. LLMReasoning is
+	// Source names which of the three producers chose this action, and is empty
+	// only on a trace recorded before actions named themselves. LLMReasoning is
 	// the model's short rationale, shown by the replay UI to explain the pick.
 	Source       string `json:"source,omitempty"`
 	LLMReasoning string `json:"llm_reasoning,omitempty"`

internal/verifier/marshal.go

@@ -611,6 +611,10 @@ type wireAction struct {
 	ToX            int    `json:"toX"`
 	ToY            int    `json:"toY"`
 	DurationMillis int    `json:"durationMillis"`
+	// Source names the generator that produced the action: the spec's setup or
+	// the action root. Empty from the candidate enumeration, which serializes
+	// actions nothing has chosen yet.
+	Source string `json:"source"`
 }
 
 // DecodeAction turns one serialized action (the flat camelCase wire contract)
@@ -623,6 +627,15 @@ func DecodeAction(raw json.RawMessage) (Action, error) {
 	if err := json.Unmarshal(raw, &wire); err != nil {
 		return Action{}, fmt.Errorf("decode action: %w", err)
 	}
+	action, err := actionFromWire(wire)
+	if err != nil {
+		return Action{}, err
+	}
+	action.Source = wire.Source
+	return action, nil
+}
+
+func actionFromWire(wire wireAction) (Action, error) {
 	switch wire.Kind {
 	case "Tap":
 		return Action{Kind: ActionKindTap, On: wire.Selector, X: wire.X, Y: wire.Y}, nil

internal/verifier/policy_parity_test.go

@@ -184,12 +184,15 @@ func modelOfferedActions(t *testing.T, verb string) map[string]Action {
 
 // actionIdentity keys an action by everything except the values the policy owns
 // rather than the candidate set: the typed text, which the seeded arm draws from
-// the edge-case corpus and the model writes itself, and a swipe's drag distance,
-// which the seeded arm draws and the enumeration lists at a nominal length.
+// the edge-case corpus and the model writes itself, a swipe's drag distance,
+// which the seeded arm draws and the enumeration lists at a nominal length, and
+// the source, which names who produced an action rather than what it does (the
+// enumeration names nobody, because the model has not chosen yet).
 // Comparing those would compare policies instead of action spaces. A swipe's
 // direction is NOT policy-owned, so it survives as the sign of the drag.
 func actionIdentity(action Action) string {
 	action.Text = ""
+	action.Source = ""
 	if action.Kind == ActionKindSwipe {
 		action.ToX = sign(action.ToX - action.FromX)
 		action.ToY = sign(action.ToY - action.FromY)

internal/verifier/setup_action_test.go

@@ -96,6 +96,58 @@ export const actionsRoot = taps;
 	}
 }
 
+// TestNextActionNamesTheGeneratorThatProducedIt is the native half of the
+// cross-host marker contract: for the same spec shape, a setup-driven step and
+// an action-root step must name different producers. The web half asserts the
+// same two names in pkg/spec/test/web-runtime.test.ts, so a match on both sides
+// proves the engines agree without either invoking the other. A per-action rate
+// divides by the root's steps only, and an unnamed producer inflates it by
+// however many steps the login consumed.
+func TestNextActionNamesTheGeneratorThatProducedIt(t *testing.T) {
+	spec := `
+import { Tap, actions, extract, taps } from "@sanderling/spec";
+const signIn = extract("signIn", state => state.ax.find("id:SignIn"));
+export const setup = actions(() => (signIn.current ? [Tap({ on: "id:SignIn" })] : []));
+export const actionsRoot = taps;
+`
+	v := loadBundled(t, spec, enumTreeJSON)
+	action, err := v.NextAction()
+	if err != nil {
+		t.Fatalf("NextAction: %v", err)
+	}
+	if action.On != "id:SignIn" || action.Source != "setup" {
+		t.Errorf("NextAction = %+v, want the Tap on id:SignIn named setup", action)
+	}
+
+	pushTree(t, v, policyTreeJSON)
+	action, err = v.NextAction()
+	if err != nil {
+		t.Fatalf("NextAction after setup went quiet: %v", err)
+	}
+	if action.Source != "seeded" {
+		t.Errorf("NextAction = %+v, want the action root's tap named seeded", action)
+	}
+}
+
+// TestSetupActionNamesSetup: the model policy reaches setup through its own
+// entry, which must name the producer the same way the seeded entry does, or
+// the two arms' per-action rates divide by different things.
+func TestSetupActionNamesSetup(t *testing.T) {
+	spec := `
+import { Tap, actions, taps } from "@sanderling/spec";
+export const setup = actions(() => [Tap({ on: "id:SignIn" })]);
+export const actionsRoot = taps;
+`
+	v := loadBundled(t, spec, enumTreeJSON)
+	action, err := v.SetupAction()
+	if err != nil {
+		t.Fatalf("SetupAction: %v", err)
+	}
+	if action.Source != "setup" {
+		t.Errorf("SetupAction = %+v, want it named setup", action)
+	}
+}
+
 // TestSetupGeneratorDrawsUnderTheModelPolicy: setup walks through the picker
 // with its rng under both policies, so a generator there is not the divergence
 // the enumeration refuses and must keep drawing. Enumerating before every setup

internal/verifier/types.go

@@ -38,6 +38,10 @@ type Action struct {
 	// when the apply call failed and nothing can say whether the action
 	// reached the app. The spec is told which of the two it is.
 	Applied bool
+	// Source names the generator that produced this action, "setup" or
+	// "seeded", as the runtime entry tagged it. Empty on an action the runner
+	// built itself (the model policy's pick), which the runner names instead.
+	Source string
 	// Relaunched, like Applied, is meaningful only on state.lastAction: the
 	// runner brought the app back to the foreground after this action, so the
 	// two readings the spec compares straddle a restart. The action still