fix(campaign): the action count leaves the setup's login out on a model run

Defects per thousand actions divided by every dispatched step, so a
spec whose setup logs in inflated the denominator by however many steps
that took. It is the same error the run gate had, and it does not cancel
between arms.

A model run is separable because only an llm-selected action stamps
next_action.source. A seeded run is not: its setup returns through the
same entry with no marker, and 11261 dispatched steps across the 169
recorded runs carry no source at all, so excluding on it blind would
report every seeded run as having explored nothing. The seeded arm
counts as before and a test pins that.
This commit is contained in:
pj committed 2026-08-18 16:52:22 +05:30
1 parent b278fc869d
commit 5f1f50c2fb
4 files changed
+164 -20

No files matched your search

+101 -1
View File
@@ -34,13 +34,51 @@ func skippedActionStep(index int, reason string) trace.Step {
return step
}
func setupStep(index int) trace.Step {
step := observedStep(index)
step.NextAction = &trace.Action{
Kind: "InputText",
Selector: "testTag:LoginScreen > testTag:LoginEmail",
Text: "[email protected]",
}
return step
}
func modelStep(index int) trace.Step {
step := actingStep(index)
step.NextAction.Source = "llm"
return step
}
func skippedModelStep(index int, reason string) trace.Step {
step := modelStep(index)
step.ActionSkipped = reason
return step
}
func writeRunDirectory(t *testing.T, seedDirectory, name string, steps []trace.Step) string {
t.Helper()
return writeRunDirectoryWithMeta(
t,
seedDirectory,
name,
trace.Meta{Seed: 11, Platform: "web", Arm: "seeded-baseline"},
steps,
)
}
func writeRunDirectoryWithMeta(
t *testing.T,
seedDirectory, name string,
declared trace.Meta,
steps []trace.Step,
) string {
t.Helper()
directory := filepath.Join(seedDirectory, name)
if err := os.MkdirAll(directory, 0o755); err != nil {
t.Fatal(err)
}
meta, err := json.Marshal(trace.Meta{Seed: 11, Platform: "web", Arm: "seeded-baseline"})
meta, err := json.Marshal(declared)
if err != nil {
t.Fatal(err)
}
@@ -111,6 +149,68 @@ func TestSummarizeRun_CountsOnlyStepsThatDispatchedAnAction(t *testing.T) {
}
}
func TestSummarizeRun_ModelRunLeavesTheSetupsLoginOutOfTheActionCount(t *testing.T) {
seedDirectory := t.TempDir()
writeRunDirectoryWithMeta(t, seedDirectory, "20260812-090000",
trace.Meta{Seed: 11, Platform: "web", Arm: "llm-identifier", Generator: "llm"},
[]trace.Step{
setupStep(1),
setupStep(2),
setupStep(3),
modelStep(4),
skippedModelStep(5, "unresolved_selector"),
modelStep(6),
})
_, summary, err := summarizeRun(seedDirectory)
if err != nil {
t.Fatal(err)
}
if summary.Steps != 6 {
t.Errorf("steps: got %d, want 6", summary.Steps)
}
if summary.Actions != 2 {
t.Errorf("actions: got %d, want 2 (three login steps were setup's and one generator action was thrown away)", summary.Actions)
}
}
func TestSummarizeRun_ModelRunWithoutSetupCountsEveryDispatchedStep(t *testing.T) {
seedDirectory := t.TempDir()
writeRunDirectoryWithMeta(t, seedDirectory, "20260812-090000",
trace.Meta{Seed: 11, Platform: "web", Arm: "llm-identifier", Generator: "llm"},
[]trace.Step{modelStep(1), modelStep(2), modelStep(3), modelStep(4)})
_, summary, err := summarizeRun(seedDirectory)
if err != nil {
t.Fatal(err)
}
if summary.Actions != 4 {
t.Errorf("actions: got %d, want 4", summary.Actions)
}
}
func TestSummarizeRun_SeededRunCountsSetupBecauseItsTraceCannotNameIt(t *testing.T) {
seedDirectory := t.TempDir()
writeRunDirectoryWithMeta(t, seedDirectory, "20260812-090000",
trace.Meta{Seed: 11, Platform: "web", Arm: "seeded-baseline", Generator: "seeded"},
[]trace.Step{
setupStep(1),
setupStep(2),
setupStep(3),
actingStep(4),
actingStep(5),
})
_, summary, err := summarizeRun(seedDirectory)
if err != nil {
t.Fatal(err)
}
if summary.Actions != 5 {
t.Errorf("actions: got %d, want 5: the seeded picker stamps no source, so excluding "+
"unstamped steps would count its whole run as setup", summary.Actions)
}
}
func TestSummarizeRun_SkipReasonsEachSuppressTheAction(t *testing.T) {
for _, reason := range []string{
"no_target", "unresolved_selector", "missing_key",