mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 03:57:09 +00:00
fix(campaign): the action count leaves the setup's login out on a model run
Defects per thousand actions divided by every dispatched step, so a spec whose setup logs in inflated the denominator by however many steps that took. It is the same error the run gate had, and it does not cancel between arms. A model run is separable because only an llm-selected action stamps next_action.source. A seeded run is not: its setup returns through the same entry with no marker, and 11261 dispatched steps across the 169 recorded runs carry no source at all, so excluding on it blind would report every seeded run as having explored nothing. The seeded arm counts as before and a test pins that.
This commit is contained in:
1 parent
b278fc869d
commit
5f1f50c2fb
4 files changed
+164
-20
No files matched your search
@@ -34,13 +34,51 @@ func skippedActionStep(index int, reason string) trace.Step {
|
||||
return step
|
||||
}
|
||||
|
||||
func setupStep(index int) trace.Step {
|
||||
step := observedStep(index)
|
||||
step.NextAction = &trace.Action{
|
||||
Kind: "InputText",
|
||||
Selector: "testTag:LoginScreen > testTag:LoginEmail",
|
||||
Text: "[email protected]",
|
||||
}
|
||||
return step
|
||||
}
|
||||
|
||||
func modelStep(index int) trace.Step {
|
||||
step := actingStep(index)
|
||||
step.NextAction.Source = "llm"
|
||||
return step
|
||||
}
|
||||
|
||||
func skippedModelStep(index int, reason string) trace.Step {
|
||||
step := modelStep(index)
|
||||
step.ActionSkipped = reason
|
||||
return step
|
||||
}
|
||||
|
||||
func writeRunDirectory(t *testing.T, seedDirectory, name string, steps []trace.Step) string {
|
||||
t.Helper()
|
||||
return writeRunDirectoryWithMeta(
|
||||
t,
|
||||
seedDirectory,
|
||||
name,
|
||||
trace.Meta{Seed: 11, Platform: "web", Arm: "seeded-baseline"},
|
||||
steps,
|
||||
)
|
||||
}
|
||||
|
||||
func writeRunDirectoryWithMeta(
|
||||
t *testing.T,
|
||||
seedDirectory, name string,
|
||||
declared trace.Meta,
|
||||
steps []trace.Step,
|
||||
) string {
|
||||
t.Helper()
|
||||
directory := filepath.Join(seedDirectory, name)
|
||||
if err := os.MkdirAll(directory, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta, err := json.Marshal(trace.Meta{Seed: 11, Platform: "web", Arm: "seeded-baseline"})
|
||||
meta, err := json.Marshal(declared)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -111,6 +149,68 @@ func TestSummarizeRun_CountsOnlyStepsThatDispatchedAnAction(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarizeRun_ModelRunLeavesTheSetupsLoginOutOfTheActionCount(t *testing.T) {
|
||||
seedDirectory := t.TempDir()
|
||||
writeRunDirectoryWithMeta(t, seedDirectory, "20260812-090000",
|
||||
trace.Meta{Seed: 11, Platform: "web", Arm: "llm-identifier", Generator: "llm"},
|
||||
[]trace.Step{
|
||||
setupStep(1),
|
||||
setupStep(2),
|
||||
setupStep(3),
|
||||
modelStep(4),
|
||||
skippedModelStep(5, "unresolved_selector"),
|
||||
modelStep(6),
|
||||
})
|
||||
|
||||
_, summary, err := summarizeRun(seedDirectory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if summary.Steps != 6 {
|
||||
t.Errorf("steps: got %d, want 6", summary.Steps)
|
||||
}
|
||||
if summary.Actions != 2 {
|
||||
t.Errorf("actions: got %d, want 2 (three login steps were setup's and one generator action was thrown away)", summary.Actions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarizeRun_ModelRunWithoutSetupCountsEveryDispatchedStep(t *testing.T) {
|
||||
seedDirectory := t.TempDir()
|
||||
writeRunDirectoryWithMeta(t, seedDirectory, "20260812-090000",
|
||||
trace.Meta{Seed: 11, Platform: "web", Arm: "llm-identifier", Generator: "llm"},
|
||||
[]trace.Step{modelStep(1), modelStep(2), modelStep(3), modelStep(4)})
|
||||
|
||||
_, summary, err := summarizeRun(seedDirectory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if summary.Actions != 4 {
|
||||
t.Errorf("actions: got %d, want 4", summary.Actions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarizeRun_SeededRunCountsSetupBecauseItsTraceCannotNameIt(t *testing.T) {
|
||||
seedDirectory := t.TempDir()
|
||||
writeRunDirectoryWithMeta(t, seedDirectory, "20260812-090000",
|
||||
trace.Meta{Seed: 11, Platform: "web", Arm: "seeded-baseline", Generator: "seeded"},
|
||||
[]trace.Step{
|
||||
setupStep(1),
|
||||
setupStep(2),
|
||||
setupStep(3),
|
||||
actingStep(4),
|
||||
actingStep(5),
|
||||
})
|
||||
|
||||
_, summary, err := summarizeRun(seedDirectory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if summary.Actions != 5 {
|
||||
t.Errorf("actions: got %d, want 5: the seeded picker stamps no source, so excluding "+
|
||||
"unstamped steps would count its whole run as setup", summary.Actions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarizeRun_SkipReasonsEachSuppressTheAction(t *testing.T) {
|
||||
for _, reason := range []string{
|
||||
"no_target", "unresolved_selector", "missing_key",
|
||||
|
||||
Reference in new issue
Block a user