mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 03:57:09 +00:00
fix(analyze): divide by actions that ran
Defects per thousand actions counted every step, including steps that chose nothing and steps whose action was never dispatched. The inflation is policy-dependent, so it does not cancel between arms: on the fixture campaign the model arm's yield was reported at 60.3 per thousand against a true 120.7, because half its steps did nothing. A runs.jsonl without the count is refused by name and line rather than read as zero actions, which would report every per-action rate wrongly. The report also carries steps beside actions now, so the gap is visible rather than folded into a denominator. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
f6d562e3dc
commit
2cbe03c3fb
6 files changed
+162
-36
No files matched your search
@@ -10,6 +10,7 @@ func violatingRun(seed int64, steps, origin int, properties ...string) classifie
|
||||
return classifiedRun{
|
||||
Seed: seed,
|
||||
Steps: steps,
|
||||
Actions: steps,
|
||||
DurationMillis: 60_000,
|
||||
OriginStep: origin,
|
||||
Violated: true,
|
||||
@@ -18,7 +19,7 @@ func violatingRun(seed int64, steps, origin int, properties ...string) classifie
|
||||
}
|
||||
|
||||
func cleanRun(seed int64, steps int) classifiedRun {
|
||||
return classifiedRun{Seed: seed, Steps: steps, DurationMillis: 60_000}
|
||||
return classifiedRun{Seed: seed, Steps: steps, Actions: steps, DurationMillis: 60_000}
|
||||
}
|
||||
|
||||
func TestSummarize_ArmWhereNoRunViolated(t *testing.T) {
|
||||
@@ -76,6 +77,54 @@ func TestSummarize_ArmWhereEveryRunViolated(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A step that chose no action, and a step whose action was never dispatched,
|
||||
// left the app untouched. Counting them would inflate the denominator of every
|
||||
// per-action rate, and the inflation differs by arm so it does not cancel.
|
||||
func TestSummarize_CountsDispatchedActionsNotSteps(t *testing.T) {
|
||||
summary := summarize(arm{
|
||||
Name: "declines",
|
||||
Budget: 40,
|
||||
Runs: []classifiedRun{
|
||||
{Seed: 1, Steps: 40, Actions: 10, DurationMillis: 3_600_000,
|
||||
Violated: true, OriginStep: 12, ViolatedProperties: []string{"cartTotal"}},
|
||||
{Seed: 2, Steps: 40, Actions: 6, DurationMillis: 3_600_000},
|
||||
},
|
||||
})
|
||||
if summary.TotalSteps != 80 {
|
||||
t.Errorf("total steps %d, want 80", summary.TotalSteps)
|
||||
}
|
||||
if summary.TotalActions != 16 {
|
||||
t.Errorf("total actions %d, want 16 dispatched of 80 steps", summary.TotalActions)
|
||||
}
|
||||
if summary.DefectsPerThousandActions == nil {
|
||||
t.Fatal("no defects per thousand actions")
|
||||
}
|
||||
expected := 1000.0 / 16.0
|
||||
if math.Abs(*summary.DefectsPerThousandActions-expected) > 1e-9 {
|
||||
t.Errorf("defects per thousand actions %v, want %v", *summary.DefectsPerThousandActions, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarize_ArmThatDispatchedNothingHasNoPerActionRate(t *testing.T) {
|
||||
summary := summarize(arm{
|
||||
Name: "inert",
|
||||
Budget: 20,
|
||||
Runs: []classifiedRun{
|
||||
{Seed: 1, Steps: 20, Actions: 0, DurationMillis: 3_600_000,
|
||||
Violated: true, OriginStep: 3, ViolatedProperties: []string{"cartTotal"}},
|
||||
},
|
||||
})
|
||||
if summary.TotalActions != 0 || summary.TotalSteps != 20 {
|
||||
t.Errorf("steps %d actions %d, want 20 and 0", summary.TotalSteps, summary.TotalActions)
|
||||
}
|
||||
if summary.DefectsPerThousandActions != nil {
|
||||
t.Errorf("defects per thousand actions %v, want none with nothing dispatched", *summary.DefectsPerThousandActions)
|
||||
}
|
||||
if summary.DefectsPerHour == nil || *summary.DefectsPerHour != 1 {
|
||||
t.Errorf("defects per hour %v, want 1: the run still consumed an hour", summary.DefectsPerHour)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarize_ArmWithNoUsableRunsAfterExclusions(t *testing.T) {
|
||||
summary := summarize(arm{
|
||||
Name: "broken",
|
||||
|
||||
Reference in new issue
Block a user