mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 12:07:09 +00:00
fix(analyze): divide by actions that ran
Defects per thousand actions counted every step, including steps that chose nothing and steps whose action was never dispatched. The inflation is policy-dependent, so it does not cancel between arms: on the fixture campaign the model arm's yield was reported at 60.3 per thousand against a true 120.7, because half its steps did nothing. A runs.jsonl without the count is refused by name and line rather than read as zero actions, which would report every per-action rate wrongly. The report also carries steps beside actions now, so the gap is visible rather than folded into a denominator. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
f6d562e3dc
commit
2cbe03c3fb
6 files changed
+162
-36
No files matched your search
@@ -94,18 +94,26 @@ func writeCampaign(t *testing.T, directory string, declared map[string]any, reco
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassify_CarriesTheDispatchedActionCount(t *testing.T) {
|
||||
actions := 7
|
||||
item := classify(runRecord{Seed: 4, Steps: 30, Actions: &actions}, 50)
|
||||
if item.Steps != 30 || item.Actions != 7 {
|
||||
t.Errorf("run %+v, want 30 steps and 7 actions", item)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGroupArms_PoolsDirectoriesSharingAnArmAndReportsMissingSeeds(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeCampaign(t, filepath.Join(root, "north"), map[string]any{
|
||||
"arm": "seeded", "max_steps": 40, "seeds": []int{1, 2, 3},
|
||||
}, []map[string]any{
|
||||
{"seed": 1, "exit_code": 0, "steps": 40},
|
||||
{"seed": 2, "exit_code": 0, "steps": 9, "first_violation_origin_step": 9},
|
||||
{"seed": 1, "exit_code": 0, "steps": 40, "actions": 33},
|
||||
{"seed": 2, "exit_code": 0, "steps": 9, "actions": 8, "first_violation_origin_step": 9},
|
||||
})
|
||||
writeCampaign(t, filepath.Join(root, "south"), map[string]any{
|
||||
"arm": "seeded", "max_steps": 40, "seeds": []int{4},
|
||||
}, []map[string]any{
|
||||
{"seed": 4, "exit_code": 0, "steps": 40},
|
||||
{"seed": 4, "exit_code": 0, "steps": 40, "actions": 40},
|
||||
})
|
||||
|
||||
arms, err := groupArms([]string{filepath.Join(root, "north"), filepath.Join(root, "south")})
|
||||
@@ -129,9 +137,9 @@ func TestGroupArms_PoolsDirectoriesSharingAnArmAndReportsMissingSeeds(t *testing
|
||||
func TestGroupArms_RejectsDisagreeingStepBudgets(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeCampaign(t, filepath.Join(root, "a"), map[string]any{"arm": "seeded", "max_steps": 40, "seeds": []int{1}},
|
||||
[]map[string]any{{"seed": 1, "exit_code": 0, "steps": 40}})
|
||||
[]map[string]any{{"seed": 1, "exit_code": 0, "steps": 40, "actions": 40}})
|
||||
writeCampaign(t, filepath.Join(root, "b"), map[string]any{"arm": "seeded", "max_steps": 80, "seeds": []int{2}},
|
||||
[]map[string]any{{"seed": 2, "exit_code": 0, "steps": 80}})
|
||||
[]map[string]any{{"seed": 2, "exit_code": 0, "steps": 80, "actions": 80}})
|
||||
|
||||
_, err := groupArms([]string{filepath.Join(root, "a"), filepath.Join(root, "b")})
|
||||
if err == nil || !strings.Contains(err.Error(), "different budgets") {
|
||||
@@ -153,7 +161,8 @@ func TestGroupArms_ReportsBadRecordLines(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
directory := filepath.Join(root, "a")
|
||||
writeCampaign(t, directory, map[string]any{"arm": "seeded", "max_steps": 40, "seeds": []int{1}}, nil)
|
||||
if err := os.WriteFile(filepath.Join(directory, recordsFileName), []byte("{\"seed\":1}\nnot json\n"), 0o644); err != nil {
|
||||
if err := os.WriteFile(filepath.Join(directory, recordsFileName),
|
||||
[]byte("{\"seed\":1,\"actions\":0}\nnot json\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := groupArms([]string{directory})
|
||||
@@ -161,3 +170,35 @@ func TestGroupArms_ReportsBadRecordLines(t *testing.T) {
|
||||
t.Fatalf("error %v, want the offending line number", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A runs.jsonl written before the campaign counted dispatched actions has no
|
||||
// such field. Reading the absence as zero would divide by zero, so the whole
|
||||
// campaign is refused instead.
|
||||
func TestGroupArms_RefusesRecordsWithoutADispatchedActionCount(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
directory := filepath.Join(root, "old-format")
|
||||
writeCampaign(t, directory, map[string]any{"arm": "seeded", "max_steps": 40, "seeds": []int{1, 2}},
|
||||
[]map[string]any{
|
||||
{"seed": 1, "exit_code": 0, "steps": 40, "actions": 40},
|
||||
{"seed": 2, "exit_code": 0, "steps": 40},
|
||||
})
|
||||
_, err := groupArms([]string{directory})
|
||||
if err == nil {
|
||||
t.Fatal("a runs.jsonl without an action count was accepted")
|
||||
}
|
||||
for _, fragment := range []string{"line 2", "actions", directory} {
|
||||
if !strings.Contains(err.Error(), fragment) {
|
||||
t.Errorf("error %q is missing %q", err, fragment)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGroupArms_RefusesAnExcludedRecordWithoutAnActionCount(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
directory := filepath.Join(root, "old-format")
|
||||
writeCampaign(t, directory, map[string]any{"arm": "seeded", "max_steps": 40, "seeds": []int{1}},
|
||||
[]map[string]any{{"seed": 1, "exit_code": 3}})
|
||||
if _, err := groupArms([]string{directory}); err == nil {
|
||||
t.Fatal("an old-format record was accepted because the run was excluded anyway")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user