mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 03:57:09 +00:00
fix(analyze): divide by actions that ran
Defects per thousand actions counted every step, including steps that chose nothing and steps whose action was never dispatched. The inflation is policy-dependent, so it does not cancel between arms: on the fixture campaign the model arm's yield was reported at 60.3 per thousand against a true 120.7, because half its steps did nothing. A runs.jsonl without the count is refused by name and line rather than read as zero actions, which would report every per-action rate wrongly. The report also carries steps beside actions now, so the gap is visible rather than folded into a denominator. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
f6d562e3dc
commit
2cbe03c3fb
6 files changed
+162
-36
No files matched your search
@@ -39,6 +39,11 @@ type runRecord struct {
|
||||
Steps int `json:"steps"`
|
||||
FirstViolationOriginStep *int `json:"first_violation_origin_step"`
|
||||
ViolatedProperties []string `json:"violated_properties"`
|
||||
// Actions is the count of steps that dispatched an action, and it is a
|
||||
// pointer so that a runs.jsonl written before the campaign tool counted
|
||||
// them is refused rather than read as an arm that acted zero times. The
|
||||
// campaign tool always emits the field, so its absence dates the file.
|
||||
Actions *int `json:"actions"`
|
||||
}
|
||||
|
||||
// Exclusion reasons. A run that failed or timed out is missing data, not a
|
||||
@@ -55,6 +60,7 @@ const (
|
||||
type classifiedRun struct {
|
||||
Seed int64
|
||||
Steps int
|
||||
Actions int
|
||||
DurationMillis int64
|
||||
OriginStep int
|
||||
Violated bool
|
||||
@@ -110,6 +116,12 @@ func loadCampaign(directory string) (manifest, []runRecord, error) {
|
||||
if err := json.Unmarshal([]byte(raw), &record); err != nil {
|
||||
return manifest{}, nil, fmt.Errorf("%s line %d in %s: %w", recordsFileName, lineNumber, directory, err)
|
||||
}
|
||||
if record.Actions == nil {
|
||||
return manifest{}, nil, fmt.Errorf("%s line %d in %s has no actions count: it was written before "+
|
||||
"dispatched actions were counted, and reading the missing count as zero would report every "+
|
||||
"per-action rate wrongly; re-run the campaign to produce it",
|
||||
recordsFileName, lineNumber, directory)
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
@@ -127,6 +139,9 @@ func classify(record runRecord, budget int) classifiedRun {
|
||||
DurationMillis: record.DurationMillis,
|
||||
ViolatedProperties: slices.Clone(record.ViolatedProperties),
|
||||
}
|
||||
if record.Actions != nil {
|
||||
item.Actions = *record.Actions
|
||||
}
|
||||
switch {
|
||||
case record.LaunchError != "":
|
||||
item.ExcludedBecause = reasonLaunchError
|
||||
|
||||
Reference in new issue
Block a user