fix(analyze): divide by actions that ran

Defects per thousand actions counted every step, including steps that chose
nothing and steps whose action was never dispatched. The inflation is
policy-dependent, so it does not cancel between arms: on the fixture campaign
the model arm's yield was reported at 60.3 per thousand against a true 120.7,
because half its steps did nothing.

A runs.jsonl without the count is refused by name and line rather than read as
zero actions, which would report every per-action rate wrongly. The report also
carries steps beside actions now, so the gap is visible rather than folded into
a denominator.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
pj committed 2026-08-12 23:43:19 +05:30
1 parent f6d562e3dc
commit 2cbe03c3fb
6 files changed
+162 -36

No files matched your search

+15
View File
@@ -39,6 +39,11 @@ type runRecord struct {
Steps int `json:"steps"`
FirstViolationOriginStep *int `json:"first_violation_origin_step"`
ViolatedProperties []string `json:"violated_properties"`
// Actions is the count of steps that dispatched an action, and it is a
// pointer so that a runs.jsonl written before the campaign tool counted
// them is refused rather than read as an arm that acted zero times. The
// campaign tool always emits the field, so its absence dates the file.
Actions *int `json:"actions"`
}
// Exclusion reasons. A run that failed or timed out is missing data, not a
@@ -55,6 +60,7 @@ const (
type classifiedRun struct {
Seed int64
Steps int
Actions int
DurationMillis int64
OriginStep int
Violated bool
@@ -110,6 +116,12 @@ func loadCampaign(directory string) (manifest, []runRecord, error) {
if err := json.Unmarshal([]byte(raw), &record); err != nil {
return manifest{}, nil, fmt.Errorf("%s line %d in %s: %w", recordsFileName, lineNumber, directory, err)
}
if record.Actions == nil {
return manifest{}, nil, fmt.Errorf("%s line %d in %s has no actions count: it was written before "+
"dispatched actions were counted, and reading the missing count as zero would report every "+
"per-action rate wrongly; re-run the campaign to produce it",
recordsFileName, lineNumber, directory)
}
records = append(records, record)
}
if err := scanner.Err(); err != nil {
@@ -127,6 +139,9 @@ func classify(record runRecord, budget int) classifiedRun {
DurationMillis: record.DurationMillis,
ViolatedProperties: slices.Clone(record.ViolatedProperties),
}
if record.Actions != nil {
item.Actions = *record.Actions
}
switch {
case record.LaunchError != "":
item.ExcludedBecause = reasonLaunchError