mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 12:07:09 +00:00
fix(campaign): the action count leaves the setup's login out on a model run
Defects per thousand actions divided by every dispatched step, so a spec whose setup logs in inflated the denominator by however many steps that took. It is the same error the run gate had, and it does not cancel between arms. A model run is separable because only an llm-selected action stamps next_action.source. A seeded run is not: its setup returns through the same entry with no marker, and 11261 dispatched steps across the 169 recorded runs carry no source at all, so excluding on it blind would report every seeded run as having explored nothing. The seeded arm counts as before and a test pins that.
This commit is contained in:
1 parent
b278fc869d
commit
5f1f50c2fb
4 files changed
+164
-20
No files matched your search
@@ -52,10 +52,12 @@ type runRecord struct {
|
||||
// It is what the survival analysis times the event by; see eventStep.
|
||||
FirstViolationDetectedStep *int `json:"first_violation_detected_step"`
|
||||
ViolatedProperties []string `json:"violated_properties"`
|
||||
// Actions is the count of steps that dispatched an action, and it is a
|
||||
// pointer so that a runs.jsonl written before the campaign tool counted
|
||||
// them is refused rather than read as an arm that acted zero times. The
|
||||
// campaign tool always emits the field, so its absence dates the file.
|
||||
// Actions is the count of steps on which the action generator dispatched an
|
||||
// action, which excludes the steps the spec's setup drove before the
|
||||
// generator was consulted wherever the trace names them. It is a pointer so
|
||||
// that a runs.jsonl written before the campaign tool counted them is refused
|
||||
// rather than read as an arm that acted zero times. The campaign tool always
|
||||
// emits the field, so its absence dates the file.
|
||||
Actions *int `json:"actions"`
|
||||
}
|
||||
|
||||
|
||||
@@ -34,7 +34,8 @@ func writeReport(result analysis, out io.Writer) {
|
||||
fmt.Fprintln(out)
|
||||
fmt.Fprintln(out, "a detection is one distinct property violated in one run; run hours sum the time the runs worked,")
|
||||
fmt.Fprintln(out, "on the monotonic clock, so a host that slept mid-run is not charged for the sleep")
|
||||
fmt.Fprintln(out, "actions count the steps that dispatched one; the rest chose nothing or had the choice thrown away")
|
||||
fmt.Fprintln(out, "actions count the steps the generator dispatched one on; the rest chose nothing, had the choice")
|
||||
fmt.Fprintln(out, "thrown away, or were the spec's setup driving the app into position before the generator ran")
|
||||
writeTable(out, []string{"arm", "steps", "actions", "run hours", "detections", "defects/1k actions", "defects/hour", "distinct defects", "found in one run"},
|
||||
func(add func(...string)) {
|
||||
for _, summary := range result.Arms {
|
||||
|
||||
Reference in new issue
Block a user