mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 12:07:09 +00:00
fix(analyze): compare arms on censored runs, not on flattened step counts
stepTimes threw the censoring flag away and handed the rank-sum a plain number per run, so a run the wall clock stopped at step 12 was ranked as one that violated at step 12. That was defensible while every clean run sat at the budget, the largest value any run could take, and it stopped being defensible when a clean run started being censored where it stopped. Twenty runs clean at step 12 against twenty violations at step 100 read a12 0.000 and p 4.683e-10 from the rank-sum, in the same report as a log-rank reading p 1.0000. The pairwise comparison is now the Gehan test over the observations themselves, and the report says how many run pairs censoring left with no order between them, which is how much of the effect size is the null value rather than an observation.
This commit is contained in:
1 parent
b7ee23942e
commit
f0726a1e61
6 files changed
+144
-52
No files matched your search
@@ -45,11 +45,15 @@ type pairwiseResult struct {
|
||||
Second string `json:"second"`
|
||||
FirstSize int `json:"first_size"`
|
||||
SecondSize int `json:"second_size"`
|
||||
Statistic float64 `json:"mann_whitney_u"`
|
||||
Statistic float64 `json:"u"`
|
||||
A12 float64 `json:"a12"`
|
||||
// Unordered is how many of the run pairs behind U and A12 have no order
|
||||
// between them, because they were tied or because censoring stopped one run
|
||||
// before the other violated. Each of them counts as half, so it is also how
|
||||
// much of the effect size is the null value rather than an observation.
|
||||
Unordered int `json:"unordered_pairs"`
|
||||
PValue float64 `json:"p_value"`
|
||||
HolmPValue float64 `json:"holm_p_value"`
|
||||
Exact bool `json:"exact"`
|
||||
}
|
||||
|
||||
type analysis struct {
|
||||
@@ -197,7 +201,7 @@ func comparePairs(arms []arm) []pairwiseResult {
|
||||
var pairs []pairwiseResult
|
||||
for first := 0; first < len(arms); first++ {
|
||||
for second := first + 1; second < len(arms); second++ {
|
||||
test := rankSum(arms[first].stepTimes(), arms[second].stepTimes())
|
||||
test := gehanTest(arms[first].observations(), arms[second].observations())
|
||||
pairs = append(pairs, pairwiseResult{
|
||||
First: arms[first].Name,
|
||||
Second: arms[second].Name,
|
||||
@@ -205,9 +209,9 @@ func comparePairs(arms []arm) []pairwiseResult {
|
||||
SecondSize: test.SecondSize,
|
||||
Statistic: test.Statistic,
|
||||
A12: test.A12,
|
||||
Unordered: test.Unordered,
|
||||
PValue: test.PValue,
|
||||
HolmPValue: math.NaN(),
|
||||
Exact: test.Exact,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user