mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
fix(analyze): score a seed pair by which run outlived the other
The paired path had the same defect as the unpaired one: it subtracted two step counts and handed the differences to the signed-rank test, so a pair holding a run the wall clock stopped at step 12 entered as a difference neither run supports. Twenty seeds where the first arm was still clean at step 12 and the second violated at step 5 in six of them read sign -1 and p 0.0011, pointing at the arm that never violated. A pair is now scored the way the unpaired comparison scores one and tested by the exact sign test over the pairs whose order censoring determines, which is what the log-rank stratified by seed reduces to here. The signed-rank goes with the differences it needed: a magnitude-based paired test wants a difference from every pair, and the arms censor on different clocks. The median difference stays, over the pairs where both runs violated, and says so.
This commit is contained in:
1 parent
e2f9864201
commit
1008402558
5 files changed
+212
-442
No files matched your search
@@ -140,15 +140,16 @@ func writeReport(result analysis, out io.Writer) {
|
||||
}
|
||||
|
||||
func writePaired(out io.Writer, comparison pairedComparison) {
|
||||
fmt.Fprintf(out, "\npaired per-seed difference, %s minus %s, censored runs held at the steps they ran\n",
|
||||
fmt.Fprintf(out, "\npaired per-seed contrast, %s against %s, each pair scored by which run outlived the other\n",
|
||||
comparison.First, comparison.Second)
|
||||
fmt.Fprintf(out, "%d seed pair(s): %s sooner in %d, %s sooner in %d, tied in %d\n",
|
||||
fmt.Fprintf(out, "%d seed pair(s): %s sooner in %d, %s sooner in %d, left in no order by censoring in %d\n",
|
||||
comparison.Pairs, comparison.First, comparison.FirstSooner,
|
||||
comparison.Second, comparison.SecondSooner, comparison.Tied)
|
||||
fmt.Fprintf(out, "median difference %+.1f steps, sign %+d, a12 within pairs %.3f\n",
|
||||
comparison.MedianDifference, comparison.Sign, comparison.A12)
|
||||
fmt.Fprintf(out, "wilcoxon signed-rank v %.1f, p %s, holm p %s\n",
|
||||
comparison.Statistic, formatPValue(comparison.PValue), formatPValue(comparison.HolmPValue))
|
||||
comparison.Second, comparison.SecondSooner, comparison.Unordered)
|
||||
fmt.Fprintf(out, "median difference %s over the %d pair(s) where both runs violated, sign %+d, a12 within pairs %.3f\n",
|
||||
formatStepDifference(comparison.MedianDifference), comparison.BothViolated, comparison.Sign, comparison.A12)
|
||||
fmt.Fprintf(out, "sign test over the %d ordered pair(s), p %s, holm p %s\n",
|
||||
comparison.FirstSooner+comparison.SecondSooner,
|
||||
formatPValue(comparison.PValue), formatPValue(comparison.HolmPValue))
|
||||
if len(comparison.UnpairedSeeds) > 0 {
|
||||
fmt.Fprintf(out, "%d seed(s) usable in one arm only and left out of the pairing: %v\n",
|
||||
len(comparison.UnpairedSeeds), comparison.UnpairedSeeds)
|
||||
@@ -177,6 +178,13 @@ func formatMedian(value *float64) string {
|
||||
return strconv.FormatFloat(*value, 'f', -1, 64)
|
||||
}
|
||||
|
||||
func formatStepDifference(value *float64) string {
|
||||
if value == nil {
|
||||
return "undefined"
|
||||
}
|
||||
return fmt.Sprintf("%+.1f steps", *value)
|
||||
}
|
||||
|
||||
// formatActions marks a denominator with actions whose producer nothing names,
|
||||
// because the rate beside it then divides by a count that may include the
|
||||
// login the spec's setup drove.
|
||||
|
||||
Reference in new issue
Block a user