fix(analyze): score a seed pair by which run outlived the other

The paired path had the same defect as the unpaired one: it subtracted two
step counts and handed the differences to the signed-rank test, so a pair
holding a run the wall clock stopped at step 12 entered as a difference
neither run supports. Twenty seeds where the first arm was still clean at
step 12 and the second violated at step 5 in six of them read sign -1 and
p 0.0011, pointing at the arm that never violated.

A pair is now scored the way the unpaired comparison scores one and tested by
the exact sign test over the pairs whose order censoring determines, which is
what the log-rank stratified by seed reduces to here. The signed-rank goes
with the differences it needed: a magnitude-based paired test wants a
difference from every pair, and the arms censor on different clocks. The
median difference stays, over the pairs where both runs violated, and says so.

cmd/internal-tools/analyze/paired.go

@@ -6,178 +6,73 @@ import (
 	"slices"
 )
 
-type signedRankResult struct {
-	Pairs     int     `json:"pairs"`
-	NonZero   int     `json:"non_zero_pairs"`
-	Statistic float64 `json:"signed_rank_v"`
-	PValue    float64 `json:"p_value"`
-	Exact     bool    `json:"exact"`
-}
-
-// exactSignedRankLimit matches R's wilcox.test: the exact null distribution is
-// used only below this many non-zero differences, and only when nothing is tied.
-const exactSignedRankLimit = 50
-
-// signedRank is the two-sided Wilcoxon signed-rank test over paired
-// differences. Zero differences are dropped before ranking and the statistic is
-// the sum of the ranks carried by the positive differences, which is the
-// quantity R's wilcox.test calls V. Wilcoxon (1945), "Individual Comparisons by
-// Ranking Methods", Biometrics Bulletin 1(6), 80-83.
-func signedRank(differences []float64) signedRankResult {
-	result := signedRankResult{
-		Pairs:     len(differences),
-		Statistic: math.NaN(),
-		PValue:    math.NaN(),
-	}
-	var magnitudes []float64
-	var positive []bool
-	for _, difference := range differences {
-		if difference == 0 {
-			continue
-		}
-		magnitudes = append(magnitudes, math.Abs(difference))
-		positive = append(positive, difference > 0)
-	}
-	result.NonZero = len(magnitudes)
-	if result.NonZero == 0 {
-		return result
-	}
-	ranks, tieGroups := midRanks(magnitudes)
-	statistic := 0.0
-	for index, rank := range ranks {
-		if positive[index] {
-			statistic += rank
-		}
-	}
-	result.Statistic = statistic
-
-	droppedZeros := len(differences) != result.NonZero
-	if len(tieGroups) == 0 && !droppedZeros && result.NonZero < exactSignedRankLimit {
-		result.Exact = true
-		result.PValue = exactSignedRankTwoSided(statistic, result.NonZero)
-		return result
-	}
-	result.PValue = normalSignedRankTwoSided(statistic, result.NonZero, tieGroups)
-	return result
-}
-
-// normalSignedRankTwoSided follows the large-sample branch of R's wilcox.test:
+// signTest is the exact two-sided sign test over matched pairs. Under the null
+// that neither arm reaches its first violation sooner, a pair whose order the
+// censoring determines falls either way with probability one half, so the count
+// is binomial and the two-sided p-value doubles the smaller tail. Pairs left
+// with no order carry no information and are not trials.
 //
-//	mean     = n(n+1)/4
-//	variance = n(n+1)(2n+1)/24 - sum(t^3 - t)/48
-//
-// where t runs over the sizes of the groups tied on the absolute difference.
-// The 0.5 shift toward the null mean is the continuity correction.
-func normalSignedRankTwoSided(statistic float64, count int, tieGroups []int) float64 {
-	variance := signedRankVariance(count, tieGroups)
-	if variance <= 0 {
-		return 1
+// It is the seed-matched form of the comparison the unpaired test makes, and
+// it is what the log-rank stratified by seed reduces to with one run per arm in
+// each stratum. The magnitude-based alternatives are not available: a
+// difference in steps needs both runs to have violated, and a paired test built
+// on scores of censored times, the paired Prentice-Wilcoxon among them, is
+// centred at zero under the null only when the two arms censor alike, which is
+// exactly what the wall clock stops them from doing.
+func signTest(favouringFirst, favouringSecond int) float64 {
+	trials := favouringFirst + favouringSecond
+	if trials == 0 {
+		return math.NaN()
 	}
-	size := float64(count)
-	centered := statistic - size*(size+1)/4
-	correction := 0.0
-	switch {
-	case centered > 0:
-		correction = 0.5
-	case centered < 0:
-		correction = -0.5
+	tail := 0.0
+	for count := 0; count <= min(favouringFirst, favouringSecond); count++ {
+		tail += math.Exp(logBinomialCoefficient(trials, count) - float64(trials)*math.Ln2)
 	}
-	z := (centered - correction) / math.Sqrt(variance)
-	tail := math.Min(standardNormalUpperTail(z), standardNormalUpperTail(-z))
 	return math.Min(2*tail, 1)
 }
 
-func signedRankVariance(count int, tieGroups []int) float64 {
-	size := float64(count)
-	tieAdjustment := 0.0
-	for _, group := range tieGroups {
-		tied := float64(group)
-		tieAdjustment += tied*tied*tied - tied
-	}
-	return size*(size+1)*(2*size+1)/24 - tieAdjustment/48
-}
-
-// exactSignedRankTwoSided doubles the smaller exact tail, as R's wilcox.test
-// does.
-func exactSignedRankTwoSided(statistic float64, count int) float64 {
-	if statistic > float64(count)*float64(count+1)/4 {
-		return math.Min(2*exactSignedRankUpperTail(statistic, count), 1)
-	}
-	return math.Min(2*exactSignedRankLowerTail(statistic, count), 1)
-}
-
-// exactSignedRankUpperTail is P(V >= statistic) under the null with no ties.
-func exactSignedRankUpperTail(statistic float64, count int) float64 {
-	counts := exactSignedRankCounts(count)
-	total, tail := 0.0, 0.0
-	for value, weight := range counts {
-		total += weight
-		if float64(value) >= statistic {
-			tail += weight
-		}
-	}
-	return tail / total
-}
-
-func exactSignedRankLowerTail(statistic float64, count int) float64 {
-	counts := exactSignedRankCounts(count)
-	total, tail := 0.0, 0.0
-	for value, weight := range counts {
-		total += weight
-		if float64(value) <= statistic {
-			tail += weight
-		}
-	}
-	return tail / total
-}
-
-// exactSignedRankCounts returns the number of sign assignments producing each
-// value of V from 0 to n(n+1)/2. V is the sum of the ranks held by the positive
-// differences, so the count is a subset-sum tally over the ranks 1 to n.
-func exactSignedRankCounts(count int) []float64 {
-	high := count * (count + 1) / 2
-	table := make([]float64, high+1)
-	table[0] = 1
-	for rank := 1; rank <= count; rank++ {
-		for sum := high; sum >= rank; sum-- {
-			if table[sum-rank] != 0 {
-				table[sum] += table[sum-rank]
-			}
-		}
-	}
-	return table
+func logBinomialCoefficient(trials, chosen int) float64 {
+	all, _ := math.Lgamma(float64(trials + 1))
+	picked, _ := math.Lgamma(float64(chosen + 1))
+	rest, _ := math.Lgamma(float64(trials-chosen) + 1)
+	return all - picked - rest
 }
 
 // pairedComparison is the seed-matched contrast the actuation ablation reports.
-// The difference is the first arm's steps to first violation less the second's,
-// so a positive median means the second arm reached its first violation sooner,
-// and Sign carries that direction as a number the decision rule can read.
+// A pair is scored the way the unpaired comparison scores one, by which run
+// outlived the other, so Sign is +1 when the second arm is the one seen to
+// violate sooner across the pairs whose order censoring determines.
 type pairedComparison struct {
-	First            string  `json:"first"`
-	Second           string  `json:"second"`
-	Pairs            int     `json:"pairs"`
-	UnpairedSeeds    []int64 `json:"unpaired_seeds,omitempty"`
-	MedianDifference float64 `json:"median_step_difference"`
-	Sign             int     `json:"sign"`
-	FirstSooner      int     `json:"first_sooner"`
-	SecondSooner     int     `json:"second_sooner"`
-	Tied             int     `json:"tied"`
+	First         string  `json:"first"`
+	Second        string  `json:"second"`
+	Pairs         int     `json:"pairs"`
+	UnpairedSeeds []int64 `json:"unpaired_seeds,omitempty"`
+	Sign          int     `json:"sign"`
+	FirstSooner   int     `json:"first_sooner"`
+	SecondSooner  int     `json:"second_sooner"`
+	// Unordered is the pairs the censoring leaves in no order, either because
+	// both runs ended clean or because the run that stopped first stopped before
+	// the other violated. They are not evidence either way and are not trials.
+	Unordered int `json:"unordered_pairs"`
+	// MedianDifference is in steps and is undefined unless some pair has both
+	// runs violating, which is the only shape a difference in steps can be read
+	// off. BothViolated says how many pairs it summarizes, because it describes
+	// those pairs and not the sample.
+	MedianDifference *float64 `json:"median_step_difference"`
+	BothViolated     int      `json:"both_violated_pairs"`
 	// A12 is the within-pair form of the Vargha-Delaney effect size, the share
-	// of matched seeds on which the first arm took more steps, counting a tie as
-	// half. A matched design has no reason to compare the two arms as pooled
-	// bags of runs when each seed has a partner.
+	// of matched seeds on which the first arm took more steps, an unordered pair
+	// counting as half. A matched design has no reason to compare the two arms
+	// as pooled bags of runs when each seed has a partner.
 	A12        float64 `json:"a12_within_pairs"`
-	Statistic  float64 `json:"signed_rank_v"`
 	PValue     float64 `json:"p_value"`
 	HolmPValue float64 `json:"holm_p_value"`
-	Exact      bool    `json:"exact"`
 }
 
-// pairArms matches the two arms by seed and contrasts them pair by pair.
-// Censored runs enter at the steps they ran, the same convention the unpaired
-// comparison uses. A seed usable in one arm and not the other is named rather
-// than dropped silently, because that is a host that lost a run and it is what
-// the campaign manifest exists to make visible.
+// pairArms matches the two arms by seed and contrasts them pair by pair. A seed
+// usable in one arm and not the other is named rather than dropped silently,
+// because that is a host that lost a run and it is what the campaign manifest
+// exists to make visible.
 func pairArms(first, second arm) (pairedComparison, error) {
 	firstBySeed, err := usableBySeed(first)
 	if err != nil {
@@ -192,7 +87,6 @@ func pairArms(first, second arm) (pairedComparison, error) {
 		First:      first.Name,
 		Second:     second.Name,
 		A12:        math.NaN(),
-		Statistic:  math.NaN(),
 		PValue:     math.NaN(),
 		HolmPValue: math.NaN(),
 	}
@@ -204,34 +98,38 @@ func pairArms(first, second arm) (pairedComparison, error) {
 			comparison.UnpairedSeeds = append(comparison.UnpairedSeeds, seed)
 			continue
 		}
-		difference := observationOf(left, first.Budget).Steps - observationOf(right, second.Budget).Steps
-		differences = append(differences, difference)
-		switch {
-		case difference < 0:
-			comparison.FirstSooner++
-		case difference > 0:
+		leftRun := observationOf(left, first.Budget)
+		rightRun := observationOf(right, second.Budget)
+		comparison.Pairs++
+		switch outlives(leftRun, rightRun) {
+		case 1:
 			comparison.SecondSooner++
+		case -1:
+			comparison.FirstSooner++
 		default:
-			comparison.Tied++
+			comparison.Unordered++
+		}
+		if leftRun.Event && rightRun.Event {
+			comparison.BothViolated++
+			differences = append(differences, leftRun.Steps-rightRun.Steps)
 		}
 	}
-	comparison.Pairs = len(differences)
 	if comparison.Pairs == 0 {
 		return comparison, nil
 	}
 
-	comparison.MedianDifference = medianOf(differences)
+	if len(differences) > 0 {
+		median := medianOf(differences)
+		comparison.MedianDifference = &median
+	}
 	switch {
-	case comparison.MedianDifference > 0:
+	case comparison.SecondSooner > comparison.FirstSooner:
 		comparison.Sign = 1
-	case comparison.MedianDifference < 0:
+	case comparison.FirstSooner > comparison.SecondSooner:
 		comparison.Sign = -1
 	}
-	comparison.A12 = (float64(comparison.SecondSooner) + 0.5*float64(comparison.Tied)) / float64(comparison.Pairs)
-	test := signedRank(differences)
-	comparison.Statistic = test.Statistic
-	comparison.PValue = test.PValue
-	comparison.Exact = test.Exact
+	comparison.A12 = (float64(comparison.SecondSooner) + 0.5*float64(comparison.Unordered)) / float64(comparison.Pairs)
+	comparison.PValue = signTest(comparison.FirstSooner, comparison.SecondSooner)
 	return comparison, nil
 }