sanderling/cmd/internal-tools/analyze/paired.go

package main

import (
	"fmt"
	"math"
	"slices"
)

// signTest is the exact two-sided sign test over matched pairs. Under the null
// that neither arm reaches its first violation sooner, a pair whose order the
// censoring determines falls either way with probability one half, so the count
// is binomial and the two-sided p-value doubles the smaller tail. Pairs left
// with no order carry no information and are not trials.
//
// It is the seed-matched form of the comparison the unpaired test makes, and
// it is what the log-rank stratified by seed reduces to with one run per arm in
// each stratum. The magnitude-based alternatives are not available: a
// difference in steps needs both runs to have violated, and a paired test built
// on scores of censored times, the paired Prentice-Wilcoxon among them, is
// centred at zero under the null only when the two arms censor alike, which is
// exactly what the wall clock stops them from doing.
func signTest(favouringFirst, favouringSecond int) float64 {
	trials := favouringFirst + favouringSecond
	if trials == 0 {
		return math.NaN()
	}
	tail := 0.0
	for count := 0; count <= min(favouringFirst, favouringSecond); count++ {
		tail += math.Exp(logBinomialCoefficient(trials, count) - float64(trials)*math.Ln2)
	}
	return math.Min(2*tail, 1)
}

func logBinomialCoefficient(trials, chosen int) float64 {
	all, _ := math.Lgamma(float64(trials + 1))
	picked, _ := math.Lgamma(float64(chosen + 1))
	rest, _ := math.Lgamma(float64(trials-chosen) + 1)
	return all - picked - rest
}

// pairedComparison is the seed-matched contrast the actuation ablation reports.
// A pair is scored the way the unpaired comparison scores one, by which run
// outlived the other, so Sign is +1 when the second arm is the one seen to
// violate sooner across the pairs whose order censoring determines.
type pairedComparison struct {
	First         string  `json:"first"`
	Second        string  `json:"second"`
	Pairs         int     `json:"pairs"`
	UnpairedSeeds []int64 `json:"unpaired_seeds,omitempty"`
	Sign          int     `json:"sign"`
	FirstSooner   int     `json:"first_sooner"`
	SecondSooner  int     `json:"second_sooner"`
	// Unordered is the pairs the censoring leaves in no order, either because
	// both runs ended clean or because the run that stopped first stopped before
	// the other violated. They are not evidence either way and are not trials.
	Unordered int `json:"unordered_pairs"`
	// MedianDifference is in steps and is undefined unless some pair has both
	// runs violating, which is the only shape a difference in steps can be read
	// off. BothViolated says how many pairs it summarizes, because it describes
	// those pairs and not the sample.
	MedianDifference *float64 `json:"median_step_difference"`
	BothViolated     int      `json:"both_violated_pairs"`
	// A12 is the within-pair form of the Vargha-Delaney effect size, the share
	// of matched seeds on which the first arm took more steps, an unordered pair
	// counting as half. A matched design has no reason to compare the two arms
	// as pooled bags of runs when each seed has a partner.
	A12        float64 `json:"a12_within_pairs"`
	PValue     float64 `json:"p_value"`
	HolmPValue float64 `json:"holm_p_value"`
}

// pairArms matches the two arms by seed and contrasts them pair by pair. A seed
// usable in one arm and not the other is named rather than dropped silently,
// because that is a host that lost a run and it is what the campaign manifest
// exists to make visible.
func pairArms(first, second arm) (pairedComparison, error) {
	firstBySeed, err := usableBySeed(first)
	if err != nil {
		return pairedComparison{}, err
	}
	secondBySeed, err := usableBySeed(second)
	if err != nil {
		return pairedComparison{}, err
	}

	comparison := pairedComparison{
		First:      first.Name,
		Second:     second.Name,
		A12:        math.NaN(),
		PValue:     math.NaN(),
		HolmPValue: math.NaN(),
	}
	var differences []float64
	for _, seed := range sortedSeeds(firstBySeed, secondBySeed) {
		left, inFirst := firstBySeed[seed]
		right, inSecond := secondBySeed[seed]
		if !inFirst || !inSecond {
			comparison.UnpairedSeeds = append(comparison.UnpairedSeeds, seed)
			continue
		}
		leftRun := observationOf(left, first.Budget)
		rightRun := observationOf(right, second.Budget)
		comparison.Pairs++
		switch outlives(leftRun, rightRun) {
		case 1:
			comparison.SecondSooner++
		case -1:
			comparison.FirstSooner++
		default:
			comparison.Unordered++
		}
		if leftRun.Event && rightRun.Event {
			comparison.BothViolated++
			differences = append(differences, leftRun.Steps-rightRun.Steps)
		}
	}
	if comparison.Pairs == 0 {
		return comparison, nil
	}

	if len(differences) > 0 {
		median := medianOf(differences)
		comparison.MedianDifference = &median
	}
	switch {
	case comparison.SecondSooner > comparison.FirstSooner:
		comparison.Sign = 1
	case comparison.FirstSooner > comparison.SecondSooner:
		comparison.Sign = -1
	}
	comparison.A12 = (float64(comparison.SecondSooner) + 0.5*float64(comparison.Unordered)) / float64(comparison.Pairs)
	comparison.PValue = signTest(comparison.FirstSooner, comparison.SecondSooner)
	return comparison, nil
}

func usableBySeed(current arm) (map[int64]classifiedRun, error) {
	bySeed := map[int64]classifiedRun{}
	for _, item := range current.Runs {
		if item.ExcludedBecause != "" {
			continue
		}
		if _, repeated := bySeed[item.Seed]; repeated {
			return nil, fmt.Errorf("arm %q has more than one usable run for seed %d: a seed-matched "+
				"comparison cannot choose between them", current.Name, item.Seed)
		}
		bySeed[item.Seed] = item
	}
	return bySeed, nil
}

func sortedSeeds(sets ...map[int64]classifiedRun) []int64 {
	var seeds []int64
	seen := map[int64]bool{}
	for _, set := range sets {
		for seed := range set {
			if seen[seed] {
				continue
			}
			seen[seed] = true
			seeds = append(seeds, seed)
		}
	}
	slices.Sort(seeds)
	return seeds
}

func medianOf(values []float64) float64 {
	sorted := slices.Sorted(slices.Values(values))
	middle := len(sorted) / 2
	if len(sorted)%2 == 1 {
		return sorted[middle]
	}
	return (sorted[middle-1] + sorted[middle]) / 2
}