mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 20:17:09 +00:00
fix(analyze): score a seed pair by which run outlived the other
The paired path had the same defect as the unpaired one: it subtracted two step counts and handed the differences to the signed-rank test, so a pair holding a run the wall clock stopped at step 12 entered as a difference neither run supports. Twenty seeds where the first arm was still clean at step 12 and the second violated at step 5 in six of them read sign -1 and p 0.0011, pointing at the arm that never violated. A pair is now scored the way the unpaired comparison scores one and tested by the exact sign test over the pairs whose order censoring determines, which is what the log-rank stratified by seed reduces to here. The signed-rank goes with the differences it needed: a magnitude-based paired test wants a difference from every pair, and the arms censor on different clocks. The median difference stays, over the pairs where both runs violated, and says so.
This commit is contained in:
1 parent
e2f9864201
commit
1008402558
5 files changed
+212
-442
No files matched your search
@@ -96,3 +96,23 @@ func TestRun_EffectSizeFollowsWhatCensoringDetermines(t *testing.T) {
|
||||
"and none of the early arm's twenty had violated by step 12", pair.A12)
|
||||
}
|
||||
}
|
||||
|
||||
// The seed-matched contrast reads the same censored runs and reaches the same
|
||||
// conclusion or it is not measuring the same thing.
|
||||
func TestRun_PairedContrastFollowsWhatCensoringDetermines(t *testing.T) {
|
||||
late := append(violatedAt(6, 5), violatedAt(14, 100)...)
|
||||
earlyDirectory, lateDirectory := wallClockArms(t, stoppedShort(20, 12), late)
|
||||
result := analyseCampaigns(t, "--paired", earlyDirectory, lateDirectory)
|
||||
|
||||
paired := *result.Paired
|
||||
if paired.First != "early" || paired.Second != "late" {
|
||||
t.Fatalf("paired %s minus %s, want early minus late", paired.First, paired.Second)
|
||||
}
|
||||
if paired.Sign != 1 {
|
||||
t.Errorf("sign %+d, want +1: the late arm is the one seen to violate first, in the six pairs "+
|
||||
"where the order is determined at all", paired.Sign)
|
||||
}
|
||||
if paired.A12 <= 0.5 {
|
||||
t.Errorf("a12 within pairs %.4f, want above 0.5", paired.A12)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user