package main import ( "bytes" "context" "encoding/json" "io" "os" "path/filepath" "slices" "strings" "testing" "time" ) // stubSanderling answers `version`, writes a run directory shaped like the one // `sanderling test` produces, violates on seed 5 and fails on seed 7. const stubSanderling = `#!/bin/sh if [ "$1" = "version" ]; then echo "stub 9.9.9" exit 0 fi seed="" output="" while [ $# -gt 0 ]; do case "$1" in --seed) seed="$2"; shift 2 ;; --output) output="$2"; shift 2 ;; *) shift ;; esac done echo "stub run seed=$seed" run="$output/20260101-000000" mkdir -p "$run" echo "{\"seed\":$seed,\"platform\":\"web\"}" > "$run/meta.json" { echo '{"step":1,"hierarchy":{"elements":[]}}' echo '{"step":2,"hierarchy":{"elements":[]}}' if [ "$seed" = "5" ]; then echo '{"step":3,"hierarchy":{"elements":[]},"violations":["cartTotalMatches"],"witnesses":{"cartTotalMatches":{"reason":"total drifted","step":2,"detected_step":3}}}' else echo '{"step":3,"hierarchy":{"elements":[]}}' fi } > "$run/trace.jsonl" if [ "$seed" = "7" ]; then echo "driver crashed" >&2 exit 3 fi ` func TestRun_EndToEndAgainstStubBinary(t *testing.T) { directory := t.TempDir() binaryPath := filepath.Join(directory, "stub-sanderling") if err := os.WriteFile(binaryPath, []byte(stubSanderling), 0o755); err != nil { t.Fatal(err) } campaignDirectory := filepath.Join(directory, "campaign") var stdout bytes.Buffer err := run([]string{ "--spec", "/specs/folio.ts", "--bundle-id", "app.folio", "--platform", "web", "--arm", "seeded-web", "--generator", "seeded", "--max-steps", "50", "--duration", "30s", "--seeds", "4-5,7", "--devices", "worker-a,worker-b", "--sanderling", binaryPath, "--output", campaignDirectory, "--", "--clear-data=false", }, &stdout, io.Discard) if err == nil || !strings.Contains(err.Error(), "1 of 3 runs failed") { t.Fatalf("expected the failing seed to be reported, got %v", err) } body, err := os.ReadFile(filepath.Join(campaignDirectory, manifestFileName)) if err != nil { t.Fatal(err) } var recorded manifest if err := json.Unmarshal(body, &recorded); err != nil { t.Fatal(err) } if recorded.SanderlingVersion != "stub 9.9.9" { t.Errorf("version: got %q", recorded.SanderlingVersion) } if !slices.Equal(recorded.Seeds, []int64{4, 5, 7}) { t.Errorf("intended seeds: got %v", recorded.Seeds) } if slices.Contains(recorded.ArgumentTemplate, "--device") { t.Errorf("web template should carry no device flag: %v", recorded.ArgumentTemplate) } if recorded.ArgumentTemplate[len(recorded.ArgumentTemplate)-1] != "--clear-data=false" { t.Errorf("template lost the passthrough flag: %v", recorded.ArgumentTemplate) } records := readRecords(t, campaignDirectory) if len(records) != 3 { t.Fatalf("records: got %d, want 3", len(records)) } bySeed := map[int64]runRecord{} for _, record := range records { bySeed[record.Seed] = record } for _, seed := range []int64{4, 5, 7} { record, ok := bySeed[seed] if !ok { t.Fatalf("seed %d is missing from %s", seed, recordsFileName) } if record.Steps != 3 { t.Errorf("seed %d steps: got %d, want 3", seed, record.Steps) } if record.RunDirectory == "" { t.Errorf("seed %d has no run directory", seed) } if !slices.Contains([]string{"worker-a", "worker-b"}, record.Device) { t.Errorf("seed %d device: got %q", seed, record.Device) } } violating := bySeed[5] if violating.FirstViolationOriginStep == nil || *violating.FirstViolationOriginStep != 2 { t.Fatalf("seed 5 origin step: got %v, want 2", violating.FirstViolationOriginStep) } if *violating.FirstViolationDetectedStep != 3 || violating.FirstViolationReason != "total drifted" { t.Errorf("seed 5 violation: %+v", violating) } if bySeed[4].FirstViolationOriginStep != nil { t.Errorf("seed 4 should be censored at the budget: %+v", bySeed[4]) } if bySeed[7].ExitCode != 3 { t.Errorf("seed 7 exit code: got %d, want 3", bySeed[7].ExitCode) } log, err := os.ReadFile(filepath.Join(campaignDirectory, "seed-7", "sanderling.log")) if err != nil { t.Fatal(err) } if !strings.Contains(string(log), "driver crashed") { t.Errorf("seed 7 log lost the stderr line: %q", log) } if !strings.Contains(stdout.String(), "outcome=violation@2") { t.Errorf("progress output: %q", stdout.String()) } } func TestExecuteCommand_RunTimeoutSignalsSoTheRunReapsItsChildren(t *testing.T) { directory := t.TempDir() marker := filepath.Join(directory, "child-reaped") trapped := filepath.Join(directory, "trap-installed") script := filepath.Join(directory, "wedged") body := "#!/bin/sh\n" + "sleep 300 &\n" + "child=$!\n" + "trap 'kill $child; echo reaped > " + marker + "; exit 143' TERM\n" + "echo installed > " + trapped + "\n" + "wait $child\n" if err := os.WriteFile(script, []byte(body), 0o755); err != nil { t.Fatal(err) } // Cancel only once the script has installed its trap. A fixed deadline // races the shell under a loaded machine, and a run signalled before its // trap exists fails this test for a reason it does not test. ctx, cancel := context.WithCancel(context.Background()) defer cancel() finished := make(chan error, 1) go func() { _, err := executeCommand(ctx, script, nil, io.Discard) finished <- err }() waitForFile(t, trapped) cancel() if err := <-finished; err != nil { t.Fatalf("execute: %v", err) } if _, err := os.Stat(marker); err != nil { t.Fatal("the run timeout killed the run outright, so it never reaped its own children: " + "an unattended sweep leaks one sidecar per wedged run") } } func waitForFile(t *testing.T, path string) { t.Helper() deadline := time.Now().Add(30 * time.Second) for time.Now().Before(deadline) { if _, err := os.Stat(path); err == nil { return } time.Sleep(5 * time.Millisecond) } t.Fatalf("%s never appeared", path) }