diff --git a/cmd/internal-tools/implementation-sweep/end_to_end_test.go b/cmd/internal-tools/implementation-sweep/end_to_end_test.go new file mode 100644 index 0000000..3817520 --- /dev/null +++ b/cmd/internal-tools/implementation-sweep/end_to_end_test.go @@ -0,0 +1,482 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "math/rand" + "net" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" +) + +// The test binary doubles as the stub implementation's server and as the +// fetcher the stub campaign uses, so the sweep drives a real preview process +// over a real port and the served URL is answered by a real HTTP server. +func TestMain(m *testing.M) { + switch { + case os.Getenv("SWEEP_TEST_SERVE_PORT") != "": + serveUntilKilled(os.Getenv("SWEEP_TEST_SERVE_PORT")) + case os.Getenv("SWEEP_TEST_FETCH_URL") != "": + recordFetch( + os.Getenv("SWEEP_TEST_FETCH_URL"), + os.Getenv("SWEEP_TEST_FETCH_LOG"), + ) + default: + os.Exit(m.Run()) + } +} + +func serveUntilKilled(port string) { + handler := http.HandlerFunc( + func(writer http.ResponseWriter, request *http.Request) { + fmt.Fprintf(writer, "%s %s", port, request.URL.RequestURI()) + }, + ) + http.ListenAndServe("localhost:"+port, handler) +} + +func recordFetch(url, logPath string) { + line := "" + response, err := http.Get(url) + if err != nil { + line = fmt.Sprintf("%s -> error %v\n", url, err) + } else { + body, _ := io.ReadAll(response.Body) + response.Body.Close() + line = fmt.Sprintf("%s -> %d %s\n", url, response.StatusCode, body) + } + logFile, err := os.OpenFile( + logPath, + os.O_CREATE|os.O_WRONLY|os.O_APPEND, + 0o644, + ) + if err != nil { + return + } + logFile.WriteString(line) + logFile.Close() +} + +// stubBun answers install, fails to build impl-02, and serves the preview from +// the test binary on the port it was given. +const stubBun = `#!/bin/sh +echo "$PWD $*" >> "%[1]s" +if [ "$1" = "run" ] && [ "$2" = "build" ]; then + case "$PWD" in *impl-02) echo "TS2322: type error" >&2; exit 1 ;; esac + exit 0 +fi +if [ "$1" = "run" ] && [ "$2" = "preview" ]; then + port="" + previous="" + for argument in "$@"; do + if [ "$previous" = "--port" ]; then port="$argument"; fi + previous="$argument" + done + SWEEP_TEST_SERVE_PORT="$port" exec "%[2]s" +fi +exit 0 +` + +// stubCampaign records the argv it was handed and whether the sweep manifest +// was already on disk when it ran, fetches the URL it was told to drive, writes +// the campaign directory the real tool would write, and fails impl-03 seed 4. +const stubCampaign = `#!/bin/sh +output="" +seed="" +url="" +previous="" +for argument in "$@"; do + case "$previous" in + --output) output="$argument" ;; + --seeds) seed="$argument" ;; + --bundle-id) url="$argument" ;; + esac + previous="$argument" +done +manifest=missing +if [ -f "%[1]s" ]; then manifest=present; fi +echo "manifest=$manifest argv: $*" >> "%[2]s" +SWEEP_TEST_FETCH_URL="$url" SWEEP_TEST_FETCH_LOG="%[3]s" "%[4]s" +mkdir -p "$output" +printf '{"arm":"stub","seeds":[%%s]}\n' "$seed" > "$output/campaign.json" +echo "stub campaign seed=$seed url=$url" +case "$output" in + *impl-03/seed-4) exit 1 ;; +esac +exit 0 +` + +func TestRun_EndToEndAgainstStubBunAndCampaign(t *testing.T) { + root := t.TempDir() + implementations := filepath.Join(root, "implementations") + for _, name := range []string{"impl-01", "impl-02", "impl-03"} { + if err := os.MkdirAll(filepath.Join(implementations, name), 0o755); err != nil { + t.Fatal(err) + } + } + specPath := filepath.Join(root, "relay.ts") + if err := os.WriteFile(specPath, []byte("export const properties = [];\n"), 0o644); err != nil { + t.Fatal(err) + } + output := filepath.Join(root, "campaigns") + bunLog := filepath.Join(root, "bun.log") + campaignLog := filepath.Join(root, "campaign.log") + fetchLog := filepath.Join(root, "fetch.log") + testBinary, err := filepath.Abs(os.Args[0]) + if err != nil { + t.Fatal(err) + } + bunPath := writeScript( + t, + filepath.Join(root, "stub-bun"), + fmt.Sprintf(stubBun, bunLog, testBinary), + ) + campaignPath := writeScript( + t, + filepath.Join(root, "stub-campaign"), + fmt.Sprintf( + stubCampaign, + filepath.Join(output, manifestFileName), + campaignLog, + fetchLog, + testBinary, + ), + ) + sanderlingPath := writeScript( + t, + filepath.Join(root, "stub-sanderling"), + "#!/bin/sh\nexit 0\n", + ) + basePort := freePortRange(t, 3) + + var stdout bytes.Buffer + err = run([]string{ + "--implementations", implementations, + "--spec", specPath, + "--seeds", "4-5", + "--max-steps", "40", + "--duration", "30s", + "--concurrency", "2", + "--base-port", fmt.Sprint(basePort), + "--output", output, + "--bun", bunPath, + "--campaign", campaignPath, + "--sanderling", sanderlingPath, + }, &stdout, io.Discard) + if err == nil || + !strings.Contains(err.Error(), "1 of 3 implementations never ran") { + t.Fatalf( + "expected the failed build and the failed campaign to be reported, got %v", + err, + ) + } + + var recorded manifest + manifestBody, err := os.ReadFile(filepath.Join(output, manifestFileName)) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(manifestBody, &recorded); err != nil { + t.Fatal(err) + } + if !slices.Equal(recorded.Seeds, []int64{4, 5}) { + t.Errorf("intended seeds: got %v", recorded.Seeds) + } + if len(recorded.Implementations) != 3 { + t.Fatalf("intended implementations: got %v", recorded.Implementations) + } + for index, planned := range recorded.Implementations { + wantPort := basePort + index + if planned.Port != wantPort { + t.Errorf( + "%s port: got %d, want %d", + planned.Name, + planned.Port, + wantPort, + ) + } + wantURL := fmt.Sprintf("http://localhost:%d/?seed={seed}", wantPort) + if planned.URLTemplate != wantURL { + t.Errorf( + "%s url template: got %q, want %q", + planned.Name, + planned.URLTemplate, + wantURL, + ) + } + } + if recorded.Generator != "seeded" || recorded.MaxSteps != 40 { + t.Errorf( + "manifest generator/budget: got %q/%d", + recorded.Generator, + recorded.MaxSteps, + ) + } + + // impl-02 fails its build, so the two implementations either side of it + // still have to reach the campaign tool with both seeds. + campaignLines := readLines(t, campaignLog) + if len(campaignLines) != 4 { + t.Fatalf( + "campaign invocations: got %d, want 4:\n%s", + len(campaignLines), + strings.Join(campaignLines, "\n"), + ) + } + seen := map[string]bool{} + for _, line := range campaignLines { + if !strings.HasPrefix(line, "manifest=present") { + t.Errorf( + "a campaign ran before the sweep manifest was written: %q", + line, + ) + } + arguments := strings.Fields(strings.SplitN(line, "argv: ", 2)[1]) + arm := argumentValue(arguments, "--arm") + seed := argumentValue(arguments, "--seeds") + bundle := argumentValue(arguments, "--bundle-id") + port := basePort + slices.Index( + []string{"impl-01", "impl-02", "impl-03"}, + arm, + ) + wantBundle := fmt.Sprintf("http://localhost:%d/?seed=%s", port, seed) + if bundle != wantBundle { + t.Errorf( + "%s seed %s: bundle id %q, want %q", + arm, + seed, + bundle, + wantBundle, + ) + } + if got := argumentValue(arguments, "--output"); got != filepath.Join( + output, + arm, + "seed-"+seed, + ) { + t.Errorf("%s seed %s: campaign output %q", arm, seed, got) + } + if got := argumentValue(arguments, "--sanderling"); got != sanderlingPath { + t.Errorf("%s seed %s: sanderling path %q", arm, seed, got) + } + seen[arm+"/"+seed] = true + } + for _, want := range []string{"impl-01/4", "impl-01/5", "impl-03/4", "impl-03/5"} { + if !seen[want] { + t.Errorf("%s never reached the campaign tool", want) + } + } + + // What the served page actually saw: the right port for the + // implementation, carrying the same seed the campaign was given. + fetched := readLines(t, fetchLog) + for _, want := range []string{ + fmt.Sprintf("http://localhost:%d/?seed=4 -> 200 %d /?seed=4", basePort, basePort), + fmt.Sprintf("http://localhost:%d/?seed=5 -> 200 %d /?seed=5", basePort, basePort), + fmt.Sprintf("http://localhost:%d/?seed=4 -> 200 %d /?seed=4", basePort+2, basePort+2), + fmt.Sprintf("http://localhost:%d/?seed=5 -> 200 %d /?seed=5", basePort+2, basePort+2), + } { + if !slices.Contains(fetched, want) { + t.Errorf( + "the served page never saw %q:\n%s", + want, + strings.Join(fetched, "\n"), + ) + } + } + + records := readRecords(t, filepath.Join(output, recordsFileName)) + if len(records) != 3 { + t.Fatalf("implementation records: got %d, want 3", len(records)) + } + byName := map[string]implementationRecord{} + for _, record := range records { + byName[record.Name] = record + } + failed := byName["impl-02"] + if failed.FailedStage != stageBuild || len(failed.Runs) != 0 { + t.Errorf( + "impl-02: got stage %q with %d runs, want a build failure and no runs", + failed.FailedStage, + len(failed.Runs), + ) + } + if !strings.Contains(failed.Error, "build.log") { + t.Errorf("impl-02 error should point at its log: %q", failed.Error) + } + buildLog, err := os.ReadFile( + filepath.Join(output, "impl-02", stageBuild+".log"), + ) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(buildLog), "TS2322") { + t.Errorf("impl-02 build log lost the compiler error: %q", buildLog) + } + for _, name := range []string{"impl-01", "impl-03"} { + record := byName[name] + if record.FailedStage != "" || len(record.Runs) != 2 { + t.Errorf( + "%s: stage %q with %d runs, want no failure and 2 runs", + name, + record.FailedStage, + len(record.Runs), + ) + } + if record.MonotonicMillis <= 0 { + t.Errorf( + "%s took %d ms, so nothing timed how long it worked", + name, + record.MonotonicMillis, + ) + } + for _, run := range record.Runs { + if run.MonotonicMillis <= 0 { + t.Errorf( + "%s seed %d took %d ms, so nothing timed the campaign", + name, + run.Seed, + run.MonotonicMillis, + ) + } + } + } + if exit := byName["impl-03"].Runs[0].ExitCode; exit != 1 { + t.Errorf("impl-03 seed 4 exit code: got %d, want 1", exit) + } + if exit := byName["impl-03"].Runs[1].ExitCode; exit != 0 { + t.Errorf( + "impl-03 seed 5 ran after seed 4 failed and should have exited 0, got %d", + exit, + ) + } + + for _, name := range []string{"impl-01", "impl-03"} { + for _, seed := range []string{"4", "5"} { + directory := filepath.Join(output, name, "seed-"+seed) + if _, err := os.Stat(filepath.Join(directory, "campaign.json")); err != nil { + t.Errorf( + "%s seed %s: no campaign directory: %v", + name, + seed, + err, + ) + } + log, err := os.ReadFile(filepath.Join(directory, "campaign.log")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(log), "stub campaign seed="+seed) { + t.Errorf( + "%s seed %s: campaign output was not captured: %q", + name, + seed, + log, + ) + } + } + } + + installed := readLines(t, bunLog) + for _, name := range []string{"impl-01", "impl-02", "impl-03"} { + if !slices.Contains( + installed, + filepath.Join(implementations, name)+" install", + ) { + t.Errorf( + "%s was never installed:\n%s", + name, + strings.Join(installed, "\n"), + ) + } + } + + // Every preview server the sweep started is gone with it: a leaked one + // holds its port, and the next sweep would be served by the old build. + client := &http.Client{Timeout: 2 * time.Second} + for _, port := range []int{basePort, basePort + 2} { + if response, err := client.Get(readinessURL(port)); err == nil { + response.Body.Close() + t.Errorf("port %d is still served after the sweep finished", port) + } + } + if !strings.Contains(stdout.String(), "failed at build") { + t.Errorf( + "progress output does not name the build failure: %q", + stdout.String(), + ) + } +} + +func writeScript(t *testing.T, path, body string) string { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o755); err != nil { + t.Fatal(err) + } + return path +} + +func readLines(t *testing.T, path string) []string { + t.Helper() + body, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var lines []string + scanner := bufio.NewScanner(strings.NewReader(string(body))) + for scanner.Scan() { + if line := strings.TrimSpace(scanner.Text()); line != "" { + lines = append(lines, line) + } + } + return lines +} + +func readRecords(t *testing.T, path string) []implementationRecord { + t.Helper() + var records []implementationRecord + for _, line := range readLines(t, path) { + var record implementationRecord + if err := json.Unmarshal([]byte(line), &record); err != nil { + t.Fatalf("%s: %v", line, err) + } + records = append(records, record) + } + return records +} + +// freePortRange finds count consecutive free ports, which is what the sweep +// hands out: one port per implementation from --base-port upwards. +func freePortRange(t *testing.T, count int) int { + t.Helper() + for range 100 { + base := 20000 + rand.Intn(20000) + if portsAreFree(base, count) { + return base + } + } + t.Fatalf("no run of %d free ports", count) + return 0 +} + +func portsAreFree(base, count int) bool { + for offset := range count { + listener, err := net.Listen( + "tcp", + fmt.Sprintf("localhost:%d", base+offset), + ) + if err != nil { + return false + } + listener.Close() + } + return true +} diff --git a/cmd/internal-tools/implementation-sweep/main.go b/cmd/internal-tools/implementation-sweep/main.go new file mode 100644 index 0000000..7cf2502 --- /dev/null +++ b/cmd/internal-tools/implementation-sweep/main.go @@ -0,0 +1,284 @@ +// Command implementation-sweep runs one identical campaign against every model +// implementation of a single requirement. It installs, builds and serves each +// implementation on its own port, then hands the campaign tool the same seed +// slice, the same step budget and the same generator for all of them, so a +// difference between implementations is not a difference in exploration. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "path/filepath" + "strconv" + "syscall" + "time" + + "github.com/priyanshujain/sanderling/internal/seedspec" +) + +// The generator and the platform are fixed rather than exposed: the +// pre-registration runs the seeded policy against a served web build, and a +// sweep that could quietly run something else records a comparison nobody made. +const ( + generator = "seeded" + platform = "web" +) + +// defaultConcurrency is how many implementations are built, served and swept at +// once. fleet.md measured eight concurrent web campaigns clean at about 1.1 GB +// resident each, parallel efficiency 0.83 at eight against 0.87 at six, and a +// knee at twelve to sixteen, on a contended laptop it says to re-measure before +// trusting anything above eight. Six sits at the better efficiency, costs about +// 7 GB of a 64 GB host, and leaves the Android emulator farm that shares that +// host its four to six slots. Each worker here also carries a vite server the +// fleet measurement did not include. +const defaultConcurrency = 6 + +// defaultBasePort is the first port handed out. Vite's own defaults are 5173 +// and 4173, so a sweep starting here does not collide with a dev server someone +// left running. +const defaultBasePort = 5300 + +type config struct { + implementationsDirectory string + specPath string + outputDirectory string + seeds []int64 + maxSteps int + duration time.Duration + concurrency int + basePort int + bunPath string + campaignPath string + sanderlingPath string + extraArguments []string +} + +const usage = `implementation-sweep runs one seeded campaign per model implementation. + +Usage: + implementation-sweep --implementations --spec --seeds + --max-steps --output [flags] + [-- ] + +Each impl-* directory under --implementations is installed, built and served on +its own port, and every one is swept with the same seeds and the same step +budget. An implementation that fails to install, build or serve is recorded and +the sweep moves on to the next one. + +Everything after a bare -- reaches every sanderling test call through the +campaign tool. +` + +func parseArguments(arguments []string, stderr io.Writer) (config, error) { + flagSet := flag.NewFlagSet("implementation-sweep", flag.ContinueOnError) + flagSet.SetOutput(stderr) + flagSet.Usage = func() { + fmt.Fprint(stderr, usage) + flagSet.PrintDefaults() + } + var configuration config + var seedSpecification string + flagSet.StringVar( + &configuration.implementationsDirectory, + "implementations", + "", + "directory holding impl-01 to impl-NN (required)", + ) + flagSet.StringVar( + &configuration.specPath, + "spec", + "", + "path to the property set every implementation is run against (required)", + ) + flagSet.StringVar( + &seedSpecification, + "seeds", + "", + "seeds every implementation runs: ranges and lists, e.g. 1-10,20 (required)", + ) + flagSet.IntVar( + &configuration.maxSteps, + "max-steps", + 0, + "per-run step budget, identical across implementations (required, must be positive)", + ) + flagSet.DurationVar( + &configuration.duration, + "duration", + 5*time.Minute, + "per-run wall-clock ceiling passed to each campaign", + ) + flagSet.IntVar( + &configuration.concurrency, + "concurrency", + defaultConcurrency, + "implementations built, served and swept at once", + ) + flagSet.IntVar( + &configuration.basePort, + "base-port", + defaultBasePort, + "first port served; each implementation takes the next one in name order", + ) + flagSet.StringVar( + &configuration.outputDirectory, + "output", + "", + "campaign tree to create (required)", + ) + flagSet.StringVar( + &configuration.bunPath, + "bun", + "bun", + "bun binary that installs, builds and serves each implementation", + ) + flagSet.StringVar( + &configuration.campaignPath, + "campaign", + "campaign", + "campaign binary to invoke per implementation and seed", + ) + flagSet.StringVar( + &configuration.sanderlingPath, + "sanderling", + "sanderling", + "sanderling binary each campaign invokes", + ) + if err := flagSet.Parse(arguments); err != nil { + return config{}, err + } + configuration.extraArguments = flagSet.Args() + + for name, value := range map[string]string{ + "--implementations": configuration.implementationsDirectory, + "--spec": configuration.specPath, + "--seeds": seedSpecification, + "--output": configuration.outputDirectory, + } { + if value == "" { + return config{}, fmt.Errorf("%s is required", name) + } + } + if configuration.maxSteps <= 0 { + return config{}, fmt.Errorf( + "--max-steps must be positive: every implementation needs the same step budget", + ) + } + if configuration.duration <= 0 { + return config{}, fmt.Errorf( + "--duration must be positive: %s", + configuration.duration, + ) + } + if configuration.concurrency <= 0 { + return config{}, fmt.Errorf( + "--concurrency must be positive: %d", + configuration.concurrency, + ) + } + if configuration.basePort < 1024 || configuration.basePort > 65535 { + return config{}, fmt.Errorf( + "--base-port %d is outside 1024-65535", + configuration.basePort, + ) + } + seeds, err := seedspec.Parse(seedSpecification) + if err != nil { + return config{}, fmt.Errorf("--seeds: %w", err) + } + configuration.seeds = seeds + for name, value := range map[string]*string{ + "--implementations": &configuration.implementationsDirectory, + "--spec": &configuration.specPath, + "--output": &configuration.outputDirectory, + } { + absolute, err := filepath.Abs(*value) + if err != nil { + return config{}, fmt.Errorf("%s: %w", name, err) + } + *value = absolute + } + return configuration, nil +} + +// servedURL is the one place a seed becomes a URL. The same seed is also handed +// to the campaign as --seeds, which reaches sanderling as --seed and fixes the +// exploration, while the scaffold reads ?seed= and fixes the latency and the +// outcome of every send. A violation replays only when both carry the same +// number, so both come from the seed argument here and never from two flags. +func servedURL(port int, seed string) string { + return fmt.Sprintf("http://localhost:%d/?seed=%s", port, seed) +} + +func readinessURL(port int) string { + return fmt.Sprintf("http://localhost:%d/", port) +} + +func campaignDirectory( + configuration config, + target implementation, + seed string, +) string { + return filepath.Join( + configuration.outputDirectory, + target.Name, + "seed-"+seed, + ) +} + +// campaignArguments builds one campaign invocation. The seed is a string +// because it lands in two arguments, --seeds and the ?seed= of --bundle-id, +// and passing it once keeps them from drifting apart. +func campaignArguments( + configuration config, + target implementation, + seed string, +) []string { + arguments := []string{ + "--spec", configuration.specPath, + "--bundle-id", servedURL(target.Port, seed), + "--platform", platform, + "--arm", target.Name, + "--generator", generator, + "--max-steps", strconv.Itoa(configuration.maxSteps), + "--duration", configuration.duration.String(), + "--seeds", seed, + "--sanderling", configuration.sanderlingPath, + "--output", campaignDirectory(configuration, target, seed), + } + if len(configuration.extraArguments) > 0 { + arguments = append(arguments, "--") + arguments = append(arguments, configuration.extraArguments...) + } + return arguments +} + +func run(arguments []string, stdout, stderr io.Writer) error { + configuration, err := parseArguments(arguments, stderr) + if err != nil { + return err + } + ctx, cancel := signal.NotifyContext( + context.Background(), + os.Interrupt, + syscall.SIGTERM, + ) + defer cancel() + return runSweep(ctx, configuration, stdout) +} + +func main() { + if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil { + if errors.Is(err, flag.ErrHelp) { + return + } + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} diff --git a/cmd/internal-tools/implementation-sweep/main_test.go b/cmd/internal-tools/implementation-sweep/main_test.go new file mode 100644 index 0000000..f9d4d6d --- /dev/null +++ b/cmd/internal-tools/implementation-sweep/main_test.go @@ -0,0 +1,214 @@ +package main + +import ( + "io" + "net/url" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + "time" +) + +func baseArguments() []string { + return []string{ + "--implementations", "/e4/implementations", + "--spec", "/e4/relay.ts", + "--seeds", "1-3", + "--max-steps", "400", + "--output", "/campaigns/e4", + } +} + +func TestParseArguments_DefaultsAndSeeds(t *testing.T) { + configuration, err := parseArguments(baseArguments(), io.Discard) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(configuration.seeds, []int64{1, 2, 3}) { + t.Errorf("seeds: got %v", configuration.seeds) + } + if configuration.concurrency != defaultConcurrency { + t.Errorf( + "concurrency default: got %d, want %d", + configuration.concurrency, + defaultConcurrency, + ) + } + if configuration.basePort != defaultBasePort { + t.Errorf( + "base port default: got %d, want %d", + configuration.basePort, + defaultBasePort, + ) + } + if configuration.duration != 5*time.Minute { + t.Errorf("duration default: got %s", configuration.duration) + } + for name, got := range map[string]string{ + "bun": configuration.bunPath, + "campaign": configuration.campaignPath, + "sanderling": configuration.sanderlingPath, + } { + if got != name { + t.Errorf("%s path default: got %q", name, got) + } + } +} + +func TestParseArguments_Rejections(t *testing.T) { + cases := []struct { + name string + arguments []string + want string + }{ + { + "missing implementations", + []string{ + "--spec", + "s", + "--seeds", + "1", + "--max-steps", + "10", + "--output", + "o", + }, + "--implementations is required", + }, + { + "missing spec", + []string{ + "--implementations", + "i", + "--seeds", + "1", + "--max-steps", + "10", + "--output", + "o", + }, + "--spec is required", + }, + { + "missing output", + []string{ + "--implementations", + "i", + "--spec", + "s", + "--seeds", + "1", + "--max-steps", + "10", + }, + "--output is required", + }, + { + "zero max steps", + append(baseArguments(), "--max-steps", "0"), + "--max-steps must be positive", + }, + { + "zero concurrency", + append(baseArguments(), "--concurrency", "0"), + "--concurrency must be positive", + }, + { + "privileged base port", + append(baseArguments(), "--base-port", "80"), + "outside 1024-65535", + }, + { + "seed zero", + append(baseArguments(), "--seeds", "0,1"), + "not reproducible", + }, + } + for _, testCase := range cases { + _, err := parseArguments(testCase.arguments, io.Discard) + if err == nil { + t.Errorf("%s: expected error", testCase.name) + continue + } + if !strings.Contains(err.Error(), testCase.want) { + t.Errorf( + "%s: got %q, want it to contain %q", + testCase.name, + err, + testCase.want, + ) + } + } +} + +// The seed reaches two independent things, the campaign's own seed and the +// scaffold's failure stream, and a replay reproduces neither unless they carry +// the same number. +func TestCampaignArguments_OneSeedReachesBothTheCampaignAndTheURL( + t *testing.T, +) { + configuration, err := parseArguments( + append(baseArguments(), "--", "--clear-data=false"), + io.Discard, + ) + if err != nil { + t.Fatal(err) + } + target := implementation{ + Name: "impl-07", + Directory: "/e4/implementations/impl-07", + Port: 5306, + } + for _, seed := range []string{"1", "42"} { + arguments := campaignArguments(configuration, target, seed) + if got := argumentValue(arguments, "--seeds"); got != seed { + t.Errorf("--seeds: got %q, want %q", got, seed) + } + bundle := argumentValue(arguments, "--bundle-id") + parsed, err := url.Parse(bundle) + if err != nil { + t.Fatalf("--bundle-id %q: %v", bundle, err) + } + if got := parsed.Query().Get("seed"); got != seed { + t.Errorf( + "served URL seed: got %q, want %q (from %q)", + got, + seed, + bundle, + ) + } + if parsed.Host != "localhost:"+strconv.Itoa(target.Port) { + t.Errorf( + "served host: got %q, want the implementation's own port %d", + parsed.Host, + target.Port, + ) + } + for flagName, want := range map[string]string{ + "--arm": "impl-07", + "--platform": "web", + "--generator": "seeded", + "--max-steps": "400", + "--spec": "/e4/relay.ts", + "--output": filepath.Join("/campaigns/e4", "impl-07", "seed-"+seed), + } { + if got := argumentValue(arguments, flagName); got != want { + t.Errorf("%s: got %q, want %q", flagName, got, want) + } + } + if arguments[len(arguments)-2] != "--" || + arguments[len(arguments)-1] != "--clear-data=false" { + t.Errorf("passthrough flags lost: %v", arguments) + } + } +} + +func argumentValue(arguments []string, name string) string { + index := slices.Index(arguments, name) + if index < 0 || index+1 >= len(arguments) { + return "" + } + return arguments[index+1] +} diff --git a/cmd/internal-tools/implementation-sweep/manifest.go b/cmd/internal-tools/implementation-sweep/manifest.go new file mode 100644 index 0000000..e1f377e --- /dev/null +++ b/cmd/internal-tools/implementation-sweep/manifest.go @@ -0,0 +1,117 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" +) + +const ( + manifestFileName = "sweep.json" + recordsFileName = "implementations.jsonl" + seedPlaceholder = "{seed}" +) + +// plannedImplementation is one implementation the sweep intends to run, with +// the port it is served on and the URL every seed is served at. +type plannedImplementation struct { + Name string `json:"name"` + Directory string `json:"directory"` + Port int `json:"port"` + URLTemplate string `json:"url_template"` +} + +// manifest is sweep.json: what the sweep INTENDED to run, written before the +// first install so a host that dropped an implementation or a seed shows up as +// a missing run rather than as a smaller sample. +type manifest struct { + Generator string `json:"generator"` + Platform string `json:"platform"` + SpecPath string `json:"spec_path"` + MaxSteps int `json:"max_steps"` + DurationMillis int64 `json:"duration_millis"` + Seeds []int64 `json:"seeds"` + Implementations []plannedImplementation `json:"implementations"` + Concurrency int `json:"concurrency"` + Host string `json:"host"` + BunPath string `json:"bun_path"` + CampaignPath string `json:"campaign_path"` + SanderlingPath string `json:"sanderling_path"` + StartedAt time.Time `json:"started_at"` +} + +func buildManifest( + configuration config, + implementations []implementation, + host string, + startedAt time.Time, +) manifest { + planned := make([]plannedImplementation, 0, len(implementations)) + for _, target := range implementations { + planned = append(planned, plannedImplementation{ + Name: target.Name, + Directory: target.Directory, + Port: target.Port, + URLTemplate: servedURL(target.Port, seedPlaceholder), + }) + } + return manifest{ + Generator: generator, + Platform: platform, + SpecPath: configuration.specPath, + MaxSteps: configuration.maxSteps, + DurationMillis: configuration.duration.Milliseconds(), + Seeds: configuration.seeds, + Implementations: planned, + Concurrency: configuration.concurrency, + Host: host, + BunPath: configuration.bunPath, + CampaignPath: configuration.campaignPath, + SanderlingPath: configuration.sanderlingPath, + StartedAt: startedAt, + } +} + +func writeManifest(directory string, value manifest) error { + body, err := json.MarshalIndent(value, "", " ") + if err != nil { + return fmt.Errorf("marshal manifest: %w", err) + } + return os.WriteFile( + filepath.Join(directory, manifestFileName), + append(body, '\n'), + 0o644, + ) +} + +// runRecord is one campaign, which is one implementation at one seed. +type runRecord struct { + Seed int64 `json:"seed"` + URL string `json:"url"` + ExitCode int `json:"exit_code"` + LaunchError string `json:"launch_error,omitempty"` + CampaignDirectory string `json:"campaign_directory"` + MonotonicMillis int64 `json:"monotonic_millis"` +} + +// implementationRecord is one line of implementations.jsonl. FailedStage names +// the step that stopped this implementation, and an implementation that never +// got past install, build or serve carries no runs at all. +type implementationRecord struct { + Name string `json:"implementation"` + Directory string `json:"directory"` + Port int `json:"port"` + FailedStage string `json:"failed_stage,omitempty"` + Error string `json:"error,omitempty"` + StartedAt time.Time `json:"started_at"` + MonotonicMillis int64 `json:"monotonic_millis"` + Runs []runRecord `json:"runs"` +} + +const ( + stageInstall = "install" + stageBuild = "build" + stageServe = "serve" +) diff --git a/cmd/internal-tools/implementation-sweep/serve.go b/cmd/internal-tools/implementation-sweep/serve.go new file mode 100644 index 0000000..666aa2a --- /dev/null +++ b/cmd/internal-tools/implementation-sweep/serve.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "syscall" + "time" +) + +const ( + // serverStartTimeout covers a cold vite start on a host already running + // five other implementations. + serverStartTimeout = 90 * time.Second + serverPollInterval = 250 * time.Millisecond + serverShutdownGrace = 10 * time.Second +) + +// server is one implementation's preview server. It runs in its own process +// group so that stopping it takes the whole vite tree with it: a leaked server +// holds its port, and the next sweep against that implementation would be +// served by the previous build. +type server struct { + command *exec.Cmd + logFile *os.File + exited chan struct{} +} + +func startServer( + ctx context.Context, + configuration config, + target implementation, + logPath string, +) (*server, error) { + logFile, err := os.Create(logPath) + if err != nil { + return nil, err + } + command := exec.CommandContext(ctx, configuration.bunPath, + "run", "preview", "--port", fmt.Sprint(target.Port), "--strictPort") + command.Dir = target.Directory + command.Stdout = logFile + command.Stderr = logFile + command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := command.Start(); err != nil { + logFile.Close() + return nil, err + } + running := &server{ + command: command, + logFile: logFile, + exited: make(chan struct{}), + } + go func() { + command.Wait() + close(running.exited) + }() + return running, nil +} + +// waitReady polls the served page until it answers. A server that exits first +// is reported as such rather than waited on for the full timeout, because the +// usual cause is a port already taken and that answer is in the log. +func (s *server) waitReady(ctx context.Context, url string) error { + client := &http.Client{Timeout: 5 * time.Second} + deadline := time.Now().Add(serverStartTimeout) + for { + select { + case <-s.exited: + return fmt.Errorf("server exited before it answered %s", url) + case <-ctx.Done(): + return ctx.Err() + default: + } + response, err := client.Get(url) + if err == nil { + io.Copy(io.Discard, response.Body) + response.Body.Close() + if response.StatusCode == http.StatusOK { + return nil + } + } + if time.Now().After(deadline) { + return fmt.Errorf( + "server did not answer %s within %s", + url, + serverStartTimeout, + ) + } + time.Sleep(serverPollInterval) + } +} + +func (s *server) stop() { + if s.command.Process != nil { + group := -s.command.Process.Pid + syscall.Kill(group, syscall.SIGTERM) + select { + case <-s.exited: + case <-time.After(serverShutdownGrace): + syscall.Kill(group, syscall.SIGKILL) + <-s.exited + } + } + s.logFile.Close() +} diff --git a/cmd/internal-tools/implementation-sweep/serve_test.go b/cmd/internal-tools/implementation-sweep/serve_test.go new file mode 100644 index 0000000..070935b --- /dev/null +++ b/cmd/internal-tools/implementation-sweep/serve_test.go @@ -0,0 +1,119 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "syscall" + "testing" + "time" +) + +// bunSpawningAServer serves from a child process and then waits, which is the +// shape of `bun run preview`: the port belongs to something below the process +// the sweep started. +const bunSpawningAServer = `#!/bin/sh +port="" +previous="" +for argument in "$@"; do + if [ "$previous" = "--port" ]; then port="$argument"; fi + previous="$argument" +done +SWEEP_TEST_SERVE_PORT="$port" "%[1]s" & +wait +` + +func TestServerStop_TakesTheProcessBelowItWithTheServer(t *testing.T) { + directory := t.TempDir() + testBinary, err := filepath.Abs(os.Args[0]) + if err != nil { + t.Fatal(err) + } + bunPath := writeScript( + t, + filepath.Join(directory, "stub-bun"), + fmt.Sprintf(bunSpawningAServer, testBinary), + ) + port := freePortRange(t, 1) + target := implementation{Name: "impl-01", Directory: directory, Port: port} + + // Background rather than the test context: only stop() may end this + // server, or a leak would be hidden by the context being cancelled. + running, err := startServer( + context.Background(), + config{bunPath: bunPath}, + target, + filepath.Join(directory, "serve.log"), + ) + if err != nil { + t.Fatal(err) + } + t.Cleanup( + func() { syscall.Kill(-running.command.Process.Pid, syscall.SIGKILL) }, + ) + if err := running.waitReady(context.Background(), readinessURL(port)); err != nil { + t.Fatal(err) + } + + running.stop() + + client := &http.Client{Timeout: time.Second} + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + response, err := client.Get(readinessURL(port)) + if err != nil { + return + } + response.Body.Close() + time.Sleep(100 * time.Millisecond) + } + t.Fatalf( + "port %d is still served after stop(): the server below bun outlived the sweep and holds the port", + port, + ) +} + +func TestServerWaitReady_ReportsAServerThatExited(t *testing.T) { + directory := t.TempDir() + bunPath := writeScript( + t, + filepath.Join(directory, "stub-bun"), + "#!/bin/sh\necho 'port is already in use' >&2\nexit 1\n", + ) + port := freePortRange(t, 1) + + running, err := startServer( + context.Background(), + config{bunPath: bunPath}, + implementation{ + Name: "impl-01", + Directory: directory, + Port: port, + }, + filepath.Join(directory, "serve.log"), + ) + if err != nil { + t.Fatal(err) + } + defer running.stop() + + started := time.Now() + err = running.waitReady(context.Background(), readinessURL(port)) + if err == nil { + t.Fatal( + "a server that exited should not be waited on until the start timeout", + ) + } + if elapsed := time.Since(started); elapsed > 30*time.Second { + t.Errorf("waited %s for a server that had already exited", elapsed) + } + log, err := os.ReadFile(filepath.Join(directory, "serve.log")) + if err != nil { + t.Fatal(err) + } + if string(log) == "" { + t.Error("serve.log did not capture why the server exited") + } +} diff --git a/cmd/internal-tools/implementation-sweep/sweep.go b/cmd/internal-tools/implementation-sweep/sweep.go new file mode 100644 index 0000000..defba0e --- /dev/null +++ b/cmd/internal-tools/implementation-sweep/sweep.go @@ -0,0 +1,391 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// implementation is one directory under --implementations, with the port it +// owns for the whole sweep. The port comes from the implementation's position +// in name order rather than from a pool, so the manifest can name the URL every +// run was served from before anything has been served. +type implementation struct { + Name string + Directory string + Port int +} + +const implementationPrefix = "impl-" + +func discoverImplementations( + directory string, + basePort int, +) ([]implementation, error) { + entries, err := os.ReadDir(directory) + if err != nil { + return nil, err + } + var found []implementation + for _, entry := range entries { + if !entry.IsDir() || + !strings.HasPrefix(entry.Name(), implementationPrefix) { + continue + } + found = append(found, implementation{ + Name: entry.Name(), + Directory: filepath.Join(directory, entry.Name()), + }) + } + if len(found) == 0 { + return nil, fmt.Errorf( + "no %s* directories in %s", + implementationPrefix, + directory, + ) + } + sort.Slice( + found, + func(i, j int) bool { return found[i].Name < found[j].Name }, + ) + if basePort+len(found)-1 > 65535 { + return nil, fmt.Errorf( + "--base-port %d leaves no room for %d implementations", + basePort, + len(found), + ) + } + for index := range found { + found[index].Port = basePort + index + } + return found, nil +} + +// resolveBinaries turns bun, campaign and sanderling into absolute paths before +// anything is installed. Each campaign runs from the sweep's own directory +// rather than the implementation's, so a relative --sanderling would otherwise +// resolve against the wrong one, and a binary that is missing altogether has to +// stop the sweep here rather than fail once per implementation and seed. +func resolveBinaries(configuration *config) error { + for name, value := range map[string]*string{ + "--bun": &configuration.bunPath, + "--campaign": &configuration.campaignPath, + "--sanderling": &configuration.sanderlingPath, + } { + resolved, err := exec.LookPath(*value) + if err != nil { + return fmt.Errorf("%s: %w", name, err) + } + absolute, err := filepath.Abs(resolved) + if err != nil { + return fmt.Errorf("%s: %w", name, err) + } + *value = absolute + } + return nil +} + +type sweep struct { + configuration config + stdout io.Writer + records io.Writer + mutex sync.Mutex + stalled int + failedRuns int + totalRuns int +} + +func runSweep( + ctx context.Context, + configuration config, + stdout io.Writer, +) error { + if _, err := os.Stat(filepath.Join(configuration.outputDirectory, manifestFileName)); err == nil { + return fmt.Errorf( + "%s already exists in %s: pick a fresh --output so two sweeps do not share a directory", + manifestFileName, + configuration.outputDirectory, + ) + } + implementations, err := discoverImplementations( + configuration.implementationsDirectory, + configuration.basePort, + ) + if err != nil { + return err + } + if err := resolveBinaries(&configuration); err != nil { + return err + } + if _, err := os.Stat(configuration.specPath); err != nil { + return fmt.Errorf("--spec: %w", err) + } + if err := os.MkdirAll(configuration.outputDirectory, 0o755); err != nil { + return fmt.Errorf("create sweep dir: %w", err) + } + host, _ := os.Hostname() + if err := writeManifest(configuration.outputDirectory, buildManifest(configuration, implementations, host, time.Now().UTC())); err != nil { + return fmt.Errorf("write %s: %w", manifestFileName, err) + } + + recordsFile, err := os.OpenFile( + filepath.Join(configuration.outputDirectory, recordsFileName), + os.O_CREATE|os.O_WRONLY|os.O_APPEND, + 0o644, + ) + if err != nil { + return fmt.Errorf("open %s: %w", recordsFileName, err) + } + defer recordsFile.Close() + + running := &sweep{ + configuration: configuration, + stdout: stdout, + records: recordsFile, + } + fmt.Fprintf( + stdout, + "sweep: %d implementations, %d seeds each, %d at a time, %s\n", + len( + implementations, + ), + len(configuration.seeds), + configuration.concurrency, + configuration.outputDirectory, + ) + running.work(ctx, implementations) + + fmt.Fprintf( + stdout, + "sweep complete: %d of %d implementations never ran, %d of %d campaigns failed\n", + running.stalled, + len(implementations), + running.failedRuns, + running.totalRuns, + ) + if running.stalled > 0 || running.failedRuns > 0 { + return fmt.Errorf( + "%d of %d implementations never ran and %d of %d campaigns failed; see %s", + running.stalled, + len(implementations), + running.failedRuns, + running.totalRuns, + recordsFileName, + ) + } + return nil +} + +func (s *sweep) work(ctx context.Context, implementations []implementation) { + queue := make(chan implementation, len(implementations)) + for _, target := range implementations { + queue <- target + } + close(queue) + + workers := min(s.configuration.concurrency, len(implementations)) + var waitGroup sync.WaitGroup + for range workers { + waitGroup.Add(1) + go func() { + defer waitGroup.Done() + for target := range queue { + if ctx.Err() != nil { + return + } + s.report(s.runImplementation(ctx, target)) + } + }() + } + waitGroup.Wait() +} + +// runImplementation carries one implementation from install to its last seed. +// Every failure it can meet is returned in the record: one implementation that +// cannot install, build or serve must not cost the other twenty-three their +// runs. +func (s *sweep) runImplementation( + ctx context.Context, + target implementation, +) (record implementationRecord) { + record = implementationRecord{ + Name: target.Name, + Directory: target.Directory, + Port: target.Port, + StartedAt: time.Now().UTC(), + } + started := time.Now() + defer func() { record.MonotonicMillis = time.Since(started).Milliseconds() }() + + directory := filepath.Join(s.configuration.outputDirectory, target.Name) + if err := os.MkdirAll(directory, 0o755); err != nil { + record.FailedStage = stageInstall + record.Error = err.Error() + return record + } + for _, step := range []struct { + stage string + arguments []string + }{ + {stageInstall, []string{"install"}}, + {stageBuild, []string{"run", "build"}}, + } { + logPath := filepath.Join(directory, step.stage+".log") + exitCode, err := runCommand( + ctx, + target.Directory, + s.configuration.bunPath, + step.arguments, + logPath, + ) + if err != nil { + record.FailedStage = step.stage + record.Error = err.Error() + return record + } + if exitCode != 0 { + record.FailedStage = step.stage + record.Error = fmt.Sprintf( + "bun %s exited %d, see %s", + strings.Join(step.arguments, " "), + exitCode, + logPath, + ) + return record + } + } + + running, err := startServer( + ctx, + s.configuration, + target, + filepath.Join(directory, "serve.log"), + ) + if err != nil { + record.FailedStage = stageServe + record.Error = err.Error() + return record + } + defer running.stop() + if err := running.waitReady(ctx, readinessURL(target.Port)); err != nil { + record.FailedStage = stageServe + record.Error = err.Error() + return record + } + + for _, seed := range s.configuration.seeds { + if ctx.Err() != nil { + return record + } + record.Runs = append(record.Runs, s.runSeed(ctx, target, seed)) + } + return record +} + +func (s *sweep) runSeed( + ctx context.Context, + target implementation, + seed int64, +) (record runRecord) { + seedText := strconv.FormatInt(seed, 10) + directory := campaignDirectory(s.configuration, target, seedText) + record = runRecord{ + Seed: seed, + URL: servedURL(target.Port, seedText), + CampaignDirectory: directory, + } + started := time.Now() + defer func() { record.MonotonicMillis = time.Since(started).Milliseconds() }() + + if err := os.MkdirAll(directory, 0o755); err != nil { + record.ExitCode = -1 + record.LaunchError = err.Error() + return record + } + exitCode, err := runCommand( + ctx, + "", + s.configuration.campaignPath, + campaignArguments( + s.configuration, + target, + seedText, + ), + filepath.Join(directory, "campaign.log"), + ) + record.ExitCode = exitCode + if err != nil { + record.LaunchError = err.Error() + } + return record +} + +// runCommand runs one step of the pipeline with its output in logPath. An +// empty directory keeps the sweep's own working directory, which is what the +// campaign tool gets: it has no reason to run inside an implementation. +func runCommand( + ctx context.Context, + directory, binary string, + arguments []string, + logPath string, +) (int, error) { + logFile, err := os.Create(logPath) + if err != nil { + return -1, err + } + defer logFile.Close() + command := exec.CommandContext(ctx, binary, arguments...) + command.Dir = directory + command.Stdout = logFile + command.Stderr = logFile + err = command.Run() + if err == nil { + return 0, nil + } + var exitError *exec.ExitError + if errors.As(err, &exitError) { + return exitError.ExitCode(), nil + } + return -1, err +} + +func (s *sweep) report(record implementationRecord) { + s.mutex.Lock() + defer s.mutex.Unlock() + if record.FailedStage != "" { + s.stalled++ + } + s.totalRuns += len(record.Runs) + for _, run := range record.Runs { + if run.ExitCode != 0 { + s.failedRuns++ + } + } + if err := json.NewEncoder(s.records).Encode(record); err != nil { + fmt.Fprintf(s.stdout, "warning: %s record: %v\n", record.Name, err) + } + elapsed := time.Duration(record.MonotonicMillis) * time.Millisecond + if record.FailedStage != "" { + fmt.Fprintf(s.stdout, "%s port=%d failed at %s: %s (%s)\n", + record.Name, record.Port, record.FailedStage, record.Error, elapsed) + return + } + failed := 0 + for _, run := range record.Runs { + if run.ExitCode != 0 { + failed++ + } + } + fmt.Fprintf(s.stdout, "%s port=%d campaigns=%d failed=%d elapsed=%s\n", + record.Name, record.Port, len(record.Runs), failed, elapsed) +} diff --git a/cmd/internal-tools/implementation-sweep/sweep_test.go b/cmd/internal-tools/implementation-sweep/sweep_test.go new file mode 100644 index 0000000..f0e3521 --- /dev/null +++ b/cmd/internal-tools/implementation-sweep/sweep_test.go @@ -0,0 +1,114 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestDiscoverImplementations_NameOrderAndOnePortEach(t *testing.T) { + directory := t.TempDir() + for _, name := range []string{"impl-03", "impl-01", "impl-10", "impl-02", "scaffold", ".DS_Store"} { + if err := os.MkdirAll(filepath.Join(directory, name), 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(directory, "impl-notes.md"), []byte("not a directory"), 0o644); err != nil { + t.Fatal(err) + } + + found, err := discoverImplementations(directory, 5300) + if err != nil { + t.Fatal(err) + } + want := []implementation{ + {Name: "impl-01", Port: 5300}, + {Name: "impl-02", Port: 5301}, + {Name: "impl-03", Port: 5302}, + {Name: "impl-10", Port: 5303}, + } + if len(found) != len(want) { + t.Fatalf( + "got %d implementations, want %d: %v", + len(found), + len(want), + found, + ) + } + for index, target := range found { + if target.Name != want[index].Name || target.Port != want[index].Port { + t.Errorf( + "position %d: got %s on %d, want %s on %d", + index, + target.Name, + target.Port, + want[index].Name, + want[index].Port, + ) + } + if target.Directory != filepath.Join(directory, want[index].Name) { + t.Errorf("%s directory: got %q", target.Name, target.Directory) + } + } +} + +func TestDiscoverImplementations_EmptyDirectoryIsRefused(t *testing.T) { + _, err := discoverImplementations(t.TempDir(), 5300) + if err == nil || !strings.Contains(err.Error(), "no impl-* directories") { + t.Fatalf("got %v, want a refusal naming impl-*", err) + } +} + +// A binary that is not there fails once, before anything is installed, rather +// than twenty-four times after the sweep has spent its build time. +func TestRunSweep_StopsBeforeItInstallsAnythingWhenABinaryIsMissing( + t *testing.T, +) { + implementations := t.TempDir() + if err := os.MkdirAll(filepath.Join(implementations, "impl-01"), 0o755); err != nil { + t.Fatal(err) + } + output := filepath.Join(t.TempDir(), "campaigns") + configuration := config{ + implementationsDirectory: implementations, + outputDirectory: output, + basePort: 5300, + concurrency: 1, + bunPath: "bun", + campaignPath: "campaign-that-is-not-installed", + sanderlingPath: "sanderling", + } + err := runSweep(t.Context(), configuration, os.Stdout) + if err == nil || !strings.Contains(err.Error(), "--campaign") { + t.Fatalf("got %v, want the missing campaign binary named", err) + } + if _, err := os.Stat(output); !os.IsNotExist(err) { + t.Errorf( + "the sweep created %s before it checked it could run: %v", + output, + err, + ) + } +} + +func TestRunSweep_RefusesADirectoryThatAlreadyHoldsASweep(t *testing.T) { + implementations := t.TempDir() + if err := os.MkdirAll(filepath.Join(implementations, "impl-01"), 0o755); err != nil { + t.Fatal(err) + } + output := t.TempDir() + if err := os.WriteFile(filepath.Join(output, manifestFileName), []byte("{}"), 0o644); err != nil { + t.Fatal(err) + } + configuration := config{ + implementationsDirectory: implementations, + outputDirectory: output, + basePort: 5300, + concurrency: 1, + } + if err := runSweep(t.Context(), configuration, os.Stdout); err == nil || + !strings.Contains(err.Error(), "already exists") { + t.Fatalf("got %v, want a refusal to reuse the directory", err) + } +}