diff --git a/cmd/internal-tools/implementation-sweep/end_to_end_test.go b/cmd/internal-tools/implementation-sweep/end_to_end_test.go
new file mode 100644
index 0000000..3817520
--- /dev/null
+++ b/cmd/internal-tools/implementation-sweep/end_to_end_test.go
@@ -0,0 +1,482 @@
+package main
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "math/rand"
+ "net"
+ "net/http"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+)
+
+// The test binary doubles as the stub implementation's server and as the
+// fetcher the stub campaign uses, so the sweep drives a real preview process
+// over a real port and the served URL is answered by a real HTTP server.
+func TestMain(m *testing.M) {
+ switch {
+ case os.Getenv("SWEEP_TEST_SERVE_PORT") != "":
+ serveUntilKilled(os.Getenv("SWEEP_TEST_SERVE_PORT"))
+ case os.Getenv("SWEEP_TEST_FETCH_URL") != "":
+ recordFetch(
+ os.Getenv("SWEEP_TEST_FETCH_URL"),
+ os.Getenv("SWEEP_TEST_FETCH_LOG"),
+ )
+ default:
+ os.Exit(m.Run())
+ }
+}
+
+func serveUntilKilled(port string) {
+ handler := http.HandlerFunc(
+ func(writer http.ResponseWriter, request *http.Request) {
+ fmt.Fprintf(writer, "%s %s", port, request.URL.RequestURI())
+ },
+ )
+ http.ListenAndServe("localhost:"+port, handler)
+}
+
+func recordFetch(url, logPath string) {
+ line := ""
+ response, err := http.Get(url)
+ if err != nil {
+ line = fmt.Sprintf("%s -> error %v\n", url, err)
+ } else {
+ body, _ := io.ReadAll(response.Body)
+ response.Body.Close()
+ line = fmt.Sprintf("%s -> %d %s\n", url, response.StatusCode, body)
+ }
+ logFile, err := os.OpenFile(
+ logPath,
+ os.O_CREATE|os.O_WRONLY|os.O_APPEND,
+ 0o644,
+ )
+ if err != nil {
+ return
+ }
+ logFile.WriteString(line)
+ logFile.Close()
+}
+
+// stubBun answers install, fails to build impl-02, and serves the preview from
+// the test binary on the port it was given.
+const stubBun = `#!/bin/sh
+echo "$PWD $*" >> "%[1]s"
+if [ "$1" = "run" ] && [ "$2" = "build" ]; then
+ case "$PWD" in *impl-02) echo "TS2322: type error" >&2; exit 1 ;; esac
+ exit 0
+fi
+if [ "$1" = "run" ] && [ "$2" = "preview" ]; then
+ port=""
+ previous=""
+ for argument in "$@"; do
+ if [ "$previous" = "--port" ]; then port="$argument"; fi
+ previous="$argument"
+ done
+ SWEEP_TEST_SERVE_PORT="$port" exec "%[2]s"
+fi
+exit 0
+`
+
+// stubCampaign records the argv it was handed and whether the sweep manifest
+// was already on disk when it ran, fetches the URL it was told to drive, writes
+// the campaign directory the real tool would write, and fails impl-03 seed 4.
+const stubCampaign = `#!/bin/sh
+output=""
+seed=""
+url=""
+previous=""
+for argument in "$@"; do
+ case "$previous" in
+ --output) output="$argument" ;;
+ --seeds) seed="$argument" ;;
+ --bundle-id) url="$argument" ;;
+ esac
+ previous="$argument"
+done
+manifest=missing
+if [ -f "%[1]s" ]; then manifest=present; fi
+echo "manifest=$manifest argv: $*" >> "%[2]s"
+SWEEP_TEST_FETCH_URL="$url" SWEEP_TEST_FETCH_LOG="%[3]s" "%[4]s"
+mkdir -p "$output"
+printf '{"arm":"stub","seeds":[%%s]}\n' "$seed" > "$output/campaign.json"
+echo "stub campaign seed=$seed url=$url"
+case "$output" in
+ *impl-03/seed-4) exit 1 ;;
+esac
+exit 0
+`
+
+func TestRun_EndToEndAgainstStubBunAndCampaign(t *testing.T) {
+ root := t.TempDir()
+ implementations := filepath.Join(root, "implementations")
+ for _, name := range []string{"impl-01", "impl-02", "impl-03"} {
+ if err := os.MkdirAll(filepath.Join(implementations, name), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ }
+ specPath := filepath.Join(root, "relay.ts")
+ if err := os.WriteFile(specPath, []byte("export const properties = [];\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ output := filepath.Join(root, "campaigns")
+ bunLog := filepath.Join(root, "bun.log")
+ campaignLog := filepath.Join(root, "campaign.log")
+ fetchLog := filepath.Join(root, "fetch.log")
+ testBinary, err := filepath.Abs(os.Args[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+ bunPath := writeScript(
+ t,
+ filepath.Join(root, "stub-bun"),
+ fmt.Sprintf(stubBun, bunLog, testBinary),
+ )
+ campaignPath := writeScript(
+ t,
+ filepath.Join(root, "stub-campaign"),
+ fmt.Sprintf(
+ stubCampaign,
+ filepath.Join(output, manifestFileName),
+ campaignLog,
+ fetchLog,
+ testBinary,
+ ),
+ )
+ sanderlingPath := writeScript(
+ t,
+ filepath.Join(root, "stub-sanderling"),
+ "#!/bin/sh\nexit 0\n",
+ )
+ basePort := freePortRange(t, 3)
+
+ var stdout bytes.Buffer
+ err = run([]string{
+ "--implementations", implementations,
+ "--spec", specPath,
+ "--seeds", "4-5",
+ "--max-steps", "40",
+ "--duration", "30s",
+ "--concurrency", "2",
+ "--base-port", fmt.Sprint(basePort),
+ "--output", output,
+ "--bun", bunPath,
+ "--campaign", campaignPath,
+ "--sanderling", sanderlingPath,
+ }, &stdout, io.Discard)
+ if err == nil ||
+ !strings.Contains(err.Error(), "1 of 3 implementations never ran") {
+ t.Fatalf(
+ "expected the failed build and the failed campaign to be reported, got %v",
+ err,
+ )
+ }
+
+ var recorded manifest
+ manifestBody, err := os.ReadFile(filepath.Join(output, manifestFileName))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := json.Unmarshal(manifestBody, &recorded); err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Equal(recorded.Seeds, []int64{4, 5}) {
+ t.Errorf("intended seeds: got %v", recorded.Seeds)
+ }
+ if len(recorded.Implementations) != 3 {
+ t.Fatalf("intended implementations: got %v", recorded.Implementations)
+ }
+ for index, planned := range recorded.Implementations {
+ wantPort := basePort + index
+ if planned.Port != wantPort {
+ t.Errorf(
+ "%s port: got %d, want %d",
+ planned.Name,
+ planned.Port,
+ wantPort,
+ )
+ }
+ wantURL := fmt.Sprintf("http://localhost:%d/?seed={seed}", wantPort)
+ if planned.URLTemplate != wantURL {
+ t.Errorf(
+ "%s url template: got %q, want %q",
+ planned.Name,
+ planned.URLTemplate,
+ wantURL,
+ )
+ }
+ }
+ if recorded.Generator != "seeded" || recorded.MaxSteps != 40 {
+ t.Errorf(
+ "manifest generator/budget: got %q/%d",
+ recorded.Generator,
+ recorded.MaxSteps,
+ )
+ }
+
+ // impl-02 fails its build, so the two implementations either side of it
+ // still have to reach the campaign tool with both seeds.
+ campaignLines := readLines(t, campaignLog)
+ if len(campaignLines) != 4 {
+ t.Fatalf(
+ "campaign invocations: got %d, want 4:\n%s",
+ len(campaignLines),
+ strings.Join(campaignLines, "\n"),
+ )
+ }
+ seen := map[string]bool{}
+ for _, line := range campaignLines {
+ if !strings.HasPrefix(line, "manifest=present") {
+ t.Errorf(
+ "a campaign ran before the sweep manifest was written: %q",
+ line,
+ )
+ }
+ arguments := strings.Fields(strings.SplitN(line, "argv: ", 2)[1])
+ arm := argumentValue(arguments, "--arm")
+ seed := argumentValue(arguments, "--seeds")
+ bundle := argumentValue(arguments, "--bundle-id")
+ port := basePort + slices.Index(
+ []string{"impl-01", "impl-02", "impl-03"},
+ arm,
+ )
+ wantBundle := fmt.Sprintf("http://localhost:%d/?seed=%s", port, seed)
+ if bundle != wantBundle {
+ t.Errorf(
+ "%s seed %s: bundle id %q, want %q",
+ arm,
+ seed,
+ bundle,
+ wantBundle,
+ )
+ }
+ if got := argumentValue(arguments, "--output"); got != filepath.Join(
+ output,
+ arm,
+ "seed-"+seed,
+ ) {
+ t.Errorf("%s seed %s: campaign output %q", arm, seed, got)
+ }
+ if got := argumentValue(arguments, "--sanderling"); got != sanderlingPath {
+ t.Errorf("%s seed %s: sanderling path %q", arm, seed, got)
+ }
+ seen[arm+"/"+seed] = true
+ }
+ for _, want := range []string{"impl-01/4", "impl-01/5", "impl-03/4", "impl-03/5"} {
+ if !seen[want] {
+ t.Errorf("%s never reached the campaign tool", want)
+ }
+ }
+
+ // What the served page actually saw: the right port for the
+ // implementation, carrying the same seed the campaign was given.
+ fetched := readLines(t, fetchLog)
+ for _, want := range []string{
+ fmt.Sprintf("http://localhost:%d/?seed=4 -> 200 %d /?seed=4", basePort, basePort),
+ fmt.Sprintf("http://localhost:%d/?seed=5 -> 200 %d /?seed=5", basePort, basePort),
+ fmt.Sprintf("http://localhost:%d/?seed=4 -> 200 %d /?seed=4", basePort+2, basePort+2),
+ fmt.Sprintf("http://localhost:%d/?seed=5 -> 200 %d /?seed=5", basePort+2, basePort+2),
+ } {
+ if !slices.Contains(fetched, want) {
+ t.Errorf(
+ "the served page never saw %q:\n%s",
+ want,
+ strings.Join(fetched, "\n"),
+ )
+ }
+ }
+
+ records := readRecords(t, filepath.Join(output, recordsFileName))
+ if len(records) != 3 {
+ t.Fatalf("implementation records: got %d, want 3", len(records))
+ }
+ byName := map[string]implementationRecord{}
+ for _, record := range records {
+ byName[record.Name] = record
+ }
+ failed := byName["impl-02"]
+ if failed.FailedStage != stageBuild || len(failed.Runs) != 0 {
+ t.Errorf(
+ "impl-02: got stage %q with %d runs, want a build failure and no runs",
+ failed.FailedStage,
+ len(failed.Runs),
+ )
+ }
+ if !strings.Contains(failed.Error, "build.log") {
+ t.Errorf("impl-02 error should point at its log: %q", failed.Error)
+ }
+ buildLog, err := os.ReadFile(
+ filepath.Join(output, "impl-02", stageBuild+".log"),
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(buildLog), "TS2322") {
+ t.Errorf("impl-02 build log lost the compiler error: %q", buildLog)
+ }
+ for _, name := range []string{"impl-01", "impl-03"} {
+ record := byName[name]
+ if record.FailedStage != "" || len(record.Runs) != 2 {
+ t.Errorf(
+ "%s: stage %q with %d runs, want no failure and 2 runs",
+ name,
+ record.FailedStage,
+ len(record.Runs),
+ )
+ }
+ if record.MonotonicMillis <= 0 {
+ t.Errorf(
+ "%s took %d ms, so nothing timed how long it worked",
+ name,
+ record.MonotonicMillis,
+ )
+ }
+ for _, run := range record.Runs {
+ if run.MonotonicMillis <= 0 {
+ t.Errorf(
+ "%s seed %d took %d ms, so nothing timed the campaign",
+ name,
+ run.Seed,
+ run.MonotonicMillis,
+ )
+ }
+ }
+ }
+ if exit := byName["impl-03"].Runs[0].ExitCode; exit != 1 {
+ t.Errorf("impl-03 seed 4 exit code: got %d, want 1", exit)
+ }
+ if exit := byName["impl-03"].Runs[1].ExitCode; exit != 0 {
+ t.Errorf(
+ "impl-03 seed 5 ran after seed 4 failed and should have exited 0, got %d",
+ exit,
+ )
+ }
+
+ for _, name := range []string{"impl-01", "impl-03"} {
+ for _, seed := range []string{"4", "5"} {
+ directory := filepath.Join(output, name, "seed-"+seed)
+ if _, err := os.Stat(filepath.Join(directory, "campaign.json")); err != nil {
+ t.Errorf(
+ "%s seed %s: no campaign directory: %v",
+ name,
+ seed,
+ err,
+ )
+ }
+ log, err := os.ReadFile(filepath.Join(directory, "campaign.log"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(log), "stub campaign seed="+seed) {
+ t.Errorf(
+ "%s seed %s: campaign output was not captured: %q",
+ name,
+ seed,
+ log,
+ )
+ }
+ }
+ }
+
+ installed := readLines(t, bunLog)
+ for _, name := range []string{"impl-01", "impl-02", "impl-03"} {
+ if !slices.Contains(
+ installed,
+ filepath.Join(implementations, name)+" install",
+ ) {
+ t.Errorf(
+ "%s was never installed:\n%s",
+ name,
+ strings.Join(installed, "\n"),
+ )
+ }
+ }
+
+ // Every preview server the sweep started is gone with it: a leaked one
+ // holds its port, and the next sweep would be served by the old build.
+ client := &http.Client{Timeout: 2 * time.Second}
+ for _, port := range []int{basePort, basePort + 2} {
+ if response, err := client.Get(readinessURL(port)); err == nil {
+ response.Body.Close()
+ t.Errorf("port %d is still served after the sweep finished", port)
+ }
+ }
+ if !strings.Contains(stdout.String(), "failed at build") {
+ t.Errorf(
+ "progress output does not name the build failure: %q",
+ stdout.String(),
+ )
+ }
+}
+
+func writeScript(t *testing.T, path, body string) string {
+ t.Helper()
+ if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ return path
+}
+
+func readLines(t *testing.T, path string) []string {
+ t.Helper()
+ body, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var lines []string
+ scanner := bufio.NewScanner(strings.NewReader(string(body)))
+ for scanner.Scan() {
+ if line := strings.TrimSpace(scanner.Text()); line != "" {
+ lines = append(lines, line)
+ }
+ }
+ return lines
+}
+
+func readRecords(t *testing.T, path string) []implementationRecord {
+ t.Helper()
+ var records []implementationRecord
+ for _, line := range readLines(t, path) {
+ var record implementationRecord
+ if err := json.Unmarshal([]byte(line), &record); err != nil {
+ t.Fatalf("%s: %v", line, err)
+ }
+ records = append(records, record)
+ }
+ return records
+}
+
+// freePortRange finds count consecutive free ports, which is what the sweep
+// hands out: one port per implementation from --base-port upwards.
+func freePortRange(t *testing.T, count int) int {
+ t.Helper()
+ for range 100 {
+ base := 20000 + rand.Intn(20000)
+ if portsAreFree(base, count) {
+ return base
+ }
+ }
+ t.Fatalf("no run of %d free ports", count)
+ return 0
+}
+
+func portsAreFree(base, count int) bool {
+ for offset := range count {
+ listener, err := net.Listen(
+ "tcp",
+ fmt.Sprintf("localhost:%d", base+offset),
+ )
+ if err != nil {
+ return false
+ }
+ listener.Close()
+ }
+ return true
+}
diff --git a/cmd/internal-tools/implementation-sweep/main.go b/cmd/internal-tools/implementation-sweep/main.go
new file mode 100644
index 0000000..7cf2502
--- /dev/null
+++ b/cmd/internal-tools/implementation-sweep/main.go
@@ -0,0 +1,284 @@
+// Command implementation-sweep runs one identical campaign against every model
+// implementation of a single requirement. It installs, builds and serves each
+// implementation on its own port, then hands the campaign tool the same seed
+// slice, the same step budget and the same generator for all of them, so a
+// difference between implementations is not a difference in exploration.
+package main
+
+import (
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "os/signal"
+ "path/filepath"
+ "strconv"
+ "syscall"
+ "time"
+
+ "github.com/priyanshujain/sanderling/internal/seedspec"
+)
+
+// The generator and the platform are fixed rather than exposed: the
+// pre-registration runs the seeded policy against a served web build, and a
+// sweep that could quietly run something else records a comparison nobody made.
+const (
+ generator = "seeded"
+ platform = "web"
+)
+
+// defaultConcurrency is how many implementations are built, served and swept at
+// once. fleet.md measured eight concurrent web campaigns clean at about 1.1 GB
+// resident each, parallel efficiency 0.83 at eight against 0.87 at six, and a
+// knee at twelve to sixteen, on a contended laptop it says to re-measure before
+// trusting anything above eight. Six sits at the better efficiency, costs about
+// 7 GB of a 64 GB host, and leaves the Android emulator farm that shares that
+// host its four to six slots. Each worker here also carries a vite server the
+// fleet measurement did not include.
+const defaultConcurrency = 6
+
+// defaultBasePort is the first port handed out. Vite's own defaults are 5173
+// and 4173, so a sweep starting here does not collide with a dev server someone
+// left running.
+const defaultBasePort = 5300
+
+type config struct {
+ implementationsDirectory string
+ specPath string
+ outputDirectory string
+ seeds []int64
+ maxSteps int
+ duration time.Duration
+ concurrency int
+ basePort int
+ bunPath string
+ campaignPath string
+ sanderlingPath string
+ extraArguments []string
+}
+
+const usage = `implementation-sweep runs one seeded campaign per model implementation.
+
+Usage:
+ implementation-sweep --implementations
--spec --seeds
+ --max-steps --output [flags]
+ [-- ]
+
+Each impl-* directory under --implementations is installed, built and served on
+its own port, and every one is swept with the same seeds and the same step
+budget. An implementation that fails to install, build or serve is recorded and
+the sweep moves on to the next one.
+
+Everything after a bare -- reaches every sanderling test call through the
+campaign tool.
+`
+
+func parseArguments(arguments []string, stderr io.Writer) (config, error) {
+ flagSet := flag.NewFlagSet("implementation-sweep", flag.ContinueOnError)
+ flagSet.SetOutput(stderr)
+ flagSet.Usage = func() {
+ fmt.Fprint(stderr, usage)
+ flagSet.PrintDefaults()
+ }
+ var configuration config
+ var seedSpecification string
+ flagSet.StringVar(
+ &configuration.implementationsDirectory,
+ "implementations",
+ "",
+ "directory holding impl-01 to impl-NN (required)",
+ )
+ flagSet.StringVar(
+ &configuration.specPath,
+ "spec",
+ "",
+ "path to the property set every implementation is run against (required)",
+ )
+ flagSet.StringVar(
+ &seedSpecification,
+ "seeds",
+ "",
+ "seeds every implementation runs: ranges and lists, e.g. 1-10,20 (required)",
+ )
+ flagSet.IntVar(
+ &configuration.maxSteps,
+ "max-steps",
+ 0,
+ "per-run step budget, identical across implementations (required, must be positive)",
+ )
+ flagSet.DurationVar(
+ &configuration.duration,
+ "duration",
+ 5*time.Minute,
+ "per-run wall-clock ceiling passed to each campaign",
+ )
+ flagSet.IntVar(
+ &configuration.concurrency,
+ "concurrency",
+ defaultConcurrency,
+ "implementations built, served and swept at once",
+ )
+ flagSet.IntVar(
+ &configuration.basePort,
+ "base-port",
+ defaultBasePort,
+ "first port served; each implementation takes the next one in name order",
+ )
+ flagSet.StringVar(
+ &configuration.outputDirectory,
+ "output",
+ "",
+ "campaign tree to create (required)",
+ )
+ flagSet.StringVar(
+ &configuration.bunPath,
+ "bun",
+ "bun",
+ "bun binary that installs, builds and serves each implementation",
+ )
+ flagSet.StringVar(
+ &configuration.campaignPath,
+ "campaign",
+ "campaign",
+ "campaign binary to invoke per implementation and seed",
+ )
+ flagSet.StringVar(
+ &configuration.sanderlingPath,
+ "sanderling",
+ "sanderling",
+ "sanderling binary each campaign invokes",
+ )
+ if err := flagSet.Parse(arguments); err != nil {
+ return config{}, err
+ }
+ configuration.extraArguments = flagSet.Args()
+
+ for name, value := range map[string]string{
+ "--implementations": configuration.implementationsDirectory,
+ "--spec": configuration.specPath,
+ "--seeds": seedSpecification,
+ "--output": configuration.outputDirectory,
+ } {
+ if value == "" {
+ return config{}, fmt.Errorf("%s is required", name)
+ }
+ }
+ if configuration.maxSteps <= 0 {
+ return config{}, fmt.Errorf(
+ "--max-steps must be positive: every implementation needs the same step budget",
+ )
+ }
+ if configuration.duration <= 0 {
+ return config{}, fmt.Errorf(
+ "--duration must be positive: %s",
+ configuration.duration,
+ )
+ }
+ if configuration.concurrency <= 0 {
+ return config{}, fmt.Errorf(
+ "--concurrency must be positive: %d",
+ configuration.concurrency,
+ )
+ }
+ if configuration.basePort < 1024 || configuration.basePort > 65535 {
+ return config{}, fmt.Errorf(
+ "--base-port %d is outside 1024-65535",
+ configuration.basePort,
+ )
+ }
+ seeds, err := seedspec.Parse(seedSpecification)
+ if err != nil {
+ return config{}, fmt.Errorf("--seeds: %w", err)
+ }
+ configuration.seeds = seeds
+ for name, value := range map[string]*string{
+ "--implementations": &configuration.implementationsDirectory,
+ "--spec": &configuration.specPath,
+ "--output": &configuration.outputDirectory,
+ } {
+ absolute, err := filepath.Abs(*value)
+ if err != nil {
+ return config{}, fmt.Errorf("%s: %w", name, err)
+ }
+ *value = absolute
+ }
+ return configuration, nil
+}
+
+// servedURL is the one place a seed becomes a URL. The same seed is also handed
+// to the campaign as --seeds, which reaches sanderling as --seed and fixes the
+// exploration, while the scaffold reads ?seed= and fixes the latency and the
+// outcome of every send. A violation replays only when both carry the same
+// number, so both come from the seed argument here and never from two flags.
+func servedURL(port int, seed string) string {
+ return fmt.Sprintf("http://localhost:%d/?seed=%s", port, seed)
+}
+
+func readinessURL(port int) string {
+ return fmt.Sprintf("http://localhost:%d/", port)
+}
+
+func campaignDirectory(
+ configuration config,
+ target implementation,
+ seed string,
+) string {
+ return filepath.Join(
+ configuration.outputDirectory,
+ target.Name,
+ "seed-"+seed,
+ )
+}
+
+// campaignArguments builds one campaign invocation. The seed is a string
+// because it lands in two arguments, --seeds and the ?seed= of --bundle-id,
+// and passing it once keeps them from drifting apart.
+func campaignArguments(
+ configuration config,
+ target implementation,
+ seed string,
+) []string {
+ arguments := []string{
+ "--spec", configuration.specPath,
+ "--bundle-id", servedURL(target.Port, seed),
+ "--platform", platform,
+ "--arm", target.Name,
+ "--generator", generator,
+ "--max-steps", strconv.Itoa(configuration.maxSteps),
+ "--duration", configuration.duration.String(),
+ "--seeds", seed,
+ "--sanderling", configuration.sanderlingPath,
+ "--output", campaignDirectory(configuration, target, seed),
+ }
+ if len(configuration.extraArguments) > 0 {
+ arguments = append(arguments, "--")
+ arguments = append(arguments, configuration.extraArguments...)
+ }
+ return arguments
+}
+
+func run(arguments []string, stdout, stderr io.Writer) error {
+ configuration, err := parseArguments(arguments, stderr)
+ if err != nil {
+ return err
+ }
+ ctx, cancel := signal.NotifyContext(
+ context.Background(),
+ os.Interrupt,
+ syscall.SIGTERM,
+ )
+ defer cancel()
+ return runSweep(ctx, configuration, stdout)
+}
+
+func main() {
+ if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil {
+ if errors.Is(err, flag.ErrHelp) {
+ return
+ }
+ fmt.Fprintf(os.Stderr, "error: %v\n", err)
+ os.Exit(1)
+ }
+}
diff --git a/cmd/internal-tools/implementation-sweep/main_test.go b/cmd/internal-tools/implementation-sweep/main_test.go
new file mode 100644
index 0000000..f9d4d6d
--- /dev/null
+++ b/cmd/internal-tools/implementation-sweep/main_test.go
@@ -0,0 +1,214 @@
+package main
+
+import (
+ "io"
+ "net/url"
+ "path/filepath"
+ "slices"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+)
+
+func baseArguments() []string {
+ return []string{
+ "--implementations", "/e4/implementations",
+ "--spec", "/e4/relay.ts",
+ "--seeds", "1-3",
+ "--max-steps", "400",
+ "--output", "/campaigns/e4",
+ }
+}
+
+func TestParseArguments_DefaultsAndSeeds(t *testing.T) {
+ configuration, err := parseArguments(baseArguments(), io.Discard)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Equal(configuration.seeds, []int64{1, 2, 3}) {
+ t.Errorf("seeds: got %v", configuration.seeds)
+ }
+ if configuration.concurrency != defaultConcurrency {
+ t.Errorf(
+ "concurrency default: got %d, want %d",
+ configuration.concurrency,
+ defaultConcurrency,
+ )
+ }
+ if configuration.basePort != defaultBasePort {
+ t.Errorf(
+ "base port default: got %d, want %d",
+ configuration.basePort,
+ defaultBasePort,
+ )
+ }
+ if configuration.duration != 5*time.Minute {
+ t.Errorf("duration default: got %s", configuration.duration)
+ }
+ for name, got := range map[string]string{
+ "bun": configuration.bunPath,
+ "campaign": configuration.campaignPath,
+ "sanderling": configuration.sanderlingPath,
+ } {
+ if got != name {
+ t.Errorf("%s path default: got %q", name, got)
+ }
+ }
+}
+
+func TestParseArguments_Rejections(t *testing.T) {
+ cases := []struct {
+ name string
+ arguments []string
+ want string
+ }{
+ {
+ "missing implementations",
+ []string{
+ "--spec",
+ "s",
+ "--seeds",
+ "1",
+ "--max-steps",
+ "10",
+ "--output",
+ "o",
+ },
+ "--implementations is required",
+ },
+ {
+ "missing spec",
+ []string{
+ "--implementations",
+ "i",
+ "--seeds",
+ "1",
+ "--max-steps",
+ "10",
+ "--output",
+ "o",
+ },
+ "--spec is required",
+ },
+ {
+ "missing output",
+ []string{
+ "--implementations",
+ "i",
+ "--spec",
+ "s",
+ "--seeds",
+ "1",
+ "--max-steps",
+ "10",
+ },
+ "--output is required",
+ },
+ {
+ "zero max steps",
+ append(baseArguments(), "--max-steps", "0"),
+ "--max-steps must be positive",
+ },
+ {
+ "zero concurrency",
+ append(baseArguments(), "--concurrency", "0"),
+ "--concurrency must be positive",
+ },
+ {
+ "privileged base port",
+ append(baseArguments(), "--base-port", "80"),
+ "outside 1024-65535",
+ },
+ {
+ "seed zero",
+ append(baseArguments(), "--seeds", "0,1"),
+ "not reproducible",
+ },
+ }
+ for _, testCase := range cases {
+ _, err := parseArguments(testCase.arguments, io.Discard)
+ if err == nil {
+ t.Errorf("%s: expected error", testCase.name)
+ continue
+ }
+ if !strings.Contains(err.Error(), testCase.want) {
+ t.Errorf(
+ "%s: got %q, want it to contain %q",
+ testCase.name,
+ err,
+ testCase.want,
+ )
+ }
+ }
+}
+
+// The seed reaches two independent things, the campaign's own seed and the
+// scaffold's failure stream, and a replay reproduces neither unless they carry
+// the same number.
+func TestCampaignArguments_OneSeedReachesBothTheCampaignAndTheURL(
+ t *testing.T,
+) {
+ configuration, err := parseArguments(
+ append(baseArguments(), "--", "--clear-data=false"),
+ io.Discard,
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ target := implementation{
+ Name: "impl-07",
+ Directory: "/e4/implementations/impl-07",
+ Port: 5306,
+ }
+ for _, seed := range []string{"1", "42"} {
+ arguments := campaignArguments(configuration, target, seed)
+ if got := argumentValue(arguments, "--seeds"); got != seed {
+ t.Errorf("--seeds: got %q, want %q", got, seed)
+ }
+ bundle := argumentValue(arguments, "--bundle-id")
+ parsed, err := url.Parse(bundle)
+ if err != nil {
+ t.Fatalf("--bundle-id %q: %v", bundle, err)
+ }
+ if got := parsed.Query().Get("seed"); got != seed {
+ t.Errorf(
+ "served URL seed: got %q, want %q (from %q)",
+ got,
+ seed,
+ bundle,
+ )
+ }
+ if parsed.Host != "localhost:"+strconv.Itoa(target.Port) {
+ t.Errorf(
+ "served host: got %q, want the implementation's own port %d",
+ parsed.Host,
+ target.Port,
+ )
+ }
+ for flagName, want := range map[string]string{
+ "--arm": "impl-07",
+ "--platform": "web",
+ "--generator": "seeded",
+ "--max-steps": "400",
+ "--spec": "/e4/relay.ts",
+ "--output": filepath.Join("/campaigns/e4", "impl-07", "seed-"+seed),
+ } {
+ if got := argumentValue(arguments, flagName); got != want {
+ t.Errorf("%s: got %q, want %q", flagName, got, want)
+ }
+ }
+ if arguments[len(arguments)-2] != "--" ||
+ arguments[len(arguments)-1] != "--clear-data=false" {
+ t.Errorf("passthrough flags lost: %v", arguments)
+ }
+ }
+}
+
+func argumentValue(arguments []string, name string) string {
+ index := slices.Index(arguments, name)
+ if index < 0 || index+1 >= len(arguments) {
+ return ""
+ }
+ return arguments[index+1]
+}
diff --git a/cmd/internal-tools/implementation-sweep/manifest.go b/cmd/internal-tools/implementation-sweep/manifest.go
new file mode 100644
index 0000000..e1f377e
--- /dev/null
+++ b/cmd/internal-tools/implementation-sweep/manifest.go
@@ -0,0 +1,117 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "time"
+)
+
+const (
+ manifestFileName = "sweep.json"
+ recordsFileName = "implementations.jsonl"
+ seedPlaceholder = "{seed}"
+)
+
+// plannedImplementation is one implementation the sweep intends to run, with
+// the port it is served on and the URL every seed is served at.
+type plannedImplementation struct {
+ Name string `json:"name"`
+ Directory string `json:"directory"`
+ Port int `json:"port"`
+ URLTemplate string `json:"url_template"`
+}
+
+// manifest is sweep.json: what the sweep INTENDED to run, written before the
+// first install so a host that dropped an implementation or a seed shows up as
+// a missing run rather than as a smaller sample.
+type manifest struct {
+ Generator string `json:"generator"`
+ Platform string `json:"platform"`
+ SpecPath string `json:"spec_path"`
+ MaxSteps int `json:"max_steps"`
+ DurationMillis int64 `json:"duration_millis"`
+ Seeds []int64 `json:"seeds"`
+ Implementations []plannedImplementation `json:"implementations"`
+ Concurrency int `json:"concurrency"`
+ Host string `json:"host"`
+ BunPath string `json:"bun_path"`
+ CampaignPath string `json:"campaign_path"`
+ SanderlingPath string `json:"sanderling_path"`
+ StartedAt time.Time `json:"started_at"`
+}
+
+func buildManifest(
+ configuration config,
+ implementations []implementation,
+ host string,
+ startedAt time.Time,
+) manifest {
+ planned := make([]plannedImplementation, 0, len(implementations))
+ for _, target := range implementations {
+ planned = append(planned, plannedImplementation{
+ Name: target.Name,
+ Directory: target.Directory,
+ Port: target.Port,
+ URLTemplate: servedURL(target.Port, seedPlaceholder),
+ })
+ }
+ return manifest{
+ Generator: generator,
+ Platform: platform,
+ SpecPath: configuration.specPath,
+ MaxSteps: configuration.maxSteps,
+ DurationMillis: configuration.duration.Milliseconds(),
+ Seeds: configuration.seeds,
+ Implementations: planned,
+ Concurrency: configuration.concurrency,
+ Host: host,
+ BunPath: configuration.bunPath,
+ CampaignPath: configuration.campaignPath,
+ SanderlingPath: configuration.sanderlingPath,
+ StartedAt: startedAt,
+ }
+}
+
+func writeManifest(directory string, value manifest) error {
+ body, err := json.MarshalIndent(value, "", " ")
+ if err != nil {
+ return fmt.Errorf("marshal manifest: %w", err)
+ }
+ return os.WriteFile(
+ filepath.Join(directory, manifestFileName),
+ append(body, '\n'),
+ 0o644,
+ )
+}
+
+// runRecord is one campaign, which is one implementation at one seed.
+type runRecord struct {
+ Seed int64 `json:"seed"`
+ URL string `json:"url"`
+ ExitCode int `json:"exit_code"`
+ LaunchError string `json:"launch_error,omitempty"`
+ CampaignDirectory string `json:"campaign_directory"`
+ MonotonicMillis int64 `json:"monotonic_millis"`
+}
+
+// implementationRecord is one line of implementations.jsonl. FailedStage names
+// the step that stopped this implementation, and an implementation that never
+// got past install, build or serve carries no runs at all.
+type implementationRecord struct {
+ Name string `json:"implementation"`
+ Directory string `json:"directory"`
+ Port int `json:"port"`
+ FailedStage string `json:"failed_stage,omitempty"`
+ Error string `json:"error,omitempty"`
+ StartedAt time.Time `json:"started_at"`
+ MonotonicMillis int64 `json:"monotonic_millis"`
+ Runs []runRecord `json:"runs"`
+}
+
+const (
+ stageInstall = "install"
+ stageBuild = "build"
+ stageServe = "serve"
+)
diff --git a/cmd/internal-tools/implementation-sweep/serve.go b/cmd/internal-tools/implementation-sweep/serve.go
new file mode 100644
index 0000000..666aa2a
--- /dev/null
+++ b/cmd/internal-tools/implementation-sweep/serve.go
@@ -0,0 +1,109 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "os/exec"
+ "syscall"
+ "time"
+)
+
+const (
+ // serverStartTimeout covers a cold vite start on a host already running
+ // five other implementations.
+ serverStartTimeout = 90 * time.Second
+ serverPollInterval = 250 * time.Millisecond
+ serverShutdownGrace = 10 * time.Second
+)
+
+// server is one implementation's preview server. It runs in its own process
+// group so that stopping it takes the whole vite tree with it: a leaked server
+// holds its port, and the next sweep against that implementation would be
+// served by the previous build.
+type server struct {
+ command *exec.Cmd
+ logFile *os.File
+ exited chan struct{}
+}
+
+func startServer(
+ ctx context.Context,
+ configuration config,
+ target implementation,
+ logPath string,
+) (*server, error) {
+ logFile, err := os.Create(logPath)
+ if err != nil {
+ return nil, err
+ }
+ command := exec.CommandContext(ctx, configuration.bunPath,
+ "run", "preview", "--port", fmt.Sprint(target.Port), "--strictPort")
+ command.Dir = target.Directory
+ command.Stdout = logFile
+ command.Stderr = logFile
+ command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+ if err := command.Start(); err != nil {
+ logFile.Close()
+ return nil, err
+ }
+ running := &server{
+ command: command,
+ logFile: logFile,
+ exited: make(chan struct{}),
+ }
+ go func() {
+ command.Wait()
+ close(running.exited)
+ }()
+ return running, nil
+}
+
+// waitReady polls the served page until it answers. A server that exits first
+// is reported as such rather than waited on for the full timeout, because the
+// usual cause is a port already taken and that answer is in the log.
+func (s *server) waitReady(ctx context.Context, url string) error {
+ client := &http.Client{Timeout: 5 * time.Second}
+ deadline := time.Now().Add(serverStartTimeout)
+ for {
+ select {
+ case <-s.exited:
+ return fmt.Errorf("server exited before it answered %s", url)
+ case <-ctx.Done():
+ return ctx.Err()
+ default:
+ }
+ response, err := client.Get(url)
+ if err == nil {
+ io.Copy(io.Discard, response.Body)
+ response.Body.Close()
+ if response.StatusCode == http.StatusOK {
+ return nil
+ }
+ }
+ if time.Now().After(deadline) {
+ return fmt.Errorf(
+ "server did not answer %s within %s",
+ url,
+ serverStartTimeout,
+ )
+ }
+ time.Sleep(serverPollInterval)
+ }
+}
+
+func (s *server) stop() {
+ if s.command.Process != nil {
+ group := -s.command.Process.Pid
+ syscall.Kill(group, syscall.SIGTERM)
+ select {
+ case <-s.exited:
+ case <-time.After(serverShutdownGrace):
+ syscall.Kill(group, syscall.SIGKILL)
+ <-s.exited
+ }
+ }
+ s.logFile.Close()
+}
diff --git a/cmd/internal-tools/implementation-sweep/serve_test.go b/cmd/internal-tools/implementation-sweep/serve_test.go
new file mode 100644
index 0000000..070935b
--- /dev/null
+++ b/cmd/internal-tools/implementation-sweep/serve_test.go
@@ -0,0 +1,119 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "os"
+ "path/filepath"
+ "syscall"
+ "testing"
+ "time"
+)
+
+// bunSpawningAServer serves from a child process and then waits, which is the
+// shape of `bun run preview`: the port belongs to something below the process
+// the sweep started.
+const bunSpawningAServer = `#!/bin/sh
+port=""
+previous=""
+for argument in "$@"; do
+ if [ "$previous" = "--port" ]; then port="$argument"; fi
+ previous="$argument"
+done
+SWEEP_TEST_SERVE_PORT="$port" "%[1]s" &
+wait
+`
+
+func TestServerStop_TakesTheProcessBelowItWithTheServer(t *testing.T) {
+ directory := t.TempDir()
+ testBinary, err := filepath.Abs(os.Args[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+ bunPath := writeScript(
+ t,
+ filepath.Join(directory, "stub-bun"),
+ fmt.Sprintf(bunSpawningAServer, testBinary),
+ )
+ port := freePortRange(t, 1)
+ target := implementation{Name: "impl-01", Directory: directory, Port: port}
+
+ // Background rather than the test context: only stop() may end this
+ // server, or a leak would be hidden by the context being cancelled.
+ running, err := startServer(
+ context.Background(),
+ config{bunPath: bunPath},
+ target,
+ filepath.Join(directory, "serve.log"),
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(
+ func() { syscall.Kill(-running.command.Process.Pid, syscall.SIGKILL) },
+ )
+ if err := running.waitReady(context.Background(), readinessURL(port)); err != nil {
+ t.Fatal(err)
+ }
+
+ running.stop()
+
+ client := &http.Client{Timeout: time.Second}
+ deadline := time.Now().Add(5 * time.Second)
+ for time.Now().Before(deadline) {
+ response, err := client.Get(readinessURL(port))
+ if err != nil {
+ return
+ }
+ response.Body.Close()
+ time.Sleep(100 * time.Millisecond)
+ }
+ t.Fatalf(
+ "port %d is still served after stop(): the server below bun outlived the sweep and holds the port",
+ port,
+ )
+}
+
+func TestServerWaitReady_ReportsAServerThatExited(t *testing.T) {
+ directory := t.TempDir()
+ bunPath := writeScript(
+ t,
+ filepath.Join(directory, "stub-bun"),
+ "#!/bin/sh\necho 'port is already in use' >&2\nexit 1\n",
+ )
+ port := freePortRange(t, 1)
+
+ running, err := startServer(
+ context.Background(),
+ config{bunPath: bunPath},
+ implementation{
+ Name: "impl-01",
+ Directory: directory,
+ Port: port,
+ },
+ filepath.Join(directory, "serve.log"),
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer running.stop()
+
+ started := time.Now()
+ err = running.waitReady(context.Background(), readinessURL(port))
+ if err == nil {
+ t.Fatal(
+ "a server that exited should not be waited on until the start timeout",
+ )
+ }
+ if elapsed := time.Since(started); elapsed > 30*time.Second {
+ t.Errorf("waited %s for a server that had already exited", elapsed)
+ }
+ log, err := os.ReadFile(filepath.Join(directory, "serve.log"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(log) == "" {
+ t.Error("serve.log did not capture why the server exited")
+ }
+}
diff --git a/cmd/internal-tools/implementation-sweep/sweep.go b/cmd/internal-tools/implementation-sweep/sweep.go
new file mode 100644
index 0000000..defba0e
--- /dev/null
+++ b/cmd/internal-tools/implementation-sweep/sweep.go
@@ -0,0 +1,391 @@
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+)
+
+// implementation is one directory under --implementations, with the port it
+// owns for the whole sweep. The port comes from the implementation's position
+// in name order rather than from a pool, so the manifest can name the URL every
+// run was served from before anything has been served.
+type implementation struct {
+ Name string
+ Directory string
+ Port int
+}
+
+const implementationPrefix = "impl-"
+
+func discoverImplementations(
+ directory string,
+ basePort int,
+) ([]implementation, error) {
+ entries, err := os.ReadDir(directory)
+ if err != nil {
+ return nil, err
+ }
+ var found []implementation
+ for _, entry := range entries {
+ if !entry.IsDir() ||
+ !strings.HasPrefix(entry.Name(), implementationPrefix) {
+ continue
+ }
+ found = append(found, implementation{
+ Name: entry.Name(),
+ Directory: filepath.Join(directory, entry.Name()),
+ })
+ }
+ if len(found) == 0 {
+ return nil, fmt.Errorf(
+ "no %s* directories in %s",
+ implementationPrefix,
+ directory,
+ )
+ }
+ sort.Slice(
+ found,
+ func(i, j int) bool { return found[i].Name < found[j].Name },
+ )
+ if basePort+len(found)-1 > 65535 {
+ return nil, fmt.Errorf(
+ "--base-port %d leaves no room for %d implementations",
+ basePort,
+ len(found),
+ )
+ }
+ for index := range found {
+ found[index].Port = basePort + index
+ }
+ return found, nil
+}
+
+// resolveBinaries turns bun, campaign and sanderling into absolute paths before
+// anything is installed. Each campaign runs from the sweep's own directory
+// rather than the implementation's, so a relative --sanderling would otherwise
+// resolve against the wrong one, and a binary that is missing altogether has to
+// stop the sweep here rather than fail once per implementation and seed.
+func resolveBinaries(configuration *config) error {
+ for name, value := range map[string]*string{
+ "--bun": &configuration.bunPath,
+ "--campaign": &configuration.campaignPath,
+ "--sanderling": &configuration.sanderlingPath,
+ } {
+ resolved, err := exec.LookPath(*value)
+ if err != nil {
+ return fmt.Errorf("%s: %w", name, err)
+ }
+ absolute, err := filepath.Abs(resolved)
+ if err != nil {
+ return fmt.Errorf("%s: %w", name, err)
+ }
+ *value = absolute
+ }
+ return nil
+}
+
+type sweep struct {
+ configuration config
+ stdout io.Writer
+ records io.Writer
+ mutex sync.Mutex
+ stalled int
+ failedRuns int
+ totalRuns int
+}
+
+func runSweep(
+ ctx context.Context,
+ configuration config,
+ stdout io.Writer,
+) error {
+ if _, err := os.Stat(filepath.Join(configuration.outputDirectory, manifestFileName)); err == nil {
+ return fmt.Errorf(
+ "%s already exists in %s: pick a fresh --output so two sweeps do not share a directory",
+ manifestFileName,
+ configuration.outputDirectory,
+ )
+ }
+ implementations, err := discoverImplementations(
+ configuration.implementationsDirectory,
+ configuration.basePort,
+ )
+ if err != nil {
+ return err
+ }
+ if err := resolveBinaries(&configuration); err != nil {
+ return err
+ }
+ if _, err := os.Stat(configuration.specPath); err != nil {
+ return fmt.Errorf("--spec: %w", err)
+ }
+ if err := os.MkdirAll(configuration.outputDirectory, 0o755); err != nil {
+ return fmt.Errorf("create sweep dir: %w", err)
+ }
+ host, _ := os.Hostname()
+ if err := writeManifest(configuration.outputDirectory, buildManifest(configuration, implementations, host, time.Now().UTC())); err != nil {
+ return fmt.Errorf("write %s: %w", manifestFileName, err)
+ }
+
+ recordsFile, err := os.OpenFile(
+ filepath.Join(configuration.outputDirectory, recordsFileName),
+ os.O_CREATE|os.O_WRONLY|os.O_APPEND,
+ 0o644,
+ )
+ if err != nil {
+ return fmt.Errorf("open %s: %w", recordsFileName, err)
+ }
+ defer recordsFile.Close()
+
+ running := &sweep{
+ configuration: configuration,
+ stdout: stdout,
+ records: recordsFile,
+ }
+ fmt.Fprintf(
+ stdout,
+ "sweep: %d implementations, %d seeds each, %d at a time, %s\n",
+ len(
+ implementations,
+ ),
+ len(configuration.seeds),
+ configuration.concurrency,
+ configuration.outputDirectory,
+ )
+ running.work(ctx, implementations)
+
+ fmt.Fprintf(
+ stdout,
+ "sweep complete: %d of %d implementations never ran, %d of %d campaigns failed\n",
+ running.stalled,
+ len(implementations),
+ running.failedRuns,
+ running.totalRuns,
+ )
+ if running.stalled > 0 || running.failedRuns > 0 {
+ return fmt.Errorf(
+ "%d of %d implementations never ran and %d of %d campaigns failed; see %s",
+ running.stalled,
+ len(implementations),
+ running.failedRuns,
+ running.totalRuns,
+ recordsFileName,
+ )
+ }
+ return nil
+}
+
+func (s *sweep) work(ctx context.Context, implementations []implementation) {
+ queue := make(chan implementation, len(implementations))
+ for _, target := range implementations {
+ queue <- target
+ }
+ close(queue)
+
+ workers := min(s.configuration.concurrency, len(implementations))
+ var waitGroup sync.WaitGroup
+ for range workers {
+ waitGroup.Add(1)
+ go func() {
+ defer waitGroup.Done()
+ for target := range queue {
+ if ctx.Err() != nil {
+ return
+ }
+ s.report(s.runImplementation(ctx, target))
+ }
+ }()
+ }
+ waitGroup.Wait()
+}
+
+// runImplementation carries one implementation from install to its last seed.
+// Every failure it can meet is returned in the record: one implementation that
+// cannot install, build or serve must not cost the other twenty-three their
+// runs.
+func (s *sweep) runImplementation(
+ ctx context.Context,
+ target implementation,
+) (record implementationRecord) {
+ record = implementationRecord{
+ Name: target.Name,
+ Directory: target.Directory,
+ Port: target.Port,
+ StartedAt: time.Now().UTC(),
+ }
+ started := time.Now()
+ defer func() { record.MonotonicMillis = time.Since(started).Milliseconds() }()
+
+ directory := filepath.Join(s.configuration.outputDirectory, target.Name)
+ if err := os.MkdirAll(directory, 0o755); err != nil {
+ record.FailedStage = stageInstall
+ record.Error = err.Error()
+ return record
+ }
+ for _, step := range []struct {
+ stage string
+ arguments []string
+ }{
+ {stageInstall, []string{"install"}},
+ {stageBuild, []string{"run", "build"}},
+ } {
+ logPath := filepath.Join(directory, step.stage+".log")
+ exitCode, err := runCommand(
+ ctx,
+ target.Directory,
+ s.configuration.bunPath,
+ step.arguments,
+ logPath,
+ )
+ if err != nil {
+ record.FailedStage = step.stage
+ record.Error = err.Error()
+ return record
+ }
+ if exitCode != 0 {
+ record.FailedStage = step.stage
+ record.Error = fmt.Sprintf(
+ "bun %s exited %d, see %s",
+ strings.Join(step.arguments, " "),
+ exitCode,
+ logPath,
+ )
+ return record
+ }
+ }
+
+ running, err := startServer(
+ ctx,
+ s.configuration,
+ target,
+ filepath.Join(directory, "serve.log"),
+ )
+ if err != nil {
+ record.FailedStage = stageServe
+ record.Error = err.Error()
+ return record
+ }
+ defer running.stop()
+ if err := running.waitReady(ctx, readinessURL(target.Port)); err != nil {
+ record.FailedStage = stageServe
+ record.Error = err.Error()
+ return record
+ }
+
+ for _, seed := range s.configuration.seeds {
+ if ctx.Err() != nil {
+ return record
+ }
+ record.Runs = append(record.Runs, s.runSeed(ctx, target, seed))
+ }
+ return record
+}
+
+func (s *sweep) runSeed(
+ ctx context.Context,
+ target implementation,
+ seed int64,
+) (record runRecord) {
+ seedText := strconv.FormatInt(seed, 10)
+ directory := campaignDirectory(s.configuration, target, seedText)
+ record = runRecord{
+ Seed: seed,
+ URL: servedURL(target.Port, seedText),
+ CampaignDirectory: directory,
+ }
+ started := time.Now()
+ defer func() { record.MonotonicMillis = time.Since(started).Milliseconds() }()
+
+ if err := os.MkdirAll(directory, 0o755); err != nil {
+ record.ExitCode = -1
+ record.LaunchError = err.Error()
+ return record
+ }
+ exitCode, err := runCommand(
+ ctx,
+ "",
+ s.configuration.campaignPath,
+ campaignArguments(
+ s.configuration,
+ target,
+ seedText,
+ ),
+ filepath.Join(directory, "campaign.log"),
+ )
+ record.ExitCode = exitCode
+ if err != nil {
+ record.LaunchError = err.Error()
+ }
+ return record
+}
+
+// runCommand runs one step of the pipeline with its output in logPath. An
+// empty directory keeps the sweep's own working directory, which is what the
+// campaign tool gets: it has no reason to run inside an implementation.
+func runCommand(
+ ctx context.Context,
+ directory, binary string,
+ arguments []string,
+ logPath string,
+) (int, error) {
+ logFile, err := os.Create(logPath)
+ if err != nil {
+ return -1, err
+ }
+ defer logFile.Close()
+ command := exec.CommandContext(ctx, binary, arguments...)
+ command.Dir = directory
+ command.Stdout = logFile
+ command.Stderr = logFile
+ err = command.Run()
+ if err == nil {
+ return 0, nil
+ }
+ var exitError *exec.ExitError
+ if errors.As(err, &exitError) {
+ return exitError.ExitCode(), nil
+ }
+ return -1, err
+}
+
+func (s *sweep) report(record implementationRecord) {
+ s.mutex.Lock()
+ defer s.mutex.Unlock()
+ if record.FailedStage != "" {
+ s.stalled++
+ }
+ s.totalRuns += len(record.Runs)
+ for _, run := range record.Runs {
+ if run.ExitCode != 0 {
+ s.failedRuns++
+ }
+ }
+ if err := json.NewEncoder(s.records).Encode(record); err != nil {
+ fmt.Fprintf(s.stdout, "warning: %s record: %v\n", record.Name, err)
+ }
+ elapsed := time.Duration(record.MonotonicMillis) * time.Millisecond
+ if record.FailedStage != "" {
+ fmt.Fprintf(s.stdout, "%s port=%d failed at %s: %s (%s)\n",
+ record.Name, record.Port, record.FailedStage, record.Error, elapsed)
+ return
+ }
+ failed := 0
+ for _, run := range record.Runs {
+ if run.ExitCode != 0 {
+ failed++
+ }
+ }
+ fmt.Fprintf(s.stdout, "%s port=%d campaigns=%d failed=%d elapsed=%s\n",
+ record.Name, record.Port, len(record.Runs), failed, elapsed)
+}
diff --git a/cmd/internal-tools/implementation-sweep/sweep_test.go b/cmd/internal-tools/implementation-sweep/sweep_test.go
new file mode 100644
index 0000000..f0e3521
--- /dev/null
+++ b/cmd/internal-tools/implementation-sweep/sweep_test.go
@@ -0,0 +1,114 @@
+package main
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+func TestDiscoverImplementations_NameOrderAndOnePortEach(t *testing.T) {
+ directory := t.TempDir()
+ for _, name := range []string{"impl-03", "impl-01", "impl-10", "impl-02", "scaffold", ".DS_Store"} {
+ if err := os.MkdirAll(filepath.Join(directory, name), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := os.WriteFile(filepath.Join(directory, "impl-notes.md"), []byte("not a directory"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+
+ found, err := discoverImplementations(directory, 5300)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := []implementation{
+ {Name: "impl-01", Port: 5300},
+ {Name: "impl-02", Port: 5301},
+ {Name: "impl-03", Port: 5302},
+ {Name: "impl-10", Port: 5303},
+ }
+ if len(found) != len(want) {
+ t.Fatalf(
+ "got %d implementations, want %d: %v",
+ len(found),
+ len(want),
+ found,
+ )
+ }
+ for index, target := range found {
+ if target.Name != want[index].Name || target.Port != want[index].Port {
+ t.Errorf(
+ "position %d: got %s on %d, want %s on %d",
+ index,
+ target.Name,
+ target.Port,
+ want[index].Name,
+ want[index].Port,
+ )
+ }
+ if target.Directory != filepath.Join(directory, want[index].Name) {
+ t.Errorf("%s directory: got %q", target.Name, target.Directory)
+ }
+ }
+}
+
+func TestDiscoverImplementations_EmptyDirectoryIsRefused(t *testing.T) {
+ _, err := discoverImplementations(t.TempDir(), 5300)
+ if err == nil || !strings.Contains(err.Error(), "no impl-* directories") {
+ t.Fatalf("got %v, want a refusal naming impl-*", err)
+ }
+}
+
+// A binary that is not there fails once, before anything is installed, rather
+// than twenty-four times after the sweep has spent its build time.
+func TestRunSweep_StopsBeforeItInstallsAnythingWhenABinaryIsMissing(
+ t *testing.T,
+) {
+ implementations := t.TempDir()
+ if err := os.MkdirAll(filepath.Join(implementations, "impl-01"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ output := filepath.Join(t.TempDir(), "campaigns")
+ configuration := config{
+ implementationsDirectory: implementations,
+ outputDirectory: output,
+ basePort: 5300,
+ concurrency: 1,
+ bunPath: "bun",
+ campaignPath: "campaign-that-is-not-installed",
+ sanderlingPath: "sanderling",
+ }
+ err := runSweep(t.Context(), configuration, os.Stdout)
+ if err == nil || !strings.Contains(err.Error(), "--campaign") {
+ t.Fatalf("got %v, want the missing campaign binary named", err)
+ }
+ if _, err := os.Stat(output); !os.IsNotExist(err) {
+ t.Errorf(
+ "the sweep created %s before it checked it could run: %v",
+ output,
+ err,
+ )
+ }
+}
+
+func TestRunSweep_RefusesADirectoryThatAlreadyHoldsASweep(t *testing.T) {
+ implementations := t.TempDir()
+ if err := os.MkdirAll(filepath.Join(implementations, "impl-01"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ output := t.TempDir()
+ if err := os.WriteFile(filepath.Join(output, manifestFileName), []byte("{}"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ configuration := config{
+ implementationsDirectory: implementations,
+ outputDirectory: output,
+ basePort: 5300,
+ concurrency: 1,
+ }
+ if err := runSweep(t.Context(), configuration, os.Stdout); err == nil ||
+ !strings.Contains(err.Error(), "already exists") {
+ t.Fatalf("got %v, want a refusal to reuse the directory", err)
+ }
+}