diff --git a/cmd/internal-tools/corpus-sweep/corpus.go b/cmd/internal-tools/corpus-sweep/corpus.go
new file mode 100644
index 0000000..cb8225b
--- /dev/null
+++ b/cmd/internal-tools/corpus-sweep/corpus.go
@@ -0,0 +1,174 @@
+package main
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+)
+
+// The corpus is tastejs/todomvc. Its examples/ directory holds 48
+// implementations of one requirement; five are excluded and the 43 that remain
+// are this experiment's population.
+//
+// The population is named here rather than read from whatever directories
+// happen to exist, so a corpus at the wrong commit fails verifyCorpus instead
+// of quietly sweeping a different sample and reporting it as this one.
+var includedImplementations = []string{
+ "angular-dart", "angular2", "angular2_es2015", "angularjs", "angularjs_require",
+ "aurelia", "backbone", "backbone_marionette", "backbone_require", "binding-scala",
+ "canjs", "canjs_require", "closure", "dijon", "dojo",
+ "duel", "elm", "emberjs", "enyo_backbone", "exoskeleton",
+ "jquery", "js_of_ocaml", "jsblocks", "knockback", "knockoutjs",
+ "knockoutjs_require", "kotlin-react", "lavaca_require", "mithril", "polymer",
+ "ractive", "react", "react-alt", "react-backbone", "reagent",
+ "riotjs", "scalajs-react", "typescript-angular", "typescript-backbone", "typescript-react",
+ "vanilla-es6", "vanillajs", "vue",
+}
+
+// excludedImplementations are the five directories under examples/ that the
+// corpus survey dropped. They are listed rather than merely omitted so
+// verifyCorpus can insist the corpus holds exactly these 48 names: an example
+// added or renamed upstream then stops the sweep rather than silently shrinking
+// or growing the sample.
+var excludedImplementations = []string{
+ "cujo", "emberjs_require", "firebase-angular", "gwt", "react-hooks",
+}
+
+const examplesDirectory = "examples"
+
+// documentPath names the served document for the implementations that do not
+// keep an index.html at the root of their example directory.
+var documentPath = map[string]string{
+ "angular-dart": "examples/angular-dart/web/index.html",
+ "duel": "examples/duel/www/index.html",
+}
+
+func documentFor(name string) string {
+ if override, ok := documentPath[name]; ok {
+ return override
+ }
+ return examplesDirectory + "/" + name + "/index.html"
+}
+
+// implementation is one member of the population, with the port it owns for the
+// whole sweep. The port is what keeps implementations apart: every one is
+// served on its own, so every one is its own web origin and localStorage keeps
+// their records in separate partitions. Four pairs in this corpus write the
+// same key, and one origin between them is one record between them.
+type implementation struct {
+ Name string
+ Document string
+ Port int
+}
+
+func (i implementation) Origin() string {
+ return fmt.Sprintf("http://127.0.0.1:%d", i.Port)
+}
+
+func (i implementation) URL() string {
+ return i.Origin() + "/" + i.Document
+}
+
+// verifyCorpus insists the corpus holds exactly the 48 example directories this
+// population was drawn from. A sweep against a different checkout would still
+// run, and its 43 arms would still be labelled, which is why the check is here
+// and not left to whoever reads the results.
+func verifyCorpus(corpusRoot string) error {
+ entries, err := os.ReadDir(filepath.Join(corpusRoot, examplesDirectory))
+ if err != nil {
+ return fmt.Errorf("--corpus: %w", err)
+ }
+ var found []string
+ for _, entry := range entries {
+ if entry.IsDir() {
+ found = append(found, entry.Name())
+ }
+ }
+ expected := slices.Concat(includedImplementations, excludedImplementations)
+ slices.Sort(expected)
+ slices.Sort(found)
+ if slices.Equal(expected, found) {
+ return nil
+ }
+ var missing, unexpected []string
+ for _, name := range expected {
+ if !slices.Contains(found, name) {
+ missing = append(missing, name)
+ }
+ }
+ for _, name := range found {
+ if !slices.Contains(expected, name) {
+ unexpected = append(unexpected, name)
+ }
+ }
+ return fmt.Errorf(
+ "%s/%s is not the corpus this population was drawn from: missing %v, unexpected %v",
+ corpusRoot,
+ examplesDirectory,
+ missing,
+ unexpected,
+ )
+}
+
+// selectImplementations resolves --implementations against the population. An
+// empty selection is the whole population, which is what a real sweep runs; a
+// named subset is for smoke runs and is recorded in the manifest like any other
+// intent.
+func selectImplementations(selection string) ([]string, error) {
+ if strings.TrimSpace(selection) == "" {
+ return slices.Clone(includedImplementations), nil
+ }
+ var names []string
+ seen := map[string]bool{}
+ for _, part := range strings.Split(selection, ",") {
+ name := strings.TrimSpace(part)
+ if name == "" {
+ return nil, fmt.Errorf("empty implementation in %q", selection)
+ }
+ if !slices.Contains(includedImplementations, name) {
+ return nil, fmt.Errorf(
+ "%q is not one of the %d implementations in this population",
+ name,
+ len(includedImplementations),
+ )
+ }
+ if seen[name] {
+ return nil, fmt.Errorf("duplicate implementation %q", name)
+ }
+ seen[name] = true
+ names = append(names, name)
+ }
+ return names, nil
+}
+
+// planImplementations gives every selected implementation its document and its
+// own port, in population order so the manifest can name the URL each arm was
+// served from before anything has been served.
+func planImplementations(
+ corpusRoot string,
+ names []string,
+ basePort int,
+) ([]implementation, error) {
+ if basePort+len(names)-1 > 65535 {
+ return nil, fmt.Errorf(
+ "--base-port %d leaves no room for %d implementations",
+ basePort,
+ len(names),
+ )
+ }
+ planned := make([]implementation, 0, len(names))
+ for index, name := range names {
+ document := documentFor(name)
+ if _, err := os.Stat(filepath.Join(corpusRoot, filepath.FromSlash(document))); err != nil {
+ return nil, fmt.Errorf("%s: %w", name, err)
+ }
+ planned = append(planned, implementation{
+ Name: name,
+ Document: document,
+ Port: basePort + index,
+ })
+ }
+ return planned, nil
+}
diff --git a/cmd/internal-tools/corpus-sweep/corpus_test.go b/cmd/internal-tools/corpus-sweep/corpus_test.go
new file mode 100644
index 0000000..1d77ceb
--- /dev/null
+++ b/cmd/internal-tools/corpus-sweep/corpus_test.go
@@ -0,0 +1,161 @@
+package main
+
+import (
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+ "testing"
+)
+
+// writeCorpus builds a tree with the shape the sweep expects: every example
+// directory the population was drawn from, each holding the document that
+// implementation is served at.
+func writeCorpus(t *testing.T, names []string) string {
+ t.Helper()
+ root := t.TempDir()
+ for _, name := range names {
+ if err := os.MkdirAll(filepath.Join(root, examplesDirectory, name), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ document := filepath.Join(root, filepath.FromSlash(documentFor(name)))
+ if err := os.MkdirAll(filepath.Dir(document), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ body := "
" + name + ""
+ if err := os.WriteFile(document, []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return root
+}
+
+func wholeCorpus(t *testing.T) string {
+ t.Helper()
+ return writeCorpus(
+ t,
+ slices.Concat(includedImplementations, excludedImplementations),
+ )
+}
+
+func TestPopulation_IsFortyThreeNamesDisjointFromTheExclusions(t *testing.T) {
+ if len(includedImplementations) != 43 {
+ t.Errorf(
+ "population size: got %d, want 43",
+ len(includedImplementations),
+ )
+ }
+ seen := map[string]bool{}
+ for _, name := range includedImplementations {
+ if seen[name] {
+ t.Errorf("%q appears twice in the population", name)
+ }
+ seen[name] = true
+ }
+ for _, name := range excludedImplementations {
+ if seen[name] {
+ t.Errorf("%q is both included and excluded", name)
+ }
+ }
+}
+
+func TestVerifyCorpus_RejectsATreeThatIsNotThisCorpus(t *testing.T) {
+ if err := verifyCorpus(wholeCorpus(t)); err != nil {
+ t.Fatalf(
+ "the corpus this population was drawn from should verify: %v",
+ err,
+ )
+ }
+
+ short := writeCorpus(
+ t,
+ slices.Concat(includedImplementations[1:], excludedImplementations),
+ )
+ err := verifyCorpus(short)
+ if err == nil {
+ t.Fatal(
+ "a corpus missing an implementation swept 42 arms and reported them as 43",
+ )
+ }
+ if !strings.Contains(err.Error(), includedImplementations[0]) {
+ t.Errorf("the error should name what is missing: %v", err)
+ }
+
+ extra := writeCorpus(
+ t,
+ slices.Concat(
+ includedImplementations,
+ excludedImplementations,
+ []string{"svelte"},
+ ),
+ )
+ err = verifyCorpus(extra)
+ if err == nil {
+ t.Fatal(
+ "a corpus with an implementation this population never drew from verified",
+ )
+ }
+ if !strings.Contains(err.Error(), "svelte") {
+ t.Errorf("the error should name what is unexpected: %v", err)
+ }
+}
+
+func TestDocumentFor_ServesTheOverriddenPathForImplementationsWithoutARootIndex(
+ t *testing.T,
+) {
+ for name, want := range map[string]string{
+ "angular-dart": "examples/angular-dart/web/index.html",
+ "duel": "examples/duel/www/index.html",
+ "vanillajs": "examples/vanillajs/index.html",
+ "react": "examples/react/index.html",
+ } {
+ if got := documentFor(name); got != want {
+ t.Errorf("%s document: got %q, want %q", name, got, want)
+ }
+ }
+}
+
+func TestPlanImplementations_RefusesAnImplementationWhoseDocumentIsMissing(
+ t *testing.T,
+) {
+ root := wholeCorpus(t)
+ if err := os.Remove(filepath.Join(root, filepath.FromSlash(documentFor("duel")))); err != nil {
+ t.Fatal(err)
+ }
+ _, err := planImplementations(root, []string{"duel"}, 5400)
+ if err == nil {
+ t.Fatal(
+ "an implementation whose document is missing would be swept as a 404 page",
+ )
+ }
+ if !strings.Contains(err.Error(), "duel") {
+ t.Errorf("the error should name the implementation: %v", err)
+ }
+}
+
+func TestSelectImplementations_DefaultsToThePopulationAndRejectsNamesOutsideIt(
+ t *testing.T,
+) {
+ all, err := selectImplementations("")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Equal(all, includedImplementations) {
+ t.Errorf(
+ "an empty selection should be the whole population, got %d names",
+ len(all),
+ )
+ }
+ subset, err := selectImplementations("react, angular2_es2015")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Equal(subset, []string{"react", "angular2_es2015"}) {
+ t.Errorf("subset: got %v", subset)
+ }
+ for _, selection := range []string{"cujo", "svelte", "react,react"} {
+ if _, err := selectImplementations(selection); err == nil {
+ t.Errorf("selection %q should have been rejected", selection)
+ }
+ }
+}
diff --git a/cmd/internal-tools/corpus-sweep/end_to_end_test.go b/cmd/internal-tools/corpus-sweep/end_to_end_test.go
new file mode 100644
index 0000000..e7179ed
--- /dev/null
+++ b/cmd/internal-tools/corpus-sweep/end_to_end_test.go
@@ -0,0 +1,468 @@
+package main
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "math/rand"
+ "net"
+ "net/http"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+)
+
+// The test binary doubles as the fetcher the stub campaign uses, so the URL a
+// campaign is handed is really requested while the sweep is serving, and what
+// came back is on disk for the test to read.
+func TestMain(m *testing.M) {
+ if url := os.Getenv("CORPUS_SWEEP_TEST_FETCH_URL"); url != "" {
+ recordFetch(url, os.Getenv("CORPUS_SWEEP_TEST_FETCH_LOG"))
+ return
+ }
+ os.Exit(m.Run())
+}
+
+func recordFetch(url, logPath string) {
+ line := ""
+ response, err := http.Get(url)
+ if err != nil {
+ line = fmt.Sprintf("%s -> error %v\n", url, err)
+ } else {
+ body, _ := io.ReadAll(response.Body)
+ response.Body.Close()
+ line = fmt.Sprintf("%s -> %d %s\n", url, response.StatusCode, body)
+ }
+ logFile, err := os.OpenFile(
+ logPath,
+ os.O_CREATE|os.O_WRONLY|os.O_APPEND,
+ 0o644,
+ )
+ if err != nil {
+ return
+ }
+ logFile.WriteString(line)
+ logFile.Close()
+}
+
+// stubCampaign records the argv it was handed and whether the sweep manifest
+// was already on disk when it ran, fetches the URL it was told to drive, writes
+// the campaign directory the real tool would write, and fails one seed.
+const stubCampaign = `#!/bin/sh
+output=""
+seed=""
+url=""
+previous=""
+for argument in "$@"; do
+ case "$previous" in
+ --output) output="$argument" ;;
+ --seeds) seed="$argument" ;;
+ --bundle-id) url="$argument" ;;
+ esac
+ previous="$argument"
+done
+manifest=missing
+if [ -f "%[1]s" ]; then manifest=present; fi
+echo "manifest=$manifest argv: $*" >> "%[2]s"
+CORPUS_SWEEP_TEST_FETCH_URL="$url" CORPUS_SWEEP_TEST_FETCH_LOG="%[3]s" "%[4]s"
+mkdir -p "$output"
+printf '{"seeds":[%%s]}\n' "$seed" > "$output/campaign.json"
+echo "stub campaign seed=$seed url=$url"
+case "$output" in
+ *angular2_es2015/seed-5) exit 1 ;;
+esac
+exit 0
+`
+
+func TestRun_EndToEndAgainstAServedCorpusAndAStubCampaign(t *testing.T) {
+ root := t.TempDir()
+ corpus := wholeCorpus(t)
+ // dojo's document is served but cannot be read, so it stalls at serve and
+ // the two implementations either side of it still have to reach the
+ // campaign tool with both seeds.
+ unreadable := filepath.Join(corpus, filepath.FromSlash(documentFor("dojo")))
+ if err := os.Chmod(unreadable, 0o000); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { os.Chmod(unreadable, 0o644) })
+
+ specPath := filepath.Join(root, "todo.ts")
+ if err := os.WriteFile(specPath, []byte("export const properties = [];\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ output := filepath.Join(root, "campaigns")
+ campaignLog := filepath.Join(root, "campaign.log")
+ fetchLog := filepath.Join(root, "fetch.log")
+ testBinary, err := filepath.Abs(os.Args[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+ campaignPath := writeScript(
+ t,
+ filepath.Join(root, "stub-campaign"),
+ fmt.Sprintf(
+ stubCampaign,
+ filepath.Join(output, manifestFileName),
+ campaignLog,
+ fetchLog,
+ testBinary,
+ ),
+ )
+ sanderlingPath := writeScript(
+ t,
+ filepath.Join(root, "stub-sanderling"),
+ "#!/bin/sh\nexit 0\n",
+ )
+ selected := []string{"angular2", "angular2_es2015", "dojo"}
+ basePort := freePortRange(t, len(selected))
+
+ var stdout bytes.Buffer
+ err = run([]string{
+ "--corpus", corpus,
+ "--spec", specPath,
+ "--implementations", strings.Join(selected, ","),
+ "--seeds", "4-5",
+ "--max-steps", "40",
+ "--duration", "30s",
+ "--concurrency", "2",
+ "--base-port", fmt.Sprint(basePort),
+ "--output", output,
+ "--campaign", campaignPath,
+ "--sanderling", sanderlingPath,
+ }, &stdout, io.Discard)
+ if err == nil ||
+ !strings.Contains(err.Error(), "1 of 3 implementations never ran") {
+ t.Fatalf(
+ "expected the stalled implementation and the failed campaign to be reported, got %v",
+ err,
+ )
+ }
+
+ recorded := readManifest(t, filepath.Join(output, manifestFileName))
+ if !slices.Equal(recorded.Seeds, []int64{4, 5}) {
+ t.Errorf("intended seeds: got %v", recorded.Seeds)
+ }
+ if len(recorded.Implementations) != len(selected) {
+ t.Fatalf("intended implementations: got %v", recorded.Implementations)
+ }
+ for index, planned := range recorded.Implementations {
+ if planned.Name != selected[index] {
+ t.Errorf(
+ "implementation %d: got %q, want %q",
+ index,
+ planned.Name,
+ selected[index],
+ )
+ }
+ wantPort := basePort + index
+ wantURL := fmt.Sprintf(
+ "http://127.0.0.1:%d/examples/%s/index.html",
+ wantPort,
+ planned.Name,
+ )
+ if planned.Port != wantPort || planned.URL != wantURL {
+ t.Errorf(
+ "%s: port %d url %q, want %d %q",
+ planned.Name,
+ planned.Port,
+ planned.URL,
+ wantPort,
+ wantURL,
+ )
+ }
+ }
+ if recorded.Generator != "seeded" || recorded.Platform != "web" ||
+ recorded.MaxSteps != 40 {
+ t.Errorf(
+ "manifest generator/platform/budget: %q/%q/%d",
+ recorded.Generator,
+ recorded.Platform,
+ recorded.MaxSteps,
+ )
+ }
+ if recorded.CorpusRoot != corpus {
+ t.Errorf(
+ "manifest corpus root: got %q, want %q",
+ recorded.CorpusRoot,
+ corpus,
+ )
+ }
+
+ campaignLines := readLines(t, campaignLog)
+ if len(campaignLines) != 4 {
+ t.Fatalf(
+ "campaign invocations: got %d, want 4:\n%s",
+ len(campaignLines),
+ strings.Join(campaignLines, "\n"),
+ )
+ }
+ seen := map[string]bool{}
+ for _, line := range campaignLines {
+ if !strings.HasPrefix(line, "manifest=present") {
+ t.Errorf(
+ "a campaign ran before the sweep manifest was written: %q",
+ line,
+ )
+ }
+ arguments := strings.Fields(strings.SplitN(line, "argv: ", 2)[1])
+ arm := argumentValue(arguments, "--arm")
+ seed := argumentValue(arguments, "--seeds")
+ port := basePort + slices.Index(selected, arm)
+ wantBundle := fmt.Sprintf(
+ "http://127.0.0.1:%d/examples/%s/index.html",
+ port,
+ arm,
+ )
+ if got := argumentValue(arguments, "--bundle-id"); got != wantBundle {
+ t.Errorf(
+ "%s seed %s: bundle id %q, want %q",
+ arm,
+ seed,
+ got,
+ wantBundle,
+ )
+ }
+ if got := argumentValue(arguments, "--output"); got != filepath.Join(
+ output,
+ arm,
+ "seed-"+seed,
+ ) {
+ t.Errorf("%s seed %s: campaign output %q", arm, seed, got)
+ }
+ if got := argumentValue(arguments, "--sanderling"); got != sanderlingPath {
+ t.Errorf("%s seed %s: sanderling path %q", arm, seed, got)
+ }
+ seen[arm+"/"+seed] = true
+ }
+ for _, want := range []string{"angular2/4", "angular2/5", "angular2_es2015/4", "angular2_es2015/5"} {
+ if !seen[want] {
+ t.Errorf("%s never reached the campaign tool", want)
+ }
+ }
+
+ // What the served page actually returned: each port answered with its own
+ // implementation's document, so no arm was driven against another's.
+ fetched := readLines(t, fetchLog)
+ for index, name := range []string{"angular2", "angular2_es2015"} {
+ want := fmt.Sprintf(
+ "http://127.0.0.1:%d/examples/%s/index.html -> 200 %s",
+ basePort+index,
+ name,
+ name,
+ )
+ matched := 0
+ for _, line := range fetched {
+ if strings.HasPrefix(line, want) {
+ matched++
+ }
+ }
+ if matched != 2 {
+ t.Errorf(
+ "%s was served its own document %d times, want 2:\n%s",
+ name,
+ matched,
+ strings.Join(fetched, "\n"),
+ )
+ }
+ }
+
+ records := readRecords(t, filepath.Join(output, recordsFileName))
+ if len(records) != 3 {
+ t.Fatalf("implementation records: got %d, want 3", len(records))
+ }
+ byName := map[string]implementationRecord{}
+ for _, record := range records {
+ byName[record.Name] = record
+ }
+ stalled := byName["dojo"]
+ if stalled.FailedStage != stageServe || len(stalled.Runs) != 0 {
+ t.Errorf(
+ "dojo: stage %q with %d runs, want a serve failure and no runs",
+ stalled.FailedStage,
+ len(stalled.Runs),
+ )
+ }
+ for _, name := range []string{"angular2", "angular2_es2015"} {
+ record := byName[name]
+ if record.FailedStage != "" || len(record.Runs) != 2 {
+ t.Errorf(
+ "%s: stage %q with %d runs, want no failure and 2 runs",
+ name,
+ record.FailedStage,
+ len(record.Runs),
+ )
+ }
+ if record.MonotonicMillis <= 0 {
+ t.Errorf(
+ "%s took %d ms, so nothing timed how long it worked",
+ name,
+ record.MonotonicMillis,
+ )
+ }
+ }
+ if exit := byName["angular2_es2015"].Runs[1].ExitCode; exit != 1 {
+ t.Errorf("angular2_es2015 seed 5 exit code: got %d, want 1", exit)
+ }
+ if exit := byName["angular2_es2015"].Runs[0].ExitCode; exit != 0 {
+ t.Errorf("angular2_es2015 seed 4 exit code: got %d, want 0", exit)
+ }
+
+ for _, name := range []string{"angular2", "angular2_es2015"} {
+ for _, seed := range []string{"4", "5"} {
+ directory := filepath.Join(output, name, "seed-"+seed)
+ if _, err := os.Stat(filepath.Join(directory, "campaign.json")); err != nil {
+ t.Errorf(
+ "%s seed %s: no campaign directory: %v",
+ name,
+ seed,
+ err,
+ )
+ }
+ log, err := os.ReadFile(filepath.Join(directory, "campaign.log"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(log), "stub campaign seed="+seed) {
+ t.Errorf(
+ "%s seed %s: campaign output was not captured: %q",
+ name,
+ seed,
+ log,
+ )
+ }
+ }
+ }
+
+ client := &http.Client{Timeout: 2 * time.Second}
+ for offset := range selected {
+ if response, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/", basePort+offset)); err == nil {
+ response.Body.Close()
+ t.Errorf(
+ "port %d is still served after the sweep finished",
+ basePort+offset,
+ )
+ }
+ }
+ if !strings.Contains(stdout.String(), "failed at serve") {
+ t.Errorf(
+ "progress output does not name the stalled implementation: %q",
+ stdout.String(),
+ )
+ }
+}
+
+func TestRun_RefusesAnOutputDirectoryThatAlreadyHoldsASweep(t *testing.T) {
+ root := t.TempDir()
+ output := filepath.Join(root, "campaigns")
+ if err := os.MkdirAll(output, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(output, manifestFileName), []byte("{}"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ specPath := filepath.Join(root, "todo.ts")
+ if err := os.WriteFile(specPath, []byte("export const properties = [];\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ err := run([]string{
+ "--corpus", wholeCorpus(t), "--spec", specPath, "--seeds", "1",
+ "--max-steps", "10", "--output", output,
+ }, io.Discard, io.Discard)
+ if err == nil || !strings.Contains(err.Error(), manifestFileName) {
+ t.Fatalf(
+ "two sweeps sharing a directory would interleave their records, got %v",
+ err,
+ )
+ }
+}
+
+func writeScript(t *testing.T, path, body string) string {
+ t.Helper()
+ if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ return path
+}
+
+func readLines(t *testing.T, path string) []string {
+ t.Helper()
+ body, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var lines []string
+ scanner := bufio.NewScanner(strings.NewReader(string(body)))
+ for scanner.Scan() {
+ if line := strings.TrimSpace(scanner.Text()); line != "" {
+ lines = append(lines, line)
+ }
+ }
+ return lines
+}
+
+func readManifest(t *testing.T, path string) manifest {
+ t.Helper()
+ body, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var recorded manifest
+ if err := json.Unmarshal(body, &recorded); err != nil {
+ t.Fatal(err)
+ }
+ return recorded
+}
+
+func readRecords(t *testing.T, path string) []implementationRecord {
+ t.Helper()
+ var records []implementationRecord
+ for _, line := range readLines(t, path) {
+ var record implementationRecord
+ if err := json.Unmarshal([]byte(line), &record); err != nil {
+ t.Fatalf("%s: %v", line, err)
+ }
+ records = append(records, record)
+ }
+ return records
+}
+
+func argumentValue(arguments []string, name string) string {
+ index := slices.Index(arguments, name)
+ if index < 0 || index+1 >= len(arguments) {
+ return ""
+ }
+ return arguments[index+1]
+}
+
+// freePortRange finds count consecutive free ports, which is what the sweep
+// hands out: one port per implementation from --base-port upwards.
+func freePortRange(t *testing.T, count int) int {
+ t.Helper()
+ for range 100 {
+ base := 20000 + rand.Intn(20000)
+ if portsAreFree(base, count) {
+ return base
+ }
+ }
+ t.Fatalf("no run of %d free ports", count)
+ return 0
+}
+
+func portsAreFree(base, count int) bool {
+ for offset := range count {
+ listener, err := net.Listen(
+ "tcp",
+ fmt.Sprintf("127.0.0.1:%d", base+offset),
+ )
+ if err != nil {
+ return false
+ }
+ listener.Close()
+ }
+ return true
+}
diff --git a/cmd/internal-tools/corpus-sweep/main.go b/cmd/internal-tools/corpus-sweep/main.go
new file mode 100644
index 0000000..c4dbc5d
--- /dev/null
+++ b/cmd/internal-tools/corpus-sweep/main.go
@@ -0,0 +1,275 @@
+// Command corpus-sweep runs one specification against every implementation in a
+// served corpus of independent implementations of the same requirement. It
+// serves each one on its own port, which is what keeps them apart: the corpus
+// holds pairs that write the same localStorage key, and one origin shared
+// between two of them is one stored record shared between them.
+package main
+
+import (
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "os/signal"
+ "path/filepath"
+ "strconv"
+ "syscall"
+ "time"
+
+ "github.com/priyanshujain/sanderling/internal/seedspec"
+)
+
+// The generator and the platform are fixed rather than exposed: the
+// pre-registration runs the seeded policy against the served corpus, and a
+// sweep that could quietly run something else records a comparison nobody made.
+const (
+ generator = "seeded"
+ platform = "web"
+)
+
+// defaultConcurrency is how many implementations are swept at once. fleet.md
+// measured eight concurrent web campaigns clean at about 1.1 GB resident each,
+// parallel efficiency 0.83 at eight against 0.87 at six, and said to re-measure
+// before trusting anything above eight on a contended host. Six sits at the
+// better efficiency and leaves the emulator farm that shares the host its
+// slots. Serving costs nothing here: the corpus needs no build and no separate
+// server process, so a worker is one browser.
+const defaultConcurrency = 6
+
+// defaultBasePort starts above the range the model-implementation sweep hands
+// out, so the two can run on one host without either being served the other's
+// pages.
+const defaultBasePort = 5400
+
+type config struct {
+ corpusRoot string
+ specPath string
+ outputDirectory string
+ implementations []string
+ seeds []int64
+ maxSteps int
+ duration time.Duration
+ concurrency int
+ basePort int
+ campaignPath string
+ sanderlingPath string
+ extraArguments []string
+}
+
+const usage = `corpus-sweep runs one seeded campaign per implementation of a served corpus.
+
+Usage:
+ corpus-sweep --corpus --spec --seeds
+ --max-steps --output [flags]
+ [-- ]
+
+Every implementation is served on its own port, so every one is its own origin
+and none can read another's stored state, and every one is swept with the same
+seeds and the same step budget. Each run's arm is the implementation's name.
+
+Everything after a bare -- reaches every sanderling test call through the
+campaign tool.
+`
+
+func parseArguments(arguments []string, stderr io.Writer) (config, error) {
+ flagSet := flag.NewFlagSet("corpus-sweep", flag.ContinueOnError)
+ flagSet.SetOutput(stderr)
+ flagSet.Usage = func() {
+ fmt.Fprint(stderr, usage)
+ flagSet.PrintDefaults()
+ }
+ var configuration config
+ var seedSpecification string
+ var selection string
+ flagSet.StringVar(
+ &configuration.corpusRoot,
+ "corpus",
+ "",
+ "root of the checked-out corpus, holding examples/ (required)",
+ )
+ flagSet.StringVar(
+ &configuration.specPath,
+ "spec",
+ "",
+ "path to the property set every implementation is run against (required)",
+ )
+ flagSet.StringVar(
+ &seedSpecification,
+ "seeds",
+ "",
+ "seeds every implementation runs: ranges and lists, e.g. 1-10,20 (required)",
+ )
+ flagSet.StringVar(
+ &selection,
+ "implementations",
+ "",
+ "comma-separated subset of the population to sweep (default: all of it)",
+ )
+ flagSet.IntVar(
+ &configuration.maxSteps,
+ "max-steps",
+ 0,
+ "per-run step budget, identical across implementations (required, must be positive)",
+ )
+ flagSet.DurationVar(
+ &configuration.duration,
+ "duration",
+ 5*time.Minute,
+ "per-run wall-clock ceiling passed to each campaign",
+ )
+ flagSet.IntVar(
+ &configuration.concurrency,
+ "concurrency",
+ defaultConcurrency,
+ "implementations swept at once",
+ )
+ flagSet.IntVar(
+ &configuration.basePort,
+ "base-port",
+ defaultBasePort,
+ "first port served; each implementation takes the next one in population order",
+ )
+ flagSet.StringVar(
+ &configuration.outputDirectory,
+ "output",
+ "",
+ "campaign tree to create (required)",
+ )
+ flagSet.StringVar(
+ &configuration.campaignPath,
+ "campaign",
+ "campaign",
+ "campaign binary to invoke per implementation and seed",
+ )
+ flagSet.StringVar(
+ &configuration.sanderlingPath,
+ "sanderling",
+ "sanderling",
+ "sanderling binary each campaign invokes",
+ )
+ if err := flagSet.Parse(arguments); err != nil {
+ return config{}, err
+ }
+ configuration.extraArguments = flagSet.Args()
+
+ for name, value := range map[string]string{
+ "--corpus": configuration.corpusRoot,
+ "--spec": configuration.specPath,
+ "--seeds": seedSpecification,
+ "--output": configuration.outputDirectory,
+ } {
+ if value == "" {
+ return config{}, fmt.Errorf("%s is required", name)
+ }
+ }
+ if configuration.maxSteps <= 0 {
+ return config{}, fmt.Errorf(
+ "--max-steps must be positive: every implementation needs the same step budget",
+ )
+ }
+ if configuration.duration <= 0 {
+ return config{}, fmt.Errorf(
+ "--duration must be positive: %s",
+ configuration.duration,
+ )
+ }
+ if configuration.concurrency <= 0 {
+ return config{}, fmt.Errorf(
+ "--concurrency must be positive: %d",
+ configuration.concurrency,
+ )
+ }
+ if configuration.basePort < 1024 || configuration.basePort > 65535 {
+ return config{}, fmt.Errorf(
+ "--base-port %d is outside 1024-65535",
+ configuration.basePort,
+ )
+ }
+ seeds, err := seedspec.Parse(seedSpecification)
+ if err != nil {
+ return config{}, fmt.Errorf("--seeds: %w", err)
+ }
+ configuration.seeds = seeds
+ names, err := selectImplementations(selection)
+ if err != nil {
+ return config{}, fmt.Errorf("--implementations: %w", err)
+ }
+ configuration.implementations = names
+ for name, value := range map[string]*string{
+ "--corpus": &configuration.corpusRoot,
+ "--spec": &configuration.specPath,
+ "--output": &configuration.outputDirectory,
+ } {
+ absolute, err := filepath.Abs(*value)
+ if err != nil {
+ return config{}, fmt.Errorf("%s: %w", name, err)
+ }
+ *value = absolute
+ }
+ return configuration, nil
+}
+
+func campaignDirectory(
+ configuration config,
+ target implementation,
+ seed string,
+) string {
+ return filepath.Join(
+ configuration.outputDirectory,
+ target.Name,
+ "seed-"+seed,
+ )
+}
+
+// campaignArguments builds one campaign invocation. The arm is the
+// implementation's name, so every run in the tree can be attributed to the
+// implementation it came from without reading back which port served it.
+func campaignArguments(
+ configuration config,
+ target implementation,
+ seed string,
+) []string {
+ arguments := []string{
+ "--spec", configuration.specPath,
+ "--bundle-id", target.URL(),
+ "--platform", platform,
+ "--arm", target.Name,
+ "--generator", generator,
+ "--max-steps", strconv.Itoa(configuration.maxSteps),
+ "--duration", configuration.duration.String(),
+ "--seeds", seed,
+ "--sanderling", configuration.sanderlingPath,
+ "--output", campaignDirectory(configuration, target, seed),
+ }
+ if len(configuration.extraArguments) > 0 {
+ arguments = append(arguments, "--")
+ arguments = append(arguments, configuration.extraArguments...)
+ }
+ return arguments
+}
+
+func run(arguments []string, stdout, stderr io.Writer) error {
+ configuration, err := parseArguments(arguments, stderr)
+ if err != nil {
+ return err
+ }
+ ctx, cancel := signal.NotifyContext(
+ context.Background(),
+ os.Interrupt,
+ syscall.SIGTERM,
+ )
+ defer cancel()
+ return runSweep(ctx, configuration, stdout)
+}
+
+func main() {
+ if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil {
+ if errors.Is(err, flag.ErrHelp) {
+ return
+ }
+ fmt.Fprintf(os.Stderr, "error: %v\n", err)
+ os.Exit(1)
+ }
+}
diff --git a/cmd/internal-tools/corpus-sweep/manifest.go b/cmd/internal-tools/corpus-sweep/manifest.go
new file mode 100644
index 0000000..31c84ac
--- /dev/null
+++ b/cmd/internal-tools/corpus-sweep/manifest.go
@@ -0,0 +1,132 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "time"
+)
+
+const (
+ manifestFileName = "sweep.json"
+ recordsFileName = "implementations.jsonl"
+)
+
+// plannedImplementation is one implementation the sweep intends to run, with
+// the origin that keeps its stored state its own and the URL every seed is
+// driven at.
+type plannedImplementation struct {
+ Name string `json:"name"`
+ Document string `json:"document"`
+ Port int `json:"port"`
+ Origin string `json:"origin"`
+ URL string `json:"url"`
+}
+
+// manifest is sweep.json: what the sweep INTENDED to run, written before the
+// first run so a host that dropped an implementation or a seed shows up as a
+// missing run rather than as a smaller sample.
+type manifest struct {
+ Generator string `json:"generator"`
+ Platform string `json:"platform"`
+ SpecPath string `json:"spec_path"`
+ CorpusRoot string `json:"corpus_root"`
+ CorpusCommit string `json:"corpus_commit"`
+ MaxSteps int `json:"max_steps"`
+ DurationMillis int64 `json:"duration_millis"`
+ Seeds []int64 `json:"seeds"`
+ Implementations []plannedImplementation `json:"implementations"`
+ Concurrency int `json:"concurrency"`
+ Host string `json:"host"`
+ CampaignPath string `json:"campaign_path"`
+ SanderlingPath string `json:"sanderling_path"`
+ StartedAt time.Time `json:"started_at"`
+}
+
+func buildManifest(
+ configuration config,
+ implementations []implementation,
+ host string,
+ startedAt time.Time,
+) manifest {
+ planned := make([]plannedImplementation, 0, len(implementations))
+ for _, target := range implementations {
+ planned = append(planned, plannedImplementation{
+ Name: target.Name,
+ Document: target.Document,
+ Port: target.Port,
+ Origin: target.Origin(),
+ URL: target.URL(),
+ })
+ }
+ return manifest{
+ Generator: generator,
+ Platform: platform,
+ SpecPath: configuration.specPath,
+ CorpusRoot: configuration.corpusRoot,
+ CorpusCommit: corpusCommit(configuration.corpusRoot),
+ MaxSteps: configuration.maxSteps,
+ DurationMillis: configuration.duration.Milliseconds(),
+ Seeds: configuration.seeds,
+ Implementations: planned,
+ Concurrency: configuration.concurrency,
+ Host: host,
+ CampaignPath: configuration.campaignPath,
+ SanderlingPath: configuration.sanderlingPath,
+ StartedAt: startedAt,
+ }
+}
+
+// corpusCommit records which checkout was swept. It is empty rather than fatal
+// for a corpus that is not a git working tree, because the population check has
+// already established the corpus holds the right examples.
+func corpusCommit(corpusRoot string) string {
+ command := exec.Command("git", "-C", corpusRoot, "rev-parse", "HEAD")
+ output, err := command.Output()
+ if err != nil {
+ return ""
+ }
+ return strings.TrimSpace(string(output))
+}
+
+func writeManifest(directory string, value manifest) error {
+ body, err := json.MarshalIndent(value, "", " ")
+ if err != nil {
+ return fmt.Errorf("marshal manifest: %w", err)
+ }
+ return os.WriteFile(
+ filepath.Join(directory, manifestFileName),
+ append(body, '\n'),
+ 0o644,
+ )
+}
+
+// runRecord is one campaign, which is one implementation at one seed.
+type runRecord struct {
+ Seed int64 `json:"seed"`
+ ExitCode int `json:"exit_code"`
+ LaunchError string `json:"launch_error,omitempty"`
+ CampaignDirectory string `json:"campaign_directory"`
+ MonotonicMillis int64 `json:"monotonic_millis"`
+}
+
+// implementationRecord is one line of implementations.jsonl. FailedStage names
+// the step that stopped this implementation, and one that never got served
+// carries no runs at all.
+type implementationRecord struct {
+ Name string `json:"implementation"`
+ Document string `json:"document"`
+ Port int `json:"port"`
+ Origin string `json:"origin"`
+ URL string `json:"url"`
+ FailedStage string `json:"failed_stage,omitempty"`
+ Error string `json:"error,omitempty"`
+ StartedAt time.Time `json:"started_at"`
+ MonotonicMillis int64 `json:"monotonic_millis"`
+ Runs []runRecord `json:"runs"`
+}
+
+const stageServe = "serve"
diff --git a/cmd/internal-tools/corpus-sweep/origin_test.go b/cmd/internal-tools/corpus-sweep/origin_test.go
new file mode 100644
index 0000000..2e7f3b3
--- /dev/null
+++ b/cmd/internal-tools/corpus-sweep/origin_test.go
@@ -0,0 +1,194 @@
+package main
+
+import (
+ "fmt"
+ "io"
+ "net/url"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// collisionPairs are implementations in this corpus that write the same
+// localStorage key as each other. Served from one origin they share one stored
+// record: in the corpus survey angular2_es2015 crashed at bootstrap on a record
+// angular2 had written, which is a violation belonging to no implementation.
+var collisionPairs = [][2]string{
+ {"angular2", "angular2_es2015"},
+ {"backbone", "backbone_require"},
+ {"canjs", "canjs_require"},
+ {"react", "typescript-react"},
+}
+
+func originOf(t *testing.T, rawURL string) string {
+ t.Helper()
+ parsed, err := url.Parse(rawURL)
+ if err != nil {
+ t.Fatalf("parse %q: %v", rawURL, err)
+ }
+ return parsed.Scheme + "://" + parsed.Host
+}
+
+// The whole population is swept so the assertion covers every implementation
+// rather than the four pairs already known to collide: the survey found those
+// four, and an unexamined fifth would be just as damaging.
+func TestSweep_GivesEveryImplementationAnOriginNoOtherImplementationShares(
+ t *testing.T,
+) {
+ root := t.TempDir()
+ corpus := wholeCorpus(t)
+ specPath := filepath.Join(root, "todo.ts")
+ if err := os.WriteFile(specPath, []byte("export const properties = [];\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ output := filepath.Join(root, "campaigns")
+ campaignLog := filepath.Join(root, "campaign.log")
+ fetchLog := filepath.Join(root, "fetch.log")
+ testBinary, err := filepath.Abs(os.Args[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+ campaignPath := writeScript(
+ t,
+ filepath.Join(root, "stub-campaign"),
+ fmt.Sprintf(
+ stubCampaign,
+ filepath.Join(output, manifestFileName),
+ campaignLog,
+ fetchLog,
+ testBinary,
+ ),
+ )
+ sanderlingPath := writeScript(
+ t,
+ filepath.Join(root, "stub-sanderling"),
+ "#!/bin/sh\nexit 0\n",
+ )
+
+ err = run([]string{
+ "--corpus", corpus,
+ "--spec", specPath,
+ "--seeds", "1",
+ "--max-steps", "10",
+ "--duration", "30s",
+ "--concurrency", "8",
+ "--base-port", fmt.Sprint(freePortRange(t, len(includedImplementations))),
+ "--output", output,
+ "--campaign", campaignPath,
+ "--sanderling", sanderlingPath,
+ }, io.Discard, io.Discard)
+ if err != nil {
+ t.Fatalf("sweep: %v", err)
+ }
+
+ // What the sweep declared it would serve each implementation from.
+ recorded := readManifest(t, filepath.Join(output, manifestFileName))
+ if len(recorded.Implementations) != len(includedImplementations) {
+ t.Fatalf(
+ "intended implementations: got %d, want %d",
+ len(recorded.Implementations),
+ len(includedImplementations),
+ )
+ }
+ plannedOrigin := map[string]string{}
+ ownerOfOrigin := map[string]string{}
+ for _, planned := range recorded.Implementations {
+ if owner, taken := ownerOfOrigin[planned.Origin]; taken {
+ t.Errorf(
+ "%s and %s are both served from %s, so they share one localStorage",
+ owner,
+ planned.Name,
+ planned.Origin,
+ )
+ }
+ ownerOfOrigin[planned.Origin] = planned.Name
+ plannedOrigin[planned.Name] = planned.Origin
+ if got := originOf(t, planned.URL); got != planned.Origin {
+ t.Errorf(
+ "%s: url %q is not under the origin %q the manifest claims",
+ planned.Name,
+ planned.URL,
+ planned.Origin,
+ )
+ }
+ }
+ for _, pair := range collisionPairs {
+ if plannedOrigin[pair[0]] == plannedOrigin[pair[1]] {
+ t.Errorf(
+ "%s and %s write the same localStorage key and are both served from %s",
+ pair[0],
+ pair[1],
+ plannedOrigin[pair[0]],
+ )
+ }
+ }
+
+ // What actually reached the driver. The web driver parses the origin to
+ // clear out of --bundle-id, so two arms sharing a bundle-id origin clear
+ // and repopulate one another's storage however carefully they are labelled.
+ drivenOrigin := map[string]string{}
+ for _, line := range readLines(t, campaignLog) {
+ arguments := strings.Fields(strings.SplitN(line, "argv: ", 2)[1])
+ arm := argumentValue(arguments, "--arm")
+ if arm == "" {
+ t.Fatalf(
+ "a campaign ran with no arm, so its runs cannot be attributed: %q",
+ line,
+ )
+ }
+ drivenOrigin[arm] = originOf(t, argumentValue(arguments, "--bundle-id"))
+ }
+ if len(drivenOrigin) != len(includedImplementations) {
+ t.Fatalf(
+ "arms that reached the campaign tool: got %d, want %d",
+ len(drivenOrigin),
+ len(includedImplementations),
+ )
+ }
+ armOfOrigin := map[string]string{}
+ for arm, origin := range drivenOrigin {
+ if other, taken := armOfOrigin[origin]; taken {
+ t.Errorf(
+ "arms %s and %s were both driven at %s",
+ other,
+ arm,
+ origin,
+ )
+ }
+ armOfOrigin[origin] = arm
+ if origin != plannedOrigin[arm] {
+ t.Errorf(
+ "%s was driven at %s but the manifest promised %s",
+ arm,
+ origin,
+ plannedOrigin[arm],
+ )
+ }
+ }
+
+ // And what each origin answered with, which is the check that the ports
+ // are not merely distinct but each carries its own implementation.
+ served := map[string]string{}
+ for _, line := range readLines(t, fetchLog) {
+ requested, body, found := strings.Cut(line, " -> 200 ")
+ if !found {
+ t.Errorf("a served page did not answer: %q", line)
+ continue
+ }
+ served[originOf(t, requested)] = body
+ }
+ for arm, origin := range drivenOrigin {
+ if want := "" + arm + ""; !strings.Contains(
+ served[origin],
+ want,
+ ) {
+ t.Errorf(
+ "%s at %s was served %q, which is not its own document",
+ arm,
+ origin,
+ served[origin],
+ )
+ }
+ }
+}
diff --git a/cmd/internal-tools/corpus-sweep/serve.go b/cmd/internal-tools/corpus-sweep/serve.go
new file mode 100644
index 0000000..d69730f
--- /dev/null
+++ b/cmd/internal-tools/corpus-sweep/serve.go
@@ -0,0 +1,178 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "net"
+ "net/http"
+ "path"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "time"
+)
+
+const (
+ readinessTimeout = 5 * time.Second
+ readinessInterval = 100 * time.Millisecond
+)
+
+// stubScriptPath is served from every implementation's own origin in place of a
+// dependency that no longer answers.
+const stubScriptPath = "/__corpus-sweep__/stub.js"
+
+// deadDependency is polyfill.io, which was shut down after this corpus was
+// pinned. One implementation loads it from its index.html; the request fails
+// and the application works regardless, but it fails slowly, once per run, on
+// every run. The corpus is served from this process, so the reference is
+// rewritten to a local no-op on the way out.
+var deadDependency = regexp.MustCompile(`https?://polyfill\.io/[^"'\s>]*`)
+
+// staticServer serves the whole corpus tree on one implementation's port. The
+// tree rather than the implementation's own directory, because an example that
+// asks for a path above itself has to resolve the same way it would in the
+// upstream repository. Only one implementation is ever visited on this port, so
+// the origin still belongs to it alone.
+type staticServer struct {
+ implementation implementation
+ listener net.Listener
+ server *http.Server
+}
+
+func startStaticServer(
+ corpusRoot string,
+ target implementation,
+) (*staticServer, error) {
+ listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", target.Port))
+ if err != nil {
+ return nil, fmt.Errorf("serve %s: %w", target.Name, err)
+ }
+ server := &http.Server{Handler: corpusHandler(corpusRoot)}
+ running := &staticServer{
+ implementation: target,
+ listener: listener,
+ server: server,
+ }
+ go server.Serve(listener)
+ return running, nil
+}
+
+func (s *staticServer) stop() {
+ s.server.Close()
+}
+
+// waitReady confirms the served document answers before any run drives it. A
+// wrong document path would otherwise reach the driver as a 404 page, and a
+// sweep of 404 pages produces clean runs for every implementation.
+//
+// Only a transport error is retried. The listener is bound before the sweep
+// starts, so a status that is not 200 is the server's answer about this
+// document and waiting will not change it.
+func (s *staticServer) waitReady(ctx context.Context) error {
+ url := s.implementation.URL()
+ client := &http.Client{Timeout: 5 * time.Second}
+ deadline := time.Now().Add(readinessTimeout)
+ var lastErr error
+ for {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ response, err := client.Get(url)
+ if err == nil {
+ io.Copy(io.Discard, response.Body)
+ response.Body.Close()
+ if response.StatusCode == http.StatusOK {
+ return nil
+ }
+ return fmt.Errorf(
+ "%s answered %d, not the document",
+ url,
+ response.StatusCode,
+ )
+ }
+ lastErr = err
+ if time.Now().After(deadline) {
+ return fmt.Errorf(
+ "%s did not answer within %s: %w",
+ url,
+ readinessTimeout,
+ lastErr,
+ )
+ }
+ time.Sleep(readinessInterval)
+ }
+}
+
+func corpusHandler(corpusRoot string) http.Handler {
+ files := http.FileServer(http.Dir(corpusRoot))
+ return http.HandlerFunc(
+ func(writer http.ResponseWriter, request *http.Request) {
+ cleaned := path.Clean(
+ "/" + strings.TrimPrefix(request.URL.Path, "/"),
+ )
+ if cleaned == stubScriptPath {
+ writer.Header().Set("Content-Type", "application/javascript")
+ io.WriteString(
+ writer,
+ "/* corpus-sweep: dependency removed upstream */\n",
+ )
+ return
+ }
+ if !strings.HasSuffix(cleaned, ".html") {
+ files.ServeHTTP(writer, request)
+ return
+ }
+ body, modified, err := readDocument(corpusRoot, cleaned)
+ if err != nil {
+ // Never the file server's fallback: it answers a document it
+ // cannot read with a 200 directory listing, and a run driven at a
+ // listing explores nothing and comes back clean.
+ http.Error(writer, err.Error(), documentStatus(err))
+ return
+ }
+ rewritten := deadDependency.ReplaceAll(body, []byte(stubScriptPath))
+ http.ServeContent(
+ writer,
+ request,
+ path.Base(cleaned),
+ modified,
+ bytes.NewReader(rewritten),
+ )
+ },
+ )
+}
+
+func documentStatus(err error) int {
+ switch {
+ case errors.Is(err, fs.ErrNotExist):
+ return http.StatusNotFound
+ case errors.Is(err, fs.ErrPermission):
+ return http.StatusForbidden
+ default:
+ return http.StatusInternalServerError
+ }
+}
+
+func readDocument(corpusRoot, cleaned string) ([]byte, time.Time, error) {
+ file, err := http.Dir(corpusRoot).Open(cleaned)
+ if err != nil {
+ return nil, time.Time{}, err
+ }
+ defer file.Close()
+ info, err := file.Stat()
+ if err != nil || info.IsDir() {
+ return nil, time.Time{}, fmt.Errorf(
+ "%s is not a document",
+ filepath.FromSlash(cleaned),
+ )
+ }
+ body, err := io.ReadAll(file)
+ if err != nil {
+ return nil, time.Time{}, err
+ }
+ return body, info.ModTime(), nil
+}
diff --git a/cmd/internal-tools/corpus-sweep/serve_test.go b/cmd/internal-tools/corpus-sweep/serve_test.go
new file mode 100644
index 0000000..8aa1439
--- /dev/null
+++ b/cmd/internal-tools/corpus-sweep/serve_test.go
@@ -0,0 +1,214 @@
+package main
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+)
+
+func get(t *testing.T, url string) (int, string) {
+ t.Helper()
+ client := &http.Client{Timeout: 5 * time.Second}
+ response, err := client.Get(url)
+ if err != nil {
+ t.Fatalf("GET %s: %v", url, err)
+ }
+ defer response.Body.Close()
+ body, err := io.ReadAll(response.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return response.StatusCode, string(body)
+}
+
+func TestStaticServer_ServesEachImplementationsDocumentFromItsOwnPort(
+ t *testing.T,
+) {
+ root := wholeCorpus(t)
+ planned, err := planImplementations(
+ root,
+ []string{"angular-dart", "duel", "react"},
+ freePortRange(t, 3),
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, target := range planned {
+ server, err := startStaticServer(root, target)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer server.stop()
+ if err := server.waitReady(context.Background()); err != nil {
+ t.Fatalf("%s: %v", target.Name, err)
+ }
+ status, body := get(t, target.URL())
+ if status != http.StatusOK ||
+ !strings.Contains(body, ""+target.Name+"") {
+ t.Errorf(
+ "%s at %s: got %d %q",
+ target.Name,
+ target.URL(),
+ status,
+ body,
+ )
+ }
+ }
+}
+
+func TestStaticServer_ReplacesTheDependencyThatNoLongerAnswers(t *testing.T) {
+ root := wholeCorpus(t)
+ document := filepath.Join(root, filepath.FromSlash(documentFor("aurelia")))
+ body := ``
+ if err := os.WriteFile(document, []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ planned, err := planImplementations(
+ root,
+ []string{"aurelia"},
+ freePortRange(t, 1),
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ server, err := startStaticServer(root, planned[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer server.stop()
+
+ status, served := get(t, planned[0].URL())
+ if status != http.StatusOK {
+ t.Fatalf("document answered %d", status)
+ }
+ if strings.Contains(served, "polyfill.io") {
+ t.Errorf(
+ "a request to a host that no longer answers reaches the browser once per run: %q",
+ served,
+ )
+ }
+ if !strings.Contains(served, `src="`+stubScriptPath+`"`) {
+ t.Errorf(
+ "the reference was removed rather than pointed at a local no-op: %q",
+ served,
+ )
+ }
+
+ stubStatus, stub := get(t, planned[0].Origin()+stubScriptPath)
+ if stubStatus != http.StatusOK || stub == "" {
+ t.Errorf("stub script: got %d %q", stubStatus, stub)
+ }
+}
+
+// An example that asks for a path above its own directory has to resolve the
+// same way it does upstream, which is why the whole tree is served rather than
+// the one directory.
+func TestStaticServer_ServesPathsAboveTheImplementationsOwnDirectory(
+ t *testing.T,
+) {
+ root := wholeCorpus(t)
+ shared := filepath.Join(root, "node_modules", "todomvc-common")
+ if err := os.MkdirAll(shared, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(shared, "base.css"), []byte("body{}"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ planned, err := planImplementations(
+ root,
+ []string{"react"},
+ freePortRange(t, 1),
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ server, err := startStaticServer(root, planned[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer server.stop()
+
+ status, body := get(
+ t,
+ planned[0].Origin()+"/node_modules/todomvc-common/base.css",
+ )
+ if status != http.StatusOK || body != "body{}" {
+ t.Errorf("shared asset: got %d %q", status, body)
+ }
+}
+
+func TestStaticServerWaitReady_ReportsADocumentThatDoesNotAnswer(t *testing.T) {
+ root := wholeCorpus(t)
+ document := filepath.Join(root, filepath.FromSlash(documentFor("dojo")))
+ if err := os.Chmod(document, 0o000); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { os.Chmod(document, 0o644) })
+ planned, err := planImplementations(
+ root,
+ []string{"dojo"},
+ freePortRange(t, 1),
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ server, err := startStaticServer(root, planned[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer server.stop()
+
+ started := time.Now()
+ err = server.waitReady(context.Background())
+ if err == nil {
+ t.Fatal(
+ "a document that does not answer would be swept as an error page and come back clean",
+ )
+ }
+ if elapsed := time.Since(started); elapsed > readinessTimeout {
+ t.Errorf(
+ "waited %s for a status that was never going to change",
+ elapsed,
+ )
+ }
+}
+
+func TestStaticServerStop_ReleasesThePort(t *testing.T) {
+ root := wholeCorpus(t)
+ planned, err := planImplementations(
+ root,
+ []string{"vue"},
+ freePortRange(t, 1),
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ server, err := startStaticServer(root, planned[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := server.waitReady(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ server.stop()
+
+ client := &http.Client{Timeout: time.Second}
+ deadline := time.Now().Add(3 * time.Second)
+ for time.Now().Before(deadline) {
+ response, err := client.Get(planned[0].URL())
+ if err != nil {
+ return
+ }
+ response.Body.Close()
+ time.Sleep(50 * time.Millisecond)
+ }
+ t.Fatalf(
+ "port %d is still served after stop(), so the next sweep cannot bind it",
+ planned[0].Port,
+ )
+}
diff --git a/cmd/internal-tools/corpus-sweep/sweep.go b/cmd/internal-tools/corpus-sweep/sweep.go
new file mode 100644
index 0000000..39d56be
--- /dev/null
+++ b/cmd/internal-tools/corpus-sweep/sweep.go
@@ -0,0 +1,307 @@
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strconv"
+ "sync"
+ "time"
+)
+
+// resolveBinaries turns campaign and sanderling into absolute paths before
+// anything is served. Each campaign runs from the sweep's own directory, and a
+// binary that is missing altogether has to stop the sweep here rather than fail
+// once per implementation and seed.
+func resolveBinaries(configuration *config) error {
+ for name, value := range map[string]*string{
+ "--campaign": &configuration.campaignPath,
+ "--sanderling": &configuration.sanderlingPath,
+ } {
+ resolved, err := exec.LookPath(*value)
+ if err != nil {
+ return fmt.Errorf("%s: %w", name, err)
+ }
+ absolute, err := filepath.Abs(resolved)
+ if err != nil {
+ return fmt.Errorf("%s: %w", name, err)
+ }
+ *value = absolute
+ }
+ return nil
+}
+
+type sweep struct {
+ configuration config
+ stdout io.Writer
+ records io.Writer
+ servers map[string]*staticServer
+ mutex sync.Mutex
+ stalled int
+ failedRuns int
+ totalRuns int
+}
+
+func runSweep(
+ ctx context.Context,
+ configuration config,
+ stdout io.Writer,
+) error {
+ if _, err := os.Stat(filepath.Join(configuration.outputDirectory, manifestFileName)); err == nil {
+ return fmt.Errorf(
+ "%s already exists in %s: pick a fresh --output so two sweeps do not share a directory",
+ manifestFileName,
+ configuration.outputDirectory,
+ )
+ }
+ if err := verifyCorpus(configuration.corpusRoot); err != nil {
+ return err
+ }
+ implementations, err := planImplementations(
+ configuration.corpusRoot,
+ configuration.implementations,
+ configuration.basePort,
+ )
+ if err != nil {
+ return err
+ }
+ if err := resolveBinaries(&configuration); err != nil {
+ return err
+ }
+ if _, err := os.Stat(configuration.specPath); err != nil {
+ return fmt.Errorf("--spec: %w", err)
+ }
+ if err := os.MkdirAll(configuration.outputDirectory, 0o755); err != nil {
+ return fmt.Errorf("create sweep dir: %w", err)
+ }
+ host, _ := os.Hostname()
+ if err := writeManifest(configuration.outputDirectory, buildManifest(configuration, implementations, host, time.Now().UTC())); err != nil {
+ return fmt.Errorf("write %s: %w", manifestFileName, err)
+ }
+
+ recordsFile, err := os.OpenFile(
+ filepath.Join(configuration.outputDirectory, recordsFileName),
+ os.O_CREATE|os.O_WRONLY|os.O_APPEND,
+ 0o644,
+ )
+ if err != nil {
+ return fmt.Errorf("open %s: %w", recordsFileName, err)
+ }
+ defer recordsFile.Close()
+
+ running := &sweep{
+ configuration: configuration,
+ stdout: stdout,
+ records: recordsFile,
+ }
+ // Every port is bound before any run starts. A port already taken means
+ // that implementation cannot have an origin of its own, and continuing
+ // without one is what the separate origins are there to prevent.
+ if err := running.serveAll(implementations); err != nil {
+ return err
+ }
+ defer running.stopAll()
+
+ fmt.Fprintf(
+ stdout,
+ "sweep: %d implementations, %d seeds each, %d at a time, %s\n",
+ len(
+ implementations,
+ ),
+ len(configuration.seeds),
+ configuration.concurrency,
+ configuration.outputDirectory,
+ )
+ running.work(ctx, implementations)
+
+ fmt.Fprintf(
+ stdout,
+ "sweep complete: %d of %d implementations never ran, %d of %d campaigns failed\n",
+ running.stalled,
+ len(implementations),
+ running.failedRuns,
+ running.totalRuns,
+ )
+ if running.stalled > 0 || running.failedRuns > 0 {
+ return fmt.Errorf(
+ "%d of %d implementations never ran and %d of %d campaigns failed; see %s",
+ running.stalled,
+ len(implementations),
+ running.failedRuns,
+ running.totalRuns,
+ recordsFileName,
+ )
+ }
+ return nil
+}
+
+func (s *sweep) serveAll(implementations []implementation) error {
+ s.servers = make(map[string]*staticServer, len(implementations))
+ for _, target := range implementations {
+ server, err := startStaticServer(s.configuration.corpusRoot, target)
+ if err != nil {
+ s.stopAll()
+ return err
+ }
+ s.servers[target.Name] = server
+ }
+ return nil
+}
+
+func (s *sweep) stopAll() {
+ for _, server := range s.servers {
+ server.stop()
+ }
+}
+
+func (s *sweep) work(ctx context.Context, implementations []implementation) {
+ queue := make(chan implementation, len(implementations))
+ for _, target := range implementations {
+ queue <- target
+ }
+ close(queue)
+
+ workers := min(s.configuration.concurrency, len(implementations))
+ var waitGroup sync.WaitGroup
+ for range workers {
+ waitGroup.Add(1)
+ go func() {
+ defer waitGroup.Done()
+ for target := range queue {
+ if ctx.Err() != nil {
+ return
+ }
+ s.report(s.runImplementation(ctx, target))
+ }
+ }()
+ }
+ waitGroup.Wait()
+}
+
+// runImplementation carries one implementation through all of its seeds. A
+// document that does not answer is recorded and the sweep moves on: one
+// implementation must not cost the other forty-two their runs.
+func (s *sweep) runImplementation(
+ ctx context.Context,
+ target implementation,
+) (record implementationRecord) {
+ record = implementationRecord{
+ Name: target.Name,
+ Document: target.Document,
+ Port: target.Port,
+ Origin: target.Origin(),
+ URL: target.URL(),
+ StartedAt: time.Now().UTC(),
+ }
+ started := time.Now()
+ defer func() { record.MonotonicMillis = time.Since(started).Milliseconds() }()
+
+ if err := os.MkdirAll(filepath.Join(s.configuration.outputDirectory, target.Name), 0o755); err != nil {
+ record.FailedStage = stageServe
+ record.Error = err.Error()
+ return record
+ }
+ if err := s.servers[target.Name].waitReady(ctx); err != nil {
+ record.FailedStage = stageServe
+ record.Error = err.Error()
+ return record
+ }
+
+ for _, seed := range s.configuration.seeds {
+ if ctx.Err() != nil {
+ return record
+ }
+ record.Runs = append(record.Runs, s.runSeed(ctx, target, seed))
+ }
+ return record
+}
+
+func (s *sweep) runSeed(
+ ctx context.Context,
+ target implementation,
+ seed int64,
+) (record runRecord) {
+ seedText := strconv.FormatInt(seed, 10)
+ directory := campaignDirectory(s.configuration, target, seedText)
+ record = runRecord{Seed: seed, CampaignDirectory: directory}
+ started := time.Now()
+ defer func() { record.MonotonicMillis = time.Since(started).Milliseconds() }()
+
+ if err := os.MkdirAll(directory, 0o755); err != nil {
+ record.ExitCode = -1
+ record.LaunchError = err.Error()
+ return record
+ }
+ exitCode, err := runCommand(
+ ctx,
+ s.configuration.campaignPath,
+ campaignArguments(
+ s.configuration,
+ target,
+ seedText,
+ ),
+ filepath.Join(directory, "campaign.log"),
+ )
+ record.ExitCode = exitCode
+ if err != nil {
+ record.LaunchError = err.Error()
+ }
+ return record
+}
+
+func runCommand(
+ ctx context.Context,
+ binary string,
+ arguments []string,
+ logPath string,
+) (int, error) {
+ logFile, err := os.Create(logPath)
+ if err != nil {
+ return -1, err
+ }
+ defer logFile.Close()
+ command := exec.CommandContext(ctx, binary, arguments...)
+ command.Stdout = logFile
+ command.Stderr = logFile
+ err = command.Run()
+ if err == nil {
+ return 0, nil
+ }
+ var exitError *exec.ExitError
+ if errors.As(err, &exitError) {
+ return exitError.ExitCode(), nil
+ }
+ return -1, err
+}
+
+func (s *sweep) report(record implementationRecord) {
+ s.mutex.Lock()
+ defer s.mutex.Unlock()
+ if record.FailedStage != "" {
+ s.stalled++
+ }
+ s.totalRuns += len(record.Runs)
+ failed := 0
+ for _, run := range record.Runs {
+ if run.ExitCode != 0 {
+ failed++
+ }
+ }
+ s.failedRuns += failed
+ if err := json.NewEncoder(s.records).Encode(record); err != nil {
+ fmt.Fprintf(s.stdout, "warning: %s record: %v\n", record.Name, err)
+ }
+ elapsed := time.Duration(record.MonotonicMillis) * time.Millisecond
+ if record.FailedStage != "" {
+ fmt.Fprintf(s.stdout, "%s port=%d failed at %s: %s (%s)\n",
+ record.Name, record.Port, record.FailedStage, record.Error, elapsed)
+ return
+ }
+ fmt.Fprintf(s.stdout, "%s origin=%s campaigns=%d failed=%d elapsed=%s\n",
+ record.Name, record.Origin, len(record.Runs), failed, elapsed)
+}