feat(corpus-sweep): run one specification against a served corpus of implementations

same fixed campaign as implementation-sweep, over a corpus that needs no build. each implementation gets its own port: the corpus holds pairs that write the same localStorage key, and one shared origin is one stored record shared between them.

cmd/internal-tools/corpus-sweep/main.go

@@ -0,0 +1,275 @@
+// Command corpus-sweep runs one specification against every implementation in a
+// served corpus of independent implementations of the same requirement. It
+// serves each one on its own port, which is what keeps them apart: the corpus
+// holds pairs that write the same localStorage key, and one origin shared
+// between two of them is one stored record shared between them.
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strconv"
+	"syscall"
+	"time"
+
+	"github.com/priyanshujain/sanderling/internal/seedspec"
+)
+
+// The generator and the platform are fixed rather than exposed: the
+// pre-registration runs the seeded policy against the served corpus, and a
+// sweep that could quietly run something else records a comparison nobody made.
+const (
+	generator = "seeded"
+	platform  = "web"
+)
+
+// defaultConcurrency is how many implementations are swept at once. fleet.md
+// measured eight concurrent web campaigns clean at about 1.1 GB resident each,
+// parallel efficiency 0.83 at eight against 0.87 at six, and said to re-measure
+// before trusting anything above eight on a contended host. Six sits at the
+// better efficiency and leaves the emulator farm that shares the host its
+// slots. Serving costs nothing here: the corpus needs no build and no separate
+// server process, so a worker is one browser.
+const defaultConcurrency = 6
+
+// defaultBasePort starts above the range the model-implementation sweep hands
+// out, so the two can run on one host without either being served the other's
+// pages.
+const defaultBasePort = 5400
+
+type config struct {
+	corpusRoot      string
+	specPath        string
+	outputDirectory string
+	implementations []string
+	seeds           []int64
+	maxSteps        int
+	duration        time.Duration
+	concurrency     int
+	basePort        int
+	campaignPath    string
+	sanderlingPath  string
+	extraArguments  []string
+}
+
+const usage = `corpus-sweep runs one seeded campaign per implementation of a served corpus.
+
+Usage:
+  corpus-sweep --corpus <dir> --spec <path> --seeds <spec>
+               --max-steps <n> --output <dir> [flags]
+               [-- <sanderling test flags>]
+
+Every implementation is served on its own port, so every one is its own origin
+and none can read another's stored state, and every one is swept with the same
+seeds and the same step budget. Each run's arm is the implementation's name.
+
+Everything after a bare -- reaches every sanderling test call through the
+campaign tool.
+`
+
+func parseArguments(arguments []string, stderr io.Writer) (config, error) {
+	flagSet := flag.NewFlagSet("corpus-sweep", flag.ContinueOnError)
+	flagSet.SetOutput(stderr)
+	flagSet.Usage = func() {
+		fmt.Fprint(stderr, usage)
+		flagSet.PrintDefaults()
+	}
+	var configuration config
+	var seedSpecification string
+	var selection string
+	flagSet.StringVar(
+		&configuration.corpusRoot,
+		"corpus",
+		"",
+		"root of the checked-out corpus, holding examples/ (required)",
+	)
+	flagSet.StringVar(
+		&configuration.specPath,
+		"spec",
+		"",
+		"path to the property set every implementation is run against (required)",
+	)
+	flagSet.StringVar(
+		&seedSpecification,
+		"seeds",
+		"",
+		"seeds every implementation runs: ranges and lists, e.g. 1-10,20 (required)",
+	)
+	flagSet.StringVar(
+		&selection,
+		"implementations",
+		"",
+		"comma-separated subset of the population to sweep (default: all of it)",
+	)
+	flagSet.IntVar(
+		&configuration.maxSteps,
+		"max-steps",
+		0,
+		"per-run step budget, identical across implementations (required, must be positive)",
+	)
+	flagSet.DurationVar(
+		&configuration.duration,
+		"duration",
+		5*time.Minute,
+		"per-run wall-clock ceiling passed to each campaign",
+	)
+	flagSet.IntVar(
+		&configuration.concurrency,
+		"concurrency",
+		defaultConcurrency,
+		"implementations swept at once",
+	)
+	flagSet.IntVar(
+		&configuration.basePort,
+		"base-port",
+		defaultBasePort,
+		"first port served; each implementation takes the next one in population order",
+	)
+	flagSet.StringVar(
+		&configuration.outputDirectory,
+		"output",
+		"",
+		"campaign tree to create (required)",
+	)
+	flagSet.StringVar(
+		&configuration.campaignPath,
+		"campaign",
+		"campaign",
+		"campaign binary to invoke per implementation and seed",
+	)
+	flagSet.StringVar(
+		&configuration.sanderlingPath,
+		"sanderling",
+		"sanderling",
+		"sanderling binary each campaign invokes",
+	)
+	if err := flagSet.Parse(arguments); err != nil {
+		return config{}, err
+	}
+	configuration.extraArguments = flagSet.Args()
+
+	for name, value := range map[string]string{
+		"--corpus": configuration.corpusRoot,
+		"--spec":   configuration.specPath,
+		"--seeds":  seedSpecification,
+		"--output": configuration.outputDirectory,
+	} {
+		if value == "" {
+			return config{}, fmt.Errorf("%s is required", name)
+		}
+	}
+	if configuration.maxSteps <= 0 {
+		return config{}, fmt.Errorf(
+			"--max-steps must be positive: every implementation needs the same step budget",
+		)
+	}
+	if configuration.duration <= 0 {
+		return config{}, fmt.Errorf(
+			"--duration must be positive: %s",
+			configuration.duration,
+		)
+	}
+	if configuration.concurrency <= 0 {
+		return config{}, fmt.Errorf(
+			"--concurrency must be positive: %d",
+			configuration.concurrency,
+		)
+	}
+	if configuration.basePort < 1024 || configuration.basePort > 65535 {
+		return config{}, fmt.Errorf(
+			"--base-port %d is outside 1024-65535",
+			configuration.basePort,
+		)
+	}
+	seeds, err := seedspec.Parse(seedSpecification)
+	if err != nil {
+		return config{}, fmt.Errorf("--seeds: %w", err)
+	}
+	configuration.seeds = seeds
+	names, err := selectImplementations(selection)
+	if err != nil {
+		return config{}, fmt.Errorf("--implementations: %w", err)
+	}
+	configuration.implementations = names
+	for name, value := range map[string]*string{
+		"--corpus": &configuration.corpusRoot,
+		"--spec":   &configuration.specPath,
+		"--output": &configuration.outputDirectory,
+	} {
+		absolute, err := filepath.Abs(*value)
+		if err != nil {
+			return config{}, fmt.Errorf("%s: %w", name, err)
+		}
+		*value = absolute
+	}
+	return configuration, nil
+}
+
+func campaignDirectory(
+	configuration config,
+	target implementation,
+	seed string,
+) string {
+	return filepath.Join(
+		configuration.outputDirectory,
+		target.Name,
+		"seed-"+seed,
+	)
+}
+
+// campaignArguments builds one campaign invocation. The arm is the
+// implementation's name, so every run in the tree can be attributed to the
+// implementation it came from without reading back which port served it.
+func campaignArguments(
+	configuration config,
+	target implementation,
+	seed string,
+) []string {
+	arguments := []string{
+		"--spec", configuration.specPath,
+		"--bundle-id", target.URL(),
+		"--platform", platform,
+		"--arm", target.Name,
+		"--generator", generator,
+		"--max-steps", strconv.Itoa(configuration.maxSteps),
+		"--duration", configuration.duration.String(),
+		"--seeds", seed,
+		"--sanderling", configuration.sanderlingPath,
+		"--output", campaignDirectory(configuration, target, seed),
+	}
+	if len(configuration.extraArguments) > 0 {
+		arguments = append(arguments, "--")
+		arguments = append(arguments, configuration.extraArguments...)
+	}
+	return arguments
+}
+
+func run(arguments []string, stdout, stderr io.Writer) error {
+	configuration, err := parseArguments(arguments, stderr)
+	if err != nil {
+		return err
+	}
+	ctx, cancel := signal.NotifyContext(
+		context.Background(),
+		os.Interrupt,
+		syscall.SIGTERM,
+	)
+	defer cancel()
+	return runSweep(ctx, configuration, stdout)
+}
+
+func main() {
+	if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return
+		}
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+}