// Command corpus-sweep runs one specification against every implementation in a
// served corpus of independent implementations of the same requirement. It
// serves each one on its own port, which is what keeps them apart: the corpus
// holds pairs that write the same localStorage key, and one origin shared
// between two of them is one stored record shared between them.
package main
import (
"context"
"errors"
"flag"
"fmt"
"io"
"os"
"os/signal"
"path/filepath"
"strconv"
"syscall"
"time"
"github.com/priyanshujain/sanderling/internal/seedspec"
)
// The generator and the platform are fixed rather than exposed: the
// pre-registration runs the seeded policy against the served corpus, and a
// sweep that could quietly run something else records a comparison nobody made.
const (
generator = "seeded"
platform = "web"
)
// defaultConcurrency is how many implementations are swept at once. fleet.md
// measured eight concurrent web campaigns clean at about 1.1 GB resident each,
// parallel efficiency 0.83 at eight against 0.87 at six, and said to re-measure
// before trusting anything above eight on a contended host. Six sits at the
// better efficiency and leaves the emulator farm that shares the host its
// slots. Serving costs nothing here: the corpus needs no build and no separate
// server process, so a worker is one browser.
const defaultConcurrency = 6
// defaultBasePort starts above the range the model-implementation sweep hands
// out, so the two can run on one host without either being served the other's
// pages.
const defaultBasePort = 5400
type config struct {
corpusRoot string
specPath string
outputDirectory string
implementations []string
seeds []int64
maxSteps int
duration time.Duration
concurrency int
basePort int
campaignPath string
sanderlingPath string
extraArguments []string
}
const usage = `corpus-sweep runs one seeded campaign per implementation of a served corpus.
Usage:
corpus-sweep --corpus
--spec --seeds
--max-steps --output [flags]
[-- ]
Every implementation is served on its own port, so every one is its own origin
and none can read another's stored state, and every one is swept with the same
seeds and the same step budget. Each run's arm is the implementation's name.
Everything after a bare -- reaches every sanderling test call through the
campaign tool.
`
func parseArguments(arguments []string, stderr io.Writer) (config, error) {
flagSet := flag.NewFlagSet("corpus-sweep", flag.ContinueOnError)
flagSet.SetOutput(stderr)
flagSet.Usage = func() {
fmt.Fprint(stderr, usage)
flagSet.PrintDefaults()
}
var configuration config
var seedSpecification string
var selection string
flagSet.StringVar(
&configuration.corpusRoot,
"corpus",
"",
"root of the checked-out corpus, holding examples/ (required)",
)
flagSet.StringVar(
&configuration.specPath,
"spec",
"",
"path to the property set every implementation is run against (required)",
)
flagSet.StringVar(
&seedSpecification,
"seeds",
"",
"seeds every implementation runs: ranges and lists, e.g. 1-10,20 (required)",
)
flagSet.StringVar(
&selection,
"implementations",
"",
"comma-separated subset of the population to sweep (default: all of it)",
)
flagSet.IntVar(
&configuration.maxSteps,
"max-steps",
0,
"per-run step budget, identical across implementations (required, must be positive)",
)
flagSet.DurationVar(
&configuration.duration,
"duration",
5*time.Minute,
"per-run wall-clock ceiling passed to each campaign",
)
flagSet.IntVar(
&configuration.concurrency,
"concurrency",
defaultConcurrency,
"implementations swept at once",
)
flagSet.IntVar(
&configuration.basePort,
"base-port",
defaultBasePort,
"first port served; each implementation takes the next one in population order",
)
flagSet.StringVar(
&configuration.outputDirectory,
"output",
"",
"campaign tree to create (required)",
)
flagSet.StringVar(
&configuration.campaignPath,
"campaign",
"campaign",
"campaign binary to invoke per implementation and seed",
)
flagSet.StringVar(
&configuration.sanderlingPath,
"sanderling",
"sanderling",
"sanderling binary each campaign invokes",
)
if err := flagSet.Parse(arguments); err != nil {
return config{}, err
}
configuration.extraArguments = flagSet.Args()
for name, value := range map[string]string{
"--corpus": configuration.corpusRoot,
"--spec": configuration.specPath,
"--seeds": seedSpecification,
"--output": configuration.outputDirectory,
} {
if value == "" {
return config{}, fmt.Errorf("%s is required", name)
}
}
if configuration.maxSteps <= 0 {
return config{}, fmt.Errorf(
"--max-steps must be positive: every implementation needs the same step budget",
)
}
if configuration.duration <= 0 {
return config{}, fmt.Errorf(
"--duration must be positive: %s",
configuration.duration,
)
}
if configuration.concurrency <= 0 {
return config{}, fmt.Errorf(
"--concurrency must be positive: %d",
configuration.concurrency,
)
}
if configuration.basePort < 1024 || configuration.basePort > 65535 {
return config{}, fmt.Errorf(
"--base-port %d is outside 1024-65535",
configuration.basePort,
)
}
seeds, err := seedspec.Parse(seedSpecification)
if err != nil {
return config{}, fmt.Errorf("--seeds: %w", err)
}
configuration.seeds = seeds
names, err := selectImplementations(selection)
if err != nil {
return config{}, fmt.Errorf("--implementations: %w", err)
}
configuration.implementations = names
for name, value := range map[string]*string{
"--corpus": &configuration.corpusRoot,
"--spec": &configuration.specPath,
"--output": &configuration.outputDirectory,
} {
absolute, err := filepath.Abs(*value)
if err != nil {
return config{}, fmt.Errorf("%s: %w", name, err)
}
*value = absolute
}
return configuration, nil
}
func campaignDirectory(
configuration config,
target implementation,
seed string,
) string {
return filepath.Join(
configuration.outputDirectory,
target.Name,
"seed-"+seed,
)
}
// campaignArguments builds one campaign invocation. The arm is the
// implementation's name, so every run in the tree can be attributed to the
// implementation it came from without reading back which port served it.
func campaignArguments(
configuration config,
target implementation,
seed string,
) []string {
arguments := []string{
"--spec", configuration.specPath,
"--bundle-id", target.URL(),
"--platform", platform,
"--arm", target.Name,
"--generator", generator,
"--max-steps", strconv.Itoa(configuration.maxSteps),
"--duration", configuration.duration.String(),
"--seeds", seed,
"--sanderling", configuration.sanderlingPath,
"--output", campaignDirectory(configuration, target, seed),
}
if len(configuration.extraArguments) > 0 {
arguments = append(arguments, "--")
arguments = append(arguments, configuration.extraArguments...)
}
return arguments
}
func run(arguments []string, stdout, stderr io.Writer) error {
configuration, err := parseArguments(arguments, stderr)
if err != nil {
return err
}
ctx, cancel := signal.NotifyContext(
context.Background(),
os.Interrupt,
syscall.SIGTERM,
)
defer cancel()
return runSweep(ctx, configuration, stdout)
}
func main() {
if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil {
if errors.Is(err, flag.ErrHelp) {
return
}
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}
}