package main import ( "context" "encoding/json" "errors" "fmt" "io" "os" "os/exec" "path/filepath" "strconv" "sync" "time" ) // resolveBinaries turns campaign and sanderling into absolute paths before // anything is served. Each campaign runs from the sweep's own directory, and a // binary that is missing altogether has to stop the sweep here rather than fail // once per implementation and seed. Every one that is missing is named // together, in flag order: stopping at the first turns that single stop into // one rerun per missing binary. func resolveBinaries(configuration *config) error { var missing []error for _, binary := range []struct { name string value *string }{ {"--campaign", &configuration.campaignPath}, {"--sanderling", &configuration.sanderlingPath}, } { resolved, err := exec.LookPath(*binary.value) if err != nil { missing = append(missing, fmt.Errorf("%s: %w", binary.name, err)) continue } absolute, err := filepath.Abs(resolved) if err != nil { missing = append(missing, fmt.Errorf("%s: %w", binary.name, err)) continue } *binary.value = absolute } return errors.Join(missing...) } type sweep struct { configuration config stdout io.Writer records io.Writer servers map[string]*staticServer mutex sync.Mutex stalled int failedRuns int totalRuns int } func runSweep( ctx context.Context, configuration config, stdout io.Writer, ) error { if _, err := os.Stat(filepath.Join(configuration.outputDirectory, manifestFileName)); err == nil { return fmt.Errorf( "%s already exists in %s: pick a fresh --output so two sweeps do not share a directory", manifestFileName, configuration.outputDirectory, ) } if err := verifyCorpus(configuration.corpusRoot); err != nil { return err } implementations, err := planImplementations( configuration.corpusRoot, configuration.implementations, configuration.basePort, ) if err != nil { return err } if err := resolveBinaries(&configuration); err != nil { return err } if _, err := os.Stat(configuration.specPath); err != nil { return fmt.Errorf("--spec: %w", err) } if err := os.MkdirAll(configuration.outputDirectory, 0o755); err != nil { return fmt.Errorf("create sweep dir: %w", err) } host, _ := os.Hostname() if err := writeManifest(configuration.outputDirectory, buildManifest(configuration, implementations, host, time.Now().UTC())); err != nil { return fmt.Errorf("write %s: %w", manifestFileName, err) } recordsFile, err := os.OpenFile( filepath.Join(configuration.outputDirectory, recordsFileName), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644, ) if err != nil { return fmt.Errorf("open %s: %w", recordsFileName, err) } defer recordsFile.Close() running := &sweep{ configuration: configuration, stdout: stdout, records: recordsFile, } // Every port is bound before any run starts. A port already taken means // that implementation cannot have an origin of its own, and continuing // without one is what the separate origins are there to prevent. if err := running.serveAll(implementations); err != nil { return err } defer running.stopAll() fmt.Fprintf( stdout, "sweep: %d implementations, %d seeds each, %d at a time, %s\n", len( implementations, ), len(configuration.seeds), configuration.concurrency, configuration.outputDirectory, ) running.work(ctx, implementations) fmt.Fprintf( stdout, "sweep complete: %d of %d implementations never ran, %d of %d campaigns failed\n", running.stalled, len(implementations), running.failedRuns, running.totalRuns, ) if running.stalled > 0 || running.failedRuns > 0 { return fmt.Errorf( "%d of %d implementations never ran and %d of %d campaigns failed; see %s", running.stalled, len(implementations), running.failedRuns, running.totalRuns, recordsFileName, ) } return nil } func (s *sweep) serveAll(implementations []implementation) error { s.servers = make(map[string]*staticServer, len(implementations)) for _, target := range implementations { server, err := startStaticServer(s.configuration.corpusRoot, target) if err != nil { s.stopAll() return err } s.servers[target.Name] = server } return nil } func (s *sweep) stopAll() { for _, server := range s.servers { server.stop() } } func (s *sweep) work(ctx context.Context, implementations []implementation) { queue := make(chan implementation, len(implementations)) for _, target := range implementations { queue <- target } close(queue) workers := min(s.configuration.concurrency, len(implementations)) var waitGroup sync.WaitGroup for range workers { waitGroup.Add(1) go func() { defer waitGroup.Done() for target := range queue { if ctx.Err() != nil { return } s.report(s.runImplementation(ctx, target)) } }() } waitGroup.Wait() } // runImplementation carries one implementation through all of its seeds. A // document that does not answer is recorded and the sweep moves on: one // implementation must not cost the other forty-two their runs. func (s *sweep) runImplementation( ctx context.Context, target implementation, ) (record implementationRecord) { record = implementationRecord{ Name: target.Name, Document: target.Document, Port: target.Port, Origin: target.Origin(), URL: target.URL(), StartedAt: time.Now().UTC(), } started := time.Now() defer func() { record.MonotonicMillis = time.Since(started).Milliseconds() }() if err := os.MkdirAll(filepath.Join(s.configuration.outputDirectory, target.Name), 0o755); err != nil { record.FailedStage = stageServe record.Error = err.Error() return record } if err := s.servers[target.Name].waitReady(ctx); err != nil { record.FailedStage = stageServe record.Error = err.Error() return record } for _, seed := range s.configuration.seeds { if ctx.Err() != nil { return record } record.Runs = append(record.Runs, s.runSeed(ctx, target, seed)) } return record } func (s *sweep) runSeed( ctx context.Context, target implementation, seed int64, ) (record runRecord) { seedText := strconv.FormatInt(seed, 10) directory := campaignDirectory(s.configuration, target, seedText) record = runRecord{Seed: seed, CampaignDirectory: directory} started := time.Now() defer func() { record.MonotonicMillis = time.Since(started).Milliseconds() }() if err := os.MkdirAll(directory, 0o755); err != nil { record.ExitCode = -1 record.LaunchError = err.Error() return record } exitCode, err := runCommand( ctx, s.configuration.campaignPath, campaignArguments( s.configuration, target, seedText, ), filepath.Join(directory, "campaign.log"), ) record.ExitCode = exitCode if err != nil { record.LaunchError = err.Error() } return record } func runCommand( ctx context.Context, binary string, arguments []string, logPath string, ) (int, error) { logFile, err := os.Create(logPath) if err != nil { return -1, err } defer logFile.Close() command := exec.CommandContext(ctx, binary, arguments...) command.Stdout = logFile command.Stderr = logFile err = command.Run() if err == nil { return 0, nil } var exitError *exec.ExitError if errors.As(err, &exitError) { return exitError.ExitCode(), nil } return -1, err } func (s *sweep) report(record implementationRecord) { s.mutex.Lock() defer s.mutex.Unlock() if record.FailedStage != "" { s.stalled++ } s.totalRuns += len(record.Runs) failed := 0 for _, run := range record.Runs { if run.ExitCode != 0 { failed++ } } s.failedRuns += failed if err := json.NewEncoder(s.records).Encode(record); err != nil { fmt.Fprintf(s.stdout, "warning: %s record: %v\n", record.Name, err) } elapsed := time.Duration(record.MonotonicMillis) * time.Millisecond if record.FailedStage != "" { fmt.Fprintf(s.stdout, "%s port=%d failed at %s: %s (%s)\n", record.Name, record.Port, record.FailedStage, record.Error, elapsed) return } fmt.Fprintf(s.stdout, "%s origin=%s campaigns=%d failed=%d elapsed=%s\n", record.Name, record.Origin, len(record.Runs), failed, elapsed) }