mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 03:57:09 +00:00
feat(oracle-reduction): replay stored traces under four reduced oracles
re-evaluates each trace offline under the full engine, a crash-only detector, a single-state check and a single-step property triple, and reports what each refutes: the oracles vary while the traces stay fixed, which separates a defect an oracle cannot express from one an explorer never reached. a disagreement with the verdicts a run recorded exits nonzero rather than being counted as a finding.
This commit is contained in:
pj committed
1 parent
a0a8c9c710
commit
a45ba76d8e
7 files changed
+2182
No files matched your search
@@ -0,0 +1,312 @@
|
||||
// Command oracle-reduction re-evaluates stored traces offline under four
|
||||
// oracles and reports what each one refutes: the full engine, a crash-only
|
||||
// detector, a single-state check, and a single-step property triple. The
|
||||
// oracles vary while the traces stay fixed, which is what separates a defect an
|
||||
// oracle cannot express from one an explorer never reached.
|
||||
//
|
||||
// The offline engine has to reproduce the verdicts each run recorded. A
|
||||
// disagreement is a bug here or a gap in the trace, so it is reported as a
|
||||
// mismatch and exits nonzero rather than being counted as a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/priyanshujain/sanderling/internal/testrun"
|
||||
)
|
||||
|
||||
const usage = `oracle-reduction replays stored traces under reduced oracles.
|
||||
|
||||
Usage:
|
||||
oracle-reduction --runs <dir> [--output <path>] [--spec <path>]
|
||||
|
||||
--runs is scanned recursively; every directory holding meta.json and
|
||||
trace.jsonl is one trace. Each trace's spec is bundled from the path its
|
||||
meta.json records unless --spec overrides it.
|
||||
|
||||
Exit status is 2 when the offline engine disagreed with any run's recorded
|
||||
verdicts, which blocks the experiment rather than reporting the difference as
|
||||
noise.
|
||||
|
||||
A reduced oracle whose rewrite no longer states a property reports
|
||||
cannot_express for it rather than a verdict, and the property is temporal-only
|
||||
when the other reductions also fail to state or refute it. A property whose
|
||||
window is longer than the two observations a triple spans is reported with
|
||||
single_step_truncates_window, which is the form that decides it.
|
||||
`
|
||||
|
||||
type config struct {
|
||||
runsRoot string
|
||||
outputPath string
|
||||
specPath string
|
||||
// hostExtractors re-runs the spec's extractor getters over each stored
|
||||
// hierarchy instead of replaying the values a web run's page computed. It
|
||||
// asks a different question of the same trace: whether the stored tree
|
||||
// alone carries what the properties read.
|
||||
hostExtractors bool
|
||||
}
|
||||
|
||||
func parseArguments(arguments []string, stderr io.Writer) (config, error) {
|
||||
flagSet := flag.NewFlagSet("oracle-reduction", flag.ContinueOnError)
|
||||
flagSet.SetOutput(stderr)
|
||||
flagSet.Usage = func() {
|
||||
fmt.Fprint(stderr, usage)
|
||||
flagSet.PrintDefaults()
|
||||
}
|
||||
var configuration config
|
||||
flagSet.StringVar(
|
||||
&configuration.runsRoot,
|
||||
"runs",
|
||||
"",
|
||||
"directory tree holding the run directories to replay (required)",
|
||||
)
|
||||
flagSet.StringVar(
|
||||
&configuration.outputPath,
|
||||
"output",
|
||||
"",
|
||||
"file to write the per-trace JSONL to (default stdout)",
|
||||
)
|
||||
flagSet.StringVar(
|
||||
&configuration.specPath,
|
||||
"spec",
|
||||
"",
|
||||
"spec to bundle instead of the one each meta.json records",
|
||||
)
|
||||
flagSet.BoolVar(
|
||||
&configuration.hostExtractors,
|
||||
"host-extractors",
|
||||
false,
|
||||
"re-run the spec's extractors over each stored hierarchy instead of replaying the values a web run's page computed",
|
||||
)
|
||||
if err := flagSet.Parse(arguments); err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
if configuration.runsRoot == "" {
|
||||
return config{}, fmt.Errorf("--runs is required")
|
||||
}
|
||||
return configuration, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
configuration, err := parseArguments(os.Args[1:], os.Stderr)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "oracle-reduction: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
code, err := run(configuration, os.Stdout, os.Stderr)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "oracle-reduction: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func run(configuration config, stdout, stderr io.Writer) (int, error) {
|
||||
directories, err := discoverRuns(configuration.runsRoot)
|
||||
if err != nil {
|
||||
return 1, err
|
||||
}
|
||||
if len(directories) == 0 {
|
||||
return 1, fmt.Errorf(
|
||||
"no run directories under %s",
|
||||
configuration.runsRoot,
|
||||
)
|
||||
}
|
||||
|
||||
output := stdout
|
||||
if configuration.outputPath != "" {
|
||||
file, createErr := os.Create(configuration.outputPath)
|
||||
if createErr != nil {
|
||||
return 1, createErr
|
||||
}
|
||||
defer file.Close()
|
||||
output = file
|
||||
}
|
||||
encoder := json.NewEncoder(output)
|
||||
|
||||
var reports []runReport
|
||||
rejected := 0
|
||||
for _, directory := range directories {
|
||||
loaded, loadErr := loadRun(directory)
|
||||
if loadErr != nil {
|
||||
rejected++
|
||||
fmt.Fprintf(stderr, "skipped %s: %v\n", directory, loadErr)
|
||||
continue
|
||||
}
|
||||
specPath := loaded.Meta.SpecPath
|
||||
if configuration.specPath != "" {
|
||||
specPath = configuration.specPath
|
||||
}
|
||||
bundle, bundleErr := testrun.BundleSpec(specPath, loaded.Meta.Seed)
|
||||
if bundleErr != nil {
|
||||
return 1, fmt.Errorf(
|
||||
"%s: bundle %s: %w",
|
||||
directory,
|
||||
specPath,
|
||||
bundleErr,
|
||||
)
|
||||
}
|
||||
if loaded.Meta.BundleSHA256 != "" &&
|
||||
bundle.SHA256 != loaded.Meta.BundleSHA256 {
|
||||
// The bundler writes each module's path into the output relative to
|
||||
// the working directory, so this differs whenever the replay is
|
||||
// invoked from somewhere else than the run was. What a changed spec
|
||||
// would actually break is caught by the property set, the extractor
|
||||
// names and the residual comparison below.
|
||||
fmt.Fprintf(stderr,
|
||||
"note: %s bundles to %s here and the run recorded %s\n",
|
||||
directory, bundle.SHA256[:12], loaded.Meta.BundleSHA256[:12])
|
||||
}
|
||||
report, replayErr := replay(
|
||||
loaded,
|
||||
string(bundle.JavaScript),
|
||||
configuration.hostExtractors,
|
||||
)
|
||||
if replayErr != nil {
|
||||
return 1, fmt.Errorf("%s: %w", directory, replayErr)
|
||||
}
|
||||
if err := encoder.Encode(report); err != nil {
|
||||
return 1, err
|
||||
}
|
||||
reports = append(reports, report)
|
||||
}
|
||||
|
||||
summarize(reports, rejected, stderr)
|
||||
for _, report := range reports {
|
||||
if !report.Valid {
|
||||
return 2, nil
|
||||
}
|
||||
}
|
||||
if len(reports) == 0 {
|
||||
return 1, fmt.Errorf(
|
||||
"every run directory was rejected; nothing was replayed",
|
||||
)
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func summarize(reports []runReport, rejected int, out io.Writer) {
|
||||
invalid := 0
|
||||
crashed := 0
|
||||
weakest := map[string]int{}
|
||||
byClass := map[string]map[string]int{}
|
||||
inexpressible := map[string]map[string]bool{}
|
||||
unmatched := 0
|
||||
engineRefutations := 0
|
||||
for _, report := range reports {
|
||||
if !report.Valid {
|
||||
invalid++
|
||||
}
|
||||
if report.CrashOnly.Fired {
|
||||
crashed++
|
||||
}
|
||||
for _, property := range report.Properties {
|
||||
recordInexpressible(
|
||||
inexpressible,
|
||||
"single-state",
|
||||
property.SingleState,
|
||||
property.Property,
|
||||
)
|
||||
recordInexpressible(
|
||||
inexpressible,
|
||||
"single-step",
|
||||
property.SingleStep,
|
||||
property.Property,
|
||||
)
|
||||
if !property.Engine.Refuted {
|
||||
if property.SingleState.Refuted || property.SingleStep.Refuted {
|
||||
unmatched++
|
||||
}
|
||||
continue
|
||||
}
|
||||
engineRefutations++
|
||||
weakest[property.Weakest]++
|
||||
if byClass[property.Class] == nil {
|
||||
byClass[property.Class] = map[string]int{}
|
||||
}
|
||||
byClass[property.Class][property.Weakest]++
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintf(
|
||||
out,
|
||||
"\ntraces replayed: %d (rejected: %d)\n",
|
||||
len(reports),
|
||||
rejected,
|
||||
)
|
||||
fmt.Fprintf(
|
||||
out,
|
||||
"validity: %d of %d reproduced the recorded verdicts exactly\n",
|
||||
len(reports)-invalid,
|
||||
len(reports),
|
||||
)
|
||||
fmt.Fprintf(out, "traces where crash-only fired: %d\n", crashed)
|
||||
fmt.Fprintf(out, "engine refutations: %d\n", engineRefutations)
|
||||
if engineRefutations > 0 {
|
||||
fmt.Fprintf(out, "weakest refuting oracle: %s\n", counts(weakest))
|
||||
for _, class := range sortedKeys(byClass) {
|
||||
fmt.Fprintf(out, " %s: %s\n", class, counts(byClass[class]))
|
||||
}
|
||||
fmt.Fprintf(out, "temporal-only fraction: %.3f\n",
|
||||
float64(weakest["temporal-only"])/float64(engineRefutations))
|
||||
}
|
||||
fmt.Fprintf(
|
||||
out,
|
||||
"properties a reduced oracle cannot express: %s\n",
|
||||
counts(distinct(inexpressible)),
|
||||
)
|
||||
fmt.Fprintf(
|
||||
out,
|
||||
"reduced-oracle refutations the engine did not make: %d\n",
|
||||
unmatched,
|
||||
)
|
||||
}
|
||||
|
||||
// recordInexpressible counts a property once per oracle however many traces it
|
||||
// appears on, because whether an oracle can state a property is a fact about
|
||||
// the property and not about the run.
|
||||
func recordInexpressible(
|
||||
seen map[string]map[string]bool,
|
||||
oracle string,
|
||||
finding refutation,
|
||||
property string,
|
||||
) {
|
||||
if !finding.CannotExpress {
|
||||
return
|
||||
}
|
||||
if seen[oracle] == nil {
|
||||
seen[oracle] = map[string]bool{}
|
||||
}
|
||||
seen[oracle][property] = true
|
||||
}
|
||||
|
||||
func distinct(seen map[string]map[string]bool) map[string]int {
|
||||
sizes := map[string]int{}
|
||||
for oracle, properties := range seen {
|
||||
sizes[oracle] = len(properties)
|
||||
}
|
||||
return sizes
|
||||
}
|
||||
|
||||
func counts(values map[string]int) string {
|
||||
parts := make([]string, 0, len(values))
|
||||
for _, key := range sortedKeys(values) {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", key, values[key]))
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func sortedKeys[V any](values map[string]V) []string {
|
||||
keys := make([]string, 0, len(values))
|
||||
for key := range values {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
Reference in new issue
Block a user