mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
feat(oracle-reduction): replay stored traces under four reduced oracles
re-evaluates each trace offline under the full engine, a crash-only detector, a single-state check and a single-step property triple, and reports what each refutes: the oracles vary while the traces stay fixed, which separates a defect an oracle cannot express from one an explorer never reached. a disagreement with the verdicts a run recorded exits nonzero rather than being counted as a finding.
This commit is contained in:
pj committed
1 parent
a0a8c9c710
commit
a45ba76d8e
7 files changed
+2182
No files matched your search
@@ -0,0 +1,549 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/priyanshujain/sanderling/internal/ltl"
|
||||
"github.com/priyanshujain/sanderling/internal/trace"
|
||||
"github.com/priyanshujain/sanderling/internal/verifier"
|
||||
)
|
||||
|
||||
// foregroundLossReason is the skip reason internal/runner records when the app
|
||||
// was no longer the foreground process at action time. It is the only
|
||||
// foreground signal a stored trace carries, and it is what the crash-only
|
||||
// oracle reads for "the application is no longer the foreground process".
|
||||
const foregroundLossReason = "app_left_foreground"
|
||||
|
||||
// refutation is one oracle's finding for one property on one trace. A reduced
|
||||
// oracle has three of them and not two: CannotExpress says its rewrite no
|
||||
// longer states the property, which is neither a refutation nor a clean bill.
|
||||
type refutation struct {
|
||||
Refuted bool `json:"refuted"`
|
||||
CannotExpress bool `json:"cannot_express,omitempty"`
|
||||
// Step is the observation whose evaluation produced the violation;
|
||||
// OriginStep is the observation whose obligation failed, which for a
|
||||
// deferred one is earlier.
|
||||
Step int `json:"step,omitempty"`
|
||||
OriginStep int `json:"origin_step,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
IsError bool `json:"is_error,omitempty"`
|
||||
}
|
||||
|
||||
type propertyReport struct {
|
||||
Property string `json:"property"`
|
||||
Class string `json:"class"`
|
||||
TopLevel string `json:"top_level"`
|
||||
Engine refutation `json:"engine"`
|
||||
SingleState refutation `json:"single_state"`
|
||||
SingleStep refutation `json:"single_step"`
|
||||
// SingleStepTruncatesWindow marks a property whose window the triple had to
|
||||
// shorten to two observations, so what its single-step column refutes is a
|
||||
// stronger property than the one the author wrote.
|
||||
SingleStepTruncatesWindow bool `json:"single_step_truncates_window,omitempty"`
|
||||
// Weakest names the weakest oracle that refutes this property on this
|
||||
// trace, and is empty unless the engine refuted it. "temporal-only" means
|
||||
// no reduced oracle did.
|
||||
Weakest string `json:"weakest_refuting_oracle,omitempty"`
|
||||
}
|
||||
|
||||
type crashReport struct {
|
||||
Fired bool `json:"fired"`
|
||||
FirstStep int `json:"first_step,omitempty"`
|
||||
ExceptionSteps []int `json:"exception_steps,omitempty"`
|
||||
ForegroundLossSteps []int `json:"foreground_loss_steps,omitempty"`
|
||||
ErrorLogSteps []int `json:"error_log_steps,omitempty"`
|
||||
}
|
||||
|
||||
// mismatch is one disagreement between the offline engine and the verdicts the
|
||||
// run recorded. Any of these blocks the trace's result.
|
||||
type mismatch struct {
|
||||
Property string `json:"property"`
|
||||
Field string `json:"field"`
|
||||
Online string `json:"online"`
|
||||
Offline string `json:"offline"`
|
||||
}
|
||||
|
||||
type runReport struct {
|
||||
Run string `json:"run"`
|
||||
Seed int64 `json:"seed"`
|
||||
Platform string `json:"platform"`
|
||||
Arm string `json:"arm,omitempty"`
|
||||
Generator string `json:"generator,omitempty"`
|
||||
// ReplayMode says where the extractor values under replay came from:
|
||||
// "page-extractor-values" reuses what the page computed in V8 and the run
|
||||
// evaluated against, "host-extractors" re-runs the spec's getters over the
|
||||
// stored hierarchy.
|
||||
ReplayMode string `json:"replay_mode"`
|
||||
StepsObserved int `json:"steps_observed"`
|
||||
StepsSkipped int `json:"steps_skipped"`
|
||||
Valid bool `json:"valid"`
|
||||
Mismatches []mismatch `json:"mismatches,omitempty"`
|
||||
// ResidualMismatches counts every step whose replayed residual formula
|
||||
// differed from the recorded one, of which Mismatches carries the first
|
||||
// few. The recorded residual is the engine's whole pending state, so
|
||||
// agreement on it is a stronger claim than agreement on the verdicts.
|
||||
ResidualMismatches int `json:"residual_mismatches"`
|
||||
CrashOnly crashReport `json:"crash_only"`
|
||||
Properties []propertyReport `json:"properties"`
|
||||
ActionsUnbounded int `json:"actions_without_resolved_bounds"`
|
||||
ScrollActions int `json:"scroll_last_actions"`
|
||||
}
|
||||
|
||||
// witnessRecord is one violation as either side reports it: the step it was
|
||||
// recorded at, plus the witness the engine attached.
|
||||
type witnessRecord struct {
|
||||
RecordedStep int
|
||||
OriginStep int
|
||||
DetectedStep int
|
||||
Reason string
|
||||
IsError bool
|
||||
}
|
||||
|
||||
// replay re-evaluates one run offline under all four oracles. The engine's
|
||||
// offline verdicts are compared against the ones the run recorded, and the
|
||||
// report is marked invalid on any disagreement: a mismatch is a bug here or a
|
||||
// gap in the trace, not a finding.
|
||||
func replay(
|
||||
run loadedRun,
|
||||
bundleJavaScript string,
|
||||
hostExtractors bool,
|
||||
) (runReport, error) {
|
||||
engine, err := verifier.New(
|
||||
verifier.WithSeed(uint64(run.Meta.Seed)),
|
||||
verifier.WithPlatform(run.Meta.Platform),
|
||||
verifier.WithAppPackage(run.Meta.BundleID),
|
||||
)
|
||||
if err != nil {
|
||||
return runReport{}, fmt.Errorf("verifier: %w", err)
|
||||
}
|
||||
if err := engine.Load(bundleJavaScript); err != nil {
|
||||
return runReport{}, fmt.Errorf("load spec: %w", err)
|
||||
}
|
||||
formulas, err := engine.PropertyFormulas()
|
||||
if err != nil {
|
||||
return runReport{}, err
|
||||
}
|
||||
|
||||
singleState := map[string]*ltl.Evaluator{}
|
||||
singleStep := map[string]*ltl.Evaluator{}
|
||||
for name, formula := range formulas {
|
||||
singleState[name] = ltl.NewEvaluator(singleStateFormula(formula))
|
||||
singleStep[name] = ltl.NewEvaluator(singleStepFormula(formula))
|
||||
}
|
||||
|
||||
usePageValues := run.Meta.Platform == "web" && !hostExtractors
|
||||
report := runReport{
|
||||
Run: run.Directory,
|
||||
Seed: run.Meta.Seed,
|
||||
Platform: run.Meta.Platform,
|
||||
Arm: run.Meta.Arm,
|
||||
Generator: run.Meta.Generator,
|
||||
ReplayMode: "host-extractors",
|
||||
}
|
||||
var folded []map[int]json.RawMessage
|
||||
if usePageValues {
|
||||
report.ReplayMode = "page-extractor-values"
|
||||
folded, err = extractorFold(run.Steps, engine.ExtractorNames())
|
||||
if err != nil {
|
||||
return runReport{}, err
|
||||
}
|
||||
}
|
||||
|
||||
offline := map[string]witnessRecord{}
|
||||
stateFired := map[string]int{}
|
||||
stepFired := map[string]int{}
|
||||
var residualMismatches []mismatch
|
||||
var lastAction *verifier.Action
|
||||
for position, step := range run.Steps {
|
||||
if step.NextAction != nil && step.NextAction.Selector != "" &&
|
||||
step.NextAction.ResolvedBounds == nil {
|
||||
report.ActionsUnbounded++
|
||||
}
|
||||
if step.SkippedVerification {
|
||||
report.StepsSkipped++
|
||||
residualMismatches = append(
|
||||
residualMismatches,
|
||||
compareResiduals(step, engine.Residuals())...)
|
||||
lastAction = lastActionFor(step)
|
||||
continue
|
||||
}
|
||||
if lastAction != nil && lastAction.Kind == verifier.ActionKindScroll {
|
||||
report.ScrollActions++
|
||||
}
|
||||
if err := engine.PushSnapshot(verifier.SnapshotInput{
|
||||
Tree: step.Hierarchy,
|
||||
LastAction: lastAction,
|
||||
StepTime: step.Timestamp,
|
||||
StepIndex: step.Index,
|
||||
RunStart: run.Meta.StartedAt,
|
||||
Logs: traceLogs(step.Logs),
|
||||
Exceptions: traceExceptions(step.Exceptions),
|
||||
}); err != nil {
|
||||
return runReport{}, fmt.Errorf("step %d push: %w", step.Index, err)
|
||||
}
|
||||
if usePageValues {
|
||||
skipped, overrideErr := engine.OverrideExtractorValues(
|
||||
folded[position],
|
||||
)
|
||||
if overrideErr != nil {
|
||||
return runReport{}, fmt.Errorf(
|
||||
"step %d override: %w",
|
||||
step.Index,
|
||||
overrideErr,
|
||||
)
|
||||
}
|
||||
if skipped > 0 {
|
||||
return runReport{}, fmt.Errorf(
|
||||
"step %d: %d recorded extractor values fell outside the spec's extractor list",
|
||||
step.Index,
|
||||
skipped,
|
||||
)
|
||||
}
|
||||
}
|
||||
engine.EvaluateProperties()
|
||||
for _, name := range engine.NewlyViolatedProperties() {
|
||||
offline[name] = witnessFrom(engine.Witness(name), step.Index)
|
||||
}
|
||||
residualMismatches = append(
|
||||
residualMismatches,
|
||||
compareResiduals(step, engine.Residuals())...)
|
||||
for name := range formulas {
|
||||
recordFiring(
|
||||
stateFired,
|
||||
name,
|
||||
singleState[name].ObserveAtStep(step.Timestamp, step.Index),
|
||||
step.Index,
|
||||
)
|
||||
recordFiring(
|
||||
stepFired,
|
||||
name,
|
||||
singleStep[name].ObserveAtStep(step.Timestamp, step.Index),
|
||||
step.Index,
|
||||
)
|
||||
}
|
||||
report.StepsObserved++
|
||||
lastAction = lastActionFor(step)
|
||||
}
|
||||
|
||||
finalizeIndex := run.finalizeIndex()
|
||||
for _, name := range engine.Finalize() {
|
||||
offline[name] = witnessFrom(engine.Witness(name), finalizeIndex)
|
||||
}
|
||||
for name := range formulas {
|
||||
recordFiring(
|
||||
stateFired,
|
||||
name,
|
||||
singleState[name].Finalize(),
|
||||
finalizeIndex,
|
||||
)
|
||||
recordFiring(
|
||||
stepFired,
|
||||
name,
|
||||
singleStep[name].Finalize(),
|
||||
finalizeIndex,
|
||||
)
|
||||
}
|
||||
|
||||
report.CrashOnly = crashOnly(run)
|
||||
report.Mismatches = compareVerdicts(onlineVerdicts(run), offline)
|
||||
report.ResidualMismatches = len(residualMismatches)
|
||||
if len(residualMismatches) > reportedResiduals {
|
||||
residualMismatches = residualMismatches[:reportedResiduals]
|
||||
}
|
||||
report.Mismatches = append(report.Mismatches, residualMismatches...)
|
||||
report.Valid = len(report.Mismatches) == 0
|
||||
|
||||
names := make([]string, 0, len(formulas))
|
||||
for name := range formulas {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
property := propertyReport{
|
||||
Property: name,
|
||||
Class: propertyClass(formulas[name]),
|
||||
TopLevel: topLevelForm(formulas[name]),
|
||||
Engine: engineRefutation(offline, name),
|
||||
SingleState: reducedRefutation(
|
||||
singleStateExpresses(formulas[name]),
|
||||
singleState[name],
|
||||
stateFired[name],
|
||||
),
|
||||
SingleStep: reducedRefutation(
|
||||
singleStepExpresses(formulas[name]),
|
||||
singleStep[name],
|
||||
stepFired[name],
|
||||
),
|
||||
|
||||
SingleStepTruncatesWindow: truncatesWindow(formulas[name]),
|
||||
}
|
||||
property.Weakest = weakestOracle(property, report.CrashOnly)
|
||||
report.Properties = append(report.Properties, property)
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
// reportedResiduals caps how many residual differences one trace lists. The
|
||||
// count of all of them is reported alongside; the first few are what says
|
||||
// where the two engines parted.
|
||||
const reportedResiduals = 5
|
||||
|
||||
// compareResiduals checks the replayed pending state against the one the step
|
||||
// recorded. A step the verifier skipped still recorded the residual it was
|
||||
// holding, so those steps assert that a skipped step advanced nothing.
|
||||
func compareResiduals(
|
||||
step trace.Step,
|
||||
replayed map[string]ltl.Formula,
|
||||
) []mismatch {
|
||||
if len(step.Residuals) == 0 {
|
||||
return nil
|
||||
}
|
||||
var mismatches []mismatch
|
||||
names := make([]string, 0, len(step.Residuals))
|
||||
for name := range step.Residuals {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
formula, ok := replayed[name]
|
||||
if !ok {
|
||||
mismatches = append(mismatches, mismatch{
|
||||
Property: name,
|
||||
Field: fmt.Sprintf("residual at step %d", step.Index),
|
||||
Online: string(step.Residuals[name]),
|
||||
Offline: "property not registered",
|
||||
})
|
||||
continue
|
||||
}
|
||||
encoded, err := json.Marshal(formula)
|
||||
if err != nil {
|
||||
encoded = []byte(fmt.Sprintf("%q", err.Error()))
|
||||
}
|
||||
if !bytes.Equal(encoded, step.Residuals[name]) {
|
||||
mismatches = append(mismatches, mismatch{
|
||||
Property: name,
|
||||
Field: fmt.Sprintf("residual at step %d", step.Index),
|
||||
Online: string(step.Residuals[name]),
|
||||
Offline: string(encoded),
|
||||
})
|
||||
}
|
||||
}
|
||||
return mismatches
|
||||
}
|
||||
|
||||
func witnessFrom(witness *verifier.Witness, recordedStep int) witnessRecord {
|
||||
record := witnessRecord{RecordedStep: recordedStep}
|
||||
if witness == nil {
|
||||
return record
|
||||
}
|
||||
record.OriginStep = witness.Step
|
||||
record.DetectedStep = witness.DetectedStep
|
||||
record.Reason = witness.Reason
|
||||
record.IsError = witness.IsError
|
||||
return record
|
||||
}
|
||||
|
||||
// onlineVerdicts reads the violations the run recorded, including the
|
||||
// end-of-run record a finalized liveness obligation is written to.
|
||||
func onlineVerdicts(run loadedRun) map[string]witnessRecord {
|
||||
recorded := map[string]witnessRecord{}
|
||||
steps := run.Steps
|
||||
if run.Finalize != nil {
|
||||
steps = append(append([]trace.Step(nil), steps...), *run.Finalize)
|
||||
}
|
||||
for _, step := range steps {
|
||||
for _, name := range step.Violations {
|
||||
record := witnessRecord{RecordedStep: step.Index}
|
||||
if witness, ok := step.Witnesses[name]; ok {
|
||||
record.OriginStep = witness.Step
|
||||
record.DetectedStep = witness.DetectedStep
|
||||
record.Reason = witness.Reason
|
||||
record.IsError = witness.IsError
|
||||
}
|
||||
recorded[name] = record
|
||||
}
|
||||
}
|
||||
return recorded
|
||||
}
|
||||
|
||||
func compareVerdicts(online, offline map[string]witnessRecord) []mismatch {
|
||||
var mismatches []mismatch
|
||||
names := map[string]bool{}
|
||||
for name := range online {
|
||||
names[name] = true
|
||||
}
|
||||
for name := range offline {
|
||||
names[name] = true
|
||||
}
|
||||
ordered := make([]string, 0, len(names))
|
||||
for name := range names {
|
||||
ordered = append(ordered, name)
|
||||
}
|
||||
sort.Strings(ordered)
|
||||
|
||||
for _, name := range ordered {
|
||||
recorded, wasRecorded := online[name]
|
||||
replayed, wasReplayed := offline[name]
|
||||
switch {
|
||||
case wasRecorded && !wasReplayed:
|
||||
mismatches = append(mismatches, mismatch{
|
||||
Property: name, Field: "violated",
|
||||
Online: fmt.Sprintf(
|
||||
"violated at step %d",
|
||||
recorded.RecordedStep,
|
||||
),
|
||||
Offline: "not violated",
|
||||
})
|
||||
continue
|
||||
case !wasRecorded && wasReplayed:
|
||||
mismatches = append(mismatches, mismatch{
|
||||
Property: name, Field: "violated",
|
||||
Online: "not violated",
|
||||
Offline: fmt.Sprintf(
|
||||
"violated at step %d",
|
||||
replayed.RecordedStep,
|
||||
),
|
||||
})
|
||||
continue
|
||||
case !wasRecorded:
|
||||
continue
|
||||
}
|
||||
for _, field := range []struct {
|
||||
name string
|
||||
online string
|
||||
offline string
|
||||
}{
|
||||
{"step", fmt.Sprint(recorded.RecordedStep), fmt.Sprint(replayed.RecordedStep)},
|
||||
{"origin_step", fmt.Sprint(recorded.OriginStep), fmt.Sprint(replayed.OriginStep)},
|
||||
{"detected_step", fmt.Sprint(recorded.DetectedStep), fmt.Sprint(replayed.DetectedStep)},
|
||||
{"reason", recorded.Reason, replayed.Reason},
|
||||
} {
|
||||
if field.online != field.offline {
|
||||
mismatches = append(mismatches, mismatch{
|
||||
Property: name, Field: field.name,
|
||||
Online: field.online, Offline: field.offline,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
return mismatches
|
||||
}
|
||||
|
||||
// crashOnly fires where the application left the foreground or an error
|
||||
// surface was recorded. Error-level log lines are reported alongside rather
|
||||
// than folded in: a console error is not a crash, and an analysis that wants
|
||||
// the looser detector can read the steps from here.
|
||||
func crashOnly(run loadedRun) crashReport {
|
||||
report := crashReport{}
|
||||
for _, step := range run.Steps {
|
||||
if len(step.Exceptions) > 0 {
|
||||
report.ExceptionSteps = append(report.ExceptionSteps, step.Index)
|
||||
}
|
||||
if step.ActionSkipped == foregroundLossReason {
|
||||
report.ForegroundLossSteps = append(
|
||||
report.ForegroundLossSteps,
|
||||
step.Index,
|
||||
)
|
||||
}
|
||||
for _, entry := range step.Logs {
|
||||
if entry.Level == "E" || entry.Level == "F" {
|
||||
report.ErrorLogSteps = append(report.ErrorLogSteps, step.Index)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
first := 0
|
||||
for _, step := range append(append([]int(nil), report.ExceptionSteps...), report.ForegroundLossSteps...) {
|
||||
if first == 0 || step < first {
|
||||
first = step
|
||||
}
|
||||
}
|
||||
report.Fired = first != 0
|
||||
report.FirstStep = first
|
||||
return report
|
||||
}
|
||||
|
||||
func engineRefutation(
|
||||
offline map[string]witnessRecord,
|
||||
name string,
|
||||
) refutation {
|
||||
record, ok := offline[name]
|
||||
if !ok {
|
||||
return refutation{}
|
||||
}
|
||||
return refutation{
|
||||
Refuted: true,
|
||||
Step: record.RecordedStep,
|
||||
OriginStep: record.OriginStep,
|
||||
Reason: record.Reason,
|
||||
IsError: record.IsError,
|
||||
}
|
||||
}
|
||||
|
||||
// reducedRefutation reports a reduced oracle's finding, or its inability to
|
||||
// state the property at all. The evaluator is driven either way so that the
|
||||
// verdict a reduction would have reached is never what decides whether it was
|
||||
// entitled to reach one.
|
||||
func reducedRefutation(
|
||||
expresses bool,
|
||||
evaluator *ltl.Evaluator,
|
||||
firedAt int,
|
||||
) refutation {
|
||||
if !expresses {
|
||||
return refutation{CannotExpress: true}
|
||||
}
|
||||
return evaluatorRefutation(evaluator, firedAt)
|
||||
}
|
||||
|
||||
func evaluatorRefutation(evaluator *ltl.Evaluator, firedAt int) refutation {
|
||||
violation := evaluator.Violation()
|
||||
if violation == nil {
|
||||
return refutation{}
|
||||
}
|
||||
return refutation{
|
||||
Refuted: true,
|
||||
Step: firedAt,
|
||||
OriginStep: violation.Step,
|
||||
Reason: violation.Reason,
|
||||
IsError: violation.IsError,
|
||||
}
|
||||
}
|
||||
|
||||
// recordFiring keeps the first observation at which a reduced oracle latched,
|
||||
// which the evaluator itself does not carry.
|
||||
func recordFiring(
|
||||
fired map[string]int,
|
||||
name string,
|
||||
verdict ltl.Verdict,
|
||||
step int,
|
||||
) {
|
||||
if verdict != ltl.VerdictViolated {
|
||||
return
|
||||
}
|
||||
if _, ok := fired[name]; ok {
|
||||
return
|
||||
}
|
||||
fired[name] = step
|
||||
}
|
||||
|
||||
// weakestOracle names the weakest oracle that refutes a defect the engine
|
||||
// refuted, in the order a crash detector, a single-state check and a
|
||||
// single-step triple are weaker than the engine.
|
||||
func weakestOracle(property propertyReport, crash crashReport) string {
|
||||
if !property.Engine.Refuted {
|
||||
return ""
|
||||
}
|
||||
switch {
|
||||
case crash.Fired:
|
||||
return "crash-only"
|
||||
case property.SingleState.Refuted:
|
||||
return "single-state"
|
||||
case property.SingleStep.Refuted:
|
||||
return "single-step"
|
||||
default:
|
||||
return "temporal-only"
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user