package runner import ( "bytes" "context" "encoding/json" "errors" "fmt" "strconv" "testing" "time" "github.com/priyanshujain/sanderling/internal/driver" ) // carrierSpec registers one extractor whose value the page supplies. It stands // in for every spec whose getters carry state across steps (folio's last-seen // Home total, its submit counters): what matters is that ASKING the page for // the value is what advances it. const carrierSpec = ` import { actions, extract } from "@sanderling/spec"; const carrier = extract("carrier", () => 0); globalThis.properties = {}; globalThis.actions = actions(() => []); ` // carrierWebDriver is a web target that alternates between a cross-fading // hierarchy (which the runner discards as transitional) and a settled one, and // whose page-side extractor advances a counter on every evaluation - exactly // what a spec-authored carrier does in V8. type carrierWebDriver struct { webDriverBase transitional bool snapshots int reads int } func (d *carrierWebDriver) Snapshot(ctx context.Context) (string, driver.Image, error) { _, image, err := d.Driver.Snapshot(ctx) d.snapshots++ if !d.transitional { return `{"attributes":{"resource-id":"HomeScreen"},"children":[]}`, image, err } // A genuine cross-fade: two live routes, and a tree that keeps changing // between retries so the runner spends its whole retry budget on it. return fmt.Sprintf(`{"attributes":{"resource-id":"root"},"children":[ {"attributes":{"resource-id":"HomeScreen","text":"frame-%d"},"children":[]}, {"attributes":{"resource-id":"LedgerScreen"},"children":[]} ]}`, d.snapshots), image, err } // A web target says so. The runner's per-step hierarchy reread is android-only, // and a fake claiming android would take a path no chrome run takes. func (d *carrierWebDriver) Health(context.Context) (driver.Health, error) { return driver.Health{Ready: true, Version: "fake", Platform: "web"}, nil } func (d *carrierWebDriver) EvaluateExtractors(context.Context) (map[int]json.RawMessage, error) { d.reads++ return map[int]json.RawMessage{0: json.RawMessage(strconv.Itoa(d.reads))}, nil } // NextActionFromV8 runs once per step, after the hierarchy fetch, so flipping // here makes every other step a cross-fade. func (d *carrierWebDriver) NextActionFromV8(context.Context) (json.RawMessage, error) { d.transitional = !d.transitional return json.RawMessage(`{"kind":"Tap","x":5,"y":5}`), nil } // TestRunner_TransitionalStepNeverAdvancesThePageCarrier pins the ordering the // web path depends on. The page-side extractors must run only on steps the // verifier accepts: their getters advance spec state every time they evaluate, // so evaluating them on a step whose values are then discarded leaves the page // one window ahead of the verifier. The next accepted pair then brackets two // committed transactions while having counted one submit, and the property // convicts an app that did nothing wrong. func TestRunner_TransitionalStepNeverAdvancesThePageCarrier(t *testing.T) { state := newHarnessWithSpec(t, carrierSpec) web := &carrierWebDriver{webDriverBase: webDriverBase{Driver: state.mock}} summary := state.run(t, Options{Duration: 30 * time.Second, MaxSteps: 5, Driver: web}) if summary.Steps != 5 { t.Fatalf("steps = %d, want 5", summary.Steps) } verified, transitional := 0, 0 previous := 0 for _, line := range readTraceLines(t, state.writer.Directory()) { if line.Transitional { transitional++ continue } verified++ change, ok := line.ExtractorChanges["carrier"] if !ok { t.Fatalf("step %d: no carrier value reached the verifier", line.Step) } current, convErr := strconv.Atoi(string(change.Curr)) if convErr != nil { t.Fatalf("step %d: carrier value %s: %v", line.Step, change.Curr, convErr) } if current != previous+1 { t.Errorf("step %d: carrier went %d -> %d; the page advanced it on a "+ "step the verifier discarded, so the verifier's window is wider "+ "than the one the spec counted actions over", line.Step, previous, current) } previous = current } if verified == 0 || transitional == 0 { t.Fatalf("need both kinds of step to prove anything: %d verified, %d transitional", verified, transitional) } if web.reads != verified { t.Errorf("the page evaluated its extractors %d time(s) across %d verified step(s); "+ "every evaluation the verifier does not use still advances spec state", web.reads, verified) } } // installFailsWebDriver is a web target whose page cannot take the runner's // lastAction: an older published @sanderling/spec runtime, a bundle that never // installed, a tab that navigated away from it. type installFailsWebDriver struct { webDriverBase } func (d *installFailsWebDriver) EvaluateExtractors(context.Context) (map[int]json.RawMessage, error) { return map[int]json.RawMessage{0: json.RawMessage(`1`)}, nil } func (d *installFailsWebDriver) NextActionFromV8(context.Context) (json.RawMessage, error) { return json.RawMessage(`{"kind":"Tap","x":5,"y":5}`), nil } func (d *installFailsWebDriver) SetLastAction(context.Context, json.RawMessage) error { return errors.New("__sanderlingSetLastAction__ is not a function") } // TestRunner_LastActionInstallFailureFailsTheRun covers the other half of the // same trust boundary. A run that cannot install lastAction in the page cannot // apply the page's extractor values either, so the step keeps goja's // dump-derived readings while the step before it holds the page's, and a delta // property compares two producers and fires. Downgraded to a warning that is a // green run reporting a violation nobody can reproduce. func TestRunner_LastActionInstallFailureFailsTheRun(t *testing.T) { state := newHarnessWithSpec(t, carrierSpec) web := &installFailsWebDriver{webDriverBase: webDriverBase{Driver: state.mock}} _, err := state.tryRun(t, Options{Duration: 2 * time.Second, MaxSteps: 3, Driver: web}) if err == nil { t.Fatal("Run succeeded with a page that cannot take lastAction; the run " + "reported green while its extractor values came from two engines") } if !bytes.Contains([]byte(err.Error()), []byte("install last action")) { t.Errorf("Run error = %v, want it to name the failed lastAction install", err) } } // logInstallFailsWebDriver takes lastAction and refuses the logs, the shape a // page carrying an older published @sanderling/spec runtime has: it knows the // action setter and not the log one. type logInstallFailsWebDriver struct { *installFailsWebDriver } func (d *logInstallFailsWebDriver) SetLastAction(context.Context, json.RawMessage) error { return nil } func (d *logInstallFailsWebDriver) SetLogs(context.Context, json.RawMessage) error { return errors.New("__sanderlingSetLogs__ is not a function") } // TestRunner_LogInstallFailureFailsTheRun holds the log channel to the same // standard as the action one. The driver having the console errors decides // nothing on web: the page's reading of every extractor replaces the host's, so // a run that cannot put the entries back into the page evaluates noLogcatErrors // against an empty array and reports green on a console full of errors. // Continuing past this is the vacuity the whole install exists to prevent. func TestRunner_LogInstallFailureFailsTheRun(t *testing.T) { state := newHarnessWithSpec(t, carrierSpec) web := &logInstallFailsWebDriver{ installFailsWebDriver: &installFailsWebDriver{webDriverBase: webDriverBase{Driver: state.mock}}, } _, err := state.tryRun(t, Options{Duration: 2 * time.Second, MaxSteps: 3, Driver: web}) if err == nil { t.Fatal("Run succeeded with a page that cannot take the step's logs; " + "every property reading the log stream ran against an empty array") } if !bytes.Contains([]byte(err.Error()), []byte("install logs")) { t.Errorf("Run error = %v, want it to name the failed log install", err) } }