sanderling/internal/runner/web_carrier_test.go

package runner

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"testing"
	"time"

	"github.com/priyanshujain/sanderling/internal/driver"
)

// carrierSpec registers one extractor whose value the page supplies. It stands
// in for every spec whose getters carry state across steps (folio's last-seen
// Home total, its submit counters): what matters is that ASKING the page for
// the value is what advances it.
const carrierSpec = `
import { actions, extract } from "@sanderling/spec";
const carrier = extract("carrier", () => 0);
globalThis.properties = {};
globalThis.actions = actions(() => []);
`

// carrierWebDriver is a web target that alternates between a cross-fading
// hierarchy (which the runner discards as transitional) and a settled one, and
// whose page-side extractor advances a counter on every evaluation - exactly
// what a spec-authored carrier does in V8.
type carrierWebDriver struct {
	webDriverBase
	transitional bool
	snapshots    int
	reads        int
}

func (d *carrierWebDriver) Snapshot(ctx context.Context) (string, driver.Image, error) {
	_, image, err := d.Driver.Snapshot(ctx)
	d.snapshots++
	if !d.transitional {
		return `{"attributes":{"resource-id":"HomeScreen"},"children":[]}`, image, err
	}
	// A genuine cross-fade: two live routes, and a tree that keeps changing
	// between retries so the runner spends its whole retry budget on it.
	return fmt.Sprintf(`{"attributes":{"resource-id":"root"},"children":[
	  {"attributes":{"resource-id":"HomeScreen","text":"frame-%d"},"children":[]},
	  {"attributes":{"resource-id":"LedgerScreen"},"children":[]}
	]}`, d.snapshots), image, err
}

// A web target says so. The runner's per-step hierarchy reread is android-only,
// and a fake claiming android would take a path no chrome run takes.
func (d *carrierWebDriver) Health(context.Context) (driver.Health, error) {
	return driver.Health{Ready: true, Version: "fake", Platform: "web"}, nil
}

func (d *carrierWebDriver) EvaluateExtractors(context.Context) (map[int]json.RawMessage, error) {
	d.reads++
	return map[int]json.RawMessage{0: json.RawMessage(strconv.Itoa(d.reads))}, nil
}

// NextActionFromV8 runs once per step, after the hierarchy fetch, so flipping
// here makes every other step a cross-fade.
func (d *carrierWebDriver) NextActionFromV8(context.Context) (json.RawMessage, error) {
	d.transitional = !d.transitional
	return json.RawMessage(`{"kind":"Tap","x":5,"y":5}`), nil
}

// TestRunner_TransitionalStepNeverAdvancesThePageCarrier pins the ordering the
// web path depends on. The page-side extractors must run only on steps the
// verifier accepts: their getters advance spec state every time they evaluate,
// so evaluating them on a step whose values are then discarded leaves the page
// one window ahead of the verifier. The next accepted pair then brackets two
// committed transactions while having counted one submit, and the property
// convicts an app that did nothing wrong.
func TestRunner_TransitionalStepNeverAdvancesThePageCarrier(t *testing.T) {
	state := newHarnessWithSpec(t, carrierSpec)
	web := &carrierWebDriver{webDriverBase: webDriverBase{Driver: state.mock}}

	summary := state.run(t, Options{Duration: 30 * time.Second, MaxSteps: 5, Driver: web})
	if summary.Steps != 5 {
		t.Fatalf("steps = %d, want 5", summary.Steps)
	}

	verified, transitional := 0, 0
	previous := 0
	for _, line := range readTraceLines(t, state.writer.Directory()) {
		if line.Transitional {
			transitional++
			continue
		}
		verified++
		change, ok := line.ExtractorChanges["carrier"]
		if !ok {
			t.Fatalf("step %d: no carrier value reached the verifier", line.Step)
		}
		current, convErr := strconv.Atoi(string(change.Curr))
		if convErr != nil {
			t.Fatalf("step %d: carrier value %s: %v", line.Step, change.Curr, convErr)
		}
		if current != previous+1 {
			t.Errorf("step %d: carrier went %d -> %d; the page advanced it on a "+
				"step the verifier discarded, so the verifier's window is wider "+
				"than the one the spec counted actions over",
				line.Step, previous, current)
		}
		previous = current
	}
	if verified == 0 || transitional == 0 {
		t.Fatalf("need both kinds of step to prove anything: %d verified, %d transitional",
			verified, transitional)
	}
	if web.reads != verified {
		t.Errorf("the page evaluated its extractors %d time(s) across %d verified step(s); "+
			"every evaluation the verifier does not use still advances spec state",
			web.reads, verified)
	}
}

// installFailsWebDriver is a web target whose page cannot take the runner's
// lastAction: an older published @sanderling/spec runtime, a bundle that never
// installed, a tab that navigated away from it.
type installFailsWebDriver struct {
	webDriverBase
}

func (d *installFailsWebDriver) EvaluateExtractors(context.Context) (map[int]json.RawMessage, error) {
	return map[int]json.RawMessage{0: json.RawMessage(`1`)}, nil
}

func (d *installFailsWebDriver) NextActionFromV8(context.Context) (json.RawMessage, error) {
	return json.RawMessage(`{"kind":"Tap","x":5,"y":5}`), nil
}

func (d *installFailsWebDriver) SetLastAction(context.Context, json.RawMessage) error {
	return errors.New("__sanderlingSetLastAction__ is not a function")
}

// TestRunner_LastActionInstallFailureFailsTheRun covers the other half of the
// same trust boundary. A run that cannot install lastAction in the page cannot
// apply the page's extractor values either, so the step keeps goja's
// dump-derived readings while the step before it holds the page's, and a delta
// property compares two producers and fires. Downgraded to a warning that is a
// green run reporting a violation nobody can reproduce.
func TestRunner_LastActionInstallFailureFailsTheRun(t *testing.T) {
	state := newHarnessWithSpec(t, carrierSpec)
	web := &installFailsWebDriver{webDriverBase: webDriverBase{Driver: state.mock}}

	_, err := state.tryRun(t, Options{Duration: 2 * time.Second, MaxSteps: 3, Driver: web})
	if err == nil {
		t.Fatal("Run succeeded with a page that cannot take lastAction; the run " +
			"reported green while its extractor values came from two engines")
	}
	if !bytes.Contains([]byte(err.Error()), []byte("install last action")) {
		t.Errorf("Run error = %v, want it to name the failed lastAction install", err)
	}
}

// logInstallFailsWebDriver takes lastAction and refuses the logs, the shape a
// page carrying an older published @sanderling/spec runtime has: it knows the
// action setter and not the log one.
type logInstallFailsWebDriver struct {
	*installFailsWebDriver
}

func (d *logInstallFailsWebDriver) SetLastAction(context.Context, json.RawMessage) error {
	return nil
}

func (d *logInstallFailsWebDriver) SetLogs(context.Context, json.RawMessage) error {
	return errors.New("__sanderlingSetLogs__ is not a function")
}

// TestRunner_LogInstallFailureFailsTheRun holds the log channel to the same
// standard as the action one. The driver having the console errors decides
// nothing on web: the page's reading of every extractor replaces the host's, so
// a run that cannot put the entries back into the page evaluates noLogcatErrors
// against an empty array and reports green on a console full of errors.
// Continuing past this is the vacuity the whole install exists to prevent.
func TestRunner_LogInstallFailureFailsTheRun(t *testing.T) {
	state := newHarnessWithSpec(t, carrierSpec)
	web := &logInstallFailsWebDriver{
		installFailsWebDriver: &installFailsWebDriver{webDriverBase: webDriverBase{Driver: state.mock}},
	}

	_, err := state.tryRun(t, Options{Duration: 2 * time.Second, MaxSteps: 3, Driver: web})
	if err == nil {
		t.Fatal("Run succeeded with a page that cannot take the step's logs; " +
			"every property reading the log stream ran against an empty array")
	}
	if !bytes.Contains([]byte(err.Error()), []byte("install logs")) {
		t.Errorf("Run error = %v, want it to name the failed log install", err)
	}
}