sanderling/internal/runner/web_carrier_test.go

package runner

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"testing"
	"time"

	"github.com/priyanshujain/sanderling/internal/driver"
	mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
	"github.com/priyanshujain/sanderling/internal/trace"
)

// carrierSpec registers one extractor whose value the page supplies. It stands
// in for every spec whose getters carry state across steps (folio's last-seen
// Home total, its submit counters): what matters is that ASKING the page for
// the value is what advances it.
const carrierSpec = `
import { actions, extract } from "@sanderling/spec";
const carrier = extract("carrier", () => 0);
globalThis.properties = {};
globalThis.actions = actions(() => []);
`

// carrierWebDriver is a web target that alternates between a cross-fading
// hierarchy (which the runner discards as transitional) and a settled one, and
// whose page-side extractor advances a counter on every evaluation - exactly
// what a spec-authored carrier does in V8.
type carrierWebDriver struct {
	*mockdriver.Driver
	transitional bool
	snapshots    int
	reads        int
}

func (d *carrierWebDriver) Snapshot(ctx context.Context) (string, driver.Image, error) {
	_, image, err := d.Driver.Snapshot(ctx)
	d.snapshots++
	if !d.transitional {
		return `{"attributes":{"resource-id":"HomeScreen"},"children":[]}`, image, err
	}
	// A genuine cross-fade: two live routes, and a tree that keeps changing
	// between retries so the runner spends its whole retry budget on it.
	return fmt.Sprintf(`{"attributes":{"resource-id":"root"},"children":[
	  {"attributes":{"resource-id":"HomeScreen","text":"frame-%d"},"children":[]},
	  {"attributes":{"resource-id":"LedgerScreen"},"children":[]}
	]}`, d.snapshots), image, err
}

func (d *carrierWebDriver) InstallBundle(context.Context, []byte) error { return nil }

// A web target says so. The runner's per-step hierarchy reread is android-only,
// and a fake claiming android would take a path no chrome run takes.
func (d *carrierWebDriver) Health(context.Context) (driver.Health, error) {
	return driver.Health{Ready: true, Version: "fake", Platform: "web"}, nil
}

func (d *carrierWebDriver) EvaluateExtractors(context.Context) (map[int]json.RawMessage, error) {
	d.reads++
	return map[int]json.RawMessage{0: json.RawMessage(strconv.Itoa(d.reads))}, nil
}

// NextActionFromV8 runs once per step, after the hierarchy fetch, so flipping
// here makes every other step a cross-fade.
func (d *carrierWebDriver) NextActionFromV8(context.Context) (json.RawMessage, error) {
	d.transitional = !d.transitional
	return json.RawMessage(`{"kind":"Tap","x":5,"y":5}`), nil
}

func (d *carrierWebDriver) SetLastAction(context.Context, json.RawMessage) error { return nil }

func (d *carrierWebDriver) SetLogs(context.Context, json.RawMessage) error { return nil }

// TestRunner_TransitionalStepNeverAdvancesThePageCarrier pins the ordering the
// web path depends on. The page-side extractors must run only on steps the
// verifier accepts: their getters advance spec state every time they evaluate,
// so evaluating them on a step whose values are then discarded leaves the page
// one window ahead of the verifier. The next accepted pair then brackets two
// committed transactions while having counted one submit, and the property
// convicts an app that did nothing wrong.
func TestRunner_TransitionalStepNeverAdvancesThePageCarrier(t *testing.T) {
	state := newHarnessWithSpec(t, carrierSpec)
	web := &carrierWebDriver{Driver: state.mock}

	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
	defer cancel()
	summary, err := Run(ctx, Options{
		Duration:    30 * time.Second,
		IdleTimeout: 20 * time.Millisecond,
		MaxSteps:    5,
		Driver:      web,
		Verifier:    state.verifier,
		TraceWriter: state.writer,
	})
	if err != nil {
		t.Fatalf("Run: %v", err)
	}
	if summary.Steps != 5 {
		t.Fatalf("steps = %d, want 5", summary.Steps)
	}

	type traceLine struct {
		Step             int                              `json:"step"`
		Transitional     bool                             `json:"transitional"`
		ExtractorChanges map[string]trace.ExtractorChange `json:"extractor_changes"`
	}
	body, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl"))
	if err != nil {
		t.Fatal(err)
	}
	verified, transitional := 0, 0
	previous := 0
	for _, raw := range bytes.Split(bytes.TrimSpace(body), []byte("\n")) {
		var line traceLine
		if err := json.Unmarshal(raw, &line); err != nil {
			t.Fatalf("decode trace line: %v", err)
		}
		if line.Transitional {
			transitional++
			continue
		}
		verified++
		change, ok := line.ExtractorChanges["carrier"]
		if !ok {
			t.Fatalf("step %d: no carrier value reached the verifier", line.Step)
		}
		current, convErr := strconv.Atoi(string(change.Curr))
		if convErr != nil {
			t.Fatalf("step %d: carrier value %s: %v", line.Step, change.Curr, convErr)
		}
		if current != previous+1 {
			t.Errorf("step %d: carrier went %d -> %d; the page advanced it on a "+
				"step the verifier discarded, so the verifier's window is wider "+
				"than the one the spec counted actions over",
				line.Step, previous, current)
		}
		previous = current
	}
	if verified == 0 || transitional == 0 {
		t.Fatalf("need both kinds of step to prove anything: %d verified, %d transitional",
			verified, transitional)
	}
	if web.reads != verified {
		t.Errorf("the page evaluated its extractors %d time(s) across %d verified step(s); "+
			"every evaluation the verifier does not use still advances spec state",
			web.reads, verified)
	}
}

// installFailsWebDriver is a web target whose page cannot take the runner's
// lastAction: an older published @sanderling/spec runtime, a bundle that never
// installed, a tab that navigated away from it.
type installFailsWebDriver struct {
	*mockdriver.Driver
}

func (d *installFailsWebDriver) InstallBundle(context.Context, []byte) error { return nil }

func (d *installFailsWebDriver) EvaluateExtractors(context.Context) (map[int]json.RawMessage, error) {
	return map[int]json.RawMessage{0: json.RawMessage(`1`)}, nil
}

func (d *installFailsWebDriver) NextActionFromV8(context.Context) (json.RawMessage, error) {
	return json.RawMessage(`{"kind":"Tap","x":5,"y":5}`), nil
}

func (d *installFailsWebDriver) SetLastAction(context.Context, json.RawMessage) error {
	return errors.New("__sanderlingSetLastAction__ is not a function")
}

func (d *installFailsWebDriver) SetLogs(context.Context, json.RawMessage) error { return nil }

// TestRunner_LastActionInstallFailureFailsTheRun covers the other half of the
// same trust boundary. A run that cannot install lastAction in the page cannot
// apply the page's extractor values either, so the step keeps goja's
// dump-derived readings while the step before it holds the page's, and a delta
// property compares two producers and fires. Downgraded to a warning that is a
// green run reporting a violation nobody can reproduce.
func TestRunner_LastActionInstallFailureFailsTheRun(t *testing.T) {
	state := newHarnessWithSpec(t, carrierSpec)
	web := &installFailsWebDriver{Driver: state.mock}

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	_, err := Run(ctx, Options{
		Duration:    2 * time.Second,
		IdleTimeout: 20 * time.Millisecond,
		MaxSteps:    3,
		Driver:      web,
		Verifier:    state.verifier,
		TraceWriter: state.writer,
	})
	if err == nil {
		t.Fatal("Run succeeded with a page that cannot take lastAction; the run " +
			"reported green while its extractor values came from two engines")
	}
	if !bytes.Contains([]byte(err.Error()), []byte("install last action")) {
		t.Errorf("Run error = %v, want it to name the failed lastAction install", err)
	}
}

// logInstallFailsWebDriver takes lastAction and refuses the logs, the shape a
// page carrying an older published @sanderling/spec runtime has: it knows the
// action setter and not the log one.
type logInstallFailsWebDriver struct {
	*installFailsWebDriver
}

func (d *logInstallFailsWebDriver) SetLastAction(context.Context, json.RawMessage) error {
	return nil
}

func (d *logInstallFailsWebDriver) SetLogs(context.Context, json.RawMessage) error {
	return errors.New("__sanderlingSetLogs__ is not a function")
}

// TestRunner_LogInstallFailureFailsTheRun holds the log channel to the same
// standard as the action one. The driver having the console errors decides
// nothing on web: the page's reading of every extractor replaces the host's, so
// a run that cannot put the entries back into the page evaluates noLogcatErrors
// against an empty array and reports green on a console full of errors.
// Continuing past this is the vacuity the whole install exists to prevent.
func TestRunner_LogInstallFailureFailsTheRun(t *testing.T) {
	state := newHarnessWithSpec(t, carrierSpec)
	web := &logInstallFailsWebDriver{
		installFailsWebDriver: &installFailsWebDriver{Driver: state.mock},
	}

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	_, err := Run(ctx, Options{
		Duration:    2 * time.Second,
		IdleTimeout: 20 * time.Millisecond,
		MaxSteps:    3,
		Driver:      web,
		Verifier:    state.verifier,
		TraceWriter: state.writer,
	})
	if err == nil {
		t.Fatal("Run succeeded with a page that cannot take the step's logs; " +
			"every property reading the log stream ran against an empty array")
	}
	if !bytes.Contains([]byte(err.Error()), []byte("install logs")) {
		t.Errorf("Run error = %v, want it to name the failed log install", err)
	}
}