feat(chrome): read the page's exceptions and navigations, and hold the picker state across them

a page navigation replaces the runtime, so the seeded picker restarted the seed's stream at its first draw on every reload and a trace could not tell a reload from a generator repeating itself. the driver now drains the main-frame navigations it saw, reports the page's buffered uncaught errors so state.exceptions is the page's list on the goja host too, and carries the picker's draw position out of v8 and back in around each decision.

internal/driver/chrome/driver.go

@@ -32,6 +32,14 @@ type Driver struct {
 
 	logsMu sync.Mutex
 	logs   []driver.LogEntry
+
+	navigationsMu sync.Mutex
+	navigations   []driver.Navigation
+
+	// pickerState is the seeded picker's draw position, held here rather than
+	// in the page: a navigation replaces the page's runtime, and a runtime that
+	// starts over restarts the seed's stream at its first draw.
+	pickerState string
 }
 
 // New creates a new ChromeDriver. Call Terminate when done.
@@ -94,6 +102,19 @@ func New() *Driver {
 		d.logsMu.Unlock()
 	})
 
+	chromedp.ListenTarget(tabCtx, func(ev any) {
+		e, ok := ev.(*page.EventFrameNavigated)
+		if !ok || e.Frame == nil || e.Frame.ParentID != "" {
+			return
+		}
+		d.navigationsMu.Lock()
+		d.navigations = append(d.navigations, driver.Navigation{
+			URL:        e.Frame.URL,
+			UnixMillis: time.Now().UnixMilli(),
+		})
+		d.navigationsMu.Unlock()
+	})
+
 	return d
 }
 
@@ -133,9 +154,22 @@ func (d *Driver) Launch(ctx context.Context, bundleID string, clearState bool, _
 		})()`, &dims)); err == nil && dims[0] > 0 && dims[1] > 0 {
 		_ = chromedp.Run(runCtx, chromedp.EmulateViewport(dims[0], dims[1]))
 	}
+	// The opening navigation is the harness arriving, not the app navigating.
+	_, _ = d.Navigations(ctx)
 	return nil
 }
 
+// Navigations returns the document-replacing main-frame navigations seen since
+// the last call and forgets them. Each one replaced the page's runtime, which
+// is what separates "the app reloaded" from "the picker repeated itself".
+func (d *Driver) Navigations(context.Context) ([]driver.Navigation, error) {
+	d.navigationsMu.Lock()
+	defer d.navigationsMu.Unlock()
+	drained := d.navigations
+	d.navigations = nil
+	return drained, nil
+}
+
 // clearState wipes the target's stored data before the application loads.
 // Script cannot do it: the tab still sits on about:blank, whose opaque origin
 // denies storage access, so `localStorage.clear()` throws SecurityError and
@@ -1078,19 +1112,87 @@ new Promise((resolve, reject) => {
   read();
 })`
 
+// Exceptions returns the uncaught errors and unhandled rejections the page
+// runtime has buffered so far. The buffer is cumulative, which is what
+// state.exceptions means inside the page (buildState in
+// pkg/spec/src/web-runtime.ts), so the host and the page read one list.
+func (d *Driver) Exceptions(ctx context.Context) ([]driver.Exception, error) {
+	const script = `JSON.stringify(window.__sanderlingExceptions__ ? window.__sanderlingExceptions__() : [])`
+	var encoded string
+	runCtx, cancel := d.runCtx(ctx)
+	defer cancel()
+	if err := chromedp.Run(runCtx, chromedp.Evaluate(script, &encoded)); err != nil {
+		return nil, fmt.Errorf("evaluate exceptions: %w", err)
+	}
+	if encoded == "" || encoded == "[]" {
+		return nil, nil
+	}
+	var captured []struct {
+		Class      string `json:"class"`
+		Message    string `json:"message"`
+		StackTrace string `json:"stackTrace"`
+		UnixMillis int64  `json:"unixMillis"`
+	}
+	if err := json.Unmarshal([]byte(encoded), &captured); err != nil {
+		return nil, fmt.Errorf("decode exceptions %s: %w", encoded, err)
+	}
+	result := make([]driver.Exception, 0, len(captured))
+	for _, entry := range captured {
+		result = append(result, driver.Exception{
+			Class:      entry.Class,
+			Message:    entry.Message,
+			StackTrace: entry.StackTrace,
+			UnixMillis: entry.UnixMillis,
+		})
+	}
+	return result, nil
+}
+
+// nextActionScript puts the carried draw position back before the picker
+// decides and reads the new one out afterwards, in the one evaluation, so no
+// navigation can land between the restore and the draw.
+const nextActionScript = `((carried) => {
+  if (!window.__sanderlingNextAction__) return "{}";
+  if (carried !== "" && window.__sanderlingRestorePickerState__) {
+    window.__sanderlingRestorePickerState__(carried);
+  }
+  const action = window.__sanderlingNextAction__();
+  const state = window.__sanderlingPickerState__ ? window.__sanderlingPickerState__() : "";
+  return JSON.stringify({action, state});
+})(%s)`
+
 // NextActionFromV8 invokes the bundle-installed action generator and returns
 // the resulting Action JSON. Returns an empty json.RawMessage when the
 // generator declines to act this tick.
+//
+// The picker's draw position rides along: it lives here rather than in the
+// page, because a page that navigates gets a fresh runtime whose picker would
+// otherwise start the seed's stream over at its first draw on every reload.
 func (d *Driver) NextActionFromV8(ctx context.Context) (json.RawMessage, error) {
-	const script = `JSON.stringify(window.__sanderlingNextAction__ ? window.__sanderlingNextAction__() : null)`
+	script := fmt.Sprintf(nextActionScript, strconv.Quote(d.pickerState))
 	var encoded string
 	runCtx, cancel := d.runCtx(ctx)
 	defer cancel()
 	if err := chromedp.Run(runCtx, chromedp.Evaluate(script, &encoded)); err != nil {
 		return nil, fmt.Errorf("evaluate next action: %w", err)
 	}
-	if encoded == "" || encoded == "null" {
+	if encoded == "" {
 		return nil, nil
 	}
-	return json.RawMessage(encoded), nil
+	var decoded struct {
+		Action json.RawMessage `json:"action"`
+		State  string          `json:"state"`
+	}
+	if err := json.Unmarshal([]byte(encoded), &decoded); err != nil {
+		return nil, fmt.Errorf("decode next action %s: %w", encoded, err)
+	}
+	// An empty state means the page had no runtime to ask, so the position we
+	// already hold is still the run's position.
+	if decoded.State != "" {
+		d.pickerState = decoded.State
+	}
+	if len(decoded.Action) == 0 || string(decoded.Action) == "null" {
+		return nil, nil
+	}
+	return decoded.Action, nil
 }

internal/driver/chrome/picker_stream_test.go

@@ -0,0 +1,154 @@
+//go:build browser
+
+package chrome
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/chromedp/chromedp"
+
+	"github.com/priyanshujain/sanderling/internal/bundler"
+)
+
+const streamSeed = 1
+
+// A seed describes one stream of actions, and a web page can end it.
+//
+// The picker's draw position lived in the page, and the bundle is registered to
+// run at every freshly-navigated document, so any navigation built a new picker
+// at the seed's first draw. A page that submits a form, follows a link or
+// reloads therefore replayed draw one forever: on the angular-dart TodoMVC
+// implementation, whose form GET-submits on Enter, seed 1 chose PressKey enter
+// on 200 of 200 steps and created no todo at all. The run reported clean, so
+// nothing but this distinguishes it from a seed that chose badly.
+func TestNextActionFromV8_ReloadDoesNotRestartTheSeedStream(t *testing.T) {
+	server := httptest.NewServer(http.FileServer(http.Dir("testdata")))
+	defer server.Close()
+	url := server.URL + "/picker-stream.html"
+
+	const calls = 8
+	uninterrupted := pickerStream(t, url, calls, calls)
+	if distinctActions(uninterrupted) < 2 {
+		t.Fatalf("the uninterrupted stream never varies (%v); a restart would be invisible", uninterrupted)
+	}
+	reloaded := pickerStream(t, url, calls, calls/2)
+
+	for index := range uninterrupted {
+		if reloaded[index] != uninterrupted[index] {
+			t.Fatalf("action %d after a reload is %s, uninterrupted the seed chose %s\nreloaded:      %v\nuninterrupted: %v",
+				index, reloaded[index], uninterrupted[index], reloaded, uninterrupted)
+		}
+	}
+}
+
+// A navigation the trace cannot see is a run nobody can read: an analysis has
+// no way to tell an app that reloaded from a generator that repeated itself.
+func TestNavigations_ReportTheDocumentThatReplacedThePage(t *testing.T) {
+	server := httptest.NewServer(http.FileServer(http.Dir("testdata")))
+	defer server.Close()
+	url := server.URL + "/picker-stream.html"
+
+	d := New()
+	defer d.Terminate(context.Background())
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	if err := d.Launch(ctx, url, false, nil); err != nil {
+		t.Fatalf("Launch: %v", err)
+	}
+
+	opening, err := d.Navigations(ctx)
+	if err != nil {
+		t.Fatalf("Navigations: %v", err)
+	}
+	if len(opening) != 0 {
+		t.Fatalf("the run's own opening navigation was reported as the app navigating: %v", opening)
+	}
+
+	if err := chromedp.Run(d.tabCtx, chromedp.Reload()); err != nil {
+		t.Fatalf("reload: %v", err)
+	}
+	reported, err := d.Navigations(ctx)
+	if err != nil {
+		t.Fatalf("Navigations: %v", err)
+	}
+	if len(reported) != 1 {
+		t.Fatalf("a reload reported %d navigations, want 1: %v", len(reported), reported)
+	}
+	if reported[0].URL != url {
+		t.Errorf("navigation URL = %q, want %q", reported[0].URL, url)
+	}
+	if reported[0].UnixMillis == 0 {
+		t.Error("navigation carries no timestamp")
+	}
+
+	drained, err := d.Navigations(ctx)
+	if err != nil {
+		t.Fatalf("Navigations: %v", err)
+	}
+	if len(drained) != 0 {
+		t.Errorf("navigations were reported twice: %v", drained)
+	}
+}
+
+// pickerStream drives the real seeded picker over the page and returns the
+// action it chose on each call, reloading the page once after reloadAfter
+// calls. Reloading past the call count leaves the stream uninterrupted.
+func pickerStream(t *testing.T, url string, calls, reloadAfter int) []string {
+	t.Helper()
+	d := New()
+	defer d.Terminate(context.Background())
+	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+	defer cancel()
+	if err := d.Launch(ctx, url, false, nil); err != nil {
+		t.Fatalf("Launch: %v", err)
+	}
+	installPickerStreamProbe(ctx, t, d)
+
+	chosen := make([]string, 0, calls)
+	for call := 0; call < calls; call++ {
+		if call == reloadAfter {
+			if err := chromedp.Run(d.tabCtx, chromedp.Reload()); err != nil {
+				t.Fatalf("reload before call %d: %v", call, err)
+			}
+		}
+		action, err := d.NextActionFromV8(ctx)
+		if err != nil {
+			t.Fatalf("NextActionFromV8 call %d: %v", call, err)
+		}
+		if len(action) == 0 {
+			t.Fatalf("call %d chose nothing; the page has tappable targets", call)
+		}
+		chosen = append(chosen, string(action))
+	}
+	return chosen
+}
+
+func installPickerStreamProbe(ctx context.Context, t *testing.T, d *Driver) {
+	t.Helper()
+	specSource := filepath.Join(repoRootDir(t), "pkg", "spec")
+	probe, err := bundler.BundleWeb(bundler.WebOptions{
+		EntryFile:      filepath.Join(specSource, "test", "picker-stream-probe.ts"),
+		WebRuntimeFile: filepath.Join(specSource, "src", "web-runtime.ts"),
+		Defines:        map[string]string{"SANDERLING_SEED": strconv.Itoa(streamSeed)},
+	})
+	if err != nil {
+		t.Fatalf("bundle picker stream probe: %v", err)
+	}
+	if err := d.InstallBundle(ctx, probe.JavaScript); err != nil {
+		t.Fatalf("install picker stream probe: %v", err)
+	}
+}
+
+func distinctActions(actions []string) int {
+	seen := map[string]struct{}{}
+	for _, action := range actions {
+		seen[action] = struct{}{}
+	}
+	return len(seen)
+}

internal/driver/chrome/testdata/picker-stream.html

@@ -0,0 +1,14 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>picker stream</title>
+<style>
+  button { display: block; width: 120px; height: 30px; margin: 10px; }
+</style>
+<form action="" method="get">
+  <input id="entry" placeholder="what needs to be done">
+</form>
+<button id="one">one</button>
+<button id="two">two</button>
+<button id="three">three</button>
+<button id="four">four</button>
+<button id="five">five</button>

pkg/spec/src/pcg.ts

@@ -33,6 +33,19 @@ export class Pcg {
     this.lo = lo & MASK64;
   }
 
+  // state and restore carry the draw position out of and back into one engine.
+  // A web run's runtime is reinstalled by every page navigation, so the host
+  // holds this pair and puts it back, and the seed's stream is one stream
+  // whatever the page does.
+  state(): { hi: bigint; lo: bigint } {
+    return { hi: this.hi, lo: this.lo };
+  }
+
+  restore(hi: bigint, lo: bigint): void {
+    this.hi = hi & MASK64;
+    this.lo = lo & MASK64;
+  }
+
   // next advances the 128-bit LCG state and returns the new (hi, lo) pair.
   private next(): { hi: bigint; lo: bigint } {
     // 128-bit multiply of state by the 128-bit multiplier, keeping 128 bits.

pkg/spec/src/runtime-entry.ts

@@ -161,6 +161,20 @@ export function installRuntime(
   // picker's rng scope. It brackets those calls with this so a multi-item
   // sampler refuses rather than handing back its first item forever.
   defineLockedGlobal("__sanderlingSetEnumeratingCandidates__", setEnumeratingCandidates);
+  // The picker's draw position, as "hi,lo" decimal (a bigint pair survives no
+  // JSON hop). A web run's runtime is reinstalled by every page navigation, so
+  // the host reads the position back after each decision and puts it in place
+  // before the next one; without that the seed's stream restarts at its first
+  // draw every time the page reloads.
+  defineLockedGlobal("__sanderlingPickerState__", () => {
+    const { hi, lo } = rng.state();
+    return `${hi},${lo}`;
+  });
+  defineLockedGlobal("__sanderlingRestorePickerState__", (state: string) => {
+    const [hi, lo] = state.split(",");
+    if (hi === undefined || lo === undefined) return;
+    rng.restore(BigInt(hi), BigInt(lo));
+  });
   defineLockedGlobal("__sanderlingExtractors__", () => evaluateExtractors());
   // __sanderlingSetupAction__ walks ONLY the setup generator once, for the LLM
   // action generator (Go), which drives selection itself and must not run the

pkg/spec/src/web-runtime.ts

@@ -672,6 +672,12 @@ defineLockedGlobal("__sanderlingSetLastAction__", (value: unknown) => {
   lastAction = value ?? null;
 });
 
+// The host reads the same buffer buildState puts behind state.exceptions, so
+// the goja-side state.exceptions is the page's list rather than the empty one
+// it held before, and the trace records an error surface an offline oracle can
+// read back.
+defineLockedGlobal("__sanderlingExceptions__", () => capturedExceptions.slice());
+
 // writable:false stops a page script from shadowing the runtime via plain
 // assignment (the realistic in-page threat). configurable:true is required so
 // unit tests sharing one process can reinstall a fake via defineProperty; a

pkg/spec/test/pcg.test.ts

@@ -56,3 +56,17 @@ for (const c of cases) {
     });
   }
 }
+
+// A web run's runtime is reinstalled by every page navigation, so the host
+// carries this pair across and the seed's stream stays one stream. A restore
+// that lost a bit would silently re-explore ground the seed already covered.
+test("a restored state continues the sequence it was taken from", () => {
+  const source = new Pcg(1n, 0n);
+  for (let draw = 0; draw < 5; draw++) source.uint64();
+  const carried = source.state();
+  const continuation = [source.uint64(), source.uint64(), source.uint64()];
+
+  const fresh = new Pcg(1n, 0n);
+  fresh.restore(carried.hi, carried.lo);
+  assert.deepEqual([fresh.uint64(), fresh.uint64(), fresh.uint64()], continuation);
+});

pkg/spec/test/picker-stream-probe.ts

@@ -0,0 +1,11 @@
+// The seeded picker, over a page that gets navigated out from under it.
+//
+// internal/driver/chrome/picker_stream_test.go bundles this through the
+// production web bundler and asks for one action at a time. `taps` over a page
+// whose targets never move draws exactly once per call, so the actions it hands
+// back ARE the seed's draw stream, and a stream that restarts is visible in
+// them.
+
+import { taps } from "../src/actions.ts";
+
+export const actionsRoot = taps;