Merge origin/master into llm-recording-and-analysis

Brings in #73, which landed web fact-parity work overlapping this branch:
selector-tagged ax handles, aria-disabled in `enabled`, shadow-DOM traversal
and a `scrollable`/`editable` dump, plus a third folio property and two new
testTags on HomeScreen.

Six files conflicted. The option-carrying ones took the union of both sides'
fields, so `--label-source` and `--exit-on-violation` both reach the pipeline.
In web-runtime.ts both sides changed how an element is described: master gave
`elementHandle` a selector to tag handles with, this branch gave it raw
attribute names and a field hint. Both survive, and `enabled` now answers
through master's isEnabled while `editable` stays.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX

internal/runner/runner.go

@@ -31,6 +31,11 @@ type Options struct {
 	// positive value stops the loop once that many steps have run.
 	MaxSteps int
 
+	// StopOnViolation ends the step loop as soon as a step records a
+	// violation, so a run that exists to find one bug stops at the evidence
+	// instead of spending the rest of its budget past it.
+	StopOnViolation bool
+
 	BundleID    string
 	Driver      driver.DeviceDriver
 	Verifier    *verifier.Verifier
@@ -74,6 +79,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
 	if logger == nil {
 		logger = slog.Default()
 	}
+	options.IdleTimeout = resolveIdleTimeout(options)
 
 	// Gate on the app actually being on top before acting, so the first
 	// action never fires against a leftover screen or a system dialog. Done
@@ -87,6 +93,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
 	if err != nil {
 		return Summary{}, err
 	}
+	_, pageExtractors := extractorSource.(webSource)
 
 	summary := Summary{StartTime: time.Now()}
 	deadline := summary.StartTime.Add(options.Duration)
@@ -122,9 +129,8 @@ func Run(ctx context.Context, options Options) (Summary, error) {
 		var logs []verifier.LogEntry
 
 		// gctx is bound to the errgroup so a returned error (or outer
-		// cancellation) propagates to siblings - notably the V8 extractor
-		// goroutine, whose CDP round-trip can otherwise outrun the step
-		// budget on a hung tab.
+		// cancellation) propagates to every sibling read rather than leaving
+		// one blocked on a hung device.
 		g, gctx := errgroup.WithContext(ctx)
 		si := stepIndex
 		// fetchSyncedState issues a single Snapshot RPC so hierarchy and
@@ -143,16 +149,6 @@ func Run(ctx context.Context, options Options) (Summary, error) {
 			logs = collectLogs(gctx, options.Driver, logSince)
 			return nil
 		})
-		var v8Overrides map[int]json.RawMessage
-		g.Go(func() error {
-			overrides, err := extractorSource.ExtractorOverrides(gctx)
-			if err != nil {
-				logger.Warn("v8 extractor evaluation failed", "step", si, "err", err)
-				return nil
-			}
-			v8Overrides = overrides
-			return nil
-		})
 		// All goroutines write to local variables and return nil, so the Wait
 		// error is always nil; ignored intentionally.
 		_ = g.Wait()
@@ -195,6 +191,29 @@ func Run(ctx context.Context, options Options) (Summary, error) {
 		var witnesses map[string]trace.Witness
 		skippedVerification := false
 		if !transitional {
+			// The page-side extractors evaluate only on steps the verifier will
+			// accept, which is why this read waits for the tree instead of
+			// racing it. A spec's extractor getters carry state across steps
+			// (folio's last-seen Home total, its submit counters) and that state
+			// advances every time they run: evaluating them on a step whose
+			// values are then thrown away leaves the page one window ahead of
+			// the verifier, so the next accepted pair brackets two committed
+			// transactions while having counted one submit, and the property
+			// convicts a healthy app. It costs the latency the read used to hide
+			// behind the hierarchy fetch; the fetch is what decides whether this
+			// step counts at all, so it has to go first.
+			//
+			// lastAction is the same value PushSnapshot hands the goja state
+			// below: the two engines evaluate this step against one action.
+			v8Overrides, overridesErr := extractorSource.ExtractorOverrides(ctx, lastAction)
+			if overridesErr != nil {
+				// Not a warning. Without the page's values this step's
+				// extractors keep goja's dump-derived readings while the
+				// previous step holds the page's, and a delta property then
+				// compares two producers and fires on an app that did nothing
+				// wrong.
+				return summary, fmt.Errorf("step %d extractor overrides: %w", stepIndex, overridesErr)
+			}
 			if err := options.Verifier.PushSnapshot(verifier.SnapshotInput{
 				Tree:          tree,
 				ScreenshotPNG: screenshotPNG,
@@ -206,13 +225,26 @@ func Run(ctx context.Context, options Options) (Summary, error) {
 			}); err != nil {
 				return summary, fmt.Errorf("step %d push: %w", stepIndex, err)
 			}
+			// Every failure below leaves some extractors holding the page's
+			// value and the rest holding goja's reading of the dump, and a
+			// property comparing previous to current across that split fires
+			// on a healthy app. Each also means the two engines loaded
+			// different bundles, which nothing downstream can reconcile.
+			if pageExtractors && len(v8Overrides) != options.Verifier.ExtractorCount() {
+				return summary, fmt.Errorf(
+					"step %d: the page reported values for %d of the spec's %d extractors; "+
+						"the page and the host are running different bundles",
+					stepIndex, len(v8Overrides), options.Verifier.ExtractorCount())
+			}
 			skipped, overrideErr := options.Verifier.OverrideExtractorValues(v8Overrides)
 			if overrideErr != nil {
-				logger.Warn("v8 override apply failed", "step", stepIndex, "err", overrideErr)
+				return summary, fmt.Errorf("step %d apply extractor overrides: %w", stepIndex, overrideErr)
 			}
 			if skipped > 0 {
-				logger.Warn("v8 override skipped out-of-range entries",
-					"step", stepIndex, "skipped", skipped, "have", len(v8Overrides))
+				return summary, fmt.Errorf(
+					"step %d: %d of %d extractor overrides fell outside the spec's extractor list; "+
+						"the page and the host are running different bundles",
+					stepIndex, skipped, len(v8Overrides))
 			}
 			options.Verifier.EvaluateProperties()
 			violations = options.Verifier.NewlyViolatedProperties()
@@ -317,6 +349,12 @@ func Run(ctx context.Context, options Options) (Summary, error) {
 		summary.Steps = stepIndex
 		if len(violations) > 0 {
 			summary.Violations = append(summary.Violations, violationRecords(violations, witnesses, stepIndex)...)
+			// The step is already written, so the trace ends on the state that
+			// produced the violation. Finalize below still runs, so pending
+			// liveness obligations are reported alongside it.
+			if options.StopOnViolation {
+				break
+			}
 		}
 		// Wait actions are themselves a settling: skip the idle poll. Actions
 		// that mutate the UI fall through to WaitForIdle so the next step's
@@ -391,12 +429,36 @@ func validate(options Options) error {
 	if options.Duration <= 0 {
 		return errors.New("runner: Duration must be positive")
 	}
-	if options.IdleTimeout <= 0 {
-		options.IdleTimeout = 2 * time.Second
-	}
 	return nil
 }
 
+// defaultIdleTimeout is the settle budget a caller that names none gets.
+const defaultIdleTimeout = 2 * time.Second
+
+// idleTimeoutFloor is a driver that knows how long its own settle can take.
+// Declared here rather than in the driver package (like lastActionInstaller in
+// source.go) so the mobile drivers stay untouched.
+type idleTimeoutFloor interface {
+	MinIdleTimeout() time.Duration
+}
+
+// resolveIdleTimeout settles the per-step settle budget: the caller's value,
+// defaulted when unset, and raised to whatever the driver says its own settle
+// needs. The chrome driver's settle waits for the DOM to go quiet and only then
+// opens its route-transition window; handed less than their sum it is cut off
+// mid-transition, and the step samples the screen the app is leaving. A driver
+// that reports no floor keeps the caller's value exactly.
+func resolveIdleTimeout(options Options) time.Duration {
+	timeout := options.IdleTimeout
+	if timeout <= 0 {
+		timeout = defaultIdleTimeout
+	}
+	if floor, ok := options.Driver.(idleTimeoutFloor); ok {
+		timeout = max(timeout, floor.MinIdleTimeout())
+	}
+	return timeout
+}
+
 // ensureForeground keeps the app under test in the foreground. When the driver
 // can report the foreground app and it no longer matches the bundle under test,
 // the app is relaunched. Returns true when a relaunch happened so the caller

internal/runner/runner_test.go

@@ -2343,3 +2343,99 @@ func TestRunner_SkipsActionWhenOverlayStealsFocusAtApplyTime(t *testing.T) {
 		t.Errorf("no step recorded action_skipped=%q, so the undispatched action looks executed", actionSkippedForeground)
 	}
 }
+
+// TestRunner_StopOnViolationEndsAtTheFirstViolation pins the gate CI runs on:
+// a step budget of 8 against a spec that only violates on the third step must
+// end on step 3 and write nothing after it, so the trace's last state is the
+// one that produced the violation.
+func TestRunner_StopOnViolationEndsAtTheFirstViolation(t *testing.T) {
+	const thirdStepViolationSpec = `
+import { actions, always, extract } from "@sanderling/spec";
+let observed = 0;
+const tick = extract(() => ++observed);
+globalThis.properties = {
+  staysUnderThree: always(() => tick.current < 3),
+};
+globalThis.actions = actions(() => []);
+`
+	state := newHarnessWithSpec(t, thirdStepViolationSpec)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	summary, err := Run(ctx, Options{
+		Duration:        time.Hour,
+		IdleTimeout:     20 * time.Millisecond,
+		MaxSteps:        8,
+		StopOnViolation: true,
+		Driver:          state.mock,
+		Verifier:        state.verifier,
+		TraceWriter:     state.writer,
+	})
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if !containsProperty(summary.Violations, "staysUnderThree") {
+		t.Fatalf("expected staysUnderThree to fire, got %v", summary.Violations)
+	}
+	if summary.Steps != 3 {
+		t.Errorf("steps: got %d, want 3 (the run must stop at the violating step, not run the 8-step budget)",
+			summary.Steps)
+	}
+	for _, step := range traceStepIndices(t, state.writer.Directory()) {
+		if step > summary.Steps {
+			t.Errorf("trace kept stepping after the violation: found step %d past step %d",
+				step, summary.Steps)
+		}
+	}
+}
+
+// TestRunner_WithoutStopOnViolationRunsTheWholeBudget is the other half: the
+// default must stay a full-budget fuzz run, so turning the flag on is the only
+// thing that shortens a run.
+func TestRunner_WithoutStopOnViolationRunsTheWholeBudget(t *testing.T) {
+	state := newHarnessWithSpec(t, violationSpec)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	summary, err := Run(ctx, Options{
+		Duration:    time.Hour,
+		IdleTimeout: 20 * time.Millisecond,
+		MaxSteps:    4,
+		Driver:      state.mock,
+		Verifier:    state.verifier,
+		TraceWriter: state.writer,
+	})
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if summary.Steps != 4 {
+		t.Errorf("steps: got %d, want 4; a violation must not shorten a default run", summary.Steps)
+	}
+}
+
+// traceStepIndices reads every step index the trace recorded, so a test can
+// assert on what the run actually wrote rather than on the summary alone.
+func traceStepIndices(t *testing.T, directory string) []int {
+	t.Helper()
+	file, err := os.Open(filepath.Join(directory, "trace.jsonl"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+	var steps []int
+	scanner := bufio.NewScanner(file)
+	scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
+	for scanner.Scan() {
+		var line struct {
+			Step int `json:"step"`
+		}
+		if err := json.Unmarshal(scanner.Bytes(), &line); err != nil {
+			t.Fatalf("trace line decode: %v", err)
+		}
+		steps = append(steps, line.Step)
+	}
+	if err := scanner.Err(); err != nil {
+		t.Fatalf("scan trace: %v", err)
+	}
+	return steps
+}

internal/runner/source.go

@@ -25,8 +25,23 @@ type ActionSource interface {
 // ExtractorSource yields per-step extractor overrides the runner applies after
 // PushSnapshot. The mobile path has none (returns nil); the web path returns the
 // values its extractors computed in V8 against the real DOM.
+//
+// lastAction is the action the previous step actually applied, the same value
+// PushSnapshot hands the goja state. The web path has to install it in the page
+// before its extractors run: a spec extractor reading state.lastAction runs in
+// V8 there, and V8 has no way to know what the runner dispatched.
 type ExtractorSource interface {
-	ExtractorOverrides(ctx context.Context) (map[int]json.RawMessage, error)
+	ExtractorOverrides(
+		ctx context.Context,
+		lastAction *verifier.Action,
+	) (map[int]json.RawMessage, error)
+}
+
+// lastActionInstaller is the web driver's channel for the previous step's
+// action. It is declared here rather than folded into driver.WebDriver so the
+// mobile drivers stay untouched; every web driver must implement it.
+type lastActionInstaller interface {
+	SetLastAction(ctx context.Context, encoded json.RawMessage) error
 }
 
 // gojaSource drives both action selection and (trivially) extractor overrides
@@ -40,7 +55,10 @@ func (s gojaSource) NextAction(context.Context, int) (verifier.Action, error) {
 	return s.verifier.NextAction()
 }
 
-func (gojaSource) ExtractorOverrides(context.Context) (map[int]json.RawMessage, error) {
+func (gojaSource) ExtractorOverrides(
+	context.Context,
+	*verifier.Action,
+) (map[int]json.RawMessage, error) {
 	return nil, nil
 }
 
@@ -61,7 +79,24 @@ func (s webSource) NextAction(ctx context.Context, _ int) (verifier.Action, erro
 	return verifier.DecodeAction(raw)
 }
 
-func (s webSource) ExtractorOverrides(ctx context.Context) (map[int]json.RawMessage, error) {
+// ExtractorOverrides installs the previous step's action in the page, then
+// reads back what the spec's extractors computed against the live DOM. The
+// install is not best-effort: a web driver that cannot take it leaves
+// state.lastAction null in V8, which silently turns every action-gated
+// property vacuously true, so it is reported as an error instead.
+func (s webSource) ExtractorOverrides(
+	ctx context.Context,
+	lastAction *verifier.Action,
+) (map[int]json.RawMessage, error) {
+	installer, ok := s.web.(lastActionInstaller)
+	if !ok {
+		return nil, fmt.Errorf(
+			"web driver %T cannot install state.lastAction; every property gated "+
+				"on the last action would be vacuously true", s.web)
+	}
+	if err := installer.SetLastAction(ctx, verifier.EncodeLastAction(lastAction)); err != nil {
+		return nil, fmt.Errorf("install last action: %w", err)
+	}
 	return s.web.EvaluateExtractors(ctx)
 }
 

internal/runner/web_carrier_test.go

@@ -0,0 +1,196 @@
+package runner
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/priyanshujain/sanderling/internal/driver"
+	mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
+	"github.com/priyanshujain/sanderling/internal/trace"
+)
+
+// carrierSpec registers one extractor whose value the page supplies. It stands
+// in for every spec whose getters carry state across steps (folio's last-seen
+// Home total, its submit counters): what matters is that ASKING the page for
+// the value is what advances it.
+const carrierSpec = `
+import { actions, extract } from "@sanderling/spec";
+const carrier = extract("carrier", () => 0);
+globalThis.properties = {};
+globalThis.actions = actions(() => []);
+`
+
+// carrierWebDriver is a web target that alternates between a cross-fading
+// hierarchy (which the runner discards as transitional) and a settled one, and
+// whose page-side extractor advances a counter on every evaluation - exactly
+// what a spec-authored carrier does in V8.
+type carrierWebDriver struct {
+	*mockdriver.Driver
+	transitional bool
+	snapshots    int
+	reads        int
+}
+
+func (d *carrierWebDriver) Snapshot(ctx context.Context) (string, driver.Image, error) {
+	_, image, err := d.Driver.Snapshot(ctx)
+	d.snapshots++
+	if !d.transitional {
+		return `{"attributes":{"resource-id":"HomeScreen"},"children":[]}`, image, err
+	}
+	// A genuine cross-fade: two live routes, and a tree that keeps changing
+	// between retries so the runner spends its whole retry budget on it.
+	return fmt.Sprintf(`{"attributes":{"resource-id":"root"},"children":[
+	  {"attributes":{"resource-id":"HomeScreen","text":"frame-%d"},"children":[]},
+	  {"attributes":{"resource-id":"LedgerScreen"},"children":[]}
+	]}`, d.snapshots), image, err
+}
+
+func (d *carrierWebDriver) InstallBundle(context.Context, []byte) error { return nil }
+
+func (d *carrierWebDriver) EvaluateExtractors(context.Context) (map[int]json.RawMessage, error) {
+	d.reads++
+	return map[int]json.RawMessage{0: json.RawMessage(strconv.Itoa(d.reads))}, nil
+}
+
+// NextActionFromV8 runs once per step, after the hierarchy fetch, so flipping
+// here makes every other step a cross-fade.
+func (d *carrierWebDriver) NextActionFromV8(context.Context) (json.RawMessage, error) {
+	d.transitional = !d.transitional
+	return json.RawMessage(`{"kind":"Tap","x":5,"y":5}`), nil
+}
+
+func (d *carrierWebDriver) SetLastAction(context.Context, json.RawMessage) error { return nil }
+
+// TestRunner_TransitionalStepNeverAdvancesThePageCarrier pins the ordering the
+// web path depends on. The page-side extractors must run only on steps the
+// verifier accepts: their getters advance spec state every time they evaluate,
+// so evaluating them on a step whose values are then discarded leaves the page
+// one window ahead of the verifier. The next accepted pair then brackets two
+// committed transactions while having counted one submit, and the property
+// convicts an app that did nothing wrong.
+func TestRunner_TransitionalStepNeverAdvancesThePageCarrier(t *testing.T) {
+	state := newHarnessWithSpec(t, carrierSpec)
+	web := &carrierWebDriver{Driver: state.mock}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	summary, err := Run(ctx, Options{
+		Duration:    30 * time.Second,
+		IdleTimeout: 20 * time.Millisecond,
+		MaxSteps:    5,
+		Driver:      web,
+		Verifier:    state.verifier,
+		TraceWriter: state.writer,
+	})
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if summary.Steps != 5 {
+		t.Fatalf("steps = %d, want 5", summary.Steps)
+	}
+
+	type traceLine struct {
+		Step             int                              `json:"step"`
+		Transitional     bool                             `json:"transitional"`
+		ExtractorChanges map[string]trace.ExtractorChange `json:"extractor_changes"`
+	}
+	body, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	verified, transitional := 0, 0
+	previous := 0
+	for _, raw := range bytes.Split(bytes.TrimSpace(body), []byte("\n")) {
+		var line traceLine
+		if err := json.Unmarshal(raw, &line); err != nil {
+			t.Fatalf("decode trace line: %v", err)
+		}
+		if line.Transitional {
+			transitional++
+			continue
+		}
+		verified++
+		change, ok := line.ExtractorChanges["carrier"]
+		if !ok {
+			t.Fatalf("step %d: no carrier value reached the verifier", line.Step)
+		}
+		current, convErr := strconv.Atoi(string(change.Curr))
+		if convErr != nil {
+			t.Fatalf("step %d: carrier value %s: %v", line.Step, change.Curr, convErr)
+		}
+		if current != previous+1 {
+			t.Errorf("step %d: carrier went %d -> %d; the page advanced it on a "+
+				"step the verifier discarded, so the verifier's window is wider "+
+				"than the one the spec counted actions over",
+				line.Step, previous, current)
+		}
+		previous = current
+	}
+	if verified == 0 || transitional == 0 {
+		t.Fatalf("need both kinds of step to prove anything: %d verified, %d transitional",
+			verified, transitional)
+	}
+	if web.reads != verified {
+		t.Errorf("the page evaluated its extractors %d time(s) across %d verified step(s); "+
+			"every evaluation the verifier does not use still advances spec state",
+			web.reads, verified)
+	}
+}
+
+// installFailsWebDriver is a web target whose page cannot take the runner's
+// lastAction: an older published @sanderling/spec runtime, a bundle that never
+// installed, a tab that navigated away from it.
+type installFailsWebDriver struct {
+	*mockdriver.Driver
+}
+
+func (d *installFailsWebDriver) InstallBundle(context.Context, []byte) error { return nil }
+
+func (d *installFailsWebDriver) EvaluateExtractors(context.Context) (map[int]json.RawMessage, error) {
+	return map[int]json.RawMessage{0: json.RawMessage(`1`)}, nil
+}
+
+func (d *installFailsWebDriver) NextActionFromV8(context.Context) (json.RawMessage, error) {
+	return json.RawMessage(`{"kind":"Tap","x":5,"y":5}`), nil
+}
+
+func (d *installFailsWebDriver) SetLastAction(context.Context, json.RawMessage) error {
+	return errors.New("__sanderlingSetLastAction__ is not a function")
+}
+
+// TestRunner_LastActionInstallFailureFailsTheRun covers the other half of the
+// same trust boundary. A run that cannot install lastAction in the page cannot
+// apply the page's extractor values either, so the step keeps goja's
+// dump-derived readings while the step before it holds the page's, and a delta
+// property compares two producers and fires. Downgraded to a warning that is a
+// green run reporting a violation nobody can reproduce.
+func TestRunner_LastActionInstallFailureFailsTheRun(t *testing.T) {
+	state := newHarnessWithSpec(t, carrierSpec)
+	web := &installFailsWebDriver{Driver: state.mock}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	_, err := Run(ctx, Options{
+		Duration:    2 * time.Second,
+		IdleTimeout: 20 * time.Millisecond,
+		MaxSteps:    3,
+		Driver:      web,
+		Verifier:    state.verifier,
+		TraceWriter: state.writer,
+	})
+	if err == nil {
+		t.Fatal("Run succeeded with a page that cannot take lastAction; the run " +
+			"reported green while its extractor values came from two engines")
+	}
+	if !bytes.Contains([]byte(err.Error()), []byte("install last action")) {
+		t.Errorf("Run error = %v, want it to name the failed lastAction install", err)
+	}
+}

internal/runner/web_extractor_trace_test.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -44,6 +45,8 @@ func (d *webMockDriver) NextActionFromV8(context.Context) (json.RawMessage, erro
 	return nil, nil
 }
 
+func (d *webMockDriver) SetLastAction(context.Context, json.RawMessage) error { return nil }
+
 // TestRunner_TraceRecordsTheValueTheVerdictUsed fails if the trace and the
 // verdict disagree about an extractor. A witness is only an explanation of a
 // violation if it holds the state the violated property was evaluated against.
@@ -116,3 +119,48 @@ func TestRunner_TraceRecordsTheValueTheVerdictUsed(t *testing.T) {
 		t.Error("no witness reached the trace; nothing was compared")
 	}
 }
+
+// splitTableSpec registers two extractors whose goja bodies both answer "goja".
+// The page below reports only the first, so index 1 keeps goja's dump-derived
+// reading while index 0 holds the page's.
+const splitTableSpec = `
+import { actions, extract } from "@sanderling/spec";
+extract("first", () => "goja");
+extract("second", () => "goja");
+globalThis.properties = {};
+globalThis.actions = actions(() => []);
+`
+
+// TestRunner_PartialExtractorTableIsFatal pins the failure the runner used to
+// let through. JSON.stringify drops an undefined-valued key, so a page whose
+// extractors are mostly undefined off their own screen reported a table with
+// holes in it, and the run completed with half the extractors reading from V8
+// and half from goja. A delta property spanning that split convicts an app that
+// did nothing wrong, which is worse than a crash: it is a green report of a bug
+// that is not there, or a red one for a bug nobody can reproduce.
+func TestRunner_PartialExtractorTableIsFatal(t *testing.T) {
+	state := newHarnessWithSpec(t, splitTableSpec)
+	web := &webMockDriver{
+		Driver:    state.mock,
+		overrides: map[int]json.RawMessage{0: json.RawMessage(`"v8"`)},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_, err := Run(ctx, Options{
+		Duration:    time.Hour,
+		IdleTimeout: 20 * time.Millisecond,
+		MaxSteps:    2,
+		Driver:      web,
+		Verifier:    state.verifier,
+		TraceWriter: state.writer,
+	})
+	if err == nil {
+		t.Fatal("the run completed on a page that reported 1 of 2 extractors; " +
+			"the second extractor silently kept goja's value")
+	}
+	const want = "the page reported values for 1 of the spec's 2 extractors"
+	if !strings.Contains(err.Error(), want) {
+		t.Errorf("Run failed with %q, want it to name the split: %q", err, want)
+	}
+}

internal/runner/web_last_action_test.go

@@ -0,0 +1,78 @@
+package runner
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
+)
+
+// On web the spec's extractors run in the page, so state.lastAction has to be
+// installed there by the runner. It used to be hardcoded null in
+// pkg/spec/src/web-runtime.ts, which made every property gated on the last
+// action (folio's submitMovesBalanceByTypedAmount, for one) vacuously true on
+// web: no failure, no warning, just a green run that proved nothing.
+
+const lastActionSpec = `
+import { actions } from "@sanderling/spec";
+globalThis.actions = actions(() => []);
+globalThis.properties = {};
+`
+
+// tappingWebDriver is a web target whose V8 picker always taps one named
+// control, so the runner has a real applied action to report on the next step.
+type tappingWebDriver struct {
+	*mockdriver.Driver
+	installed []string
+}
+
+func (d *tappingWebDriver) InstallBundle(context.Context, []byte) error { return nil }
+
+func (d *tappingWebDriver) EvaluateExtractors(context.Context) (map[int]json.RawMessage, error) {
+	return nil, nil
+}
+
+func (d *tappingWebDriver) NextActionFromV8(context.Context) (json.RawMessage, error) {
+	return json.RawMessage(`{"kind":"Tap","x":12,"y":34,"selector":"id:TxnSubmit"}`), nil
+}
+
+func (d *tappingWebDriver) SetLastAction(_ context.Context, encoded json.RawMessage) error {
+	d.installed = append(d.installed, string(encoded))
+	return nil
+}
+
+func TestRunner_WebInstallsLastActionInThePage(t *testing.T) {
+	state := newHarnessWithSpec(t, lastActionSpec)
+	web := &tappingWebDriver{Driver: state.mock}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if _, err := Run(ctx, Options{
+		Duration:    time.Hour,
+		IdleTimeout: 20 * time.Millisecond,
+		MaxSteps:    3,
+		Driver:      web,
+		Verifier:    state.verifier,
+		TraceWriter: state.writer,
+	}); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	if len(web.installed) < 2 {
+		t.Fatalf("the page was handed lastAction %d time(s); the web path never installed it",
+			len(web.installed))
+	}
+	// Step 1 has no previous action, exactly as the goja host reports it.
+	if web.installed[0] != "null" {
+		t.Errorf("step 1 installed %s, want null", web.installed[0])
+	}
+	// Every later step carries what the runner actually applied. The shape is
+	// the goja host's (internal/verifier/marshal.go lastActionFields), pinned
+	// against it by TestLastAction_WebJSONMatchesTheGojaObject.
+	const want = `{"kind":"Tap","on":"id:TxnSubmit"}`
+	if web.installed[1] != want {
+		t.Errorf("step 2 installed %s, want %s", web.installed[1], want)
+	}
+}