Merge branch 'web-logs-reach-the-state' into correctness-and-spec-skills

internal/driver/chrome/driver.go

@@ -887,6 +887,30 @@ func (d *Driver) SetLastAction(ctx context.Context, encoded json.RawMessage) err
 	return nil
 }
 
+// SetLogs installs the entries this step's log fetch returned as state.logs
+// inside the page runtime. The page cannot derive them: console output reaches
+// the driver over CDP and nothing in the page reads it back. Without this call
+// every web state.logs is empty, and since the page's reading of an extractor
+// replaces the host's, the default noLogcatErrors then reports green on a run
+// whose console was full of errors.
+//
+// Unguarded for the same reason as SetLastAction: on a page with no setter,
+// "the page cannot accept logs" has to fail the run rather than be reported as
+// a successful install.
+func (d *Driver) SetLogs(ctx context.Context, encoded json.RawMessage) error {
+	payload := strings.TrimSpace(string(encoded))
+	if payload == "" {
+		payload = "[]"
+	}
+	script := fmt.Sprintf(`window.__sanderlingSetLogs__(%s)`, payload)
+	runCtx, cancel := d.runCtx(ctx)
+	defer cancel()
+	if err := chromedp.Run(runCtx, chromedp.Evaluate(script, nil)); err != nil {
+		return fmt.Errorf("set logs: %w", err)
+	}
+	return nil
+}
+
 // extractorScript resolves the extractor table once the page is not mid route
 // transition, giving up on that wait after %d ms.
 //

internal/driver/chrome/driver_test.go

@@ -860,6 +860,54 @@ func TestSetLastAction_ReportsAPageThatCannotTakeIt(t *testing.T) {
 	}
 }
 
+// TestSetLogs_ReportsAPageThatCannotTakeThem is the same install on the channel
+// the log properties hang off. The driver holding a console error changes
+// nothing on web: the page's reading of every extractor replaces the host's, so
+// unless the entries are put back into the page, noLogcatErrors counts an empty
+// array and stays green through a run full of errors.
+func TestSetLogs_ReportsAPageThatCannotTakeThem(t *testing.T) {
+	const withSetter = `<body><script>
+	  window.__logsSeen = null;
+	  window.__sanderlingSetLogs__ = function (value) { window.__logsSeen = value; };
+	</script></body>`
+	const withoutSetter = `<body><div id="app">no sanderling runtime here</div></body>`
+	pages := map[string]string{"/with": withSetter, "/without": withoutSetter}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		_, _ = w.Write([]byte(pages[r.URL.Path]))
+	}))
+	defer server.Close()
+
+	d := New()
+	defer d.Terminate(context.Background())
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	if err := d.Launch(ctx, server.URL+"/with", false, nil); err != nil {
+		t.Fatalf("Launch: %v", err)
+	}
+	logs := json.RawMessage(`[{"unixMillis":1,"level":"E","tag":"console","message":"boom"}]`)
+	if err := d.SetLogs(ctx, logs); err != nil {
+		t.Fatalf("SetLogs on a page that defines the setter: %v", err)
+	}
+	var seen []map[string]any
+	if err := chromedp.Run(d.tabCtx,
+		chromedp.Evaluate(`window.__logsSeen`, &seen)); err != nil {
+		t.Fatalf("read installed logs: %v", err)
+	}
+	if len(seen) != 1 || seen[0]["level"] != "E" || seen[0]["message"] != "boom" {
+		t.Errorf("the page received %v, want the error-level entry the driver captured", seen)
+	}
+
+	if err := d.Launch(ctx, server.URL+"/without", false, nil); err != nil {
+		t.Fatalf("Launch: %v", err)
+	}
+	if err := d.SetLogs(ctx, logs); err == nil {
+		t.Error("SetLogs reported success on a page with no setter; " +
+			"a runtime that cannot take the step's logs is indistinguishable from one that did")
+	}
+}
+
 // TestEvaluateExtractors_ReportsAMissingTable is the same failure on the other
 // sampler. An empty override map is what a spec with no extractors returns, so
 // treating a missing table as {} makes "this page has no sanderling runtime"

internal/runner/runner.go

@@ -163,7 +163,7 @@ func Run(ctx context.Context, options Options) (Summary, error) {
 		})
 		logSince := lastLogTime
 		g.Go(func() error {
-			logs = collectLogs(gctx, options.Driver, logSince)
+			logs = collectLogs(gctx, options.Driver, logger, si, logSince)
 			return nil
 		})
 		// All goroutines write to local variables and return nil, so the Wait
@@ -222,9 +222,10 @@ func Run(ctx context.Context, options Options) (Summary, error) {
 			// behind the hierarchy fetch; the fetch is what decides whether this
 			// step counts at all, so it has to go first.
 			//
-			// lastAction is the same value PushSnapshot hands the goja state
-			// below: the two engines evaluate this step against one action.
-			v8Overrides, overridesErr := extractorSource.ExtractorOverrides(ctx, lastAction)
+			// lastAction and logs are the same values PushSnapshot hands the
+			// goja state below: the two engines evaluate this step against one
+			// action and one set of log entries.
+			v8Overrides, overridesErr := extractorSource.ExtractorOverrides(ctx, lastAction, logs)
 			if overridesErr != nil {
 				// Not a warning. Without the page's values this step's
 				// extractors keep goja's dump-derived readings while the
@@ -823,11 +824,23 @@ func applyAction(ctx context.Context, drv driver.DeviceDriver, action verifier.A
 }
 
 // collectLogs pulls recent error-level log entries from the driver since the
-// previous fetch. A failure is warned-on but not fatal: log capture is a
-// best-effort observability channel, not a correctness dependency.
-func collectLogs(ctx context.Context, drv driver.DeviceDriver, since time.Time) []verifier.LogEntry {
+// previous fetch. A failure is warned-on but not fatal: one unreadable fetch on
+// a flaky device should not end a run. It is not free either. This fetch is the
+// whole evidence base for state.logs, so a step that could not make it leaves
+// every log property (the default noLogcatErrors included) holding on an empty
+// slice, and that has to be visible in the run's output rather than read as the
+// app having logged nothing.
+func collectLogs(
+	ctx context.Context,
+	drv driver.DeviceDriver,
+	logger *slog.Logger,
+	step int,
+	since time.Time,
+) []verifier.LogEntry {
 	entries, err := drv.RecentLogs(ctx, since, "E")
 	if err != nil {
+		logger.Warn("log fetch failed; log properties hold vacuously this step",
+			"step", step, "err", err)
 		return nil
 	}
 	result := make([]verifier.LogEntry, 0, len(entries))

internal/runner/source.go

@@ -24,14 +24,15 @@ type ActionSource interface {
 // PushSnapshot. The mobile path has none (returns nil); the web path returns the
 // values its extractors computed in V8 against the real DOM.
 //
-// lastAction is the action the previous step actually applied, the same value
-// PushSnapshot hands the goja state. The web path has to install it in the page
-// before its extractors run: a spec extractor reading state.lastAction runs in
-// V8 there, and V8 has no way to know what the runner dispatched.
+// lastAction and logs are what PushSnapshot hands the goja state. The web path
+// has to install both in the page before its extractors run: a spec extractor
+// reading state.lastAction or state.logs runs in V8 there, and V8 knows neither
+// what the runner dispatched nor what the driver's log fetch returned.
 type ExtractorSource interface {
 	ExtractorOverrides(
 		ctx context.Context,
 		lastAction *verifier.Action,
+		logs []verifier.LogEntry,
 	) (map[int]json.RawMessage, error)
 }
 
@@ -42,6 +43,13 @@ type lastActionInstaller interface {
 	SetLastAction(ctx context.Context, encoded json.RawMessage) error
 }
 
+// logInstaller is the same channel for the entries this step's log fetch
+// returned. Console output reaches the driver over CDP, so the page can only
+// learn about it from the runner.
+type logInstaller interface {
+	SetLogs(ctx context.Context, encoded json.RawMessage) error
+}
+
 // gojaSource drives both action selection and (trivially) extractor overrides
 // for the mobile path, where the goja-bundled picker runs in-process and no V8
 // extractor values exist.
@@ -56,6 +64,7 @@ func (s gojaSource) NextAction(context.Context) (verifier.Action, error) {
 func (gojaSource) ExtractorOverrides(
 	context.Context,
 	*verifier.Action,
+	[]verifier.LogEntry,
 ) (map[int]json.RawMessage, error) {
 	return nil, nil
 }
@@ -77,24 +86,35 @@ func (s webSource) NextAction(ctx context.Context) (verifier.Action, error) {
 	return verifier.DecodeAction(raw)
 }
 
-// ExtractorOverrides installs the previous step's action in the page, then
-// reads back what the spec's extractors computed against the live DOM. The
-// install is not best-effort: a web driver that cannot take it leaves
-// state.lastAction null in V8, which silently turns every action-gated
-// property vacuously true, so it is reported as an error instead.
+// ExtractorOverrides installs the previous step's action and this step's log
+// entries in the page, then reads back what the spec's extractors computed
+// against the live DOM. Neither install is best-effort: a web driver that
+// cannot take them leaves state.lastAction null and state.logs empty in V8,
+// which silently turns every action-gated property and every log property
+// vacuously true, so both are reported as errors instead.
 func (s webSource) ExtractorOverrides(
 	ctx context.Context,
 	lastAction *verifier.Action,
+	logs []verifier.LogEntry,
 ) (map[int]json.RawMessage, error) {
-	installer, ok := s.web.(lastActionInstaller)
+	actions, ok := s.web.(lastActionInstaller)
 	if !ok {
 		return nil, fmt.Errorf(
 			"web driver %T cannot install state.lastAction; every property gated "+
 				"on the last action would be vacuously true", s.web)
 	}
-	if err := installer.SetLastAction(ctx, verifier.EncodeLastAction(lastAction)); err != nil {
+	if err := actions.SetLastAction(ctx, verifier.EncodeLastAction(lastAction)); err != nil {
 		return nil, fmt.Errorf("install last action: %w", err)
 	}
+	entries, ok := s.web.(logInstaller)
+	if !ok {
+		return nil, fmt.Errorf(
+			"web driver %T cannot install state.logs; every property reading the "+
+				"log stream would be vacuously true", s.web)
+	}
+	if err := entries.SetLogs(ctx, verifier.EncodeLogs(logs)); err != nil {
+		return nil, fmt.Errorf("install logs: %w", err)
+	}
 	return s.web.EvaluateExtractors(ctx)
 }
 

internal/runner/web_carrier_test.go

@@ -75,6 +75,8 @@ func (d *carrierWebDriver) NextActionFromV8(context.Context) (json.RawMessage, e
 
 func (d *carrierWebDriver) SetLastAction(context.Context, json.RawMessage) error { return nil }
 
+func (d *carrierWebDriver) SetLogs(context.Context, json.RawMessage) error { return nil }
+
 // TestRunner_TransitionalStepNeverAdvancesThePageCarrier pins the ordering the
 // web path depends on. The page-side extractors must run only on steps the
 // verifier accepts: their getters advance spec state every time they evaluate,
@@ -172,6 +174,8 @@ func (d *installFailsWebDriver) SetLastAction(context.Context, json.RawMessage)
 	return errors.New("__sanderlingSetLastAction__ is not a function")
 }
 
+func (d *installFailsWebDriver) SetLogs(context.Context, json.RawMessage) error { return nil }
+
 // TestRunner_LastActionInstallFailureFailsTheRun covers the other half of the
 // same trust boundary. A run that cannot install lastAction in the page cannot
 // apply the page's extractor values either, so the step keeps goja's
@@ -200,3 +204,49 @@ func TestRunner_LastActionInstallFailureFailsTheRun(t *testing.T) {
 		t.Errorf("Run error = %v, want it to name the failed lastAction install", err)
 	}
 }
+
+// logInstallFailsWebDriver takes lastAction and refuses the logs, the shape a
+// page carrying an older published @sanderling/spec runtime has: it knows the
+// action setter and not the log one.
+type logInstallFailsWebDriver struct {
+	*installFailsWebDriver
+}
+
+func (d *logInstallFailsWebDriver) SetLastAction(context.Context, json.RawMessage) error {
+	return nil
+}
+
+func (d *logInstallFailsWebDriver) SetLogs(context.Context, json.RawMessage) error {
+	return errors.New("__sanderlingSetLogs__ is not a function")
+}
+
+// TestRunner_LogInstallFailureFailsTheRun holds the log channel to the same
+// standard as the action one. The driver having the console errors decides
+// nothing on web: the page's reading of every extractor replaces the host's, so
+// a run that cannot put the entries back into the page evaluates noLogcatErrors
+// against an empty array and reports green on a console full of errors.
+// Continuing past this is the vacuity the whole install exists to prevent.
+func TestRunner_LogInstallFailureFailsTheRun(t *testing.T) {
+	state := newHarnessWithSpec(t, carrierSpec)
+	web := &logInstallFailsWebDriver{
+		installFailsWebDriver: &installFailsWebDriver{Driver: state.mock},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	_, err := Run(ctx, Options{
+		Duration:    2 * time.Second,
+		IdleTimeout: 20 * time.Millisecond,
+		MaxSteps:    3,
+		Driver:      web,
+		Verifier:    state.verifier,
+		TraceWriter: state.writer,
+	})
+	if err == nil {
+		t.Fatal("Run succeeded with a page that cannot take the step's logs; " +
+			"every property reading the log stream ran against an empty array")
+	}
+	if !bytes.Contains([]byte(err.Error()), []byte("install logs")) {
+		t.Errorf("Run error = %v, want it to name the failed log install", err)
+	}
+}

internal/runner/web_extractor_trace_test.go

@@ -47,6 +47,8 @@ func (d *webMockDriver) NextActionFromV8(context.Context) (json.RawMessage, erro
 
 func (d *webMockDriver) SetLastAction(context.Context, json.RawMessage) error { return nil }
 
+func (d *webMockDriver) SetLogs(context.Context, json.RawMessage) error { return nil }
+
 // TestRunner_TraceRecordsTheValueTheVerdictUsed fails if the trace and the
 // verdict disagree about an extractor. A witness is only an explanation of a
 // violation if it holds the state the violated property was evaluated against.

internal/runner/web_last_action_test.go

@@ -1,12 +1,16 @@
 package runner
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
+	"log/slog"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/priyanshujain/sanderling/internal/driver"
 	mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
 )
 
@@ -26,7 +30,8 @@ globalThis.properties = {};
 // control, so the runner has a real applied action to report on the next step.
 type tappingWebDriver struct {
 	*mockdriver.Driver
-	installed []string
+	installed     []string
+	installedLogs []string
 }
 
 func (d *tappingWebDriver) InstallBundle(context.Context, []byte) error { return nil }
@@ -44,6 +49,11 @@ func (d *tappingWebDriver) SetLastAction(_ context.Context, encoded json.RawMess
 	return nil
 }
 
+func (d *tappingWebDriver) SetLogs(_ context.Context, encoded json.RawMessage) error {
+	d.installedLogs = append(d.installedLogs, string(encoded))
+	return nil
+}
+
 func TestRunner_WebInstallsLastActionInThePage(t *testing.T) {
 	state := newHarnessWithSpec(t, lastActionSpec)
 	web := &tappingWebDriver{Driver: state.mock}
@@ -78,6 +88,73 @@ func TestRunner_WebInstallsLastActionInThePage(t *testing.T) {
 	}
 }
 
+// The same hole on the other channel: state.logs was hardcoded [] in
+// pkg/spec/src/web-runtime.ts, and because the page's reading of an extractor
+// replaces the host's on web, the driver's error-level entries never reached a
+// property. The default noLogcatErrors counted an empty array on every run.
+func TestRunner_WebInstallsTheStepsLogsInThePage(t *testing.T) {
+	state := newHarnessWithSpec(t, lastActionSpec)
+	state.mock.LogEntries = []driver.LogEntry{
+		{UnixMillis: 1700000000123, Level: "E", Tag: "console", Message: "boom from the page"},
+	}
+	web := &tappingWebDriver{Driver: state.mock}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if _, err := Run(ctx, Options{
+		Duration:    time.Hour,
+		IdleTimeout: 20 * time.Millisecond,
+		MaxSteps:    2,
+		Driver:      web,
+		Verifier:    state.verifier,
+		TraceWriter: state.writer,
+	}); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	if len(web.installedLogs) == 0 {
+		t.Fatal("the page was never handed the step's logs; every property reading " +
+			"state.logs evaluated against the empty array the page starts with")
+	}
+	// The shape is the goja host's (internal/verifier/marshal.go logFields),
+	// pinned against it by TestLogs_WebJSONMatchesTheGojaObject.
+	const want = `[{"unixMillis":1700000000123,"level":"E","tag":"console","message":"boom from the page"}]`
+	if web.installedLogs[0] != want {
+		t.Errorf("step 1 installed %s, want %s", web.installedLogs[0], want)
+	}
+}
+
+// A log fetch that fails decides the verdict of every log property: they all
+// evaluate against an empty slice and hold. That is not a fact about the app,
+// so the step it happened on has to be visible in the run's output. It used to
+// be dropped in silence, under a comment claiming it was warned about.
+func TestRunner_ReportsALogFetchItCouldNotMake(t *testing.T) {
+	state := newHarnessWithSpec(t, lastActionSpec)
+	state.mock.Failures[mockdriver.ActionRecentLogs] = errors.New("adb: device offline")
+
+	var buffer bytes.Buffer
+	logger := slog.New(slog.NewTextHandler(&buffer, &slog.HandlerOptions{Level: slog.LevelWarn}))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if _, err := Run(ctx, Options{
+		Duration:    time.Hour,
+		IdleTimeout: 20 * time.Millisecond,
+		MaxSteps:    2,
+		Driver:      state.mock,
+		Verifier:    state.verifier,
+		TraceWriter: state.writer,
+		Logger:      logger,
+	}); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	if !strings.Contains(buffer.String(), "adb: device offline") {
+		t.Errorf("the run never reported the failed log fetch, so noLogcatErrors "+
+			"held on evidence nobody collected; log was %q", buffer.String())
+	}
+}
+
 // failingTapWebDriver dispatches the tap and then fails the call, the shape an
 // RPC deadline takes: the page has the click, the runner has an error.
 type failingTapWebDriver struct {

internal/verifier/marshal.go

@@ -468,19 +468,44 @@ func runtimeMillis(stepTime, runStart time.Time) int64 {
 	return stepTime.Sub(runStart).Milliseconds()
 }
 
+// logFields is the ONE description of a state.logs entry, for the same reason
+// lastActionFields is: the goja host turns it into a JS object (logsArray) and
+// the web host receives the same fields as JSON (EncodeLogs), so a property
+// counting error-level lines cannot read one shape on native and another on web.
+func logFields(entry LogEntry) []actionField {
+	return []actionField{
+		{key: "unixMillis", value: entry.UnixMillis},
+		{key: "level", value: entry.Level},
+		{key: "tag", value: entry.Tag},
+		{key: "message", value: entry.Message},
+	}
+}
+
 func logsArray(runtime *goja.Runtime, logs []LogEntry) *goja.Object {
 	array := runtime.NewArray()
 	for index, entry := range logs {
-		item := runtime.NewObject()
-		_ = item.Set("unixMillis", entry.UnixMillis)
-		_ = item.Set("level", entry.Level)
-		_ = item.Set("tag", entry.Tag)
-		_ = item.Set("message", entry.Message)
-		_ = array.Set(fmt.Sprintf("%d", index), item)
+		_ = array.Set(fmt.Sprintf("%d", index), objectFromFields(runtime, logFields(entry)))
 	}
 	return array
 }
 
+// EncodeLogs renders this step's log entries for the web host, which has no
+// Go-side state object to read: the runner pushes this JSON into the page
+// before each extractor evaluation. No entries encodes as an empty array, the
+// same value the goja host reports for a step whose log fetch found nothing.
+func EncodeLogs(logs []LogEntry) json.RawMessage {
+	var buffer bytes.Buffer
+	buffer.WriteByte('[')
+	for index, entry := range logs {
+		if index > 0 {
+			buffer.WriteByte(',')
+		}
+		buffer.Write(encodeFields(logFields(entry)))
+	}
+	buffer.WriteByte(']')
+	return buffer.Bytes()
+}
+
 func exceptionsArray(runtime *goja.Runtime, exceptions []Exception) *goja.Object {
 	array := runtime.NewArray()
 	for index, exception := range exceptions {

internal/verifier/marshal_test.go

@@ -282,3 +282,50 @@ func TestLastAction_ReportsARelaunchSeparatelyFromTheDispatch(t *testing.T) {
 		})
 	}
 }
+
+// TestLogs_WebJSONMatchesTheGojaObject pins state.logs to ONE shape across the
+// two hosts, for the same reason lastAction is pinned. On web the page's
+// reading of every extractor replaces the host's, so state.logs is whatever
+// EncodeLogs put in the page: a field this side renames or cases differently
+// leaves the default noLogcatErrors counting nothing on web while it counts on
+// native, with nothing reporting that it never saw an entry.
+func TestLogs_WebJSONMatchesTheGojaObject(t *testing.T) {
+	verifier := newVerifier(t)
+	mustLoad(t, verifier, `
+		globalThis.lines = __sanderling__.extract(state => JSON.stringify(state.logs));
+	`)
+
+	for _, testCase := range []struct {
+		name string
+		logs []LogEntry
+	}{
+		{"none", nil},
+		{"empty", []LogEntry{}},
+		{
+			"one error",
+			[]LogEntry{{UnixMillis: 1700000000123, Level: "E", Tag: "console", Message: "boom from the page"}},
+		},
+		{
+			"mixed levels",
+			[]LogEntry{
+				{UnixMillis: 1, Level: "E", Tag: "console", Message: `say "hi" <b> & co`},
+				{UnixMillis: 2, Level: "W", Tag: "AndroidRuntime", Message: "a warning"},
+			},
+		},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			if err := verifier.PushSnapshot(SnapshotInput{
+				Snapshots: Snapshots{},
+				Logs:      testCase.logs,
+			}); err != nil {
+				t.Fatal(err)
+			}
+			handle := verifier.runtime.GlobalObject().Get("lines").ToObject(verifier.runtime)
+			goja := handle.Get("current").String()
+			web := string(EncodeLogs(testCase.logs))
+			if goja != web {
+				t.Errorf("the two hosts disagree on state.logs\n goja: %s\n  web: %s", goja, web)
+			}
+		})
+	}
+}

pkg/spec/src/web-runtime.ts

@@ -445,6 +445,15 @@ if (typeof globalThis.addEventListener === "function") {
 // that reads state.lastAction vacuously true on web.
 let lastAction: unknown = null;
 
+// logs is what the driver captured between the previous step and this one,
+// pushed in by the Go runner (via __sanderlingSetLogs__) before each extractor
+// evaluation, in the shape internal/verifier/marshal.go builds for goja. The
+// page cannot derive it: console output reaches the runner over CDP and nothing
+// in the page reads it back. Hardcoding [] here, as this file used to, makes
+// every spec property that reads state.logs vacuously true on web, the default
+// noLogcatErrors included, because the page's reading is the one that wins.
+let logs: unknown[] = [];
+
 function buildState(): unknown {
   return {
     snapshots: {},
@@ -453,7 +462,7 @@ function buildState(): unknown {
     window,
     lastAction,
     time: 0,
-    logs: [],
+    logs,
     exceptions: capturedExceptions.slice(),
   };
 }
@@ -502,6 +511,11 @@ defineLockedGlobal("__sanderlingSetLastAction__", (value: unknown) => {
   lastAction = value ?? null;
 });
 
+// The host calls this once per step too, alongside __sanderlingSetLastAction__.
+defineLockedGlobal("__sanderlingSetLogs__", (value: unknown) => {
+  logs = Array.isArray(value) ? value : [];
+});
+
 // writable:false stops a page script from shadowing the runtime via plain
 // assignment (the realistic in-page threat). configurable:true is required so
 // unit tests sharing one process can reinstall a fake via defineProperty; a

pkg/spec/test/web-runtime.test.ts

@@ -365,6 +365,35 @@ test("state.lastAction is null when the host pushed nothing", () => {
   assert.equal(lastActionSeenByASpec(null), null);
 });
 
+// state.logs is the same kind of hole. Console output reaches the runner over
+// CDP, so the page cannot read it back, and while the web runtime hardcoded []
+// there the default noLogcatErrors counted an empty array on every web run: a
+// page whose console was full of errors went green having checked nothing.
+function logsSeenByASpec(pushed: unknown): unknown {
+  const setLogs = (globalThis as Record<string, unknown>).__sanderlingSetLogs__ as (
+    value: unknown,
+  ) => void;
+  __testing__.extractors.length = 0;
+  __testing__.runtime.extract((state) => (state as { logs: unknown }).logs);
+  let out: Record<number, { value?: unknown }> = {};
+  withState(() => {
+    setLogs(pushed);
+    out = __testing__.evaluateExtractors();
+  });
+  return readingOf(out, 0);
+}
+
+test("state.logs carries the entries the host pushed", () => {
+  const entries = [
+    { unixMillis: 1700000000123, level: "E", tag: "console", message: "boom from the page" },
+  ];
+  assert.deepEqual(logsSeenByASpec(entries), entries);
+});
+
+test("state.logs is empty when the host pushed no entries", () => {
+  assert.deepEqual(logsSeenByASpec([]), []);
+});
+
 // sanitize runs over every extractor's return value before it leaves the
 // runtime. A user extractor that returns a page object reachable from
 // document/window can be self-referential, carry functions, or nest deeply;

test/browser/console_levels_test.go

@@ -81,6 +81,33 @@ func TestBrowserConsoleErrorReachesTheSpec(t *testing.T) {
 	}
 }
 
+// TestBrowserConsoleErrorFiresTheLogProperty drives the same page through the
+// whole bundle -> run -> verify pipeline instead of hand-assembling a snapshot.
+// The driver holding the entry is not enough on web: every extractor reading is
+// replaced by the one the page computed, so state.logs is whatever the page
+// says it is, and a page that answers "no logs" leaves noLogcatErrors green on
+// a run whose console was full of errors.
+func TestBrowserConsoleErrorFiresTheLogProperty(t *testing.T) {
+	violations := runFixture(t, "console-levels")
+	if !slices.Contains(violations, "noLogcatErrors") {
+		t.Fatalf("a page calling console.error ran a whole run without noLogcatErrors firing; violations=%v", violations)
+	}
+}
+
+// TestBrowserQuietPageKeepsTheLogPropertySatisfied is the other half: a page
+// whose console never reaches the error level must leave noLogcatErrors alone,
+// so the property is reporting what the page logged rather than being on
+// whenever the run is web.
+func TestBrowserQuietPageKeepsTheLogPropertySatisfied(t *testing.T) {
+	violations := runFixture(t, "console-quiet")
+	if slices.Contains(violations, "noLogcatErrors") {
+		t.Errorf("noLogcatErrors fired on a page that logged nothing at error level; violations=%v", violations)
+	}
+	if !slices.Contains(violations, "counterNeverMoves") {
+		t.Fatalf("nothing was ever pressed, so the run proves nothing about a property that can fire; violations=%v", violations)
+	}
+}
+
 // TestBrowserConsoleLevelsMapToTheLogcatScale pins what each console verb
 // becomes once it crosses the driver. driver.LogEntry.Level is the single-letter
 // logcat scale on every platform, so a spec asking for warnings or debug lines

test/browser/testdata/console-quiet/index.html

@@ -0,0 +1,23 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>console-quiet</title>
+  </head>
+  <body>
+    <button id="press" style="width:200px;height:80px">press</button>
+    <div id="count">0</div>
+    <script>
+      // Noisy below the error level. A run that reports these as errors is
+      // reporting a healthy page as broken.
+      console.warn("a warning");
+      console.log("a plain log");
+      console.info("an info line");
+      let presses = 0;
+      document.getElementById("press").addEventListener("click", function () {
+        presses += 1;
+        document.getElementById("count").textContent = String(presses);
+      });
+    </script>
+  </body>
+</html>

test/browser/testdata/console-quiet/spec.ts

@@ -0,0 +1,15 @@
+import { always, extract, taps } from "@sanderling/spec";
+import { noLogcatErrors } from "@sanderling/spec/defaults/properties";
+
+const presses = extract((s) => {
+  const el = s.ax.find({ id: "count" });
+  return el ? parseInt(el.text, 10) || 0 : 0;
+}).named("presses");
+
+// The run has to actually be driving the page, or noLogcatErrors staying
+// satisfied says nothing about whether it can fire at all.
+const counterNeverMoves = always(() => presses.current === 0);
+
+export const properties = { noLogcatErrors, counterNeverMoves };
+
+export const actionsRoot = taps;