From a4c2a6b91c54545e0a9f0f52801d0576cd6520ed Mon Sep 17 00:00:00 2001 From: PJ Date: Sun, 16 Aug 2026 01:23:04 +0530 Subject: [PATCH 1/7] docs(driver): state the log level scale on LogEntry Claude-Session: https://claude.ai/code/session_01ShuAy8q8ZfPi8KHxwc8JpQ --- internal/driver/driver.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 649d673..ca9366d 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -82,6 +82,12 @@ type FocusedWindowChecker interface { FocusedWindowApp(ctx context.Context) (string, error) } +// LogEntry is one line of device log. Level is logcat's single-letter scale on +// every platform: "V", "D", "I", "W", "E", "F", ordered as written. The runner +// fetches at "E" and the default properties count entries whose level equals +// "E", so a driver that spells a level any other way empties the channel +// without failing anything: the entries never arrive and every property reading +// state.logs holds vacuously. type LogEntry struct { UnixMillis int64 Level string From 9b1fbdf60f8b83f4be20b400179deb3a4fd6a199 Mon Sep 17 00:00:00 2001 From: PJ Date: Sun, 16 Aug 2026 01:23:17 +0530 Subject: [PATCH 2/7] fix(chrome): keep a log entry the level scale cannot rank Claude-Session: https://claude.ai/code/session_01ShuAy8q8ZfPi8KHxwc8JpQ --- internal/driver/chrome/driver.go | 10 ++++++++- internal/driver/chrome/logs_test.go | 33 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 internal/driver/chrome/logs_test.go diff --git a/internal/driver/chrome/driver.go b/internal/driver/chrome/driver.go index fac3554..37d48a8 100644 --- a/internal/driver/chrome/driver.go +++ b/internal/driver/chrome/driver.go @@ -712,9 +712,17 @@ func (d *Driver) Metrics(ctx context.Context, _ string) (driver.Metrics, error) }, nil } +// meetsLevel keeps an entry whose level the scale cannot rank. Ranking an +// unknown level below every threshold drops it, and a dropped entry is +// indistinguishable from a quiet app: the caller sees silence and reports it as +// health. func meetsLevel(level, minLevel string) bool { order := map[string]int{"V": 0, "D": 1, "I": 2, "W": 3, "E": 4, "F": 5} - return order[level] >= order[minLevel] + rank, ranked := order[level] + if !ranked { + return true + } + return rank >= order[minLevel] } func pngDimensions(png []byte) (int, int) { diff --git a/internal/driver/chrome/logs_test.go b/internal/driver/chrome/logs_test.go new file mode 100644 index 0000000..3457b9a --- /dev/null +++ b/internal/driver/chrome/logs_test.go @@ -0,0 +1,33 @@ +package chrome + +import "testing" + +// A level the scale does not name is unknown, not verbose. Ranking it below +// every threshold is what silently emptied the web log channel: the entries +// existed, the filter dropped them, and the run reported nothing. Evidence the +// filter cannot rank has to reach the caller, who can at least see it. +func TestMeetsLevel(t *testing.T) { + cases := []struct { + level string + minLevel string + want bool + }{ + {"E", "E", true}, + {"F", "E", true}, + {"W", "E", false}, + {"I", "E", false}, + {"D", "E", false}, + {"V", "E", false}, + {"W", "W", true}, + {"I", "W", false}, + {"D", "V", true}, + {"ERROR", "E", true}, + {"WARNING", "E", true}, + {"", "E", true}, + } + for _, tc := range cases { + if got := meetsLevel(tc.level, tc.minLevel); got != tc.want { + t.Errorf("meetsLevel(%q, %q) = %v, want %v", tc.level, tc.minLevel, got, tc.want) + } + } +} From beb036d2c1dc8fa035ab96e0547ee01b77bdc321 Mon Sep 17 00:00:00 2001 From: PJ Date: Sun, 16 Aug 2026 01:23:45 +0530 Subject: [PATCH 3/7] fix(chrome): record console levels on the logcat scale Claude-Session: https://claude.ai/code/session_01ShuAy8q8ZfPi8KHxwc8JpQ --- internal/driver/chrome/driver.go | 22 +++++++++++++++++----- internal/driver/chrome/logs_test.go | 29 ++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/internal/driver/chrome/driver.go b/internal/driver/chrome/driver.go index 37d48a8..dfa177a 100644 --- a/internal/driver/chrome/driver.go +++ b/internal/driver/chrome/driver.go @@ -79,14 +79,10 @@ func New() *Driver { } } } - level := strings.ToUpper(string(e.Type)) - if level == "LOG" { - level = "I" - } d.logsMu.Lock() d.logs = append(d.logs, driver.LogEntry{ UnixMillis: int64(e.Timestamp.Time().UnixMilli()), - Level: level, + Level: consoleLevel(e.Type), Tag: "console", Message: strings.Join(parts, " "), }) @@ -712,6 +708,22 @@ func (d *Driver) Metrics(ctx context.Context, _ string) (driver.Metrics, error) }, nil } +// consoleLevel places a console call on driver.LogEntry's logcat scale. The +// verbs a spec acts on are all named here; the rest are info rather than "E" +// because promoting them would convict an app of an error it never logged. +func consoleLevel(apiType runtime.APIType) string { + switch apiType { + case runtime.APITypeError, runtime.APITypeAssert: + return "E" + case runtime.APITypeWarning: + return "W" + case runtime.APITypeDebug: + return "D" + default: + return "I" + } +} + // meetsLevel keeps an entry whose level the scale cannot rank. Ranking an // unknown level below every threshold drops it, and a dropped entry is // indistinguishable from a quiet app: the caller sees silence and reports it as diff --git a/internal/driver/chrome/logs_test.go b/internal/driver/chrome/logs_test.go index 3457b9a..4cb81ea 100644 --- a/internal/driver/chrome/logs_test.go +++ b/internal/driver/chrome/logs_test.go @@ -1,6 +1,33 @@ package chrome -import "testing" +import ( + "testing" + + "github.com/chromedp/cdproto/runtime" +) + +// Every console verb has to land on the logcat scale driver.LogEntry declares: +// the runner fetches at "E" and the default properties count entries whose +// level equals "E", so a level spelled any other way is an error the spec never +// sees. A verb with no mapping is info, which is honest about severity without +// fabricating an error the page never logged. +func TestConsoleLevel(t *testing.T) { + cases := map[runtime.APIType]string{ + runtime.APITypeError: "E", + runtime.APITypeAssert: "E", + runtime.APITypeWarning: "W", + runtime.APITypeDebug: "D", + runtime.APITypeLog: "I", + runtime.APITypeInfo: "I", + runtime.APITypeTable: "I", + runtime.APIType("countReset"): "I", + } + for apiType, want := range cases { + if got := consoleLevel(apiType); got != want { + t.Errorf("consoleLevel(%q) = %q, want %q", apiType, got, want) + } + } +} // A level the scale does not name is unknown, not verbose. Ranking it below // every threshold is what silently emptied the web log channel: the entries From fdc616247a3c7fda9962a02c89f7a57096071177 Mon Sep 17 00:00:00 2001 From: PJ Date: Sun, 16 Aug 2026 01:23:58 +0530 Subject: [PATCH 4/7] test(browser): drive a console error through to the spec Claude-Session: https://claude.ai/code/session_01ShuAy8q8ZfPi8KHxwc8JpQ --- test/browser/console_levels_test.go | 174 ++++++++++++++++++ .../testdata/console-levels/index.html | 24 +++ test/browser/testdata/console-levels/spec.ts | 6 + 3 files changed, 204 insertions(+) create mode 100644 test/browser/console_levels_test.go create mode 100644 test/browser/testdata/console-levels/index.html create mode 100644 test/browser/testdata/console-levels/spec.ts diff --git a/test/browser/console_levels_test.go b/test/browser/console_levels_test.go new file mode 100644 index 0000000..ac3d625 --- /dev/null +++ b/test/browser/console_levels_test.go @@ -0,0 +1,174 @@ +//go:build browser + +package browser_test + +import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/priyanshujain/sanderling/internal/driver" + "github.com/priyanshujain/sanderling/internal/driver/chrome" + "github.com/priyanshujain/sanderling/internal/hierarchy" + "github.com/priyanshujain/sanderling/internal/verifier" +) + +// TestBrowserConsoleErrorReachesTheSpec drives a page that calls console.error +// and follows the entry the whole way a run does: the driver's log fetch at the +// runner's minimum level, then into the verifier state a property reads. The +// default noLogcatErrors counts entries whose level is "E", so a driver that +// spells the level any other way leaves the property permanently satisfied on +// web with nothing reporting that it never saw anything. +func TestBrowserConsoleErrorReachesTheSpec(t *testing.T) { + ctx, driverInstance, since := launchConsoleFixture(t) + + entries, err := driverInstance.RecentLogs(ctx, since, "E") + if err != nil { + t.Fatalf("recent logs: %v", err) + } + if len(entries) != 1 { + t.Fatalf("the runner's error-level fetch returned %d entries, want the page's one console.error: %+v", len(entries), entries) + } + if entries[0].Level != "E" { + t.Errorf("console.error arrived as level %q, want %q", entries[0].Level, "E") + } + if !strings.Contains(entries[0].Message, "boom from the page") { + t.Errorf("console.error arrived as %q, want the page's message", entries[0].Message) + } + + dump, err := driverInstance.Hierarchy(ctx) + if err != nil { + t.Fatalf("hierarchy: %v", err) + } + tree, err := hierarchy.Parse(dump) + if err != nil { + t.Fatalf("parse hierarchy: %v", err) + } + + gojaBundle, _ := bundleSpec(t, filepath.Join(testdataDir(t), "console-levels", "spec.ts")) + verifierInstance, err := verifier.New( + verifier.WithSeed(fixtureSeed), + verifier.WithPlatform("web"), + ) + if err != nil { + t.Fatalf("verifier: %v", err) + } + if err := verifierInstance.Load(string(gojaBundle)); err != nil { + t.Fatalf("load spec: %v", err) + } + if err := verifierInstance.PushSnapshot(verifier.SnapshotInput{ + Tree: tree, + Logs: asVerifierLogs(entries), + }); err != nil { + t.Fatalf("push snapshot: %v", err) + } + verifierInstance.EvaluateProperties() + if violated := verifierInstance.NewlyViolatedProperties(); !slices.Contains(violated, "noLogcatErrors") { + t.Fatalf("a console.error on the page left noLogcatErrors satisfied; violations=%v", violated) + } +} + +// TestBrowserConsoleLevelsMapToTheLogcatScale pins what each console verb +// becomes once it crosses the driver. driver.LogEntry.Level is the single-letter +// logcat scale on every platform, so a spec asking for warnings or debug lines +// by letter has to get the same answer on web as it does on Android, and a +// console verb the driver has no mapping for still has to arrive rather than be +// silently discarded. +func TestBrowserConsoleLevelsMapToTheLogcatScale(t *testing.T) { + ctx, driverInstance, since := launchConsoleFixture(t) + + entries, err := driverInstance.RecentLogs(ctx, since, "V") + if err != nil { + t.Fatalf("recent logs: %v", err) + } + if len(entries) != 6 { + t.Fatalf("the page made 6 console calls, the driver kept %d: %+v", len(entries), entries) + } + + wantLevels := map[string]string{ + "boom from the page": "E", + "a warning": "W", + "a plain log": "I", + "a debug line": "D", + "an info line": "I", + } + seen := map[string]bool{} + for _, entry := range entries { + if !slices.Contains([]string{"V", "D", "I", "W", "E", "F"}, entry.Level) { + t.Errorf("entry %q carries level %q, which is not on the logcat scale a spec compares against", entry.Message, entry.Level) + } + for message, level := range wantLevels { + if !strings.Contains(entry.Message, message) { + continue + } + seen[message] = true + if entry.Level != level { + t.Errorf("console message %q arrived as level %q, want %q", message, entry.Level, level) + } + } + } + for message := range wantLevels { + if !seen[message] { + t.Errorf("console message %q never reached the driver's log buffer", message) + } + } + + errorsOnly, err := driverInstance.RecentLogs(ctx, since, "E") + if err != nil { + t.Fatalf("recent logs: %v", err) + } + if len(errorsOnly) != 1 { + t.Fatalf("the error-level fetch kept %d of the 6 entries, want only the console.error: %+v", len(errorsOnly), errorsOnly) + } + warningsUp, err := driverInstance.RecentLogs(ctx, since, "W") + if err != nil { + t.Fatalf("recent logs: %v", err) + } + if len(warningsUp) != 2 { + t.Fatalf("the warning-level fetch kept %d entries, want the console.error and the console.warn: %+v", len(warningsUp), warningsUp) + } +} + +// launchConsoleFixture serves the console-levels page and drives headless Chrome +// to it, returning the driver plus the instant before the page ran so a log +// fetch can ask for everything the page emitted. +func launchConsoleFixture(t *testing.T) (context.Context, *chrome.Driver, time.Time) { + t.Helper() + + server := httptest.NewServer(http.FileServer(http.Dir(testdataDir(t)))) + t.Cleanup(server.Close) + + driverInstance := chrome.New() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = driverInstance.Terminate(ctx) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + t.Cleanup(cancel) + + since := time.Now() + if err := driverInstance.Launch(ctx, server.URL+"/console-levels/", false, nil); err != nil { + t.Fatalf("launch: %v", err) + } + return ctx, driverInstance, since +} + +func asVerifierLogs(entries []driver.LogEntry) []verifier.LogEntry { + out := make([]verifier.LogEntry, 0, len(entries)) + for _, entry := range entries { + out = append(out, verifier.LogEntry{ + UnixMillis: entry.UnixMillis, + Level: entry.Level, + Tag: entry.Tag, + Message: entry.Message, + }) + } + return out +} diff --git a/test/browser/testdata/console-levels/index.html b/test/browser/testdata/console-levels/index.html new file mode 100644 index 0000000..56da280 --- /dev/null +++ b/test/browser/testdata/console-levels/index.html @@ -0,0 +1,24 @@ + + + + + console-levels + + + +
0
+ + + diff --git a/test/browser/testdata/console-levels/spec.ts b/test/browser/testdata/console-levels/spec.ts new file mode 100644 index 0000000..e4a513d --- /dev/null +++ b/test/browser/testdata/console-levels/spec.ts @@ -0,0 +1,6 @@ +import { taps } from "@sanderling/spec"; +import { noLogcatErrors } from "@sanderling/spec/defaults/properties"; + +export const properties = { noLogcatErrors }; + +export const actionsRoot = taps; From 546c80885e2046699ba2a5babbbe608a8691cf2d Mon Sep 17 00:00:00 2001 From: PJ Date: Sun, 16 Aug 2026 01:27:02 +0530 Subject: [PATCH 5/7] docs(ioscompanion): name the vacuity behind the empty log slice Claude-Session: https://claude.ai/code/session_01ShuAy8q8ZfPi8KHxwc8JpQ --- internal/driver/ioscompanion/driver.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/driver/ioscompanion/driver.go b/internal/driver/ioscompanion/driver.go index aa12589..d335b44 100644 --- a/internal/driver/ioscompanion/driver.go +++ b/internal/driver/ioscompanion/driver.go @@ -1084,7 +1084,10 @@ func (d *Driver) WaitForIdle(ctx context.Context, _ time.Duration) error { } // RecentLogs returns no entries: the companion log RPC is a follow-up, so v1 -// reports an empty slice rather than failing. +// reports an empty slice rather than failing. Every property reading state.logs +// therefore holds vacuously on iOS and nothing says so; closing it means +// tailing idb's streaming log RPC and mapping os_log levels onto the +// single-letter scale driver.LogEntry declares. func (d *Driver) RecentLogs(_ context.Context, _ time.Time, _ string) ([]driver.LogEntry, error) { return []driver.LogEntry{}, nil } From 59da61fecb2fc1711232127246d84e94dc147e4b Mon Sep 17 00:00:00 2001 From: PJ Date: Sun, 16 Aug 2026 01:29:58 +0530 Subject: [PATCH 6/7] fix(chrome): keep the message of an object console argument Claude-Session: https://claude.ai/code/session_01ShuAy8q8ZfPi8KHxwc8JpQ --- internal/driver/chrome/driver.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/internal/driver/chrome/driver.go b/internal/driver/chrome/driver.go index dfa177a..db51c01 100644 --- a/internal/driver/chrome/driver.go +++ b/internal/driver/chrome/driver.go @@ -70,13 +70,21 @@ func New() *Driver { } var parts []string for _, arg := range e.Args { - if arg.Value != nil { - var s string - if err := json.Unmarshal(arg.Value, &s); err == nil { - parts = append(parts, s) - } else { - parts = append(parts, string(arg.Value)) + // An object argument, which is what console.error(err) passes, + // carries no value at all: CDP sends a description instead. Reading + // only the value logged those calls with an empty message, so the + // entry named a level and nothing a reader could act on. + if arg.Value == nil { + if arg.Description != "" { + parts = append(parts, arg.Description) } + continue + } + var s string + if err := json.Unmarshal(arg.Value, &s); err == nil { + parts = append(parts, s) + } else { + parts = append(parts, string(arg.Value)) } } d.logsMu.Lock() From b131c69e7bd9643a8ea840c98a5da28f0a3344c3 Mon Sep 17 00:00:00 2001 From: PJ Date: Sun, 16 Aug 2026 01:29:58 +0530 Subject: [PATCH 7/7] test(browser): cover console.error with an error object Claude-Session: https://claude.ai/code/session_01ShuAy8q8ZfPi8KHxwc8JpQ --- test/browser/console_levels_test.go | 38 +++++++++++++------ .../testdata/console-levels/index.html | 1 + 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/test/browser/console_levels_test.go b/test/browser/console_levels_test.go index ac3d625..a1b81de 100644 --- a/test/browser/console_levels_test.go +++ b/test/browser/console_levels_test.go @@ -31,14 +31,22 @@ func TestBrowserConsoleErrorReachesTheSpec(t *testing.T) { if err != nil { t.Fatalf("recent logs: %v", err) } - if len(entries) != 1 { - t.Fatalf("the runner's error-level fetch returned %d entries, want the page's one console.error: %+v", len(entries), entries) + if len(entries) != 2 { + t.Fatalf("the runner's error-level fetch returned %d entries, want the page's two console.error calls: %+v", len(entries), entries) } - if entries[0].Level != "E" { - t.Errorf("console.error arrived as level %q, want %q", entries[0].Level, "E") + for _, entry := range entries { + if entry.Level != "E" { + t.Errorf("console.error %q arrived as level %q, want %q", entry.Message, entry.Level, "E") + } } - if !strings.Contains(entries[0].Message, "boom from the page") { - t.Errorf("console.error arrived as %q, want the page's message", entries[0].Message) + // console.error(err) is the ordinary way a page reports a failure, and the + // argument is then an object rather than a string. An entry that arrives + // with the right level and no message names nothing a reader can act on. + if !messageSeen(entries, "boom from the page") { + t.Errorf("the page's console.error string never arrived: %+v", entries) + } + if !messageSeen(entries, "object arg detail") { + t.Errorf("console.error(new Error(...)) arrived with no message: %+v", entries) } dump, err := driverInstance.Hierarchy(ctx) @@ -86,8 +94,8 @@ func TestBrowserConsoleLevelsMapToTheLogcatScale(t *testing.T) { if err != nil { t.Fatalf("recent logs: %v", err) } - if len(entries) != 6 { - t.Fatalf("the page made 6 console calls, the driver kept %d: %+v", len(entries), entries) + if len(entries) != 7 { + t.Fatalf("the page made 7 console calls, the driver kept %d: %+v", len(entries), entries) } wantLevels := map[string]string{ @@ -122,15 +130,15 @@ func TestBrowserConsoleLevelsMapToTheLogcatScale(t *testing.T) { if err != nil { t.Fatalf("recent logs: %v", err) } - if len(errorsOnly) != 1 { - t.Fatalf("the error-level fetch kept %d of the 6 entries, want only the console.error: %+v", len(errorsOnly), errorsOnly) + if len(errorsOnly) != 2 { + t.Fatalf("the error-level fetch kept %d of the 7 entries, want only the console.error calls: %+v", len(errorsOnly), errorsOnly) } warningsUp, err := driverInstance.RecentLogs(ctx, since, "W") if err != nil { t.Fatalf("recent logs: %v", err) } - if len(warningsUp) != 2 { - t.Fatalf("the warning-level fetch kept %d entries, want the console.error and the console.warn: %+v", len(warningsUp), warningsUp) + if len(warningsUp) != 3 { + t.Fatalf("the warning-level fetch kept %d entries, want the console.error calls and the console.warn: %+v", len(warningsUp), warningsUp) } } @@ -160,6 +168,12 @@ func launchConsoleFixture(t *testing.T) (context.Context, *chrome.Driver, time.T return ctx, driverInstance, since } +func messageSeen(entries []driver.LogEntry, want string) bool { + return slices.ContainsFunc(entries, func(entry driver.LogEntry) bool { + return strings.Contains(entry.Message, want) + }) +} + func asVerifierLogs(entries []driver.LogEntry) []verifier.LogEntry { out := make([]verifier.LogEntry, 0, len(entries)) for _, entry := range entries { diff --git a/test/browser/testdata/console-levels/index.html b/test/browser/testdata/console-levels/index.html index 56da280..1683068 100644 --- a/test/browser/testdata/console-levels/index.html +++ b/test/browser/testdata/console-levels/index.html @@ -9,6 +9,7 @@
0