From c8d52cb7d56d45bf8e9d56bcd0505d6fb6289b01 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 18 Aug 2026 20:14:59 +0530 Subject: [PATCH] test(browser): the exit code a dead run and a violated one actually leave Drives the built binary against a page with nothing to tap and reads the process status, then the same run through campaign to pin what lands in runs.jsonl: exit_code 1 there is a detection the analysis drops as missing data. --- test/browser/dead_run_test.go | 180 ++++++++++++++++++ .../testdata/nothing-to-tap/index.html | 10 + test/browser/testdata/nothing-to-tap/spec.ts | 9 + .../testdata/nothing-to-tap/violated-spec.ts | 9 + 4 files changed, 208 insertions(+) create mode 100644 test/browser/dead_run_test.go create mode 100644 test/browser/testdata/nothing-to-tap/index.html create mode 100644 test/browser/testdata/nothing-to-tap/spec.ts create mode 100644 test/browser/testdata/nothing-to-tap/violated-spec.ts diff --git a/test/browser/dead_run_test.go b/test/browser/dead_run_test.go new file mode 100644 index 0000000..c67df5b --- /dev/null +++ b/test/browser/dead_run_test.go @@ -0,0 +1,180 @@ +//go:build browser + +package browser_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// The nothing-to-tap fixture is a page holding one line of static text, so the +// seeded generator finds no candidate on any step and the run ends having +// driven the app zero times. It is the shape a provider outage gives a model +// arm, reproduced without a provider. +const deadRunFixture = "nothing-to-tap" + +// TestBrowserDeadRunExitCodes drives the real binary against that page and +// reads the status the shell would: the refusal is only worth what the process +// exit code says, and CI, campaign and the analysis all read it rather than the +// error value. +func TestBrowserDeadRunExitCodes(t *testing.T) { + binary := buildBinary(t, "./cmd/sanderling") + pageURL := servePage(t) + + for _, testCase := range []struct { + name string + spec string + flags []string + wantExit int + wantSays string + }{ + { + name: "a run that explored nothing is refused", + spec: "spec.ts", + wantExit: 1, + wantSays: "the action generator drove the app in none of them", + }, + { + name: "the property-free flag waives the refusal it names and no other", + spec: "spec.ts", + flags: []string{"--allow-no-properties"}, + wantExit: 1, + wantSays: "the action generator drove the app in none of them", + }, + { + name: "the dead-run flag waives the dead-run refusal", + spec: "spec.ts", + flags: []string{"--allow-no-generator-actions"}, + wantExit: 0, + }, + { + // The campaign case: no flags are passed there, and a run that + // recorded a violation has produced the verdict the refusal exists + // to demand. + name: "a recorded violation carries the run without any flag", + spec: "violated-spec.ts", + wantExit: 0, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + arguments := append(runFlags(t, testCase.spec, pageURL), testCase.flags...) + exitCode, output := runBinary(t, binary, arguments) + + if exitCode != testCase.wantExit { + t.Fatalf("exit code: got %d, want %d\n%s", exitCode, testCase.wantExit, output) + } + if testCase.wantSays != "" && !strings.Contains(output, testCase.wantSays) { + t.Errorf("the run never said %q:\n%s", testCase.wantSays, output) + } + if testCase.wantExit != 0 && !strings.Contains(output, "--allow-no-generator-actions") { + t.Errorf("the refusal never names the flag that waives it:\n%s", output) + } + }) + } +} + +// TestBrowserCampaignRecordsAViolationFoundWithoutTheGenerator follows the +// whole chain a campaign is read through. A campaign passes no +// --exit-on-violation, so a refusal here writes exit_code 1 and the analysis +// drops the run as missing data: the detection the campaign exists to count +// disappears into the excluded pile. +func TestBrowserCampaignRecordsAViolationFoundWithoutTheGenerator(t *testing.T) { + sanderling := buildBinary(t, "./cmd/sanderling") + campaign := buildBinary(t, "./cmd/internal-tools/campaign") + pageURL := servePage(t) + campaignDirectory := filepath.Join(t.TempDir(), "campaign") + + exitCode, output := runBinary(t, campaign, []string{ + "--spec", fixtureSpec(t, "violated-spec.ts"), + "--bundle-id", pageURL, + "--platform", "web", + "--arm", "dead-run", + "--generator", "seeded", + "--max-steps", "3", + "--duration", "60s", + "--seeds", "1", + "--sanderling", sanderling, + "--output", campaignDirectory, + }) + if exitCode != 0 { + t.Fatalf("campaign exit code: got %d, want 0\n%s", exitCode, output) + } + + body, err := os.ReadFile(filepath.Join(campaignDirectory, "runs.jsonl")) + if err != nil { + t.Fatal(err) + } + var record struct { + ExitCode int `json:"exit_code"` + Actions int `json:"actions"` + ViolatedProperties []string `json:"violated_properties"` + } + if err := json.Unmarshal(body, &record); err != nil { + t.Fatalf("runs.jsonl: %v\n%s", err, body) + } + + if record.ExitCode != 0 { + t.Errorf("exit_code: got %d, want 0. The analysis excludes a non-zero record as "+ + "missing data, so this violation would never reach the survival analysis", record.ExitCode) + } + if record.Actions != 0 { + t.Fatalf("actions: got %d, want 0: the fixture offers the generator nothing to drive", record.Actions) + } + if len(record.ViolatedProperties) != 1 || record.ViolatedProperties[0] != "bannerIsEmpty" { + t.Errorf("violated_properties: got %v, want [bannerIsEmpty]", record.ViolatedProperties) + } +} + +func runFlags(t *testing.T, spec, pageURL string) []string { + t.Helper() + return []string{ + "test", + "--platform", "web", + "--spec", fixtureSpec(t, spec), + "--bundle-id", pageURL, + "--duration", "60s", + "--max-steps", "3", + "--seed", "1", + "--output", t.TempDir(), + } +} + +func fixtureSpec(t *testing.T, name string) string { + t.Helper() + return filepath.Join(testdataDir(t), deadRunFixture, name) +} + +func servePage(t *testing.T) string { + t.Helper() + server := httptest.NewServer(http.FileServer(http.Dir(testdataDir(t)))) + t.Cleanup(server.Close) + return server.URL + "/" + deadRunFixture + "/" +} + +func buildBinary(t *testing.T, packagePath string) string { + t.Helper() + binary := filepath.Join(t.TempDir(), filepath.Base(packagePath)) + build := exec.Command("go", "build", "-o", binary, packagePath) + build.Dir = repoRoot(t) + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("build %s: %v\n%s", packagePath, err, output) + } + return binary +} + +func runBinary(t *testing.T, binary string, arguments []string) (int, string) { + t.Helper() + command := exec.Command(binary, arguments...) + command.Dir = repoRoot(t) + output, err := command.CombinedOutput() + if command.ProcessState == nil { + t.Fatalf("%s never ran: %v", binary, err) + } + return command.ProcessState.ExitCode(), string(output) +} diff --git a/test/browser/testdata/nothing-to-tap/index.html b/test/browser/testdata/nothing-to-tap/index.html new file mode 100644 index 0000000..f4e0a63 --- /dev/null +++ b/test/browser/testdata/nothing-to-tap/index.html @@ -0,0 +1,10 @@ + + + + + nothing-to-tap + + + + + diff --git a/test/browser/testdata/nothing-to-tap/spec.ts b/test/browser/testdata/nothing-to-tap/spec.ts new file mode 100644 index 0000000..61542a0 --- /dev/null +++ b/test/browser/testdata/nothing-to-tap/spec.ts @@ -0,0 +1,9 @@ +import { always, extract, taps } from "@sanderling/spec"; + +const banner = extract((s) => s.ax.find({ id: "banner" })?.text ?? "").named("banner"); + +const bannerStays = always(() => banner.current === "nothing here answers a tap"); + +export const properties = { bannerStays }; + +export const actionsRoot = taps; diff --git a/test/browser/testdata/nothing-to-tap/violated-spec.ts b/test/browser/testdata/nothing-to-tap/violated-spec.ts new file mode 100644 index 0000000..9010c67 --- /dev/null +++ b/test/browser/testdata/nothing-to-tap/violated-spec.ts @@ -0,0 +1,9 @@ +import { always, extract, taps } from "@sanderling/spec"; + +const banner = extract((s) => s.ax.find({ id: "banner" })?.text ?? "").named("banner"); + +const bannerIsEmpty = always(() => banner.current === ""); + +export const properties = { bannerIsEmpty }; + +export const actionsRoot = taps;