test(browser): the exit code a dead run and a violated one actually leave

Drives the built binary against a page with nothing to tap and reads the
process status, then the same run through campaign to pin what lands in
runs.jsonl: exit_code 1 there is a detection the analysis drops as missing
data.

test/browser/dead_run_test.go

@@ -0,0 +1,180 @@
+//go:build browser
+
+package browser_test
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// The nothing-to-tap fixture is a page holding one line of static text, so the
+// seeded generator finds no candidate on any step and the run ends having
+// driven the app zero times. It is the shape a provider outage gives a model
+// arm, reproduced without a provider.
+const deadRunFixture = "nothing-to-tap"
+
+// TestBrowserDeadRunExitCodes drives the real binary against that page and
+// reads the status the shell would: the refusal is only worth what the process
+// exit code says, and CI, campaign and the analysis all read it rather than the
+// error value.
+func TestBrowserDeadRunExitCodes(t *testing.T) {
+	binary := buildBinary(t, "./cmd/sanderling")
+	pageURL := servePage(t)
+
+	for _, testCase := range []struct {
+		name     string
+		spec     string
+		flags    []string
+		wantExit int
+		wantSays string
+	}{
+		{
+			name:     "a run that explored nothing is refused",
+			spec:     "spec.ts",
+			wantExit: 1,
+			wantSays: "the action generator drove the app in none of them",
+		},
+		{
+			name:     "the property-free flag waives the refusal it names and no other",
+			spec:     "spec.ts",
+			flags:    []string{"--allow-no-properties"},
+			wantExit: 1,
+			wantSays: "the action generator drove the app in none of them",
+		},
+		{
+			name:     "the dead-run flag waives the dead-run refusal",
+			spec:     "spec.ts",
+			flags:    []string{"--allow-no-generator-actions"},
+			wantExit: 0,
+		},
+		{
+			// The campaign case: no flags are passed there, and a run that
+			// recorded a violation has produced the verdict the refusal exists
+			// to demand.
+			name:     "a recorded violation carries the run without any flag",
+			spec:     "violated-spec.ts",
+			wantExit: 0,
+		},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			arguments := append(runFlags(t, testCase.spec, pageURL), testCase.flags...)
+			exitCode, output := runBinary(t, binary, arguments)
+
+			if exitCode != testCase.wantExit {
+				t.Fatalf("exit code: got %d, want %d\n%s", exitCode, testCase.wantExit, output)
+			}
+			if testCase.wantSays != "" && !strings.Contains(output, testCase.wantSays) {
+				t.Errorf("the run never said %q:\n%s", testCase.wantSays, output)
+			}
+			if testCase.wantExit != 0 && !strings.Contains(output, "--allow-no-generator-actions") {
+				t.Errorf("the refusal never names the flag that waives it:\n%s", output)
+			}
+		})
+	}
+}
+
+// TestBrowserCampaignRecordsAViolationFoundWithoutTheGenerator follows the
+// whole chain a campaign is read through. A campaign passes no
+// --exit-on-violation, so a refusal here writes exit_code 1 and the analysis
+// drops the run as missing data: the detection the campaign exists to count
+// disappears into the excluded pile.
+func TestBrowserCampaignRecordsAViolationFoundWithoutTheGenerator(t *testing.T) {
+	sanderling := buildBinary(t, "./cmd/sanderling")
+	campaign := buildBinary(t, "./cmd/internal-tools/campaign")
+	pageURL := servePage(t)
+	campaignDirectory := filepath.Join(t.TempDir(), "campaign")
+
+	exitCode, output := runBinary(t, campaign, []string{
+		"--spec", fixtureSpec(t, "violated-spec.ts"),
+		"--bundle-id", pageURL,
+		"--platform", "web",
+		"--arm", "dead-run",
+		"--generator", "seeded",
+		"--max-steps", "3",
+		"--duration", "60s",
+		"--seeds", "1",
+		"--sanderling", sanderling,
+		"--output", campaignDirectory,
+	})
+	if exitCode != 0 {
+		t.Fatalf("campaign exit code: got %d, want 0\n%s", exitCode, output)
+	}
+
+	body, err := os.ReadFile(filepath.Join(campaignDirectory, "runs.jsonl"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var record struct {
+		ExitCode           int      `json:"exit_code"`
+		Actions            int      `json:"actions"`
+		ViolatedProperties []string `json:"violated_properties"`
+	}
+	if err := json.Unmarshal(body, &record); err != nil {
+		t.Fatalf("runs.jsonl: %v\n%s", err, body)
+	}
+
+	if record.ExitCode != 0 {
+		t.Errorf("exit_code: got %d, want 0. The analysis excludes a non-zero record as "+
+			"missing data, so this violation would never reach the survival analysis", record.ExitCode)
+	}
+	if record.Actions != 0 {
+		t.Fatalf("actions: got %d, want 0: the fixture offers the generator nothing to drive", record.Actions)
+	}
+	if len(record.ViolatedProperties) != 1 || record.ViolatedProperties[0] != "bannerIsEmpty" {
+		t.Errorf("violated_properties: got %v, want [bannerIsEmpty]", record.ViolatedProperties)
+	}
+}
+
+func runFlags(t *testing.T, spec, pageURL string) []string {
+	t.Helper()
+	return []string{
+		"test",
+		"--platform", "web",
+		"--spec", fixtureSpec(t, spec),
+		"--bundle-id", pageURL,
+		"--duration", "60s",
+		"--max-steps", "3",
+		"--seed", "1",
+		"--output", t.TempDir(),
+	}
+}
+
+func fixtureSpec(t *testing.T, name string) string {
+	t.Helper()
+	return filepath.Join(testdataDir(t), deadRunFixture, name)
+}
+
+func servePage(t *testing.T) string {
+	t.Helper()
+	server := httptest.NewServer(http.FileServer(http.Dir(testdataDir(t))))
+	t.Cleanup(server.Close)
+	return server.URL + "/" + deadRunFixture + "/"
+}
+
+func buildBinary(t *testing.T, packagePath string) string {
+	t.Helper()
+	binary := filepath.Join(t.TempDir(), filepath.Base(packagePath))
+	build := exec.Command("go", "build", "-o", binary, packagePath)
+	build.Dir = repoRoot(t)
+	if output, err := build.CombinedOutput(); err != nil {
+		t.Fatalf("build %s: %v\n%s", packagePath, err, output)
+	}
+	return binary
+}
+
+func runBinary(t *testing.T, binary string, arguments []string) (int, string) {
+	t.Helper()
+	command := exec.Command(binary, arguments...)
+	command.Dir = repoRoot(t)
+	output, err := command.CombinedOutput()
+	if command.ProcessState == nil {
+		t.Fatalf("%s never ran: %v", binary, err)
+	}
+	return command.ProcessState.ExitCode(), string(output)
+}

test/browser/testdata/nothing-to-tap/index.html

@@ -0,0 +1,10 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>nothing-to-tap</title>
+  </head>
+  <body>
+    <p id="banner">nothing here answers a tap</p>
+  </body>
+</html>

test/browser/testdata/nothing-to-tap/spec.ts

@@ -0,0 +1,9 @@
+import { always, extract, taps } from "@sanderling/spec";
+
+const banner = extract((s) => s.ax.find({ id: "banner" })?.text ?? "").named("banner");
+
+const bannerStays = always(() => banner.current === "nothing here answers a tap");
+
+export const properties = { bannerStays };
+
+export const actionsRoot = taps;

test/browser/testdata/nothing-to-tap/violated-spec.ts

@@ -0,0 +1,9 @@
+import { always, extract, taps } from "@sanderling/spec";
+
+const banner = extract((s) => s.ax.find({ id: "banner" })?.text ?? "").named("banner");
+
+const bannerIsEmpty = always(() => banner.current === "");
+
+export const properties = { bannerIsEmpty };
+
+export const actionsRoot = taps;