ci: add dispatch workflows for folio and the replay ui

.github/scripts/folio-run.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# Runs examples/folio/sanderling/spec.ts against one platform and checks the
+# exit code the job expects. Kept out of the workflow YAML so it can be run by
+# hand, which is how it was calibrated:
+#
+#   SEED=3 MAX_STEPS=240 .github/scripts/folio-run.sh android
+#
+# android and ios expect exit 2: folio's double-submit bug is still there, and a
+# run that no longer finds it is a regression in the fuzzer, not a pass. web
+# expects a clean run that reached the transaction screen; it cannot observe the
+# submit bug (see docs/development/ci.md).
+set -uo pipefail
+
+platform="${1:?usage: folio-run.sh android|ios|web}"
+seed="${SEED:-1}"
+max_steps="${MAX_STEPS:-240}"
+duration="${DURATION:-20m}"
+sanderling="${SANDERLING:-./bin/sanderling}"
+output="runs/folio-$platform"
+spec="examples/folio/sanderling/spec.ts"
+summary="${GITHUB_STEP_SUMMARY:-/dev/null}"
+
+folio_args=(--bundle-id app.folio)
+case "$platform" in
+  android)
+    folio_args+=(--android-app-path
+      examples/folio/app/androidApp/build/outputs/apk/debug/androidApp-debug.apk)
+    ;;
+  ios)
+    folio_args+=(--platform ios
+      --ios-app-path examples/folio/app/iosApp/build/Build/Products/Debug-iphonesimulator/iosApp.app
+      --ios-device "${IOS_DEVICE:-iPhone 16 Pro}")
+    ;;
+  web)
+    dist="examples/folio/app/webApp/build/dist/wasmJs/developmentExecutable"
+    port="${PORT:-8791}"
+    # The stock static servers do not set COOP/COEP, and without cross-origin
+    # isolation the app's sqlite worker never starts, so folio loads to a blank
+    # canvas and every step observes an empty accessibility tree.
+    python3 - "$dist" "$port" <<'PY' &
+import functools, http.server, sys
+
+class Isolated(http.server.SimpleHTTPRequestHandler):
+    def end_headers(self):
+        self.send_header("Cross-Origin-Opener-Policy", "same-origin")
+        self.send_header("Cross-Origin-Embedder-Policy", "require-corp")
+        self.send_header("Cross-Origin-Resource-Policy", "cross-origin")
+        super().end_headers()
+
+    def log_message(self, *args):
+        pass
+
+directory, port = sys.argv[1], int(sys.argv[2])
+handler = functools.partial(Isolated, directory=directory)
+http.server.HTTPServer(("127.0.0.1", port), handler).serve_forever()
+PY
+    server_pid=$!
+    trap 'kill "$server_pid" 2>/dev/null' EXIT
+    for _ in $(seq 1 30); do
+      curl -sf "http://127.0.0.1:$port/index.html" >/dev/null && break
+      sleep 1
+    done
+    folio_args=(--platform web --bundle-id "http://127.0.0.1:$port/index.html")
+    ;;
+  *)
+    echo "unknown platform: $platform" >&2
+    exit 64
+    ;;
+esac
+
+"$sanderling" test \
+  --spec "$spec" \
+  "${folio_args[@]}" \
+  --duration "$duration" \
+  --max-steps "$max_steps" \
+  --seed "$seed" \
+  --exit-on-violation \
+  --output "$output"
+code=$?
+
+run_dir="$(ls -d "$output"/*/ 2>/dev/null | tail -1)"
+steps=0
+[ -n "$run_dir" ] && steps=$(wc -l < "$run_dir/trace.jsonl" | tr -d ' ')
+violated=$(grep -ho '"violations":\[[^]]*\]' "$run_dir/trace.jsonl" 2>/dev/null | head -1)
+
+{
+  echo "### folio on $platform"
+  echo
+  echo "- seed \`$seed\`, budget $max_steps steps / $duration"
+  echo "- $steps steps recorded, exit $code"
+  [ -n "$violated" ] && echo "- $violated"
+} >> "$summary"
+
+if [ "$platform" = "web" ]; then
+  # The web leg is a health gate: the same spec has to log in and drive the app
+  # as far as the transaction screen. Reaching it is the evidence; the submit
+  # property cannot fire here.
+  if [ "$code" -ne 0 ]; then
+    echo "folio/web: expected a clean run, got exit $code" >&2
+    exit 1
+  fi
+  if ! grep -q '"AddTransactionScreen"' "$run_dir/trace.jsonl"; then
+    echo "folio/web: the run never reached AddTransactionScreen, so it never got past login" >&2
+    exit 1
+  fi
+  echo "folio/web: clean run over $steps steps, reached the transaction screen"
+  exit 0
+fi
+
+case "$code" in
+  2) echo "folio/$platform: found the submit bug in $steps steps"; exit 0 ;;
+  0) echo "folio/$platform: the run finished clean; the double-submit bug was NOT found in $steps steps (seed $seed)" >&2; exit 1 ;;
+  *) echo "folio/$platform: the harness failed with exit $code" >&2; exit "$code" ;;
+esac

.github/workflows/folio.yml

@@ -0,0 +1,244 @@
+name: folio
+
+# One spec, three platforms. Dispatch-only: each job boots a device or a
+# browser, builds the folio app for that platform, and runs
+# examples/folio/sanderling/spec.ts against it.
+#
+# android and ios are expect-the-bug jobs: folio double-submits a transaction on
+# a double tap, so the run is supposed to end with exit 2. Exit 0 means the
+# fuzzer stopped finding a bug that is still there; exit 1 means the harness
+# broke. The two are worth telling apart, which is why --exit-on-violation exits
+# 2 and not 1.
+#
+# web is a health gate instead: the same spec drives the wasmJs build through
+# login and into the transaction flow, but it cannot observe the double submit.
+# The property keys off state.lastAction, which the web runtime does not report,
+# and off the action's selector, which the web picker does not carry (it emits
+# coordinates). Both are fixable, neither is a small fix; see
+# docs/development/ci.md.
+
+on:
+  workflow_dispatch:
+    inputs:
+      platforms:
+        description: which legs to run
+        type: choice
+        options: [all, android, ios, web]
+        default: all
+      seed:
+        description: seed override (0 = each job's calibrated seed)
+        default: "0"
+      duration:
+        description: wall-clock budget per run
+        default: 20m
+      max-steps:
+        description: step budget override (0 = each job's calibrated budget)
+        default: "0"
+
+jobs:
+  android:
+    if: ${{ inputs.platforms == 'all' || inputs.platforms == 'android' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: "17"
+
+      - name: Set up Android SDK
+        uses: android-actions/setup-android@v3
+
+      - name: Set up bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: "1.3.13"
+
+      - name: Cache Gradle
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: folio-gradle-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            folio-gradle-${{ runner.os }}-
+
+      # Without this the emulator falls back to software rendering and every
+      # step costs several seconds.
+      - name: Enable KVM
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
+            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+
+      - name: Build the folio APK
+        run: ./gradlew :app:androidApp:assembleDebug
+        working-directory: examples/folio
+
+      - name: Build sanderling
+        run: make sanderling-android
+
+      - name: Fuzz folio on an emulator
+        uses: reactivecircus/android-emulator-runner@v2
+        with:
+          api-level: 34
+          target: google_apis
+          arch: x86_64
+          profile: pixel_6
+          emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim
+          disable-animations: true
+          script: .github/scripts/folio-run.sh android
+        env:
+          SEED: ${{ inputs.seed != '0' && inputs.seed || '3' }}
+          MAX_STEPS: ${{ inputs.max-steps != '0' && inputs.max-steps || '240' }}
+          DURATION: ${{ inputs.duration }}
+
+      - name: Upload the run
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: folio-android
+          path: runs/
+          retention-days: 14
+
+  ios:
+    if: ${{ inputs.platforms == 'all' || inputs.platforms == 'ios' }}
+    runs-on: macos-15
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Set up bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: "1.3.13"
+
+      - name: Install idb-companion and xcodegen
+        run: brew install idb-companion xcodegen
+
+      # Both asset tarballs are built by the prepare scripts, and the runner
+      # bundle is an xcodebuild of companion/Sources. Keyed on the scripts and
+      # the versions the Makefile embeds, so a later run reuses them.
+      - name: Cache the companion and runner bundles
+        uses: actions/cache@v4
+        with:
+          path: |
+            internal/driver/ioscompanion/companionassets/assets
+            internal/driver/ioscompanion/runnerassets/assets
+          key: ios-assets-${{ runner.os }}-${{ hashFiles('internal/driver/ioscompanion/companionassets/prepare.sh', 'companion/prepare.sh', 'companion/project.yml', 'companion/Sources/**') }}
+
+      - name: Build sanderling
+        run: make sanderling-ios
+
+      - name: Boot a simulator
+        run: |
+          xcrun simctl boot "$IOS_DEVICE" || true
+          xcrun simctl bootstatus "$IOS_DEVICE" -b
+        env:
+          IOS_DEVICE: iPhone 16 Pro
+
+      - name: Build and install folio
+        run: just ios
+        working-directory: examples/folio
+        env:
+          IOS_DEVICE: iPhone 16 Pro
+
+      - name: Fuzz folio on the simulator
+        run: .github/scripts/folio-run.sh ios
+        env:
+          SEED: ${{ inputs.seed != '0' && inputs.seed || '1' }}
+          MAX_STEPS: ${{ inputs.max-steps != '0' && inputs.max-steps || '240' }}
+          DURATION: ${{ inputs.duration }}
+          IOS_DEVICE: iPhone 16 Pro
+
+      - name: Upload the run
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: folio-ios
+          path: runs/
+          retention-days: 14
+
+  web:
+    if: ${{ inputs.platforms == 'all' || inputs.platforms == 'web' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: "17"
+
+      - name: Set up bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: "1.3.13"
+
+      - name: Cache Gradle
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: folio-gradle-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            folio-gradle-${{ runner.os }}-
+
+      - name: Set up Chrome
+        uses: browser-actions/setup-chrome@v1
+        with:
+          chrome-version: stable
+
+      - name: Allow Chrome under unprivileged user namespaces
+        run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+
+      - name: Verify headless Chrome starts
+        run: |
+          chrome --version
+          chrome --headless --no-sandbox --disable-gpu --disable-dev-shm-usage \
+            --dump-dom 'data:text/html,<title>ok</title>'
+
+      - name: Build the folio wasmJs app
+        run: ./gradlew :app:webApp:wasmJsBrowserDevelopmentExecutableDistribution
+        working-directory: examples/folio
+
+      - name: Build sanderling
+        run: make sanderling-web
+
+      - name: Fuzz folio in the browser
+        run: .github/scripts/folio-run.sh web
+        env:
+          SEED: ${{ inputs.seed != '0' && inputs.seed || '1' }}
+          MAX_STEPS: ${{ inputs.max-steps != '0' && inputs.max-steps || '200' }}
+          DURATION: ${{ inputs.duration }}
+
+      - name: Upload the run
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: folio-web
+          path: runs/
+          retention-days: 14

.github/workflows/replay-ui.yml

@@ -0,0 +1,123 @@
+name: replay-ui
+
+# Sanderling fuzzing sanderling's own replay UI. Dispatch-only: it takes minutes
+# and it is a demo of the product loop, not a merge gate.
+#
+# The shape is: produce a real trace, serve it with `sanderling replay`, then run
+# a spec against that UI. Any violation fails the job, because every property in
+# replay-ui/sanderling/spec.ts is a cross-panel agreement that holds for any
+# trace.
+
+on:
+  workflow_dispatch:
+    inputs:
+      seed:
+        description: seed for the dogfood run
+        default: "3"
+      max-steps:
+        description: step budget for the dogfood run
+        default: "80"
+
+jobs:
+  dogfood:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Set up bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: "1.3.13"
+
+      # Pinned stable plus the AppArmor sysctl: the same setup ci.yml's browser
+      # job needs to get headless Chrome up on ubuntu-latest.
+      - name: Set up Chrome
+        uses: browser-actions/setup-chrome@v1
+        with:
+          chrome-version: stable
+
+      - name: Allow Chrome under unprivileged user namespaces
+        run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+
+      - name: Verify headless Chrome starts
+        run: |
+          chrome --version
+          chrome --headless --no-sandbox --disable-gpu --disable-dev-shm-usage \
+            --dump-dom 'data:text/html,<title>ok</title>'
+
+      # The UI the spec drives is the one embedded in this binary, so the build
+      # has to come after any change to replay-ui/src.
+      - name: Build sanderling
+        run: make sanderling-web
+
+      # A trace with a violation and uncaught exceptions in it, so the UI has
+      # something to render in every panel the spec looks at. No
+      # --exit-on-violation here: the run is the fixture, and stopping it at the
+      # first violation would leave a four-step trace to fuzz.
+      - name: Record a fixture trace
+        run: |
+          python3 -m http.server 8792 --bind 127.0.0.1 \
+            --directory test/browser/testdata/throwing &
+          for _ in $(seq 1 30); do
+            curl -sf http://127.0.0.1:8792/ >/dev/null && break
+            sleep 1
+          done
+          ./bin/sanderling test \
+            --platform web \
+            --spec test/browser/testdata/throwing/spec.ts \
+            --bundle-id http://127.0.0.1:8792/ \
+            --duration 5m --max-steps 25 --seed 7 \
+            --output runs/fixture
+
+      - name: Serve the trace with sanderling replay
+        run: |
+          # Flags before the positional argument: Go's flag package stops
+          # parsing at the first non-flag word.
+          ./bin/sanderling replay --port 8793 --no-open runs/fixture &
+          for _ in $(seq 1 30); do
+            curl -sf http://127.0.0.1:8793/api/runs >/dev/null && break
+            sleep 1
+          done
+          run_id="$(ls runs/fixture | head -1)"
+          echo "RUN_URL=http://127.0.0.1:8793/runs/$run_id/steps/1" >> "$GITHUB_ENV"
+          curl -sf "http://127.0.0.1:8793/runs/$run_id/steps/1" >/dev/null
+
+      - name: Fuzz the replay UI
+        run: |
+          ./bin/sanderling test \
+            --platform web \
+            --spec replay-ui/sanderling/spec.ts \
+            --bundle-id "$RUN_URL" \
+            --duration 10m \
+            --max-steps "${{ inputs.max-steps }}" \
+            --seed "${{ inputs.seed }}" \
+            --exit-on-violation \
+            --output runs/dogfood
+
+      - name: Summarise
+        if: always()
+        run: |
+          {
+            echo "### replay-ui dogfood"
+            echo
+            echo "- seed \`${{ inputs.seed }}\`, budget ${{ inputs.max-steps }} steps"
+            for dir in runs/dogfood/*/; do
+              steps=$(wc -l < "$dir/trace.jsonl" | tr -d ' ')
+              violations=$(grep -c '"violations":\[' "$dir/trace.jsonl" || true)
+              echo "- $steps steps recorded, $violations step(s) with violations"
+            done
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload runs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: replay-ui-runs
+          path: runs/
+          retention-days: 14