diff --git a/.github/scripts/folio-run.sh b/.github/scripts/folio-run.sh new file mode 100755 index 0000000..576f21f --- /dev/null +++ b/.github/scripts/folio-run.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Runs examples/folio/sanderling/spec.ts against one platform and checks the +# exit code the job expects. Kept out of the workflow YAML so it can be run by +# hand, which is how it was calibrated: +# +# SEED=3 MAX_STEPS=240 .github/scripts/folio-run.sh android +# +# android and ios expect exit 2: folio's double-submit bug is still there, and a +# run that no longer finds it is a regression in the fuzzer, not a pass. web +# expects a clean run that reached the transaction screen; it cannot observe the +# submit bug (see docs/development/ci.md). +set -uo pipefail + +platform="${1:?usage: folio-run.sh android|ios|web}" +seed="${SEED:-1}" +max_steps="${MAX_STEPS:-240}" +duration="${DURATION:-20m}" +sanderling="${SANDERLING:-./bin/sanderling}" +output="runs/folio-$platform" +spec="examples/folio/sanderling/spec.ts" +summary="${GITHUB_STEP_SUMMARY:-/dev/null}" + +folio_args=(--bundle-id app.folio) +case "$platform" in + android) + folio_args+=(--android-app-path + examples/folio/app/androidApp/build/outputs/apk/debug/androidApp-debug.apk) + ;; + ios) + folio_args+=(--platform ios + --ios-app-path examples/folio/app/iosApp/build/Build/Products/Debug-iphonesimulator/iosApp.app + --ios-device "${IOS_DEVICE:-iPhone 16 Pro}") + ;; + web) + dist="examples/folio/app/webApp/build/dist/wasmJs/developmentExecutable" + port="${PORT:-8791}" + # The stock static servers do not set COOP/COEP, and without cross-origin + # isolation the app's sqlite worker never starts, so folio loads to a blank + # canvas and every step observes an empty accessibility tree. + python3 - "$dist" "$port" <<'PY' & +import functools, http.server, sys + +class Isolated(http.server.SimpleHTTPRequestHandler): + def end_headers(self): + self.send_header("Cross-Origin-Opener-Policy", "same-origin") + self.send_header("Cross-Origin-Embedder-Policy", "require-corp") + self.send_header("Cross-Origin-Resource-Policy", "cross-origin") + super().end_headers() + + def log_message(self, *args): + pass + +directory, port = sys.argv[1], int(sys.argv[2]) +handler = functools.partial(Isolated, directory=directory) +http.server.HTTPServer(("127.0.0.1", port), handler).serve_forever() +PY + server_pid=$! + trap 'kill "$server_pid" 2>/dev/null' EXIT + for _ in $(seq 1 30); do + curl -sf "http://127.0.0.1:$port/index.html" >/dev/null && break + sleep 1 + done + folio_args=(--platform web --bundle-id "http://127.0.0.1:$port/index.html") + ;; + *) + echo "unknown platform: $platform" >&2 + exit 64 + ;; +esac + +"$sanderling" test \ + --spec "$spec" \ + "${folio_args[@]}" \ + --duration "$duration" \ + --max-steps "$max_steps" \ + --seed "$seed" \ + --exit-on-violation \ + --output "$output" +code=$? + +run_dir="$(ls -d "$output"/*/ 2>/dev/null | tail -1)" +steps=0 +[ -n "$run_dir" ] && steps=$(wc -l < "$run_dir/trace.jsonl" | tr -d ' ') +violated=$(grep -ho '"violations":\[[^]]*\]' "$run_dir/trace.jsonl" 2>/dev/null | head -1) + +{ + echo "### folio on $platform" + echo + echo "- seed \`$seed\`, budget $max_steps steps / $duration" + echo "- $steps steps recorded, exit $code" + [ -n "$violated" ] && echo "- $violated" +} >> "$summary" + +if [ "$platform" = "web" ]; then + # The web leg is a health gate: the same spec has to log in and drive the app + # as far as the transaction screen. Reaching it is the evidence; the submit + # property cannot fire here. + if [ "$code" -ne 0 ]; then + echo "folio/web: expected a clean run, got exit $code" >&2 + exit 1 + fi + if ! grep -q '"AddTransactionScreen"' "$run_dir/trace.jsonl"; then + echo "folio/web: the run never reached AddTransactionScreen, so it never got past login" >&2 + exit 1 + fi + echo "folio/web: clean run over $steps steps, reached the transaction screen" + exit 0 +fi + +case "$code" in + 2) echo "folio/$platform: found the submit bug in $steps steps"; exit 0 ;; + 0) echo "folio/$platform: the run finished clean; the double-submit bug was NOT found in $steps steps (seed $seed)" >&2; exit 1 ;; + *) echo "folio/$platform: the harness failed with exit $code" >&2; exit "$code" ;; +esac diff --git a/.github/workflows/folio.yml b/.github/workflows/folio.yml new file mode 100644 index 0000000..94f1dcf --- /dev/null +++ b/.github/workflows/folio.yml @@ -0,0 +1,244 @@ +name: folio + +# One spec, three platforms. Dispatch-only: each job boots a device or a +# browser, builds the folio app for that platform, and runs +# examples/folio/sanderling/spec.ts against it. +# +# android and ios are expect-the-bug jobs: folio double-submits a transaction on +# a double tap, so the run is supposed to end with exit 2. Exit 0 means the +# fuzzer stopped finding a bug that is still there; exit 1 means the harness +# broke. The two are worth telling apart, which is why --exit-on-violation exits +# 2 and not 1. +# +# web is a health gate instead: the same spec drives the wasmJs build through +# login and into the transaction flow, but it cannot observe the double submit. +# The property keys off state.lastAction, which the web runtime does not report, +# and off the action's selector, which the web picker does not carry (it emits +# coordinates). Both are fixable, neither is a small fix; see +# docs/development/ci.md. + +on: + workflow_dispatch: + inputs: + platforms: + description: which legs to run + type: choice + options: [all, android, ios, web] + default: all + seed: + description: seed override (0 = each job's calibrated seed) + default: "0" + duration: + description: wall-clock budget per run + default: 20m + max-steps: + description: step budget override (0 = each job's calibrated budget) + default: "0" + +jobs: + android: + if: ${{ inputs.platforms == 'all' || inputs.platforms == 'android' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + + - name: Set up Android SDK + uses: android-actions/setup-android@v3 + + - name: Set up bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.3.13" + + - name: Cache Gradle + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: folio-gradle-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + folio-gradle-${{ runner.os }}- + + # Without this the emulator falls back to software rendering and every + # step costs several seconds. + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - name: Build the folio APK + run: ./gradlew :app:androidApp:assembleDebug + working-directory: examples/folio + + - name: Build sanderling + run: make sanderling-android + + - name: Fuzz folio on an emulator + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 34 + target: google_apis + arch: x86_64 + profile: pixel_6 + emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim + disable-animations: true + script: .github/scripts/folio-run.sh android + env: + SEED: ${{ inputs.seed != '0' && inputs.seed || '3' }} + MAX_STEPS: ${{ inputs.max-steps != '0' && inputs.max-steps || '240' }} + DURATION: ${{ inputs.duration }} + + - name: Upload the run + if: always() + uses: actions/upload-artifact@v4 + with: + name: folio-android + path: runs/ + retention-days: 14 + + ios: + if: ${{ inputs.platforms == 'all' || inputs.platforms == 'ios' }} + runs-on: macos-15 + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Set up bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.3.13" + + - name: Install idb-companion and xcodegen + run: brew install idb-companion xcodegen + + # Both asset tarballs are built by the prepare scripts, and the runner + # bundle is an xcodebuild of companion/Sources. Keyed on the scripts and + # the versions the Makefile embeds, so a later run reuses them. + - name: Cache the companion and runner bundles + uses: actions/cache@v4 + with: + path: | + internal/driver/ioscompanion/companionassets/assets + internal/driver/ioscompanion/runnerassets/assets + key: ios-assets-${{ runner.os }}-${{ hashFiles('internal/driver/ioscompanion/companionassets/prepare.sh', 'companion/prepare.sh', 'companion/project.yml', 'companion/Sources/**') }} + + - name: Build sanderling + run: make sanderling-ios + + - name: Boot a simulator + run: | + xcrun simctl boot "$IOS_DEVICE" || true + xcrun simctl bootstatus "$IOS_DEVICE" -b + env: + IOS_DEVICE: iPhone 16 Pro + + - name: Build and install folio + run: just ios + working-directory: examples/folio + env: + IOS_DEVICE: iPhone 16 Pro + + - name: Fuzz folio on the simulator + run: .github/scripts/folio-run.sh ios + env: + SEED: ${{ inputs.seed != '0' && inputs.seed || '1' }} + MAX_STEPS: ${{ inputs.max-steps != '0' && inputs.max-steps || '240' }} + DURATION: ${{ inputs.duration }} + IOS_DEVICE: iPhone 16 Pro + + - name: Upload the run + if: always() + uses: actions/upload-artifact@v4 + with: + name: folio-ios + path: runs/ + retention-days: 14 + + web: + if: ${{ inputs.platforms == 'all' || inputs.platforms == 'web' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + + - name: Set up bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.3.13" + + - name: Cache Gradle + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: folio-gradle-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + folio-gradle-${{ runner.os }}- + + - name: Set up Chrome + uses: browser-actions/setup-chrome@v1 + with: + chrome-version: stable + + - name: Allow Chrome under unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Verify headless Chrome starts + run: | + chrome --version + chrome --headless --no-sandbox --disable-gpu --disable-dev-shm-usage \ + --dump-dom 'data:text/html,ok' + + - name: Build the folio wasmJs app + run: ./gradlew :app:webApp:wasmJsBrowserDevelopmentExecutableDistribution + working-directory: examples/folio + + - name: Build sanderling + run: make sanderling-web + + - name: Fuzz folio in the browser + run: .github/scripts/folio-run.sh web + env: + SEED: ${{ inputs.seed != '0' && inputs.seed || '1' }} + MAX_STEPS: ${{ inputs.max-steps != '0' && inputs.max-steps || '200' }} + DURATION: ${{ inputs.duration }} + + - name: Upload the run + if: always() + uses: actions/upload-artifact@v4 + with: + name: folio-web + path: runs/ + retention-days: 14 diff --git a/.github/workflows/replay-ui.yml b/.github/workflows/replay-ui.yml new file mode 100644 index 0000000..1efd5f4 --- /dev/null +++ b/.github/workflows/replay-ui.yml @@ -0,0 +1,123 @@ +name: replay-ui + +# Sanderling fuzzing sanderling's own replay UI. Dispatch-only: it takes minutes +# and it is a demo of the product loop, not a merge gate. +# +# The shape is: produce a real trace, serve it with `sanderling replay`, then run +# a spec against that UI. Any violation fails the job, because every property in +# replay-ui/sanderling/spec.ts is a cross-panel agreement that holds for any +# trace. + +on: + workflow_dispatch: + inputs: + seed: + description: seed for the dogfood run + default: "3" + max-steps: + description: step budget for the dogfood run + default: "80" + +jobs: + dogfood: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Set up bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.3.13" + + # Pinned stable plus the AppArmor sysctl: the same setup ci.yml's browser + # job needs to get headless Chrome up on ubuntu-latest. + - name: Set up Chrome + uses: browser-actions/setup-chrome@v1 + with: + chrome-version: stable + + - name: Allow Chrome under unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Verify headless Chrome starts + run: | + chrome --version + chrome --headless --no-sandbox --disable-gpu --disable-dev-shm-usage \ + --dump-dom 'data:text/html,ok' + + # The UI the spec drives is the one embedded in this binary, so the build + # has to come after any change to replay-ui/src. + - name: Build sanderling + run: make sanderling-web + + # A trace with a violation and uncaught exceptions in it, so the UI has + # something to render in every panel the spec looks at. No + # --exit-on-violation here: the run is the fixture, and stopping it at the + # first violation would leave a four-step trace to fuzz. + - name: Record a fixture trace + run: | + python3 -m http.server 8792 --bind 127.0.0.1 \ + --directory test/browser/testdata/throwing & + for _ in $(seq 1 30); do + curl -sf http://127.0.0.1:8792/ >/dev/null && break + sleep 1 + done + ./bin/sanderling test \ + --platform web \ + --spec test/browser/testdata/throwing/spec.ts \ + --bundle-id http://127.0.0.1:8792/ \ + --duration 5m --max-steps 25 --seed 7 \ + --output runs/fixture + + - name: Serve the trace with sanderling replay + run: | + # Flags before the positional argument: Go's flag package stops + # parsing at the first non-flag word. + ./bin/sanderling replay --port 8793 --no-open runs/fixture & + for _ in $(seq 1 30); do + curl -sf http://127.0.0.1:8793/api/runs >/dev/null && break + sleep 1 + done + run_id="$(ls runs/fixture | head -1)" + echo "RUN_URL=http://127.0.0.1:8793/runs/$run_id/steps/1" >> "$GITHUB_ENV" + curl -sf "http://127.0.0.1:8793/runs/$run_id/steps/1" >/dev/null + + - name: Fuzz the replay UI + run: | + ./bin/sanderling test \ + --platform web \ + --spec replay-ui/sanderling/spec.ts \ + --bundle-id "$RUN_URL" \ + --duration 10m \ + --max-steps "${{ inputs.max-steps }}" \ + --seed "${{ inputs.seed }}" \ + --exit-on-violation \ + --output runs/dogfood + + - name: Summarise + if: always() + run: | + { + echo "### replay-ui dogfood" + echo + echo "- seed \`${{ inputs.seed }}\`, budget ${{ inputs.max-steps }} steps" + for dir in runs/dogfood/*/; do + steps=$(wc -l < "$dir/trace.jsonl" | tr -d ' ') + violations=$(grep -c '"violations":\[' "$dir/trace.jsonl" || true) + echo "- $steps steps recorded, $violations step(s) with violations" + done + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload runs + if: always() + uses: actions/upload-artifact@v4 + with: + name: replay-ui-runs + path: runs/ + retention-days: 14