diff --git a/.env.local.example b/.env.local.example index 879acdd..f2b8f72 100644 --- a/.env.local.example +++ b/.env.local.example @@ -5,7 +5,7 @@ # / `release-android-local` / `release-npm-dry` Make targets don't need any # of these. They're snapshot/local-only. # -# In CI, these are provided via GitHub Actions secrets (see .github/workflows/release.yml). +# In CI, these are provided via GitHub Actions secrets (see .github/workflows/ci.yml). # npm automation token (bypasses 2FA). # Create at npmjs.com → Access Tokens → Generate New Token → Automation. diff --git a/.github/actions/folio-simulator/action.yml b/.github/actions/folio-simulator/action.yml deleted file mode 100644 index 36dcd1f..0000000 --- a/.github/actions/folio-simulator/action.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: folio on a simulator -description: Boot an iOS simulator, install folio on it, and leave the app stopped. - -inputs: - device: - description: simulator device name - default: iPhone 16 Pro - -runs: - using: composite - steps: - - name: Boot a simulator - shell: bash - run: | - xcrun simctl boot "$IOS_DEVICE" || true - xcrun simctl bootstatus "$IOS_DEVICE" -b - env: - IOS_DEVICE: ${{ inputs.device }} - - - name: Build and install folio - shell: bash - working-directory: examples/folio - run: just ios - env: - IOS_DEVICE: ${{ inputs.device }} - - # `just ios` leaves the app running, and the run's first act is to clear - # its state. Stopping it here means the run always opens the same way. - - name: Stop the app before the run - shell: bash - run: xcrun simctl terminate booted app.folio || true diff --git a/.github/actions/replay-ui-fixture/action.yml b/.github/actions/replay-ui-fixture/action.yml deleted file mode 100644 index 11d5be3..0000000 --- a/.github/actions/replay-ui-fixture/action.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: replay ui fixture -description: Record a trace with sanderling, then serve it with sanderling replay. - -outputs: - url: - description: the step page of the served run, for a spec to drive - value: ${{ steps.serve.outputs.url }} - -runs: - using: composite - steps: - # A trace with a violation and uncaught exceptions in it, so the UI has - # something to render in every panel the spec looks at. No - # --exit-on-violation here: the run is the fixture, and stopping it at the - # first violation would leave a four-step trace to fuzz. - - name: Record a fixture trace - shell: bash - run: | - python3 -m http.server 8792 --bind 127.0.0.1 \ - --directory test/browser/testdata/throwing & - ready="" - for _ in $(seq 1 30); do - curl -sf http://127.0.0.1:8792/ >/dev/null && { ready=1; break; } - sleep 1 - done - if [ -z "$ready" ]; then - echo "the fixture http server never answered on 127.0.0.1:8792" >&2 - exit 1 - fi - ./bin/sanderling test \ - --platform web \ - --spec test/browser/testdata/throwing/spec.ts \ - --bundle-id http://127.0.0.1:8792/ \ - --duration 5m --max-steps 25 --seed 7 \ - --output runs/fixture - - - name: Serve the trace with sanderling replay - id: serve - shell: bash - run: | - # Flags before the positional argument: Go's flag package stops - # parsing at the first non-flag word. - ./bin/sanderling replay --port 8793 --no-open runs/fixture & - ready="" - for _ in $(seq 1 30); do - curl -sf http://127.0.0.1:8793/api/runs >/dev/null && { ready=1; break; } - sleep 1 - done - if [ -z "$ready" ]; then - echo "sanderling replay never served /api/runs on 127.0.0.1:8793" >&2 - exit 1 - fi - run_id="$(ls runs/fixture | head -1)" - echo "url=http://127.0.0.1:8793/runs/$run_id/steps/1" >> "$GITHUB_OUTPUT" - curl -sf "http://127.0.0.1:8793/runs/$run_id/steps/1" >/dev/null diff --git a/.github/scripts/folio-run.sh b/.github/scripts/folio-run.sh index 35f88a3..a746506 100755 --- a/.github/scripts/folio-run.sh +++ b/.github/scripts/folio-run.sh @@ -292,7 +292,7 @@ case "$code" in ;; 0) echo "folio/$platform: the run finished clean; the double-submit bug was NOT found in $steps steps (seed $seed)" >&2 - echo "folio/$platform: the spec ran without throwing, so this is the fuzzer no longer reaching the bug, not a broken spec" >&2 + echo "folio/$platform: the spec ran without throwing, so this is the run no longer reaching the bug, not a broken spec" >&2 exit 1 ;; *) echo "folio/$platform: the harness failed with exit $code" >&2; exit "$code" ;; diff --git a/.github/scripts/workflow-refs.sh b/.github/scripts/workflow-refs.sh index 8871dcd..546d223 100755 --- a/.github/scripts/workflow-refs.sh +++ b/.github/scripts/workflow-refs.sh @@ -1,17 +1,17 @@ #!/usr/bin/env bash -# Checks that everything the workflows name actually exists: composite actions, -# make targets, and the scripts a run: block invokes. +# Checks that everything the workflow names actually exists: composite actions, +# make targets, and the scripts a run: block invokes. Then checks that no run: +# block interpolates a `${{ }}`. # # This is the class actionlint does not cover. `uses: ./.github/actions/typo` -# lints clean and fails only when the job runs, and these workflows are -# dispatch-only or push-triggered, so that first run is after merge. +# lints clean and fails only when the job runs, and the folio jobs and the +# release job never run on a pull request, so that first run is after merge. set -euo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" ROOT="$root" python3 - <<'PY' import glob -import json import os import re import sys @@ -75,14 +75,6 @@ for path in sorted(glob.glob(os.path.join(root, ".github/workflows/*.yml"))): commands = re.sub(r"#[^\n]*", "", body) for name in re.findall(r"\bmake\s+([a-z][a-z0-9-]*)\b", commands): wanted.add(name) - # `make "sanderling-$SANDERLING"` is resolved from the matrix that feeds it - if "sanderling-$SANDERLING" in body: - table = re.search(r"examples='(\[.*?\])'", body, re.S) - if table is None: - sys.exit("workflow-refs: %s builds a make target from $SANDERLING but its " - "examples table could not be read" % rel(path)) - for entry in json.loads(table.group(1)): - wanted.add("sanderling-%s" % entry["sanderling"]) for name in sorted(wanted): report(name in targets, "make %s" % name) @@ -97,6 +89,51 @@ for name in sorted(scripts): if os.path.isfile(full): report(os.access(full, os.X_OK), "%s is executable" % name) +# --- expressions in a run: block --------------------------------------------- +# A `${{ }}` is substituted into the script text before bash reads the line, so +# an expression carrying text someone else wrote runs as a command. Values reach +# a run: block through env instead. actionlint flags only the contexts it knows +# are attacker-controlled, and a matrix value or a dispatch input is not on that +# list. +print("\nrun: blocks free of ${{ }}:") + + +def run_blocks(text): + lines = text.split("\n") + i = 0 + while i < len(lines): + head = re.match(r"^(\s*(?:-\s+)?)run:(.*)$", lines[i]) + if head is None: + i += 1 + continue + column, rest = len(head.group(1)), head.group(2).strip() + start, body = i + 1, [] + if rest in ("|", "|-", "|+", ">", ">-", ">+", ""): + i += 1 + while i < len(lines) and (not lines[i].strip() + or len(lines[i]) - len(lines[i].lstrip()) > column): + body.append(lines[i]) + i += 1 + else: + body.append(rest) + i += 1 + yield start, "\n".join(body) + + +blocks = 0 +for path in workflow_files(): + hits = [] + for line, body in run_blocks(open(path).read()): + blocks += 1 + hits += ["line %d: %s" % (line, hit) for hit in re.findall(r"\$\{\{.*?\}\}", body, re.S)] + report(not hits, rel(path), " (%s)" % ("; ".join(hits) if hits else "clean")) + +# Same reason as the local action count above: a scanner that reads no run: +# block at all would pass every file it never looked at. +if blocks == 0: + sys.exit("workflow-refs: found no run: block at all, so this check is not " + "reading the workflows it claims to read") + print("\n%d references checked" % checked) if problems: sys.exit("workflow-refs: unresolved: %s" % ", ".join(problems)) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1585226..d8d5fc6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,19 +1,20 @@ name: ci on: - # Runs on PRs (opened / synchronize / reopened, which are the defaults) and - # manual dispatch only. We deliberately don't run on direct pushes to master: - # master is PR-merge-only, and PR validation already covers the merge - # commit via the `synchronize` event on the PR branch. pull_request: + push: + branches: [master] + tags: ["v*"] workflow_dispatch: permissions: contents: read +# A superseded pull request run is waste. A run that publishes is not, so only +# a pull request cancels. concurrency: group: ci-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: test: @@ -125,10 +126,8 @@ jobs: - name: Drive web fixtures through headless Chrome run: make test-browser - # Four workflows and four composite actions, and the ones that fuzz the - # examples are dispatch-only, which GitHub refuses to dispatch until they are - # on the default branch. Their first real run is therefore after merge, so a - # bad expression or a missing action would land before anything caught it. + # The folio jobs and the release job never run on a pull request, so a bad + # expression or a missing action in them would land before anything caught it. workflows: runs-on: ubuntu-latest steps: @@ -147,3 +146,445 @@ jobs: # the job runs. - name: Check that the workflow references resolve run: .github/scripts/workflow-refs.sh + + folio-android: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + SEED: "9" + MAX_STEPS: "200" + DURATION: 20m + steps: + - uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + + - name: Set up bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: "1.3.13" + + - name: Build the folio app + uses: ./.github/actions/folio-app + with: + platform: android + + - name: Build sanderling + run: make sanderling-android + + - name: Run the spec on an emulator + uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2.38.0 + with: + api-level: 34 + target: google_apis + arch: x86_64 + emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim + disable-animations: true + script: .github/scripts/folio-run.sh android + + - name: Upload the run + if: always() + uses: actions/upload-artifact@v7 + with: + name: folio-android + path: runs/ + retention-days: 14 + + folio-ios: + if: github.event_name != 'pull_request' + runs-on: macos-15 + timeout-minutes: 90 + env: + SEED: "7" + MAX_STEPS: "240" + DURATION: 20m + IOS_DEVICE: iPhone 16 Pro + steps: + - uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + + - name: Set up bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: "1.3.13" + + - name: Build the folio app + uses: ./.github/actions/folio-app + with: + platform: ios + + - name: Build sanderling + run: make sanderling-ios + + - name: Boot a simulator + run: | + xcrun simctl boot "$IOS_DEVICE" || true + xcrun simctl bootstatus "$IOS_DEVICE" -b + + - name: Build and install folio + working-directory: examples/folio + run: just ios + + # `just ios` leaves the app running, and the run's first act is to clear + # its state. Stopping it here means the run always opens the same way. + - name: Stop the app before the run + run: xcrun simctl terminate booted app.folio || true + + - name: Run the spec + run: .github/scripts/folio-run.sh ios + + - name: Upload the run + if: always() + uses: actions/upload-artifact@v7 + with: + name: folio-ios + path: runs/ + retention-days: 14 + + folio-web: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + SEED: "3" + MAX_STEPS: "240" + DURATION: 20m + steps: + - uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + + - name: Set up bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: "1.3.13" + + - name: Set up headless Chrome + uses: ./.github/actions/headless-chrome + + - name: Build the folio app + uses: ./.github/actions/folio-app + with: + platform: web + + - name: Build sanderling + run: make sanderling-web + + - name: Run the spec + run: .github/scripts/folio-run.sh web + + - name: Upload the run + if: always() + uses: actions/upload-artifact@v7 + with: + name: folio-web + path: runs/ + retention-days: 14 + + replay-ui: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + SEED: "3" + MAX_STEPS: "80" + DURATION: 10m + steps: + - uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + + - name: Set up bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: "1.3.13" + + - name: Set up headless Chrome + uses: ./.github/actions/headless-chrome + + # The UI the spec drives is the one embedded in this binary, so the build + # has to come after any change to replay-ui/src. + - name: Build sanderling + run: make sanderling-web + + # A trace with a violation and uncaught exceptions in it, so the UI has + # something to render in every panel the spec looks at. No + # --exit-on-violation here: the run is the fixture, and stopping it at the + # first violation would leave a four-step trace to run against. + - name: Record a fixture trace + run: | + python3 -m http.server 8792 --bind 127.0.0.1 \ + --directory test/browser/testdata/throwing & + ready="" + for _ in $(seq 1 30); do + curl -sf http://127.0.0.1:8792/ >/dev/null && { ready=1; break; } + sleep 1 + done + if [ -z "$ready" ]; then + echo "the fixture http server never answered on 127.0.0.1:8792" >&2 + exit 1 + fi + ./bin/sanderling test \ + --platform web \ + --spec test/browser/testdata/throwing/spec.ts \ + --bundle-id http://127.0.0.1:8792/ \ + --duration 5m --max-steps 25 --seed 7 \ + --output runs/fixture + + - name: Serve the trace with sanderling replay + id: fixture + run: | + # Flags before the positional argument: Go's flag package stops + # parsing at the first non-flag word. + ./bin/sanderling replay --port 8793 --no-open runs/fixture & + ready="" + for _ in $(seq 1 30); do + curl -sf http://127.0.0.1:8793/api/runs >/dev/null && { ready=1; break; } + sleep 1 + done + if [ -z "$ready" ]; then + echo "sanderling replay never served /api/runs on 127.0.0.1:8793" >&2 + exit 1 + fi + run_id="$(basename "$(find runs/fixture -mindepth 1 -maxdepth 1 | head -1)")" + echo "url=http://127.0.0.1:8793/runs/$run_id/steps/1" >> "$GITHUB_OUTPUT" + curl -sf "http://127.0.0.1:8793/runs/$run_id/steps/1" >/dev/null + + # The url goes through env rather than into the script text: a `${{ }}` is + # substituted before bash ever sees the line. + - name: Run the spec + run: | + ./bin/sanderling test \ + --platform web \ + --spec replay-ui/sanderling/spec.ts \ + --bundle-id "$RUN_URL" \ + --duration "$DURATION" \ + --max-steps "$MAX_STEPS" \ + --seed "$SEED" \ + --exit-on-violation \ + --output runs/replay-ui + env: + RUN_URL: ${{ steps.fixture.outputs.url }} + + # Exit 0 above means no property returned false. It does not mean any + # property was ever evaluated against real content: they all decline to + # judge when the elements they read are absent, so a run that never + # rendered the step page is green and worthless. This step is what tells + # the two apart, and it fails the job when nothing was judged. folio's + # jobs make the same call inside folio-run.sh, where the exit code it is + # judging is in scope. + - name: Classify the run + if: always() + run: .github/scripts/replay-ui-summary.sh runs/replay-ui + + - name: Upload the run + if: always() + uses: actions/upload-artifact@v7 + with: + name: replay-ui-runs + path: runs/ + retention-days: 14 + + # On a tag this publishes @sanderling/spec at the tag's version and the CLI to + # GitHub Releases. On master it publishes @sanderling/spec only, and only when + # pkg/spec/package.json carries a version npm does not have yet. + release: + if: github.ref == 'refs/heads/master' || github.ref_type == 'tag' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + # A refname is attacker-controlled text and git permits backtick, `$`, + # `(`, `;`, `&` and `|` in it, so it goes through env: a `${{ }}` is + # substituted before bash ever sees the line. Every step below reads these + # outputs rather than the refname, and nothing reaches a shell before it + # has matched the pattern. The pattern is anchored and admits no newline, + # which is what stops the value below forging a second $GITHUB_OUTPUT key. + - name: Validate the tag + id: tag + if: github.ref_type == 'tag' + run: | + pattern='^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z]+(\.[0-9A-Za-z]+)*)?$' + if [[ ! "$TAG" =~ $pattern ]]; then + echo "release: refusing to publish from '$TAG'" >&2 + echo "release: a release tag is vMAJOR.MINOR.PATCH with an optional -prerelease, e.g. v0.1.0 or v0.0.1-rc1" >&2 + exit 1 + fi + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "version=${TAG#v}" >> "$GITHUB_OUTPUT" + env: + TAG: ${{ github.ref_name }} + + - uses: actions/checkout@v7 + with: + # Empty on master, where the commit that triggered the run is the one + # to publish and master may have moved on since. + ref: ${{ steps.tag.outputs.tag || github.sha }} + # GoReleaser reads the tag history for its changelog. + fetch-depth: 0 + # `npm ci` below runs dependency lifecycle scripts, and no step in + # this job needs the git credential afterwards. + persist-credentials: false + + - name: Set up Node 22 + uses: actions/setup-node@v7 + with: + node-version: "22" + registry-url: "https://registry.npmjs.org" + cache: npm + cache-dependency-path: pkg/spec/package-lock.json + + - name: Install dependencies + working-directory: pkg/spec + run: npm ci + + - name: Stamp version + if: github.ref_type == 'tag' + working-directory: pkg/spec + run: npm version "$VERSION" --no-git-tag-version --allow-same-version + env: + VERSION: ${{ steps.tag.outputs.version }} + + # npm refuses a version it already has, so most merges to master have + # nothing to publish and must not be red for it. The registry is asked + # rather than the diff of package.json: that answer is still right after a + # revert, after a merge that publishes nothing, and after a publish that + # failed halfway. Only stdout decides, because `npm view` on a version + # that does not exist is empty on some npm releases and an error on + # others, and an unreachable registry must end in a publish that fails + # loudly rather than a skip that looks like success. + - name: Ask npm whether this version is already published + id: version + working-directory: pkg/spec + run: | + version="$(node -p 'require("./package.json").version')" + # Held to the pattern the tag is held to above, and for the same + # reason: a value with a newline in it would forge a second key. + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z]+(\.[0-9A-Za-z]+)*)?$ ]]; then + echo "release: pkg/spec/package.json carries '$version', which is not a version this publishes" >&2 + exit 1 + fi + published="$(npm view "@sanderling/spec@$version" version 2>/dev/null || true)" + if [ -n "$published" ]; then + echo "npm already has @sanderling/spec@$version, nothing to publish" + echo "publish=false" >> "$GITHUB_OUTPUT" + else + echo "publishing @sanderling/spec@$version" + echo "publish=true" >> "$GITHUB_OUTPUT" + fi + echo "version=$version" >> "$GITHUB_OUTPUT" + + - name: Publish @sanderling/spec to npm + if: steps.version.outputs.publish == 'true' + working-directory: pkg/spec + # npm tag pre-releases (e.g. 0.1.0-rc1) as "next" so npm install @sanderling/spec + # keeps resolving the latest stable. + run: | + if [[ "$VERSION" == *-* ]]; then + npm publish --access public --tag next + else + npm publish --access public + fi + # The publish credential is scoped to the one step that publishes rather + # than to the job, so no other step runs with it in reach. + env: + VERSION: ${{ steps.version.outputs.version }} + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + - name: Set up Go + if: github.ref_type == 'tag' + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + + - name: Set up JDK 17 + if: github.ref_type == 'tag' + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "17" + + - name: Set up Android SDK + if: github.ref_type == 'tag' + uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1 + + - name: Cache Gradle + if: github.ref_type == 'tag' + uses: actions/cache@v6 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: gradle-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + gradle-${{ runner.os }}- + + - name: Build sidecar JAR + if: github.ref_type == 'tag' + run: make sidecar + + - name: Publish the sanderling CLI to GitHub Releases + if: github.ref_type == 'tag' + uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 + with: + version: "~> v2" + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # The docs used to build only when docs/ or the Makefile changed. A path + # filter here would have to sit on the whole workflow, so the site is rebuilt + # on every merge instead: it is pandoc over a few pages, and a deploy of bytes + # that did not change is a no-op. + docs: + if: github.ref == 'refs/heads/master' || github.ref_type == 'tag' + runs-on: ubuntu-latest + permissions: + contents: read + pages: write + id-token: write + # Pages takes one deployment at a time. + concurrency: + group: pages + cancel-in-progress: false + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/checkout@v7 + + - name: Install pandoc + run: sudo apt-get update && sudo apt-get install -y pandoc + + - name: Build site + run: make docs + + # No include-hidden-files: v4 stopped uploading dot-files by default, and + # build/site has none. It is pandoc output plus a copy of docs/_assets, + # which holds three ordinary files. _assets is underscore-prefixed, not + # hidden, and deploy-pages serves the artifact without running Jekyll, so + # it needs no .nojekyll either. + - uses: actions/upload-pages-artifact@v5 + with: + path: build/site + + - uses: actions/deploy-pages@v5 + id: deployment diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index f42f60c..0000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: docs - -on: - push: - branches: [master] - paths: - - "docs/**" - - "Makefile" - - ".github/workflows/docs.yml" - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: pages - cancel-in-progress: false - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - - name: Install pandoc - run: sudo apt-get update && sudo apt-get install -y pandoc - - - name: Build site - run: make docs - - # No include-hidden-files: v4 stopped uploading dot-files by default, and - # build/site has none. It is pandoc output plus a copy of docs/_assets, - # which holds three ordinary files. _assets is underscore-prefixed, not - # hidden, and deploy-pages serves the artifact without running Jekyll, so - # it needs no .nojekyll either. - - uses: actions/upload-pages-artifact@v5 - with: - path: build/site - - deploy: - needs: build - runs-on: ubuntu-latest - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - steps: - - uses: actions/deploy-pages@v5 - id: deployment diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml deleted file mode 100644 index 08780bc..0000000 --- a/.github/workflows/examples.yml +++ /dev/null @@ -1,193 +0,0 @@ -name: examples - -# Every example sanderling ships, fuzzed the same way: build sanderling for a -# platform, bring the target up, run a spec against it, classify the trace it -# wrote, upload the run. Only the bring-up differs, and that lives in the -# per-target actions under .github/actions/. -# -# Dispatch-only: these take tens of minutes and they demonstrate the product -# loop, they do not gate a merge. -# -# folio on ios and in the browser expect the bug: folio double-submits a -# transaction on a double tap, so the run is supposed to end with exit 2. Exit 0 -# means the fuzzer stopped finding a bug that is still there; exit 1 means the -# harness broke. The two are worth telling apart, which is why -# --exit-on-violation exits 2 and not 1. -# -# folio on android is a health gate. It convicts in four runs out of five, which -# is real evidence but not a gate: the fifth would report a regression it had -# not found. Its budget is set so the conviction it usually gets is a bonus. -# -# the replay ui leg fuzzes sanderling's own replay UI, and any violation fails -# it. Its properties are cross-panel agreements that hold for any trace, so none -# of them needs recalibrating when the fixture changes. - -on: - workflow_dispatch: - inputs: - targets: - description: which examples to fuzz - type: choice - options: [all, folio, android, ios, web, replay-ui] - default: all - seed: - description: seed override (0 = each target's calibrated seed) - default: "0" - max-steps: - description: step budget override (0 = each target's calibrated budget) - default: "0" - duration: - description: wall-clock budget override (empty = each target's calibrated budget) - default: "" - -permissions: - contents: read - -jobs: - plan: - runs-on: ubuntu-latest - permissions: {} - outputs: - examples: ${{ steps.pick.outputs.examples }} - steps: - # The matrix is built here rather than written out under strategy.matrix - # because a job-level `if:` cannot read the matrix context, so a static - # matrix has no way to leave a leg out. jq -c keeps the value on one line, - # which is what makes the $GITHUB_OUTPUT write below safe. - - name: Pick the examples to fuzz - id: pick - run: | - examples='[ - {"target":"android","name":"folio on android","app":"folio", - "runs-on":"ubuntu-latest","timeout":90,"sanderling":"android", - "seed":"9","max-steps":"200","duration":"20m","artifact":"folio-android"}, - {"target":"ios","name":"folio on ios","app":"folio", - "runs-on":"macos-15","timeout":90,"sanderling":"ios", - "seed":"7","max-steps":"240","duration":"20m","artifact":"folio-ios"}, - {"target":"web","name":"folio in the browser","app":"folio", - "runs-on":"ubuntu-latest","timeout":60,"sanderling":"web","chrome":true, - "seed":"3","max-steps":"240","duration":"20m","artifact":"folio-web"}, - {"target":"replay-ui","name":"the replay ui","app":"replay-ui", - "runs-on":"ubuntu-latest","timeout":45,"sanderling":"web","chrome":true, - "seed":"3","max-steps":"80","duration":"10m","artifact":"replay-ui-runs"} - ]' - picked="$(jq -c --arg want "$TARGETS" \ - 'map(select($want == "all" or .target == $want or .app == $want))' \ - <<<"$examples")" - if [ "$picked" = "[]" ]; then - echo "examples: '$TARGETS' selects no example, so this dispatch would run nothing" >&2 - exit 1 - fi - echo "examples=$picked" >> "$GITHUB_OUTPUT" - env: - TARGETS: ${{ inputs.targets }} - - fuzz: - needs: plan - name: fuzz ${{ matrix.name }} - runs-on: ${{ matrix.runs-on }} - timeout-minutes: ${{ matrix.timeout }} - strategy: - # Each leg is its own evidence. One target failing must not cancel the - # others, which is how these ran as separate jobs. - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.examples) }} - env: - SEED: ${{ inputs.seed != '0' && inputs.seed || matrix.seed }} - MAX_STEPS: ${{ inputs.max-steps != '0' && inputs.max-steps || matrix.max-steps }} - DURATION: ${{ inputs.duration != '' && inputs.duration || matrix.duration }} - IOS_DEVICE: iPhone 16 Pro - steps: - - uses: actions/checkout@v7 - - - name: Set up Go - uses: actions/setup-go@v7 - with: - go-version-file: go.mod - cache: true - - - name: Set up bun - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 - with: - bun-version: "1.3.13" - - - name: Set up headless Chrome - if: matrix.chrome - uses: ./.github/actions/headless-chrome - - - name: Build the folio app - if: matrix.app == 'folio' - uses: ./.github/actions/folio-app - with: - platform: ${{ matrix.target }} - - # The UI the replay-ui spec drives is the one embedded in this binary, so - # the build has to come after any change to replay-ui/src. - - name: Build sanderling - run: make "sanderling-$SANDERLING" - env: - SANDERLING: ${{ matrix.sanderling }} - - - name: Put folio on the simulator - if: matrix.target == 'ios' - uses: ./.github/actions/folio-simulator - - - name: Serve a trace to fuzz - id: fixture - if: matrix.target == 'replay-ui' - uses: ./.github/actions/replay-ui-fixture - - - name: Fuzz folio on an emulator - if: matrix.target == 'android' - uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2.38.0 - with: - api-level: 34 - target: google_apis - arch: x86_64 - emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim - disable-animations: true - script: .github/scripts/folio-run.sh android - - - name: Fuzz folio - if: matrix.app == 'folio' && matrix.target != 'android' - run: .github/scripts/folio-run.sh "$TARGET" - env: - TARGET: ${{ matrix.target }} - - # Inputs go through env rather than into the script text: a `${{ }}` is - # substituted before bash ever sees the line, so a seed of `$(id)` would - # run as a command. - - name: Fuzz the replay UI - if: matrix.target == 'replay-ui' - run: | - ./bin/sanderling test \ - --platform web \ - --spec replay-ui/sanderling/spec.ts \ - --bundle-id "$RUN_URL" \ - --duration "$DURATION" \ - --max-steps "$MAX_STEPS" \ - --seed "$SEED" \ - --exit-on-violation \ - --output runs/replay-ui - env: - RUN_URL: ${{ steps.fixture.outputs.url }} - - # Exit 0 above means no property returned false. It does not mean any - # property was ever evaluated against real content: they all decline to - # judge when the elements they read are absent, so a run that never - # rendered the step page is green and worthless. This step is what tells - # the two apart, and it fails the job when nothing was judged. folio's - # legs make the same call inside folio-run.sh, where the exit code it is - # judging is in scope. - - name: Classify the replay UI run - if: ${{ always() && matrix.target == 'replay-ui' }} - run: .github/scripts/replay-ui-summary.sh runs/replay-ui - - - name: Upload the run - if: always() - uses: actions/upload-artifact@v7 - with: - name: ${{ matrix.artifact }} - path: runs/ - retention-days: 14 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 0dc1982..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,144 +0,0 @@ -name: release - -on: - push: - tags: - - "v*" - workflow_dispatch: - inputs: - tag: - description: "Tag to release (e.g. v0.0.1-rc1). Must already exist." - required: true - type: string - -permissions: - contents: read - -concurrency: - group: release-${{ github.ref }} - cancel-in-progress: false - -jobs: - resolve-tag: - name: Resolve and validate the tag - runs-on: ubuntu-latest - permissions: {} - outputs: - tag: ${{ steps.tag.outputs.tag }} - version: ${{ steps.tag.outputs.version }} - steps: - # A refname is attacker-controlled text and git permits backtick, `$`, - # `(`, `;`, `&` and `|` in it, so it goes through env: a `${{ }}` is - # substituted before bash ever sees the line. Every later job reads these - # outputs rather than the refname, and nothing reaches a shell before it - # has matched the pattern. The pattern is anchored and admits no newline, - # which is what stops the value below forging a second $GITHUB_OUTPUT key. - - name: Validate the tag - id: tag - run: | - pattern='^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z]+(\.[0-9A-Za-z]+)*)?$' - if [[ ! "$TAG" =~ $pattern ]]; then - echo "release: refusing to publish from '$TAG'" >&2 - echo "release: a release tag is vMAJOR.MINOR.PATCH with an optional -prerelease, e.g. v0.1.0 or v0.0.1-rc1" >&2 - exit 1 - fi - echo "tag=$TAG" >> "$GITHUB_OUTPUT" - echo "version=${TAG#v}" >> "$GITHUB_OUTPUT" - env: - TAG: ${{ inputs.tag || github.ref_name }} - - release-npm: - name: Publish @sanderling/spec to npm - needs: resolve-tag - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ needs.resolve-tag.outputs.tag }} - # `npm ci` below runs dependency lifecycle scripts, and no step in - # this job needs the git credential afterwards. - persist-credentials: false - - - name: Set up Node 22 - uses: actions/setup-node@v7 - with: - node-version: "22" - registry-url: "https://registry.npmjs.org" - cache: npm - cache-dependency-path: pkg/spec/package-lock.json - - - name: Install dependencies - working-directory: pkg/spec - run: npm ci - - - name: Stamp version - working-directory: pkg/spec - run: npm version "$VERSION" --no-git-tag-version --allow-same-version - env: - VERSION: ${{ needs.resolve-tag.outputs.version }} - - - name: Publish - working-directory: pkg/spec - # npm tag pre-releases (e.g. 0.1.0-rc1) as "next" so npm install @sanderling/spec - # keeps resolving the latest stable. - run: | - if [[ "$VERSION" == *-* ]]; then - npm publish --access public --tag next - else - npm publish --access public - fi - # The publish credential is scoped to the one step that publishes rather - # than to the job, so no other step runs with it in reach. - env: - VERSION: ${{ needs.resolve-tag.outputs.version }} - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - release-cli: - name: Publish sanderling CLI to GitHub Releases - needs: resolve-tag - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ needs.resolve-tag.outputs.tag }} - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@v7 - with: - go-version-file: go.mod - cache: true - - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - distribution: temurin - java-version: "17" - - - name: Set up Android SDK - uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1 - - - name: Cache Gradle - uses: actions/cache@v6 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: gradle-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - gradle-${{ runner.os }}- - - - name: Build sidecar JAR - run: make sidecar - - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 - with: - version: "~> v2" - args: release --clean - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}