pj/sanderling
1
0 Fork 0
mirror of https://github.com/priyanshujain/sanderling.git synced 2026-10-02 11:07:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'one-workflow' into correctness-and-spec-skills

Browse Source
This commit is contained in:
pj committed 2026-08-16 03:33:05 +05:30
2 parents 1a16b4b424 e326216461
commit ccc4941df3
9 files changed
+502 -497
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.env.local.example
+1 -1
View File
@@ -5,7 +5,7 @@
# / `release-android-local` / `release-npm-dry` Make targets don't need any
# of these. They're snapshot/local-only.
#
# In CI, these are provided via GitHub Actions secrets (see .github/workflows/release.yml).
# In CI, these are provided via GitHub Actions secrets (see .github/workflows/ci.yml).
# npm automation token (bypasses 2FA).
# Create at npmjs.com → Access Tokens → Generate New Token → Automation.
.github/actions/folio-simulator/action.yml
-31
View File
@@ -1,31 +0,0 @@
name: folio on a simulator
description: Boot an iOS simulator, install folio on it, and leave the app stopped.
inputs:
device:
description: simulator device name
default: iPhone 16 Pro
runs:
using: composite
steps:
- name: Boot a simulator
shell: bash
run: |
xcrun simctl boot "$IOS_DEVICE" || true
xcrun simctl bootstatus "$IOS_DEVICE" -b
env:
IOS_DEVICE: ${{ inputs.device }}
- name: Build and install folio
shell: bash
working-directory: examples/folio
run: just ios
env:
IOS_DEVICE: ${{ inputs.device }}
# `just ios` leaves the app running, and the run's first act is to clear
# its state. Stopping it here means the run always opens the same way.
- name: Stop the app before the run
shell: bash
run: xcrun simctl terminate booted app.folio || true
.github/actions/replay-ui-fixture/action.yml
-55
View File
@@ -1,55 +0,0 @@
name: replay ui fixture
description: Record a trace with sanderling, then serve it with sanderling replay.
outputs:
url:
description: the step page of the served run, for a spec to drive
value: ${{ steps.serve.outputs.url }}
runs:
using: composite
steps:
# A trace with a violation and uncaught exceptions in it, so the UI has
# something to render in every panel the spec looks at. No
# --exit-on-violation here: the run is the fixture, and stopping it at the
# first violation would leave a four-step trace to fuzz.
- name: Record a fixture trace
shell: bash
run: |
python3 -m http.server 8792 --bind 127.0.0.1 \
--directory test/browser/testdata/throwing &
ready=""
for _ in $(seq 1 30); do
curl -sf http://127.0.0.1:8792/ >/dev/null && { ready=1; break; }
sleep 1
done
if [ -z "$ready" ]; then
echo "the fixture http server never answered on 127.0.0.1:8792" >&2
exit 1
fi
./bin/sanderling test \
--platform web \
--spec test/browser/testdata/throwing/spec.ts \
--bundle-id http://127.0.0.1:8792/ \
--duration 5m --max-steps 25 --seed 7 \
--output runs/fixture
- name: Serve the trace with sanderling replay
id: serve
shell: bash
run: |
# Flags before the positional argument: Go's flag package stops
# parsing at the first non-flag word.
./bin/sanderling replay --port 8793 --no-open runs/fixture &
ready=""
for _ in $(seq 1 30); do
curl -sf http://127.0.0.1:8793/api/runs >/dev/null && { ready=1; break; }
sleep 1
done
if [ -z "$ready" ]; then
echo "sanderling replay never served /api/runs on 127.0.0.1:8793" >&2
exit 1
fi
run_id="$(ls runs/fixture | head -1)"
echo "url=http://127.0.0.1:8793/runs/$run_id/steps/1" >> "$GITHUB_OUTPUT"
curl -sf "http://127.0.0.1:8793/runs/$run_id/steps/1" >/dev/null
.github/scripts/folio-run.sh
+1 -1
View File
@@ -292,7 +292,7 @@ case "$code" in
;;
0)
echo "folio/$platform: the run finished clean; the double-submit bug was NOT found in $steps steps (seed $seed)" >&2
echo "folio/$platform: the spec ran without throwing, so this is the fuzzer no longer reaching the bug, not a broken spec" >&2
echo "folio/$platform: the spec ran without throwing, so this is the run no longer reaching the bug, not a broken spec" >&2
exit 1
;;
*) echo "folio/$platform: the harness failed with exit $code" >&2; exit "$code" ;;
.github/scripts/workflow-refs.sh
+50 -13
View File
@@ -1,17 +1,17 @@
#!/usr/bin/env bash
# Checks that everything the workflows name actually exists: composite actions,
# make targets, and the scripts a run: block invokes.
# Checks that everything the workflow names actually exists: composite actions,
# make targets, and the scripts a run: block invokes. Then checks that no run:
# block interpolates a `${{ }}`.
#
# This is the class actionlint does not cover. `uses: ./.github/actions/typo`
# lints clean and fails only when the job runs, and these workflows are
# dispatch-only or push-triggered, so that first run is after merge.
# lints clean and fails only when the job runs, and the folio jobs and the
# release job never run on a pull request, so that first run is after merge.
set -euo pipefail
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
ROOT="$root" python3 - <<'PY'
import glob
import json
import os
import re
import sys
@@ -75,14 +75,6 @@ for path in sorted(glob.glob(os.path.join(root, ".github/workflows/*.yml"))):
commands = re.sub(r"#[^\n]*", "", body)
for name in re.findall(r"\bmake\s+([a-z][a-z0-9-]*)\b", commands):
wanted.add(name)
# `make "sanderling-$SANDERLING"` is resolved from the matrix that feeds it
if "sanderling-$SANDERLING" in body:
table = re.search(r"examples='(\[.*?\])'", body, re.S)
if table is None:
sys.exit("workflow-refs: %s builds a make target from $SANDERLING but its "
"examples table could not be read" % rel(path))
for entry in json.loads(table.group(1)):
wanted.add("sanderling-%s" % entry["sanderling"])
for name in sorted(wanted):
report(name in targets, "make %s" % name)
@@ -97,6 +89,51 @@ for name in sorted(scripts):
if os.path.isfile(full):
report(os.access(full, os.X_OK), "%s is executable" % name)
# --- expressions in a run: block ---------------------------------------------
# A `${{ }}` is substituted into the script text before bash reads the line, so
# an expression carrying text someone else wrote runs as a command. Values reach
# a run: block through env instead. actionlint flags only the contexts it knows
# are attacker-controlled, and a matrix value or a dispatch input is not on that
# list.
print("\nrun: blocks free of ${{ }}:")
def run_blocks(text):
lines = text.split("\n")
i = 0
while i < len(lines):
head = re.match(r"^(\s*(?:-\s+)?)run:(.*)$", lines[i])
if head is None:
i += 1
continue
column, rest = len(head.group(1)), head.group(2).strip()
start, body = i + 1, []
if rest in ("|", "|-", "|+", ">", ">-", ">+", ""):
i += 1
while i < len(lines) and (not lines[i].strip()
or len(lines[i]) - len(lines[i].lstrip()) > column):
body.append(lines[i])
i += 1
else:
body.append(rest)
i += 1
yield start, "\n".join(body)
blocks = 0
for path in workflow_files():
hits = []
for line, body in run_blocks(open(path).read()):
blocks += 1
hits += ["line %d: %s" % (line, hit) for hit in re.findall(r"\$\{\{.*?\}\}", body, re.S)]
report(not hits, rel(path), " (%s)" % ("; ".join(hits) if hits else "clean"))
# Same reason as the local action count above: a scanner that reads no run:
# block at all would pass every file it never looked at.
if blocks == 0:
sys.exit("workflow-refs: found no run: block at all, so this check is not "
"reading the workflows it claims to read")
print("\n%d references checked" % checked)
if problems:
sys.exit("workflow-refs: unresolved: %s" % ", ".join(problems))
.github/workflows/ci.yml
+450 -9
View File
@@ -1,19 +1,20 @@
name: ci
on:
# Runs on PRs (opened / synchronize / reopened, which are the defaults) and
# manual dispatch only. We deliberately don't run on direct pushes to master:
# master is PR-merge-only, and PR validation already covers the merge
# commit via the `synchronize` event on the PR branch.
pull_request:
push:
branches: [master]
tags: ["v*"]
workflow_dispatch:
permissions:
contents: read
# A superseded pull request run is waste. A run that publishes is not, so only
# a pull request cancels.
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
test:
@@ -125,10 +126,8 @@ jobs:
- name: Drive web fixtures through headless Chrome
run: make test-browser
# Four workflows and four composite actions, and the ones that fuzz the
# examples are dispatch-only, which GitHub refuses to dispatch until they are
# on the default branch. Their first real run is therefore after merge, so a
# bad expression or a missing action would land before anything caught it.
# The folio jobs and the release job never run on a pull request, so a bad
# expression or a missing action in them would land before anything caught it.
workflows:
runs-on: ubuntu-latest
steps:
@@ -147,3 +146,445 @@ jobs:
# the job runs.
- name: Check that the workflow references resolve
run: .github/scripts/workflow-refs.sh
folio-android:
if: github.event_name != 'pull_request'
runs-on: ubuntu-latest
timeout-minutes: 90
env:
SEED: "9"
MAX_STEPS: "200"
DURATION: 20m
steps:
- uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: go.mod
cache: true
- name: Set up bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: "1.3.13"
- name: Build the folio app
uses: ./.github/actions/folio-app
with:
platform: android
- name: Build sanderling
run: make sanderling-android
- name: Run the spec on an emulator
uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2.38.0
with:
api-level: 34
target: google_apis
arch: x86_64
emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim
disable-animations: true
script: .github/scripts/folio-run.sh android
- name: Upload the run
if: always()
uses: actions/upload-artifact@v7
with:
name: folio-android
path: runs/
retention-days: 14
folio-ios:
if: github.event_name != 'pull_request'
runs-on: macos-15
timeout-minutes: 90
env:
SEED: "7"
MAX_STEPS: "240"
DURATION: 20m
IOS_DEVICE: iPhone 16 Pro
steps:
- uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: go.mod
cache: true
- name: Set up bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: "1.3.13"
- name: Build the folio app
uses: ./.github/actions/folio-app
with:
platform: ios
- name: Build sanderling
run: make sanderling-ios
- name: Boot a simulator
run: |
xcrun simctl boot "$IOS_DEVICE" || true
xcrun simctl bootstatus "$IOS_DEVICE" -b
- name: Build and install folio
working-directory: examples/folio
run: just ios
# `just ios` leaves the app running, and the run's first act is to clear
# its state. Stopping it here means the run always opens the same way.
- name: Stop the app before the run
run: xcrun simctl terminate booted app.folio || true
- name: Run the spec
run: .github/scripts/folio-run.sh ios
- name: Upload the run
if: always()
uses: actions/upload-artifact@v7
with:
name: folio-ios
path: runs/
retention-days: 14
folio-web:
if: github.event_name != 'pull_request'
runs-on: ubuntu-latest
timeout-minutes: 60
env:
SEED: "3"
MAX_STEPS: "240"
DURATION: 20m
steps:
- uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: go.mod
cache: true
- name: Set up bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: "1.3.13"
- name: Set up headless Chrome
uses: ./.github/actions/headless-chrome
- name: Build the folio app
uses: ./.github/actions/folio-app
with:
platform: web
- name: Build sanderling
run: make sanderling-web
- name: Run the spec
run: .github/scripts/folio-run.sh web
- name: Upload the run
if: always()
uses: actions/upload-artifact@v7
with:
name: folio-web
path: runs/
retention-days: 14
replay-ui:
runs-on: ubuntu-latest
timeout-minutes: 45
env:
SEED: "3"
MAX_STEPS: "80"
DURATION: 10m
steps:
- uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: go.mod
cache: true
- name: Set up bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: "1.3.13"
- name: Set up headless Chrome
uses: ./.github/actions/headless-chrome
# The UI the spec drives is the one embedded in this binary, so the build
# has to come after any change to replay-ui/src.
- name: Build sanderling
run: make sanderling-web
# A trace with a violation and uncaught exceptions in it, so the UI has
# something to render in every panel the spec looks at. No
# --exit-on-violation here: the run is the fixture, and stopping it at the
# first violation would leave a four-step trace to run against.
- name: Record a fixture trace
run: |
python3 -m http.server 8792 --bind 127.0.0.1 \
--directory test/browser/testdata/throwing &
ready=""
for _ in $(seq 1 30); do
curl -sf http://127.0.0.1:8792/ >/dev/null && { ready=1; break; }
sleep 1
done
if [ -z "$ready" ]; then
echo "the fixture http server never answered on 127.0.0.1:8792" >&2
exit 1
fi
./bin/sanderling test \
--platform web \
--spec test/browser/testdata/throwing/spec.ts \
--bundle-id http://127.0.0.1:8792/ \
--duration 5m --max-steps 25 --seed 7 \
--output runs/fixture
- name: Serve the trace with sanderling replay
id: fixture
run: |
# Flags before the positional argument: Go's flag package stops
# parsing at the first non-flag word.
./bin/sanderling replay --port 8793 --no-open runs/fixture &
ready=""
for _ in $(seq 1 30); do
curl -sf http://127.0.0.1:8793/api/runs >/dev/null && { ready=1; break; }
sleep 1
done
if [ -z "$ready" ]; then
echo "sanderling replay never served /api/runs on 127.0.0.1:8793" >&2
exit 1
fi
run_id="$(basename "$(find runs/fixture -mindepth 1 -maxdepth 1 | head -1)")"
echo "url=http://127.0.0.1:8793/runs/$run_id/steps/1" >> "$GITHUB_OUTPUT"
curl -sf "http://127.0.0.1:8793/runs/$run_id/steps/1" >/dev/null
# The url goes through env rather than into the script text: a `${{ }}` is
# substituted before bash ever sees the line.
- name: Run the spec
run: |
./bin/sanderling test \
--platform web \
--spec replay-ui/sanderling/spec.ts \
--bundle-id "$RUN_URL" \
--duration "$DURATION" \
--max-steps "$MAX_STEPS" \
--seed "$SEED" \
--exit-on-violation \
--output runs/replay-ui
env:
RUN_URL: ${{ steps.fixture.outputs.url }}
# Exit 0 above means no property returned false. It does not mean any
# property was ever evaluated against real content: they all decline to
# judge when the elements they read are absent, so a run that never
# rendered the step page is green and worthless. This step is what tells
# the two apart, and it fails the job when nothing was judged. folio's
# jobs make the same call inside folio-run.sh, where the exit code it is
# judging is in scope.
- name: Classify the run
if: always()
run: .github/scripts/replay-ui-summary.sh runs/replay-ui
- name: Upload the run
if: always()
uses: actions/upload-artifact@v7
with:
name: replay-ui-runs
path: runs/
retention-days: 14
# On a tag this publishes @sanderling/spec at the tag's version and the CLI to
# GitHub Releases. On master it publishes @sanderling/spec only, and only when
# pkg/spec/package.json carries a version npm does not have yet.
release:
if: github.ref == 'refs/heads/master' || github.ref_type == 'tag'
runs-on: ubuntu-latest
permissions:
contents: write
steps:
# A refname is attacker-controlled text and git permits backtick, `$`,
# `(`, `;`, `&` and `|` in it, so it goes through env: a `${{ }}` is
# substituted before bash ever sees the line. Every step below reads these
# outputs rather than the refname, and nothing reaches a shell before it
# has matched the pattern. The pattern is anchored and admits no newline,
# which is what stops the value below forging a second $GITHUB_OUTPUT key.
- name: Validate the tag
id: tag
if: github.ref_type == 'tag'
run: |
pattern='^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z]+(\.[0-9A-Za-z]+)*)?$'
if [[ ! "$TAG" =~ $pattern ]]; then
echo "release: refusing to publish from '$TAG'" >&2
echo "release: a release tag is vMAJOR.MINOR.PATCH with an optional -prerelease, e.g. v0.1.0 or v0.0.1-rc1" >&2
exit 1
fi
echo "tag=$TAG" >> "$GITHUB_OUTPUT"
echo "version=${TAG#v}" >> "$GITHUB_OUTPUT"
env:
TAG: ${{ github.ref_name }}
- uses: actions/checkout@v7
with:
# Empty on master, where the commit that triggered the run is the one
# to publish and master may have moved on since.
ref: ${{ steps.tag.outputs.tag || github.sha }}
# GoReleaser reads the tag history for its changelog.
fetch-depth: 0
# `npm ci` below runs dependency lifecycle scripts, and no step in
# this job needs the git credential afterwards.
persist-credentials: false
- name: Set up Node 22
uses: actions/setup-node@v7
with:
node-version: "22"
registry-url: "https://registry.npmjs.org"
cache: npm
cache-dependency-path: pkg/spec/package-lock.json
- name: Install dependencies
working-directory: pkg/spec
run: npm ci
- name: Stamp version
if: github.ref_type == 'tag'
working-directory: pkg/spec
run: npm version "$VERSION" --no-git-tag-version --allow-same-version
env:
VERSION: ${{ steps.tag.outputs.version }}
# npm refuses a version it already has, so most merges to master have
# nothing to publish and must not be red for it. The registry is asked
# rather than the diff of package.json: that answer is still right after a
# revert, after a merge that publishes nothing, and after a publish that
# failed halfway. Only stdout decides, because `npm view` on a version
# that does not exist is empty on some npm releases and an error on
# others, and an unreachable registry must end in a publish that fails
# loudly rather than a skip that looks like success.
- name: Ask npm whether this version is already published
id: version
working-directory: pkg/spec
run: |
version="$(node -p 'require("./package.json").version')"
# Held to the pattern the tag is held to above, and for the same
# reason: a value with a newline in it would forge a second key.
if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z]+(\.[0-9A-Za-z]+)*)?$ ]]; then
echo "release: pkg/spec/package.json carries '$version', which is not a version this publishes" >&2
exit 1
fi
published="$(npm view "@sanderling/spec@$version" version 2>/dev/null || true)"
if [ -n "$published" ]; then
echo "npm already has @sanderling/spec@$version, nothing to publish"
echo "publish=false" >> "$GITHUB_OUTPUT"
else
echo "publishing @sanderling/spec@$version"
echo "publish=true" >> "$GITHUB_OUTPUT"
fi
echo "version=$version" >> "$GITHUB_OUTPUT"
- name: Publish @sanderling/spec to npm
if: steps.version.outputs.publish == 'true'
working-directory: pkg/spec
# npm tag pre-releases (e.g. 0.1.0-rc1) as "next" so npm install @sanderling/spec
# keeps resolving the latest stable.
run: |
if [[ "$VERSION" == *-* ]]; then
npm publish --access public --tag next
else
npm publish --access public
fi
# The publish credential is scoped to the one step that publishes rather
# than to the job, so no other step runs with it in reach.
env:
VERSION: ${{ steps.version.outputs.version }}
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
- name: Set up Go
if: github.ref_type == 'tag'
uses: actions/setup-go@v7
with:
go-version-file: go.mod
cache: true
- name: Set up JDK 17
if: github.ref_type == 'tag'
uses: actions/setup-java@v5
with:
distribution: temurin
java-version: "17"
- name: Set up Android SDK
if: github.ref_type == 'tag'
uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1
- name: Cache Gradle
if: github.ref_type == 'tag'
uses: actions/cache@v6
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
restore-keys: |
gradle-${{ runner.os }}-
- name: Build sidecar JAR
if: github.ref_type == 'tag'
run: make sidecar
- name: Publish the sanderling CLI to GitHub Releases
if: github.ref_type == 'tag'
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
with:
version: "~> v2"
args: release --clean
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# The docs used to build only when docs/ or the Makefile changed. A path
# filter here would have to sit on the whole workflow, so the site is rebuilt
# on every merge instead: it is pandoc over a few pages, and a deploy of bytes
# that did not change is a no-op.
docs:
if: github.ref == 'refs/heads/master' || github.ref_type == 'tag'
runs-on: ubuntu-latest
permissions:
contents: read
pages: write
id-token: write
# Pages takes one deployment at a time.
concurrency:
group: pages
cancel-in-progress: false
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
steps:
- uses: actions/checkout@v7
- name: Install pandoc
run: sudo apt-get update && sudo apt-get install -y pandoc
- name: Build site
run: make docs
# No include-hidden-files: v4 stopped uploading dot-files by default, and
# build/site has none. It is pandoc output plus a copy of docs/_assets,
# which holds three ordinary files. _assets is underscore-prefixed, not
# hidden, and deploy-pages serves the artifact without running Jekyll, so
# it needs no .nojekyll either.
- uses: actions/upload-pages-artifact@v5
with:
path: build/site
- uses: actions/deploy-pages@v5
id: deployment
.github/workflows/docs.yml
-50
View File
@@ -1,50 +0,0 @@
name: docs
on:
push:
branches: [master]
paths:
- "docs/**"
- "Makefile"
- ".github/workflows/docs.yml"
workflow_dispatch:
permissions:
contents: read
pages: write
id-token: write
concurrency:
group: pages
cancel-in-progress: false
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Install pandoc
run: sudo apt-get update && sudo apt-get install -y pandoc
- name: Build site
run: make docs
# No include-hidden-files: v4 stopped uploading dot-files by default, and
# build/site has none. It is pandoc output plus a copy of docs/_assets,
# which holds three ordinary files. _assets is underscore-prefixed, not
# hidden, and deploy-pages serves the artifact without running Jekyll, so
# it needs no .nojekyll either.
- uses: actions/upload-pages-artifact@v5
with:
path: build/site
deploy:
needs: build
runs-on: ubuntu-latest
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
steps:
- uses: actions/deploy-pages@v5
id: deployment
.github/workflows/examples.yml
-193
View File
@@ -1,193 +0,0 @@
name: examples
# Every example sanderling ships, fuzzed the same way: build sanderling for a
# platform, bring the target up, run a spec against it, classify the trace it
# wrote, upload the run. Only the bring-up differs, and that lives in the
# per-target actions under .github/actions/.
#
# Dispatch-only: these take tens of minutes and they demonstrate the product
# loop, they do not gate a merge.
#
# folio on ios and in the browser expect the bug: folio double-submits a
# transaction on a double tap, so the run is supposed to end with exit 2. Exit 0
# means the fuzzer stopped finding a bug that is still there; exit 1 means the
# harness broke. The two are worth telling apart, which is why
# --exit-on-violation exits 2 and not 1.
#
# folio on android is a health gate. It convicts in four runs out of five, which
# is real evidence but not a gate: the fifth would report a regression it had
# not found. Its budget is set so the conviction it usually gets is a bonus.
#
# the replay ui leg fuzzes sanderling's own replay UI, and any violation fails
# it. Its properties are cross-panel agreements that hold for any trace, so none
# of them needs recalibrating when the fixture changes.
on:
workflow_dispatch:
inputs:
targets:
description: which examples to fuzz
type: choice
options: [all, folio, android, ios, web, replay-ui]
default: all
seed:
description: seed override (0 = each target's calibrated seed)
default: "0"
max-steps:
description: step budget override (0 = each target's calibrated budget)
default: "0"
duration:
description: wall-clock budget override (empty = each target's calibrated budget)
default: ""
permissions:
contents: read
jobs:
plan:
runs-on: ubuntu-latest
permissions: {}
outputs:
examples: ${{ steps.pick.outputs.examples }}
steps:
# The matrix is built here rather than written out under strategy.matrix
# because a job-level `if:` cannot read the matrix context, so a static
# matrix has no way to leave a leg out. jq -c keeps the value on one line,
# which is what makes the $GITHUB_OUTPUT write below safe.
- name: Pick the examples to fuzz
id: pick
run: |
examples='[
{"target":"android","name":"folio on android","app":"folio",
"runs-on":"ubuntu-latest","timeout":90,"sanderling":"android",
"seed":"9","max-steps":"200","duration":"20m","artifact":"folio-android"},
{"target":"ios","name":"folio on ios","app":"folio",
"runs-on":"macos-15","timeout":90,"sanderling":"ios",
"seed":"7","max-steps":"240","duration":"20m","artifact":"folio-ios"},
{"target":"web","name":"folio in the browser","app":"folio",
"runs-on":"ubuntu-latest","timeout":60,"sanderling":"web","chrome":true,
"seed":"3","max-steps":"240","duration":"20m","artifact":"folio-web"},
{"target":"replay-ui","name":"the replay ui","app":"replay-ui",
"runs-on":"ubuntu-latest","timeout":45,"sanderling":"web","chrome":true,
"seed":"3","max-steps":"80","duration":"10m","artifact":"replay-ui-runs"}
]'
picked="$(jq -c --arg want "$TARGETS" \
'map(select($want == "all" or .target == $want or .app == $want))' \
<<<"$examples")"
if [ "$picked" = "[]" ]; then
echo "examples: '$TARGETS' selects no example, so this dispatch would run nothing" >&2
exit 1
fi
echo "examples=$picked" >> "$GITHUB_OUTPUT"
env:
TARGETS: ${{ inputs.targets }}
fuzz:
needs: plan
name: fuzz ${{ matrix.name }}
runs-on: ${{ matrix.runs-on }}
timeout-minutes: ${{ matrix.timeout }}
strategy:
# Each leg is its own evidence. One target failing must not cancel the
# others, which is how these ran as separate jobs.
fail-fast: false
matrix:
include: ${{ fromJSON(needs.plan.outputs.examples) }}
env:
SEED: ${{ inputs.seed != '0' && inputs.seed || matrix.seed }}
MAX_STEPS: ${{ inputs.max-steps != '0' && inputs.max-steps || matrix.max-steps }}
DURATION: ${{ inputs.duration != '' && inputs.duration || matrix.duration }}
IOS_DEVICE: iPhone 16 Pro
steps:
- uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: go.mod
cache: true
- name: Set up bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: "1.3.13"
- name: Set up headless Chrome
if: matrix.chrome
uses: ./.github/actions/headless-chrome
- name: Build the folio app
if: matrix.app == 'folio'
uses: ./.github/actions/folio-app
with:
platform: ${{ matrix.target }}
# The UI the replay-ui spec drives is the one embedded in this binary, so
# the build has to come after any change to replay-ui/src.
- name: Build sanderling
run: make "sanderling-$SANDERLING"
env:
SANDERLING: ${{ matrix.sanderling }}
- name: Put folio on the simulator
if: matrix.target == 'ios'
uses: ./.github/actions/folio-simulator
- name: Serve a trace to fuzz
id: fixture
if: matrix.target == 'replay-ui'
uses: ./.github/actions/replay-ui-fixture
- name: Fuzz folio on an emulator
if: matrix.target == 'android'
uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2.38.0
with:
api-level: 34
target: google_apis
arch: x86_64
emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim
disable-animations: true
script: .github/scripts/folio-run.sh android
- name: Fuzz folio
if: matrix.app == 'folio' && matrix.target != 'android'
run: .github/scripts/folio-run.sh "$TARGET"
env:
TARGET: ${{ matrix.target }}
# Inputs go through env rather than into the script text: a `${{ }}` is
# substituted before bash ever sees the line, so a seed of `$(id)` would
# run as a command.
- name: Fuzz the replay UI
if: matrix.target == 'replay-ui'
run: |
./bin/sanderling test \
--platform web \
--spec replay-ui/sanderling/spec.ts \
--bundle-id "$RUN_URL" \
--duration "$DURATION" \
--max-steps "$MAX_STEPS" \
--seed "$SEED" \
--exit-on-violation \
--output runs/replay-ui
env:
RUN_URL: ${{ steps.fixture.outputs.url }}
# Exit 0 above means no property returned false. It does not mean any
# property was ever evaluated against real content: they all decline to
# judge when the elements they read are absent, so a run that never
# rendered the step page is green and worthless. This step is what tells
# the two apart, and it fails the job when nothing was judged. folio's
# legs make the same call inside folio-run.sh, where the exit code it is
# judging is in scope.
- name: Classify the replay UI run
if: ${{ always() && matrix.target == 'replay-ui' }}
run: .github/scripts/replay-ui-summary.sh runs/replay-ui
- name: Upload the run
if: always()
uses: actions/upload-artifact@v7
with:
name: ${{ matrix.artifact }}
path: runs/
retention-days: 14
.github/workflows/release.yml
-144
View File
@@ -1,144 +0,0 @@
name: release
on:
push:
tags:
- "v*"
workflow_dispatch:
inputs:
tag:
description: "Tag to release (e.g. v0.0.1-rc1). Must already exist."
required: true
type: string
permissions:
contents: read
concurrency:
group: release-${{ github.ref }}
cancel-in-progress: false
jobs:
resolve-tag:
name: Resolve and validate the tag
runs-on: ubuntu-latest
permissions: {}
outputs:
tag: ${{ steps.tag.outputs.tag }}
version: ${{ steps.tag.outputs.version }}
steps:
# A refname is attacker-controlled text and git permits backtick, `$`,
# `(`, `;`, `&` and `|` in it, so it goes through env: a `${{ }}` is
# substituted before bash ever sees the line. Every later job reads these
# outputs rather than the refname, and nothing reaches a shell before it
# has matched the pattern. The pattern is anchored and admits no newline,
# which is what stops the value below forging a second $GITHUB_OUTPUT key.
- name: Validate the tag
id: tag
run: |
pattern='^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z]+(\.[0-9A-Za-z]+)*)?$'
if [[ ! "$TAG" =~ $pattern ]]; then
echo "release: refusing to publish from '$TAG'" >&2
echo "release: a release tag is vMAJOR.MINOR.PATCH with an optional -prerelease, e.g. v0.1.0 or v0.0.1-rc1" >&2
exit 1
fi
echo "tag=$TAG" >> "$GITHUB_OUTPUT"
echo "version=${TAG#v}" >> "$GITHUB_OUTPUT"
env:
TAG: ${{ inputs.tag || github.ref_name }}
release-npm:
name: Publish @sanderling/spec to npm
needs: resolve-tag
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@v7
with:
ref: ${{ needs.resolve-tag.outputs.tag }}
# `npm ci` below runs dependency lifecycle scripts, and no step in
# this job needs the git credential afterwards.
persist-credentials: false
- name: Set up Node 22
uses: actions/setup-node@v7
with:
node-version: "22"
registry-url: "https://registry.npmjs.org"
cache: npm
cache-dependency-path: pkg/spec/package-lock.json
- name: Install dependencies
working-directory: pkg/spec
run: npm ci
- name: Stamp version
working-directory: pkg/spec
run: npm version "$VERSION" --no-git-tag-version --allow-same-version
env:
VERSION: ${{ needs.resolve-tag.outputs.version }}
- name: Publish
working-directory: pkg/spec
# npm tag pre-releases (e.g. 0.1.0-rc1) as "next" so npm install @sanderling/spec
# keeps resolving the latest stable.
run: |
if [[ "$VERSION" == *-* ]]; then
npm publish --access public --tag next
else
npm publish --access public
fi
# The publish credential is scoped to the one step that publishes rather
# than to the job, so no other step runs with it in reach.
env:
VERSION: ${{ needs.resolve-tag.outputs.version }}
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
release-cli:
name: Publish sanderling CLI to GitHub Releases
needs: resolve-tag
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v7
with:
ref: ${{ needs.resolve-tag.outputs.tag }}
fetch-depth: 0
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: go.mod
cache: true
- name: Set up JDK 17
uses: actions/setup-java@v5
with:
distribution: temurin
java-version: "17"
- name: Set up Android SDK
uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1
- name: Cache Gradle
uses: actions/cache@v6
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
restore-keys: |
gradle-${{ runner.os }}-
- name: Build sidecar JAR
run: make sidecar
- name: Run GoReleaser
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
with:
version: "~> v2"
args: release --clean
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}