mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 20:17:09 +00:00
ci(folio): pin the recalibrated seeds and drop android to a health gate
web 3 and ios 7 convict 3 runs out of 3 with an exactly 2x witness. android convicts 2 in 5 because the same seed does not walk the same trajectory there, so it proves the app runs instead.
This commit is contained in:
1 parent
21ae7fde83
commit
a9334b2927
2 files changed
+33
-7
No files matched your search
@@ -5,8 +5,10 @@
|
|||||||
#
|
#
|
||||||
# SEED=3 MAX_STEPS=240 .github/scripts/folio-run.sh android
|
# SEED=3 MAX_STEPS=240 .github/scripts/folio-run.sh android
|
||||||
#
|
#
|
||||||
# Every platform expects exit 2: folio's double-submit bug is still there, and a
|
# web and ios expect exit 2: folio's double-submit bug is still there, and a run
|
||||||
# run that no longer finds it is a regression in the fuzzer, not a pass.
|
# that no longer finds it is a regression in the fuzzer, not a pass. android is a
|
||||||
|
# health gate instead, because a run there is not a function of its seed; see
|
||||||
|
# docs/development/ci.md.
|
||||||
set -uo pipefail
|
set -uo pipefail
|
||||||
|
|
||||||
platform="${1:?usage: folio-run.sh android|ios|web}"
|
platform="${1:?usage: folio-run.sh android|ios|web}"
|
||||||
@@ -96,6 +98,26 @@ violated=$(grep -ho '"violations":\[[^]]*\]' "$run_dir/trace.jsonl" 2>/dev/null
|
|||||||
[ -n "$violated" ] && echo "- $violated"
|
[ -n "$violated" ] && echo "- $violated"
|
||||||
} >> "$summary"
|
} >> "$summary"
|
||||||
|
|
||||||
|
if [ "$platform" = "android" ]; then
|
||||||
|
# A health gate, not a conviction gate. The android hierarchy dump can show
|
||||||
|
# two screens mid-transition, and such a step applies no action, so the same
|
||||||
|
# seed walks a different trajectory each run and the bug turns up in about two
|
||||||
|
# runs in five. Demanding a conviction would cry wolf more often than it would
|
||||||
|
# catch the regression it exists to catch. Reaching the transaction screen is
|
||||||
|
# what this leg proves: the app built, installed, launched and drove.
|
||||||
|
case "$code" in
|
||||||
|
0) ;;
|
||||||
|
2) echo "folio/android: found the submit bug in $steps steps (a bonus, not required)" ;;
|
||||||
|
*) echo "folio/android: the harness failed with exit $code" >&2; exit "$code" ;;
|
||||||
|
esac
|
||||||
|
if ! grep -q '"AddTransactionScreen"' "$run_dir/trace.jsonl"; then
|
||||||
|
echo "folio/android: the run never reached AddTransactionScreen, so it never got past login" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "folio/android: healthy run over $steps steps, reached the transaction screen"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
case "$code" in
|
case "$code" in
|
||||||
2) echo "folio/$platform: found the submit bug in $steps steps"; exit 0 ;;
|
2) echo "folio/$platform: found the submit bug in $steps steps"; exit 0 ;;
|
||||||
0) echo "folio/$platform: the run finished clean; the double-submit bug was NOT found in $steps steps (seed $seed)" >&2; exit 1 ;;
|
0) echo "folio/$platform: the run finished clean; the double-submit bug was NOT found in $steps steps (seed $seed)" >&2; exit 1 ;;
|
||||||
|
|||||||
@@ -4,11 +4,15 @@ name: folio
|
|||||||
# browser, builds the folio app for that platform, and runs
|
# browser, builds the folio app for that platform, and runs
|
||||||
# examples/folio/sanderling/spec.ts against it.
|
# examples/folio/sanderling/spec.ts against it.
|
||||||
#
|
#
|
||||||
# All three are expect-the-bug jobs: folio double-submits a transaction on a
|
# web and ios are expect-the-bug jobs: folio double-submits a transaction on a
|
||||||
# double tap, so the run is supposed to end with exit 2. Exit 0 means the fuzzer
|
# double tap, so the run is supposed to end with exit 2. Exit 0 means the fuzzer
|
||||||
# stopped finding a bug that is still there; exit 1 means the harness broke. The
|
# stopped finding a bug that is still there; exit 1 means the harness broke. The
|
||||||
# two are worth telling apart, which is why --exit-on-violation exits 2 and not
|
# two are worth telling apart, which is why --exit-on-violation exits 2 and not
|
||||||
# 1.
|
# 1.
|
||||||
|
#
|
||||||
|
# android is a health gate. The same seed does not walk the same trajectory
|
||||||
|
# there, so the bug turns up in roughly two runs in five, and a conviction gate
|
||||||
|
# would report a regression it had not found.
|
||||||
|
|
||||||
on:
|
on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
@@ -92,8 +96,8 @@ jobs:
|
|||||||
disable-animations: true
|
disable-animations: true
|
||||||
script: .github/scripts/folio-run.sh android
|
script: .github/scripts/folio-run.sh android
|
||||||
env:
|
env:
|
||||||
SEED: ${{ inputs.seed != '0' && inputs.seed || '3' }}
|
SEED: ${{ inputs.seed != '0' && inputs.seed || '9' }}
|
||||||
MAX_STEPS: ${{ inputs.max-steps != '0' && inputs.max-steps || '240' }}
|
MAX_STEPS: ${{ inputs.max-steps != '0' && inputs.max-steps || '120' }}
|
||||||
DURATION: ${{ inputs.duration }}
|
DURATION: ${{ inputs.duration }}
|
||||||
|
|
||||||
- name: Upload the run
|
- name: Upload the run
|
||||||
@@ -173,7 +177,7 @@ jobs:
|
|||||||
- name: Fuzz folio on the simulator
|
- name: Fuzz folio on the simulator
|
||||||
run: .github/scripts/folio-run.sh ios
|
run: .github/scripts/folio-run.sh ios
|
||||||
env:
|
env:
|
||||||
SEED: ${{ inputs.seed != '0' && inputs.seed || '1' }}
|
SEED: ${{ inputs.seed != '0' && inputs.seed || '7' }}
|
||||||
MAX_STEPS: ${{ inputs.max-steps != '0' && inputs.max-steps || '240' }}
|
MAX_STEPS: ${{ inputs.max-steps != '0' && inputs.max-steps || '240' }}
|
||||||
DURATION: ${{ inputs.duration }}
|
DURATION: ${{ inputs.duration }}
|
||||||
IOS_DEVICE: iPhone 16 Pro
|
IOS_DEVICE: iPhone 16 Pro
|
||||||
@@ -244,7 +248,7 @@ jobs:
|
|||||||
- name: Fuzz folio in the browser
|
- name: Fuzz folio in the browser
|
||||||
run: .github/scripts/folio-run.sh web
|
run: .github/scripts/folio-run.sh web
|
||||||
env:
|
env:
|
||||||
SEED: ${{ inputs.seed != '0' && inputs.seed || '1' }}
|
SEED: ${{ inputs.seed != '0' && inputs.seed || '3' }}
|
||||||
MAX_STEPS: ${{ inputs.max-steps != '0' && inputs.max-steps || '240' }}
|
MAX_STEPS: ${{ inputs.max-steps != '0' && inputs.max-steps || '240' }}
|
||||||
DURATION: ${{ inputs.duration }}
|
DURATION: ${{ inputs.duration }}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user