mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
test(ci): cover the folio classifier's verdicts
21 cases through a stubbed sanderling: every exit path, the drift check, a missing trace, a zero-byte trace, an empty glob and a truncated line. Asserts the flags that reached the binary, not just the exit code. Invoked as bash -eo pipefail -c, which is what a run: block does. Running folio-run.sh itself under -e would kill it at the first non-zero sanderling test, which is the exit code it exists to read.
This commit is contained in:
pj committed
1 parent
cd3bc51a0d
commit
41c6c1e4b9
1 file changed
+284
Executable
+284
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env bash
|
||||
# Drives folio-run.sh through a stubbed `sanderling` binary and checks the
|
||||
# verdict it reaches from each shape of trace. What is under test is the
|
||||
# classification, not the fuzzer: the stub writes the trace the run would have
|
||||
# written and exits the code the run would have exited.
|
||||
#
|
||||
# folio-run.sh is invoked as `bash -eo pipefail -c <script>`, which is what a
|
||||
# `run:` block does: -e is set on the shell that calls the script, and does not
|
||||
# cross the shebang into it. Running the script itself under -e would kill it at
|
||||
# the first non-zero `sanderling test`, which is the exit code it exists to read.
|
||||
set -euo pipefail
|
||||
|
||||
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
script="$here/folio-run.sh"
|
||||
spec="$here/../../examples/folio/sanderling/spec.ts"
|
||||
work="$(mktemp -d)"
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
|
||||
failed=0
|
||||
case_dir=""
|
||||
status=0
|
||||
spec_override=""
|
||||
|
||||
run() { # <case> <platform> <stub exit code> [none] ; trace lines on stdin
|
||||
local name="$1" platform="$2" code="$3" trace_mode="${4:-file}"
|
||||
case_dir="$work/$name"
|
||||
mkdir -p "$case_dir"
|
||||
cat > "$case_dir/trace.jsonl"
|
||||
printf '#!/usr/bin/env bash\n' > "$case_dir/sanderling"
|
||||
cat >> "$case_dir/sanderling" <<'STUB'
|
||||
printf '%s\n' "$@" > "$STUB_ARGV"
|
||||
out="" ; prev=""
|
||||
for arg in "$@"; do
|
||||
[ "$prev" = "--output" ] && out="$arg"
|
||||
prev="$arg"
|
||||
done
|
||||
if [ "$STUB_TRACE_MODE" = file ]; then
|
||||
mkdir -p "$out/20260815-101500"
|
||||
cp "$STUB_TRACE" "$out/20260815-101500/trace.jsonl"
|
||||
fi
|
||||
exit "$STUB_CODE"
|
||||
STUB
|
||||
chmod +x "$case_dir/sanderling"
|
||||
status=0
|
||||
cd "$case_dir"
|
||||
STUB_ARGV="$case_dir/argv" STUB_TRACE="$case_dir/trace.jsonl" \
|
||||
STUB_TRACE_MODE="$trace_mode" STUB_CODE="$code" \
|
||||
SANDERLING="$case_dir/sanderling" SPEC="${spec_override:-$spec}" \
|
||||
GITHUB_STEP_SUMMARY="$case_dir/summary.md" \
|
||||
SEED=7 MAX_STEPS=240 DURATION=20m \
|
||||
bash -eo pipefail -c "'$script' '$platform'" \
|
||||
> "$case_dir/out" 2> "$case_dir/err" || status=$?
|
||||
cd "$here"
|
||||
spec_override=""
|
||||
}
|
||||
|
||||
fail() { echo "FAIL: $*" >&2; failed=1; }
|
||||
|
||||
expect_status() { # <want> <case>
|
||||
[ "$status" = "$1" ] || fail "$2: exit $status, want $1"
|
||||
}
|
||||
|
||||
# grep reads both files directly: piping `cat` into `grep -q` returns 141 under
|
||||
# pipefail, because grep leaves on the first match and cat takes the SIGPIPE.
|
||||
expect_says() { # <text> <case>
|
||||
grep -qF -- "$1" "$case_dir/out" "$case_dir/err" || fail "$2: nothing said '$1'"
|
||||
}
|
||||
|
||||
expect_silent() { # <text> <case>
|
||||
if grep -qF -- "$1" "$case_dir/out" "$case_dir/err"; then
|
||||
fail "$2: should not have said '$1'"
|
||||
fi
|
||||
}
|
||||
|
||||
expect_summary() { # <text> <case>
|
||||
grep -qF -- "$1" "$case_dir/summary.md" || fail "$2: summary has no '$1'"
|
||||
}
|
||||
|
||||
expect_argv() { # <text> <case>
|
||||
grep -qxF -- "$1" "$case_dir/argv" || fail "$2: '$1' never reached the binary"
|
||||
}
|
||||
|
||||
convicting='{"violations":["submitCommitsOneTransactionPerAction"],"witnesses":{"submitCommitsOneTransactionPerAction":{"is_error":false,"reason":"one tap, two rows"}}}'
|
||||
unrelated='{"violations":["newAccountBalanceIsZero"],"witnesses":{"newAccountBalanceIsZero":{"is_error":false,"reason":"opened at 4.00"}}}'
|
||||
threw='{"violations":["submitMovesBalanceByAtMostTypedAmount"],"witnesses":{"submitMovesBalanceByAtMostTypedAmount":{"is_error":true,"reason":"TypeError: cannot read text of undefined"}}}'
|
||||
on_txn='{"route":"AddTransactionScreen","violations":[]}'
|
||||
off_txn='{"route":"HomeScreen","violations":[]}'
|
||||
|
||||
# --- the flags the calibrated runs were measured with reach the binary --------
|
||||
|
||||
run argv-ios ios 2 <<TRACE
|
||||
$on_txn
|
||||
$convicting
|
||||
TRACE
|
||||
expect_status 0 argv-ios
|
||||
expect_argv "--exit-on-violation" argv-ios
|
||||
expect_argv "--platform" argv-ios
|
||||
expect_argv "ios" argv-ios
|
||||
expect_argv "--seed" argv-ios
|
||||
expect_argv "7" argv-ios
|
||||
expect_argv "240" argv-ios
|
||||
expect_argv "20m" argv-ios
|
||||
expect_argv "iPhone 16 Pro" argv-ios
|
||||
|
||||
run argv-android android 2 <<TRACE
|
||||
$on_txn
|
||||
$convicting
|
||||
TRACE
|
||||
expect_status 0 argv-android
|
||||
expect_argv "--exit-on-violation" argv-android
|
||||
expect_argv "app.folio" argv-android
|
||||
expect_argv "examples/folio/app/androidApp/build/outputs/apk/debug/androidApp-debug.apk" argv-android
|
||||
|
||||
# --- ios and web: a conviction is required -----------------------------------
|
||||
|
||||
run convicted ios 2 <<TRACE
|
||||
$on_txn
|
||||
$convicting
|
||||
TRACE
|
||||
expect_status 0 convicted
|
||||
expect_says "found the submit bug in 2 steps (submitCommitsOneTransactionPerAction)" convicted
|
||||
expect_summary "- convicted on: submitCommitsOneTransactionPerAction" convicted
|
||||
|
||||
# Exit 2 with a violation of a real but ungated property is not this leg's bug.
|
||||
run other-violation ios 2 <<TRACE
|
||||
$on_txn
|
||||
$unrelated
|
||||
TRACE
|
||||
expect_status 1 other-violation
|
||||
expect_says "not the double-submit this leg gates on" other-violation
|
||||
expect_summary "- also violated: newAccountBalanceIsZero" other-violation
|
||||
|
||||
# A predicate that threw reaches exit 2 by the identical path, and is not a
|
||||
# verdict about folio at all.
|
||||
run threw-ios ios 2 <<TRACE
|
||||
$on_txn
|
||||
$threw
|
||||
TRACE
|
||||
expect_status 1 threw-ios
|
||||
expect_says "a predicate threw, so exit 2 is not a verdict about folio" threw-ios
|
||||
expect_says "TypeError: cannot read text of undefined" threw-ios
|
||||
expect_silent "found the submit bug" threw-ios
|
||||
expect_summary "**a predicate threw**" threw-ios
|
||||
|
||||
# The bug is still in folio, so a clean run is the fuzzer no longer reaching it.
|
||||
run clean-ios ios 0 <<TRACE
|
||||
$on_txn
|
||||
$on_txn
|
||||
TRACE
|
||||
expect_status 1 clean-ios
|
||||
expect_says "the double-submit bug was NOT found in 2 steps" clean-ios
|
||||
|
||||
run harness-ios ios 3 <<TRACE
|
||||
$on_txn
|
||||
TRACE
|
||||
expect_status 3 harness-ios
|
||||
expect_says "the harness failed with exit 3" harness-ios
|
||||
|
||||
# --- android: a health gate, with a conviction as a bonus --------------------
|
||||
|
||||
run android-convicted android 2 <<TRACE
|
||||
$on_txn
|
||||
$convicting
|
||||
TRACE
|
||||
expect_status 0 android-convicted
|
||||
expect_says "found the submit bug in 2 steps (a bonus, not required)" android-convicted
|
||||
|
||||
run android-other android 2 <<TRACE
|
||||
$on_txn
|
||||
$unrelated
|
||||
TRACE
|
||||
expect_status 0 android-other
|
||||
expect_says "judging health only" android-other
|
||||
|
||||
run android-healthy android 0 <<TRACE
|
||||
$on_txn
|
||||
$on_txn
|
||||
TRACE
|
||||
expect_status 0 android-healthy
|
||||
expect_says "healthy run over 2 steps, reached the transaction screen" android-healthy
|
||||
|
||||
# Health means it got to the screen the double-submit lives on. Anything less is
|
||||
# a leg that proved nothing, however green the exit code.
|
||||
run android-stalled android 0 <<TRACE
|
||||
$off_txn
|
||||
{"route":"LedgerScreen","violations":[]}
|
||||
TRACE
|
||||
expect_status 1 android-stalled
|
||||
expect_says "never reached AddTransactionScreen over 2 steps" android-stalled
|
||||
expect_says "routes the trace does record: HomeScreen,LedgerScreen" android-stalled
|
||||
|
||||
# The thrown-predicate check has to bite on android too: this is the leg whose
|
||||
# gate is loose enough to swallow it.
|
||||
run threw-android android 2 <<TRACE
|
||||
$on_txn
|
||||
$threw
|
||||
TRACE
|
||||
expect_status 1 threw-android
|
||||
expect_says "a predicate threw" threw-android
|
||||
expect_silent "judging health only" threw-android
|
||||
|
||||
run harness-android android 4 <<TRACE
|
||||
$on_txn
|
||||
TRACE
|
||||
expect_status 4 harness-android
|
||||
expect_says "the harness failed with exit 4" harness-android
|
||||
|
||||
# --- traces that are not there, or are there and say nothing -----------------
|
||||
|
||||
# Exit 2 and no run directory at all: the glob matches nothing, and an empty
|
||||
# trace must not be judged as a folio that behaved.
|
||||
run no-trace ios 2 none </dev/null
|
||||
expect_status 1 no-trace
|
||||
expect_says "wrote no trace" no-trace
|
||||
expect_silent "Traceback" no-trace
|
||||
|
||||
run no-trace-android android 0 none </dev/null
|
||||
expect_status 1 no-trace-android
|
||||
expect_says "wrote no trace" no-trace-android
|
||||
|
||||
# A zero-byte trace: the file is there, so the missing-trace check passes and
|
||||
# the classification has to survive reading nothing out of it.
|
||||
run zero-byte ios 0 file </dev/null
|
||||
expect_status 1 zero-byte
|
||||
expect_says "NOT found in 0 steps" zero-byte
|
||||
expect_silent "Traceback" zero-byte
|
||||
|
||||
# A line the recorder truncated is skipped, not fatal, and the violation on the
|
||||
# readable line is still found.
|
||||
run malformed ios 2 <<TRACE
|
||||
$on_txn
|
||||
{"violations":["submitCommitsOne
|
||||
$convicting
|
||||
TRACE
|
||||
expect_status 0 malformed
|
||||
expect_says "found the submit bug" malformed
|
||||
expect_silent "Traceback" malformed
|
||||
|
||||
run bad-platform windows 0 none </dev/null
|
||||
expect_status 64 bad-platform
|
||||
expect_says "unknown platform: windows" bad-platform
|
||||
|
||||
# --- the gate names still exist in the spec they gate ------------------------
|
||||
|
||||
# Nothing but this check ties GATED_PROPERTIES to the spec. Rename a gated
|
||||
# property and the classification matches nothing: ios and web report a
|
||||
# different bug, android reclassifies a conviction as health and stays green.
|
||||
sed 's/submitCommitsOneTransactionPerAction/submitCommitsOneTxnPerAction/g' \
|
||||
"$spec" > "$work/renamed-spec.ts"
|
||||
spec_override="$work/renamed-spec.ts"
|
||||
run drift-renamed ios 2 <<TRACE
|
||||
$on_txn
|
||||
$convicting
|
||||
TRACE
|
||||
expect_status 1 drift-renamed
|
||||
expect_says "no longer declares submitCommitsOneTransactionPerAction" drift-renamed
|
||||
if [ -e "$case_dir/argv" ]; then
|
||||
fail "drift-renamed: the run started before the gate was checked"
|
||||
fi
|
||||
|
||||
spec_override="$work/absent-spec.ts"
|
||||
run drift-missing ios 2 none </dev/null
|
||||
expect_status 1 drift-missing
|
||||
expect_says "cannot read" drift-missing
|
||||
|
||||
echo "export const notProperties = { a };" > "$work/shapeless-spec.ts"
|
||||
spec_override="$work/shapeless-spec.ts"
|
||||
run drift-shapeless ios 2 none </dev/null
|
||||
expect_status 1 drift-shapeless
|
||||
expect_says "declares no \`export const properties" drift-shapeless
|
||||
|
||||
# And the same check against the spec as it stands: this is the assertion that
|
||||
# fails at `make test` when someone renames a property without moving the gate.
|
||||
run gate-matches-spec ios 2 <<TRACE
|
||||
$on_txn
|
||||
$convicting
|
||||
TRACE
|
||||
expect_status 0 gate-matches-spec
|
||||
expect_silent "no longer declares" gate-matches-spec
|
||||
|
||||
if [ "$failed" = 0 ]; then
|
||||
echo "folio-run.sh: ok"
|
||||
fi
|
||||
exit "$failed"
|
||||
Reference in new issue
Block a user