mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
pj
41c6c1e4b9
21 cases through a stubbed sanderling: every exit path, the drift check, a missing trace, a zero-byte trace, an empty glob and a truncated line. Asserts the flags that reached the binary, not just the exit code. Invoked as bash -eo pipefail -c, which is what a run: block does. Running folio-run.sh itself under -e would kill it at the first non-zero sanderling test, which is the exit code it exists to read.
285 lines
9.4 KiB
Bash
Executable File
285 lines
9.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Drives folio-run.sh through a stubbed `sanderling` binary and checks the
|
|
# verdict it reaches from each shape of trace. What is under test is the
|
|
# classification, not the fuzzer: the stub writes the trace the run would have
|
|
# written and exits the code the run would have exited.
|
|
#
|
|
# folio-run.sh is invoked as `bash -eo pipefail -c <script>`, which is what a
|
|
# `run:` block does: -e is set on the shell that calls the script, and does not
|
|
# cross the shebang into it. Running the script itself under -e would kill it at
|
|
# the first non-zero `sanderling test`, which is the exit code it exists to read.
|
|
set -euo pipefail
|
|
|
|
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
script="$here/folio-run.sh"
|
|
spec="$here/../../examples/folio/sanderling/spec.ts"
|
|
work="$(mktemp -d)"
|
|
trap 'rm -rf "$work"' EXIT
|
|
|
|
failed=0
|
|
case_dir=""
|
|
status=0
|
|
spec_override=""
|
|
|
|
run() { # <case> <platform> <stub exit code> [none] ; trace lines on stdin
|
|
local name="$1" platform="$2" code="$3" trace_mode="${4:-file}"
|
|
case_dir="$work/$name"
|
|
mkdir -p "$case_dir"
|
|
cat > "$case_dir/trace.jsonl"
|
|
printf '#!/usr/bin/env bash\n' > "$case_dir/sanderling"
|
|
cat >> "$case_dir/sanderling" <<'STUB'
|
|
printf '%s\n' "$@" > "$STUB_ARGV"
|
|
out="" ; prev=""
|
|
for arg in "$@"; do
|
|
[ "$prev" = "--output" ] && out="$arg"
|
|
prev="$arg"
|
|
done
|
|
if [ "$STUB_TRACE_MODE" = file ]; then
|
|
mkdir -p "$out/20260815-101500"
|
|
cp "$STUB_TRACE" "$out/20260815-101500/trace.jsonl"
|
|
fi
|
|
exit "$STUB_CODE"
|
|
STUB
|
|
chmod +x "$case_dir/sanderling"
|
|
status=0
|
|
cd "$case_dir"
|
|
STUB_ARGV="$case_dir/argv" STUB_TRACE="$case_dir/trace.jsonl" \
|
|
STUB_TRACE_MODE="$trace_mode" STUB_CODE="$code" \
|
|
SANDERLING="$case_dir/sanderling" SPEC="${spec_override:-$spec}" \
|
|
GITHUB_STEP_SUMMARY="$case_dir/summary.md" \
|
|
SEED=7 MAX_STEPS=240 DURATION=20m \
|
|
bash -eo pipefail -c "'$script' '$platform'" \
|
|
> "$case_dir/out" 2> "$case_dir/err" || status=$?
|
|
cd "$here"
|
|
spec_override=""
|
|
}
|
|
|
|
fail() { echo "FAIL: $*" >&2; failed=1; }
|
|
|
|
expect_status() { # <want> <case>
|
|
[ "$status" = "$1" ] || fail "$2: exit $status, want $1"
|
|
}
|
|
|
|
# grep reads both files directly: piping `cat` into `grep -q` returns 141 under
|
|
# pipefail, because grep leaves on the first match and cat takes the SIGPIPE.
|
|
expect_says() { # <text> <case>
|
|
grep -qF -- "$1" "$case_dir/out" "$case_dir/err" || fail "$2: nothing said '$1'"
|
|
}
|
|
|
|
expect_silent() { # <text> <case>
|
|
if grep -qF -- "$1" "$case_dir/out" "$case_dir/err"; then
|
|
fail "$2: should not have said '$1'"
|
|
fi
|
|
}
|
|
|
|
expect_summary() { # <text> <case>
|
|
grep -qF -- "$1" "$case_dir/summary.md" || fail "$2: summary has no '$1'"
|
|
}
|
|
|
|
expect_argv() { # <text> <case>
|
|
grep -qxF -- "$1" "$case_dir/argv" || fail "$2: '$1' never reached the binary"
|
|
}
|
|
|
|
convicting='{"violations":["submitCommitsOneTransactionPerAction"],"witnesses":{"submitCommitsOneTransactionPerAction":{"is_error":false,"reason":"one tap, two rows"}}}'
|
|
unrelated='{"violations":["newAccountBalanceIsZero"],"witnesses":{"newAccountBalanceIsZero":{"is_error":false,"reason":"opened at 4.00"}}}'
|
|
threw='{"violations":["submitMovesBalanceByAtMostTypedAmount"],"witnesses":{"submitMovesBalanceByAtMostTypedAmount":{"is_error":true,"reason":"TypeError: cannot read text of undefined"}}}'
|
|
on_txn='{"route":"AddTransactionScreen","violations":[]}'
|
|
off_txn='{"route":"HomeScreen","violations":[]}'
|
|
|
|
# --- the flags the calibrated runs were measured with reach the binary --------
|
|
|
|
run argv-ios ios 2 <<TRACE
|
|
$on_txn
|
|
$convicting
|
|
TRACE
|
|
expect_status 0 argv-ios
|
|
expect_argv "--exit-on-violation" argv-ios
|
|
expect_argv "--platform" argv-ios
|
|
expect_argv "ios" argv-ios
|
|
expect_argv "--seed" argv-ios
|
|
expect_argv "7" argv-ios
|
|
expect_argv "240" argv-ios
|
|
expect_argv "20m" argv-ios
|
|
expect_argv "iPhone 16 Pro" argv-ios
|
|
|
|
run argv-android android 2 <<TRACE
|
|
$on_txn
|
|
$convicting
|
|
TRACE
|
|
expect_status 0 argv-android
|
|
expect_argv "--exit-on-violation" argv-android
|
|
expect_argv "app.folio" argv-android
|
|
expect_argv "examples/folio/app/androidApp/build/outputs/apk/debug/androidApp-debug.apk" argv-android
|
|
|
|
# --- ios and web: a conviction is required -----------------------------------
|
|
|
|
run convicted ios 2 <<TRACE
|
|
$on_txn
|
|
$convicting
|
|
TRACE
|
|
expect_status 0 convicted
|
|
expect_says "found the submit bug in 2 steps (submitCommitsOneTransactionPerAction)" convicted
|
|
expect_summary "- convicted on: submitCommitsOneTransactionPerAction" convicted
|
|
|
|
# Exit 2 with a violation of a real but ungated property is not this leg's bug.
|
|
run other-violation ios 2 <<TRACE
|
|
$on_txn
|
|
$unrelated
|
|
TRACE
|
|
expect_status 1 other-violation
|
|
expect_says "not the double-submit this leg gates on" other-violation
|
|
expect_summary "- also violated: newAccountBalanceIsZero" other-violation
|
|
|
|
# A predicate that threw reaches exit 2 by the identical path, and is not a
|
|
# verdict about folio at all.
|
|
run threw-ios ios 2 <<TRACE
|
|
$on_txn
|
|
$threw
|
|
TRACE
|
|
expect_status 1 threw-ios
|
|
expect_says "a predicate threw, so exit 2 is not a verdict about folio" threw-ios
|
|
expect_says "TypeError: cannot read text of undefined" threw-ios
|
|
expect_silent "found the submit bug" threw-ios
|
|
expect_summary "**a predicate threw**" threw-ios
|
|
|
|
# The bug is still in folio, so a clean run is the fuzzer no longer reaching it.
|
|
run clean-ios ios 0 <<TRACE
|
|
$on_txn
|
|
$on_txn
|
|
TRACE
|
|
expect_status 1 clean-ios
|
|
expect_says "the double-submit bug was NOT found in 2 steps" clean-ios
|
|
|
|
run harness-ios ios 3 <<TRACE
|
|
$on_txn
|
|
TRACE
|
|
expect_status 3 harness-ios
|
|
expect_says "the harness failed with exit 3" harness-ios
|
|
|
|
# --- android: a health gate, with a conviction as a bonus --------------------
|
|
|
|
run android-convicted android 2 <<TRACE
|
|
$on_txn
|
|
$convicting
|
|
TRACE
|
|
expect_status 0 android-convicted
|
|
expect_says "found the submit bug in 2 steps (a bonus, not required)" android-convicted
|
|
|
|
run android-other android 2 <<TRACE
|
|
$on_txn
|
|
$unrelated
|
|
TRACE
|
|
expect_status 0 android-other
|
|
expect_says "judging health only" android-other
|
|
|
|
run android-healthy android 0 <<TRACE
|
|
$on_txn
|
|
$on_txn
|
|
TRACE
|
|
expect_status 0 android-healthy
|
|
expect_says "healthy run over 2 steps, reached the transaction screen" android-healthy
|
|
|
|
# Health means it got to the screen the double-submit lives on. Anything less is
|
|
# a leg that proved nothing, however green the exit code.
|
|
run android-stalled android 0 <<TRACE
|
|
$off_txn
|
|
{"route":"LedgerScreen","violations":[]}
|
|
TRACE
|
|
expect_status 1 android-stalled
|
|
expect_says "never reached AddTransactionScreen over 2 steps" android-stalled
|
|
expect_says "routes the trace does record: HomeScreen,LedgerScreen" android-stalled
|
|
|
|
# The thrown-predicate check has to bite on android too: this is the leg whose
|
|
# gate is loose enough to swallow it.
|
|
run threw-android android 2 <<TRACE
|
|
$on_txn
|
|
$threw
|
|
TRACE
|
|
expect_status 1 threw-android
|
|
expect_says "a predicate threw" threw-android
|
|
expect_silent "judging health only" threw-android
|
|
|
|
run harness-android android 4 <<TRACE
|
|
$on_txn
|
|
TRACE
|
|
expect_status 4 harness-android
|
|
expect_says "the harness failed with exit 4" harness-android
|
|
|
|
# --- traces that are not there, or are there and say nothing -----------------
|
|
|
|
# Exit 2 and no run directory at all: the glob matches nothing, and an empty
|
|
# trace must not be judged as a folio that behaved.
|
|
run no-trace ios 2 none </dev/null
|
|
expect_status 1 no-trace
|
|
expect_says "wrote no trace" no-trace
|
|
expect_silent "Traceback" no-trace
|
|
|
|
run no-trace-android android 0 none </dev/null
|
|
expect_status 1 no-trace-android
|
|
expect_says "wrote no trace" no-trace-android
|
|
|
|
# A zero-byte trace: the file is there, so the missing-trace check passes and
|
|
# the classification has to survive reading nothing out of it.
|
|
run zero-byte ios 0 file </dev/null
|
|
expect_status 1 zero-byte
|
|
expect_says "NOT found in 0 steps" zero-byte
|
|
expect_silent "Traceback" zero-byte
|
|
|
|
# A line the recorder truncated is skipped, not fatal, and the violation on the
|
|
# readable line is still found.
|
|
run malformed ios 2 <<TRACE
|
|
$on_txn
|
|
{"violations":["submitCommitsOne
|
|
$convicting
|
|
TRACE
|
|
expect_status 0 malformed
|
|
expect_says "found the submit bug" malformed
|
|
expect_silent "Traceback" malformed
|
|
|
|
run bad-platform windows 0 none </dev/null
|
|
expect_status 64 bad-platform
|
|
expect_says "unknown platform: windows" bad-platform
|
|
|
|
# --- the gate names still exist in the spec they gate ------------------------
|
|
|
|
# Nothing but this check ties GATED_PROPERTIES to the spec. Rename a gated
|
|
# property and the classification matches nothing: ios and web report a
|
|
# different bug, android reclassifies a conviction as health and stays green.
|
|
sed 's/submitCommitsOneTransactionPerAction/submitCommitsOneTxnPerAction/g' \
|
|
"$spec" > "$work/renamed-spec.ts"
|
|
spec_override="$work/renamed-spec.ts"
|
|
run drift-renamed ios 2 <<TRACE
|
|
$on_txn
|
|
$convicting
|
|
TRACE
|
|
expect_status 1 drift-renamed
|
|
expect_says "no longer declares submitCommitsOneTransactionPerAction" drift-renamed
|
|
if [ -e "$case_dir/argv" ]; then
|
|
fail "drift-renamed: the run started before the gate was checked"
|
|
fi
|
|
|
|
spec_override="$work/absent-spec.ts"
|
|
run drift-missing ios 2 none </dev/null
|
|
expect_status 1 drift-missing
|
|
expect_says "cannot read" drift-missing
|
|
|
|
echo "export const notProperties = { a };" > "$work/shapeless-spec.ts"
|
|
spec_override="$work/shapeless-spec.ts"
|
|
run drift-shapeless ios 2 none </dev/null
|
|
expect_status 1 drift-shapeless
|
|
expect_says "declares no \`export const properties" drift-shapeless
|
|
|
|
# And the same check against the spec as it stands: this is the assertion that
|
|
# fails at `make test` when someone renames a property without moving the gate.
|
|
run gate-matches-spec ios 2 <<TRACE
|
|
$on_txn
|
|
$convicting
|
|
TRACE
|
|
expect_status 0 gate-matches-spec
|
|
expect_silent "no longer declares" gate-matches-spec
|
|
|
|
if [ "$failed" = 0 ]; then
|
|
echo "folio-run.sh: ok"
|
|
fi
|
|
exit "$failed"
|