diff --git a/.github/scripts/folio-run.sh b/.github/scripts/folio-run.sh index 51fcf06..de00edd 100755 --- a/.github/scripts/folio-run.sh +++ b/.github/scripts/folio-run.sh @@ -138,7 +138,10 @@ for line in lines: for names in (convictions, thrown, other): print(", ".join(names)) PY -) +) || { + echo "folio/$platform: could not classify $trace, so exit $code cannot be read as a verdict" >&2 + exit 1 +} convicted=$(printf '%s\n' "$classified" | sed -n '1p') thrown=$(printf '%s\n' "$classified" | sed -n '2p') other=$(printf '%s\n' "$classified" | sed -n '3p') @@ -164,6 +167,14 @@ if [ -n "$thrown" ]; then exit 1 fi +# Only 0 and 2, the two codes that claim the run completed: any other code is a +# harness failure, which the branches below already report as one. Without this, +# a missing trace is judged as an empty trace and reported as a verdict on folio. +if [ ! -f "$trace" ] && { [ "$code" = 0 ] || [ "$code" = 2 ]; }; then + echo "folio/$platform: the run exited $code but wrote no trace under $output/, so there is nothing to judge" >&2 + exit 1 +fi + if [ "$platform" = "android" ]; then # A health gate, not a conviction gate: android convicts in four runs out of # five, and a gate that fails the fifth would report a regression it had not @@ -180,7 +191,7 @@ if [ "$platform" = "android" ]; then ;; *) echo "folio/android: the harness failed with exit $code" >&2; exit "$code" ;; esac - if ! grep -q '"AddTransactionScreen"' "$trace" 2>/dev/null; then + if ! grep -q '"AddTransactionScreen"' "$trace"; then echo "folio/android: the run never reached AddTransactionScreen, so it never got past login" >&2 exit 1 fi diff --git a/docs/development/ci.md b/docs/development/ci.md index 502795a..db0e291 100644 --- a/docs/development/ci.md +++ b/docs/development/ci.md @@ -52,6 +52,7 @@ before it decides: | a violation whose witness carries `is_error` | red: a predicate threw, and a thrown predicate is recorded as a violation like any other | | a violation of any other property (`newAccountBalanceIsZero` fires on one android seed) | red: a real finding, but not the one this leg gates on | | no violation and exit 0 | red: the fuzzer stopped finding a bug that is still there | +| no trace at all, on exit 0 or 2 | red: the run recorded nothing, so there is no verdict to read | | exit 1, or any other code | red: the harness broke, and the code propagates | The first two rows are why the check is worth the code it takes. A `TypeError`