From b2f9335f5d9fe72b81dc75252a4b5337cdc41df1 Mon Sep 17 00:00:00 2001 From: PJ Date: Sat, 15 Aug 2026 12:47:03 +0530 Subject: [PATCH] ci(folio): a missing trace is not a verdict with no trace the android gate ran its grep against ./trace.jsonl and reported "never reached AddTransactionScreen, so it never got past login", which is not what happened. the web and ios branches had the same misdiagnosis on exit 0. same class, one line up: the classifier's own failure was swallowed, so with the evidence reader dead the gate printed a healthy run and exited 0. --- .github/scripts/folio-run.sh | 15 +++++++++++++-- docs/development/ci.md | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/scripts/folio-run.sh b/.github/scripts/folio-run.sh index 51fcf06..de00edd 100755 --- a/.github/scripts/folio-run.sh +++ b/.github/scripts/folio-run.sh @@ -138,7 +138,10 @@ for line in lines: for names in (convictions, thrown, other): print(", ".join(names)) PY -) +) || { + echo "folio/$platform: could not classify $trace, so exit $code cannot be read as a verdict" >&2 + exit 1 +} convicted=$(printf '%s\n' "$classified" | sed -n '1p') thrown=$(printf '%s\n' "$classified" | sed -n '2p') other=$(printf '%s\n' "$classified" | sed -n '3p') @@ -164,6 +167,14 @@ if [ -n "$thrown" ]; then exit 1 fi +# Only 0 and 2, the two codes that claim the run completed: any other code is a +# harness failure, which the branches below already report as one. Without this, +# a missing trace is judged as an empty trace and reported as a verdict on folio. +if [ ! -f "$trace" ] && { [ "$code" = 0 ] || [ "$code" = 2 ]; }; then + echo "folio/$platform: the run exited $code but wrote no trace under $output/, so there is nothing to judge" >&2 + exit 1 +fi + if [ "$platform" = "android" ]; then # A health gate, not a conviction gate: android convicts in four runs out of # five, and a gate that fails the fifth would report a regression it had not @@ -180,7 +191,7 @@ if [ "$platform" = "android" ]; then ;; *) echo "folio/android: the harness failed with exit $code" >&2; exit "$code" ;; esac - if ! grep -q '"AddTransactionScreen"' "$trace" 2>/dev/null; then + if ! grep -q '"AddTransactionScreen"' "$trace"; then echo "folio/android: the run never reached AddTransactionScreen, so it never got past login" >&2 exit 1 fi diff --git a/docs/development/ci.md b/docs/development/ci.md index 502795a..db0e291 100644 --- a/docs/development/ci.md +++ b/docs/development/ci.md @@ -52,6 +52,7 @@ before it decides: | a violation whose witness carries `is_error` | red: a predicate threw, and a thrown predicate is recorded as a violation like any other | | a violation of any other property (`newAccountBalanceIsZero` fires on one android seed) | red: a real finding, but not the one this leg gates on | | no violation and exit 0 | red: the fuzzer stopped finding a bug that is still there | +| no trace at all, on exit 0 or 2 | red: the run recorded nothing, so there is no verdict to read | | exit 1, or any other code | red: the harness broke, and the code propagates | The first two rows are why the check is worth the code it takes. A `TypeError`