From 71e3e25131dc1edc9e71fc555751f9cb53f5d94f Mon Sep 17 00:00:00 2001 From: PJ Date: Sun, 7 Jun 2026 22:33:21 +0530 Subject: [PATCH] refactor(conformance): move the gate harness out of examples/folio --- conformance/.gitignore | 1 + {examples/folio => conformance}/gates.sh | 19 ++++++++++--------- .../testdata}/g1-nonzero-exit/exit_status | 0 .../testdata}/g1-nonzero-exit/output.log | 0 .../testdata}/g1-nonzero-exit/trace.jsonl | 0 .../testdata}/g2-benign-only/exit_status | 0 .../testdata}/g2-benign-only/output.log | 0 .../testdata}/g2-benign-only/trace.jsonl | 0 .../testdata}/g2-real-error/exit_status | 0 .../testdata}/g2-real-error/output.log | 0 .../testdata}/g2-real-error/trace.jsonl | 0 .../testdata}/g3-dirty-field/exit_status | 0 .../testdata}/g3-dirty-field/output.log | 0 .../testdata}/g3-dirty-field/trace.jsonl | 0 .../testdata}/g4-doubled-text/exit_status | 0 .../testdata}/g4-doubled-text/output.log | 0 .../testdata}/g4-doubled-text/trace.jsonl | 0 .../testdata}/g5-slow-p95/exit_status | 0 .../testdata}/g5-slow-p95/output.log | 0 .../testdata}/g5-slow-p95/trace.jsonl | 0 .../testdata}/generate.py | 0 .../testdata}/pass/exit_status | 0 .../testdata}/pass/output.log | 0 .../testdata}/pass/trace.jsonl | 0 24 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 conformance/.gitignore rename {examples/folio => conformance}/gates.sh (95%) rename {examples/folio/gates-testdata => conformance/testdata}/g1-nonzero-exit/exit_status (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g1-nonzero-exit/output.log (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g1-nonzero-exit/trace.jsonl (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g2-benign-only/exit_status (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g2-benign-only/output.log (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g2-benign-only/trace.jsonl (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g2-real-error/exit_status (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g2-real-error/output.log (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g2-real-error/trace.jsonl (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g3-dirty-field/exit_status (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g3-dirty-field/output.log (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g3-dirty-field/trace.jsonl (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g4-doubled-text/exit_status (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g4-doubled-text/output.log (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g4-doubled-text/trace.jsonl (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g5-slow-p95/exit_status (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g5-slow-p95/output.log (100%) rename {examples/folio/gates-testdata => conformance/testdata}/g5-slow-p95/trace.jsonl (100%) rename {examples/folio/gates-testdata => conformance/testdata}/generate.py (100%) rename {examples/folio/gates-testdata => conformance/testdata}/pass/exit_status (100%) rename {examples/folio/gates-testdata => conformance/testdata}/pass/output.log (100%) rename {examples/folio/gates-testdata => conformance/testdata}/pass/trace.jsonl (100%) diff --git a/conformance/.gitignore b/conformance/.gitignore new file mode 100644 index 0000000..a1e0396 --- /dev/null +++ b/conformance/.gitignore @@ -0,0 +1 @@ +runs/ diff --git a/examples/folio/gates.sh b/conformance/gates.sh similarity index 95% rename from examples/folio/gates.sh rename to conformance/gates.sh index 1c1458d..81d580b 100755 --- a/examples/folio/gates.sh +++ b/conformance/gates.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Scripted conformance gate for the iOS simulator driver. Runs five serial, -# non-overlapping 3-minute fuzz runs against the folio app and scores five -# gates (G1..G5) over the captured traces and output. Exits non-zero if any -# gate fails. +# non-overlapping 3-minute fuzz runs against the folio example app +# (examples/folio) and scores five gates (G1..G5) over the captured traces and +# output. Exits non-zero if any gate fails. # # Backends: # BACKEND=simulator (default) drive the booted iOS simulator @@ -25,6 +25,7 @@ set -euo pipefail script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +folio_directory="$(cd "${script_directory}/../examples/folio" && pwd)" BACKEND="${BACKEND:-simulator}" RUNS="${RUNS:-5}" @@ -35,8 +36,8 @@ SANDERLING="${SANDERLING:-sanderling}" IOS_DEVICE="${IOS_DEVICE:-iPhone 17 Pro}" bundle_id="app.folio" -spec_path="${script_directory}/sanderling/spec.ts" -ios_app="${script_directory}/app/iosApp/build/Build/Products/Debug-iphonesimulator/iosApp.app" +spec_path="${folio_directory}/sanderling/spec.ts" +ios_app="${folio_directory}/app/iosApp/build/Build/Products/Debug-iphonesimulator/iosApp.app" # The companion binary, embedded for simulator runs. Referenced by file name # only for the orphan-process check; prose elsewhere says "the companion". @@ -252,12 +253,12 @@ collect_run_artifacts() { run_gates() { if [[ "$BACKEND" == "simulator" ]]; then echo "preparing folio build for the simulator backend" - ( cd "$script_directory" && just ios >/dev/null ) + ( cd "$folio_directory" && just ios >/dev/null ) fi local timestamp timestamp="$(date +%Y%m%d-%H%M%S)" - local gate_root="${script_directory}/sanderling/gates/${timestamp}" + local gate_root="${script_directory}/runs/${timestamp}" mkdir -p "$gate_root" echo "gate root: ${gate_root}" echo "backend=${BACKEND} runs=${RUNS} duration=${DURATION} seed=${SEED} p95_limit_ms=${P95_LIMIT_MS}" @@ -338,10 +339,10 @@ run_gates() { # ---- offline self-test ----------------------------------------------------- # Exercises every analyzer against canned passing/failing run directories under -# gates-testdata/ so the parsing logic can be checked without a device. +# testdata/ so the parsing logic can be checked without a device. self_test() { - local testdata="${script_directory}/gates-testdata" + local testdata="${script_directory}/testdata" local failures=0 assert() { diff --git a/examples/folio/gates-testdata/g1-nonzero-exit/exit_status b/conformance/testdata/g1-nonzero-exit/exit_status similarity index 100% rename from examples/folio/gates-testdata/g1-nonzero-exit/exit_status rename to conformance/testdata/g1-nonzero-exit/exit_status diff --git a/examples/folio/gates-testdata/g1-nonzero-exit/output.log b/conformance/testdata/g1-nonzero-exit/output.log similarity index 100% rename from examples/folio/gates-testdata/g1-nonzero-exit/output.log rename to conformance/testdata/g1-nonzero-exit/output.log diff --git a/examples/folio/gates-testdata/g1-nonzero-exit/trace.jsonl b/conformance/testdata/g1-nonzero-exit/trace.jsonl similarity index 100% rename from examples/folio/gates-testdata/g1-nonzero-exit/trace.jsonl rename to conformance/testdata/g1-nonzero-exit/trace.jsonl diff --git a/examples/folio/gates-testdata/g2-benign-only/exit_status b/conformance/testdata/g2-benign-only/exit_status similarity index 100% rename from examples/folio/gates-testdata/g2-benign-only/exit_status rename to conformance/testdata/g2-benign-only/exit_status diff --git a/examples/folio/gates-testdata/g2-benign-only/output.log b/conformance/testdata/g2-benign-only/output.log similarity index 100% rename from examples/folio/gates-testdata/g2-benign-only/output.log rename to conformance/testdata/g2-benign-only/output.log diff --git a/examples/folio/gates-testdata/g2-benign-only/trace.jsonl b/conformance/testdata/g2-benign-only/trace.jsonl similarity index 100% rename from examples/folio/gates-testdata/g2-benign-only/trace.jsonl rename to conformance/testdata/g2-benign-only/trace.jsonl diff --git a/examples/folio/gates-testdata/g2-real-error/exit_status b/conformance/testdata/g2-real-error/exit_status similarity index 100% rename from examples/folio/gates-testdata/g2-real-error/exit_status rename to conformance/testdata/g2-real-error/exit_status diff --git a/examples/folio/gates-testdata/g2-real-error/output.log b/conformance/testdata/g2-real-error/output.log similarity index 100% rename from examples/folio/gates-testdata/g2-real-error/output.log rename to conformance/testdata/g2-real-error/output.log diff --git a/examples/folio/gates-testdata/g2-real-error/trace.jsonl b/conformance/testdata/g2-real-error/trace.jsonl similarity index 100% rename from examples/folio/gates-testdata/g2-real-error/trace.jsonl rename to conformance/testdata/g2-real-error/trace.jsonl diff --git a/examples/folio/gates-testdata/g3-dirty-field/exit_status b/conformance/testdata/g3-dirty-field/exit_status similarity index 100% rename from examples/folio/gates-testdata/g3-dirty-field/exit_status rename to conformance/testdata/g3-dirty-field/exit_status diff --git a/examples/folio/gates-testdata/g3-dirty-field/output.log b/conformance/testdata/g3-dirty-field/output.log similarity index 100% rename from examples/folio/gates-testdata/g3-dirty-field/output.log rename to conformance/testdata/g3-dirty-field/output.log diff --git a/examples/folio/gates-testdata/g3-dirty-field/trace.jsonl b/conformance/testdata/g3-dirty-field/trace.jsonl similarity index 100% rename from examples/folio/gates-testdata/g3-dirty-field/trace.jsonl rename to conformance/testdata/g3-dirty-field/trace.jsonl diff --git a/examples/folio/gates-testdata/g4-doubled-text/exit_status b/conformance/testdata/g4-doubled-text/exit_status similarity index 100% rename from examples/folio/gates-testdata/g4-doubled-text/exit_status rename to conformance/testdata/g4-doubled-text/exit_status diff --git a/examples/folio/gates-testdata/g4-doubled-text/output.log b/conformance/testdata/g4-doubled-text/output.log similarity index 100% rename from examples/folio/gates-testdata/g4-doubled-text/output.log rename to conformance/testdata/g4-doubled-text/output.log diff --git a/examples/folio/gates-testdata/g4-doubled-text/trace.jsonl b/conformance/testdata/g4-doubled-text/trace.jsonl similarity index 100% rename from examples/folio/gates-testdata/g4-doubled-text/trace.jsonl rename to conformance/testdata/g4-doubled-text/trace.jsonl diff --git a/examples/folio/gates-testdata/g5-slow-p95/exit_status b/conformance/testdata/g5-slow-p95/exit_status similarity index 100% rename from examples/folio/gates-testdata/g5-slow-p95/exit_status rename to conformance/testdata/g5-slow-p95/exit_status diff --git a/examples/folio/gates-testdata/g5-slow-p95/output.log b/conformance/testdata/g5-slow-p95/output.log similarity index 100% rename from examples/folio/gates-testdata/g5-slow-p95/output.log rename to conformance/testdata/g5-slow-p95/output.log diff --git a/examples/folio/gates-testdata/g5-slow-p95/trace.jsonl b/conformance/testdata/g5-slow-p95/trace.jsonl similarity index 100% rename from examples/folio/gates-testdata/g5-slow-p95/trace.jsonl rename to conformance/testdata/g5-slow-p95/trace.jsonl diff --git a/examples/folio/gates-testdata/generate.py b/conformance/testdata/generate.py similarity index 100% rename from examples/folio/gates-testdata/generate.py rename to conformance/testdata/generate.py diff --git a/examples/folio/gates-testdata/pass/exit_status b/conformance/testdata/pass/exit_status similarity index 100% rename from examples/folio/gates-testdata/pass/exit_status rename to conformance/testdata/pass/exit_status diff --git a/examples/folio/gates-testdata/pass/output.log b/conformance/testdata/pass/output.log similarity index 100% rename from examples/folio/gates-testdata/pass/output.log rename to conformance/testdata/pass/output.log diff --git a/examples/folio/gates-testdata/pass/trace.jsonl b/conformance/testdata/pass/trace.jsonl similarity index 100% rename from examples/folio/gates-testdata/pass/trace.jsonl rename to conformance/testdata/pass/trace.jsonl