#!/usr/bin/env bash # Scripted conformance gate for the mobile drivers. Runs five serial, # non-overlapping 3-minute fuzz runs against the folio example app # (examples/folio) and scores five gates (G1..G5) over the captured traces and # output. Exits non-zero if any gate fails. # # Backends: # BACKEND=simulator (default) drive the booted iOS simulator # BACKEND=device drive an attached physical iPhone via the # driver's runner-only device path; select it # with IOS_DEVICE="" (passed as --ios-device) # BACKEND=android drive an Android device/emulator over the JVM # sidecar; select a specific device with # ANDROID_DEVICE="" (passed as --device) # # Usage: # ./gates.sh run the simulator gates # BACKEND=device IOS_DEVICE="iPhone" ./gates.sh # BACKEND=android ANDROID_DEVICE="663c91b1" ./gates.sh # RUNS=1 SEEDS=303 ./gates.sh re-run just the seed that failed # ./gates.sh --self-test run the offline analyzer tests only # # Tunables (environment): # RUNS=5 number of serial runs # DURATION=3m per-run fuzz duration # SEEDS="101 202 303 404 505" # one fuzz seed per run, whitespace separated, consumed in # order. The values differ so the runs explore different # action streams, and they are fixed so a failing gate can # be re-run exactly; the results table prints the seed each # run used. A 0 is rejected: sanderling test reads --seed 0 # as "derive the seed from the clock", which is the one # thing a gate cannot reproduce. # P95_LIMIT_MS G5 p95 step-latency ceiling in ms (default 2500 for iOS, # 5500 for the android backend's higher and more variable # per-step USB cost, especially cold right after a reboot) # SANDERLING=sanderling binary to invoke set -euo pipefail script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" folio_directory="$(cd "${script_directory}/../examples/folio" && pwd)" BACKEND="${BACKEND:-simulator}" RUNS="${RUNS:-5}" DURATION="${DURATION:-3m}" SEEDS="${SEEDS:-101 202 303 404 505}" # The p95 ceiling is backend-specific: a physical Android device drives every # step over USB (snapshot + settle + adb round-trips), so its per-step floor is # several times the iOS simulator's in-process cost. 2500ms was calibrated on # the simulator; holding a physical device to it would force ripping out the # settle/retry logic the correctness gates depend on. Override with P95_LIMIT_MS. if [[ "$BACKEND" == "android" ]]; then P95_LIMIT_MS="${P95_LIMIT_MS:-5500}" else P95_LIMIT_MS="${P95_LIMIT_MS:-2500}" fi SANDERLING="${SANDERLING:-sanderling}" IOS_DEVICE="${IOS_DEVICE:-iPhone 17 Pro}" ANDROID_DEVICE="${ANDROID_DEVICE:-}" if [[ -n "${SEED:-}" ]]; then echo "gates.sh: SEED is not read any more; set SEEDS to one seed per run" >&2 exit 2 fi bundle_id="app.folio" spec_path="${folio_directory}/sanderling/spec.ts" android_apk="${folio_directory}/app/androidApp/build/outputs/apk/debug/androidApp-debug.apk" # The built app bundle differs by SDK: the simulator build lands under # Debug-iphonesimulator, the device build under Debug-iphoneos. if [[ "$BACKEND" == "device" ]]; then ios_app="${folio_directory}/app/iosApp/build/Build/Products/Debug-iphoneos/iosApp.app" else ios_app="${folio_directory}/app/iosApp/build/Build/Products/Debug-iphonesimulator/iosApp.app" fi # Run adb against the selected Android device, or the only one if unset. adb_target() { if [[ -n "$ANDROID_DEVICE" ]]; then adb -s "$ANDROID_DEVICE" "$@"; else adb "$@"; fi } # The companion binary, embedded for simulator runs. Referenced by file name # only for the orphan-process check; prose elsewhere says "the companion". companion_process_name="idb_companion" # Known-benign stderr lines the companion always prints. These are matched as # fixed substrings and excluded from the G2 ERROR scan. Keep this list tight: # only lines that are provably harmless and emitted on every healthy run. benign_stderr_substrings=( # The dynamic linker reports the same Objective-C class registered by two # loaded images. The companion runs fine; this is cosmetic. The real line # reads "objc[]: Class ...", so the fixed substring is "]: Class". "]: Class" "is implemented in both" "One of the two will be used. Which one is undefined." # gRPC and absl emit informational banner lines on startup. "WARNING: All log messages before absl::InitializeLog()" ) # ---- gate analyzers (pure, operate on a single run directory) -------------- # G2 helper: strip benign companion noise, then report sanderling ERROR lines. # sanderling's progress logger renders error-level records as lines beginning # "error:" (see internal/testrun/progress.go). We also catch the upper-case # ERROR token (word-bounded, so ERRORS/ERRORLESS and path fragments do not # trip the gate) for safety against future handlers. error_lines_in() { local output_file="$1" local filtered filtered="$(cat "$output_file")" local pattern for pattern in "${benign_stderr_substrings[@]}"; do filtered="$(printf '%s\n' "$filtered" | grep -vF "$pattern" || true)" done printf '%s\n' "$filtered" | grep -E '(^error:|\bERROR\b)' || true } # G1: process exit status recorded by the runner loop. gate_exit_zero() { local run_directory="$1" [[ "$(cat "${run_directory}/exit_status")" == "0" ]] } # G2: no sanderling ERROR lines after filtering benign companion noise. gate_no_error_lines() { local run_directory="$1" local found found="$(error_lines_in "${run_directory}/output.log")" [[ -z "$found" ]] } # G3: the first hierarchy snapshot shows the login screen with empty email and # password fields (clear-state proof). Reads the first trace line carrying a # hierarchy and asserts LoginEmail/LoginPassword carry no text. gate_clear_state() { local run_directory="$1" local trace_file="${run_directory}/trace.jsonl" [[ -f "$trace_file" ]] || return 1 local verdict verdict="$(jq -s -r ' [ .[] | select(.hierarchy != null) ] as $withHierarchy | if ($withHierarchy | length) == 0 then "fail:no-hierarchy" else ($withHierarchy[0].hierarchy.elements // []) as $elements | ($elements | map(select(.resourceId == "LoginEmail")) | first) as $email | ($elements | map(select(.resourceId == "LoginPassword")) | first) as $password | if $email == null or $password == null then "fail:no-login-fields" elif (($email.attrs.text // $email.text // "") != "") then "fail:email-not-empty" elif (($password.attrs.text // $password.text // "") != "") then "fail:password-not-empty" else "pass" end end ' "$trace_file")" [[ "$verdict" == "pass" ]] } # G4: no doubled text after InputText. For each InputText action targeting a # field, the field's value in the NEXT hierarchy must not read as the input # applied twice (catches append-vs-replace and double-paste bugs). The action # chosen at step N is applied before step N+1 is observed, so the effect lands # in the following snapshot. # # Two signals, because the typed value is not always in the trace: a target the # platform reports no secure fact for has its typed value written as a fixed # placeholder (internal/verifier/redaction.go), and android reports that fact # for nothing, so on that backend every InputText records the placeholder. The # OBSERVED value is never redacted, and a field holding one string twice over is # the doubling itself. A value that is a single character repeated is exempt: # the input corpus types "a" 4096 times and a pair of spaces, and neither can be # told apart from its own doubling. gate_no_doubled_text() { local run_directory="$1" local trace_file="${run_directory}/trace.jsonl" [[ -f "$trace_file" ]] || return 1 local verdict verdict="$(jq -s -r --arg redacted "[redacted]" ' # Map a selector like "testTag:LoginScreen > testTag:LoginEmail" to its # target field id: the token after the final ":" of the last segment. An # action typed at coordinates carries no selector, and jq splits an empty # string into no segments at all, so the fallbacks keep such a step from # aborting the whole analysis. def target_field(selector): ((selector | split(">") | last // "") | gsub("^\\s+|\\s+$";"")) as $last | ($last | split(":") | last // ""); def self_doubled(value): (value | length) as $n | ($n / 2 | floor) as $half | $n > 1 and ($n % 2 == 0) and (value[0:$half] == value[$half:]) and ((value | explode | unique | length) > 1); [ .[] | select(.hierarchy != null) ] as $steps | reduce range(0; ($steps | length)) as $i ([]; ($steps[$i]) as $current | (if $i + 1 < ($steps | length) then $steps[$i + 1] else null end) as $next | if ($current.next_action.kind == "InputText") and ($next != null) then (target_field($current.next_action.selector // "")) as $field | ($current.next_action.text // "") as $typed | (($next.hierarchy.elements // []) | map(select(.resourceId == $field)) | first) as $element | if $element != null then (($element.attrs.text // $element.text // "")) as $value | if self_doubled($value) or ($typed != "" and $typed != $redacted and ($value | contains($typed + $typed))) then . + [{field: $field, value: $value}] else . end else . end else . end) | if length == 0 then "pass" else "fail:" + (.[0].field) + ":" + (.[0].value[0:60]) end ' "$trace_file")" [[ "$verdict" == "pass" ]] } # Emit one step-latency sample per consecutive trace-step pair, in # milliseconds, on stdout. The trace records a wall-clock timestamp per step; # the latency of a step is the gap to the next step's observation. Used by G5, # which aggregates samples across all runs before computing the p95. Timestamps # are RFC3339 with fractional seconds and a numeric offset, so they are parsed # with python3 rather than jq's UTC-only fromdateiso8601. emit_step_latencies() { local trace_file="$1" [[ -f "$trace_file" ]] || return 0 jq -r 'select(.timestamp != null) | .timestamp' "$trace_file" \ | python3 -c ' import sys from datetime import datetime stamps = [datetime.fromisoformat(line.strip()) for line in sys.stdin if line.strip()] for earlier, later in zip(stamps, stamps[1:]): print(int((later - earlier).total_seconds() * 1000)) ' } # Compute the p95 (nearest-rank) of the latency samples on stdin, in ms. p95_of() { python3 -c ' import sys, math samples = sorted(int(float(line)) for line in sys.stdin if line.strip()) if not samples: print(0) sys.exit(0) rank = max(1, math.ceil(0.95 * len(samples))) print(samples[rank - 1]) ' } # G5 orphan check: report any lingering companion, runner session (the hybrid # simulator driver hosts an in-simulator runner), and, on the device backend, # the device runner session. The usbmux tunnel is an in-process forwarder that # dies with sanderling, so it leaves no process to check. Empty output is clean. orphan_processes() { local found="" if [[ "$BACKEND" == "android" ]]; then # sanderling SIGTERMs the JVM sidecar on shutdown; a survivor is an orphan. if pgrep -f "sanderling-sidecar.*\.jar" >/dev/null 2>&1; then found+="sidecar "; fi printf '%s' "$found" return fi if pgrep -f "$companion_process_name" >/dev/null 2>&1; then found+="companion " fi if pgrep -f "sanderling-runner.*xctestrun" >/dev/null 2>&1; then found+="runner-session " fi if pgrep -f "CompanionRunnerUITests-Runner" >/dev/null 2>&1; then found+="runner-app " fi if [[ "$BACKEND" == "device" ]]; then # The device test session that hosts the runner. Its destination carries # platform=iOS,id=. if pgrep -f "xctestrun.*platform=iOS,id=" >/dev/null 2>&1; then found+="device-session " fi fi printf '%s' "$found" } # ---- run orchestration ----------------------------------------------------- # Split a seed list into the global seed_list, one entry per requested run. # sanderling test reads --seed 0 as "derive the seed from the clock", so a 0 # here would make that run unreproducible and is refused. seed_list=() select_seeds() { local requested="$1" local -a candidates=() read -r -a candidates <<<"$2" if [[ "${#candidates[@]}" -lt "$requested" ]]; then echo "gates.sh: SEEDS has ${#candidates[@]} value(s) but RUNS=${requested}; give one seed per run" >&2 return 1 fi local index seed for ((index = 0; index < ${#candidates[@]}; index++)); do seed="${candidates[index]}" case "$seed" in '' | *[!0-9]*) echo "gates.sh: SEEDS takes whitespace-separated positive integers; got \"${seed}\"" >&2 return 1 ;; esac if [[ "$((10#$seed))" -eq 0 ]]; then echo "gates.sh: SEEDS may not contain 0; sanderling test derives a clock seed from 0, so a gate configured with it cannot be reproduced" >&2 return 1 fi done seed_list=("${candidates[@]:0:requested}") } invoke_sanderling() { local output_directory="$1" local output_log="$2" local exit_status_file="$3" local seed="$4" local platform target_flags=() if [[ "$BACKEND" == "android" ]]; then # pm clear is blocked on some OEM ROMs, so a fresh install (which wipes # /data/data) provides the clean clear-state start; --clear-data=false # then skips the sidecar's pm clear. Mirrors the iOS per-run reinstall. platform=android adb_target uninstall "$bundle_id" >/dev/null 2>&1 || true # A transient install hiccup must score this run as a failure, not abort the # whole harness under `set -e` and discard the other runs' data. if ! adb_target install "$android_apk" >"$output_log" 2>&1; then echo "adb install failed for ${android_apk}; recording run as a failure" >>"$output_log" printf '1' >"$exit_status_file" return fi target_flags=(--clear-data=false) [[ -n "$ANDROID_DEVICE" ]] && target_flags+=(--device "$ANDROID_DEVICE") else # The iOS backends pass --ios-app-path so each run reinstalls the current # build for a clean start (device via devicectl, simulator via simctl). platform=ios target_flags=(--ios-device "$IOS_DEVICE" --ios-app-path "$ios_app") fi local status=0 "$SANDERLING" test \ --platform "$platform" \ --spec "$spec_path" \ --bundle-id "$bundle_id" \ "${target_flags[@]}" \ --duration "$DURATION" \ --seed "$seed" \ --output "$output_directory" \ >"$output_log" 2>&1 || status=$? printf '%s' "$status" >"$exit_status_file" } # Locate the run directory sanderling created under output_directory (it nests # a timestamped subdirectory) and normalise its artifacts up one level. collect_run_artifacts() { local output_directory="$1" [[ -f "${output_directory}/trace.jsonl" ]] && return 0 local produced produced="$(find "$output_directory" -mindepth 2 -name 'trace.jsonl' 2>/dev/null | head -1 || true)" if [[ -n "$produced" ]]; then cp "$produced" "${output_directory}/trace.jsonl" 2>/dev/null || true fi } run_gates() { select_seeds "$RUNS" "$SEEDS" || exit 2 if [[ "$BACKEND" == "android" ]]; then # Build only; invoke_sanderling reinstalls per run via adb (gradle's ddmlib # install is flaky on some physical devices). echo "preparing folio android build" ( cd "$folio_directory" && just build >/dev/null ) # A physical device, unlike an emulator, lets system UI steal the foreground # from the app the fuzzer is exploring. Keep the screen on so it never # re-locks, silence the autofill save-password prompt that pops over the # login form, and stop Play Protect from intercepting the per-run reinstall. # The device must already be unlocked (a secure lock cannot be opened here). adb_target shell svc power stayon true >/dev/null 2>&1 || true adb_target shell settings put secure autofill_service null >/dev/null 2>&1 || true adb_target shell settings put global verifier_verify_adb_installs 0 >/dev/null 2>&1 || true elif [[ "$BACKEND" == "simulator" ]]; then echo "preparing folio build for the simulator backend" ( cd "$folio_directory" && just ios >/dev/null ) else echo "preparing folio device build" ( cd "$folio_directory" && just ios-device >/dev/null ) fi local timestamp timestamp="$(date +%Y%m%d-%H%M%S)" local gate_root="${script_directory}/runs/${timestamp}" mkdir -p "$gate_root" echo "gate root: ${gate_root}" echo "backend=${BACKEND} runs=${RUNS} duration=${DURATION} seeds=${seed_list[*]} p95_limit_ms=${P95_LIMIT_MS}" local all_latencies="${gate_root}/all-latencies.txt" : >"$all_latencies" local -a g1 g2 g3 g4 g5 local run_index for ((run_index = 1; run_index <= RUNS; run_index++)); do local run_directory="${gate_root}/run-${run_index}" local seed="${seed_list[run_index - 1]}" mkdir -p "$run_directory" printf '%s' "$seed" >"${run_directory}/seed" echo "run ${run_index}/${RUNS} seed=${seed} -> ${run_directory}" invoke_sanderling "$run_directory" "${run_directory}/output.log" "${run_directory}/exit_status" "$seed" collect_run_artifacts "$run_directory" g1[run_index]=$(gate_exit_zero "$run_directory" && echo PASS || echo FAIL) g2[run_index]=$(gate_no_error_lines "$run_directory" && echo PASS || echo FAIL) g3[run_index]=$(gate_clear_state "$run_directory" && echo PASS || echo FAIL) g4[run_index]=$(gate_no_doubled_text "$run_directory" && echo PASS || echo FAIL) emit_step_latencies "${run_directory}/trace.jsonl" >>"$all_latencies" local orphans orphans="$(orphan_processes)" if [[ -n "$orphans" ]]; then g5[run_index]="FAIL" echo " orphaned processes after run ${run_index}: ${orphans}" else g5[run_index]="PENDING" fi done local p95 p95="$(p95_of <"$all_latencies")" local final_orphans final_orphans="$(orphan_processes)" # G5 is global: p95 is computed over every run's samples and an orphan after # any single run fails the whole gate. Decide once, then stamp every row. local g5_global="PASS" if [[ "$p95" -ge "$P95_LIMIT_MS" ]]; then g5_global="FAIL" fi if [[ -n "$final_orphans" ]]; then g5_global="FAIL" echo "orphaned processes at end: ${final_orphans}" fi for ((run_index = 1; run_index <= RUNS; run_index++)); do if [[ "${g5[run_index]}" == "FAIL" ]]; then g5_global="FAIL" fi done for ((run_index = 1; run_index <= RUNS; run_index++)); do g5[run_index]="$g5_global" done echo printf 'run seed G1 G2 G3 G4 G5\n' local verdict="PASS" for ((run_index = 1; run_index <= RUNS; run_index++)); do printf '%-4s %-6s %-5s %-5s %-5s %-5s %-5s\n' \ "$run_index" "${seed_list[run_index - 1]}" \ "${g1[run_index]}" "${g2[run_index]}" "${g3[run_index]}" "${g4[run_index]}" "${g5[run_index]}" for cell in "${g1[run_index]}" "${g2[run_index]}" "${g3[run_index]}" "${g4[run_index]}" "${g5[run_index]}"; do [[ "$cell" == "PASS" ]] || verdict="FAIL" done done echo echo "p95 step latency: ${p95}ms (limit ${P95_LIMIT_MS}ms)" if [[ "$verdict" == "PASS" ]]; then echo "GATES PASS" return 0 fi echo "GATES FAIL" echo "reproduce one run with: RUNS=1 SEEDS= BACKEND=${BACKEND} $0" return 1 } # ---- offline self-test ----------------------------------------------------- # Exercises every analyzer against canned passing/failing run directories under # testdata/ so the parsing logic can be checked without a device. self_test() { local testdata="${script_directory}/testdata" local failures=0 # The self-test fixtures (g5-slow-p95 = 4000ms) were calibrated against the # 2500ms ceiling, so pin it here. Without this the backend-dependent default # (5500ms under BACKEND=android) would rate the slow fixture as a PASS and the # offline, device-free analyzer check would fail purely from an env var. local P95_LIMIT_MS=2500 assert() { local label="$1" expected="$2" actual="$3" if [[ "$expected" == "$actual" ]]; then printf 'ok %s\n' "$label" else printf 'FAIL %s (expected %s, got %s)\n' "$label" "$expected" "$actual" failures=$((failures + 1)) fi } assert "G1 pass run exits zero" PASS \ "$(gate_exit_zero "${testdata}/pass" && echo PASS || echo FAIL)" assert "G1 fail run nonzero exit" FAIL \ "$(gate_exit_zero "${testdata}/g1-nonzero-exit" && echo PASS || echo FAIL)" assert "G2 pass run no errors" PASS \ "$(gate_no_error_lines "${testdata}/pass" && echo PASS || echo FAIL)" assert "G2 benign noise tolerated" PASS \ "$(gate_no_error_lines "${testdata}/g2-benign-only" && echo PASS || echo FAIL)" assert "G2 real error caught" FAIL \ "$(gate_no_error_lines "${testdata}/g2-real-error" && echo PASS || echo FAIL)" assert "G3 pass clear state" PASS \ "$(gate_clear_state "${testdata}/pass" && echo PASS || echo FAIL)" assert "G3 dirty fields caught" FAIL \ "$(gate_clear_state "${testdata}/g3-dirty-field" && echo PASS || echo FAIL)" assert "G4 pass no doubling" PASS \ "$(gate_no_doubled_text "${testdata}/pass" && echo PASS || echo FAIL)" assert "G4 doubled text caught with the typed value redacted" FAIL \ "$(gate_no_doubled_text "${testdata}/g4-doubled-text" && echo PASS || echo FAIL)" assert "G4 repeated character is not doubling" PASS \ "$(gate_no_doubled_text "${testdata}/g4-repeated-character" && echo PASS || echo FAIL)" assert "G4 doubled text caught from the recorded value" FAIL \ "$(gate_no_doubled_text "${testdata}/g4-recorded-text" && echo PASS || echo FAIL)" assert "G4 input without a selector does not abort the gate" PASS \ "$(gate_no_doubled_text "${testdata}/g4-selectorless-input" && echo PASS || echo FAIL)" assert "seeds default to one distinct value per run" "101 202 303 404 505" \ "$(select_seeds 5 "101 202 303 404 505" >/dev/null 2>&1 && echo "${seed_list[*]}")" assert "seeds override honoured, extras ignored" "11 22" \ "$(select_seeds 2 "11 22 33" >/dev/null 2>&1 && echo "${seed_list[*]}")" assert "seed 0 rejected" FAIL \ "$(select_seeds 1 "0" >/dev/null 2>&1 && echo PASS || echo FAIL)" assert "padded seed 00 rejected" FAIL \ "$(select_seeds 1 "00" >/dev/null 2>&1 && echo PASS || echo FAIL)" assert "non-numeric seed rejected" FAIL \ "$(select_seeds 1 "abc" >/dev/null 2>&1 && echo PASS || echo FAIL)" assert "seed list shorter than RUNS rejected" FAIL \ "$(select_seeds 5 "101 202" >/dev/null 2>&1 && echo PASS || echo FAIL)" local pass_p95 slow_p95 pass_p95="$(emit_step_latencies "${testdata}/pass/trace.jsonl" | p95_of)" slow_p95="$(emit_step_latencies "${testdata}/g5-slow-p95/trace.jsonl" | p95_of)" assert "G5 pass p95 under limit" PASS \ "$([[ "$pass_p95" -lt "$P95_LIMIT_MS" ]] && echo PASS || echo FAIL)" assert "G5 slow p95 over limit" FAIL \ "$([[ "$slow_p95" -lt "$P95_LIMIT_MS" ]] && echo PASS || echo FAIL)" echo if [[ "$failures" -eq 0 ]]; then echo "SELF-TEST PASS" return 0 fi echo "SELF-TEST FAIL (${failures} failed)" return 1 } main() { case "${1:-}" in --self-test) self_test ;; "") run_gates ;; *) echo "usage: $0 [--self-test]" >&2; exit 2 ;; esac } main "$@"