From 194866511fbdb9e8471b6da90ef67ce97a0bf1a9 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 11 Jun 2026 09:24:40 +0530 Subject: [PATCH] fix(conformance): pin self-test p95 budget and score install failures as run failures self_test reused the backend-dependent P95_LIMIT_MS, so under BACKEND=android the 4000ms slow fixture rated PASS and the offline analyzer check failed from an env var; pin it to 2500. A per-run adb install failure ran unguarded under set -e and aborted the whole harness; guard it, record the run as a G1 failure, and continue. --- conformance/gates.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/conformance/gates.sh b/conformance/gates.sh index c3de231..10626b6 100755 --- a/conformance/gates.sh +++ b/conformance/gates.sh @@ -258,7 +258,13 @@ invoke_sanderling() { # then skips the sidecar's pm clear. Mirrors the iOS per-run reinstall. platform=android adb_target uninstall "$bundle_id" >/dev/null 2>&1 || true - adb_target install "$android_apk" >/dev/null + # A transient install hiccup must score this run as a failure, not abort the + # whole harness under `set -e` and discard the other runs' data. + if ! adb_target install "$android_apk" >"$output_log" 2>&1; then + echo "adb install failed for ${android_apk}; recording run as a failure" >>"$output_log" + printf '1' >"$exit_status_file" + return + fi target_flags=(--clear-data=false) [[ -n "$ANDROID_DEVICE" ]] && target_flags+=(--device "$ANDROID_DEVICE") else @@ -403,6 +409,11 @@ run_gates() { self_test() { local testdata="${script_directory}/testdata" local failures=0 + # The self-test fixtures (g5-slow-p95 = 4000ms) were calibrated against the + # 2500ms ceiling, so pin it here. Without this the backend-dependent default + # (5500ms under BACKEND=android) would rate the slow fixture as a PASS and the + # offline, device-free analyzer check would fail purely from an env var. + local P95_LIMIT_MS=2500 assert() { local label="$1" expected="$2" actual="$3"