From 9407b20914119d4724983e2fe65f11d892c4b7ae Mon Sep 17 00:00:00 2001 From: Jason Ross Date: Wed, 8 Jul 2026 17:04:54 -0500 Subject: [PATCH] ci(e2e): restore wedge-capture diagnostics on the matrix E2E legs (#421) Mirror the e2e-preview job's proven wedge-capture onto the API 29-36 matrix legs, now that #454 replaced reactivecircus/android-emulator-runner with the same hand-provisioned manual boot e2e-preview uses. Each leg now wraps its `./gradlew connectedDebugAndroidTest` in the #404 `timeout -k 30s $WEDGE_TIMEOUT` wrapper; on a hang (exit 124) capture_wedge dumps the smoking gun (running test via TestRunner logcat, SIGQUIT thread dumps of the app + instrumentation processes, service list / service check, dumpsys activity+window, logcat tail) into a per-API-level wedge-diagnostics-api artifact (if-no-files-found: ignore so healthy runs stay quiet). Why it cannot re-hang the legs (the #406/90dfb18 revert reason): the wrapper wraps ONLY the foreground gradle client, never the backgrounded emulator -- identical in shape to e2e-preview's run_shard. The reverted #404 wrapper wrapped reactivecircus's emulator boot; #454 removed that. WEDGE_TIMEOUT reuses preview's 1200s: healthy matrix legs run ~8.3-12.0 min (whole job), well under 20 min, which is itself well under this job's 50-min cap, so a genuine wedge is caught + captured and a false trip on a healthy run is not possible. --- .github/workflows/ci.yml | 100 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 94 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11635d8..a03b1ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -550,16 +550,26 @@ jobs: # `adb shell input keyevent 82 || true` unlock (killing the race); and a readiness gate # (boot_completed==1) BEFORE connectedDebugAndroidTest runs. Emulator flags mirror e2e-preview # (cold `-no-snapshot -accel on -gpu swiftshader_indirect`, plus `-verbose -debug ...` for boot - # diagnosability). NO #404 wedge-capture wrapper here — it was reverted from this matrix job for - # hanging all 8 legs (90dfb18); only the API-37 e2e-preview job keeps it. The gradle run retries - # ONCE on a TEST failure (parity with the replaced retry step); boot flakes are covered by the - # 2-attempt boot loop above. + # diagnosability). The gradle run is wrapped in the #404 wedge-capture `timeout` — RESTORED to + # this matrix by #421. It was reverted in #406 (90dfb18) for hanging all 8 legs, but that + # wrapper wrapped reactivecircus/android-emulator-runner's emulator boot; #454 replaced that + # with this manual boot, so the wrapper now wraps ONLY the foreground gradle client — never the + # backgrounded emulator — exactly like e2e-preview's proven run_shard, and so cannot re-hang the + # leg. See capture_wedge / run_e2e below. The gradle run retries ONCE on a TEST failure (parity + # with the replaced retry step); boot flakes are covered by the 2-attempt boot loop above. - name: Boot emulator and run E2E run: | set -euo pipefail EMU_LOG="${RUNNER_TEMP:-/tmp}/emulator-api${{ matrix.api-level }}.log" LOGCAT_LOG="${RUNNER_TEMP:-/tmp}/logcat-api${{ matrix.api-level }}.txt" DIAG_LOG="${RUNNER_TEMP:-/tmp}/boot-diagnostics-api${{ matrix.api-level }}.txt" + # Wedge (hang) smoking-gun capture (#404, restored to the matrix by #421) — see + # capture_wedge / run_e2e below. WEDGE_TIMEOUT mirrors e2e-preview's 1200s: a healthy + # matrix leg has run the whole job in ~8.3–12.0 min historically and the wrapper only wraps + # the ./gradlew invocation (a subset of that), so 20 min is generous no-false-trip headroom + # yet well under this job's 50-min cap so a genuine wedge is caught and captured. + WEDGE_LOG="${RUNNER_TEMP:-/tmp}/wedge-diagnostics-api${{ matrix.api-level }}.txt" + WEDGE_TIMEOUT=1200 GPU_MODE="swiftshader_indirect" # On a boot timeout, dump full system state into $DIAG_LOG for the artifact upload, then a @@ -629,10 +639,75 @@ jobs: adb shell settings put global transition_animation_scale 0.0 || true adb shell settings put global animator_duration_scale 0.0 || true + # WEDGE (hang) smoking-gun capture (#404, restored to the matrix by #421). On the wrapper + # `timeout` below (exit 124), grab the smoking gun WHILE this hand-provisioned emulator is + # still alive (it stays up for the rest of this step): which test was running, SIGQUIT + # (kill -3) thread dumps of the app + instrumentation processes (ART -> logcat + /data/anr), + # activity/window state, and — the boot-race crux — whether the binder services are + # published. Every probe guarded so a missing tool / dead device can't abort it; appended so + # both attempts survive. `|| true` keeps it from tripping this step's `set -e`. Identical in + # shape to e2e-preview's capture_wedge. + capture_wedge() { + { + echo "===================================================================" + echo "===== E2E WEDGE — API ${{ matrix.api-level }} — $1 =====" + echo "===== $(date -u +%FT%TZ) — after ${WEDGE_TIMEOUT}s wrapper timeout =====" + echo "===================================================================" + echo "--- snapshot: N/A — matrix cold-boots (-no-snapshot); no AVD snapshot cache ---" + echo "--- running/last instrumented test (logcat TestRunner) ---" + grep -a TestRunner "$LOGCAT_LOG" 2>/dev/null | tail -25 || true + echo "--- getprop sys.boot_completed ---" + adb shell getprop sys.boot_completed 2>&1 || true + echo "--- getprop init.svc.* (per-service init state) ---" + adb shell getprop 2>&1 | grep -a init.svc || true + echo "--- service list (are binder services published?) ---" + adb shell service list 2>&1 || true + for svc in input window activity; do + echo "--- service check $svc ---" + adb shell service check "$svc" 2>&1 || true + done + echo "--- pids ---" + APP_PID="$(adb shell pidof org.libremail.app 2>/dev/null | tr -d '\r')" || true + TEST_PID="$(adb shell pidof org.libremail.app.test 2>/dev/null | tr -d '\r')" || true + echo "app pid: ${APP_PID:-}" + echo "test pid: ${TEST_PID:-}" + echo "--- SIGQUIT (kill -3) thread dumps -> ART writes to logcat + /data/anr ---" + for pid in $APP_PID $TEST_PID; do + [ -n "$pid" ] && adb shell kill -3 "$pid" 2>&1 || true + done + sleep 5 + echo "--- /data/anr/* (SIGQUIT + ANR traces) ---" + adb shell 'cat /data/anr/* 2>/dev/null' 2>&1 || true + echo "--- dumpsys activity ---" + adb shell dumpsys activity 2>&1 || true + echo "--- dumpsys window ---" + adb shell dumpsys window 2>&1 || true + echo "--- logcat -d (tail 400 — includes the SIGQUIT thread dump) ---" + adb logcat -d 2>&1 | tail -400 || true + echo "--- emulator accel / kvm / mem / disk ---" + "$ANDROID_SDK_ROOT/emulator/emulator" -accel-check 2>&1 || true + ls -l /dev/kvm 2>&1 || true + free -h 2>&1 || true + df -h 2>&1 || true + } >> "$WEDGE_LOG" 2>&1 || true + echo "::warning::E2E API ${{ matrix.api-level }} WEDGED ($1) — see the wedge-diagnostics-api${{ matrix.api-level }} artifact" + } + # Retry the test run ONCE on failure (parity with the replaced retry step). A retried-but- # passed run is flagged as a ::warning:: because a blanket retry can mask a real regression. - run_e2e() { ./gradlew connectedDebugAndroidTest --stacktrace; } - run_e2e || { echo "::warning::API ${{ matrix.api-level }} E2E run failed — retrying once"; run_e2e; } + # The gradle run is wrapped in the #404 wedge wrapper `timeout`: a wedge (hang) trips it + # (exit 124) -> capture_wedge runs while the emulator is still alive, then the run returns + # 124 so the retry / gate still see a failure. `|| status=$?` makes the exit code survive + # this step's `set -e`; -k 30s SIGKILLs a gradle client that ignores SIGTERM. The wrapper + # wraps ONLY the foreground gradle client (NOT the backgrounded emulator), identical in + # shape to e2e-preview's run_shard — the structural reason it cannot re-hang the leg. + run_e2e() { + local status=0 + timeout -k 30s "$WEDGE_TIMEOUT" ./gradlew connectedDebugAndroidTest --stacktrace || status=$? + if [ "$status" -eq 124 ]; then capture_wedge "$1"; fi + return "$status" + } + run_e2e "attempt 1" || { echo "::warning::API ${{ matrix.api-level }} E2E run failed — retrying once"; run_e2e "attempt 2 (retry)"; } # On any E2E failure (both boot attempts failed, a hung emulator, or an earlier setup/SDK # step), snapshot device + runner state to the step log AND a file for the artifact upload — @@ -677,6 +752,19 @@ jobs: ${{ runner.temp }}/boot-diagnostics-api${{ matrix.api-level }}.txt if-no-files-found: warn + # Wedge-specific smoking gun (#404, restored to the matrix by #421): only present when the + # wrapper `timeout` tripped (a hang) on either attempt — separate from the E2E diagnostics + # artifact above. `if-no-files-found: ignore` keeps healthy runs quiet (no wedge => no file). + # Per-api-level name (upload-artifact@v7 rejects duplicate artifact names). Mirrors the + # e2e-preview job's wedge-diagnostics artifact. + - name: Upload wedge diagnostics + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wedge-diagnostics-api${{ matrix.api-level }} + path: ${{ runner.temp }}/wedge-diagnostics-api${{ matrix.api-level }}.txt + if-no-files-found: ignore + # API 37 (Android 17, preview) E2E. Its only system image is the nonstandard # android-37.0 / google_apis_ps16k (16 KB page size), which reactivecircus/android-emulator-runner # can't provision (it builds android-37 / google_apis, neither of which exists), so this job -- 2.47.3