Merge pull request #464 from JMR-dev/ci-421-matrix-wedge-capture
ci(e2e): restore wedge-capture diagnostics on the matrix E2E legs (#421)
This commit was merged in pull request #464.
This commit is contained in:
mergify[bot]
@@ -550,16 +550,26 @@ jobs:
|
||||
# `adb shell input keyevent 82 || true` unlock (killing the race); and a readiness gate
|
||||
# (boot_completed==1) BEFORE connectedDebugAndroidTest runs. Emulator flags mirror e2e-preview
|
||||
# (cold `-no-snapshot -accel on -gpu swiftshader_indirect`, plus `-verbose -debug ...` for boot
|
||||
# diagnosability). NO #404 wedge-capture wrapper here — it was reverted from this matrix job for
|
||||
# hanging all 8 legs (90dfb18); only the API-37 e2e-preview job keeps it. The gradle run retries
|
||||
# ONCE on a TEST failure (parity with the replaced retry step); boot flakes are covered by the
|
||||
# 2-attempt boot loop above.
|
||||
# diagnosability). The gradle run is wrapped in the #404 wedge-capture `timeout` — RESTORED to
|
||||
# this matrix by #421. It was reverted in #406 (90dfb18) for hanging all 8 legs, but that
|
||||
# wrapper wrapped reactivecircus/android-emulator-runner's emulator boot; #454 replaced that
|
||||
# with this manual boot, so the wrapper now wraps ONLY the foreground gradle client — never the
|
||||
# backgrounded emulator — exactly like e2e-preview's proven run_shard, and so cannot re-hang the
|
||||
# leg. See capture_wedge / run_e2e below. The gradle run retries ONCE on a TEST failure (parity
|
||||
# with the replaced retry step); boot flakes are covered by the 2-attempt boot loop above.
|
||||
- name: Boot emulator and run E2E
|
||||
run: |
|
||||
set -euo pipefail
|
||||
EMU_LOG="${RUNNER_TEMP:-/tmp}/emulator-api${{ matrix.api-level }}.log"
|
||||
LOGCAT_LOG="${RUNNER_TEMP:-/tmp}/logcat-api${{ matrix.api-level }}.txt"
|
||||
DIAG_LOG="${RUNNER_TEMP:-/tmp}/boot-diagnostics-api${{ matrix.api-level }}.txt"
|
||||
# Wedge (hang) smoking-gun capture (#404, restored to the matrix by #421) — see
|
||||
# capture_wedge / run_e2e below. WEDGE_TIMEOUT mirrors e2e-preview's 1200s: a healthy
|
||||
# matrix leg has run the whole job in ~8.3–12.0 min historically and the wrapper only wraps
|
||||
# the ./gradlew invocation (a subset of that), so 20 min is generous no-false-trip headroom
|
||||
# yet well under this job's 50-min cap so a genuine wedge is caught and captured.
|
||||
WEDGE_LOG="${RUNNER_TEMP:-/tmp}/wedge-diagnostics-api${{ matrix.api-level }}.txt"
|
||||
WEDGE_TIMEOUT=1200
|
||||
GPU_MODE="swiftshader_indirect"
|
||||
|
||||
# On a boot timeout, dump full system state into $DIAG_LOG for the artifact upload, then a
|
||||
@@ -629,10 +639,75 @@ jobs:
|
||||
adb shell settings put global transition_animation_scale 0.0 || true
|
||||
adb shell settings put global animator_duration_scale 0.0 || true
|
||||
|
||||
# WEDGE (hang) smoking-gun capture (#404, restored to the matrix by #421). On the wrapper
|
||||
# `timeout` below (exit 124), grab the smoking gun WHILE this hand-provisioned emulator is
|
||||
# still alive (it stays up for the rest of this step): which test was running, SIGQUIT
|
||||
# (kill -3) thread dumps of the app + instrumentation processes (ART -> logcat + /data/anr),
|
||||
# activity/window state, and — the boot-race crux — whether the binder services are
|
||||
# published. Every probe guarded so a missing tool / dead device can't abort it; appended so
|
||||
# both attempts survive. `|| true` keeps it from tripping this step's `set -e`. Identical in
|
||||
# shape to e2e-preview's capture_wedge.
|
||||
capture_wedge() {
|
||||
{
|
||||
echo "==================================================================="
|
||||
echo "===== E2E WEDGE — API ${{ matrix.api-level }} — $1 ====="
|
||||
echo "===== $(date -u +%FT%TZ) — after ${WEDGE_TIMEOUT}s wrapper timeout ====="
|
||||
echo "==================================================================="
|
||||
echo "--- snapshot: N/A — matrix cold-boots (-no-snapshot); no AVD snapshot cache ---"
|
||||
echo "--- running/last instrumented test (logcat TestRunner) ---"
|
||||
grep -a TestRunner "$LOGCAT_LOG" 2>/dev/null | tail -25 || true
|
||||
echo "--- getprop sys.boot_completed ---"
|
||||
adb shell getprop sys.boot_completed 2>&1 || true
|
||||
echo "--- getprop init.svc.* (per-service init state) ---"
|
||||
adb shell getprop 2>&1 | grep -a init.svc || true
|
||||
echo "--- service list (are binder services published?) ---"
|
||||
adb shell service list 2>&1 || true
|
||||
for svc in input window activity; do
|
||||
echo "--- service check $svc ---"
|
||||
adb shell service check "$svc" 2>&1 || true
|
||||
done
|
||||
echo "--- pids ---"
|
||||
APP_PID="$(adb shell pidof org.libremail.app 2>/dev/null | tr -d '\r')" || true
|
||||
TEST_PID="$(adb shell pidof org.libremail.app.test 2>/dev/null | tr -d '\r')" || true
|
||||
echo "app pid: ${APP_PID:-<none>}"
|
||||
echo "test pid: ${TEST_PID:-<none>}"
|
||||
echo "--- SIGQUIT (kill -3) thread dumps -> ART writes to logcat + /data/anr ---"
|
||||
for pid in $APP_PID $TEST_PID; do
|
||||
[ -n "$pid" ] && adb shell kill -3 "$pid" 2>&1 || true
|
||||
done
|
||||
sleep 5
|
||||
echo "--- /data/anr/* (SIGQUIT + ANR traces) ---"
|
||||
adb shell 'cat /data/anr/* 2>/dev/null' 2>&1 || true
|
||||
echo "--- dumpsys activity ---"
|
||||
adb shell dumpsys activity 2>&1 || true
|
||||
echo "--- dumpsys window ---"
|
||||
adb shell dumpsys window 2>&1 || true
|
||||
echo "--- logcat -d (tail 400 — includes the SIGQUIT thread dump) ---"
|
||||
adb logcat -d 2>&1 | tail -400 || true
|
||||
echo "--- emulator accel / kvm / mem / disk ---"
|
||||
"$ANDROID_SDK_ROOT/emulator/emulator" -accel-check 2>&1 || true
|
||||
ls -l /dev/kvm 2>&1 || true
|
||||
free -h 2>&1 || true
|
||||
df -h 2>&1 || true
|
||||
} >> "$WEDGE_LOG" 2>&1 || true
|
||||
echo "::warning::E2E API ${{ matrix.api-level }} WEDGED ($1) — see the wedge-diagnostics-api${{ matrix.api-level }} artifact"
|
||||
}
|
||||
|
||||
# Retry the test run ONCE on failure (parity with the replaced retry step). A retried-but-
|
||||
# passed run is flagged as a ::warning:: because a blanket retry can mask a real regression.
|
||||
run_e2e() { ./gradlew connectedDebugAndroidTest --stacktrace; }
|
||||
run_e2e || { echo "::warning::API ${{ matrix.api-level }} E2E run failed — retrying once"; run_e2e; }
|
||||
# The gradle run is wrapped in the #404 wedge wrapper `timeout`: a wedge (hang) trips it
|
||||
# (exit 124) -> capture_wedge runs while the emulator is still alive, then the run returns
|
||||
# 124 so the retry / gate still see a failure. `|| status=$?` makes the exit code survive
|
||||
# this step's `set -e`; -k 30s SIGKILLs a gradle client that ignores SIGTERM. The wrapper
|
||||
# wraps ONLY the foreground gradle client (NOT the backgrounded emulator), identical in
|
||||
# shape to e2e-preview's run_shard — the structural reason it cannot re-hang the leg.
|
||||
run_e2e() {
|
||||
local status=0
|
||||
timeout -k 30s "$WEDGE_TIMEOUT" ./gradlew connectedDebugAndroidTest --stacktrace || status=$?
|
||||
if [ "$status" -eq 124 ]; then capture_wedge "$1"; fi
|
||||
return "$status"
|
||||
}
|
||||
run_e2e "attempt 1" || { echo "::warning::API ${{ matrix.api-level }} E2E run failed — retrying once"; run_e2e "attempt 2 (retry)"; }
|
||||
|
||||
# On any E2E failure (both boot attempts failed, a hung emulator, or an earlier setup/SDK
|
||||
# step), snapshot device + runner state to the step log AND a file for the artifact upload —
|
||||
@@ -677,6 +752,19 @@ jobs:
|
||||
${{ runner.temp }}/boot-diagnostics-api${{ matrix.api-level }}.txt
|
||||
if-no-files-found: warn
|
||||
|
||||
# Wedge-specific smoking gun (#404, restored to the matrix by #421): only present when the
|
||||
# wrapper `timeout` tripped (a hang) on either attempt — separate from the E2E diagnostics
|
||||
# artifact above. `if-no-files-found: ignore` keeps healthy runs quiet (no wedge => no file).
|
||||
# Per-api-level name (upload-artifact@v7 rejects duplicate artifact names). Mirrors the
|
||||
# e2e-preview job's wedge-diagnostics artifact.
|
||||
- name: Upload wedge diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: wedge-diagnostics-api${{ matrix.api-level }}
|
||||
path: ${{ runner.temp }}/wedge-diagnostics-api${{ matrix.api-level }}.txt
|
||||
if-no-files-found: ignore
|
||||
|
||||
# API 37 (Android 17, preview) E2E. Its only system image is the nonstandard
|
||||
# android-37.0 / google_apis_ps16k (16 KB page size), which reactivecircus/android-emulator-runner
|
||||
# can't provision (it builds android-37 / google_apis, neither of which exists), so this job
|
||||
|
||||
Reference in New Issue
Block a user