fix(ci): drop matrix E2E wedge-capture that hangs all 8 legs
The `timeout -k 30s` wrapper + `capture_wedge()` added in2f32657for the matrix `e2e` job (API 29-36) reproducibly wedges every leg, while the manually-provisioned API 37 preview shard running the identical capture logic passes. Revert the two matrix "Run E2E tests" steps' `script:` blocks to main's plain script (just the backgrounded logcat stream + `./gradlew connectedDebugAndroidTest`) and drop the now-dead "Upload wedge diagnostics" step from the matrix job. Kept untouched: the job-level `timeout-minutes: 50` backstop added in27ede55, and the entire `e2e-preview` job (its own capture_wedge/watchdog and wedge-diagnostics-api37-preview-shard* upload are unaffected).
This commit is contained in:
+5
-138
@@ -530,75 +530,9 @@ jobs:
|
|||||||
# so a test failure or emulator flake is diagnosable from the uploaded artifact.
|
# so a test failure or emulator flake is diagnosable from the uploaded artifact.
|
||||||
# Backgrounded; gradle stays the last foreground command so the step's exit status is
|
# Backgrounded; gradle stays the last foreground command so the step's exit status is
|
||||||
# still the test result (a real failure still trips continue-on-error -> the retry).
|
# still the test result (a real failure still trips continue-on-error -> the retry).
|
||||||
# WEDGE CAPTURE (#404): the gradle run is wrapped in an explicit `timeout` well below the
|
|
||||||
# job's hard cap but comfortably above a normal run (~13-15 min), so a WEDGE (hang) trips
|
|
||||||
# the wrapper (exit 124) instead of hanging until the force-kill — GUARANTEEING the
|
|
||||||
# capture below runs WHILE THE EMULATOR IS STILL ALIVE (the runner action tears it down as
|
|
||||||
# soon as this script returns, so a post-step can't probe it). A normal run finishes long
|
|
||||||
# before 1200s and is unaffected. See the shared capture body in `capture_wedge`.
|
|
||||||
script: |
|
script: |
|
||||||
WEDGE_TIMEOUT=1200
|
adb logcat -v time > "$RUNNER_TEMP/logcat-api${{ matrix.api-level }}.txt" 2>&1 &
|
||||||
LOGCAT="$RUNNER_TEMP/logcat-api${{ matrix.api-level }}.txt"
|
./gradlew connectedDebugAndroidTest --stacktrace
|
||||||
WEDGE="$RUNNER_TEMP/wedge-diagnostics-api${{ matrix.api-level }}.txt"
|
|
||||||
adb logcat -v time > "$LOGCAT" 2>&1 &
|
|
||||||
# On a wedge, grab the smoking gun: which test was running, SIGQUIT (kill -3) thread
|
|
||||||
# dumps of the app + instrumentation processes (ART -> logcat + /data/anr), activity/
|
|
||||||
# window state, and — the boot-race crux — whether the binder services are published.
|
|
||||||
# Every probe guarded (|| true) so a missing tool / dead device can't abort it; appended
|
|
||||||
# (not overwritten) so a wedge on attempt 1 survives even if the retry later passes.
|
|
||||||
capture_wedge() {
|
|
||||||
{
|
|
||||||
echo "==================================================================="
|
|
||||||
echo "===== E2E WEDGE — API ${{ matrix.api-level }} — $1 ====="
|
|
||||||
echo "===== $(date -u +%FT%TZ) — after ${WEDGE_TIMEOUT}s wrapper timeout ====="
|
|
||||||
echo "==================================================================="
|
|
||||||
echo "--- snapshot: was 'Create AVD' a cache-hit (warm snapshot restore)? ---"
|
|
||||||
echo "avd-cache cache-hit: '${{ steps.avd-cache.outputs.cache-hit }}'"
|
|
||||||
echo "--- running/last instrumented test (logcat TestRunner) ---"
|
|
||||||
grep -a TestRunner "$LOGCAT" 2>/dev/null | tail -25 || true
|
|
||||||
echo "--- getprop sys.boot_completed ---"
|
|
||||||
adb shell getprop sys.boot_completed 2>&1 || true
|
|
||||||
echo "--- getprop init.svc.* (per-service init state) ---"
|
|
||||||
adb shell getprop 2>&1 | grep -a init.svc || true
|
|
||||||
echo "--- service list (are binder services published?) ---"
|
|
||||||
adb shell service list 2>&1 || true
|
|
||||||
for svc in input window activity; do
|
|
||||||
echo "--- service check $svc ---"
|
|
||||||
adb shell service check "$svc" 2>&1 || true
|
|
||||||
done
|
|
||||||
echo "--- pids ---"
|
|
||||||
APP_PID="$(adb shell pidof org.libremail.app 2>/dev/null | tr -d '\r')" || true
|
|
||||||
TEST_PID="$(adb shell pidof org.libremail.app.test 2>/dev/null | tr -d '\r')" || true
|
|
||||||
echo "app pid: ${APP_PID:-<none>}"
|
|
||||||
echo "test pid: ${TEST_PID:-<none>}"
|
|
||||||
echo "--- SIGQUIT (kill -3) thread dumps -> ART writes to logcat + /data/anr ---"
|
|
||||||
for pid in $APP_PID $TEST_PID; do
|
|
||||||
[ -n "$pid" ] && adb shell kill -3 "$pid" 2>&1 || true
|
|
||||||
done
|
|
||||||
sleep 5
|
|
||||||
echo "--- /data/anr/* (SIGQUIT + ANR traces) ---"
|
|
||||||
adb shell 'cat /data/anr/* 2>/dev/null' 2>&1 || true
|
|
||||||
echo "--- dumpsys activity ---"
|
|
||||||
adb shell dumpsys activity 2>&1 || true
|
|
||||||
echo "--- dumpsys window ---"
|
|
||||||
adb shell dumpsys window 2>&1 || true
|
|
||||||
echo "--- logcat -d (tail 400 — includes the SIGQUIT thread dump) ---"
|
|
||||||
adb logcat -d 2>&1 | tail -400 || true
|
|
||||||
echo "--- emulator accel / kvm / mem / disk ---"
|
|
||||||
"$ANDROID_SDK_ROOT/emulator/emulator" -accel-check 2>&1 || true
|
|
||||||
ls -l /dev/kvm 2>&1 || true
|
|
||||||
free -h 2>&1 || true
|
|
||||||
df -h 2>&1 || true
|
|
||||||
} >> "$WEDGE" 2>&1 || true
|
|
||||||
echo "::warning::E2E API ${{ matrix.api-level }} WEDGED ($1) — see the wedge-diagnostics-api${{ matrix.api-level }} artifact"
|
|
||||||
}
|
|
||||||
# `|| status=$?` (not `if timeout; then`) so the exit code survives regardless of `set -e`;
|
|
||||||
# -k 30s SIGKILLs a gradle client that ignores SIGTERM. 124 == wedge -> capture, then exit
|
|
||||||
# with the real status so continue-on-error still fires the retry.
|
|
||||||
status=0
|
|
||||||
timeout -k 30s "$WEDGE_TIMEOUT" ./gradlew connectedDebugAndroidTest --stacktrace || status=$?
|
|
||||||
if [ "$status" -eq 124 ]; then capture_wedge "attempt 1"; fi
|
|
||||||
exit "$status"
|
|
||||||
|
|
||||||
- name: Run E2E tests (retry after emulator boot race)
|
- name: Run E2E tests (retry after emulator boot race)
|
||||||
if: steps.e2e.outcome == 'failure'
|
if: steps.e2e.outcome == 'failure'
|
||||||
@@ -612,64 +546,10 @@ jobs:
|
|||||||
disable-animations: true
|
disable-animations: true
|
||||||
# Retry runs a fresh emulator boot; stream its logcat the same way. `>` overwrites
|
# Retry runs a fresh emulator boot; stream its logcat the same way. `>` overwrites
|
||||||
# attempt 1's file so the artifact holds the FINAL attempt's logs, matching the
|
# attempt 1's file so the artifact holds the FINAL attempt's logs, matching the
|
||||||
# failure-time dump below (which reflects this last attempt's state). Same #404 wrapper
|
# failure-time dump below (which reflects this last attempt's state).
|
||||||
# timeout + wedge capture as attempt 1 (appended to the same WEDGE file) so a wedge is
|
|
||||||
# captured on the RETRY too, not just the first attempt.
|
|
||||||
script: |
|
script: |
|
||||||
WEDGE_TIMEOUT=1200
|
adb logcat -v time > "$RUNNER_TEMP/logcat-api${{ matrix.api-level }}.txt" 2>&1 &
|
||||||
LOGCAT="$RUNNER_TEMP/logcat-api${{ matrix.api-level }}.txt"
|
./gradlew connectedDebugAndroidTest --stacktrace
|
||||||
WEDGE="$RUNNER_TEMP/wedge-diagnostics-api${{ matrix.api-level }}.txt"
|
|
||||||
adb logcat -v time > "$LOGCAT" 2>&1 &
|
|
||||||
capture_wedge() {
|
|
||||||
{
|
|
||||||
echo "==================================================================="
|
|
||||||
echo "===== E2E WEDGE — API ${{ matrix.api-level }} — $1 ====="
|
|
||||||
echo "===== $(date -u +%FT%TZ) — after ${WEDGE_TIMEOUT}s wrapper timeout ====="
|
|
||||||
echo "==================================================================="
|
|
||||||
echo "--- snapshot: was 'Create AVD' a cache-hit (warm snapshot restore)? ---"
|
|
||||||
echo "avd-cache cache-hit: '${{ steps.avd-cache.outputs.cache-hit }}'"
|
|
||||||
echo "--- running/last instrumented test (logcat TestRunner) ---"
|
|
||||||
grep -a TestRunner "$LOGCAT" 2>/dev/null | tail -25 || true
|
|
||||||
echo "--- getprop sys.boot_completed ---"
|
|
||||||
adb shell getprop sys.boot_completed 2>&1 || true
|
|
||||||
echo "--- getprop init.svc.* (per-service init state) ---"
|
|
||||||
adb shell getprop 2>&1 | grep -a init.svc || true
|
|
||||||
echo "--- service list (are binder services published?) ---"
|
|
||||||
adb shell service list 2>&1 || true
|
|
||||||
for svc in input window activity; do
|
|
||||||
echo "--- service check $svc ---"
|
|
||||||
adb shell service check "$svc" 2>&1 || true
|
|
||||||
done
|
|
||||||
echo "--- pids ---"
|
|
||||||
APP_PID="$(adb shell pidof org.libremail.app 2>/dev/null | tr -d '\r')" || true
|
|
||||||
TEST_PID="$(adb shell pidof org.libremail.app.test 2>/dev/null | tr -d '\r')" || true
|
|
||||||
echo "app pid: ${APP_PID:-<none>}"
|
|
||||||
echo "test pid: ${TEST_PID:-<none>}"
|
|
||||||
echo "--- SIGQUIT (kill -3) thread dumps -> ART writes to logcat + /data/anr ---"
|
|
||||||
for pid in $APP_PID $TEST_PID; do
|
|
||||||
[ -n "$pid" ] && adb shell kill -3 "$pid" 2>&1 || true
|
|
||||||
done
|
|
||||||
sleep 5
|
|
||||||
echo "--- /data/anr/* (SIGQUIT + ANR traces) ---"
|
|
||||||
adb shell 'cat /data/anr/* 2>/dev/null' 2>&1 || true
|
|
||||||
echo "--- dumpsys activity ---"
|
|
||||||
adb shell dumpsys activity 2>&1 || true
|
|
||||||
echo "--- dumpsys window ---"
|
|
||||||
adb shell dumpsys window 2>&1 || true
|
|
||||||
echo "--- logcat -d (tail 400 — includes the SIGQUIT thread dump) ---"
|
|
||||||
adb logcat -d 2>&1 | tail -400 || true
|
|
||||||
echo "--- emulator accel / kvm / mem / disk ---"
|
|
||||||
"$ANDROID_SDK_ROOT/emulator/emulator" -accel-check 2>&1 || true
|
|
||||||
ls -l /dev/kvm 2>&1 || true
|
|
||||||
free -h 2>&1 || true
|
|
||||||
df -h 2>&1 || true
|
|
||||||
} >> "$WEDGE" 2>&1 || true
|
|
||||||
echo "::warning::E2E API ${{ matrix.api-level }} WEDGED ($1) — see the wedge-diagnostics-api${{ matrix.api-level }} artifact"
|
|
||||||
}
|
|
||||||
status=0
|
|
||||||
timeout -k 30s "$WEDGE_TIMEOUT" ./gradlew connectedDebugAndroidTest --stacktrace || status=$?
|
|
||||||
if [ "$status" -eq 124 ]; then capture_wedge "attempt 2 (retry)"; fi
|
|
||||||
exit "$status"
|
|
||||||
|
|
||||||
# On any E2E failure (both boot attempts failed, a hung emulator, or an earlier setup/SDK
|
# On any E2E failure (both boot attempts failed, a hung emulator, or an earlier setup/SDK
|
||||||
# step), snapshot device + runner state to the step log AND a file for the artifact upload —
|
# step), snapshot device + runner state to the step log AND a file for the artifact upload —
|
||||||
@@ -712,19 +592,6 @@ jobs:
|
|||||||
${{ runner.temp }}/diagnostics-api${{ matrix.api-level }}.txt
|
${{ runner.temp }}/diagnostics-api${{ matrix.api-level }}.txt
|
||||||
if-no-files-found: warn
|
if-no-files-found: warn
|
||||||
|
|
||||||
# Wedge-specific smoking gun (#404): only present when the wrapper `timeout` tripped (a hang) —
|
|
||||||
# written by capture_wedge inside the E2E run step(s), covering BOTH the first attempt and the
|
|
||||||
# retry. Separate from the #388 e2e-diagnostics artifact above (the general failure dump).
|
|
||||||
# `if-no-files-found: ignore` keeps the overwhelmingly-common healthy run quiet (no wedge => no
|
|
||||||
# file). Per-api-level name (upload-artifact@v7 rejects duplicate names).
|
|
||||||
- name: Upload wedge diagnostics
|
|
||||||
if: always()
|
|
||||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
||||||
with:
|
|
||||||
name: wedge-diagnostics-api${{ matrix.api-level }}
|
|
||||||
path: ${{ runner.temp }}/wedge-diagnostics-api${{ matrix.api-level }}.txt
|
|
||||||
if-no-files-found: ignore
|
|
||||||
|
|
||||||
# API 37 (Android 17, preview) E2E. Its only system image is the nonstandard
|
# API 37 (Android 17, preview) E2E. Its only system image is the nonstandard
|
||||||
# android-37.0 / google_apis_ps16k (16 KB page size), which reactivecircus/android-emulator-runner
|
# android-37.0 / google_apis_ps16k (16 KB page size), which reactivecircus/android-emulator-runner
|
||||||
# can't provision (it builds android-37 / google_apis, neither of which exists), so this job
|
# can't provision (it builds android-37 / google_apis, neither of which exists), so this job
|
||||||
|
|||||||
Reference in New Issue
Block a user