diff --git a/.github/scripts/e2e-run.sh b/.github/scripts/e2e-run.sh new file mode 100755 index 0000000..b2486a1 --- /dev/null +++ b/.github/scripts/e2e-run.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# +# Runs the instrumented suite on an already-booted emulator, and makes a failure +# diagnosable without a re-run. +# +# Adapted from LibreMail's CI emulator instrumentation. Its lesson, learned there over +# several wedged merge queues, is that a red E2E leg with nothing but "exit 1" in the log +# costs more than the failure itself -- so every failure path here leaves evidence behind. +# +# WHY THIS IS A FILE rather than inline YAML: reactivecircus/android-emulator-runner splits +# its `script:` input on newlines and runs each line as its own `sh -c`. Shell functions, +# `if` blocks and traps cannot survive that, which is why the previous version had its whole +# failure handler crammed onto one unreadable line. One line calls this; this can breathe. +# +# Two failure shapes, deliberately handled differently: +# +# FAILED -- gradle returned non-zero. The reports say which test and why, so capture the +# device and runner state around it. +# WEDGED -- gradle never returned and the wrapper timeout killed it. There is no report at +# all, so the evidence has to be taken from the live device: what test was +# running, and what every process was doing. SIGQUIT is the important part -- ART +# dumps full thread stacks to logcat and /data/anr, which is how you tell a +# deadlocked test from a stuck MediaCodec from an emulator that stopped answering. +# +# Every probe is guarded with `|| true`. A diagnostic must never be the thing that turns a run +# red -- notably, a grep that matches nothing exits 1. +set -uo pipefail + +LABEL="${1:-unknown}" +APP_ID="org.libremediaconverter" +TEST_ID="org.libremediaconverter.test" + +TMP="${RUNNER_TEMP:-/tmp}" +LOGCAT_LOG="$TMP/logcat-api${LABEL}.txt" +DIAG_LOG="$TMP/diagnostics-api${LABEL}.txt" +WEDGE_LOG="$TMP/wedge-diagnostics-api${LABEL}.txt" + +# ~5 min is a healthy leg (measured across API 33-36), and this wraps only the gradle client, +# a subset of that. 20 min is generous enough never to trip on a slow-but-working run, and far +# enough under the job's 60-min cap that a genuine wedge still leaves time to capture it. +WEDGE_TIMEOUT=1200 + +# Stream logcat from now until the step ends, into a file that survives to the artifact upload. +# Without this, a failure that happens on-device leaves nothing behind: `adb logcat -d` at the +# end only has whatever is still in the ring buffer, and a chatty test run evicts the cause. +echo "===== logcat (api${LABEL}) =====" >> "$LOGCAT_LOG" +adb logcat -v time >> "$LOGCAT_LOG" 2>&1 & +LOGCAT_PID=$! + +dump_diagnostics() { + { + echo "===== E2E api${LABEL} failure diagnostics -- $(date -u +%FT%TZ) =====" + echo "--- adb devices ---"; adb devices -l 2>&1 || true + echo "--- guest memory ---"; adb shell cat /proc/meminfo 2>&1 | grep -E 'MemTotal|MemAvailable|SwapTotal' || true + echo "--- guest storage ---"; adb shell df /data 2>&1 || true + echo "--- is the app even installed? ---"; adb shell pm list packages 2>&1 | grep -a libremedia || true + echo "--- native crashes ---"; adb logcat -d -b crash 2>&1 | tail -80 || true + echo "--- runner: kvm ---"; ls -l /dev/kvm 2>&1 || true + echo "--- runner: memory ---"; free -h 2>&1 || true + echo "--- runner: disk ---"; df -h 2>&1 || true + } >> "$DIAG_LOG" 2>&1 || true + + # Also to the step log, so the common case needs no artifact download. + echo "----- FAILURE SUMMARY (api${LABEL}) -----" + adb shell cat /proc/meminfo 2>&1 | grep -E 'MemTotal|MemAvailable' || true + echo "--- native crashes (tail 60) ---" + adb logcat -d -b crash 2>&1 | tail -60 || true +} + +capture_wedge() { + { + echo "===================================================================" + echo "===== E2E WEDGE -- api${LABEL} -- $1" + echo "===== $(date -u +%FT%TZ) -- after ${WEDGE_TIMEOUT}s wrapper timeout" + echo "===================================================================" + # The single most useful line: which test was in flight when everything stopped. + echo "--- running/last instrumented test (logcat TestRunner) ---" + grep -a TestRunner "$LOGCAT_LOG" 2>/dev/null | tail -25 || true + echo "--- boot state ---" + adb shell getprop sys.boot_completed 2>&1 || true + echo "--- are the binder services published? ---" + for svc in input window activity media.player; do + echo " service check $svc:"; adb shell service check "$svc" 2>&1 || true + done + APP_PID="$(adb shell pidof "$APP_ID" 2>/dev/null | tr -d '\r')" || true + TEST_PID="$(adb shell pidof "$TEST_ID" 2>/dev/null | tr -d '\r')" || true + echo "--- pids --- app: ${APP_PID:-} test: ${TEST_PID:-}" + # SIGQUIT makes ART dump every thread's stack to logcat and /data/anr. This is what + # distinguishes a deadlocked test from a stuck native encode from a dead device. + echo "--- SIGQUIT thread dumps ---" + for pid in $APP_PID $TEST_PID; do + [ -n "$pid" ] && adb shell kill -3 "$pid" 2>&1 || true + done + sleep 5 + echo "--- /data/anr/* ---" + adb shell 'cat /data/anr/* 2>/dev/null' 2>&1 || true + echo "--- dumpsys activity ---"; adb shell dumpsys activity 2>&1 || true + echo "--- dumpsys window ---"; adb shell dumpsys window 2>&1 || true + # FFmpeg and Media3 both run through MediaCodec; a wedged transcode shows up here. + echo "--- dumpsys media.player ---"; adb shell dumpsys media.player 2>&1 || true + echo "--- logcat -d (tail 400, includes the SIGQUIT dump) ---" + adb logcat -d 2>&1 | tail -400 || true + } >> "$WEDGE_LOG" 2>&1 || true + echo "::warning::E2E api${LABEL} WEDGED ($1) -- see the wedge-diagnostics-api${LABEL} artifact" +} + +echo "::group::E2E api${LABEL}" +adb shell cat /proc/meminfo 2>&1 | grep -E 'MemTotal|MemAvailable|SwapTotal' || true + +status=0 +# -k 30s SIGKILLs a gradle client that ignores SIGTERM. The wrapper covers ONLY the foreground +# gradle client -- never the emulator, which the action owns -- so it cannot hang the leg. +timeout -k 30s "$WEDGE_TIMEOUT" \ + ./gradlew :app:connectedDebugAndroidTest -PabiFilters=x86_64 --stacktrace || status=$? +echo "::endgroup::" + +if [ "$status" -eq 0 ]; then + kill "$LOGCAT_PID" 2>/dev/null || true + exit 0 +fi + +if [ "$status" -eq 124 ]; then + capture_wedge "api${LABEL}" +else + echo "::error::E2E api${LABEL} failed (exit $status)" +fi +dump_diagnostics +kill "$LOGCAT_PID" 2>/dev/null || true +exit "$status" diff --git a/.github/workflows/status_check.yml b/.github/workflows/status_check.yml index 875ca9d..171163b 100644 --- a/.github/workflows/status_check.yml +++ b/.github/workflows/status_check.yml @@ -261,20 +261,12 @@ jobs: # dominate the APK, so shipping arm64 to an x86_64 emulator doubles the # install for code that can never run: 114 MB against 80 MB. # - # The probe lines survive from diagnosing the API 37 crash and are kept - # because a red instrumented run is otherwise near-impossible to read from a - # log alone. The first reports what the guest actually got, so a wrong - # emulator configuration is visible on a green run too; the crash dump runs - # only on failure, so a green run is unchanged. - # - # Each line here is a separate `sh -c` -- the action splits the script on - # newlines -- so the failure handler has to stay on one line. The action does - # not pass ignoreReturnCode, so a non-zero line fails the job outright: the - # probe ends in `|| true` because a grep that matches nothing exits 1, and a - # diagnostic must never be the thing that turns a run red. - script: | - adb shell cat /proc/meminfo | grep -E 'MemTotal|MemAvailable|SwapTotal' || true - ./gradlew :app:connectedDebugAndroidTest -PabiFilters=x86_64 || { echo "=== guest memory at failure ==="; adb shell cat /proc/meminfo | grep -E 'MemTotal|MemAvailable|SwapTotal'; echo "=== native crashes ==="; adb logcat -d -b crash | tail -60; exit 1; } + # Diagnostics live in .github/scripts/e2e-run.sh, not here. This action splits + # `script` on newlines and runs each line as its own `sh -c`, so a handler written + # inline has to fit on ONE line -- which is how the previous version ended up as an + # unreadable chain of semicolons. One line invokes the script; the script can use + # functions, and captures a hang as well as a failure. See its header. + script: bash .github/scripts/e2e-run.sh ${{ matrix.label }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() @@ -283,3 +275,25 @@ jobs: path: | app/build/reports/androidTests/ app/build/outputs/androidTest-results/ + if-no-files-found: warn + + # The streamed logcat and the failure dump. Uploaded always, because a leg that goes red + # once and green on re-run is exactly the one worth reading afterwards, and by then the + # emulator is long gone. + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: e2e-diagnostics-api${{ matrix.label }} + path: | + ${{ runner.temp }}/logcat-api${{ matrix.label }}.txt + ${{ runner.temp }}/diagnostics-api${{ matrix.label }}.txt + if-no-files-found: warn + + # Only exists when the wrapper timeout tripped, so `ignore` keeps healthy runs quiet + # instead of warning on every green leg. + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: e2e-wedge-api${{ matrix.label }} + path: ${{ runner.temp }}/wedge-diagnostics-api${{ matrix.label }}.txt + if-no-files-found: ignore