diff --git a/.github/workflows/status_check.yml b/.github/workflows/status_check.yml index 099267f..b3d0597 100644 --- a/.github/workflows/status_check.yml +++ b/.github/workflows/status_check.yml @@ -103,9 +103,12 @@ jobs: # --------------------------------------------------------------------------- # One runner per API level, across the whole supported range. # - # The range is the point: minSdk is 33 and targetSdk is 37, and the foreground - # service type differs across it -- none below 34, dataSync at 34, mediaProcessing - # from 35. Testing a single level would leave two thirds of that branch unexercised. + # The range is the point: minSdk is 33, and the foreground service type differs + # across it -- none below 34, dataSync at 34, mediaProcessing from 35. Testing a + # single level would leave two thirds of that branch unexercised. + # + # It stops at 36 rather than targetSdk 37 because the android-37.0 emulator image + # is broken, not because 37 does not matter. See docs/api-37-emulator-crash.md. # # FFmpeg is not built here. The AAR is committed under bin/, so a red run means the # code is broken rather than that a cross-compile hiccuped. @@ -130,12 +133,13 @@ jobs: api-level: "35" - label: "36" api-level: "36" - # API 37 ships as android-37.0. The emulator action installs - # "platforms;android-${api-level}" and there is no platforms;android-37, - # so a bare 37 fails during SDK setup before an emulator ever starts. - # system-image-api-level alone does not fix it: that only names the image. - - label: "37" - api-level: "37.0" + # No API 37 row. targetSdk is 37, but the android-37.0 emulator image + # crash-loops surfaceflinger inside its own gralloc mapper, so every test + # fails there no matter what this app does. Ruling that in took four CI + # rounds, so the evidence and the ruled-out fixes are written down rather + # than left to be rediscovered: docs/api-37-emulator-crash.md. That file + # also records what to try first when re-adding it -- note that the row + # needs api-level "37.0", since a bare 37 fails during SDK setup. steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -172,29 +176,25 @@ jobs: emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none disable-animations: true # The default userdata partition is not big enough for this APK once the - # FFmpeg libraries are in it. API 37's system image leaves the least room and - # was the level that actually failed, with "Requested internal only, but not - # enough space" -- but the margin was thin everywhere, so give them all room. + # FFmpeg libraries are in it. One level failed outright with "Requested + # internal only, but not enough space", and the margin was thin everywhere + # else, so give them all room. disk-size: 8G - # The emulator raises an undersized guest to 2560M by itself, but only for - # API levels it recognises, and it does not recognise "37.0". Every green job - # in this matrix was quietly running at 2560M while API 37 ran at the pixel_6 - # default of 1536M, lost system_server partway through installing the 82 MB - # APK, and reported it as "Can't find service: package". 2560M is not a guess - # at a sufficient value: it is the value the other four levels already pass - # at. Setting it explicitly makes the matrix uniform instead of leaving one - # level at the mercy of that heuristic. + # Pinned because the emulator's own default is not uniform: it raises an + # undersized guest to a floor that varies by API level -- 2048M at 33, 2560M + # at 34 through 36 -- and skips levels it does not recognise entirely. 2560M + # is the highest of those floors, so no level gets less memory than it + # already had, and none of them depend on that heuristic any more. ram-size: 2560M # Build only the ABI the emulator can execute. FFmpeg's native libraries # dominate the APK, so shipping arm64 to an x86_64 emulator doubles the # install for code that can never run: 114 MB against 80 MB. # - # The memory probes exist because this failure cannot be reproduced locally: - # API 37 will not boot on a workstation under either GPU mode -- host aborts - # surfaceflinger in the goldfish mapper, swiftshader_indirect segfaults the - # emulator. CI is the only instrument, so it has to report enough to be - # conclusive. The first line proves what the guest actually got regardless of - # the result; the rest runs only on failure, so a green run is unchanged. + # The probe lines survive from diagnosing the API 37 crash and are kept + # because a red instrumented run is otherwise near-impossible to read from a + # log alone. The first reports what the guest actually got, so a wrong + # emulator configuration is visible on a green run too; the crash dump runs + # only on failure, so a green run is unchanged. # # Each line here is a separate `sh -c` -- the action splits the script on # newlines -- so the failure handler has to stay on one line. The action does @@ -203,7 +203,7 @@ jobs: # diagnostic must never be the thing that turns a run red. script: | adb shell cat /proc/meminfo | grep -E 'MemTotal|MemAvailable|SwapTotal' || true - ./gradlew :app:connectedDebugAndroidTest -PabiFilters=x86_64 || { echo "=== guest memory at failure ==="; adb shell cat /proc/meminfo | grep -E 'MemTotal|MemAvailable|SwapTotal'; echo "=== kernel OOM kills ==="; adb shell dmesg | grep -iE 'oom|lowmemory|killed process' | tail -20; echo "=== native crashes ==="; adb logcat -d -b crash | tail -40; exit 1; } + ./gradlew :app:connectedDebugAndroidTest -PabiFilters=x86_64 || { echo "=== guest memory at failure ==="; adb shell cat /proc/meminfo | grep -E 'MemTotal|MemAvailable|SwapTotal'; echo "=== native crashes ==="; adb logcat -d -b crash | tail -60; exit 1; } - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() diff --git a/docs/api-37-emulator-crash.md b/docs/api-37-emulator-crash.md new file mode 100644 index 0000000..41d608c --- /dev/null +++ b/docs/api-37-emulator-crash.md @@ -0,0 +1,185 @@ +# API 37 is not tested in CI: a crash in Google's `android-37.0` emulator image + +**Status:** open upstream, worked around by removing API 37 from the E2E matrix +**Last verified:** 2026-08-21, against emulator `37.1.11.0` and system image revision 6 + +`minSdk` is 33 and `targetSdk` is 37, and the E2E matrix in +[`status_check.yml`](../.github/workflows/status_check.yml) runs API 33 through 36. +API 37 is deliberately absent. This is why. + +## Summary + +The `android-37.0` emulator system image crashes `surfaceflinger` in a loop. The app +under test never gets a working framework, so every instrumented test fails regardless +of what the app does. The bug is in the emulator image, not in this project. + +The crash is an assertion inside the emulator's own gralloc implementation: + +``` +Executable: /system/bin/surfaceflinger +signal 6 (SIGABRT), code -1 (SI_QUEUE), tid: RegionSampling +Abort message: 'Assertion failed: !rcEnc->featureInfo()->hasReadColorBufferDma' + + #03 mapper.ranchu.so GoldfishMapper::readFromHost(cb_handle_t const&) const + #04 mapper.ranchu.so GoldfishMapper::GoldfishMapper()::'lambda'(...)::__invoke + #05 libui.so android::Gralloc5Mapper::lock(...) + #06 libui.so android::GraphicBufferMapper::lock(...) + #07 libui.so android::GraphicBuffer::lockAsync(...) + #08 libui.so android::GraphicBuffer::lock(...) + #09 surfaceflinger android::RegionSamplingThread::threadMain() +``` + +`RegionSamplingThread` is SystemUI's navigation-bar luma sampling. It calls +`GraphicBuffer::lock`, which routes into `GoldfishMapper::readFromHost`, which asserts +that the host has *not* negotiated the `ReadColorBufferDma` capability. On this image +the host has, so the assertion fails and `surfaceflinger` aborts. It restarts and +aborts again. + +## Impact + +The failure surfaces in two different ways depending on how far the job gets, which is +why it took several rounds to identify: + +| Guest RAM | Where it dies | What CI reports | +|---|---|---| +| 1536 MB | during APK install | `Unknown failure: cmd: Can't find service: package` | +| 2560 MB | during the test run | `There were failing tests` — all of them | + +At 2560 MB the install succeeds and the tests actually execute, then fail wholesale. +The first failure in the report is misleading: + +``` +kotlin.UninitializedPropertyAccessException: lateinit property output has not + been initialized + at Media3EngineTest.tearDown(Media3EngineTest.kt:53) + +java.lang.IllegalStateException: WorkManager is not initialized properly. + You have explicitly disabled WorkManagerInitializer in your manifest, ... +``` + +Neither is a real defect in this project. `tearDown` throws because `setUp` never got +far enough to assign `output`, and WorkManager's `InitializationProvider` never runs +because content-provider installation fails on a framework whose `surfaceflinger` is +crash-looping. The same tests pass at API 33, 34, 35, and 36 in the same CI run, and +the first `surfaceflinger` abort is timestamped *before* the test results are reported. + +That last point is inference rather than proof: the definitive check would be running +the suite against a healthy API 37 device, which does not exist yet. If a fixed image +ships and these tests still fail, treat the WorkManager error as a real finding. + +## Environment + +Reproduced identically in two unrelated environments, so it is not specific to a host +GPU, driver, or CI runner. + +| | GitHub Actions | Local workstation | +|---|---|---| +| Host | `ubuntu-latest`, no GPU | Fedora, Intel Iris Xe (TGL GT2) | +| Emulator | `37.1.11.0` (build 15917651) | `37.1.11.0` (build 15917651) | +| GPU mode | `swiftshader_indirect` | `host` | +| Result | boots, aborts during tests | aborts before boot completes | + +System image: `system-images;android-37.0;google_apis;x86_64`, `Pkg.Revision=6`, +`AndroidVersion.ApiLevel=37.0`, `AndroidVersion.ExtensionLevel=22` + +``` +Build fingerprint: google/sdk_gphone64_x86_64/emu64xa:17/CE2A.260420.019/15611780:userdebug/dev-keys +Kernel Release: 6.12.58-android16-6-gccafb60de224-ab14828483 +``` + +## What was ruled out, and how + +Each of these was tested rather than reasoned about, because the first three attempts +at this bug were plausible fixes that turned out to address earlier, unrelated failures. + +**Guest memory.** The emulator raises an undersized guest to a minimum on its own, but +only for API levels it recognises, and it does not recognise `"37.0"`. API 33 bumps to +2048 MB and 34/35/36 to 2560 MB, while API 37 logged no bump at all and ran at the +`pixel_6` default of 1536 MB. Setting `ram-size: 2560M` explicitly fixed that asymmetry +and did change the outcome — the job got past install and into the test run — but it is +not the underlying bug. At the moment of failure the guest reported `MemTotal 2527392 +kB` with `MemAvailable 1507104 kB`: 1.5 GB free, and no OOM kills. + +**GPU mode.** Both `swiftshader_indirect` and `host` crash, with the same assertion and +the same frames. The crash is in the gralloc mapper, below the renderer. + +**Disabling the DMA feature.** `GLDMA` is the host feature that most plausibly backs the +guest's `hasReadColorBufferDma`. Launching with `-feature -GLDMA` was accepted by the +emulator — the log confirms `Feature 'GLDMA' (51) is overridden to 'disabled'` — and +`surfaceflinger` still aborted 13 times and the device never finished booting. Whatever +sets that guest capability, it is not this flag. + +**An ATD image.** `google_atd` / `aosp_atd` images are built for automated testing and +ship without the SystemUI package set, which is what drives `RegionSamplingThread` in +the first place. That would likely sidestep the bug class entirely, but **no ATD image +exists for `android-37.0`** — only `google_apis`, `google_apis_playstore`, the `ps16k` +16 KB-page variants, and Wear OS. Check again when revisiting; if an ATD image appears, +try it before anything else here. + +## Reproducing it + +Locally, with `-gpu host` so the emulator itself does not segfault on Intel graphics: + +```bash +sdkmanager --install "system-images;android-37.0;google_apis;x86_64" +echo no | avdmanager create avd -n api37_repro \ + -k "system-images;android-37.0;google_apis;x86_64" -d pixel_6 --force + +$ANDROID_HOME/emulator/emulator -avd api37_repro \ + -no-window -gpu host -noaudio -no-boot-anim -camera-back none -no-snapshot & + +# Boot never completes. Count the aborts: +adb logcat -d -b crash | grep -c hasReadColorBufferDma +``` + +`sys.boot_completed` never reaches `1`, `pgrep -f system_server` stays empty, and +`keystore2`'s watchdog logs `await_boot_completed ... Overdue` indefinitely. + +To see the CI-side form instead, restore the API 37 row in the E2E matrix of +`status_check.yml` (`api-level: "37.0"` — a bare `37` fails earlier still, during SDK +setup, because there is no `platforms;android-37`). + +## Filing this upstream + +Not yet filed. To file it: + +1. Go to and sign in with a Google account. +2. Choose **Report an issue**, then pick the component for the Android emulator — search + the component picker for "Emulator"; it sits under the Android Studio component tree. + If the picker is unclear, Android Studio's **Help → Submit Feedback** opens the same + tracker with the component preselected, and the emulator's own **Extended controls → + Help → File a bug** does likewise. +3. Title it for the mechanism, not the symptom, so it is searchable — for example: + `surfaceflinger aborts in GoldfishMapper::readFromHost (hasReadColorBufferDma) on + android-37.0 google_apis x86_64`. +4. Paste the assertion and backtrace from the top of this document, the environment + table, and the reproduction steps above. State explicitly that it reproduces on two + unrelated hosts under both GPU modes — that is the detail that stops it being closed + as a local graphics problem. +5. List what was ruled out. Bugs that arrive with `-feature -GLDMA` already eliminated + tend not to bounce back asking for it. +6. Attach: + - the guest tombstone, via `adb pull /data/tombstones` (or the `pbtombstone` output + the crash log names) + - `adb logcat -d -b crash > crash.txt` + - the emulator's own stdout log, captured by redirecting the launch command + - the AVD's `config.ini` + - a link to a failing CI job, which shows it on hardware you do not control: + + +Record the issue number here once filed. + +## When to revisit + +Re-add the API 37 row when any of these happens: + +- a new `android-37.0` system image revision ships (this was revision 6) +- an ATD image appears for API 37 +- the upstream issue is marked fixed + +Until then the coverage gap is narrow but real. `targetSdk` is 37, so the app is still +compiled and unit-tested against it, and the API-dependent behaviour this matrix exists +to exercise — the foreground service type, which is absent below 34, `dataSync` at 34, +and `mediaProcessing` from 35 — is fully covered at 35 and 36. What is *not* covered is +any behaviour Android 17 changes relative to 36. Verify that on a physical device before +release.