diff --git a/.github/scripts/e2e-report-shape-test.sh b/.github/scripts/e2e-report-shape-test.sh index f354d3c..56894c2 100755 --- a/.github/scripts/e2e-report-shape-test.sh +++ b/.github/scripts/e2e-report-shape-test.sh @@ -184,6 +184,42 @@ out="$(run_report "$root")" assert_contains "same-line annotation removed: counts 2, so it was worth 1" "$out" \ " baseline DEVIATION: the tree carries 2 tests marked \`@FailsOnEmulatorApi37\` but the baseline says 3 — update FAILS_ON_EMULATOR_API37_BASELINE" +# --------------------------------------------------------------------------- +# 4. A run the abort truncated, with fewer failures than the baseline: NOT a deviation. +# +# `expected` comes from `Starting N tests`, printed before anything can abort, so it still +# answers "is the marked set the size the baseline says". `failed` is a tally of what actually +# ran, and on a truncated run the tests after the abort never start. Measured on 2026-09-05, two +# api37-debug dispatches of the same four marked tests: 4/4/4 and then 4/3/3. Announcing the +# second as "one now passes" is the wrong reading, and #120 is the standing lesson about a notice +# that is wrong often enough to be skimmed past. +# --------------------------------------------------------------------------- +root="$(make_root "$FIXTURE_DIR" 3)" +cat > "$root/gradle.log" <<'TRUNCATED' +> Task :app:connectedDebugAndroidTest +Starting 3 tests on test(AVD) - 16 +There was 2 failure(s). +Test run failed to complete. Expected 3 tests, received 2. onError: commandError=false message=INSTRUMENTATION_ABORTED: System has crashed. +TRUNCATED +out="$(run_report "$root")" +assert_contains "truncated run: the truncation is reported" "$out" ' completed cleanly: no' +assert_absent "truncated run: the short failure count is not a deviation" "$out" 'tests failed, the baseline is' + +# --------------------------------------------------------------------------- +# 5. The same short failure count on a run that finished IS a deviation. +# +# The pair is the point: case 4 must not have bought its quiet by disabling the check outright. +# --------------------------------------------------------------------------- +root="$(make_root "$FIXTURE_DIR" 3)" +cat > "$root/gradle.log" <<'CLEAN' +> Task :app:connectedDebugAndroidTest +Starting 3 tests on test(AVD) - 16 +There was 2 failure(s). +CLEAN +out="$(run_report "$root")" +assert_contains "clean run, short by one: the deviation fires" "$out" \ + '2 tests failed, the baseline is 3' + echo if [ "$failures" -eq 0 ]; then echo "e2e-report-shape-test.sh: all checks passed" diff --git a/.github/scripts/e2e-report-shape.sh b/.github/scripts/e2e-report-shape.sh index 7b42321..48c10af 100755 --- a/.github/scripts/e2e-report-shape.sh +++ b/.github/scripts/e2e-report-shape.sh @@ -252,8 +252,22 @@ if [ -n "$baseline" ]; then if [ "$expected" != "unknown" ] && [ "$expected" != "$baseline" ]; then deviations+=("the runner started $expected tests, the baseline is $baseline") fi + # `expected` is compared on every run and `failed` only on a run that finished, and the + # difference is the truncation this file already records rather than compares. `expected` + # comes from `Starting N tests`, which is printed before anything can abort, so it answers + # "is the marked set the size the baseline says" whatever happens afterwards. `failed` is a + # tally of what actually ran: on a truncated run the tests after the abort never start, so + # comparing it to the baseline announces a deviation about the framework dying rather than + # about the test list. Measured on 2026-09-05, two api37-debug dispatches of the same four + # marked tests: 4/4/4 and then 4/3/3, the second having lost the last test to the abort. + # Announcing that as "one now passes" is exactly the wrong reading, and #120 is the standing + # lesson about a notice that is wrong often enough to be skimmed past. if [ "$failed" != "unknown" ] && [ "$failed" != "$baseline" ]; then - deviations+=("$failed tests failed, the baseline is $baseline — every test carrying the marker is expected to fail on this image, so fewer means one now passes and more means a new one joined") + if [ "$completed" = "**no**" ]; then + echo "::debug::$failed of $baseline marked tests failed, on a run the abort truncated — not compared" + else + deviations+=("$failed tests failed, the baseline is $baseline — every test carrying the marker is expected to fail on this image, so fewer means one now passes and more means a new one joined") + fi fi fi if [ -n "$marked" ] && [ "$marked" != "$baseline" ]; then diff --git a/app/src/androidTest/java/org/libremediaconverter/FailsOnEmulatorApi37.kt b/app/src/androidTest/java/org/libremediaconverter/FailsOnEmulatorApi37.kt index 8db3c5a..d53d372 100644 --- a/app/src/androidTest/java/org/libremediaconverter/FailsOnEmulatorApi37.kt +++ b/app/src/androidTest/java/org/libremediaconverter/FailsOnEmulatorApi37.kt @@ -54,13 +54,18 @@ annotation class FailsOnEmulatorApi37 * **The fourth carrier is the one to read that sentence carefully for.** * `pickingAFileThroughTheSystemPickerFillsInTheFileCard` was marked on 2026-09-05 for aborting * `system_server` rather than for failing (#108), and on the gating leg it passed two runs of - * four. The count still holds on the advisory leg, and that was **measured rather than assumed**: - * `api37-debug.yml` run 34008889182, dispatched with this annotation as its filter, reports - * `expected: 4, received: 4, failed: 4` (and `completed cleanly: no`, which is this job's normal). - * It fails there because it runs alongside the rotation test, which takes the framework down first - * — so the reason this line did not have to become two numbers is a property of the advisory leg, - * not of the test. If it ever reports three failures out of four, read that as this test having - * got lucky rather than as an image that improved. + * four. It fails on the advisory leg because the rotation test runs before it and takes the + * framework down first — measured, `api37-debug.yml` run 34008889182, which reports + * `expected: 4, received: 4, failed: 4` with the four in the order Media3, Media3, rotation, + * picker. + * + * **But a second dispatch of the identical configuration reported 4/3/3**, having lost the last + * test to the abort rather than to anything about the test list, and that is why + * `e2e-report-shape.sh` compares `failed` only on a run that finished. `expected` is compared + * always — it comes from `Starting N tests`, which is printed before anything can abort, so it is + * the field that answers "is the marked set the size this number says". Read a *clean* run + * reporting fewer failures than this as one of them now passing; read a truncated one as the + * framework having died, which is this job's normal. * * So: adding or removing a [FailsOnEmulatorApi37] means changing this number, in this file, in * the same diff. The report says so on the run itself if you forget — it prints the tree's own