From 3e9528454c59f2376ec286c85cb8fdec306203cf Mon Sep 17 00:00:00 2001 From: Jason Ross Date: Tue, 25 Aug 2026 20:20:59 -0500 Subject: [PATCH] Announce a baseline it cannot read, rather than falling quiet "A comparison was asked for" and "a number was found to compare against" were one variable, and collapsing them put the report one refactor away from being the thing #83 filed. The sed that reads FAILS_ON_EMULATOR_API37_BASELINE is anchored at the line start, so indenting the const into an object -- or renaming it, or moving it -- empties it, and the old code then skipped the whole comparison while the table kept printing exactly as before. Silent, and indistinguishable from a run that matched. Now an unreadable baseline is itself a deviation, with the notice naming the const so the fix is obvious. Verified against the real captured log of run 32865281555 three ways: baseline file absent, const indented into an object, and the committed file unchanged -- the first two announce, the third stays silent. --- .github/scripts/e2e-report-shape.sh | 26 ++++++++++++++++--- .../FailsOnEmulatorApi37.kt | 8 +++--- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.github/scripts/e2e-report-shape.sh b/.github/scripts/e2e-report-shape.sh index 865bef7..50b5f83 100755 --- a/.github/scripts/e2e-report-shape.sh +++ b/.github/scripts/e2e-report-shape.sh @@ -165,11 +165,21 @@ fi # the advisory leg should run and how many should fail. A smaller `failed` means one now passes # -- which is the trigger to delete the annotation, written down in FailsOnEmulatorApi37.kt. # --------------------------------------------------------------------------- +# +# `advisory` and `baseline` are two variables on purpose. "A comparison was asked for" and "a +# number was found to compare against" are different facts, and collapsing them is how this +# report would go quietly back to being the thing #83 filed: the `sed` below is anchored, so +# indenting the const into an object -- or renaming it, or moving it to another file -- empties +# `baseline`, and a single flag would take the whole comparison down with it while the table +# kept printing. An unreadable baseline is itself a deviation, and is announced as one. +advisory="no" baseline="" marked="" deviations=() -if [ -n "$BASELINE_FILE" ] && [ -f "$BASELINE_FILE" ]; then - baseline="$(sed -nE 's/^const val FAILS_ON_EMULATOR_API37_BASELINE = ([0-9]+).*/\1/p' "$BASELINE_FILE" | head -1)" +if [ -n "$BASELINE_FILE" ]; then + advisory="yes" + [ -f "$BASELINE_FILE" ] \ + && baseline="$(sed -nE 's/^const val FAILS_ON_EMULATOR_API37_BASELINE = ([0-9]+).*/\1/p' "$BASELINE_FILE" | head -1)" # The #81 check, verbatim: what the tree actually carries. Reported next to the baseline so a # stale baseline shows up here rather than only once the emulator disagrees with it. if [ -d "$REPO_ROOT/app/src/androidTest" ]; then @@ -178,6 +188,10 @@ if [ -n "$BASELINE_FILE" ] && [ -f "$BASELINE_FILE" ]; then fi fi +if [ "$advisory" = "yes" ] && [ -z "$baseline" ]; then + deviations+=("the committed baseline could not be read from \`$(basename -- "$BASELINE_FILE")\` — has \`FAILS_ON_EMULATOR_API37_BASELINE\` been renamed, indented into a class, or moved? Nothing was compared") +fi + if [ -n "$baseline" ]; then if [ "$no_run" = "nothing" ]; then deviations+=("no test run observed — the runner never reported starting one, where the baseline expects $baseline tests carrying \`@FailsOnEmulatorApi37\`") @@ -211,7 +225,7 @@ if [ -n "$failed_names" ]; then echo " failed tests:" printf '%s\n' "$failed_names" | sed -e 's/^/ /' fi -if [ -n "$baseline" ]; then +if [ "$advisory" = "yes" ]; then if [ "${#deviations[@]}" -eq 0 ]; then echo " baseline: matches ($baseline expected, $baseline failed)" else @@ -250,9 +264,13 @@ if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then echo "> $xml_count test XML files were present; the counts above come from the first." echo fi - if [ -n "$baseline" ]; then + if [ "$advisory" = "yes" ]; then if [ "${#deviations[@]}" -eq 0 ]; then echo "**Matches the committed baseline of $baseline** — $baseline tests carry \`@FailsOnEmulatorApi37\` and all $baseline failed, which is what this job is for." + elif [ -z "$baseline" ]; then + echo "**The committed baseline could not be read, so nothing was compared.** Announced as a notice, not an error: this job is advisory and its conclusion is unchanged by anything here." + echo + printf -- '- %s\n' "${deviations[@]}" else echo "**DEVIATION from the committed baseline of $baseline.** Announced as a notice, not an error: this job is advisory and its conclusion is unchanged by anything here." echo diff --git a/app/src/androidTest/java/org/libremediaconverter/FailsOnEmulatorApi37.kt b/app/src/androidTest/java/org/libremediaconverter/FailsOnEmulatorApi37.kt index d669ef8..a816864 100644 --- a/app/src/androidTest/java/org/libremediaconverter/FailsOnEmulatorApi37.kt +++ b/app/src/androidTest/java/org/libremediaconverter/FailsOnEmulatorApi37.kt @@ -31,9 +31,11 @@ annotation class FailsOnEmulatorApi37 * How many tests carry [FailsOnEmulatorApi37] — the advisory API 37 job's committed baseline. * * **No Kotlin reads this, and it is not stray config.** `.github/scripts/e2e-report-shape.sh` - * parses it out of this file by name, and the advisory job compares the run it just did against - * it: this many tests should start, and all of them should fail. Deleting it makes that - * comparison silently stop happening — the report keeps printing, with nothing to compare to. + * parses it out of this file by name, with a line-anchored pattern, and the advisory job compares + * the run it just did against it: this many tests should start, and all of them should fail. + * Deleting it, renaming it, or indenting it into a class stops the comparison — the report would + * keep printing with nothing to compare to, so it announces that it could not read the baseline + * rather than falling quiet. If you see that notice, this line is what it means. * * **One number, both checks, and that is what the marker means.** A test carrying it cannot pass * on this image, so the count is simultaneously how many the advisory leg runs and how many fail.