diff --git a/.github/scripts/e2e-report-shape-test.sh b/.github/scripts/e2e-report-shape-test.sh new file mode 100755 index 0000000..f354d3c --- /dev/null +++ b/.github/scripts/e2e-report-shape-test.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# +# Exercises e2e-report-shape.sh's baseline counter against fixture source, with no emulator and +# no CI run. Run it directly: +# +# .github/scripts/e2e-report-shape-test.sh +# +# WHY THIS CAN EXIST AT ALL: the counter is a pure function of the working tree. It greps +# `app/src/androidTest` for `@FailsOnEmulatorApi37` and compares the total against the number +# committed in FailsOnEmulatorApi37.kt. Nothing about that needs a device, which is the whole +# reason #120 could be measured rather than argued about. +# +# WHY A THROWAWAY REPO ROOT rather than a knob on the script. The report finds its own root from +# `BASH_SOURCE`, so a copy of it placed at `/.github/scripts/` reads `/app/src/...`. +# Building that root is three mkdirs and costs the shipped script nothing: +# +# - the REAL script is what runs, byte for byte, so reverting the matcher reddens this test +# rather than a testing-only code path beside it; +# - no environment variable exists that could point the LIVE count somewhere else, which is +# the failure mode #83 built the baseline check to prevent in the first place; +# - XML_DIR resolves inside the throwaway root, so a stale app/build/outputs left by a real +# run on a developer machine cannot leak into the numbers here. +# +# WHAT IT GUARDS (#120). The old matcher looked for the string anywhere on any line, so a KDoc +# saying `Deliberately not @FailsOnEmulatorApi37` counted as a marked test and every PR got a +# deviation notice that was wrong. The obvious repair -- count only lines that are nothing but +# the annotation -- silently stops counting `@FailsOnEmulatorApi37 @Test`, which is legal Kotlin, +# and undercounting is the direction that hides a genuine new marker. The fixture carries every +# shape at once -- three that count and three that must not, enumerated in its own header -- so +# both mistakes fail here instead of on a PR: against testdata/marker-shapes the old matcher says +# 5, own-line-only says 2, and the shipped one 3. +# +# NOT WIRED INTO CI, deliberately and as a known gap. Adding a step to the Static analysis job +# would add a new way for a gating job to go red, and #120 was explicit that nothing about it may +# change any job's status or the pass/fail rules. shellcheck still covers this file, since that +# step reads `git ls-files '*.sh'` rather than a fixed list. +set -uo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPORT="$SCRIPT_DIR/e2e-report-shape.sh" +FIXTURE_DIR="$SCRIPT_DIR/testdata/marker-shapes" +FIXTURE="$FIXTURE_DIR/MarkerShapes.kt" + +TMP="$(mktemp -d)" +# Single quotes: the path is expanded when the trap fires, not when it is set. +trap 'rm -rf -- "$TMP"' EXIT + +failures=0 + +pass() { printf 'ok %s\n' "$1"; } + +fail() { + failures=$((failures + 1)) + printf 'FAIL %s\n' "$1" + shift + printf ' %s\n' "$@" +} + +# assert_contains +# `case` rather than grep: the strings being matched carry backticks and an em dash, and this +# way neither the shell nor a regex engine gets an opinion about them. +assert_contains() { + case "$2" in + *"$3"*) pass "$1" ;; + *) fail "$1" "wanted to find: $3" "in:" "$2" ;; + esac +} + +assert_absent() { + case "$2" in + *"$3"*) fail "$1" "did NOT want to find: $3" "in:" "$2" ;; + *) pass "$1" ;; + esac +} + +# make_root +# Assembles a throwaway repo root around the given tree and prints its path. +make_root() { + local tree="$1" baseline="$2" root testdir + root="$(mktemp -d "$TMP/root.XXXXXX")" + testdir="$root/app/src/androidTest/java/org/libremediaconverter" + mkdir -p "$root/.github/scripts" "$testdir" + cp -- "$REPORT" "$root/.github/scripts/" + cp -- "$tree"/*.kt "$testdir/" + + # The synthetic stand-in for the committed baseline. Its KDoc names the marker the way the real + # file does -- in brackets, never with an `@` -- because the real file lives inside the tree + # being counted, so an `@` spelling here would add a phantom to every number below. + cat > "$testdir/FailsOnEmulatorApi37.kt" < tests, all failing -- which is what the + # advisory leg looks like when nothing has drifted. It leaves the marked count as the only + # field that can deviate, so every assertion below is about the thing under test. + cat > "$root/gradle.log" < Task :app:connectedDebugAndroidTest +Starting $baseline tests on test(AVD) - 16 +There was $baseline failure(s). +EOF + + printf '%s\n' "$root" +} + +# run_report -- stdout of the real script; its summary lands in /summary.md. +run_report() { + E2E_WEDGED_AFTER='' GITHUB_STEP_SUMMARY="$1/summary.md" \ + bash "$1/.github/scripts/e2e-report-shape.sh" 37 "$1/gradle.log" \ + "$1/app/src/androidTest/java/org/libremediaconverter/FailsOnEmulatorApi37.kt" +} + +# --------------------------------------------------------------------------- +# The fixture still carries every shape. +# +# Three of the checks below are covered twice over -- deleting a real annotation moves the count +# and fails a case further down. The two decoys are not: drop the KDoc mention and the count +# stays 3, so the precision this whole ticket is about would stop being tested and nothing would +# say so. That asymmetry is why the shapes are asserted by name rather than only by their effect +# on the total. +# --------------------------------------------------------------------------- +fixture_text="$(cat -- "$FIXTURE")" +assert_contains "fixture: the import" "$fixture_text" 'import org.libremediaconverter.FailsOnEmulatorApi37' +assert_contains "fixture: annotation own line" "$fixture_text" ' + @FailsOnEmulatorApi37 + @Test' +assert_contains "fixture: annotation with @Test on one line" "$fixture_text" '@FailsOnEmulatorApi37 @Test' +assert_contains "fixture: annotation nested and indented" "$fixture_text" ' + @FailsOnEmulatorApi37' +assert_contains "fixture: KDoc mention (this is #120)" "$fixture_text" "* Deliberately not \`@FailsOnEmulatorApi37\`" +assert_contains "fixture: commented-out annotation" "$fixture_text" '// @FailsOnEmulatorApi37' + +# --------------------------------------------------------------------------- +# 1. The fixture's three real annotations against a baseline of 3: no deviation. +# +# This one case fails under both wrong matchers -- the old one counts 5, own-line-only counts 2 -- +# which is why it is first. +# --------------------------------------------------------------------------- +root="$(make_root "$FIXTURE_DIR" 3)" +out="$(run_report "$root")" +assert_contains "3 real markers, baseline 3: reports a match" "$out" ' baseline: matches (3 expected, 3 failed)' +assert_absent "3 real markers, baseline 3: says nothing about the tree" "$out" 'the tree carries' +assert_contains "3 real markers, baseline 3: summary agrees" \ + "$(cat -- "$root/summary.md")" '**Matches the committed baseline of 3**' + +# --------------------------------------------------------------------------- +# 2. A fourth REAL annotation. The count has to move and the deviation has to fire. +# +# The important half of #120: precision was the bug, but a matcher that stopped noticing a new +# marker would have been a worse one, silently. +# --------------------------------------------------------------------------- +plus_one="$(mktemp -d "$TMP/plusone.XXXXXX")" +cp -- "$FIXTURE" "$plus_one/" +cat > "$plus_one/FourthMarker.kt" <<'EOF' +package org.libremediaconverter.fixture + +class FourthMarker { + @FailsOnEmulatorApi37 + @Test + fun addedToday() = Unit +} +EOF +root="$(make_root "$plus_one" 3)" +out="$(run_report "$root")" +assert_contains "a 4th real marker: the deviation fires, and counts 4" "$out" \ + " baseline DEVIATION: the tree carries 4 tests marked \`@FailsOnEmulatorApi37\` but the baseline says 3 — update FAILS_ON_EMULATOR_API37_BASELINE" +assert_contains "a 4th real marker: the summary carries it too" "$(cat -- "$root/summary.md")" \ + "- the tree carries 4 tests marked \`@FailsOnEmulatorApi37\` but the baseline says 3" + +# --------------------------------------------------------------------------- +# 3. Delete the same-line annotation and the count must drop to 2. +# +# This is the trap, pinned down. `@FailsOnEmulatorApi37 @Test` on one line is what separates the +# shipped matcher from `^[[:space:]]*@NAME[[:space:]]*$`, and without this case the fixture entry +# guarding it could be deleted as decoration -- case 1 would then pass under the wrong matcher. +# Here the same-line entry is worth exactly one, and it is asserted to be. +# --------------------------------------------------------------------------- +minus_same_line="$(mktemp -d "$TMP/minus.XXXXXX")" +sed -e '/@FailsOnEmulatorApi37 @Test/d' -- "$FIXTURE" > "$minus_same_line/MarkerShapes.kt" +root="$(make_root "$minus_same_line" 3)" +out="$(run_report "$root")" +assert_contains "same-line annotation removed: counts 2, so it was worth 1" "$out" \ + " baseline DEVIATION: the tree carries 2 tests marked \`@FailsOnEmulatorApi37\` but the baseline says 3 — update FAILS_ON_EMULATOR_API37_BASELINE" + +echo +if [ "$failures" -eq 0 ]; then + echo "e2e-report-shape-test.sh: all checks passed" + exit 0 +fi +echo "e2e-report-shape-test.sh: $failures check(s) failed" +exit 1 diff --git a/.github/scripts/e2e-report-shape.sh b/.github/scripts/e2e-report-shape.sh index 4521738..7b42321 100755 --- a/.github/scripts/e2e-report-shape.sh +++ b/.github/scripts/e2e-report-shape.sh @@ -30,16 +30,31 @@ # # Usage: # e2e-report-shape.sh