From e7c3e5688fcd2b384f1e33ca703eb52ed744f372 Mon Sep 17 00:00:00 2001 From: Jason Ross Date: Sun, 6 Sep 2026 09:02:40 -0500 Subject: [PATCH] Stop the match line claiming a failure count nobody measured This PR's own advisory leg caught it. With five markers and a truncated run it printed failed: 4 ... baseline: matches (5 expected, 5 failed) three lines apart. The match line has always printed the baseline twice, which was true while `failed` had to equal it to get there -- and the previous commit removed that requirement for truncated runs without noticing this line depended on it. So the truncated spelling says what happened: `matches (5 expected; 4 of 5 failed, on a run the abort truncated -- not compared)`. Pinned by a third case beside the two from that commit. Co-Authored-By: Claude Opus 5 (1M context) --- .github/scripts/e2e-report-shape-test.sh | 4 ++++ .github/scripts/e2e-report-shape.sh | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/scripts/e2e-report-shape-test.sh b/.github/scripts/e2e-report-shape-test.sh index 56894c2..7046ac3 100755 --- a/.github/scripts/e2e-report-shape-test.sh +++ b/.github/scripts/e2e-report-shape-test.sh @@ -204,6 +204,10 @@ TRUNCATED out="$(run_report "$root")" assert_contains "truncated run: the truncation is reported" "$out" ' completed cleanly: no' assert_absent "truncated run: the short failure count is not a deviation" "$out" 'tests failed, the baseline is' +# And the match line has to say what actually happened rather than repeat the baseline: PR #245's +# advisory leg printed `failed: 4` three lines above `matches (5 expected, 5 failed)`. +assert_contains "truncated run: the match line does not claim the baseline's failure count" "$out" \ + ' baseline: matches (3 expected; 2 of 3 failed, on a run the abort truncated — not compared)' # --------------------------------------------------------------------------- # 5. The same short failure count on a run that finished IS a deviation. diff --git a/.github/scripts/e2e-report-shape.sh b/.github/scripts/e2e-report-shape.sh index 48c10af..e8805f2 100755 --- a/.github/scripts/e2e-report-shape.sh +++ b/.github/scripts/e2e-report-shape.sh @@ -297,7 +297,16 @@ if [ -n "$failed_names" ]; then fi if [ "$advisory" = "yes" ]; then if [ "${#deviations[@]}" -eq 0 ]; then - echo " baseline: matches ($baseline expected, $baseline failed)" + # Two spellings, because one of them would be a lie half the time. `$baseline expected, + # $baseline failed` is only true of a run that finished; on a truncated one `failed` is a + # tally of the tests that got to run before the framework died, and printing the baseline in + # its place claims a number nobody measured. Seen on PR #245's advisory leg, which reported + # `failed: 4` three lines above `matches (5 expected, 5 failed)`. + if [ "$failed" != "unknown" ] && [ "$failed" != "$baseline" ]; then + echo " baseline: matches ($baseline expected; $failed of $baseline failed, on a run the abort truncated — not compared)" + else + echo " baseline: matches ($baseline expected, $baseline failed)" + fi else printf ' baseline DEVIATION: %s\n' "${deviations[@]}" fi -- 2.47.3