diff --git a/.github/scripts/e2e-report-shape-test.sh b/.github/scripts/e2e-report-shape-test.sh index 56894c2..7046ac3 100755 --- a/.github/scripts/e2e-report-shape-test.sh +++ b/.github/scripts/e2e-report-shape-test.sh @@ -204,6 +204,10 @@ TRUNCATED out="$(run_report "$root")" assert_contains "truncated run: the truncation is reported" "$out" ' completed cleanly: no' assert_absent "truncated run: the short failure count is not a deviation" "$out" 'tests failed, the baseline is' +# And the match line has to say what actually happened rather than repeat the baseline: PR #245's +# advisory leg printed `failed: 4` three lines above `matches (5 expected, 5 failed)`. +assert_contains "truncated run: the match line does not claim the baseline's failure count" "$out" \ + ' baseline: matches (3 expected; 2 of 3 failed, on a run the abort truncated — not compared)' # --------------------------------------------------------------------------- # 5. The same short failure count on a run that finished IS a deviation. diff --git a/.github/scripts/e2e-report-shape.sh b/.github/scripts/e2e-report-shape.sh index 48c10af..e8805f2 100755 --- a/.github/scripts/e2e-report-shape.sh +++ b/.github/scripts/e2e-report-shape.sh @@ -297,7 +297,16 @@ if [ -n "$failed_names" ]; then fi if [ "$advisory" = "yes" ]; then if [ "${#deviations[@]}" -eq 0 ]; then - echo " baseline: matches ($baseline expected, $baseline failed)" + # Two spellings, because one of them would be a lie half the time. `$baseline expected, + # $baseline failed` is only true of a run that finished; on a truncated one `failed` is a + # tally of the tests that got to run before the framework died, and printing the baseline in + # its place claims a number nobody measured. Seen on PR #245's advisory leg, which reported + # `failed: 4` three lines above `matches (5 expected, 5 failed)`. + if [ "$failed" != "unknown" ] && [ "$failed" != "$baseline" ]; then + echo " baseline: matches ($baseline expected; $failed of $baseline failed, on a run the abort truncated — not compared)" + else + echo " baseline: matches ($baseline expected, $baseline failed)" + fi else printf ' baseline DEVIATION: %s\n' "${deviations[@]}" fi