From 8556de0efb498ecd9ae5733ec105d60cea038e1f Mon Sep 17 00:00:00 2001 From: Jason Ross Date: Thu, 16 Oct 2025 19:54:21 -0500 Subject: [PATCH] release workflow fixes and updates --- .github/workflows/release.yml | 406 +++++++++++++++++++++++++--------- 1 file changed, 298 insertions(+), 108 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 435a6e2..8569e33 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,6 +25,16 @@ on: description: "Comma-separated jobs to run (e.g., build-windows,build-debian,build-arch,build-rhel)" required: true default: "build-windows,build-debian,build-arch,build-rhel" + start_from_step: + description: "Start workflow from this step (all subsequent steps will run)" + required: false + type: choice + default: "bump-version" + options: + - "bump-version" + - "merge-develop-to-main" + - "build-artifacts" + - "do-release" permissions: contents: write @@ -45,6 +55,7 @@ env: jobs: bump-version: + if: ${{ github.event.inputs.start_from_step == 'bump-version' }} runs-on: ubuntu-latest permissions: contents: write @@ -179,6 +190,7 @@ jobs: wait-for-version-pr: needs: [bump-version] + if: ${{ github.event.inputs.start_from_step == 'bump-version' }} runs-on: ubuntu-latest permissions: contents: read @@ -275,21 +287,28 @@ jobs: merge-develop-to-main: needs: [wait-for-version-pr] + if: | + ${{ + always() && + (github.event.inputs.start_from_step == 'bump-version' || github.event.inputs.start_from_step == 'merge-develop-to-main') && + (needs.wait-for-version-pr.result == 'success' || needs.wait-for-version-pr.result == 'skipped') + }} runs-on: ubuntu-latest permissions: contents: write + pull-requests: write # Ensure only one merge operation runs at a time concurrency: group: main-branch-merge cancel-in-progress: false outputs: - merge_commit_sha: ${{ steps.merge.outputs.merge_commit_sha }} - previous_main_sha: ${{ steps.merge.outputs.previous_main_sha }} + pr_number: ${{ steps.create-pr.outputs.pr_number }} + previous_main_sha: ${{ steps.check-branches.outputs.previous_main_sha }} steps: - name: Checkout code uses: actions/checkout@v4 with: - ref: main + ref: develop fetch-depth: 0 token: ${{ secrets.GITHUB_TOKEN }} @@ -298,13 +317,15 @@ jobs: git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - - name: Fast-forward main to develop - id: merge + - name: Check branches and verify merge readiness + id: check-branches + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -e git fetch origin main develop - # Store current main SHA for potential rollback + # Store current main SHA for reference PREVIOUS_MAIN_SHA=$(git rev-parse origin/main) echo "previous_main_sha=$PREVIOUS_MAIN_SHA" >> $GITHUB_OUTPUT echo "Previous main SHA: $PREVIOUS_MAIN_SHA" @@ -322,130 +343,176 @@ jobs: # Ensure develop is actually ahead DEVELOP_SHA=$(git rev-parse origin/develop) if [ "$MAIN_SHA" = "$DEVELOP_SHA" ]; then - echo "Main is already up to date with develop. Nothing to merge." - echo "merge_commit_sha=$MAIN_SHA" >> $GITHUB_OUTPUT + echo "::warning::Main is already up to date with develop. Nothing to merge." exit 0 fi - # Merge develop into main (create merge commit for revert capability) - git checkout main - git merge origin/develop --no-ff -m "Merge develop into main for release" - - echo "Successfully merged develop into main" - git log origin/main..HEAD --oneline - - # Push with error handling - if ! git push origin main; then - echo "::error::Failed to push merge commit to main" + # Check for existing open PR from develop to main + EXISTING_PR=$(gh pr list --base main --head develop --state open --json number --jq '.[0].number' || echo "") + if [ -n "$EXISTING_PR" ]; then + echo "::error::PR #$EXISTING_PR already exists from develop to main" + echo "::error::Please close or merge the existing PR before creating a new one" exit 1 fi - echo "Successfully pushed merge commit to main" - # Store the new merge commit SHA - MERGE_COMMIT_SHA=$(git rev-parse HEAD) - echo "merge_commit_sha=$MERGE_COMMIT_SHA" >> $GITHUB_OUTPUT - echo "New main SHA: $MERGE_COMMIT_SHA" + echo "✓ Ready to create PR from develop to main" - verify-main-status-checks: + - name: Create PR from develop to main + id: create-pr + env: + GH_TOKEN: ${{ secrets.CI_CD_PAT }} + run: | + set -e + VERSION="${{ github.event.inputs.new_version }}" + + # Create PR with auto-merge enabled + PR_URL=$(gh pr create \ + --base main \ + --head develop \ + --title "Release v${VERSION} - Merge develop into main" \ + --body "This PR merges develop into main for release v${VERSION}. + + **Auto-generated by release workflow** + + Once status checks pass, this PR will be automatically merged." \ + --repo ${{ github.repository }} || { + echo "::error::Failed to create PR" + exit 1 + }) + + # Extract PR number from URL + PR_NUMBER=$(echo "$PR_URL" | grep -oP '\d+$') + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT + echo "Created PR #$PR_NUMBER: $PR_URL" + + # Enable auto-merge (merge commit to maintain history) + if ! gh pr merge "$PR_NUMBER" --auto --merge --repo ${{ github.repository }}; then + echo "::error::Failed to enable auto-merge for PR #$PR_NUMBER" + exit 1 + fi + echo "Auto-merge enabled for PR #$PR_NUMBER" + + wait-for-main-pr: needs: [merge-develop-to-main] + if: | + ${{ + always() && + (github.event.inputs.start_from_step == 'bump-version' || github.event.inputs.start_from_step == 'merge-develop-to-main') && + needs.merge-develop-to-main.result == 'success' + }} runs-on: ubuntu-latest permissions: - contents: write - actions: read - # Prevent multiple rollback operations from running concurrently - concurrency: - group: main-branch-verify-and-rollback - cancel-in-progress: false + contents: read + pull-requests: read + outputs: + merge_commit_sha: ${{ steps.wait-merge.outputs.merge_commit_sha }} steps: - - name: Checkout main branch + - name: Checkout repository uses: actions/checkout@v4 - with: - ref: main - - name: Verify commit status via GitHub API - id: check-status + - name: Wait for PR status checks and merge + id: wait-merge env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -e - MERGE_COMMIT="${{ needs.merge-develop-to-main.outputs.merge_commit_sha }}" - echo "Checking status of commit: $MERGE_COMMIT" + PR_NUMBER="${{ needs.merge-develop-to-main.outputs.pr_number }}" + echo "Monitoring PR #$PR_NUMBER for status checks..." - # Wait a moment for status checks to be registered - sleep 5 - - # Get commit status using GitHub API - MAX_WAIT=300 # 5 minutes max wait for status checks to appear - SLEEP_INTERVAL=10 + MAX_WAIT=${{ github.event.inputs.pr_check_timeout || 1800 }} + INITIAL_INTERVAL=10 + MAX_INTERVAL=300 # 5 minutes maximum + BACKOFF_MULTIPLIER=1.5 + SLEEP_INTERVAL=$INITIAL_INTERVAL ELAPSED=0 + RETRY_COUNT=0 + MAX_RETRIES=3 + echo "Max wait time: ${MAX_WAIT}s, using exponential backoff (max interval: ${MAX_INTERVAL}s)" while [ $ELAPSED -lt $MAX_WAIT ]; do - # Get combined status for the commit - STATUS_RESPONSE=$(gh api "repos/${{ github.repository }}/commits/${MERGE_COMMIT}/status" --jq '{state: .state, statuses: .statuses | length, total_count: .total_count}' || echo '{"state":"pending","statuses":0,"total_count":0}') + # Get PR status with retry logic + if ! PR_STATE=$(gh pr view "$PR_NUMBER" --json state --jq '.state' --repo ${{ github.repository }} 2>&1); then + RETRY_COUNT=$((RETRY_COUNT + 1)) + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "::error::Failed to fetch PR status after $MAX_RETRIES retries" + exit 1 + fi + echo "::warning::Failed to fetch PR status (attempt $RETRY_COUNT/$MAX_RETRIES), retrying..." + sleep $((2 ** RETRY_COUNT)) + continue + fi + RETRY_COUNT=0 - STATE=$(echo "$STATUS_RESPONSE" | jq -r '.state') - TOTAL_COUNT=$(echo "$STATUS_RESPONSE" | jq -r '.total_count') + if [ "$PR_STATE" = "MERGED" ]; then + echo "✓ PR #$PR_NUMBER has been merged successfully!" - echo "Commit status: state=$STATE, checks=$TOTAL_COUNT (${ELAPSED}s elapsed)" - - if [ "$STATE" = "failure" ] || [ "$STATE" = "error" ]; then - echo "::error::Status checks failed on commit $MERGE_COMMIT" - gh api "repos/${{ github.repository }}/commits/${MERGE_COMMIT}/status" --jq '.statuses[] | select(.state == "failure" or .state == "error") | "- " + .context + ": " + .state' - exit 1 - elif [ "$STATE" = "success" ]; then - echo "✓ All status checks passed on main branch" + # Get the merge commit SHA + MERGE_COMMIT_SHA=$(gh pr view "$PR_NUMBER" --json mergeCommit --jq '.mergeCommit.oid' --repo ${{ github.repository }}) + echo "merge_commit_sha=$MERGE_COMMIT_SHA" >> $GITHUB_OUTPUT + echo "Merge commit SHA: $MERGE_COMMIT_SHA" exit 0 fi + if [ "$PR_STATE" = "CLOSED" ]; then + echo "::error::PR #$PR_NUMBER was closed without merging" + exit 1 + fi + + # Check status checks with retry logic + RETRY_COUNT=0 + if ! STATUS_JSON=$(gh pr view "$PR_NUMBER" --json statusCheckRollup --jq '.statusCheckRollup' --repo ${{ github.repository }} 2>&1); then + RETRY_COUNT=$((RETRY_COUNT + 1)) + if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then + echo "::error::Failed to fetch status checks after $MAX_RETRIES retries" + exit 1 + fi + echo "::warning::Failed to fetch status checks (attempt $RETRY_COUNT/$MAX_RETRIES), retrying..." + sleep $((2 ** RETRY_COUNT)) + continue + fi + + # Count check states + TOTAL=$(echo "$STATUS_JSON" | jq 'length') + COMPLETED=$(echo "$STATUS_JSON" | jq '[.[] | select(.conclusion != null)] | length') + SUCCESS=$(echo "$STATUS_JSON" | jq '[.[] | select(.conclusion == "SUCCESS" or .conclusion == "NEUTRAL" or .conclusion == "SKIPPED")] | length') + FAILED=$(echo "$STATUS_JSON" | jq '[.[] | select(.conclusion == "FAILURE" or .conclusion == "CANCELLED" or .conclusion == "TIMED_OUT")] | length') + + echo "Status checks: $COMPLETED/$TOTAL completed, $SUCCESS passed, $FAILED failed (interval: ${SLEEP_INTERVAL}s)" + + # Check for failures + if [ "$FAILED" -gt 0 ]; then + echo "::error::Status checks failed for PR #$PR_NUMBER" + gh pr view "$PR_NUMBER" --json statusCheckRollup --jq '.statusCheckRollup[] | select(.conclusion == "FAILURE" or .conclusion == "CANCELLED" or .conclusion == "TIMED_OUT") | "- " + .name + ": " + .conclusion' --repo ${{ github.repository }} + exit 1 + fi + + echo "Waiting for checks to complete... (${ELAPSED}s elapsed)" sleep $SLEEP_INTERVAL ELAPSED=$((ELAPSED + SLEEP_INTERVAL)) + + # Calculate next interval with exponential backoff (capped at MAX_INTERVAL) + NEXT_INTERVAL=$(awk "BEGIN {printf \"%.0f\", $SLEEP_INTERVAL * $BACKOFF_MULTIPLIER}") + if [ $NEXT_INTERVAL -gt $MAX_INTERVAL ]; then + SLEEP_INTERVAL=$MAX_INTERVAL + else + SLEEP_INTERVAL=$NEXT_INTERVAL + fi done - # If we get here, treat as success if no checks were registered - if [ "$TOTAL_COUNT" -eq 0 ]; then - echo "⚠ No status checks found for commit. Investigate." - exit 1 - fi - - echo "::warning::Status checks still pending after ${MAX_WAIT}s, ending workflow. Investigate status checks" - exit 1 - - - name: Handle status check failures with rollback - if: failure() && steps.check-status.outcome == 'failure' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -e - echo "::error::Status checks failed on main branch - initiating rollback" - - # Configure git - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - # Fetch latest - git fetch origin main - - # Get the merge commit that needs to be reverted - MERGE_COMMIT="${{ needs.merge-develop-to-main.outputs.merge_commit_sha }}" - echo "Reverting merge commit: $MERGE_COMMIT" - - # Checkout main and create revert commit - git checkout main - git revert "$MERGE_COMMIT" --no-edit -m 1 - - # Push the revert commit with error handling - if ! git push origin main; then - echo "::error::Failed to push revert commit" - exit 1 - fi - - echo "::error::Reverted merge commit $MERGE_COMMIT on main branch" - echo "::error::Workflow failed due to status check failures" + echo "::error::Timeout waiting for PR #$PR_NUMBER to merge" exit 1 build-windows: - needs: [verify-main-status-checks] - if: ${{ contains(github.event.inputs.jobs, 'build-windows') }} + needs: [wait-for-main-pr] + if: | + ${{ + always() && + contains(github.event.inputs.jobs, 'build-windows') && + (github.event.inputs.start_from_step == 'bump-version' || + github.event.inputs.start_from_step == 'merge-develop-to-main' || + github.event.inputs.start_from_step == 'build-artifacts') && + (needs.wait-for-main-pr.result == 'success' || needs.wait-for-main-pr.result == 'skipped') + }} runs-on: windows-latest steps: - name: Checkout code @@ -531,8 +598,16 @@ jobs: dist/android-file-handler-windows.sha256 build-debian: - needs: [verify-main-status-checks] - if: ${{ contains(github.event.inputs.jobs, 'build-debian') }} + needs: [wait-for-main-pr] + if: | + ${{ + always() && + contains(github.event.inputs.jobs, 'build-debian') && + (github.event.inputs.start_from_step == 'bump-version' || + github.event.inputs.start_from_step == 'merge-develop-to-main' || + github.event.inputs.start_from_step == 'build-artifacts') && + (needs.wait-for-main-pr.result == 'success' || needs.wait-for-main-pr.result == 'skipped') + }} permissions: contents: read packages: read @@ -625,8 +700,16 @@ jobs: pkg_dist_debian/** build-arch: - needs: [verify-main-status-checks] - if: ${{ contains(github.event.inputs.jobs, 'build-arch') }} + needs: [wait-for-main-pr] + if: | + ${{ + always() && + contains(github.event.inputs.jobs, 'build-arch') && + (github.event.inputs.start_from_step == 'bump-version' || + github.event.inputs.start_from_step == 'merge-develop-to-main' || + github.event.inputs.start_from_step == 'build-artifacts') && + (needs.wait-for-main-pr.result == 'success' || needs.wait-for-main-pr.result == 'skipped') + }} permissions: contents: read packages: read @@ -738,8 +821,16 @@ jobs: build-rhel: - needs: [verify-main-status-checks] - if: ${{ contains(github.event.inputs.jobs, 'build-rhel') }} + needs: [wait-for-main-pr] + if: | + ${{ + always() && + contains(github.event.inputs.jobs, 'build-rhel') && + (github.event.inputs.start_from_step == 'bump-version' || + github.event.inputs.start_from_step == 'merge-develop-to-main' || + github.event.inputs.start_from_step == 'build-artifacts') && + (needs.wait-for-main-pr.result == 'success' || needs.wait-for-main-pr.result == 'skipped') + }} permissions: contents: read packages: read @@ -829,11 +920,98 @@ jobs: dist/android-file-handler-rhel.sha256 pkg_dist_rhel/** + rollback-on-build-failure: + needs: [wait-for-main-pr, build-windows, build-debian, build-arch, build-rhel] + if: | + ${{ + always() && + (github.event.inputs.start_from_step == 'bump-version' || + github.event.inputs.start_from_step == 'merge-develop-to-main' || + github.event.inputs.start_from_step == 'build-artifacts') && + needs.wait-for-main-pr.result == 'success' && + (needs.build-windows.result == 'failure' || + needs.build-debian.result == 'failure' || + needs.build-arch.result == 'failure' || + needs.build-rhel.result == 'failure') + }} + runs-on: ubuntu-latest + permissions: + contents: write + # Prevent multiple rollback operations from running concurrently + concurrency: + group: main-branch-rollback + cancel-in-progress: false + steps: + - name: Checkout main branch + uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Rollback merge commit on build failure + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -e + echo "::error::One or more build jobs failed - initiating rollback" + + # Configure git + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # Fetch latest + git fetch origin main + + # Get the merge commit that needs to be reverted + MERGE_COMMIT="${{ needs.wait-for-main-pr.outputs.merge_commit_sha }}" + echo "Reverting merge commit: $MERGE_COMMIT" + + # Checkout main and create revert commit + git checkout main + git revert "$MERGE_COMMIT" --no-edit -m 1 + + # Push the revert commit with error handling + if ! git push origin main; then + echo "::error::Failed to push revert commit" + exit 1 + fi + + echo "::error::Reverted merge commit $MERGE_COMMIT on main branch" + echo "::error::Build failures detected:" + + # Report which builds failed + if [ "${{ needs.build-windows.result }}" = "failure" ]; then + echo "::error:: - build-windows: FAILED" + fi + if [ "${{ needs.build-debian.result }}" = "failure" ]; then + echo "::error:: - build-debian: FAILED" + fi + if [ "${{ needs.build-arch.result }}" = "failure" ]; then + echo "::error:: - build-arch: FAILED" + fi + if [ "${{ needs.build-rhel.result }}" = "failure" ]; then + echo "::error:: - build-rhel: FAILED" + fi + + exit 1 + - do-release: - needs: [build-windows, build-debian, build-arch, build-rhel] - if: ${{ needs.build-windows.result == 'success' && needs.build-debian.result == 'success' && needs.build-arch.result == 'success' && needs.build-rhel.result == 'success' }} + needs: [rollback-on-build-failure, build-windows, build-debian, build-arch, build-rhel] + if: | + ${{ + always() && + (github.event.inputs.start_from_step == 'bump-version' || + github.event.inputs.start_from_step == 'merge-develop-to-main' || + github.event.inputs.start_from_step == 'build-artifacts' || + github.event.inputs.start_from_step == 'do-release') && + needs.rollback-on-build-failure.result != 'failure' && + (needs.build-windows.result == 'success' || needs.build-windows.result == 'skipped') && + (needs.build-debian.result == 'success' || needs.build-debian.result == 'skipped') && + (needs.build-arch.result == 'success' || needs.build-arch.result == 'skipped') && + (needs.build-rhel.result == 'success' || needs.build-rhel.result == 'skipped') + }} runs-on: ubuntu-latest steps: - name: Checkout code @@ -882,8 +1060,20 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} upload-s3: - needs: [build-windows, build-debian, build-arch, build-rhel] - if: ${{ needs.build-windows.result == 'success' && needs.build-debian.result == 'success' && needs.build-arch.result == 'success' && needs.build-rhel.result == 'success' }} + needs: [rollback-on-build-failure, build-windows, build-debian, build-arch, build-rhel] + if: | + ${{ + always() && + (github.event.inputs.start_from_step == 'bump-version' || + github.event.inputs.start_from_step == 'merge-develop-to-main' || + github.event.inputs.start_from_step == 'build-artifacts' || + github.event.inputs.start_from_step == 'do-release') && + needs.rollback-on-build-failure.result != 'failure' && + (needs.build-windows.result == 'success' || needs.build-windows.result == 'skipped') && + (needs.build-debian.result == 'success' || needs.build-debian.result == 'skipped') && + (needs.build-arch.result == 'success' || needs.build-arch.result == 'skipped') && + (needs.build-rhel.result == 'success' || needs.build-rhel.result == 'skipped') + }} runs-on: ubuntu-latest steps: - name: Checkout code