From ee6d5e09606d20ce9bc11c12d9409147bbd4c2c1 Mon Sep 17 00:00:00 2001 From: Jason Ross Date: Fri, 20 Mar 2026 21:07:16 -0500 Subject: [PATCH] change artifact upload to Cloudflare R2 and auth pattern uses GCP Secrets manager --- .github/workflows/release-prefect-dagger.yml | 50 ++++---- CLAUDE.md | 9 +- ci/prefect_flow.py | 105 ++++++++++------ ci/r2_upload.py | 120 +++++++++++++++++++ pyproject.toml | 2 + 5 files changed, 228 insertions(+), 58 deletions(-) create mode 100644 ci/r2_upload.py diff --git a/.github/workflows/release-prefect-dagger.yml b/.github/workflows/release-prefect-dagger.yml index 3702bbb..299bd0e 100644 --- a/.github/workflows/release-prefect-dagger.yml +++ b/.github/workflows/release-prefect-dagger.yml @@ -4,7 +4,7 @@ # Architecture: # - Windows build runs natively on windows-latest (cannot containerize) # - Linux builds run via Dagger containers orchestrated by Prefect -# - Signing, release creation, and S3 upload handled by Prefect tasks +# - Signing, release creation, and R2 upload handled by Prefect tasks # - Container runtime: Podman (Dagger connects via Podman socket) # # Local equivalent: @@ -206,7 +206,7 @@ jobs: dist/android-file-handler-rhel.sha256 pkg_dist_rhel/** - # ── Release + S3 Upload (Prefect) ─────────────────────────────────── + # ── Release + R2 Upload (Prefect) ─────────────────────────────────── do-release: needs: [build-windows, build-linux] runs-on: ubuntu-latest @@ -246,42 +246,52 @@ jobs: poetry run python -m ci.prefect_flow release \ --github-token "$GITHUB_TOKEN" - upload-s3: + upload-r2: runs-on: ubuntu-latest needs: do-release if: needs.do-release.result == 'success' + permissions: + contents: read + id-token: write steps: - name: Checkout code uses: actions/checkout@v4 with: ref: main - - name: Install AWS CLI - run: python -m pip install --upgrade pip awscli + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: latest + virtualenvs-create: true + virtualenvs-in-project: true + + - name: Install project + CI dependencies + run: poetry install --with ci - name: Download build artifacts uses: actions/download-artifact@v4 with: merge-multiple: true - path: ./binaries + path: ./release-files - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v2 + - name: Authenticate to GCP + uses: google-github-actions/auth@v2 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} - - name: Upload artifacts to S3 + - name: Upload artifacts to Cloudflare R2 run: | - set -euo pipefail - if [ -z "${{ secrets.S3_BUCKET }}" ]; then - echo "S3_BUCKET secret not set; skipping upload" - exit 0 - fi - aws s3 sync ./binaries s3://${{ secrets.S3_BUCKET }}/builds/${{ github.run_id }}/ --acl private - env: - AWS_PAGER: "" + poetry run python -m ci.prefect_flow upload-r2 \ + --gcp-project-id "${{ secrets.GCP_PROJECT_ID }}" \ + --run-id "${{ github.run_id }}" \ + --release-dir "./release-files" sync-wiki: needs: do-release diff --git a/CLAUDE.md b/CLAUDE.md index dc2784f..dd455e0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -73,6 +73,10 @@ poetry run python -m ci.prefect_flow sign --gpg-passphrase "$GPG_PASSPHRASE" poetry run python -m ci.prefect_flow full \ --gpg-passphrase "$GPG_PASSPHRASE" \ --github-token "$GITHUB_TOKEN" + +# Upload artifacts to Cloudflare R2 (standalone) +poetry run python -m ci.prefect_flow upload-r2 \ + --gcp-project-id "$GCP_PROJECT_ID" ``` #### Podman Compose Build (Recommended for Linux) @@ -163,7 +167,7 @@ poetry run pyinstaller scripts/spec_scripts/android-file-handler-windows.spec The project uses a **Prefect + Dagger** pipeline wrapped by GitHub Actions (`.github/workflows/release-prefect-dagger.yml`): - **Dagger** runs containerized Linux builds (Debian, Arch, RHEL) using pre-built builder images -- **Prefect** orchestrates the pipeline: build → sign → release → S3 upload +- **Prefect** orchestrates the pipeline: build → sign → release → R2 upload - **GitHub Actions** provides the runner infrastructure and Windows build (cannot containerize) - **Podman** is the container runtime (Dagger connects via Podman socket) @@ -171,13 +175,14 @@ Pipeline structure: 1. `build-windows` — Native Windows build on `windows-latest` 2. `build-linux` — All Linux distros built in parallel via Prefect + Dagger 3. `do-release` — Creates GitHub release with all artifacts -4. `upload-s3` — Optional S3 upload +4. `upload-r2` — Uploads artifacts to Cloudflare R2 (credentials from GCP Secrets Manager) 5. `sync-wiki` — Wiki synchronization The CI pipeline modules live in `ci/`: - `ci/config.py` — Shared build configuration - `ci/dagger_pipeline.py` — Dagger container build definitions - `ci/prefect_flow.py` — Prefect flow orchestration and CLI +- `ci/r2_upload.py` — Cloudflare R2 upload with GCP Secrets Manager integration - `ci/signing.py` — GPG signing and SHA-256 hashing utilities ## Coding Standards diff --git a/ci/prefect_flow.py b/ci/prefect_flow.py index cb61d2b..f37858a 100644 --- a/ci/prefect_flow.py +++ b/ci/prefect_flow.py @@ -1,7 +1,7 @@ """Prefect orchestration flow for the CI/CD release pipeline. Coordinates Dagger-based Linux builds, GPG signing, GitHub release -creation, and S3 artifact upload. +creation, and Cloudflare R2 artifact upload. Usage: # Build all Linux distros (CI) @@ -10,10 +10,10 @@ Usage: # Sign artifacts in dist/ poetry run python -m ci.prefect_flow sign --gpg-passphrase "$GPG_PASSPHRASE" - # Create GitHub release + upload S3 + # Create GitHub release + upload R2 poetry run python -m ci.prefect_flow release --github-token "$GITHUB_TOKEN" - # Full pipeline (build + sign + release + S3) + # Full pipeline (build + sign + release + R2) poetry run python -m ci.prefect_flow full --gpg-passphrase "$GPG_PASSPHRASE" \\ --github-token "$GITHUB_TOKEN" """ @@ -28,6 +28,7 @@ from prefect import flow, task from ci.config import PipelineConfig from ci.dagger_pipeline import build_all_linux +from ci.r2_upload import upload_to_r2 from ci.signing import sign_and_hash # --------------------------------------------------------------------------- @@ -159,32 +160,30 @@ def task_create_github_release( print(f"GitHub release {tag} created successfully") -@task(name="upload-s3") -def task_upload_s3( +@task(name="upload-r2") +def task_upload_r2( release_dir: Path, - s3_bucket: str, + gcp_project_id: str, run_id: str, -) -> None: - """Upload release artifacts to S3. +) -> list[str]: + """Upload release artifacts to Cloudflare R2. + + Credentials are fetched from GCP Secrets Manager at runtime. Args: release_dir: Directory containing release files. - s3_bucket: S3 bucket name. + gcp_project_id: GCP project ID for Secrets Manager lookups. run_id: Unique identifier for this build run. + + Returns: + List of uploaded R2 object keys. """ - if not s3_bucket: - print("S3_BUCKET not set; skipping upload") - return + if not gcp_project_id: + print("GCP_PROJECT_ID not set; skipping R2 upload") + return [] - target = f"s3://{s3_bucket}/builds/{run_id}/" - print(f"Uploading to {target}") - - subprocess.run( - ["aws", "s3", "sync", str(release_dir), target, "--acl", "private"], - check=True, - env={**os.environ, "AWS_PAGER": ""}, - ) - print("S3 upload complete") + print(f"Uploading to Cloudflare R2 (build {run_id})") + return upload_to_r2(release_dir, gcp_project_id, run_id) # --------------------------------------------------------------------------- @@ -209,10 +208,10 @@ def flow_sign(gpg_passphrase: str) -> list[Path]: @flow(name="release-flow", log_prints=True) def flow_release( github_token: str, - s3_bucket: str = "", + gcp_project_id: str = "", run_id: str = "", ) -> None: - """Create a GitHub release and optionally upload to S3.""" + """Create a GitHub release and optionally upload to Cloudflare R2.""" config = PipelineConfig() version = task_get_version() @@ -220,30 +219,52 @@ def flow_release( task_prepare_release_files(config.project_root / "dist", release_dir) task_create_github_release(version, release_dir, github_token) - if s3_bucket: - task_upload_s3(release_dir, s3_bucket, run_id or "local") + if gcp_project_id: + task_upload_r2(release_dir, gcp_project_id, run_id or "local") + + +@flow(name="upload-r2-flow", log_prints=True) +def flow_upload_r2( + gcp_project_id: str, + run_id: str = "", + release_dir: str = "", +) -> list[str]: + """Upload release artifacts to Cloudflare R2 (standalone). + + Args: + gcp_project_id: GCP project ID for Secrets Manager lookups. + run_id: Build run identifier for R2 path. + release_dir: Path to directory containing artifacts. Defaults to + /release-files. + + Returns: + List of uploaded R2 object keys. + """ + config = PipelineConfig() + target_dir = Path(release_dir) if release_dir else config.project_root / "release-files" + return task_upload_r2(target_dir, gcp_project_id, run_id or "local") # type: ignore[return-value] @flow(name="full-pipeline", log_prints=True) def flow_full_pipeline( gpg_passphrase: str = "", github_token: str = "", - s3_bucket: str = "", + gcp_project_id: str = "", run_id: str = "", skip_build: bool = False, skip_sign: bool = False, skip_release: bool = False, ) -> None: - """Run the complete CI/CD pipeline: build → sign → release → S3. + """Run the complete CI/CD pipeline: build → sign → release → R2. Args: gpg_passphrase: GPG key passphrase for signing. github_token: GitHub token for release creation. - s3_bucket: Optional S3 bucket for artifact upload. - run_id: Build run identifier for S3 path. + gcp_project_id: GCP project ID for R2 credential lookup. + run_id: Build run identifier for R2 path. skip_build: Skip the Linux build step. skip_sign: Skip the signing step. - skip_release: Skip the release + S3 step. + skip_release: Skip the release + R2 step. """ config = PipelineConfig() @@ -266,8 +287,8 @@ def flow_full_pipeline( task_prepare_release_files(config.project_root / "dist", release_dir) task_create_github_release(version, release_dir, github_token) - if s3_bucket: - task_upload_s3(release_dir, s3_bucket, run_id or "local") + if gcp_project_id: + task_upload_r2(release_dir, gcp_project_id, run_id or "local") # --------------------------------------------------------------------------- @@ -292,14 +313,20 @@ def main() -> None: # release release_parser = subparsers.add_parser("release", help="Create GitHub release") release_parser.add_argument("--github-token", required=True, help="GitHub token") - release_parser.add_argument("--s3-bucket", default="", help="S3 bucket name") + release_parser.add_argument("--gcp-project-id", default="", help="GCP project ID for R2 credentials") release_parser.add_argument("--run-id", default="", help="Build run ID") + # upload-r2 + r2_parser = subparsers.add_parser("upload-r2", help="Upload artifacts to Cloudflare R2") + r2_parser.add_argument("--gcp-project-id", required=True, help="GCP project ID for R2 credentials") + r2_parser.add_argument("--run-id", default="", help="Build run ID") + r2_parser.add_argument("--release-dir", default="", help="Path to artifact directory") + # full full_parser = subparsers.add_parser("full", help="Run full pipeline") full_parser.add_argument("--gpg-passphrase", default="", help="GPG passphrase") full_parser.add_argument("--github-token", default="", help="GitHub token") - full_parser.add_argument("--s3-bucket", default="", help="S3 bucket name") + full_parser.add_argument("--gcp-project-id", default="", help="GCP project ID for R2 credentials") full_parser.add_argument("--run-id", default="", help="Build run ID") full_parser.add_argument("--skip-build", action="store_true") full_parser.add_argument("--skip-sign", action="store_true") @@ -314,14 +341,20 @@ def main() -> None: elif args.action == "release": flow_release( github_token=args.github_token, - s3_bucket=args.s3_bucket, + gcp_project_id=args.gcp_project_id, run_id=args.run_id, ) + elif args.action == "upload-r2": + flow_upload_r2( + gcp_project_id=args.gcp_project_id, + run_id=args.run_id, + release_dir=args.release_dir, + ) elif args.action == "full": flow_full_pipeline( gpg_passphrase=args.gpg_passphrase, github_token=args.github_token, - s3_bucket=args.s3_bucket, + gcp_project_id=args.gcp_project_id, run_id=args.run_id, skip_build=args.skip_build, skip_sign=args.skip_sign, diff --git a/ci/r2_upload.py b/ci/r2_upload.py new file mode 100644 index 0000000..eb67a35 --- /dev/null +++ b/ci/r2_upload.py @@ -0,0 +1,120 @@ +"""Cloudflare R2 artifact upload with credentials from GCP Secrets Manager.""" + +import mimetypes +from typing import Any +from pathlib import Path + +import boto3 # type: ignore[import-untyped] +from google.cloud import secretmanager # type: ignore[import-untyped] + +# GCP Secret Manager secret names for R2 credentials +_R2_ACCESS_KEY_SECRET = "r2-access-key-id" +_R2_SECRET_KEY_SECRET = "r2-secret-access-key" +_R2_ENDPOINT_SECRET = "r2-endpoint-url" +_R2_BUCKET_SECRET = "r2-bucket-name" + + +def _fetch_secret(client: Any, project_id: str, secret_id: str) -> str: + """Fetch the latest version of a secret from GCP Secrets Manager. + + Args: + client: Secret Manager client. + project_id: GCP project ID. + secret_id: Name of the secret to retrieve. + + Returns: + The secret value as a string. + + Raises: + google.api_core.exceptions.NotFound: If the secret does not exist. + """ + name = f"projects/{project_id}/secrets/{secret_id}/versions/latest" + response = client.access_secret_version(request={"name": name}) + return response.payload.data.decode("utf-8") + + +def get_r2_credentials(gcp_project_id: str) -> dict[str, str]: + """Retrieve all Cloudflare R2 credentials from GCP Secrets Manager. + + Args: + gcp_project_id: GCP project ID containing the secrets. + + Returns: + Dictionary with keys: access_key_id, secret_access_key, + endpoint_url, bucket_name. + """ + client: Any = secretmanager.SecretManagerServiceClient() # pyright: ignore + + return { + "access_key_id": _fetch_secret(client, gcp_project_id, _R2_ACCESS_KEY_SECRET), + "secret_access_key": _fetch_secret( + client, gcp_project_id, _R2_SECRET_KEY_SECRET + ), + "endpoint_url": _fetch_secret(client, gcp_project_id, _R2_ENDPOINT_SECRET), + "bucket_name": _fetch_secret(client, gcp_project_id, _R2_BUCKET_SECRET), + } + + +def upload_to_r2( + release_dir: Path, + gcp_project_id: str, + run_id: str, +) -> list[str]: + """Upload release artifacts to Cloudflare R2. + + Fetches R2 credentials from GCP Secrets Manager, then uploads all + files in the release directory to the R2 bucket under a builds// + prefix. + + Args: + release_dir: Directory containing release files to upload. + gcp_project_id: GCP project ID for Secrets Manager lookups. + run_id: Unique identifier for this build run. + + Returns: + List of uploaded R2 object keys. + + Raises: + FileNotFoundError: If release_dir does not exist. + botocore.exceptions.ClientError: If R2 upload fails. + """ + if not release_dir.is_dir(): + raise FileNotFoundError(f"Release directory not found: {release_dir}") + + credentials = get_r2_credentials(gcp_project_id) + + s3_client: Any = boto3.client( # pyright: ignore + "s3", + endpoint_url=credentials["endpoint_url"], + aws_access_key_id=credentials["access_key_id"], + aws_secret_access_key=credentials["secret_access_key"], + ) + + bucket = credentials["bucket_name"] + prefix = f"builds/{run_id}" + uploaded_keys: list[str] = [] + + for file_path in sorted(release_dir.iterdir()): + if not file_path.is_file(): + continue + + key = f"{prefix}/{file_path.name}" + content_type, _ = mimetypes.guess_type(str(file_path)) + + extra_args: dict[str, str] = {} + if content_type: + extra_args["ContentType"] = content_type + + print(f" Uploading {file_path.name} → {key}") + s3_client.upload_file( # pyright: ignore[reportUnknownMemberType] + str(file_path), + bucket, + key, + ExtraArgs=extra_args, + ) + uploaded_keys.append(key) + + print( + f"Uploaded {len(uploaded_keys)} files to R2 bucket '{bucket}' under '{prefix}/'" + ) + return uploaded_keys diff --git a/pyproject.toml b/pyproject.toml index f04a458..7dc6270 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,8 @@ optional = true [tool.poetry.group.ci.dependencies] dagger-io = ">=0.15.0" prefect = ">=3.0.0" +boto3 = ">=1.35.0" +google-cloud-secret-manager = ">=2.21.0" [tool.black] line-length = 88