diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..d3f5bf6 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,144 @@ +# Continuous deployment for the LibreMail bug-report ingest Worker + infra. +# +# MANUAL ONLY: this workflow never runs on push/PR. A maintainer triggers it from +# the Actions tab (workflow_dispatch), choosing a stack. It builds the TinyGo/Wasm +# Worker end to end (same setup as ci.yml) and then runs `pulumi up` over the +# infra/ program to deploy the Worker (with its R2 + Secrets Store + var bindings +# and Cron Triggers), the R2 bucket, and the Google Cloud DNS record. +# +# It is gated to the `production` GitHub Actions environment, so that environment's +# secrets and any required-reviewer / branch protection rules apply, and to the +# `main` branch (a guard step fails the run otherwise). Deploying is real and +# billable, hence manual + environment-gated + maintainer-run-from-main. +# +# Supply-chain note: every action (first- and third-party) is pinned to a full +# commit SHA with a trailing "# vX.Y.Z" comment, matching ci.yml / autoupdate.yml. +# +# Secrets/config the maintainer must set BEFORE the first deploy (see infra/README.md): +# production environment SECRETS (Settings > Environments > production): +# - PULUMI_ACCESS_TOKEN Pulumi Cloud access token (state backend). For a +# self-managed backend instead, set the `cloud-url` +# input + a PULUMI_CONFIG_PASSPHRASE secret. +# - CLOUDFLARE_API_TOKEN Cloudflare token scoped to Workers Scripts + R2 (+ Cron). +# - CLOUDFLARE_ACCOUNT_ID Cloudflare account id (also set as stack config). +# - GOOGLE_CREDENTIALS GCP service-account JSON with Cloud DNS admin on the zone. +# stack CONFIG (infra/Pulumi..yaml — replace every REPLACE_ME_* first): +# cloudflareAccountId, secretsStoreId, dnsManagedZone, dnsRecordName, +# dnsRecordTarget, gcp:project (+ optional r2/otel/dns overrides). +# Cloudflare Secrets Store must already hold the four secret values +# (bugreport-enc-keyring, bugreport-admin-token, github-token, +# otel-exporter-otlp-headers) under the configured secretsStoreId. + +name: CD + +on: + workflow_dispatch: + inputs: + stack: + description: 'Pulumi stack to deploy' + required: true + default: prod + type: choice + options: + - prod + - dev + +# Least privilege: the job only needs to read the repo out; Pulumi auth is via env. +permissions: + contents: read + +# Never run two deploys of the same stack concurrently; do not cancel an in-flight +# deploy (interrupting `pulumi up` can leave a stack mid-update). +concurrency: + group: cd-${{ github.event.inputs.stack }} + cancel-in-progress: false + +jobs: + deploy: + name: deploy + runs-on: ubuntu-latest + # Gate on the production environment so its secrets + protection rules apply. + environment: production + steps: + # Deploys must be cut from main. workflow_dispatch lets a user pick any ref, + # so fail loudly if this was launched from a non-main branch. + - name: Guard - deploy only from main + if: github.ref != 'refs/heads/main' + run: | + echo "::error::Deploy must be run from the 'main' branch (got '${{ github.ref }}')." + exit 1 + + - name: Check out repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + # Mirror ci.yml: cache both the root and infra/ module go.sum (infra pulls the + # heavy Pulumi SDKs) so warm runs restore deps instead of re-downloading. + - name: Set up Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: '1.26' + cache-dependency-path: | + go.sum + infra/go.sum + + # TinyGo builds the Wasm Worker (pnpm run build). Same version as ci.yml. + - name: Set up TinyGo + uses: acifani/setup-tinygo@dd8a7075d951a7595b2ef2123ed0ab1af0c13e56 # v3.0.0 + with: + tinygo-version: '0.41.1' + + - name: Set up pnpm + uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + with: + version: '10' + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '22' + cache: pnpm + + - name: Install Node dependencies + run: pnpm install --frozen-lockfile + + # TEMPORARY (tracking #26; tinygo-org/tinygo#5467): identical to ci.yml. TinyGo + # 0.41.1 vendors a net/http js/wasm overlay that fails to compile on Go 1.26; + # apply the exact upstream fix to the installed TinyGo source before building. + # git apply exits non-zero (failing loudly) if the source has drifted. + - name: Patch TinyGo net/http (temporary) + run: | + patch_file="$PWD/.ci/tinygo-net-roundtrip.patch" + tinygoroot="$(tinygo env TINYGOROOT)" + echo "Applying $patch_file to $tinygoroot/src/net/http/roundtrip_js.go" + git -C "$tinygoroot" apply --verbose "$patch_file" || { + echo "::error::TinyGo net/http patch did not apply cleanly; TinyGo source may have changed. Update or remove .ci/tinygo-net-roundtrip.patch (see #26)." + exit 1 + } + + # Produce build/worker.mjs (ES-module shim) + build/app.wasm. The infra program + # uploads the shim as the Worker's main module via the workerScriptPath config + # injected below. + - name: Build Wasm Worker + run: pnpm run build + + # Install the Pulumi CLI and run `pulumi up` over infra/. config-map injects the + # freshly built artifact path so the WorkersScript uploads the real module + # (ContentFile) instead of the placeholder. Provider + backend credentials come + # from the production environment secrets below; nothing secret is committed. + - name: Pulumi up + uses: pulumi/actions@8e5e406f4007fca908480587cb9893c07090f58d # v7.0.0 + with: + command: up + stack-name: ${{ github.event.inputs.stack }} + work-dir: infra + upsert: false + config-map: '{ "libremail-bug-report-ingest-infra:workerScriptPath": { value: "../build/worker.mjs", secret: false } }' + env: + # Pulumi state backend (Pulumi Cloud). For a self-managed backend, drop this, + # set the action's `cloud-url` input, and add PULUMI_CONFIG_PASSPHRASE. + PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} + # Cloudflare provider (Workers + R2 + Cron Triggers). + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + # GCP provider (Cloud DNS record). + GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS }} diff --git a/infra/Pulumi.dev.yaml b/infra/Pulumi.dev.yaml index e945db6..6d7cac7 100644 --- a/infra/Pulumi.dev.yaml +++ b/infra/Pulumi.dev.yaml @@ -12,8 +12,20 @@ config: # --- Program config (namespace = project name from Pulumi.yaml) --- libremail-bug-report-ingest-infra:cloudflareAccountId: REPLACE_ME_CLOUDFLARE_ACCOUNT_ID + # Cloudflare Secrets Store id holding the Worker's secrets (encryption keyring, + # admin token, GitHub token, OTLP headers). Account-specific; not itself secret. + libremail-bug-report-ingest-infra:secretsStoreId: REPLACE_ME_CLOUDFLARE_SECRETS_STORE_ID libremail-bug-report-ingest-infra:workerName: libremail-bug-report-ingest libremail-bug-report-ingest-infra:workerCompatibilityDate: "2025-06-01" + # Built Worker artifact (../build/worker.mjs from `pnpm run build`). Leave UNSET + # here: the CD workflow (.github/workflows/deploy.yml) injects it at deploy time + # via config-map so `pulumi preview` without a build still uses the placeholder. + # libremail-bug-report-ingest-infra:workerScriptPath: ../build/worker.mjs + # Plain (non-secret) Worker vars — mirror wrangler.jsonc "vars". + libremail-bug-report-ingest-infra:githubRepo: JMR-dev/LibreMail + # OTLP endpoint EMPTY disables telemetry until a collector is chosen (#17). + libremail-bug-report-ingest-infra:otelExporterOtlpEndpoint: "" + libremail-bug-report-ingest-infra:otelServiceName: libremail-bug-report-ingest libremail-bug-report-ingest-infra:r2BucketName: libremail-bug-reports-dev # R2 location hint (optional). One of: apac, eeur, enam, weur, wnam, oc. libremail-bug-report-ingest-infra:r2BucketLocation: enam diff --git a/infra/Pulumi.prod.yaml b/infra/Pulumi.prod.yaml index da4e0e1..3b1a9ab 100644 --- a/infra/Pulumi.prod.yaml +++ b/infra/Pulumi.prod.yaml @@ -12,8 +12,20 @@ config: # --- Program config (namespace = project name from Pulumi.yaml) --- libremail-bug-report-ingest-infra:cloudflareAccountId: REPLACE_ME_CLOUDFLARE_ACCOUNT_ID + # Cloudflare Secrets Store id holding the Worker's secrets (encryption keyring, + # admin token, GitHub token, OTLP headers). Account-specific; not itself secret. + libremail-bug-report-ingest-infra:secretsStoreId: REPLACE_ME_CLOUDFLARE_SECRETS_STORE_ID libremail-bug-report-ingest-infra:workerName: libremail-bug-report-ingest libremail-bug-report-ingest-infra:workerCompatibilityDate: "2025-06-01" + # Built Worker artifact (../build/worker.mjs from `pnpm run build`). Leave UNSET + # here: the CD workflow (.github/workflows/deploy.yml) injects it at deploy time + # via config-map so `pulumi preview` without a build still uses the placeholder. + # libremail-bug-report-ingest-infra:workerScriptPath: ../build/worker.mjs + # Plain (non-secret) Worker vars — mirror wrangler.jsonc "vars". + libremail-bug-report-ingest-infra:githubRepo: JMR-dev/LibreMail + # OTLP endpoint EMPTY disables telemetry until a collector is chosen (#17). + libremail-bug-report-ingest-infra:otelExporterOtlpEndpoint: "" + libremail-bug-report-ingest-infra:otelServiceName: libremail-bug-report-ingest libremail-bug-report-ingest-infra:r2BucketName: libremail-bug-reports # R2 location hint (optional). One of: apac, eeur, enam, weur, wnam, oc. libremail-bug-report-ingest-infra:r2BucketLocation: enam diff --git a/infra/README.md b/infra/README.md index 84a08f0..24198bc 100644 --- a/infra/README.md +++ b/infra/README.md @@ -10,22 +10,59 @@ It provisions, with the Pulumi Go SDK and the | Resource | Type | Purpose | | --- | --- | --- | -| `ingest-worker` | `cloudflare.WorkersScript` | The ingest Worker (`libremail-bug-report-ingest`, built in #1). | +| `ingest-worker` | `cloudflare.WorkersScript` | The ingest Worker (`libremail-bug-report-ingest`, built in #1), wired with its full binding set (R2 + Secrets Store + vars, see [Worker bindings](#worker-bindings)). | | `reports-bucket` | `cloudflare.R2Bucket` | Encrypted bug-report storage (`libremail-bug-reports`). Per [ADR 0001](../docs/decisions/encryption.md) only ciphertext is written. | +| `ingest-cron-triggers` | `cloudflare.WorkersCronTrigger` | The two Friday UTC Cron Triggers (#13) that drive the weekly publish job; bound to the ingest Worker. | | `ingest-dns-record` | `gcp.dns.RecordSet` | Google Cloud DNS record pointing the ingest hostname at the Worker's route/custom domain. | DNS authority is **Google Cloud DNS**. The managed zone is **referenced by name** (it already exists / is managed elsewhere), and this stack only adds a record to it. -> **Worker content is a placeholder.** The deployed Worker is Go compiled by -> TinyGo to Wasm plus the `syumai/workers` ES-module shim, emitted by -> `pnpm run build` into `../build/` (git-ignored, produced by CI). This program -> ships a documented placeholder module body so the resource is fully described -> and unit-testable without the artifact. Wire the real artifact at deploy time -> via the `workerScriptContent` config, or by setting `ContentFile` / -> `ContentSha256` on the Worker resource to `../build/worker.mjs` in the deploy -> pipeline. +> **Worker content: placeholder by default, real artifact at deploy.** The +> deployed Worker is Go compiled by TinyGo to Wasm plus the `syumai/workers` +> ES-module shim, emitted by `pnpm run build` into `../build/` (git-ignored, +> produced by CI). Set the **`workerScriptPath`** config to the built main module +> (`../build/worker.mjs`) and the program uploads it via `ContentFile` + +> `ContentSha256` (computed from the file). When `workerScriptPath` is unset, a +> documented placeholder module body is uploaded instead, so the resource is fully +> described and unit-testable without the artifact. The CD workflow +> ([`.github/workflows/deploy.yml`](../.github/workflows/deploy.yml)) builds the +> Worker and injects `workerScriptPath=../build/worker.mjs` at deploy time, so a +> local `pulumi preview` (no build) still works off the placeholder. +> +> Note: a TinyGo Worker is `worker.mjs` (main module) that imports `app.wasm`. +> `ContentFile` uploads the main module; if a functional deploy needs the wasm as a +> separate module part, add it alongside `worker.mjs` in the build output the +> pipeline points at. The bindings/crons below are provider-verified by the mock +> tests regardless of which content source is used. + +## Worker bindings + +The Worker resource carries the bindings the runtime code reads, matching +`wrangler.jsonc` and the Worker source (`internal/storage.*Binding`, +`worker/*_wasm.go`). This is the wiring #4 added (the gap #9 flagged); the mock +tests (`deploy_test.go`) assert every one of them: + +| Binding (JS var) | Type | Points at | +| --- | --- | --- | +| `REPORTS_BUCKET` | `r2_bucket` | `r2BucketName` (`libremail-bug-reports`). Encrypted report objects (ADR 0001). | +| `BUGREPORT_ENC_KEYRING` | `secrets_store_secret` | `secretsStoreId` / secret `bugreport-enc-keyring` — AES-256 keyring (ADR 0001). | +| `ADMIN_TOKEN` | `secrets_store_secret` | `secretsStoreId` / secret `bugreport-admin-token` — admin API token (ADR 0003). | +| `GITHUB_TOKEN` | `secrets_store_secret` | `secretsStoreId` / secret `github-token` — publish PAT (#14). | +| `OTEL_EXPORTER_OTLP_HEADERS` | `secrets_store_secret` | `secretsStoreId` / secret `otel-exporter-otlp-headers` — OTLP auth headers (#17). | +| `GITHUB_REPO` | `plain_text` | `githubRepo` (`JMR-dev/LibreMail`) — publish target (#14). | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | `plain_text` | `otelExporterOtlpEndpoint` (empty ⇒ telemetry disabled, #17). | +| `OTEL_SERVICE_NAME` | `plain_text` | `otelServiceName` (`libremail-bug-report-ingest`, #17). | + +The **Cron Triggers** resource registers both Friday UTC crons (`0 22 * * 5` and +`0 23 * * 5`) from `wrangler.jsonc` (#13); the Worker's scheduled handler gates on +the real America/Chicago rule so exactly one publishes each Friday. + +> The Secrets Store **secret values** themselves (keyring, admin token, GitHub +> token, OTLP headers) are **not** created by this stack — the maintainer stores +> them in Cloudflare Secrets Store under `secretsStoreId`. This stack only binds +> them to the Worker by name. ## Prerequisites @@ -46,18 +83,23 @@ project name, `libremail-bug-report-ingest-infra`. Provider keys use the | Key | Required | Default | Description | | --- | --- | --- | --- | | `cloudflareAccountId` | yes | — | Cloudflare account that owns the Worker + R2 bucket. | +| `secretsStoreId` | yes | — | Cloudflare **Secrets Store id** holding the Worker's secrets (keyring, admin token, GitHub token, OTLP headers). Account-specific; not itself secret. | | `dnsManagedZone` | yes | — | **Name** of the existing Google Cloud DNS managed zone. | | `dnsRecordName` | yes | — | Ingest hostname as an FQDN with a trailing dot (e.g. `bugreport.libremail.example.`). | | `dnsRecordTarget` | yes | — | CNAME target = the Worker route/custom domain (FQDN, trailing dot). | | `workerName` | no | `libremail-bug-report-ingest` | Worker script name (matches `wrangler.jsonc`). | | `workerCompatibilityDate` | no | `2025-06-01` | Worker runtime compatibility date. | -| `workerScriptContent` | no | placeholder | Override the Worker module body (normally supplied by the build pipeline). | -| `r2BucketName` | no | `libremail-bug-reports` | R2 bucket name. | +| `workerScriptPath` | no | — | Path to the built main module (`../build/worker.mjs`). When set, upload it via `ContentFile` + computed `ContentSha256`. Set by the CD workflow at deploy. | +| `workerScriptContent` | no | placeholder | Inline Worker module body used when `workerScriptPath` is unset. | +| `githubRepo` | no | `JMR-dev/LibreMail` | `GITHUB_REPO` var: `owner/repo` the weekly publish job files issues on (#14). | +| `otelExporterOtlpEndpoint` | no | `""` | `OTEL_EXPORTER_OTLP_ENDPOINT` var: OTLP base URL; empty disables telemetry (#17). | +| `otelServiceName` | no | `libremail-bug-report-ingest` | `OTEL_SERVICE_NAME` var: reported `service.name` (#17). | +| `r2BucketName` | no | `libremail-bug-reports` | R2 bucket name (also the `REPORTS_BUCKET` binding target). | | `r2BucketLocation` | no | (provider default) | R2 location hint: `apac`, `eeur`, `enam`, `weur`, `wnam`, `oc`. | | `dnsRecordType` | no | `CNAME` | DNS record type. | | `dnsTtlSeconds` | no | `300` | DNS record TTL (seconds). | | `gcpProject` | no | (from `gcp:project`) | Project the record belongs to, if different from the provider project. | -| `cloudflareZoneId` | no | — | **Reserved** for #7 (rate-limit ruleset) and Worker routes/custom domain. Unused today. | +| `cloudflareZoneId` | no | — | **Reserved** for #6/#7 (rate-limit ruleset) and Worker routes/custom domain. Unused today. | Set a non-secret value with, e.g.: @@ -82,25 +124,64 @@ Store**, bound to the Worker — it is managed there, not committed here. ## Deploying +### Via GitHub Actions (recommended) + +[`.github/workflows/deploy.yml`](../.github/workflows/deploy.yml) is a +**manual, `workflow_dispatch`-only** CD workflow. A maintainer runs it from the +Actions tab, on the **`main`** branch (a guard step fails otherwise), and picks a +`stack` (default `prod`). It reuses ci.yml's Go/TinyGo/pnpm setup + the TinyGo +net/http patch, runs `pnpm run build`, then `pulumi up --stack ` over this +program (injecting the built `../build/worker.mjs` as `workerScriptPath`). It is +gated to the **`production`** GitHub Actions environment, so that environment's +secrets and any required-reviewer / protection rules apply. + +**One-time setup before the first deploy** — the maintainer configures: + +1. **`production` environment secrets** (repo Settings → Environments → production): + + | Secret | Purpose | + | --- | --- | + | `PULUMI_ACCESS_TOKEN` | Pulumi Cloud token (state backend). Self-managed backend? Use the action's `cloud-url` input + a `PULUMI_CONFIG_PASSPHRASE` secret instead. | + | `CLOUDFLARE_API_TOKEN` | Cloudflare token scoped to Workers Scripts + R2 (+ Cron Triggers) on the account. | + | `CLOUDFLARE_ACCOUNT_ID` | Cloudflare account id. | + | `GOOGLE_CREDENTIALS` | GCP service-account JSON with Cloud DNS admin on the managed zone. | + +2. **Stack config** in `Pulumi..yaml` — replace every `REPLACE_ME_*` + (`cloudflareAccountId`, `secretsStoreId`, `dnsManagedZone`, `dnsRecordName`, + `dnsRecordTarget`, `gcp:project`). + +3. **Cloudflare Secrets Store** — under `secretsStoreId`, store the four secret + values the Worker binds: `bugreport-enc-keyring`, `bugreport-admin-token`, + `github-token`, `otel-exporter-otlp-headers`. + +4. **Pulumi stack** — create it and select it once (`pulumi stack init `); + the workflow runs with `upsert: false` and expects it to exist. + +**Trigger:** Actions → **CD** → *Run workflow* → branch `main`, stack `prod`. + +### Via the Pulumi CLI (manual) + ```console -pulumi stack select dev # or: pulumi stack init dev -# set the REPLACE_ME_* config values + secrets above, then: +pulumi stack select prod # or: pulumi stack init prod +# set the REPLACE_ME_* config values + provider secrets above, then build + deploy: +pnpm run build # produces ../build/worker.mjs (needed if workerScriptPath is set) pulumi preview pulumi up ``` -> Deployment is gated behind a maintainer check-in and the Pulumi CLI is not yet -> available in this environment, so this change ships **compile- and -> test-verified only**. Running a real `pulumi preview` against the accounts is a -> follow-up. +> No real deploy has been run from this repo yet: the mechanism (this program + +> the CD workflow) is **compile-, test-, and actionlint-verified only**. The +> maintainer supplies the credentials/config above and triggers the first deploy. ## Testing (no Pulumi CLI required) The program is exercised with the Pulumi Go SDK's mocking (`pulumi.RunErr` + `pulumi.WithMocks`), which registers resources against an in-memory monitor — no cloud calls, no CLI. The tests assert that the expected -resources are registered with the expected inputs (Worker name/account, R2 bucket -name/location, DNS type/name/target/ttl). +resources are registered with the expected inputs: Worker name/account, its full +**binding set** (R2 `REPORTS_BUCKET`, the four Secrets Store secrets, the three +plain vars) and the **Cron Triggers**, the `ContentFile`/`ContentSha256` artifact +path, R2 bucket name/location, and DNS type/name/target/ttl. ```console go vet ./... @@ -108,10 +189,10 @@ go build ./... go test ./... ``` -## Adding rate limiting later (#7) +## Adding rate limiting later (#6 ADR / #7 impl) -[The abuse/rate-limit ADR](../docs/decisions/labels-and-abuse.md) chose to +[The abuse/rate-limit ADR (#6)](../docs/decisions/labels-and-abuse.md) chose to implement ingest rate limiting as **Cloudflare Rate Limiting rules via Pulumi** (`cloudflare.NewRuleset`, phase `http_ratelimit`, scoped to a zone). That is out -of scope for this ticket. The `cloudflareZoneId` config key and the structure of -`deploy.go` leave a clean insertion point; #7 adds the ruleset resource there. +of scope for this ticket. The `cloudflareZoneId` config key and the forward note +in `deploy.go` leave a clean insertion point; #7 adds the ruleset resource there. diff --git a/infra/deploy.go b/infra/deploy.go index eeb0e60..ab9e522 100644 --- a/infra/deploy.go +++ b/infra/deploy.go @@ -1,8 +1,11 @@ package main import ( + "crypto/sha256" + "encoding/hex" "errors" "fmt" + "os" "github.com/pulumi/pulumi-cloudflare/sdk/v6/go/cloudflare" "github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/dns" @@ -13,9 +16,10 @@ import ( // Logical (Pulumi) resource names. Stable across deploys and asserted by // deploy_test.go, so treat them as part of the program's contract. const ( - resWorker = "ingest-worker" - resR2Bucket = "reports-bucket" - resDNSRecord = "ingest-dns-record" + resWorker = "ingest-worker" + resR2Bucket = "reports-bucket" + resDNSRecord = "ingest-dns-record" + resCronTrigger = "ingest-cron-triggers" ) // Defaults for config values that have a sensible project-wide default. Values @@ -35,15 +39,62 @@ const ( defaultDNSTTLSeconds = 300 ) +// Worker binding names and runtime var defaults. These mirror wrangler.jsonc and +// the names the Worker code reads at runtime (internal/storage.*Binding, +// worker/*_wasm.go), so the deployed Worker actually has the R2 bucket, the +// Secrets Store secrets, and the plain vars it expects. Changing a binding *name* +// here is a breaking change to the Worker contract. +const ( + // JS variable (binding) names the Worker reads via env.. + bindingR2Bucket = "REPORTS_BUCKET" // R2 bucket (internal/storage.BucketBinding) + bindingEncKeyring = "BUGREPORT_ENC_KEYRING" // Secrets Store: AES keyring (ADR 0001) + bindingAdminToken = "ADMIN_TOKEN" // Secrets Store: admin API bearer token (ADR 0003) + bindingGitHubToken = "GITHUB_TOKEN" // Secrets Store: publish PAT (#14) + bindingOtelHeaders = "OTEL_EXPORTER_OTLP_HEADERS" // Secrets Store: OTLP auth headers (#17) + varGitHubRepo = "GITHUB_REPO" // plain var: "owner/repo" publish target (#14) + varOtelEndpoint = "OTEL_EXPORTER_OTLP_ENDPOINT" // plain var: OTLP base URL, "" disables (#17) + varOtelServiceName = "OTEL_SERVICE_NAME" // plain var: reported service.name (#17) + + // Secrets Store secret_name values (the names of the stored secrets), from + // wrangler.jsonc secrets_store_secrets. The store_id is account-specific and + // supplied via the secretsStoreId config. + secretNameEncKeyring = "bugreport-enc-keyring" + secretNameAdminToken = "bugreport-admin-token" + secretNameGitHubToken = "github-token" + secretNameOtelHeaders = "otel-exporter-otlp-headers" + + // Cloudflare multipart binding "type" discriminators. + // https://developers.cloudflare.com/workers/configuration/multipart-upload-metadata/#bindings + bindingTypeR2Bucket = "r2_bucket" + bindingTypeSecretsStore = "secrets_store_secret" + bindingTypePlainText = "plain_text" + + // Runtime var defaults (match wrangler.jsonc "vars"). + defaultGitHubRepo = "JMR-dev/LibreMail" + defaultOtelServiceName = "libremail-bug-report-ingest" +) + +// defaultCronSchedules are the two Friday UTC Cron Triggers from wrangler.jsonc +// (#13). Cloudflare crons are UTC-only and 17:00 America/Chicago is a different +// UTC hour under CDT vs CST, so BOTH candidate hours are registered; the Worker's +// scheduled handler (internal/schedule.IsFriday1700Central) gates each fire so +// exactly one does the weekly publish and the other is a no-op. +var defaultCronSchedules = []string{ + "0 22 * * 5", // Fridays 22:00 UTC == 17:00 Central during CDT (summer) + "0 23 * * 5", // Fridays 23:00 UTC == 17:00 Central during CST (winter) +} + // placeholderWorkerScript is a stand-in module body for the Worker. // // The deployed Worker is Go compiled by TinyGo to Wasm plus the syumai/workers // ES-module shim, emitted by `pnpm run build` into ../build/ (git-ignored, -// produced by CI). Uploading that multipart Wasm artifact is the build/deploy -// pipeline's job; this program ships a documented placeholder so the resource is -// fully described and unit-testable without the artifact present. Override it at -// deploy time with the `workerScriptContent` config, or wire the real artifact -// via ContentFile/ContentSha256 in the deploy pipeline. See infra/README.md. +// produced by CI). The real, functional deploy uploads that built artifact by +// setting the `workerScriptPath` config (the CD workflow sets it to +// ../build/worker.mjs after `pnpm run build`), which switches the resource to +// ContentFile + ContentSha256. When `workerScriptPath` is unset this documented +// placeholder is uploaded instead, so the resource is fully described and +// unit-testable without the artifact present. Override the placeholder body with +// the `workerScriptContent` config. See infra/README.md. const placeholderWorkerScript = `// PLACEHOLDER - replaced at deploy time by the TinyGo -> Wasm build (see repo README). export default { async fetch() { @@ -59,14 +110,25 @@ type infraConfig struct { // cloudflareAccountId is the Cloudflare account that owns the Worker + bucket. cloudflareAccountId string // cloudflareZoneId is optional and unused today. It is reserved so that - // Worker routes / custom domains and the ingest rate-limit ruleset (#7) can + // Worker routes / custom domains and the ingest rate-limit ruleset (#6/#7) can // be added later without a config change. See the forward note in deploy. cloudflareZoneId string workerName string workerContent string + workerScriptPath string // optional path to the built main module; enables ContentFile upload compatibilityDate string + // secretsStoreId is the Cloudflare Secrets Store id that holds the Worker's + // secrets (keyring, admin token, GitHub token, OTLP headers). Account-specific, + // so required; it is NOT itself a secret (it is an id, like the account id). + secretsStoreId string + + // Plain (non-secret) Worker vars. + githubRepo string + otelEndpoint string + otelServiceName string + r2BucketName string r2BucketLocation string // optional; Cloudflare picks a location when empty @@ -89,6 +151,10 @@ func loadConfig(ctx *pulumi.Context) (*infraConfig, error) { if err != nil { return nil, fmt.Errorf("config cloudflareAccountId: %w", err) } + secretsStoreID, err := cfg.Try("secretsStoreId") + if err != nil { + return nil, fmt.Errorf("config secretsStoreId: %w", err) + } managedZone, err := cfg.Try("dnsManagedZone") if err != nil { return nil, fmt.Errorf("config dnsManagedZone: %w", err) @@ -114,7 +180,12 @@ func loadConfig(ctx *pulumi.Context) (*infraConfig, error) { cloudflareZoneId: cfg.Get("cloudflareZoneId"), workerName: getOr(cfg, "workerName", defaultWorkerName), workerContent: getOr(cfg, "workerScriptContent", placeholderWorkerScript), + workerScriptPath: cfg.Get("workerScriptPath"), compatibilityDate: getOr(cfg, "workerCompatibilityDate", defaultCompatibilityDate), + secretsStoreId: secretsStoreID, + githubRepo: getOr(cfg, "githubRepo", defaultGitHubRepo), + otelEndpoint: cfg.Get("otelExporterOtlpEndpoint"), + otelServiceName: getOr(cfg, "otelServiceName", defaultOtelServiceName), r2BucketName: getOr(cfg, "r2BucketName", defaultR2BucketName), r2BucketLocation: cfg.Get("r2BucketLocation"), gcpProject: cfg.Get("gcpProject"), @@ -134,6 +205,62 @@ func getOr(cfg *config.Config, key, def string) string { return def } +// fileSha256 returns the lowercase hex SHA-256 of the file at path. It is used to +// derive ContentSha256 for the Worker artifact upload; the Cloudflare provider +// requires ContentSha256 whenever ContentFile is set (it triggers an update when +// the built Wasm changes). +func fileSha256(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +// workerBindings returns the full binding set the Worker needs at runtime: the R2 +// bucket, the four Cloudflare Secrets Store secrets, and the three plain vars. +// This is the #9-flagged wiring gap that #4 closes — before this the deployed +// Worker had no bindings and could not read its bucket/secrets/vars. +func workerBindings(cfg *infraConfig) cloudflare.WorkersScriptBindingArray { + return cloudflare.WorkersScriptBindingArray{ + // R2 bucket: env.REPORTS_BUCKET -> the encrypted-report bucket (ADR 0001). + &cloudflare.WorkersScriptBindingArgs{ + Name: pulumi.String(bindingR2Bucket), + Type: pulumi.String(bindingTypeR2Bucket), + BucketName: pulumi.String(cfg.r2BucketName), + }, + // Secrets Store secrets, read at runtime via env..get(). + secretsStoreBinding(bindingEncKeyring, cfg.secretsStoreId, secretNameEncKeyring), + secretsStoreBinding(bindingAdminToken, cfg.secretsStoreId, secretNameAdminToken), + secretsStoreBinding(bindingGitHubToken, cfg.secretsStoreId, secretNameGitHubToken), + secretsStoreBinding(bindingOtelHeaders, cfg.secretsStoreId, secretNameOtelHeaders), + // Plain (non-secret) vars, read via cloudflare.Getenv(). + plainTextBinding(varGitHubRepo, cfg.githubRepo), + plainTextBinding(varOtelEndpoint, cfg.otelEndpoint), + plainTextBinding(varOtelServiceName, cfg.otelServiceName), + } +} + +// secretsStoreBinding builds one Cloudflare Secrets Store binding (env..get()). +func secretsStoreBinding(name, storeID, secretName string) cloudflare.WorkersScriptBindingInput { + return &cloudflare.WorkersScriptBindingArgs{ + Name: pulumi.String(name), + Type: pulumi.String(bindingTypeSecretsStore), + StoreId: pulumi.String(storeID), + SecretName: pulumi.String(secretName), + } +} + +// plainTextBinding builds one plain-text (non-secret) var binding. +func plainTextBinding(name, text string) cloudflare.WorkersScriptBindingInput { + return &cloudflare.WorkersScriptBindingArgs{ + Name: pulumi.String(name), + Type: pulumi.String(bindingTypePlainText), + Text: pulumi.String(text), + } +} + // deploy registers all resources for the bug-report ingest stack. func deploy(ctx *pulumi.Context) error { cfg, err := loadConfig(ctx) @@ -141,16 +268,29 @@ func deploy(ctx *pulumi.Context) error { return err } - // 1. Cloudflare Worker script - the ingest Worker built in #1. Content is a - // documented placeholder; the real Wasm artifact is uploaded by the build - // pipeline (see placeholderWorkerScript). - worker, err := cloudflare.NewWorkersScript(ctx, resWorker, &cloudflare.WorkersScriptArgs{ + // 1. Cloudflare Worker script - the ingest Worker built in #1, now wired with + // its full binding set (R2 + Secrets Store + vars). Content is the real + // built artifact when workerScriptPath is set (the CD workflow points it at + // ../build/worker.mjs after `pnpm run build`); otherwise a documented + // placeholder module body (see placeholderWorkerScript). + scriptArgs := &cloudflare.WorkersScriptArgs{ AccountId: pulumi.String(cfg.cloudflareAccountId), ScriptName: pulumi.String(cfg.workerName), - Content: pulumi.String(cfg.workerContent), MainModule: pulumi.String(mainModule), CompatibilityDate: pulumi.String(cfg.compatibilityDate), - }) + Bindings: workerBindings(cfg), + } + if cfg.workerScriptPath != "" { + sum, err := fileSha256(cfg.workerScriptPath) + if err != nil { + return fmt.Errorf("worker script artifact %q: %w", cfg.workerScriptPath, err) + } + scriptArgs.ContentFile = pulumi.String(cfg.workerScriptPath) + scriptArgs.ContentSha256 = pulumi.String(sum) + } else { + scriptArgs.Content = pulumi.String(cfg.workerContent) + } + worker, err := cloudflare.NewWorkersScript(ctx, resWorker, scriptArgs) if err != nil { return fmt.Errorf("worker script: %w", err) } @@ -170,7 +310,24 @@ func deploy(ctx *pulumi.Context) error { return fmt.Errorf("r2 bucket: %w", err) } - // 3. Google Cloud DNS record pointing the ingest hostname at the Worker's + // 3. Cloudflare Cron Triggers for the weekly publish job (#13). Both Friday UTC + // hours are registered (DST straddle); the Worker's scheduled handler gates + // which one publishes. Depends on the Worker script it schedules. + schedules := cloudflare.WorkersCronTriggerScheduleArray{} + for _, cron := range defaultCronSchedules { + schedules = append(schedules, &cloudflare.WorkersCronTriggerScheduleArgs{ + Cron: pulumi.String(cron), + }) + } + if _, err := cloudflare.NewWorkersCronTrigger(ctx, resCronTrigger, &cloudflare.WorkersCronTriggerArgs{ + AccountId: pulumi.String(cfg.cloudflareAccountId), + ScriptName: worker.ScriptName, + Schedules: schedules, + }); err != nil { + return fmt.Errorf("cron triggers: %w", err) + } + + // 4. Google Cloud DNS record pointing the ingest hostname at the Worker's // route/custom domain. DNS authority is Google Cloud DNS; the managed zone // is referenced by name (it is managed outside this stack) and a record is // added to it. @@ -189,17 +346,18 @@ func deploy(ctx *pulumi.Context) error { return fmt.Errorf("dns record: %w", err) } - // Forward note (#7 / docs/decisions/labels-and-abuse.md): the accepted ADR - // implements ingest rate limiting as Cloudflare Rate Limiting rules via - // Pulumi (cloudflare.NewRuleset with Phase "http_ratelimit", scoped to - // cloudflareZoneId). That is out of scope for #2, but the config - // (cloudflareZoneId) and this structure leave a clean insertion point: - // add the ruleset resource here. Worker routes / a custom domain would - // attach the same way via cloudflareZoneId. + // Forward note (#6 ADR docs/decisions/labels-and-abuse.md / #7 impl): the + // accepted ADR implements ingest rate limiting as Cloudflare Rate Limiting + // rules via Pulumi (cloudflare.NewRuleset with Phase "http_ratelimit", scoped + // to cloudflareZoneId). That is out of scope for this ticket, but the config + // (cloudflareZoneId) and this structure leave a clean insertion point: add the + // ruleset resource here. Worker routes / a custom domain would attach the same + // way via cloudflareZoneId. ctx.Export("workerName", worker.ScriptName) ctx.Export("workerAccountId", worker.AccountId) ctx.Export("r2BucketName", bucket.Name) + ctx.Export("cronSchedules", pulumi.ToStringArray(defaultCronSchedules)) ctx.Export("dnsRecordFqdn", record.Name) ctx.Export("dnsRecordTargets", record.Rrdatas) if cfg.cloudflareZoneId != "" { diff --git a/infra/deploy_test.go b/infra/deploy_test.go index 2dc2063..fef7287 100644 --- a/infra/deploy_test.go +++ b/infra/deploy_test.go @@ -1,7 +1,11 @@ package main import ( + "crypto/sha256" + "encoding/hex" "encoding/json" + "os" + "path/filepath" "testing" "github.com/pulumi/pulumi/sdk/v3/go/common/resource" @@ -14,9 +18,10 @@ const testProject = "libremail-bug-report-ingest-infra" // Resource type tokens registered by the program. const ( - tokWorkersScript = "cloudflare:index/workersScript:WorkersScript" - tokR2Bucket = "cloudflare:index/r2Bucket:R2Bucket" - tokDNSRecordSet = "gcp:dns/recordSet:RecordSet" + tokWorkersScript = "cloudflare:index/workersScript:WorkersScript" + tokR2Bucket = "cloudflare:index/r2Bucket:R2Bucket" + tokWorkersCronTrigger = "cloudflare:index/workersCronTrigger:WorkersCronTrigger" + tokDNSRecordSet = "gcp:dns/recordSet:RecordSet" ) // recordingMocks implements pulumi.MockResourceMonitor, capturing every @@ -54,6 +59,7 @@ func key(k string) string { return testProject + ":" + k } func fullConfig() map[string]string { return map[string]string{ key("cloudflareAccountId"): "cf-acct-123", + key("secretsStoreId"): "ss-store-xyz", key("dnsManagedZone"): "libremail-zone", key("dnsRecordName"): "bugreport.example.com.", key("dnsRecordTarget"): "libremail-bug-report-ingest.acme.workers.dev.", @@ -93,6 +99,34 @@ func strProp(t *testing.T, pm resource.PropertyMap, k string) string { return v.StringValue() } +// objArray returns pm[k] as a slice of PropertyMaps, failing if it is absent or +// not an array of objects. Used to walk the Worker bindings / cron schedules. +func objArray(t *testing.T, pm resource.PropertyMap, k string) []resource.PropertyMap { + t.Helper() + v, ok := pm[resource.PropertyKey(k)] + if !ok || !v.IsArray() { + t.Fatalf("property %q missing or not an array: %v", k, pm.Mappable()) + } + out := make([]resource.PropertyMap, 0, len(v.ArrayValue())) + for i, el := range v.ArrayValue() { + if !el.IsObject() { + t.Fatalf("%s[%d] is not an object: %v", k, i, el) + } + out = append(out, el.ObjectValue()) + } + return out +} + +// bindingsByName indexes the Worker's bindings array by its "name" (the JS var). +func bindingsByName(t *testing.T, in resource.PropertyMap) map[string]resource.PropertyMap { + t.Helper() + out := map[string]resource.PropertyMap{} + for _, b := range objArray(t, in, "bindings") { + out[strProp(t, b, "name")] = b + } + return out +} + func TestWorkerScriptRegistered(t *testing.T) { m := runProgram(t, fullConfig()) in := m.find(t, tokWorkersScript).Inputs @@ -114,6 +148,128 @@ func TestWorkerScriptRegistered(t *testing.T) { } } +// TestWorkerScriptBindings is the core coverage for #4/#9: the deployed Worker +// must carry the R2 bucket binding, the four Secrets Store bindings, and the +// three plain vars, with the exact names/store/secret_name the Worker reads at +// runtime (wrangler.jsonc contract). +func TestWorkerScriptBindings(t *testing.T) { + m := runProgram(t, fullConfig()) + in := m.find(t, tokWorkersScript).Inputs + bindings := bindingsByName(t, in) + + // R2 bucket binding. + r2, ok := bindings["REPORTS_BUCKET"] + if !ok { + t.Fatalf("missing REPORTS_BUCKET binding; have %v", bindingNames(bindings)) + } + if got, want := strProp(t, r2, "type"), "r2_bucket"; got != want { + t.Errorf("REPORTS_BUCKET type = %q, want %q", got, want) + } + if got, want := strProp(t, r2, "bucketName"), "libremail-bug-reports"; got != want { + t.Errorf("REPORTS_BUCKET bucketName = %q, want %q", got, want) + } + + // Secrets Store bindings: name -> secret_name (all share the one store id). + wantSecrets := map[string]string{ + "BUGREPORT_ENC_KEYRING": "bugreport-enc-keyring", + "ADMIN_TOKEN": "bugreport-admin-token", + "GITHUB_TOKEN": "github-token", + "OTEL_EXPORTER_OTLP_HEADERS": "otel-exporter-otlp-headers", + } + for name, wantSecretName := range wantSecrets { + b, ok := bindings[name] + if !ok { + t.Errorf("missing Secrets Store binding %q; have %v", name, bindingNames(bindings)) + continue + } + if got, want := strProp(t, b, "type"), "secrets_store_secret"; got != want { + t.Errorf("%s type = %q, want %q", name, got, want) + } + if got, want := strProp(t, b, "storeId"), "ss-store-xyz"; got != want { + t.Errorf("%s storeId = %q, want %q", name, got, want) + } + if got := strProp(t, b, "secretName"); got != wantSecretName { + t.Errorf("%s secretName = %q, want %q", name, got, wantSecretName) + } + } + + // Plain-text vars. + wantVars := map[string]string{ + "GITHUB_REPO": "JMR-dev/LibreMail", + "OTEL_EXPORTER_OTLP_ENDPOINT": "", + "OTEL_SERVICE_NAME": "libremail-bug-report-ingest", + } + for name, wantText := range wantVars { + b, ok := bindings[name] + if !ok { + t.Errorf("missing plain var binding %q; have %v", name, bindingNames(bindings)) + continue + } + if got, want := strProp(t, b, "type"), "plain_text"; got != want { + t.Errorf("%s type = %q, want %q", name, got, want) + } + if got := textValue(b); got != wantText { + t.Errorf("%s text = %q, want %q", name, got, wantText) + } + } +} + +// bindingNames is a small diagnostic helper for failure messages. +func bindingNames(bindings map[string]resource.PropertyMap) []string { + names := make([]string, 0, len(bindings)) + for n := range bindings { + names = append(names, n) + } + return names +} + +// textValue reads a binding's "text" property, treating an absent property as the +// empty string (an empty plain-text var may serialize either way). +func textValue(b resource.PropertyMap) string { + v, ok := b[resource.PropertyKey("text")] + if !ok { + return "" + } + if !v.IsString() { + return "" + } + return v.StringValue() +} + +// TestWorkerContentFromArtifact verifies the artifact-override path: when +// workerScriptPath is set (as the CD workflow sets it to ../build/worker.mjs +// after `pnpm run build`), the Worker uploads the built file via ContentFile + +// a computed ContentSha256 instead of the inline placeholder content. +func TestWorkerContentFromArtifact(t *testing.T) { + dir := t.TempDir() + artifact := filepath.Join(dir, "worker.mjs") + body := []byte("export default { async fetch() { return new Response('ok'); } };\n") + if err := os.WriteFile(artifact, body, 0o600); err != nil { + t.Fatalf("write artifact: %v", err) + } + sum := sha256.Sum256(body) + wantSha := hex.EncodeToString(sum[:]) + + cfg := fullConfig() + cfg[key("workerScriptPath")] = artifact + m := runProgram(t, cfg) + in := m.find(t, tokWorkersScript).Inputs + + if got, want := strProp(t, in, "contentFile"), artifact; got != want { + t.Errorf("contentFile = %q, want %q", got, want) + } + if got := strProp(t, in, "contentSha256"); got != wantSha { + t.Errorf("contentSha256 = %q, want %q", got, wantSha) + } + if _, ok := in[resource.PropertyKey("content")]; ok { + t.Error("content should be unset when workerScriptPath drives a ContentFile upload") + } + // Bindings must still be attached in the artifact path. + if _, ok := bindingsByName(t, in)["REPORTS_BUCKET"]; !ok { + t.Error("REPORTS_BUCKET binding missing on the artifact-content Worker") + } +} + func TestR2BucketRegistered(t *testing.T) { m := runProgram(t, fullConfig()) in := m.find(t, tokR2Bucket).Inputs @@ -169,13 +325,43 @@ func TestDNSRecordRegistered(t *testing.T) { } } -func TestExactlyThreeManagedResources(t *testing.T) { +// TestWorkerCronTriggers asserts the two Friday UTC crons (#13) are registered as +// a WorkersCronTrigger bound to the ingest Worker. +func TestWorkerCronTriggers(t *testing.T) { + m := runProgram(t, fullConfig()) + in := m.find(t, tokWorkersCronTrigger).Inputs + + if got, want := strProp(t, in, "scriptName"), "libremail-bug-report-ingest"; got != want { + t.Errorf("cron scriptName = %q, want %q", got, want) + } + if got, want := strProp(t, in, "accountId"), "cf-acct-123"; got != want { + t.Errorf("cron accountId = %q, want %q", got, want) + } + + var crons []string + for _, s := range objArray(t, in, "schedules") { + crons = append(crons, strProp(t, s, "cron")) + } + want := []string{"0 22 * * 5", "0 23 * * 5"} + if len(crons) != len(want) { + t.Fatalf("cron schedules = %v, want %v", crons, want) + } + for i := range want { + if crons[i] != want[i] { + t.Errorf("schedules[%d] = %q, want %q", i, crons[i], want[i]) + } + } +} + +// TestManagedResourceCounts pins the managed resource set: exactly one each of the +// Worker script, R2 bucket, cron trigger, and DNS record. +func TestManagedResourceCounts(t *testing.T) { m := runProgram(t, fullConfig()) counts := map[string]int{} for _, r := range m.resources { counts[r.TypeToken]++ } - for _, tok := range []string{tokWorkersScript, tokR2Bucket, tokDNSRecordSet} { + for _, tok := range []string{tokWorkersScript, tokR2Bucket, tokWorkersCronTrigger, tokDNSRecordSet} { if counts[tok] != 1 { t.Errorf("expected exactly 1 %s, got %d", tok, counts[tok]) }