#4 GitHub Actions CD: deploy via workflow_dispatch #46

Merged
JMR-dev merged 2 commits from ticket-4-cd-deploy into main 2026-07-02 23:56:22 +00:00
6 changed files with 645 additions and 52 deletions
+144
View File
@@ -0,0 +1,144 @@
# Continuous deployment for the LibreMail bug-report ingest Worker + infra.
#
# MANUAL ONLY: this workflow never runs on push/PR. A maintainer triggers it from
# the Actions tab (workflow_dispatch), choosing a stack. It builds the TinyGo/Wasm
# Worker end to end (same setup as ci.yml) and then runs `pulumi up` over the
# infra/ program to deploy the Worker (with its R2 + Secrets Store + var bindings
# and Cron Triggers), the R2 bucket, and the Google Cloud DNS record.
#
# It is gated to the `production` GitHub Actions environment, so that environment's
# secrets and any required-reviewer / branch protection rules apply, and to the
# `main` branch (a guard step fails the run otherwise). Deploying is real and
# billable, hence manual + environment-gated + maintainer-run-from-main.
#
# Supply-chain note: every action (first- and third-party) is pinned to a full
# commit SHA with a trailing "# vX.Y.Z" comment, matching ci.yml / autoupdate.yml.
#
# Secrets/config the maintainer must set BEFORE the first deploy (see infra/README.md):
# production environment SECRETS (Settings > Environments > production):
# - PULUMI_ACCESS_TOKEN Pulumi Cloud access token (state backend). For a
# self-managed backend instead, set the `cloud-url`
# input + a PULUMI_CONFIG_PASSPHRASE secret.
# - CLOUDFLARE_API_TOKEN Cloudflare token scoped to Workers Scripts + R2 (+ Cron).
# - CLOUDFLARE_ACCOUNT_ID Cloudflare account id (also set as stack config).
# - GOOGLE_CREDENTIALS GCP service-account JSON with Cloud DNS admin on the zone.
# stack CONFIG (infra/Pulumi.<stack>.yaml — replace every REPLACE_ME_* first):
# cloudflareAccountId, secretsStoreId, dnsManagedZone, dnsRecordName,
# dnsRecordTarget, gcp:project (+ optional r2/otel/dns overrides).
# Cloudflare Secrets Store must already hold the four secret values
# (bugreport-enc-keyring, bugreport-admin-token, github-token,
# otel-exporter-otlp-headers) under the configured secretsStoreId.
name: CD
on:
workflow_dispatch:
inputs:
stack:
description: 'Pulumi stack to deploy'
required: true
default: prod
type: choice
options:
- prod
- dev
# Least privilege: the job only needs to read the repo out; Pulumi auth is via env.
permissions:
contents: read
# Never run two deploys of the same stack concurrently; do not cancel an in-flight
# deploy (interrupting `pulumi up` can leave a stack mid-update).
concurrency:
group: cd-${{ github.event.inputs.stack }}
cancel-in-progress: false
jobs:
deploy:
name: deploy
runs-on: ubuntu-latest
# Gate on the production environment so its secrets + protection rules apply.
environment: production
steps:
# Deploys must be cut from main. workflow_dispatch lets a user pick any ref,
# so fail loudly if this was launched from a non-main branch.
- name: Guard - deploy only from main
if: github.ref != 'refs/heads/main'
run: |
echo "::error::Deploy must be run from the 'main' branch (got '${{ github.ref }}')."
exit 1
- name: Check out repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
# Mirror ci.yml: cache both the root and infra/ module go.sum (infra pulls the
# heavy Pulumi SDKs) so warm runs restore deps instead of re-downloading.
- name: Set up Go
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
with:
go-version: '1.26'
cache-dependency-path: |
go.sum
infra/go.sum
# TinyGo builds the Wasm Worker (pnpm run build). Same version as ci.yml.
- name: Set up TinyGo
uses: acifani/setup-tinygo@dd8a7075d951a7595b2ef2123ed0ab1af0c13e56 # v3.0.0
with:
tinygo-version: '0.41.1'
- name: Set up pnpm
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
with:
version: '11'
- name: Set up Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: '26'
cache: pnpm
- name: Install Node dependencies
run: pnpm install --frozen-lockfile
# TEMPORARY (tracking #26; tinygo-org/tinygo#5467): identical to ci.yml. TinyGo
# 0.41.1 vendors a net/http js/wasm overlay that fails to compile on Go 1.26;
# apply the exact upstream fix to the installed TinyGo source before building.
# git apply exits non-zero (failing loudly) if the source has drifted.
- name: Patch TinyGo net/http (temporary)
run: |
patch_file="$PWD/.ci/tinygo-net-roundtrip.patch"
tinygoroot="$(tinygo env TINYGOROOT)"
echo "Applying $patch_file to $tinygoroot/src/net/http/roundtrip_js.go"
git -C "$tinygoroot" apply --verbose "$patch_file" || {
echo "::error::TinyGo net/http patch did not apply cleanly; TinyGo source may have changed. Update or remove .ci/tinygo-net-roundtrip.patch (see #26)."
exit 1
}
# Produce build/worker.mjs (ES-module shim) + build/app.wasm. The infra program
# uploads the shim as the Worker's main module via the workerScriptPath config
# injected below.
- name: Build Wasm Worker
run: pnpm run build
# Install the Pulumi CLI and run `pulumi up` over infra/. config-map injects the
# freshly built artifact path so the WorkersScript uploads the real module
# (ContentFile) instead of the placeholder. Provider + backend credentials come
# from the production environment secrets below; nothing secret is committed.
- name: Pulumi up
uses: pulumi/actions@8e5e406f4007fca908480587cb9893c07090f58d # v7.0.0
with:
command: up
stack-name: ${{ github.event.inputs.stack }}
work-dir: infra
upsert: false
config-map: '{ "libremail-bug-report-ingest-infra:workerScriptPath": { value: "../build/worker.mjs", secret: false } }'
env:
# Pulumi state backend (Pulumi Cloud). For a self-managed backend, drop this,
# set the action's `cloud-url` input, and add PULUMI_CONFIG_PASSPHRASE.
PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
# Cloudflare provider (Workers + R2 + Cron Triggers).
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
# GCP provider (Cloud DNS record).
GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS }}
+12
View File
@@ -12,8 +12,20 @@ config:
# --- Program config (namespace = project name from Pulumi.yaml) ---
libremail-bug-report-ingest-infra:cloudflareAccountId: REPLACE_ME_CLOUDFLARE_ACCOUNT_ID
# Cloudflare Secrets Store id holding the Worker's secrets (encryption keyring,
# admin token, GitHub token, OTLP headers). Account-specific; not itself secret.
libremail-bug-report-ingest-infra:secretsStoreId: REPLACE_ME_CLOUDFLARE_SECRETS_STORE_ID
libremail-bug-report-ingest-infra:workerName: libremail-bug-report-ingest
libremail-bug-report-ingest-infra:workerCompatibilityDate: "2025-06-01"
# Built Worker artifact (../build/worker.mjs from `pnpm run build`). Leave UNSET
# here: the CD workflow (.github/workflows/deploy.yml) injects it at deploy time
# via config-map so `pulumi preview` without a build still uses the placeholder.
# libremail-bug-report-ingest-infra:workerScriptPath: ../build/worker.mjs
# Plain (non-secret) Worker vars — mirror wrangler.jsonc "vars".
libremail-bug-report-ingest-infra:githubRepo: JMR-dev/LibreMail
# OTLP endpoint EMPTY disables telemetry until a collector is chosen (#17).
libremail-bug-report-ingest-infra:otelExporterOtlpEndpoint: ""
libremail-bug-report-ingest-infra:otelServiceName: libremail-bug-report-ingest
libremail-bug-report-ingest-infra:r2BucketName: libremail-bug-reports-dev
# R2 location hint (optional). One of: apac, eeur, enam, weur, wnam, oc.
libremail-bug-report-ingest-infra:r2BucketLocation: enam
+12
View File
@@ -12,8 +12,20 @@ config:
# --- Program config (namespace = project name from Pulumi.yaml) ---
libremail-bug-report-ingest-infra:cloudflareAccountId: REPLACE_ME_CLOUDFLARE_ACCOUNT_ID
# Cloudflare Secrets Store id holding the Worker's secrets (encryption keyring,
# admin token, GitHub token, OTLP headers). Account-specific; not itself secret.
libremail-bug-report-ingest-infra:secretsStoreId: REPLACE_ME_CLOUDFLARE_SECRETS_STORE_ID
libremail-bug-report-ingest-infra:workerName: libremail-bug-report-ingest
libremail-bug-report-ingest-infra:workerCompatibilityDate: "2025-06-01"
# Built Worker artifact (../build/worker.mjs from `pnpm run build`). Leave UNSET
# here: the CD workflow (.github/workflows/deploy.yml) injects it at deploy time
# via config-map so `pulumi preview` without a build still uses the placeholder.
# libremail-bug-report-ingest-infra:workerScriptPath: ../build/worker.mjs
# Plain (non-secret) Worker vars — mirror wrangler.jsonc "vars".
libremail-bug-report-ingest-infra:githubRepo: JMR-dev/LibreMail
# OTLP endpoint EMPTY disables telemetry until a collector is chosen (#17).
libremail-bug-report-ingest-infra:otelExporterOtlpEndpoint: ""
libremail-bug-report-ingest-infra:otelServiceName: libremail-bug-report-ingest
libremail-bug-report-ingest-infra:r2BucketName: libremail-bug-reports
# R2 location hint (optional). One of: apac, eeur, enam, weur, wnam, oc.
libremail-bug-report-ingest-infra:r2BucketLocation: enam
+105 -24
View File
@@ -10,22 +10,59 @@ It provisions, with the Pulumi Go SDK and the
| Resource | Type | Purpose |
| --- | --- | --- |
| `ingest-worker` | `cloudflare.WorkersScript` | The ingest Worker (`libremail-bug-report-ingest`, built in #1). |
| `ingest-worker` | `cloudflare.WorkersScript` | The ingest Worker (`libremail-bug-report-ingest`, built in #1), wired with its full binding set (R2 + Secrets Store + vars, see [Worker bindings](#worker-bindings)). |
| `reports-bucket` | `cloudflare.R2Bucket` | Encrypted bug-report storage (`libremail-bug-reports`). Per [ADR 0001](../docs/decisions/encryption.md) only ciphertext is written. |
| `ingest-cron-triggers` | `cloudflare.WorkersCronTrigger` | The two Friday UTC Cron Triggers (#13) that drive the weekly publish job; bound to the ingest Worker. |
| `ingest-dns-record` | `gcp.dns.RecordSet` | Google Cloud DNS record pointing the ingest hostname at the Worker's route/custom domain. |
DNS authority is **Google Cloud DNS**. The managed zone is **referenced by name**
(it already exists / is managed elsewhere), and this stack only adds a record to
it.
> **Worker content is a placeholder.** The deployed Worker is Go compiled by
> TinyGo to Wasm plus the `syumai/workers` ES-module shim, emitted by
> `pnpm run build` into `../build/` (git-ignored, produced by CI). This program
> ships a documented placeholder module body so the resource is fully described
> and unit-testable without the artifact. Wire the real artifact at deploy time
> via the `workerScriptContent` config, or by setting `ContentFile` /
> `ContentSha256` on the Worker resource to `../build/worker.mjs` in the deploy
> pipeline.
> **Worker content: placeholder by default, real artifact at deploy.** The
> deployed Worker is Go compiled by TinyGo to Wasm plus the `syumai/workers`
> ES-module shim, emitted by `pnpm run build` into `../build/` (git-ignored,
> produced by CI). Set the **`workerScriptPath`** config to the built main module
> (`../build/worker.mjs`) and the program uploads it via `ContentFile` +
> `ContentSha256` (computed from the file). When `workerScriptPath` is unset, a
> documented placeholder module body is uploaded instead, so the resource is fully
> described and unit-testable without the artifact. The CD workflow
> ([`.github/workflows/deploy.yml`](../.github/workflows/deploy.yml)) builds the
> Worker and injects `workerScriptPath=../build/worker.mjs` at deploy time, so a
> local `pulumi preview` (no build) still works off the placeholder.
>
> Note: a TinyGo Worker is `worker.mjs` (main module) that imports `app.wasm`.
> `ContentFile` uploads the main module; if a functional deploy needs the wasm as a
> separate module part, add it alongside `worker.mjs` in the build output the
> pipeline points at. The bindings/crons below are provider-verified by the mock
> tests regardless of which content source is used.
## Worker bindings
The Worker resource carries the bindings the runtime code reads, matching
`wrangler.jsonc` and the Worker source (`internal/storage.*Binding`,
`worker/*_wasm.go`). This is the wiring #4 added (the gap #9 flagged); the mock
tests (`deploy_test.go`) assert every one of them:
| Binding (JS var) | Type | Points at |
| --- | --- | --- |
| `REPORTS_BUCKET` | `r2_bucket` | `r2BucketName` (`libremail-bug-reports`). Encrypted report objects (ADR 0001). |
| `BUGREPORT_ENC_KEYRING` | `secrets_store_secret` | `secretsStoreId` / secret `bugreport-enc-keyring` — AES-256 keyring (ADR 0001). |
| `ADMIN_TOKEN` | `secrets_store_secret` | `secretsStoreId` / secret `bugreport-admin-token` — admin API token (ADR 0003). |
| `GITHUB_TOKEN` | `secrets_store_secret` | `secretsStoreId` / secret `github-token` — publish PAT (#14). |
| `OTEL_EXPORTER_OTLP_HEADERS` | `secrets_store_secret` | `secretsStoreId` / secret `otel-exporter-otlp-headers` — OTLP auth headers (#17). |
| `GITHUB_REPO` | `plain_text` | `githubRepo` (`JMR-dev/LibreMail`) — publish target (#14). |
| `OTEL_EXPORTER_OTLP_ENDPOINT` | `plain_text` | `otelExporterOtlpEndpoint` (empty ⇒ telemetry disabled, #17). |
| `OTEL_SERVICE_NAME` | `plain_text` | `otelServiceName` (`libremail-bug-report-ingest`, #17). |
The **Cron Triggers** resource registers both Friday UTC crons (`0 22 * * 5` and
`0 23 * * 5`) from `wrangler.jsonc` (#13); the Worker's scheduled handler gates on
the real America/Chicago rule so exactly one publishes each Friday.
> The Secrets Store **secret values** themselves (keyring, admin token, GitHub
> token, OTLP headers) are **not** created by this stack — the maintainer stores
> them in Cloudflare Secrets Store under `secretsStoreId`. This stack only binds
> them to the Worker by name.
## Prerequisites
@@ -46,18 +83,23 @@ project name, `libremail-bug-report-ingest-infra`. Provider keys use the
| Key | Required | Default | Description |
| --- | --- | --- | --- |
| `cloudflareAccountId` | yes | — | Cloudflare account that owns the Worker + R2 bucket. |
| `secretsStoreId` | yes | — | Cloudflare **Secrets Store id** holding the Worker's secrets (keyring, admin token, GitHub token, OTLP headers). Account-specific; not itself secret. |
| `dnsManagedZone` | yes | — | **Name** of the existing Google Cloud DNS managed zone. |
| `dnsRecordName` | yes | — | Ingest hostname as an FQDN with a trailing dot (e.g. `bugreport.libremail.example.`). |
| `dnsRecordTarget` | yes | — | CNAME target = the Worker route/custom domain (FQDN, trailing dot). |
| `workerName` | no | `libremail-bug-report-ingest` | Worker script name (matches `wrangler.jsonc`). |
| `workerCompatibilityDate` | no | `2025-06-01` | Worker runtime compatibility date. |
| `workerScriptContent` | no | placeholder | Override the Worker module body (normally supplied by the build pipeline). |
| `r2BucketName` | no | `libremail-bug-reports` | R2 bucket name. |
| `workerScriptPath` | no | — | Path to the built main module (`../build/worker.mjs`). When set, upload it via `ContentFile` + computed `ContentSha256`. Set by the CD workflow at deploy. |
| `workerScriptContent` | no | placeholder | Inline Worker module body used when `workerScriptPath` is unset. |
| `githubRepo` | no | `JMR-dev/LibreMail` | `GITHUB_REPO` var: `owner/repo` the weekly publish job files issues on (#14). |
| `otelExporterOtlpEndpoint` | no | `""` | `OTEL_EXPORTER_OTLP_ENDPOINT` var: OTLP base URL; empty disables telemetry (#17). |
| `otelServiceName` | no | `libremail-bug-report-ingest` | `OTEL_SERVICE_NAME` var: reported `service.name` (#17). |
| `r2BucketName` | no | `libremail-bug-reports` | R2 bucket name (also the `REPORTS_BUCKET` binding target). |
| `r2BucketLocation` | no | (provider default) | R2 location hint: `apac`, `eeur`, `enam`, `weur`, `wnam`, `oc`. |
| `dnsRecordType` | no | `CNAME` | DNS record type. |
| `dnsTtlSeconds` | no | `300` | DNS record TTL (seconds). |
| `gcpProject` | no | (from `gcp:project`) | Project the record belongs to, if different from the provider project. |
| `cloudflareZoneId` | no | — | **Reserved** for #7 (rate-limit ruleset) and Worker routes/custom domain. Unused today. |
| `cloudflareZoneId` | no | — | **Reserved** for #6/#7 (rate-limit ruleset) and Worker routes/custom domain. Unused today. |
Set a non-secret value with, e.g.:
@@ -82,25 +124,64 @@ Store**, bound to the Worker — it is managed there, not committed here.
## Deploying
### Via GitHub Actions (recommended)
[`.github/workflows/deploy.yml`](../.github/workflows/deploy.yml) is a
**manual, `workflow_dispatch`-only** CD workflow. A maintainer runs it from the
Actions tab, on the **`main`** branch (a guard step fails otherwise), and picks a
`stack` (default `prod`). It reuses ci.yml's Go/TinyGo/pnpm setup + the TinyGo
net/http patch, runs `pnpm run build`, then `pulumi up --stack <stack>` over this
program (injecting the built `../build/worker.mjs` as `workerScriptPath`). It is
gated to the **`production`** GitHub Actions environment, so that environment's
secrets and any required-reviewer / protection rules apply.
**One-time setup before the first deploy** — the maintainer configures:
1. **`production` environment secrets** (repo Settings → Environments → production):
| Secret | Purpose |
| --- | --- |
| `PULUMI_ACCESS_TOKEN` | Pulumi Cloud token (state backend). Self-managed backend? Use the action's `cloud-url` input + a `PULUMI_CONFIG_PASSPHRASE` secret instead. |
| `CLOUDFLARE_API_TOKEN` | Cloudflare token scoped to Workers Scripts + R2 (+ Cron Triggers) on the account. |
| `CLOUDFLARE_ACCOUNT_ID` | Cloudflare account id. |
| `GOOGLE_CREDENTIALS` | GCP service-account JSON with Cloud DNS admin on the managed zone. |
2. **Stack config** in `Pulumi.<stack>.yaml` — replace every `REPLACE_ME_*`
(`cloudflareAccountId`, `secretsStoreId`, `dnsManagedZone`, `dnsRecordName`,
`dnsRecordTarget`, `gcp:project`).
3. **Cloudflare Secrets Store** — under `secretsStoreId`, store the four secret
values the Worker binds: `bugreport-enc-keyring`, `bugreport-admin-token`,
`github-token`, `otel-exporter-otlp-headers`.
4. **Pulumi stack** — create it and select it once (`pulumi stack init <stack>`);
the workflow runs with `upsert: false` and expects it to exist.
**Trigger:** Actions → **CD** → *Run workflow* → branch `main`, stack `prod`.
### Via the Pulumi CLI (manual)
```console
pulumi stack select dev # or: pulumi stack init dev
# set the REPLACE_ME_* config values + secrets above, then:
pulumi stack select prod # or: pulumi stack init prod
# set the REPLACE_ME_* config values + provider secrets above, then build + deploy:
pnpm run build # produces ../build/worker.mjs (needed if workerScriptPath is set)
pulumi preview
pulumi up
```
> Deployment is gated behind a maintainer check-in and the Pulumi CLI is not yet
> available in this environment, so this change ships **compile- and
> test-verified only**. Running a real `pulumi preview` against the accounts is a
> follow-up.
> No real deploy has been run from this repo yet: the mechanism (this program +
> the CD workflow) is **compile-, test-, and actionlint-verified only**. The
> maintainer supplies the credentials/config above and triggers the first deploy.
## Testing (no Pulumi CLI required)
The program is exercised with the Pulumi Go SDK's mocking
(`pulumi.RunErr` + `pulumi.WithMocks`), which registers resources against an
in-memory monitor — no cloud calls, no CLI. The tests assert that the expected
resources are registered with the expected inputs (Worker name/account, R2 bucket
name/location, DNS type/name/target/ttl).
resources are registered with the expected inputs: Worker name/account, its full
**binding set** (R2 `REPORTS_BUCKET`, the four Secrets Store secrets, the three
plain vars) and the **Cron Triggers**, the `ContentFile`/`ContentSha256` artifact
path, R2 bucket name/location, and DNS type/name/target/ttl.
```console
go vet ./...
@@ -108,10 +189,10 @@ go build ./...
go test ./...
```
## Adding rate limiting later (#7)
## Adding rate limiting later (#6 ADR / #7 impl)
[The abuse/rate-limit ADR](../docs/decisions/labels-and-abuse.md) chose to
[The abuse/rate-limit ADR (#6)](../docs/decisions/labels-and-abuse.md) chose to
implement ingest rate limiting as **Cloudflare Rate Limiting rules via Pulumi**
(`cloudflare.NewRuleset`, phase `http_ratelimit`, scoped to a zone). That is out
of scope for this ticket. The `cloudflareZoneId` config key and the structure of
`deploy.go` leave a clean insertion point; #7 adds the ruleset resource there.
of scope for this ticket. The `cloudflareZoneId` config key and the forward note
in `deploy.go` leave a clean insertion point; #7 adds the ruleset resource there.
+181 -23
View File
@@ -1,8 +1,11 @@
package main
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"github.com/pulumi/pulumi-cloudflare/sdk/v6/go/cloudflare"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/dns"
@@ -13,9 +16,10 @@ import (
// Logical (Pulumi) resource names. Stable across deploys and asserted by
// deploy_test.go, so treat them as part of the program's contract.
const (
resWorker = "ingest-worker"
resR2Bucket = "reports-bucket"
resDNSRecord = "ingest-dns-record"
resWorker = "ingest-worker"
resR2Bucket = "reports-bucket"
resDNSRecord = "ingest-dns-record"
resCronTrigger = "ingest-cron-triggers"
)
// Defaults for config values that have a sensible project-wide default. Values
@@ -35,15 +39,62 @@ const (
defaultDNSTTLSeconds = 300
)
// Worker binding names and runtime var defaults. These mirror wrangler.jsonc and
// the names the Worker code reads at runtime (internal/storage.*Binding,
// worker/*_wasm.go), so the deployed Worker actually has the R2 bucket, the
// Secrets Store secrets, and the plain vars it expects. Changing a binding *name*
// here is a breaking change to the Worker contract.
const (
// JS variable (binding) names the Worker reads via env.<name>.
bindingR2Bucket = "REPORTS_BUCKET" // R2 bucket (internal/storage.BucketBinding)
bindingEncKeyring = "BUGREPORT_ENC_KEYRING" // Secrets Store: AES keyring (ADR 0001)
bindingAdminToken = "ADMIN_TOKEN" // Secrets Store: admin API bearer token (ADR 0003)
bindingGitHubToken = "GITHUB_TOKEN" // Secrets Store: publish PAT (#14)
bindingOtelHeaders = "OTEL_EXPORTER_OTLP_HEADERS" // Secrets Store: OTLP auth headers (#17)
varGitHubRepo = "GITHUB_REPO" // plain var: "owner/repo" publish target (#14)
varOtelEndpoint = "OTEL_EXPORTER_OTLP_ENDPOINT" // plain var: OTLP base URL, "" disables (#17)
varOtelServiceName = "OTEL_SERVICE_NAME" // plain var: reported service.name (#17)
// Secrets Store secret_name values (the names of the stored secrets), from
// wrangler.jsonc secrets_store_secrets. The store_id is account-specific and
// supplied via the secretsStoreId config.
secretNameEncKeyring = "bugreport-enc-keyring"
secretNameAdminToken = "bugreport-admin-token"
secretNameGitHubToken = "github-token"
secretNameOtelHeaders = "otel-exporter-otlp-headers"
// Cloudflare multipart binding "type" discriminators.
// https://developers.cloudflare.com/workers/configuration/multipart-upload-metadata/#bindings
bindingTypeR2Bucket = "r2_bucket"
bindingTypeSecretsStore = "secrets_store_secret"
bindingTypePlainText = "plain_text"
// Runtime var defaults (match wrangler.jsonc "vars").
defaultGitHubRepo = "JMR-dev/LibreMail"
defaultOtelServiceName = "libremail-bug-report-ingest"
)
// defaultCronSchedules are the two Friday UTC Cron Triggers from wrangler.jsonc
// (#13). Cloudflare crons are UTC-only and 17:00 America/Chicago is a different
// UTC hour under CDT vs CST, so BOTH candidate hours are registered; the Worker's
// scheduled handler (internal/schedule.IsFriday1700Central) gates each fire so
// exactly one does the weekly publish and the other is a no-op.
var defaultCronSchedules = []string{
"0 22 * * 5", // Fridays 22:00 UTC == 17:00 Central during CDT (summer)
"0 23 * * 5", // Fridays 23:00 UTC == 17:00 Central during CST (winter)
}
// placeholderWorkerScript is a stand-in module body for the Worker.
//
// The deployed Worker is Go compiled by TinyGo to Wasm plus the syumai/workers
// ES-module shim, emitted by `pnpm run build` into ../build/ (git-ignored,
// produced by CI). Uploading that multipart Wasm artifact is the build/deploy
// pipeline's job; this program ships a documented placeholder so the resource is
// fully described and unit-testable without the artifact present. Override it at
// deploy time with the `workerScriptContent` config, or wire the real artifact
// via ContentFile/ContentSha256 in the deploy pipeline. See infra/README.md.
// produced by CI). The real, functional deploy uploads that built artifact by
// setting the `workerScriptPath` config (the CD workflow sets it to
// ../build/worker.mjs after `pnpm run build`), which switches the resource to
// ContentFile + ContentSha256. When `workerScriptPath` is unset this documented
// placeholder is uploaded instead, so the resource is fully described and
// unit-testable without the artifact present. Override the placeholder body with
// the `workerScriptContent` config. See infra/README.md.
const placeholderWorkerScript = `// PLACEHOLDER - replaced at deploy time by the TinyGo -> Wasm build (see repo README).
export default {
async fetch() {
@@ -59,14 +110,25 @@ type infraConfig struct {
// cloudflareAccountId is the Cloudflare account that owns the Worker + bucket.
cloudflareAccountId string
// cloudflareZoneId is optional and unused today. It is reserved so that
// Worker routes / custom domains and the ingest rate-limit ruleset (#7) can
// Worker routes / custom domains and the ingest rate-limit ruleset (#6/#7) can
// be added later without a config change. See the forward note in deploy.
cloudflareZoneId string
workerName string
workerContent string
workerScriptPath string // optional path to the built main module; enables ContentFile upload
compatibilityDate string
// secretsStoreId is the Cloudflare Secrets Store id that holds the Worker's
// secrets (keyring, admin token, GitHub token, OTLP headers). Account-specific,
// so required; it is NOT itself a secret (it is an id, like the account id).
secretsStoreId string
// Plain (non-secret) Worker vars.
githubRepo string
otelEndpoint string
otelServiceName string
r2BucketName string
r2BucketLocation string // optional; Cloudflare picks a location when empty
@@ -89,6 +151,10 @@ func loadConfig(ctx *pulumi.Context) (*infraConfig, error) {
if err != nil {
return nil, fmt.Errorf("config cloudflareAccountId: %w", err)
}
secretsStoreID, err := cfg.Try("secretsStoreId")
if err != nil {
return nil, fmt.Errorf("config secretsStoreId: %w", err)
}
managedZone, err := cfg.Try("dnsManagedZone")
if err != nil {
return nil, fmt.Errorf("config dnsManagedZone: %w", err)
@@ -114,7 +180,12 @@ func loadConfig(ctx *pulumi.Context) (*infraConfig, error) {
cloudflareZoneId: cfg.Get("cloudflareZoneId"),
workerName: getOr(cfg, "workerName", defaultWorkerName),
workerContent: getOr(cfg, "workerScriptContent", placeholderWorkerScript),
workerScriptPath: cfg.Get("workerScriptPath"),
compatibilityDate: getOr(cfg, "workerCompatibilityDate", defaultCompatibilityDate),
secretsStoreId: secretsStoreID,
githubRepo: getOr(cfg, "githubRepo", defaultGitHubRepo),
otelEndpoint: cfg.Get("otelExporterOtlpEndpoint"),
otelServiceName: getOr(cfg, "otelServiceName", defaultOtelServiceName),
r2BucketName: getOr(cfg, "r2BucketName", defaultR2BucketName),
r2BucketLocation: cfg.Get("r2BucketLocation"),
gcpProject: cfg.Get("gcpProject"),
@@ -134,6 +205,62 @@ func getOr(cfg *config.Config, key, def string) string {
return def
}
// fileSha256 returns the lowercase hex SHA-256 of the file at path. It is used to
// derive ContentSha256 for the Worker artifact upload; the Cloudflare provider
// requires ContentSha256 whenever ContentFile is set (it triggers an update when
// the built Wasm changes).
func fileSha256(path string) (string, error) {
data, err := os.ReadFile(path)
if err != nil {
return "", err
}
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:]), nil
}
// workerBindings returns the full binding set the Worker needs at runtime: the R2
// bucket, the four Cloudflare Secrets Store secrets, and the three plain vars.
// This is the #9-flagged wiring gap that #4 closes — before this the deployed
// Worker had no bindings and could not read its bucket/secrets/vars.
func workerBindings(cfg *infraConfig) cloudflare.WorkersScriptBindingArray {
return cloudflare.WorkersScriptBindingArray{
// R2 bucket: env.REPORTS_BUCKET -> the encrypted-report bucket (ADR 0001).
&cloudflare.WorkersScriptBindingArgs{
Name: pulumi.String(bindingR2Bucket),
Type: pulumi.String(bindingTypeR2Bucket),
BucketName: pulumi.String(cfg.r2BucketName),
},
// Secrets Store secrets, read at runtime via env.<name>.get().
secretsStoreBinding(bindingEncKeyring, cfg.secretsStoreId, secretNameEncKeyring),
secretsStoreBinding(bindingAdminToken, cfg.secretsStoreId, secretNameAdminToken),
secretsStoreBinding(bindingGitHubToken, cfg.secretsStoreId, secretNameGitHubToken),
secretsStoreBinding(bindingOtelHeaders, cfg.secretsStoreId, secretNameOtelHeaders),
// Plain (non-secret) vars, read via cloudflare.Getenv(<name>).
plainTextBinding(varGitHubRepo, cfg.githubRepo),
plainTextBinding(varOtelEndpoint, cfg.otelEndpoint),
plainTextBinding(varOtelServiceName, cfg.otelServiceName),
}
}
// secretsStoreBinding builds one Cloudflare Secrets Store binding (env.<name>.get()).
func secretsStoreBinding(name, storeID, secretName string) cloudflare.WorkersScriptBindingInput {
return &cloudflare.WorkersScriptBindingArgs{
Name: pulumi.String(name),
Type: pulumi.String(bindingTypeSecretsStore),
StoreId: pulumi.String(storeID),
SecretName: pulumi.String(secretName),
}
}
// plainTextBinding builds one plain-text (non-secret) var binding.
func plainTextBinding(name, text string) cloudflare.WorkersScriptBindingInput {
return &cloudflare.WorkersScriptBindingArgs{
Name: pulumi.String(name),
Type: pulumi.String(bindingTypePlainText),
Text: pulumi.String(text),
}
}
// deploy registers all resources for the bug-report ingest stack.
func deploy(ctx *pulumi.Context) error {
cfg, err := loadConfig(ctx)
@@ -141,16 +268,29 @@ func deploy(ctx *pulumi.Context) error {
return err
}
// 1. Cloudflare Worker script - the ingest Worker built in #1. Content is a
// documented placeholder; the real Wasm artifact is uploaded by the build
// pipeline (see placeholderWorkerScript).
worker, err := cloudflare.NewWorkersScript(ctx, resWorker, &cloudflare.WorkersScriptArgs{
// 1. Cloudflare Worker script - the ingest Worker built in #1, now wired with
// its full binding set (R2 + Secrets Store + vars). Content is the real
// built artifact when workerScriptPath is set (the CD workflow points it at
// ../build/worker.mjs after `pnpm run build`); otherwise a documented
// placeholder module body (see placeholderWorkerScript).
scriptArgs := &cloudflare.WorkersScriptArgs{
AccountId: pulumi.String(cfg.cloudflareAccountId),
ScriptName: pulumi.String(cfg.workerName),
Content: pulumi.String(cfg.workerContent),
MainModule: pulumi.String(mainModule),
CompatibilityDate: pulumi.String(cfg.compatibilityDate),
})
Bindings: workerBindings(cfg),
}
if cfg.workerScriptPath != "" {
sum, err := fileSha256(cfg.workerScriptPath)
if err != nil {
return fmt.Errorf("worker script artifact %q: %w", cfg.workerScriptPath, err)
}
scriptArgs.ContentFile = pulumi.String(cfg.workerScriptPath)
scriptArgs.ContentSha256 = pulumi.String(sum)
} else {
scriptArgs.Content = pulumi.String(cfg.workerContent)
}
worker, err := cloudflare.NewWorkersScript(ctx, resWorker, scriptArgs)
if err != nil {
return fmt.Errorf("worker script: %w", err)
}
@@ -170,7 +310,24 @@ func deploy(ctx *pulumi.Context) error {
return fmt.Errorf("r2 bucket: %w", err)
}
// 3. Google Cloud DNS record pointing the ingest hostname at the Worker's
// 3. Cloudflare Cron Triggers for the weekly publish job (#13). Both Friday UTC
// hours are registered (DST straddle); the Worker's scheduled handler gates
// which one publishes. Depends on the Worker script it schedules.
schedules := cloudflare.WorkersCronTriggerScheduleArray{}
for _, cron := range defaultCronSchedules {
schedules = append(schedules, &cloudflare.WorkersCronTriggerScheduleArgs{
Cron: pulumi.String(cron),
})
}
if _, err := cloudflare.NewWorkersCronTrigger(ctx, resCronTrigger, &cloudflare.WorkersCronTriggerArgs{
AccountId: pulumi.String(cfg.cloudflareAccountId),
ScriptName: worker.ScriptName,
Schedules: schedules,
}); err != nil {
return fmt.Errorf("cron triggers: %w", err)
}
// 4. Google Cloud DNS record pointing the ingest hostname at the Worker's
// route/custom domain. DNS authority is Google Cloud DNS; the managed zone
// is referenced by name (it is managed outside this stack) and a record is
// added to it.
@@ -189,17 +346,18 @@ func deploy(ctx *pulumi.Context) error {
return fmt.Errorf("dns record: %w", err)
}
// Forward note (#7 / docs/decisions/labels-and-abuse.md): the accepted ADR
// implements ingest rate limiting as Cloudflare Rate Limiting rules via
// Pulumi (cloudflare.NewRuleset with Phase "http_ratelimit", scoped to
// cloudflareZoneId). That is out of scope for #2, but the config
// (cloudflareZoneId) and this structure leave a clean insertion point:
// add the ruleset resource here. Worker routes / a custom domain would
// attach the same way via cloudflareZoneId.
// Forward note (#6 ADR docs/decisions/labels-and-abuse.md / #7 impl): the
// accepted ADR implements ingest rate limiting as Cloudflare Rate Limiting
// rules via Pulumi (cloudflare.NewRuleset with Phase "http_ratelimit", scoped
// to cloudflareZoneId). That is out of scope for this ticket, but the config
// (cloudflareZoneId) and this structure leave a clean insertion point: add the
// ruleset resource here. Worker routes / a custom domain would attach the same
// way via cloudflareZoneId.
ctx.Export("workerName", worker.ScriptName)
ctx.Export("workerAccountId", worker.AccountId)
ctx.Export("r2BucketName", bucket.Name)
ctx.Export("cronSchedules", pulumi.ToStringArray(defaultCronSchedules))
ctx.Export("dnsRecordFqdn", record.Name)
ctx.Export("dnsRecordTargets", record.Rrdatas)
if cfg.cloudflareZoneId != "" {
+191 -5
View File
@@ -1,7 +1,11 @@
package main
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"os"
"path/filepath"
"testing"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
@@ -14,9 +18,10 @@ const testProject = "libremail-bug-report-ingest-infra"
// Resource type tokens registered by the program.
const (
tokWorkersScript = "cloudflare:index/workersScript:WorkersScript"
tokR2Bucket = "cloudflare:index/r2Bucket:R2Bucket"
tokDNSRecordSet = "gcp:dns/recordSet:RecordSet"
tokWorkersScript = "cloudflare:index/workersScript:WorkersScript"
tokR2Bucket = "cloudflare:index/r2Bucket:R2Bucket"
tokWorkersCronTrigger = "cloudflare:index/workersCronTrigger:WorkersCronTrigger"
tokDNSRecordSet = "gcp:dns/recordSet:RecordSet"
)
// recordingMocks implements pulumi.MockResourceMonitor, capturing every
@@ -54,6 +59,7 @@ func key(k string) string { return testProject + ":" + k }
func fullConfig() map[string]string {
return map[string]string{
key("cloudflareAccountId"): "cf-acct-123",
key("secretsStoreId"): "ss-store-xyz",
key("dnsManagedZone"): "libremail-zone",
key("dnsRecordName"): "bugreport.example.com.",
key("dnsRecordTarget"): "libremail-bug-report-ingest.acme.workers.dev.",
@@ -93,6 +99,34 @@ func strProp(t *testing.T, pm resource.PropertyMap, k string) string {
return v.StringValue()
}
// objArray returns pm[k] as a slice of PropertyMaps, failing if it is absent or
// not an array of objects. Used to walk the Worker bindings / cron schedules.
func objArray(t *testing.T, pm resource.PropertyMap, k string) []resource.PropertyMap {
t.Helper()
v, ok := pm[resource.PropertyKey(k)]
if !ok || !v.IsArray() {
t.Fatalf("property %q missing or not an array: %v", k, pm.Mappable())
}
out := make([]resource.PropertyMap, 0, len(v.ArrayValue()))
for i, el := range v.ArrayValue() {
if !el.IsObject() {
t.Fatalf("%s[%d] is not an object: %v", k, i, el)
}
out = append(out, el.ObjectValue())
}
return out
}
// bindingsByName indexes the Worker's bindings array by its "name" (the JS var).
func bindingsByName(t *testing.T, in resource.PropertyMap) map[string]resource.PropertyMap {
t.Helper()
out := map[string]resource.PropertyMap{}
for _, b := range objArray(t, in, "bindings") {
out[strProp(t, b, "name")] = b
}
return out
}
func TestWorkerScriptRegistered(t *testing.T) {
m := runProgram(t, fullConfig())
in := m.find(t, tokWorkersScript).Inputs
@@ -114,6 +148,128 @@ func TestWorkerScriptRegistered(t *testing.T) {
}
}
// TestWorkerScriptBindings is the core coverage for #4/#9: the deployed Worker
// must carry the R2 bucket binding, the four Secrets Store bindings, and the
// three plain vars, with the exact names/store/secret_name the Worker reads at
// runtime (wrangler.jsonc contract).
func TestWorkerScriptBindings(t *testing.T) {
m := runProgram(t, fullConfig())
in := m.find(t, tokWorkersScript).Inputs
bindings := bindingsByName(t, in)
// R2 bucket binding.
r2, ok := bindings["REPORTS_BUCKET"]
if !ok {
t.Fatalf("missing REPORTS_BUCKET binding; have %v", bindingNames(bindings))
}
if got, want := strProp(t, r2, "type"), "r2_bucket"; got != want {
t.Errorf("REPORTS_BUCKET type = %q, want %q", got, want)
}
if got, want := strProp(t, r2, "bucketName"), "libremail-bug-reports"; got != want {
t.Errorf("REPORTS_BUCKET bucketName = %q, want %q", got, want)
}
// Secrets Store bindings: name -> secret_name (all share the one store id).
wantSecrets := map[string]string{
"BUGREPORT_ENC_KEYRING": "bugreport-enc-keyring",
"ADMIN_TOKEN": "bugreport-admin-token",
"GITHUB_TOKEN": "github-token",
"OTEL_EXPORTER_OTLP_HEADERS": "otel-exporter-otlp-headers",
}
for name, wantSecretName := range wantSecrets {
b, ok := bindings[name]
if !ok {
t.Errorf("missing Secrets Store binding %q; have %v", name, bindingNames(bindings))
continue
}
if got, want := strProp(t, b, "type"), "secrets_store_secret"; got != want {
t.Errorf("%s type = %q, want %q", name, got, want)
}
if got, want := strProp(t, b, "storeId"), "ss-store-xyz"; got != want {
t.Errorf("%s storeId = %q, want %q", name, got, want)
}
if got := strProp(t, b, "secretName"); got != wantSecretName {
t.Errorf("%s secretName = %q, want %q", name, got, wantSecretName)
}
}
// Plain-text vars.
wantVars := map[string]string{
"GITHUB_REPO": "JMR-dev/LibreMail",
"OTEL_EXPORTER_OTLP_ENDPOINT": "",
"OTEL_SERVICE_NAME": "libremail-bug-report-ingest",
}
for name, wantText := range wantVars {
b, ok := bindings[name]
if !ok {
t.Errorf("missing plain var binding %q; have %v", name, bindingNames(bindings))
continue
}
if got, want := strProp(t, b, "type"), "plain_text"; got != want {
t.Errorf("%s type = %q, want %q", name, got, want)
}
if got := textValue(b); got != wantText {
t.Errorf("%s text = %q, want %q", name, got, wantText)
}
}
}
// bindingNames is a small diagnostic helper for failure messages.
func bindingNames(bindings map[string]resource.PropertyMap) []string {
names := make([]string, 0, len(bindings))
for n := range bindings {
names = append(names, n)
}
return names
}
// textValue reads a binding's "text" property, treating an absent property as the
// empty string (an empty plain-text var may serialize either way).
func textValue(b resource.PropertyMap) string {
v, ok := b[resource.PropertyKey("text")]
if !ok {
return ""
}
if !v.IsString() {
return ""
}
return v.StringValue()
}
// TestWorkerContentFromArtifact verifies the artifact-override path: when
// workerScriptPath is set (as the CD workflow sets it to ../build/worker.mjs
// after `pnpm run build`), the Worker uploads the built file via ContentFile +
// a computed ContentSha256 instead of the inline placeholder content.
func TestWorkerContentFromArtifact(t *testing.T) {
dir := t.TempDir()
artifact := filepath.Join(dir, "worker.mjs")
body := []byte("export default { async fetch() { return new Response('ok'); } };\n")
if err := os.WriteFile(artifact, body, 0o600); err != nil {
t.Fatalf("write artifact: %v", err)
}
sum := sha256.Sum256(body)
wantSha := hex.EncodeToString(sum[:])
cfg := fullConfig()
cfg[key("workerScriptPath")] = artifact
m := runProgram(t, cfg)
in := m.find(t, tokWorkersScript).Inputs
if got, want := strProp(t, in, "contentFile"), artifact; got != want {
t.Errorf("contentFile = %q, want %q", got, want)
}
if got := strProp(t, in, "contentSha256"); got != wantSha {
t.Errorf("contentSha256 = %q, want %q", got, wantSha)
}
if _, ok := in[resource.PropertyKey("content")]; ok {
t.Error("content should be unset when workerScriptPath drives a ContentFile upload")
}
// Bindings must still be attached in the artifact path.
if _, ok := bindingsByName(t, in)["REPORTS_BUCKET"]; !ok {
t.Error("REPORTS_BUCKET binding missing on the artifact-content Worker")
}
}
func TestR2BucketRegistered(t *testing.T) {
m := runProgram(t, fullConfig())
in := m.find(t, tokR2Bucket).Inputs
@@ -169,13 +325,43 @@ func TestDNSRecordRegistered(t *testing.T) {
}
}
func TestExactlyThreeManagedResources(t *testing.T) {
// TestWorkerCronTriggers asserts the two Friday UTC crons (#13) are registered as
// a WorkersCronTrigger bound to the ingest Worker.
func TestWorkerCronTriggers(t *testing.T) {
m := runProgram(t, fullConfig())
in := m.find(t, tokWorkersCronTrigger).Inputs
if got, want := strProp(t, in, "scriptName"), "libremail-bug-report-ingest"; got != want {
t.Errorf("cron scriptName = %q, want %q", got, want)
}
if got, want := strProp(t, in, "accountId"), "cf-acct-123"; got != want {
t.Errorf("cron accountId = %q, want %q", got, want)
}
var crons []string
for _, s := range objArray(t, in, "schedules") {
crons = append(crons, strProp(t, s, "cron"))
}
want := []string{"0 22 * * 5", "0 23 * * 5"}
if len(crons) != len(want) {
t.Fatalf("cron schedules = %v, want %v", crons, want)
}
for i := range want {
if crons[i] != want[i] {
t.Errorf("schedules[%d] = %q, want %q", i, crons[i], want[i])
}
}
}
// TestManagedResourceCounts pins the managed resource set: exactly one each of the
// Worker script, R2 bucket, cron trigger, and DNS record.
func TestManagedResourceCounts(t *testing.T) {
m := runProgram(t, fullConfig())
counts := map[string]int{}
for _, r := range m.resources {
counts[r.TypeToken]++
}
for _, tok := range []string{tokWorkersScript, tokR2Bucket, tokDNSRecordSet} {
for _, tok := range []string{tokWorkersScript, tokR2Bucket, tokWorkersCronTrigger, tokDNSRecordSet} {
if counts[tok] != 1 {
t.Errorf("expected exactly 1 %s, got %d", tok, counts[tok])
}