feat(deploy) working on deployment to hetzner

.gitignore

@@ -1,4 +1,8 @@
 # Pulumi
+# gh-repo-bootstrap state key. The encrypted state beside it is fine to commit;
+# the key that decrypts it is not.
+state/.passphrase
+state/.passphrase.*
 pulumi/Pulumi.*.yaml.bak
 pulumi/.pulumi/
 pulumi/bin/
@@ -18,6 +22,7 @@ secrets/*.dec.*
 ansible/.vault_pass
 ansible/*.retry
 ansible/.facts_cache/
+ansible/.collections/
 
 # Editor
 .vscode/

ansible/group_vars/all.yml

@@ -30,7 +30,7 @@ epel_packages:
 # controller (operator laptop or Cloud Build runner) must be authed to GCP
 # via ADC. Override gcp_project in inventory if the disposable + production
 # stacks live in different projects.
-gcp_project: REPLACE-with-gcp-project-id
+gcp_project: vaultwarden-497422
 secret_prefix: vaultwarden-
 
 # Extra CIDRs allowed to reach SSH at the host nftables layer, on top of the

ansible/roles/nftables/tasks/main.yml

@@ -1,4 +1,17 @@
 ---
+# firewalld and a hand-written nftables.service ruleset can't coexist: our
+# `flush ruleset` wipes firewalld's tables, and a firewalld reload wipes ours.
+- name: Gather service facts
+  ansible.builtin.service_facts:
+
+- name: Stop and mask firewalld
+  ansible.builtin.systemd:
+    name: firewalld.service
+    state: stopped
+    enabled: false
+    masked: true
+  when: "'firewalld.service' in ansible_facts.services"
+
 - name: Install nftables base ruleset
   ansible.builtin.template:
     src: main.nft.j2

ansible/roles/quadlets/tasks/main.yml

@@ -74,6 +74,7 @@
     content: |
       VAULT_FQDN={{ vault_fqdn }}
       WG_SUBNET={{ wg_subnet }}
+      GCP_PROJECT={{ gcp_project }}
     owner: "{{ vault_user }}"
     group: "{{ vault_user }}"
     mode: "0644"

caddy/Caddyfile

@@ -20,14 +20,22 @@
 
 {$VAULT_FQDN} {
     tls {
-        # googleclouddns reads the project_id from the SA JSON pointed at
-        # by GOOGLE_APPLICATION_CREDENTIALS — no explicit project flag needed.
-        dns googleclouddns
+        # Credentials come from the SA JSON at GOOGLE_APPLICATION_CREDENTIALS,
+        # but the project does not: the module refuses to load without
+        # gcp_project ("missing Google Cloud project ID"). The zone lookup
+        # lists the project's zones, so the SA needs project-level
+        # dns.managedZones.list on top of its zone-scoped dns.admin.
+        dns googleclouddns {
+            gcp_project {$GCP_PROJECT}
+        }
     }
 
     encode zstd gzip
 
     coraza_waf {
+        # Mounts the embedded CRS filesystem; without it the @-prefixed
+        # Includes below fail with "no such file or directory".
+        load_owasp_crs
         directives `
             Include @coraza.conf-recommended
             Include @crs-setup.conf.example

caddy/Containerfile

@@ -25,7 +25,8 @@ ARG GCD_VERSION
 ENV CGO_ENABLED=0
 ENV GOFLAGS=-trimpath
 ENV GOTOOLCHAIN=local
-ENV PATH=/root/go/bin:/usr/local/go/bin:/usr/bin:/bin
+# The golang image's GOPATH is /go, so `go install` lands in /go/bin.
+ENV PATH=/go/bin:/usr/local/go/bin:/usr/bin:/bin
 
 RUN go install github.com/caddyserver/xcaddy/cmd/xcaddy@${XCADDY_VERSION}
 
@@ -38,7 +39,7 @@ FROM ${RUNTIME_IMAGE}
 
 COPY --from=builder /out/caddy /usr/local/bin/caddy
 
-LABEL org.opencontainers.image.source="https://github.com/jasonross/vaultwarden_deployment"
+LABEL org.opencontainers.image.source="https://github.com/JMR-dev/vaultwarden-deployment"
 LABEL org.opencontainers.image.description="Custom Caddy with googleclouddns DNS-01 + Coraza WAF (DetectionOnly)."
 
 EXPOSE 443

cloudbuild.yaml

@@ -32,7 +32,7 @@
 
 substitutions:
   _PULUMI_STACK: production
-  _PULUMI_STATE_BUCKET: gs://REPLACE-with-pulumi-state-bucket
+  _PULUMI_STATE_BUCKET: gs://vaultwarden-497422-pulumi-state
 
 availableSecrets:
   secretManager:

quadlets/caddy.container.j2

@@ -25,13 +25,26 @@ Network=vaultwarden.network
 # podman role) lets rootless caddy bind without CAP_NET_BIND_SERVICE.
 PublishPort=443:443
 
+# The distroless image runs Caddy as nonroot (65532). keep-id maps the host
+# vault_user onto that uid, so the bind-mounted data/log dirs (owned by
+# vault_user) are writable inside; without it they appear as root:root 0750
+# and Caddy can't open its logs or store certs.
+UserNS=keep-id:uid=65532,gid=65532
+# The host sysctl above doesn't carry into the container's own netns, where
+# nonroot would otherwise be refused :443 (and :80, which Caddy binds for
+# redirects but isn't published). Same as Docker's per-container default.
+Sysctl=net.ipv4.ip_unprivileged_port_start=0
+# Caddy's default storage is $HOME/.local/share/caddy — the container layer,
+# wiped on every auto-update recreate. Keep certs on the /data mount instead.
+Environment=XDG_DATA_HOME=/data
+
 # Caddyfile bind-mount, plus Caddy's data dir (certs, OCSP staples) and
 # log dir (access log + Coraza audit). All :Z under SELinux.
 Volume=%h/caddy/Caddyfile:/etc/caddy/Caddyfile:Z,ro
 Volume=%h/caddy/data:/data:Z
 Volume=%h/caddy/logs:/var/log/caddy:Z
 
-# VAULT_FQDN + WG_SUBNET come from the Ansible-rendered env file.
+# VAULT_FQDN, WG_SUBNET + GCP_PROJECT come from the Ansible-rendered env file.
 EnvironmentFile=%h/caddy/caddy.env
 
 # GCP SA JSON for DNS-01. Podman mounts the secret at /run/secrets/gcp-sa-dns

runbook/README.md

@@ -168,9 +168,15 @@ cp inventory/disposable.yml.example inventory/disposable.yml
 
 # Update group_vars/all.yml gcp_project to the real value.
 ansible-playbook -i inventory/disposable.yml playbook.yml \
-  --extra-vars "caddy_image=$(cd ../pulumi && pulumi -s disposable stack output imageRepo):latest"
+  --extra-vars "caddy_image=$(cd ../pulumi && pulumi -s disposable stack output imageRepo):latest" \
+  --extra-vars "{\"nftables_extra_ssh_cidrs\":[\"$(curl -s ifconfig.me)/32\"]}"
 ```
 
+The `nftables_extra_ssh_cidrs` var matters on the first run: WG isn't up yet,
+so you're SSHing over the public IP, and the nftables ruleset otherwise only
+admits SSH from the WG subnet. Once §2.4 closes the Cloud Firewall hole,
+re-run without it to drop the host-side rule too.
+
 ADC handles GCP auth for the Secret Manager lookups during the run.
 Expect ~5 minutes for the first run (package downloads dominate).
 
@@ -221,7 +227,7 @@ gcloud builds triggers create manual \
   --name=vaultwarden-deploy \
   --build-config=cloudbuild.yaml \
   --repo-type=GITHUB \
-  --repo=jasonross/vaultwarden_deployment \
+  --repo=https://github.com/JMR-dev/vaultwarden-deployment \
   --branch=main \
   --service-account=projects/<PROJECT>/serviceAccounts/vaultwarden-cb@<PROJECT>.iam.gserviceaccount.com \
   --substitutions=_PULUMI_STACK=production,_PULUMI_STATE_BUCKET=gs://<bucket>
@@ -536,13 +542,11 @@ optional.
 ### Caddy image rolls back broken
 
 ```sh
-gh auth login        # if needed
+IMAGE=$(cd pulumi && pulumi -s <stack> stack output imageRepo)
 # Find the previous SHA:
-gh api /users/<owner>/packages/container/vaultwarden-caddy/versions
-# Re-tag a known-good <sha> as :latest in AR, e.g.:
-# gcloud artifacts docker tags add \
-#   <region>-docker.pkg.dev/<project>/<repo>/vaultwarden-caddy:<good-sha> \
-#   <region>-docker.pkg.dev/<project>/<repo>/vaultwarden-caddy:latest
+gcloud artifacts docker tags list "$IMAGE"
+# Re-tag a known-good <sha> as :latest:
+gcloud artifacts docker tags add "$IMAGE:<good-sha>" "$IMAGE:latest"
 ssh ansible@10.42.0.1
 sudo -u vaultwarden XDG_RUNTIME_DIR=/run/user/1100 podman auto-update --rollback
 ```

secrets/README.md

@@ -86,7 +86,7 @@ gcp_sa_dns_json: |
 ### `vaultwarden-restic` — backup destination
 
 ```yaml
-restic_repository: s3:s3.<region>.r2.cloudflarestorage.com/<bucket>
+restic_repository: s3:https://<cloudflare-account-id>.r2.cloudflarestorage.com/<bucket>
 restic_password: <strong, randomly-generated>
 r2_access_key_id: <R2 access key>
 r2_secret_access_key: <R2 secret key>
@@ -112,7 +112,7 @@ deploys through Cloud Build. Each holds a single raw value (not YAML).
 ### `vaultwarden-hcloud-token` — Hetzner Cloud API token
 
 ```sh
-read -s -p "hcloud token: " TOK; echo
+read -rs "TOK?hcloud token: "; echo   # zsh; in bash: read -rs -p "hcloud token: " TOK
 printf '%s' "$TOK" \
   | gcloud secrets create vaultwarden-hcloud-token --replication-policy=automatic --data-file=-
 unset TOK

state/.passphrase

@@ -1 +0,0 @@
-ec50a7e952179266228b6dc838e811e23548f1d382ef9181e2f1ff00e722fa96

state/.pulumi/stacks/gh-repo-bootstrap/bootstrap.json.attrs

@@ -1 +1 @@
-{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"text/plain; charset=utf-8","user.metadata":null,"md5":"Zu5pepJvejGpWRUQmLJHCg=="}
+{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"text/plain; charset=utf-8","user.metadata":null,"md5":"n3eoL5Cm3O744gqgTp9dMA=="}

state/.pulumi/stacks/gh-repo-bootstrap/bootstrap.json.bak.attrs

@@ -1 +1 @@
-{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"text/plain; charset=utf-8","user.metadata":null,"md5":"Fwu7FOt7MRUCj6A507HpPw=="}
+{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"text/plain; charset=utf-8","user.metadata":null,"md5":"Zu5pepJvejGpWRUQmLJHCg=="}

state/Pulumi.bootstrap.yaml

@@ -1,3 +1,3 @@
-encryptionsalt: v1:vJoLalmhYaE=:v1:p/1QIOtwxsIsbTAA:Qlo64fLkjecjC7Qimlaq0wfIeDg/4w==
+encryptionsalt: v1:4OWjrUrnwfw=:v1:UZNdYinbREaMwza3:ahk9DMajp75NgHOPi3r34fDnivDIiw==
 config:
   github:owner: JMR-dev

state/Pulumi.yaml

@@ -1,4 +1,4 @@
 name: gh-repo-bootstrap
 runtime: go
 backend:
-  url: file:///home/jasonross/workspace/vaultwarden_deployment/state
+  url: file:///home/jasonross/workspace/vaultwarden-deployment/state