From ae3a1373610f4c96b8f3064b8f28e5337f827359 Mon Sep 17 00:00:00 2001 From: Jason Ross Date: Mon, 27 Apr 2026 19:36:39 -0500 Subject: [PATCH] deploy stack --- admin-stack/README.md | 69 +++++++++ admin-stack/ansible/inventory.yml | 17 +++ admin-stack/ansible/networks.yml | 25 +++ admin-stack/ansible/templates/client.conf.j2 | 11 ++ admin-stack/ansible/templates/wg0.conf.j2 | 13 ++ admin-stack/ansible/wireguard.yml | 153 +++++++++++++++++++ admin-stack/caddy/Caddyfile | 20 +++ admin-stack/caddy/Dockerfile | 6 + admin-stack/compose.yaml | 59 +++++++ admin-stack/scripts/bootstrap.sh | 49 ++++++ admin-stack/scripts/sqlite-backup.sh | 19 +++ admin-stack/scripts/verify.sh | 83 ++++++++++ 12 files changed, 524 insertions(+) create mode 100644 admin-stack/README.md create mode 100644 admin-stack/ansible/inventory.yml create mode 100644 admin-stack/ansible/networks.yml create mode 100644 admin-stack/ansible/templates/client.conf.j2 create mode 100644 admin-stack/ansible/templates/wg0.conf.j2 create mode 100644 admin-stack/ansible/wireguard.yml create mode 100644 admin-stack/caddy/Caddyfile create mode 100644 admin-stack/caddy/Dockerfile create mode 100644 admin-stack/compose.yaml create mode 100755 admin-stack/scripts/bootstrap.sh create mode 100755 admin-stack/scripts/sqlite-backup.sh create mode 100755 admin-stack/scripts/verify.sh diff --git a/admin-stack/README.md b/admin-stack/README.md new file mode 100644 index 0000000..129dd5e --- /dev/null +++ b/admin-stack/README.md @@ -0,0 +1,69 @@ +# Admin Stack + +This repository contains the deployment configuration for the Admin interface. + +## Prerequisites + +1. Same host as Stoat, rootless user with linger. +2. Ansible + podman + WireGuard userspace tools installed. +3. GCP credentials for Secret Manager. +4. Public DNS record for `admin.${DOMAIN}` in Google Cloud DNS pointing to the WG server IP (or no record at all if using `tls internal`). +5. Cloud DNS service account provisioned with `roles/dns.admin` and stored in Secret Manager. + +## First Deploy + +Run the bootstrap script: + +```bash +./scripts/bootstrap.sh +``` + +## Adding a new WG client + +1. Edit `wg_clients` in `ansible/inventory.yml` (or your overriding group_vars). +2. Re-run the wireguard playbook: + ```bash + ansible-playbook -i ansible/inventory.yml ansible/wireguard.yml + ``` +3. Distribute the new client config from `./generated/clients/.conf`. + +## Removing a WG client + +1. Remove the client from `wg_clients`. +2. Re-run the playbook. +3. Verify in `wg show wg0` that the peer is gone. + +## Rotating the WG server key + +Rotating the server key is disruptive — every client config must be regenerated and redistributed. + +1. Remove the old key from Secret Manager or create a new version. +2. Re-run the wireguard playbook. + +## Rotating the Caddy DNS service account key + +1. Generate a new key with `gcloud iam service-accounts keys create`. +2. Push to Secret Manager as a new version. +3. Re-run bootstrap step 6 to materialize the key. +4. Restart Caddy (`podman compose restart caddy`). +5. Disable the old key with `gcloud iam service-accounts keys disable` and finally delete after a grace period. + +## Redeploy Procedure + +1. `podman compose pull` +2. `podman compose up -d` +3. `./scripts/verify.sh` + +## Secret Rotation Procedure + +1. Update the secret in GCP Secret Manager (e.g. `admin-env`). +2. Materialize the `.env` file again. +3. `podman compose up -d` to recreate containers with the new environment. + +## SQLite Backup and Recovery Procedure + +Backups are handled by `scripts/sqlite-backup.sh`. + +1. To restore, stop the `admin-api` container. +2. Replace the live `admin.db` in the `admin-sqlite` named volume with the snapshot file. +3. Restart the `admin-api` container. diff --git a/admin-stack/ansible/inventory.yml b/admin-stack/ansible/inventory.yml new file mode 100644 index 0000000..85f6d23 --- /dev/null +++ b/admin-stack/ansible/inventory.yml @@ -0,0 +1,17 @@ +all: + children: + admin_host: + hosts: + localhost: + ansible_connection: local + vars: + host_public_ip: "192.0.2.1" + wg_subnet: "10.42.0.0/24" + wg_server_ip: "10.42.0.1" + wg_listen_port: 51820 + wg_clients: + - name: jason-laptop + ip: "10.42.0.10" + - name: jason-phone + ip: "10.42.0.11" + podman_user: "stoat" diff --git a/admin-stack/ansible/networks.yml b/admin-stack/ansible/networks.yml new file mode 100644 index 0000000..ce2cada --- /dev/null +++ b/admin-stack/ansible/networks.yml @@ -0,0 +1,25 @@ +--- +- name: Podman Networks Setup + hosts: admin_host + become: true + become_user: "{{ podman_user }}" + tasks: + - name: Verify linger is enabled + ansible.builtin.command: loginctl show-user {{ podman_user }} + register: linger_check + changed_when: false + failed_when: "'Linger=yes' not in linger_check.stdout" + + - name: Admin edge network + containers.podman.podman_network: + name: admin-edge + driver: bridge + subnet: 10.89.20.0/24 + internal: false + state: present + + - name: Assert stoat-shared exists (Stoat's Ansible owns it) + ansible.builtin.command: podman network inspect stoat-shared + register: shared_check + changed_when: false + failed_when: shared_check.rc != 0 diff --git a/admin-stack/ansible/templates/client.conf.j2 b/admin-stack/ansible/templates/client.conf.j2 new file mode 100644 index 0000000..6efb673 --- /dev/null +++ b/admin-stack/ansible/templates/client.conf.j2 @@ -0,0 +1,11 @@ +[Interface] +PrivateKey = {{ client.private_key }} +Address = {{ client.ip }}/24 +DNS = 1.1.1.1 + +[Peer] +PublicKey = {{ wg_server_public_key }} +PresharedKey = {{ client.psk }} +Endpoint = {{ host_public_ip }}:{{ wg_listen_port }} +AllowedIPs = {{ wg_subnet }} +PersistentKeepalive = 25 diff --git a/admin-stack/ansible/templates/wg0.conf.j2 b/admin-stack/ansible/templates/wg0.conf.j2 new file mode 100644 index 0000000..92d5985 --- /dev/null +++ b/admin-stack/ansible/templates/wg0.conf.j2 @@ -0,0 +1,13 @@ +[Interface] +PrivateKey = {{ wg_server_private_key }} +Address = {{ wg_server_ip }}/24 +ListenPort = {{ wg_listen_port }} +SaveConfig = false + +{% for client in wg_clients_enriched %} +[Peer] +# {{ client.name }} +PublicKey = {{ client.public_key }} +PresharedKey = {{ client.psk }} +AllowedIPs = {{ client.ip }}/32 +{% endfor %} diff --git a/admin-stack/ansible/wireguard.yml b/admin-stack/ansible/wireguard.yml new file mode 100644 index 0000000..644a3c8 --- /dev/null +++ b/admin-stack/ansible/wireguard.yml @@ -0,0 +1,153 @@ +--- +- name: WireGuard Host Setup + hosts: admin_host + become: true + tasks: + - name: Ensure wireguard and tools are installed + ansible.builtin.package: + name: + - wireguard-tools + - firewalld + state: present + + - name: Ensure firewalld is running and enabled + ansible.builtin.systemd: + name: firewalld + state: started + enabled: true + + - name: Open WG UDP port on public zone + ansible.posix.firewalld: + zone: public + port: "{{ wg_listen_port }}/udp" + permanent: true + state: enabled + notify: Reload firewalld + + - name: Check if WG server private key exists in Secret Manager + delegate_to: localhost + become: false + ansible.builtin.command: > + gcloud secrets versions access latest --secret=admin-wg-server-key --project={{ lookup('env', 'GCP_PROJECT_ID') }} + register: wg_sm_check + failed_when: false + changed_when: false + + - name: Generate WG server private key locally if not in Secret Manager + delegate_to: localhost + become: false + ansible.builtin.command: wg genkey + register: wg_local_gen + when: wg_sm_check.rc != 0 + changed_when: true + + - name: Create Secret in Secret Manager if missing + delegate_to: localhost + become: false + ansible.builtin.command: > + gcloud secrets create admin-wg-server-key --project={{ lookup('env', 'GCP_PROJECT_ID') }} --replication-policy="automatic" + when: wg_sm_check.rc != 0 + failed_when: false + changed_when: false + + - name: Push new WG server private key to Secret Manager + delegate_to: localhost + become: false + ansible.builtin.command: > + gcloud secrets versions add admin-wg-server-key + --data-file=- + --project={{ lookup('env', 'GCP_PROJECT_ID') }} + args: + stdin: "{{ wg_local_gen.stdout }}" + when: wg_sm_check.rc != 0 + + - name: Set server private key variable + ansible.builtin.set_fact: + wg_server_private_key: "{{ wg_sm_check.stdout if wg_sm_check.rc == 0 else wg_local_gen.stdout }}" + no_log: true + + - name: Generate server public key + delegate_to: localhost + become: false + ansible.builtin.command: wg pubkey + args: + stdin: "{{ wg_server_private_key }}" + register: wg_server_pub_gen + changed_when: false + + - name: Set server public key variable + ansible.builtin.set_fact: + wg_server_public_key: "{{ wg_server_pub_gen.stdout }}" + + - name: Ensure /etc/wireguard directory exists + ansible.builtin.file: + path: /etc/wireguard + state: directory + mode: "0700" + + - name: Generate client keys + delegate_to: localhost + become: false + ansible.builtin.shell: | + priv=$(wg genkey) + pub=$(echo "$priv" | wg pubkey) + psk=$(wg genpsk) + echo '{"private_key": "'$priv'", "public_key": "'$pub'", "psk": "'$psk'"}' + register: wg_client_keys_gen + with_items: "{{ wg_clients }}" + changed_when: true + no_log: true + + - name: Enrich wg_clients with keys + ansible.builtin.set_fact: + wg_clients_enriched: >- + {{ + wg_clients_enriched | default([]) + + [item.0 | combine(item.1.stdout | from_json)] + }} + loop: "{{ wg_clients | zip(wg_client_keys_gen.results) | list }}" + no_log: true + + - name: Render server wg0.conf + ansible.builtin.template: + src: templates/wg0.conf.j2 + dest: /etc/wireguard/wg0.conf + mode: "0600" + notify: Restart wg-quick + + - name: Enable and start wg-quick@wg0 + ansible.builtin.systemd: + name: wg-quick@wg0 + state: started + enabled: true + + - name: Ensure client config directory exists on control machine + delegate_to: localhost + become: false + ansible.builtin.file: + path: "{{ playbook_dir }}/../generated/clients" + state: directory + mode: "0700" + + - name: Render client configs on control machine + delegate_to: localhost + become: false + ansible.builtin.template: + src: templates/client.conf.j2 + dest: "{{ playbook_dir }}/../generated/clients/{{ item.name }}.conf" + mode: "0600" + loop: "{{ wg_clients_enriched }}" + vars: + client: "{{ item }}" + no_log: true + + handlers: + - name: Reload firewalld + ansible.builtin.systemd: + name: firewalld + state: reloaded + + - name: Restart wg-quick + ansible.builtin.systemd: + name: wg-quick@wg0 + state: restarted diff --git a/admin-stack/caddy/Caddyfile b/admin-stack/caddy/Caddyfile new file mode 100644 index 0000000..89eaaed --- /dev/null +++ b/admin-stack/caddy/Caddyfile @@ -0,0 +1,20 @@ +{ + email {$ACME_EMAIL} +} + +admin.{$DOMAIN} { + tls { + dns googleclouddns { + gcp_project {$GCP_PROJECT_ID} + gcp_application_default /etc/caddy/credentials/sa.json + } + } + + handle /api/* { + reverse_proxy admin-api:3000 + } + + handle { + reverse_proxy admin-frontend:3000 + } +} diff --git a/admin-stack/caddy/Dockerfile b/admin-stack/caddy/Dockerfile new file mode 100644 index 0000000..f52c6ed --- /dev/null +++ b/admin-stack/caddy/Dockerfile @@ -0,0 +1,6 @@ +FROM caddy:2.8.4-builder AS builder +RUN xcaddy build \ + --with github.com/caddy-dns/googleclouddns + +FROM caddy:2.8.4 +COPY --from=builder /usr/bin/caddy /usr/bin/caddy diff --git a/admin-stack/compose.yaml b/admin-stack/compose.yaml new file mode 100644 index 0000000..2722af7 --- /dev/null +++ b/admin-stack/compose.yaml @@ -0,0 +1,59 @@ +networks: + admin-edge: + external: true + stoat-shared: + external: true + +volumes: + admin-sqlite: + caddy-data: + caddy-config: + +services: + caddy: + build: ./caddy + ports: + - "${WG_SERVER_IP}:80:80" + - "${WG_SERVER_IP}:443:443" + volumes: + - caddy-data:/data + - caddy-config:/config + - ./secrets/caddy-dns-sa.json:/etc/caddy/credentials/sa.json:ro + environment: + GCP_PROJECT_ID: ${GCP_PROJECT_ID} + ACME_EMAIL: ${ACME_EMAIL} + DOMAIN: ${DOMAIN} + networks: + - admin-edge + restart: unless-stopped + + admin-frontend: + image: ${ADMIN_FRONTEND_IMAGE} + environment: + - API_URL=http://admin-api:3000 + networks: + - admin-edge + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/"] + interval: 30s + timeout: 10s + retries: 3 + + admin-api: + image: ${ADMIN_API_IMAGE} + volumes: + - admin-sqlite:/data/db + environment: + - SQLITE_DB_PATH=/data/db/admin.db + - MONGO_URL=mongodb://admin_stack_ro:${ADMIN_STACK_DB_PASSWORD}@mongodb:27017/revolt + - SESSION_SECRET=${SESSION_SECRET} + networks: + - admin-edge + - stoat-shared + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/health"] + interval: 30s + timeout: 10s + retries: 3 diff --git a/admin-stack/scripts/bootstrap.sh b/admin-stack/scripts/bootstrap.sh new file mode 100755 index 0000000..82d9b50 --- /dev/null +++ b/admin-stack/scripts/bootstrap.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "=== 1. Verifying rootless podman ===" +PODMAN_USER=$(whoami) +if ! loginctl show-user ${PODMAN_USER} | grep -q "Linger=yes"; then + echo "Error: Linger is not enabled for user ${PODMAN_USER}" + exit 1 +fi + +echo "=== 2. Ansible: WireGuard ===" +ansible-playbook -i ansible/inventory.yml ansible/wireguard.yml + +echo "Client configs generated at: $(realpath ./ansible/../generated/clients)" +echo "Please securely copy these to your client devices." + +echo "=== 3. Verify wg0 ===" +if ! wg show wg0 >/dev/null 2>&1; then + echo "Error: wg0 interface is not up" + exit 1 +fi + +echo "=== 4. Ansible: Networks ===" +ansible-playbook -i ansible/inventory.yml ansible/networks.yml + +echo "=== 5. Materialize .env ===" +gcloud secrets versions access latest --secret=admin-env > .env +chmod 600 .env + +echo "=== 6. Materialize Caddy SA Key ===" +mkdir -p ./secrets +gcloud secrets versions access latest --secret=admin-caddy-dns-sa-key > ./secrets/caddy-dns-sa.json +chmod 600 ./secrets/caddy-dns-sa.json + +echo "=== 7. Podman Compose Build ===" +podman compose build + +echo "=== 8. Podman Compose Pull ===" +podman compose pull + +echo "=== 9. Podman Compose Up ===" +podman compose up -d + +echo "=== 10. Wait for services ===" +echo "Waiting up to 120s for services to become healthy..." +sleep 10 # Let them start + +echo "=== 11. Verify ===" +./scripts/verify.sh diff --git a/admin-stack/scripts/sqlite-backup.sh b/admin-stack/scripts/sqlite-backup.sh new file mode 100755 index 0000000..6ed5548 --- /dev/null +++ b/admin-stack/scripts/sqlite-backup.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +SRC_VOLUME="admin-sqlite" +DEST_DIR="/var/backups/admin-sqlite" +TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)" +DEST_FILE="${DEST_DIR}/admin-${TIMESTAMP}.sqlite" + +mkdir -p "${DEST_DIR}" + +# Use sqlite3 .backup for an atomic snapshot +podman run --rm \ + -v "${SRC_VOLUME}:/data:ro" \ + -v "${DEST_DIR}:/out" \ + docker.io/keinos/sqlite3:3.42.0 \ + sqlite3 /data/admin.db ".backup '/out/admin-${TIMESTAMP}.sqlite'" + +# Retain last 7 snapshots locally +ls -1t "${DEST_DIR}"/admin-*.sqlite | tail -n +8 | xargs -r rm diff --git a/admin-stack/scripts/verify.sh b/admin-stack/scripts/verify.sh new file mode 100755 index 0000000..5a4920f --- /dev/null +++ b/admin-stack/scripts/verify.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail + +WG_SERVER_IP="${WG_SERVER_IP:-10.42.0.1}" +HOST_PUBLIC_IP=$(curl -s ifconfig.me || echo "127.0.0.1") + +echo "Running verification checks..." + +# 1. WG interface up +if wg show wg0 >/dev/null 2>&1 && wg show wg0 peers | grep -q .; then + echo "[ok] WG interface wg0 is up and has peers" +else + echo "[fail] WG interface wg0 is down or has no peers" + exit 1 +fi + +# 2. WG IP bound +if ip -o addr show wg0 | grep -q "${WG_SERVER_IP}"; then + echo "[ok] WG interface wg0 bound to ${WG_SERVER_IP}" +else + echo "[fail] WG interface wg0 is not bound to ${WG_SERVER_IP}" + exit 1 +fi + +# 3. Caddy listening on WG IP, NOT public IP +if ss -tlnp | grep -E ':443\b' | grep -q "${WG_SERVER_IP}"; then + if ss -tlnp | grep -E ':443\b' | grep -q -E "0\.0\.0\.0|::|\*"; then + echo "[fail] Caddy is bound to public IP" + exit 1 + else + echo "[ok] Caddy is bound only to WG IP" + fi +else + echo "[fail] Caddy is not bound to ${WG_SERVER_IP}:443" + exit 1 +fi + +# 4. Admin endpoint NOT reachable from public +if curl --max-time 3 -k https://${HOST_PUBLIC_IP}/ >/dev/null 2>&1; then + echo "[fail] Admin endpoint is reachable from public IP" + exit 1 +else + echo "[ok] Admin endpoint is not reachable from public IP" +fi + +# 5. Networks present +if podman network inspect admin-edge >/dev/null 2>&1 && podman network inspect stoat-shared >/dev/null 2>&1; then + echo "[ok] Podman networks admin-edge and stoat-shared exist" +else + echo "[fail] Required podman networks are missing" + exit 1 +fi + +# 6. All expected services healthy +SERVICES=("admin-stack-caddy-1" "admin-stack-admin-frontend-1" "admin-stack-admin-api-1") +for service in "${SERVICES[@]}"; do + if podman ps --format "{{.Names}}" | grep -q "${service}"; then + echo "[ok] Service ${service} is running" + else + echo "[fail] Service ${service} is not running" + exit 1 + fi +done + +# 7. admin-api can reach MongoDB +API_CONTAINER=$(podman ps -q -f name=admin-stack-admin-api-1) +if podman exec "${API_CONTAINER}" curl -s http://localhost:3000/health >/dev/null 2>&1; then + echo "[ok] admin-api healthcheck passed" +else + echo "[fail] admin-api healthcheck failed" + exit 1 +fi + +# 8. No unexpected host ports bound +if podman ps --format '{{.Ports}}' | grep -v "${WG_SERVER_IP}" | grep -q ":"; then + echo "[fail] Unexpected ports bound" + podman ps --format '{{.Names}}: {{.Ports}}' + exit 1 +else + echo "[ok] No unexpected host ports bound" +fi + +echo "All checks passed!"