deploy stack

2026-04-27 19:36:39 -05:00
parent bb1def719f
commit ae3a137361
12 changed files with 524 additions and 0 deletions
@@ -0,0 +1,69 @@
+# Admin Stack
+
+This repository contains the deployment configuration for the Admin interface.
+
+## Prerequisites
+
+1. Same host as Stoat, rootless user with linger.
+2. Ansible + podman + WireGuard userspace tools installed.
+3. GCP credentials for Secret Manager.
+4. Public DNS record for `admin.${DOMAIN}` in Google Cloud DNS pointing to the WG server IP (or no record at all if using `tls internal`).
+5. Cloud DNS service account provisioned with `roles/dns.admin` and stored in Secret Manager.
+
+## First Deploy
+
+Run the bootstrap script:
+
+```bash
+./scripts/bootstrap.sh
+```
+
+## Adding a new WG client
+
+1. Edit `wg_clients` in `ansible/inventory.yml` (or your overriding group_vars).
+2. Re-run the wireguard playbook:
+   ```bash
+   ansible-playbook -i ansible/inventory.yml ansible/wireguard.yml
+   ```
+3. Distribute the new client config from `./generated/clients/<name>.conf`.
+
+## Removing a WG client
+
+1. Remove the client from `wg_clients`.
+2. Re-run the playbook.
+3. Verify in `wg show wg0` that the peer is gone.
+
+## Rotating the WG server key
+
+Rotating the server key is disruptive — every client config must be regenerated and redistributed.
+
+1. Remove the old key from Secret Manager or create a new version.
+2. Re-run the wireguard playbook.
+
+## Rotating the Caddy DNS service account key
+
+1. Generate a new key with `gcloud iam service-accounts keys create`.
+2. Push to Secret Manager as a new version.
+3. Re-run bootstrap step 6 to materialize the key.
+4. Restart Caddy (`podman compose restart caddy`).
+5. Disable the old key with `gcloud iam service-accounts keys disable` and finally delete after a grace period.
+
+## Redeploy Procedure
+
+1. `podman compose pull`
+2. `podman compose up -d`
+3. `./scripts/verify.sh`
+
+## Secret Rotation Procedure
+
+1. Update the secret in GCP Secret Manager (e.g. `admin-env`).
+2. Materialize the `.env` file again.
+3. `podman compose up -d` to recreate containers with the new environment.
+
+## SQLite Backup and Recovery Procedure
+
+Backups are handled by `scripts/sqlite-backup.sh`.
+
+1. To restore, stop the `admin-api` container.
+2. Replace the live `admin.db` in the `admin-sqlite` named volume with the snapshot file.
+3. Restart the `admin-api` container.
@@ -0,0 +1,17 @@
+all:
+  children:
+    admin_host:
+      hosts:
+        localhost:
+          ansible_connection: local
+      vars:
+        host_public_ip: "192.0.2.1"
+        wg_subnet: "10.42.0.0/24"
+        wg_server_ip: "10.42.0.1"
+        wg_listen_port: 51820
+        wg_clients:
+          - name: jason-laptop
+            ip: "10.42.0.10"
+          - name: jason-phone
+            ip: "10.42.0.11"
+        podman_user: "stoat"
@@ -0,0 +1,25 @@
+---
+- name: Podman Networks Setup
+  hosts: admin_host
+  become: true
+  become_user: "{{ podman_user }}"
+  tasks:
+    - name: Verify linger is enabled
+      ansible.builtin.command: loginctl show-user {{ podman_user }}
+      register: linger_check
+      changed_when: false
+      failed_when: "'Linger=yes' not in linger_check.stdout"
+
+    - name: Admin edge network
+      containers.podman.podman_network:
+        name: admin-edge
+        driver: bridge
+        subnet: 10.89.20.0/24
+        internal: false
+        state: present
+
+    - name: Assert stoat-shared exists (Stoat's Ansible owns it)
+      ansible.builtin.command: podman network inspect stoat-shared
+      register: shared_check
+      changed_when: false
+      failed_when: shared_check.rc != 0
@@ -0,0 +1,11 @@
+[Interface]
+PrivateKey = {{ client.private_key }}
+Address = {{ client.ip }}/24
+DNS = 1.1.1.1
+
+[Peer]
+PublicKey = {{ wg_server_public_key }}
+PresharedKey = {{ client.psk }}
+Endpoint = {{ host_public_ip }}:{{ wg_listen_port }}
+AllowedIPs = {{ wg_subnet }}
+PersistentKeepalive = 25
@@ -0,0 +1,13 @@
+[Interface]
+PrivateKey = {{ wg_server_private_key }}
+Address = {{ wg_server_ip }}/24
+ListenPort = {{ wg_listen_port }}
+SaveConfig = false
+
+{% for client in wg_clients_enriched %}
+[Peer]
+# {{ client.name }}
+PublicKey = {{ client.public_key }}
+PresharedKey = {{ client.psk }}
+AllowedIPs = {{ client.ip }}/32
+{% endfor %}
@@ -0,0 +1,153 @@
+---
+- name: WireGuard Host Setup
+  hosts: admin_host
+  become: true
+  tasks:
+    - name: Ensure wireguard and tools are installed
+      ansible.builtin.package:
+        name:
+          - wireguard-tools
+          - firewalld
+        state: present
+
+    - name: Ensure firewalld is running and enabled
+      ansible.builtin.systemd:
+        name: firewalld
+        state: started
+        enabled: true
+
+    - name: Open WG UDP port on public zone
+      ansible.posix.firewalld:
+        zone: public
+        port: "{{ wg_listen_port }}/udp"
+        permanent: true
+        state: enabled
+      notify: Reload firewalld
+
+    - name: Check if WG server private key exists in Secret Manager
+      delegate_to: localhost
+      become: false
+      ansible.builtin.command: >
+        gcloud secrets versions access latest --secret=admin-wg-server-key --project={{ lookup('env', 'GCP_PROJECT_ID') }}
+      register: wg_sm_check
+      failed_when: false
+      changed_when: false
+
+    - name: Generate WG server private key locally if not in Secret Manager
+      delegate_to: localhost
+      become: false
+      ansible.builtin.command: wg genkey
+      register: wg_local_gen
+      when: wg_sm_check.rc != 0
+      changed_when: true
+
+    - name: Create Secret in Secret Manager if missing
+      delegate_to: localhost
+      become: false
+      ansible.builtin.command: >
+        gcloud secrets create admin-wg-server-key --project={{ lookup('env', 'GCP_PROJECT_ID') }} --replication-policy="automatic"
+      when: wg_sm_check.rc != 0
+      failed_when: false
+      changed_when: false
+
+    - name: Push new WG server private key to Secret Manager
+      delegate_to: localhost
+      become: false
+      ansible.builtin.command: >
+        gcloud secrets versions add admin-wg-server-key
+        --data-file=-
+        --project={{ lookup('env', 'GCP_PROJECT_ID') }}
+      args:
+        stdin: "{{ wg_local_gen.stdout }}"
+      when: wg_sm_check.rc != 0
+
+    - name: Set server private key variable
+      ansible.builtin.set_fact:
+        wg_server_private_key: "{{ wg_sm_check.stdout if wg_sm_check.rc == 0 else wg_local_gen.stdout }}"
+      no_log: true
+
+    - name: Generate server public key
+      delegate_to: localhost
+      become: false
+      ansible.builtin.command: wg pubkey
+      args:
+        stdin: "{{ wg_server_private_key }}"
+      register: wg_server_pub_gen
+      changed_when: false
+
+    - name: Set server public key variable
+      ansible.builtin.set_fact:
+        wg_server_public_key: "{{ wg_server_pub_gen.stdout }}"
+
+    - name: Ensure /etc/wireguard directory exists
+      ansible.builtin.file:
+        path: /etc/wireguard
+        state: directory
+        mode: "0700"
+
+    - name: Generate client keys
+      delegate_to: localhost
+      become: false
+      ansible.builtin.shell: |
+        priv=$(wg genkey)
+        pub=$(echo "$priv" | wg pubkey)
+        psk=$(wg genpsk)
+        echo '{"private_key": "'$priv'", "public_key": "'$pub'", "psk": "'$psk'"}'
+      register: wg_client_keys_gen
+      with_items: "{{ wg_clients }}"
+      changed_when: true
+      no_log: true
+
+    - name: Enrich wg_clients with keys
+      ansible.builtin.set_fact:
+        wg_clients_enriched: >-
+          {{
+            wg_clients_enriched | default([]) +
+            [item.0 | combine(item.1.stdout | from_json)]
+          }}
+      loop: "{{ wg_clients | zip(wg_client_keys_gen.results) | list }}"
+      no_log: true
+
+    - name: Render server wg0.conf
+      ansible.builtin.template:
+        src: templates/wg0.conf.j2
+        dest: /etc/wireguard/wg0.conf
+        mode: "0600"
+      notify: Restart wg-quick
+
+    - name: Enable and start wg-quick@wg0
+      ansible.builtin.systemd:
+        name: wg-quick@wg0
+        state: started
+        enabled: true
+
+    - name: Ensure client config directory exists on control machine
+      delegate_to: localhost
+      become: false
+      ansible.builtin.file:
+        path: "{{ playbook_dir }}/../generated/clients"
+        state: directory
+        mode: "0700"
+
+    - name: Render client configs on control machine
+      delegate_to: localhost
+      become: false
+      ansible.builtin.template:
+        src: templates/client.conf.j2
+        dest: "{{ playbook_dir }}/../generated/clients/{{ item.name }}.conf"
+        mode: "0600"
+      loop: "{{ wg_clients_enriched }}"
+      vars:
+        client: "{{ item }}"
+      no_log: true
+
+  handlers:
+    - name: Reload firewalld
+      ansible.builtin.systemd:
+        name: firewalld
+        state: reloaded
+
+    - name: Restart wg-quick
+      ansible.builtin.systemd:
+        name: wg-quick@wg0
+        state: restarted
@@ -0,0 +1,20 @@
+{
+    email {$ACME_EMAIL}
+}
+
+admin.{$DOMAIN} {
+    tls {
+        dns googleclouddns {
+            gcp_project {$GCP_PROJECT_ID}
+            gcp_application_default /etc/caddy/credentials/sa.json
+        }
+    }
+
+    handle /api/* {
+        reverse_proxy admin-api:3000
+    }
+
+    handle {
+        reverse_proxy admin-frontend:3000
+    }
+}
@@ -0,0 +1,6 @@
+FROM caddy:2.8.4-builder AS builder
+RUN xcaddy build \
+    --with github.com/caddy-dns/googleclouddns
+
+FROM caddy:2.8.4
+COPY --from=builder /usr/bin/caddy /usr/bin/caddy
@@ -0,0 +1,59 @@
+networks:
+  admin-edge:
+    external: true
+  stoat-shared:
+    external: true
+
+volumes:
+  admin-sqlite:
+  caddy-data:
+  caddy-config:
+
+services:
+  caddy:
+    build: ./caddy
+    ports:
+      - "${WG_SERVER_IP}:80:80"
+      - "${WG_SERVER_IP}:443:443"
+    volumes:
+      - caddy-data:/data
+      - caddy-config:/config
+      - ./secrets/caddy-dns-sa.json:/etc/caddy/credentials/sa.json:ro
+    environment:
+      GCP_PROJECT_ID: ${GCP_PROJECT_ID}
+      ACME_EMAIL: ${ACME_EMAIL}
+      DOMAIN: ${DOMAIN}
+    networks:
+      - admin-edge
+    restart: unless-stopped
+
+  admin-frontend:
+    image: ${ADMIN_FRONTEND_IMAGE}
+    environment:
+      - API_URL=http://admin-api:3000
+    networks:
+      - admin-edge
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  admin-api:
+    image: ${ADMIN_API_IMAGE}
+    volumes:
+      - admin-sqlite:/data/db
+    environment:
+      - SQLITE_DB_PATH=/data/db/admin.db
+      - MONGO_URL=mongodb://admin_stack_ro:${ADMIN_STACK_DB_PASSWORD}@mongodb:27017/revolt
+      - SESSION_SECRET=${SESSION_SECRET}
+    networks:
+      - admin-edge
+      - stoat-shared
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "=== 1. Verifying rootless podman ==="
+PODMAN_USER=$(whoami)
+if ! loginctl show-user ${PODMAN_USER} | grep -q "Linger=yes"; then
+    echo "Error: Linger is not enabled for user ${PODMAN_USER}"
+    exit 1
+fi
+
+echo "=== 2. Ansible: WireGuard ==="
+ansible-playbook -i ansible/inventory.yml ansible/wireguard.yml
+
+echo "Client configs generated at: $(realpath ./ansible/../generated/clients)"
+echo "Please securely copy these to your client devices."
+
+echo "=== 3. Verify wg0 ==="
+if ! wg show wg0 >/dev/null 2>&1; then
+    echo "Error: wg0 interface is not up"
+    exit 1
+fi
+
+echo "=== 4. Ansible: Networks ==="
+ansible-playbook -i ansible/inventory.yml ansible/networks.yml
+
+echo "=== 5. Materialize .env ==="
+gcloud secrets versions access latest --secret=admin-env > .env
+chmod 600 .env
+
+echo "=== 6. Materialize Caddy SA Key ==="
+mkdir -p ./secrets
+gcloud secrets versions access latest --secret=admin-caddy-dns-sa-key > ./secrets/caddy-dns-sa.json
+chmod 600 ./secrets/caddy-dns-sa.json
+
+echo "=== 7. Podman Compose Build ==="
+podman compose build
+
+echo "=== 8. Podman Compose Pull ==="
+podman compose pull
+
+echo "=== 9. Podman Compose Up ==="
+podman compose up -d
+
+echo "=== 10. Wait for services ==="
+echo "Waiting up to 120s for services to become healthy..."
+sleep 10 # Let them start
+
+echo "=== 11. Verify ==="
+./scripts/verify.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SRC_VOLUME="admin-sqlite"
+DEST_DIR="/var/backups/admin-sqlite"
+TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)"
+DEST_FILE="${DEST_DIR}/admin-${TIMESTAMP}.sqlite"
+
+mkdir -p "${DEST_DIR}"
+
+# Use sqlite3 .backup for an atomic snapshot
+podman run --rm \
+    -v "${SRC_VOLUME}:/data:ro" \
+    -v "${DEST_DIR}:/out" \
+    docker.io/keinos/sqlite3:3.42.0 \
+    sqlite3 /data/admin.db ".backup '/out/admin-${TIMESTAMP}.sqlite'"
+
+# Retain last 7 snapshots locally
+ls -1t "${DEST_DIR}"/admin-*.sqlite | tail -n +8 | xargs -r rm
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+WG_SERVER_IP="${WG_SERVER_IP:-10.42.0.1}"
+HOST_PUBLIC_IP=$(curl -s ifconfig.me || echo "127.0.0.1")
+
+echo "Running verification checks..."
+
+# 1. WG interface up
+if wg show wg0 >/dev/null 2>&1 && wg show wg0 peers | grep -q .; then
+    echo "[ok] WG interface wg0 is up and has peers"
+else
+    echo "[fail] WG interface wg0 is down or has no peers"
+    exit 1
+fi
+
+# 2. WG IP bound
+if ip -o addr show wg0 | grep -q "${WG_SERVER_IP}"; then
+    echo "[ok] WG interface wg0 bound to ${WG_SERVER_IP}"
+else
+    echo "[fail] WG interface wg0 is not bound to ${WG_SERVER_IP}"
+    exit 1
+fi
+
+# 3. Caddy listening on WG IP, NOT public IP
+if ss -tlnp | grep -E ':443\b' | grep -q "${WG_SERVER_IP}"; then
+    if ss -tlnp | grep -E ':443\b' | grep -q -E "0\.0\.0\.0|::|\*"; then
+        echo "[fail] Caddy is bound to public IP"
+        exit 1
+    else
+        echo "[ok] Caddy is bound only to WG IP"
+    fi
+else
+    echo "[fail] Caddy is not bound to ${WG_SERVER_IP}:443"
+    exit 1
+fi
+
+# 4. Admin endpoint NOT reachable from public
+if curl --max-time 3 -k https://${HOST_PUBLIC_IP}/ >/dev/null 2>&1; then
+    echo "[fail] Admin endpoint is reachable from public IP"
+    exit 1
+else
+    echo "[ok] Admin endpoint is not reachable from public IP"
+fi
+
+# 5. Networks present
+if podman network inspect admin-edge >/dev/null 2>&1 && podman network inspect stoat-shared >/dev/null 2>&1; then
+    echo "[ok] Podman networks admin-edge and stoat-shared exist"
+else
+    echo "[fail] Required podman networks are missing"
+    exit 1
+fi
+
+# 6. All expected services healthy
+SERVICES=("admin-stack-caddy-1" "admin-stack-admin-frontend-1" "admin-stack-admin-api-1")
+for service in "${SERVICES[@]}"; do
+    if podman ps --format "{{.Names}}" | grep -q "${service}"; then
+        echo "[ok] Service ${service} is running"
+    else
+        echo "[fail] Service ${service} is not running"
+        exit 1
+    fi
+done
+
+# 7. admin-api can reach MongoDB
+API_CONTAINER=$(podman ps -q -f name=admin-stack-admin-api-1)
+if podman exec "${API_CONTAINER}" curl -s http://localhost:3000/health >/dev/null 2>&1; then
+    echo "[ok] admin-api healthcheck passed"
+else
+    echo "[fail] admin-api healthcheck failed"
+    exit 1
+fi
+
+# 8. No unexpected host ports bound
+if podman ps --format '{{.Ports}}' | grep -v "${WG_SERVER_IP}" | grep -q ":"; then
+    echo "[fail] Unexpected ports bound"
+    podman ps --format '{{.Names}}: {{.Ports}}'
+    exit 1
+else
+    echo "[ok] No unexpected host ports bound"
+fi
+
+echo "All checks passed!"