From 924203c8d8329d9d25bfe09c9beceb8c1665fdc2 Mon Sep 17 00:00:00 2001
From: Jason Ross <jason.ross841@outlook.com>
Date: Sun, 10 May 2026 18:28:51 -0500
Subject: [PATCH 1/2] scratch disk, wg-easy, Caddy with Coraza

---
 ansible/group_vars/all/main.yml               |  23 +++-
 ansible/group_vars/all/vault.yml.example      |   5 +
 ansible/inventory/hosts.yml.example           |   3 +
 ansible/roles/fail2ban/defaults/main.yml      |   6 +
 .../action.d/nftables-forward-allports.conf   |  34 ++++++
 .../fail2ban/files/filter.d/coraza-waf.conf   |  30 +++++
 .../files/selinux/watchtower-fail2ban.te      |  17 +++
 ansible/roles/fail2ban/tasks/main.yml         |  41 ++++++-
 .../roles/fail2ban/templates/jail.local.j2    |  24 ++++
 .../firewall/templates/watchtower.nft.j2      |   2 +
 ansible/roles/luks/tasks/main.yml             |  10 ++
 .../observability/files/disk-guard.service    |   8 ++
 .../observability/files/disk-guard.timer      |  10 ++
 ansible/roles/observability/handlers/main.yml |  20 ++++
 ansible/roles/observability/tasks/main.yml    | 103 ++++++++++++++++--
 .../observability/templates/Caddyfile.j2      |  77 +++++++++++++
 .../templates/caddy-data.volume.j2            |   7 ++
 .../templates/caddy.container.j2              |  56 ++++++++++
 .../observability/templates/disk-guard.sh.j2  |  63 +++++++++++
 .../templates/grafana.container.j2            |   5 +-
 .../observability/templates/grafana.ini.j2    |   2 +-
 .../observability/templates/loki.container.j2 |   5 +-
 .../templates/mimir.container.j2              |   7 +-
 .../templates/tempo.container.j2              |   7 +-
 .../templates/wg-easy.container.j2            |  37 +++++++
 .../observability/templates/wg-easy.env.j2    |  25 +++++
 ansible/roles/wireguard/handlers/main.yml     |   7 +-
 ansible/roles/wireguard/tasks/main.yml        |  30 +++--
 build/caddy/Dockerfile                        |  38 +++++++
 tofu/buckets.tf                               |   2 +-
 tofu/outputs.tf                               |   8 +-
 tofu/server.tf                                |  22 ++++
 tofu/variables.tf                             |   6 +
 33 files changed, 687 insertions(+), 53 deletions(-)
 create mode 100644 ansible/roles/fail2ban/files/action.d/nftables-forward-allports.conf
 create mode 100644 ansible/roles/fail2ban/files/filter.d/coraza-waf.conf
 create mode 100644 ansible/roles/fail2ban/files/selinux/watchtower-fail2ban.te
 create mode 100644 ansible/roles/observability/files/disk-guard.service
 create mode 100644 ansible/roles/observability/files/disk-guard.timer
 create mode 100644 ansible/roles/observability/templates/Caddyfile.j2
 create mode 100644 ansible/roles/observability/templates/caddy-data.volume.j2
 create mode 100644 ansible/roles/observability/templates/caddy.container.j2
 create mode 100644 ansible/roles/observability/templates/disk-guard.sh.j2
 create mode 100644 ansible/roles/observability/templates/wg-easy.container.j2
 create mode 100644 ansible/roles/observability/templates/wg-easy.env.j2
 create mode 100644 build/caddy/Dockerfile

diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml
index 54c7c93..e3b2a94 100644
--- a/ansible/group_vars/all/main.yml
+++ b/ansible/group_vars/all/main.yml
@@ -40,7 +40,16 @@ admin_allow_ipv6:
 observability_data_root: /var/lib/observability
 observability_secrets_dir: /etc/observability/secrets
 observability_config_dir: /etc/observability/config
-luks_device_source: /dev/sdb
+# observability_volume_id is injected per-host from the Ansible inventory.
+# Run `tofu output -json | jq -r '.ansible_inventory.value'` to regenerate hosts.yml.
+# Leave empty in dev/staging — the LUKS role will fall back to an 80 GB loop file.
+observability_volume_id: ""
+
+# When observability_volume_id is set, use the stable Hetzner by-id path.
+# When empty, the LUKS role falls back to a loop-backed image on the root disk (dev/staging only).
+luks_device_source: >-
+  {{ (observability_volume_id | length > 0)
+     | ternary('/dev/disk/by-id/scsi-0HC_Volume_' + observability_volume_id, '/dev/sdb') }}
 luks_mapper_name: observability_data
 
 # Container images (pin in production)
@@ -48,11 +57,21 @@ image_mimir:   "docker.io/grafana/mimir:2.14.3"
 image_loki:    "docker.io/grafana/loki:3.3.2"
 image_tempo:   "docker.io/grafana/tempo:2.7.1"
 image_grafana: "docker.io/grafana/grafana:11.4.0"
+image_caddy:   "localhost/watchtower-caddy:latest"
+
+# wg-easy — WireGuard peer management web UI.
+# image built from ghcr.io; pin the tag in production.
+image_wg_easy: "ghcr.io/wg-easy/wg-easy:14"
+wg_easy_web_port: 51821   # TCP — web UI, wg0 only (covered by iifname wg0 accept in nftables)
+wg_easy_wg_port: "{{ wireguard_listen_port }}"
+
+# Indirection for vault secret
+wg_easy_password_hash: "{{ vault_wg_easy_password_hash }}"
 
 # Retention (per signal)
 mimir_retention_days: 90
 loki_retention_days:  30
-tempo_retention_days: 14
+tempo_retention_days: 30
 
 # Tenants — override in production
 tenants:
diff --git a/ansible/group_vars/all/vault.yml.example b/ansible/group_vars/all/vault.yml.example
index c32e482..60a14e2 100644
--- a/ansible/group_vars/all/vault.yml.example
+++ b/ansible/group_vars/all/vault.yml.example
@@ -12,3 +12,8 @@ vault_wireguard_peers:
   - name: admin-laptop
     public_key: REPLACE_ME
     allowed_ips: 10.8.0.10/32
+
+# bcrypt hash of the wg-easy web-UI password.
+# Generate with: python3 -c "import bcrypt; print(bcrypt.hashpw(b'YOURPASSWORD', bcrypt.gensalt(12)).decode())"
+# or: htpasswd -bnBC 12 '' YOURPASSWORD | tr -d ':\n'
+vault_wg_easy_password_hash: "REPLACE_ME_BCRYPT_HASH"
diff --git a/ansible/inventory/hosts.yml.example b/ansible/inventory/hosts.yml.example
index 060f88a..73599a2 100644
--- a/ansible/inventory/hosts.yml.example
+++ b/ansible/inventory/hosts.yml.example
@@ -4,3 +4,6 @@ all:
       ansible_host: REPLACE_WITH_TOFU_OUTPUT_IPV4
       ansible_user: root
       ansible_python_interpreter: /usr/bin/python3
+      # Hetzner Volume ID for the observability data disk.
+      # Populate from: tofu output -json | jq -r '.ansible_inventory.value'
+      observability_volume_id: REPLACE_WITH_TOFU_OUTPUT_OBSERVABILITY_VOLUME_ID
diff --git a/ansible/roles/fail2ban/defaults/main.yml b/ansible/roles/fail2ban/defaults/main.yml
index ba4a333..cce5904 100644
--- a/ansible/roles/fail2ban/defaults/main.yml
+++ b/ansible/roles/fail2ban/defaults/main.yml
@@ -10,3 +10,9 @@ fail2ban_sender: fail2ban@watchtower
 fail2ban_recidive_bantime: 1w
 fail2ban_recidive_findtime: 1d
 fail2ban_recidive_maxretry: 5
+
+# Coraza WAF jail — higher maxretry than sshd because CRS in DetectionOnly
+# mode fires on benign requests too; tune these down when SecRuleEngine=On.
+fail2ban_coraza_bantime: 1h
+fail2ban_coraza_findtime: 10m
+fail2ban_coraza_maxretry: 15
diff --git a/ansible/roles/fail2ban/files/action.d/nftables-forward-allports.conf b/ansible/roles/fail2ban/files/action.d/nftables-forward-allports.conf
new file mode 100644
index 0000000..24b2cc8
--- /dev/null
+++ b/ansible/roles/fail2ban/files/action.d/nftables-forward-allports.conf
@@ -0,0 +1,34 @@
+# /etc/fail2ban/action.d/nftables-forward-allports.conf
+#
+# Bans IPs in the nftables FORWARD hook instead of INPUT.
+#
+# Why FORWARD instead of INPUT:
+#   Rootful Podman publishes ports via nftables DNAT rules in PREROUTING.
+#   After DNAT, the routing decision sends the packet through FORWARD (not
+#   INPUT), so INPUT-based bans never see Podman-destined traffic.
+#   FORWARD priority -1 evaluates before the watchtower FORWARD chain (0),
+#   so banned WireGuard peer IPs (10.8.0.x) are dropped before the
+#   `iifname wg0 accept` rule in the watchtower table.
+#
+# Each jail using this action gets its own per-jail chain and set so that
+# multiple jails can safely coexist without shared-chain lifecycle conflicts.
+
+[Definition]
+
+actionstart = nft add table inet f2b-table
+              nft add chain inet f2b-table f2b-<name>-fwd { type filter hook forward priority -1 \; }
+              nft add set   inet f2b-table f2b-<name>-set { type ipv4_addr \; flags timeout \; }
+              nft add rule  inet f2b-table f2b-<name>-fwd ip saddr @f2b-<name>-set drop
+
+actionstop  = nft flush  chain inet f2b-table f2b-<name>-fwd
+              nft delete chain inet f2b-table f2b-<name>-fwd
+              nft delete set   inet f2b-table f2b-<name>-set
+
+actioncheck = nft list chain inet f2b-table f2b-<name>-fwd
+
+actionban   = nft add element inet f2b-table f2b-<name>-set { <ip> timeout <bantime>s }
+
+actionunban = nft delete element inet f2b-table f2b-<name>-set { <ip> }
+
+[Init]
+name = default
diff --git a/ansible/roles/fail2ban/files/filter.d/coraza-waf.conf b/ansible/roles/fail2ban/files/filter.d/coraza-waf.conf
new file mode 100644
index 0000000..159d68a
--- /dev/null
+++ b/ansible/roles/fail2ban/files/filter.d/coraza-waf.conf
@@ -0,0 +1,30 @@
+# /etc/fail2ban/filter.d/coraza-waf.conf
+#
+# Matches the Section A transaction header in a Coraza serial audit log.
+#
+# Coraza serial audit log structure (SecAuditLogType Serial):
+#   --<boundary>--A--                              ← boundary marker (ignored)
+#   [DD/Mon/YYYY:HH:MM:SS[.us] ±ZZZZ] id ip port dst_ip dst_port  ← matched
+#   --<boundary>--B--                              ← request headers (ignored)
+#   ...
+#   --<boundary>--H--                              ← rule messages (ignored)
+#   --<boundary>--Z--                              ← end marker (ignored)
+#
+# With SecAuditEngine RelevantOnly, ONLY transactions that matched a WAF rule
+# are written to the log, so every match of the Section A header == a WAF hit.
+#
+# The timestamp format is [DD/Mon/YYYY:HH:MM:SS ±HHMM] (Apache Combined style)
+# with an optional sub-second fraction. fail2ban extracts the date via
+# datepattern; the failregex captures <HOST> (client IP) from the same line.
+
+[INCLUDES]
+before = common.conf
+
+[Definition]
+
+failregex = ^\[[\d]{2}/\w+/[\d]{4}:[\d]{2}:[\d]{2}:[\d]{2}(?:\.[\d]+)? [+-][\d]{4}\] \S+ <HOST> [\d]+ \S+ [\d]+\s*$
+
+ignoreregex =
+
+# Timestamp is at the start of the line, wrapped in square brackets.
+datepattern = {^LN-BEG}\[%%d/%%b/%%Y:%%H:%%M:%%S %%z\]
diff --git a/ansible/roles/fail2ban/files/selinux/watchtower-fail2ban.te b/ansible/roles/fail2ban/files/selinux/watchtower-fail2ban.te
new file mode 100644
index 0000000..1bdf2f7
--- /dev/null
+++ b/ansible/roles/fail2ban/files/selinux/watchtower-fail2ban.te
@@ -0,0 +1,17 @@
+module watchtower-fail2ban 1.0;
+
+# Allow fail2ban to read Coraza WAF audit logs written by the containerised
+# Caddy process.  The audit log directory is bind-mounted with Podman's shared
+# `:z` relabel option, which labels files as container_file_t:s0 (no private
+# MCS category).  fail2ban runs with s0 clearance, so the MCS check passes;
+# only the type allow rule is missing from the base policy.
+
+require {
+    type fail2ban_t;
+    type container_file_t;
+    class file  { getattr open read };
+    class dir   { getattr open read search };
+}
+
+allow fail2ban_t container_file_t:dir  { getattr open read search };
+allow fail2ban_t container_file_t:file { getattr open read };
diff --git a/ansible/roles/fail2ban/tasks/main.yml b/ansible/roles/fail2ban/tasks/main.yml
index d0f943f..26c1d1f 100644
--- a/ansible/roles/fail2ban/tasks/main.yml
+++ b/ansible/roles/fail2ban/tasks/main.yml
@@ -4,11 +4,12 @@
     name: epel-release
     state: present
 
-- name: Install fail2ban
+- name: Install fail2ban and checkpolicy
   ansible.builtin.dnf:
     name:
       - fail2ban
       - fail2ban-firewalld     # provides the firewallcmd actions; harmless even if firewalld is masked
+      - checkpolicy            # provides checkmodule + semodule_package for custom SELinux modules
     state: present
     update_cache: true
 
@@ -21,6 +22,44 @@
     state: absent
   notify: Restart fail2ban
 
+# Deploy filter and action files BEFORE the jail is enabled so fail2ban can
+# find them on its first start.
+- name: Deploy fail2ban filter files
+  ansible.builtin.copy:
+    src: filter.d/
+    dest: /etc/fail2ban/filter.d/
+    mode: "0644"
+  notify: Restart fail2ban
+
+- name: Deploy fail2ban action files
+  ansible.builtin.copy:
+    src: action.d/
+    dest: /etc/fail2ban/action.d/
+    mode: "0644"
+  notify: Restart fail2ban
+
+# Install a custom SELinux policy module that allows fail2ban_t to read files
+# labelled container_file_t:s0.  The Coraza audit log directory is bind-mounted
+# into Caddy with ':z' (shared label), which sets container_file_t:s0 — no
+# private MCS categories — so fail2ban's s0 clearance can pass the MCS check,
+# but the type allow rule is still required.
+- name: Copy watchtower-fail2ban SELinux policy source
+  ansible.builtin.copy:
+    src: selinux/watchtower-fail2ban.te
+    dest: /tmp/watchtower-fail2ban.te
+    mode: "0644"
+  register: selinux_policy_src
+
+- name: Compile and install watchtower-fail2ban SELinux policy module
+  ansible.builtin.shell: |
+    set -e
+    cd /tmp
+    checkmodule -M -m -o watchtower-fail2ban.mod watchtower-fail2ban.te
+    semodule_package -o watchtower-fail2ban.pp -m watchtower-fail2ban.mod
+    semodule -i watchtower-fail2ban.pp
+  when: selinux_policy_src.changed
+  notify: Restart fail2ban
+
 - name: Render jail.local
   ansible.builtin.template:
     src: jail.local.j2
diff --git a/ansible/roles/fail2ban/templates/jail.local.j2 b/ansible/roles/fail2ban/templates/jail.local.j2
index f9a5180..17dbc3a 100644
--- a/ansible/roles/fail2ban/templates/jail.local.j2
+++ b/ansible/roles/fail2ban/templates/jail.local.j2
@@ -41,3 +41,27 @@ banaction = nftables-allports
 bantime   = {{ fail2ban_recidive_bantime }}
 findtime  = {{ fail2ban_recidive_findtime }}
 maxretry  = {{ fail2ban_recidive_maxretry }}
+
+# Coraza WAF audit log — bans WireGuard peer IPs (10.8.0.x) that trigger
+# WAF rules through Caddy.  Uses nftables-forward-allports because Podman
+# DNATs published ports, so traffic traverses FORWARD (not INPUT) and
+# INPUT-based ban actions never intercept it.
+#
+# ignoreip intentionally excludes wireguard_subnet_v4 so misbehaving VPN
+# clients can be banned.  The server's own address (127.0.0.1/::1) is still
+# whitelisted to prevent self-banning.
+#
+# Tune fail2ban_coraza_maxretry down (and SecRuleEngine to "On") once you
+# have reviewed the Coraza audit log and confirmed false-positive rate.
+[coraza-waf]
+enabled   = true
+port      = 0:65535
+protocol  = tcp
+logpath   = {{ observability_data_root }}/caddy/logs/coraza-audit-grafana.log
+backend   = auto
+filter    = coraza-waf
+banaction = nftables-forward-allports
+ignoreip  = 127.0.0.1/8 ::1
+maxretry  = {{ fail2ban_coraza_maxretry }}
+findtime  = {{ fail2ban_coraza_findtime }}
+bantime   = {{ fail2ban_coraza_bantime }}
diff --git a/ansible/roles/firewall/templates/watchtower.nft.j2 b/ansible/roles/firewall/templates/watchtower.nft.j2
index 347be4f..f9c6720 100644
--- a/ansible/roles/firewall/templates/watchtower.nft.j2
+++ b/ansible/roles/firewall/templates/watchtower.nft.j2
@@ -55,6 +55,8 @@ table inet watchtower {
 
         # Anything arriving on WireGuard is treated as trusted (only authenticated
         # peers can reach this interface). Service ports are bound to 10.8.0.1.
+        # This also implicitly allows the wg-easy web UI on TCP 51821, which
+        # is bound to 10.8.0.1 and therefore only reachable from VPN peers.
         iifname "{{ wireguard_interface }}" accept
 
         # Podman bridge networks: allow the kernel to deliver packets the
diff --git a/ansible/roles/luks/tasks/main.yml b/ansible/roles/luks/tasks/main.yml
index a72016f..16d4cfe 100644
--- a/ansible/roles/luks/tasks/main.yml
+++ b/ansible/roles/luks/tasks/main.yml
@@ -15,6 +15,16 @@
   ansible.builtin.set_fact:
     luks_use_loop_file: "{{ not luks_device_stat.stat.exists }}"
 
+- name: Fail if Hetzner Volume expected but device not found
+  ansible.builtin.fail:
+    msg: >
+      Device {{ luks_device_source }} was not found, but observability_volume_id is set
+      to '{{ observability_volume_id }}'. Ensure the Hetzner Volume is attached to the server
+      and udev has settled (try: udevadm settle --timeout=30).
+  when:
+    - observability_volume_id | length > 0
+    - not luks_device_stat.stat.exists
+
 - name: Create loop-backed LUKS image (if no real device)
   when: luks_use_loop_file
   block:
diff --git a/ansible/roles/observability/files/disk-guard.service b/ansible/roles/observability/files/disk-guard.service
new file mode 100644
index 0000000..e97a7aa
--- /dev/null
+++ b/ansible/roles/observability/files/disk-guard.service
@@ -0,0 +1,8 @@
+[Unit]
+Description=Observability disk-guard: flush services when data volume is near-full
+After=network-online.target wg-easy.service loki.service mimir.service tempo.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/disk-guard.sh
diff --git a/ansible/roles/observability/files/disk-guard.timer b/ansible/roles/observability/files/disk-guard.timer
new file mode 100644
index 0000000..3f3649b
--- /dev/null
+++ b/ansible/roles/observability/files/disk-guard.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Run observability disk-guard every 5 minutes
+
+[Timer]
+OnBootSec=5min
+OnUnitActiveSec=5min
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/ansible/roles/observability/handlers/main.yml b/ansible/roles/observability/handlers/main.yml
index 66fe0aa..ecb7c0e 100644
--- a/ansible/roles/observability/handlers/main.yml
+++ b/ansible/roles/observability/handlers/main.yml
@@ -27,13 +27,33 @@
     state: restarted
     daemon_reload: true
 
+- name: Restart caddy
+  ansible.builtin.systemd:
+    name: caddy.service
+    state: restarted
+    daemon_reload: true
+
+- name: Restart wg-easy
+  ansible.builtin.systemd:
+    name: wg-easy.service
+    state: restarted
+    daemon_reload: true
+
 - name: Restart all services
   ansible.builtin.systemd:
     name: "{{ item }}"
     state: restarted
     daemon_reload: true
   loop:
+    - wg-easy.service
+    - caddy.service
     - mimir.service
     - loki.service
     - tempo.service
     - grafana.service
+
+- name: Restart disk-guard timer
+  ansible.builtin.systemd:
+    name: disk-guard.timer
+    state: restarted
+    daemon_reload: true
diff --git a/ansible/roles/observability/tasks/main.yml b/ansible/roles/observability/tasks/main.yml
index b5ba214..dc6a3b9 100644
--- a/ansible/roles/observability/tasks/main.yml
+++ b/ansible/roles/observability/tasks/main.yml
@@ -16,8 +16,10 @@
     - { path: "{{ observability_config_dir }}/grafana/provisioning", mode: "0755" }
     - { path: "{{ observability_config_dir }}/grafana/provisioning/datasources", mode: "0755" }
     - { path: "{{ observability_config_dir }}/grafana/provisioning/dashboards", mode: "0755" }
+    - { path: "{{ observability_config_dir }}/caddy", mode: "0755" }
     - { path: /var/log/observability, mode: "0750" }
     - { path: /etc/containers/systemd, mode: "0755" }
+    - { path: /etc/observability/caddy-build, mode: "0755" }
 
 # Container processes run as non-root UIDs. Bind-mounted data dirs must be
 # owned by those UIDs or the services cannot create their WAL/index/DB files.
@@ -29,10 +31,12 @@
     owner: "{{ item.uid }}"
     group: "{{ item.gid }}"
   loop:
-    - { path: "{{ observability_data_root }}/mimir",   uid: 10001, gid: 10001 }
-    - { path: "{{ observability_data_root }}/loki",    uid: 10001, gid: 10001 }
-    - { path: "{{ observability_data_root }}/tempo",   uid: 10001, gid: 10001 }
-    - { path: "{{ observability_data_root }}/grafana", uid: 472,   gid: 472 }
+    - { path: "{{ observability_data_root }}/mimir",      uid: 10001, gid: 10001 }
+    - { path: "{{ observability_data_root }}/loki",       uid: 10001, gid: 10001 }
+    - { path: "{{ observability_data_root }}/tempo",      uid: 10001, gid: 10001 }
+    - { path: "{{ observability_data_root }}/grafana",    uid: 472,   gid: 472 }
+    - { path: "{{ observability_data_root }}/caddy",      uid: 1000,  gid: 1000 }
+    - { path: "{{ observability_data_root }}/caddy/logs", uid: 1000,  gid: 1000 }
 
 - name: Write S3 credentials env file
   ansible.builtin.template:
@@ -81,6 +85,22 @@
     mode: "0644"
   notify: Restart grafana
 
+- name: Render Caddyfile
+  ansible.builtin.template:
+    src: Caddyfile.j2
+    dest: "{{ observability_config_dir }}/caddy/Caddyfile"
+    mode: "0644"
+  notify: Restart caddy
+
+- name: Render wg-easy env file
+  ansible.builtin.template:
+    src: wg-easy.env.j2
+    dest: "{{ observability_secrets_dir }}/wg-easy.env"
+    mode: "0600"
+    owner: root
+    group: root
+  notify: Restart wg-easy
+
 - name: Apply SELinux context to config tree
   ansible.builtin.command: restorecon -R /etc/observability
   changed_when: false
@@ -102,8 +122,28 @@
     - loki-data
     - tempo-data
     - grafana-data
+    - caddy-data
   notify: Reload systemd
 
+# Copy the Caddy build context to the server so podman build runs locally.
+- name: Copy Caddy + Coraza Dockerfile to server
+  ansible.builtin.copy:
+    src: "{{ role_path }}/../../../build/caddy/Dockerfile"
+    dest: /etc/observability/caddy-build/Dockerfile
+    mode: "0644"
+  register: caddy_dockerfile_result
+
+- name: Check if Caddy + Coraza image already exists
+  ansible.builtin.command: podman image inspect localhost/watchtower-caddy:latest
+  register: caddy_image_inspect
+  changed_when: false
+  failed_when: false
+
+- name: Build Caddy + Coraza image
+  ansible.builtin.command: podman build -t localhost/watchtower-caddy:latest /etc/observability/caddy-build/
+  when: caddy_image_inspect.rc != 0 or caddy_dockerfile_result.changed
+  notify: Restart caddy
+
 - name: Deploy container quadlets
   ansible.builtin.template:
     src: "{{ item }}.container.j2"
@@ -114,6 +154,8 @@
     - loki
     - tempo
     - grafana
+    - wg-easy
+    - caddy
   notify:
     - Reload systemd
     - Restart all services
@@ -121,6 +163,48 @@
 - name: Force handlers to flush before health checks
   ansible.builtin.meta: flush_handlers
 
+- name: Deploy disk-guard script
+  ansible.builtin.template:
+    src: disk-guard.sh.j2
+    dest: /usr/local/bin/disk-guard.sh
+    mode: "0750"
+    owner: root
+    group: root
+
+- name: Deploy disk-guard systemd units
+  ansible.builtin.copy:
+    src: "{{ item }}"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: "0644"
+    owner: root
+    group: root
+  loop:
+    - disk-guard.service
+    - disk-guard.timer
+  notify:
+    - Reload systemd
+    - Restart disk-guard timer
+
+- name: Enable and start disk-guard timer
+  ansible.builtin.systemd:
+    name: disk-guard.timer
+    enabled: true
+    state: started
+    daemon_reload: true
+
+# Health checks run on the managed host (via SSH) so they work regardless of
+# whether the Ansible controller is connected to the WireGuard VPN.  The server
+# can reach its own WireGuard IP (10.8.0.1) via the local wg0 interface.
+- name: Wait for Caddy to be ready
+  ansible.builtin.uri:
+    url: "http://{{ wireguard_server_ip }}/api/health"
+    status_code: 200
+  register: caddy_ready
+  retries: 30
+  delay: 5
+  until: caddy_ready.status == 200
+  delegate_to: "{{ inventory_hostname }}"
+
 - name: Wait for Mimir /ready
   ansible.builtin.uri:
     url: "http://{{ wireguard_server_ip }}:8080/ready"
@@ -129,6 +213,7 @@
   retries: 30
   delay: 5
   until: mimir_ready.status == 200
+  delegate_to: "{{ inventory_hostname }}"
 
 - name: Wait for Loki /ready
   ansible.builtin.uri:
@@ -138,6 +223,7 @@
   retries: 30
   delay: 5
   until: loki_ready.status == 200
+  delegate_to: "{{ inventory_hostname }}"
 
 - name: Wait for Tempo /ready
   ansible.builtin.uri:
@@ -147,12 +233,5 @@
   retries: 30
   delay: 5
   until: tempo_ready.status == 200
+  delegate_to: "{{ inventory_hostname }}"
 
-- name: Wait for Grafana /api/health
-  ansible.builtin.uri:
-    url: "http://{{ wireguard_server_ip }}:3000/api/health"
-    status_code: 200
-  register: grafana_ready
-  retries: 30
-  delay: 5
-  until: grafana_ready.status == 200
diff --git a/ansible/roles/observability/templates/Caddyfile.j2 b/ansible/roles/observability/templates/Caddyfile.j2
new file mode 100644
index 0000000..5fd73bf
--- /dev/null
+++ b/ansible/roles/observability/templates/Caddyfile.j2
@@ -0,0 +1,77 @@
+{
+    # Disable the built-in ACME client; TLS is terminated inside the VPN.
+    auto_https off
+
+    # coraza_waf must be ordered before the reverse_proxy handler so WAF
+    # inspection runs before the request is forwarded upstream.
+    order coraza_waf first
+
+    # Send Caddy's own structured log to stdout; Podman + journald capture it.
+    log {
+        output stdout
+        format json
+    }
+
+    # Accept cleartext HTTP/2 (h2c) from OTLP gRPC clients on port 4317.
+    # The transport block in the :4317 site only controls Caddy → Tempo;
+    # this servers block makes Caddy accept h2c from the client side too.
+    servers :4317 {
+        protocols h1 h2c
+    }
+}
+
+# ── Grafana UI — browser-facing ───────────────────────────────────────────────
+# Full OWASP CRS in detection-only mode.  Switch SecRuleEngine to "On" after
+# reviewing Coraza audit logs and tuning false-positives for your workload.
+:80 {
+    coraza_waf {
+        load_owasp_crs
+
+        directives `
+            Include @coraza.conf-recommended
+            Include @crs-setup.conf.example
+            Include @owasp_crs/*.conf
+
+            SecRuleEngine DetectionOnly
+            SecRequestBodyAccess On
+            SecResponseBodyAccess Off
+            SecAuditEngine RelevantOnly
+            SecAuditLog /var/log/caddy/coraza-audit-grafana.log
+            SecAuditLogType Serial
+        `
+    }
+
+    reverse_proxy grafana:3000
+}
+
+# ── Loki push + query API ─────────────────────────────────────────────────────
+# Machine-to-machine over WireGuard; no CRS (structured/binary payloads).
+:3100 {
+    reverse_proxy loki:3100
+}
+
+# ── Mimir push + query API ────────────────────────────────────────────────────
+:8080 {
+    reverse_proxy mimir:8080
+}
+
+# ── Tempo query API ───────────────────────────────────────────────────────────
+:3200 {
+    reverse_proxy tempo:3200
+}
+
+# ── Tempo OTLP HTTP ingest ────────────────────────────────────────────────────
+:4318 {
+    reverse_proxy tempo:4318
+}
+
+# ── Tempo OTLP gRPC ingest ────────────────────────────────────────────────────
+# Tempo's gRPC receiver speaks h2c (plain HTTP/2 without TLS upgrade).
+# The transport block disables TLS and forces HTTP/2 on the upstream connection.
+:4317 {
+    reverse_proxy tempo:4317 {
+        transport http {
+            versions h2c
+        }
+    }
+}
diff --git a/ansible/roles/observability/templates/caddy-data.volume.j2 b/ansible/roles/observability/templates/caddy-data.volume.j2
new file mode 100644
index 0000000..327b899
--- /dev/null
+++ b/ansible/roles/observability/templates/caddy-data.volume.j2
@@ -0,0 +1,7 @@
+[Volume]
+# Caddy uses this volume for its internal state (OCSP staples, Coraza audit
+# logs, etc.).  The host path sits on the 500 GB observability data disk.
+VolumeName=caddy-data
+Device={{ observability_data_root }}/caddy
+Type=bind
+Options=bind
diff --git a/ansible/roles/observability/templates/caddy.container.j2 b/ansible/roles/observability/templates/caddy.container.j2
new file mode 100644
index 0000000..2021f90
--- /dev/null
+++ b/ansible/roles/observability/templates/caddy.container.j2
@@ -0,0 +1,56 @@
+[Unit]
+Description=Caddy reverse proxy with Coraza WAF
+# Caddy publishes ports on the WireGuard IP; wg0 must be up before the
+# PublishPort binds succeed.  wg-easy.service owns the wg0 interface.
+Wants=network-online.target observability-network.service wg-easy.service
+After=network-online.target observability-network.service wg-easy.service
+
+[Container]
+Image={{ image_caddy }}
+ContainerName=caddy
+
+# Same network as all backends; Caddy reaches them by container name.
+Network=observability.network
+
+Volume=caddy-data.volume:/var/lib/caddy:Z
+Volume={{ observability_config_dir }}/caddy/Caddyfile:/etc/caddy/Caddyfile:ro,Z
+# Separate bind mount with shared (':z') label so fail2ban on the host can
+# read the Coraza audit log.  Private (':Z') would assign a container-private
+# MCS category that fail2ban cannot access even as root.
+Volume={{ observability_data_root }}/caddy/logs:/var/log/caddy:z
+
+# Expose each service port on the WireGuard IP only.  Caddy listens on
+# 0.0.0.0 inside the container; Podman's PublishPort restricts host exposure.
+PublishPort={{ wireguard_server_ip }}:80:80
+PublishPort={{ wireguard_server_ip }}:3100:3100
+PublishPort={{ wireguard_server_ip }}:8080:8080
+PublishPort={{ wireguard_server_ip }}:3200:3200
+PublishPort={{ wireguard_server_ip }}:4317:4317
+PublishPort={{ wireguard_server_ip }}:4318:4318
+
+# Port 80 requires NET_BIND_SERVICE when running as non-root.
+AddCapability=NET_BIND_SERVICE
+
+User=1000
+Group=1000
+
+PodmanArgs=--memory=512m --memory-swap=512m
+
+[Service]
+Restart=on-failure
+RestartSec=10
+TimeoutStartSec=120
+
+# Wait for wg0 to have its address before Caddy tries to bind PublishPorts on
+# the WireGuard IP.  Times out after 2 minutes and lets systemd mark the unit
+# failed so the operator sees a clear error rather than a silent bind error.
+ExecStartPre=/bin/bash -c \
+  'for i in $(seq 60); do \
+     ip -4 addr show {{ wireguard_interface }} 2>/dev/null \
+       | grep -q "{{ wireguard_server_ip }}" && exit 0; \
+     sleep 2; \
+   done; \
+   echo "Timed out waiting for {{ wireguard_interface }}"; exit 1'
+
+[Install]
+WantedBy=multi-user.target default.target
diff --git a/ansible/roles/observability/templates/disk-guard.sh.j2 b/ansible/roles/observability/templates/disk-guard.sh.j2
new file mode 100644
index 0000000..787d9f5
--- /dev/null
+++ b/ansible/roles/observability/templates/disk-guard.sh.j2
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# Managed by Ansible — see roles/observability/templates/disk-guard.sh.j2
+#
+# Monitors disk usage on the observability data volume and calls each service's
+# ingester-flush HTTP API when the disk is near capacity, accelerating the
+# drain of in-memory chunks to Hetzner Object Storage.
+#
+# NOTE: /flush and /ingester/flush are admin-only endpoints registered outside
+# the standard auth middleware in Loki, Mimir, and Tempo. They flush all tenants
+# and do not require X-Scope-OrgID even when multitenancy is enabled.
+#
+# Thresholds:
+#   >= 80 %  WARNING — flush all ingesters
+#   >= 90 %  ERR     — flush + journal critical (operator action required)
+
+set -euo pipefail
+
+DATA_ROOT="{{ observability_data_root }}"
+MAPPER="{{ luks_mapper_name }}"
+BASE_URL="http://{{ wireguard_server_ip }}"
+
+# Verify the LUKS volume is actually mounted; if not, we'd be checking the
+# wrong (root) filesystem and triggering spurious flushes.
+if ! findmnt --source "/dev/mapper/${MAPPER}" --target "${DATA_ROOT}" > /dev/null 2>&1; then
+  systemd-cat -t disk-guard -p err \
+    printf 'LUKS volume /dev/mapper/%s is not mounted at %s — skipping disk check' \
+    "${MAPPER}" "${DATA_ROOT}"
+  exit 1
+fi
+
+USAGE=$(df --output=pcent "${DATA_ROOT}" | tail -1 | tr -d ' %')
+
+if (( USAGE < 80 )); then
+  exit 0
+fi
+
+LEVEL="WARNING"
+(( USAGE >= 90 )) && LEVEL="ERR"
+
+systemd-cat -t disk-guard -p "${LEVEL,,}" \
+  printf '%s: observability data volume at %d%% — flushing ingesters to object storage' \
+  "${LEVEL}" "${USAGE}"
+
+flush() {
+  local svc=$1 url=$2
+  local http_code
+  http_code=$(curl -s -o /dev/null -w "%{http_code}" -X POST "${url}" --max-time 10 || true)
+  if [[ "${http_code}" == "204" || "${http_code}" == "200" ]]; then
+    systemd-cat -t disk-guard -p info printf 'Flushed %s ingester (HTTP %s)' "${svc}" "${http_code}"
+  else
+    systemd-cat -t disk-guard -p warning printf 'Flush %s returned HTTP %s (check service logs)' "${svc}" "${http_code}"
+  fi
+}
+
+flush loki  "${BASE_URL}:3100/flush"
+flush mimir "${BASE_URL}:8080/ingester/flush"
+flush tempo "${BASE_URL}:3200/flush"
+
+if (( USAGE >= 90 )); then
+  systemd-cat -t disk-guard -p err \
+    printf 'ERR: observability volume at %d%% — manual intervention required (compact, scale, or extend volume)' \
+    "${USAGE}"
+fi
\ No newline at end of file
diff --git a/ansible/roles/observability/templates/grafana.container.j2 b/ansible/roles/observability/templates/grafana.container.j2
index 85728f2..b6f4f61 100644
--- a/ansible/roles/observability/templates/grafana.container.j2
+++ b/ansible/roles/observability/templates/grafana.container.j2
@@ -1,8 +1,7 @@
 [Unit]
 Description=Grafana
 Wants=network-online.target observability-network.service mimir.service loki.service tempo.service
-After=network-online.target observability-network.service mimir.service loki.service tempo.service wg-quick@{{ wireguard_interface }}.service
-Requires=wg-quick@{{ wireguard_interface }}.service
+After=network-online.target observability-network.service mimir.service loki.service tempo.service
 
 [Container]
 Image={{ image_grafana }}
@@ -18,8 +17,6 @@ Environment=GF_PATHS_PLUGINS=/var/lib/grafana/plugins
 Environment=GF_PATHS_PROVISIONING=/etc/grafana/provisioning
 Environment=GF_SECURITY_ADMIN_PASSWORD={{ grafana_admin_password }}
 
-PublishPort={{ wireguard_server_ip }}:3000:3000
-
 User=472
 Group=472
 
diff --git a/ansible/roles/observability/templates/grafana.ini.j2 b/ansible/roles/observability/templates/grafana.ini.j2
index 41d0910..842daf8 100644
--- a/ansible/roles/observability/templates/grafana.ini.j2
+++ b/ansible/roles/observability/templates/grafana.ini.j2
@@ -11,7 +11,7 @@ protocol = http
 http_addr = 0.0.0.0
 http_port = 3000
 domain = {{ wireguard_server_ip }}
-root_url = http://{{ wireguard_server_ip }}:3000/
+root_url = http://{{ wireguard_server_ip }}/
 enforce_domain = false
 
 [security]
diff --git a/ansible/roles/observability/templates/loki.container.j2 b/ansible/roles/observability/templates/loki.container.j2
index 2e13e1d..f3bdc03 100644
--- a/ansible/roles/observability/templates/loki.container.j2
+++ b/ansible/roles/observability/templates/loki.container.j2
@@ -1,8 +1,7 @@
 [Unit]
 Description=Grafana Loki
 Wants=network-online.target observability-network.service
-After=network-online.target observability-network.service wg-quick@{{ wireguard_interface }}.service
-Requires=wg-quick@{{ wireguard_interface }}.service
+After=network-online.target observability-network.service
 
 [Container]
 Image={{ image_loki }}
@@ -12,8 +11,6 @@ Volume=loki-data.volume:/var/lib/loki:Z
 Volume={{ observability_config_dir }}/loki/loki.yaml:/etc/loki/loki.yaml:ro,Z
 EnvironmentFile={{ observability_secrets_dir }}/s3.env
 
-PublishPort={{ wireguard_server_ip }}:3100:3100
-
 Exec=-config.file=/etc/loki/loki.yaml -config.expand-env=true
 
 User=10001
diff --git a/ansible/roles/observability/templates/mimir.container.j2 b/ansible/roles/observability/templates/mimir.container.j2
index 2914f32..b7d67b0 100644
--- a/ansible/roles/observability/templates/mimir.container.j2
+++ b/ansible/roles/observability/templates/mimir.container.j2
@@ -1,8 +1,7 @@
 [Unit]
 Description=Grafana Mimir
 Wants=network-online.target observability-network.service
-After=network-online.target observability-network.service wg-quick@{{ wireguard_interface }}.service
-Requires=wg-quick@{{ wireguard_interface }}.service
+After=network-online.target observability-network.service
 
 [Container]
 Image={{ image_mimir }}
@@ -12,10 +11,6 @@ Volume=mimir-data.volume:/var/lib/mimir:Z
 Volume={{ observability_config_dir }}/mimir/mimir.yaml:/etc/mimir/mimir.yaml:ro,Z
 EnvironmentFile={{ observability_secrets_dir }}/s3.env
 
-# Bind only to WireGuard interface
-PublishPort={{ wireguard_server_ip }}:8080:8080
-PublishPort={{ wireguard_server_ip }}:9095:9095
-
 # -config.expand-env enables ${AWS_*} substitution from EnvironmentFile.
 Exec=-config.file=/etc/mimir/mimir.yaml -config.expand-env=true
 
diff --git a/ansible/roles/observability/templates/tempo.container.j2 b/ansible/roles/observability/templates/tempo.container.j2
index 1da2b42..9945e1b 100644
--- a/ansible/roles/observability/templates/tempo.container.j2
+++ b/ansible/roles/observability/templates/tempo.container.j2
@@ -1,8 +1,7 @@
 [Unit]
 Description=Grafana Tempo
 Wants=network-online.target observability-network.service
-After=network-online.target observability-network.service wg-quick@{{ wireguard_interface }}.service
-Requires=wg-quick@{{ wireguard_interface }}.service
+After=network-online.target observability-network.service
 
 [Container]
 Image={{ image_tempo }}
@@ -12,10 +11,6 @@ Volume=tempo-data.volume:/var/lib/tempo:Z
 Volume={{ observability_config_dir }}/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro,Z
 EnvironmentFile={{ observability_secrets_dir }}/s3.env
 
-PublishPort={{ wireguard_server_ip }}:3200:3200
-PublishPort={{ wireguard_server_ip }}:4317:4317
-PublishPort={{ wireguard_server_ip }}:4318:4318
-
 Exec=-config.file=/etc/tempo/tempo.yaml -config.expand-env=true
 
 User=10001
diff --git a/ansible/roles/observability/templates/wg-easy.container.j2 b/ansible/roles/observability/templates/wg-easy.container.j2
new file mode 100644
index 0000000..966274c
--- /dev/null
+++ b/ansible/roles/observability/templates/wg-easy.container.j2
@@ -0,0 +1,37 @@
+[Unit]
+Description=wg-easy WireGuard peer management
+# wg-easy manages the host wg0 interface directly; it must start after the
+# network is up but has no dependency on any observability container.
+Wants=network-online.target
+After=network-online.target
+
+[Container]
+Image={{ image_wg_easy }}
+ContainerName=wg-easy
+
+# Host network required so wg-easy can create/configure the wg0 interface on
+# the real host namespace.  Container network isolation is intentionally bypassed.
+Network=host
+
+# Capabilities for WireGuard interface management
+AddCapability=NET_ADMIN
+AddCapability=NET_RAW
+AddCapability=SYS_MODULE
+
+# Disable SELinux labelling — the container needs to write /etc/wireguard on
+# the host and bind /dev/net/tun; the default container policy would deny this.
+SecurityLabelDisable=true
+
+# /etc/wireguard is bind-mounted WITHOUT :Z so the host directory keeps its
+# original SELinux context and `wg` tooling outside the container still works.
+Volume=/etc/wireguard:/etc/wireguard
+
+EnvironmentFile={{ observability_secrets_dir }}/wg-easy.env
+
+[Service]
+Restart=on-failure
+RestartSec=10
+TimeoutStartSec=120
+
+[Install]
+WantedBy=multi-user.target default.target
diff --git a/ansible/roles/observability/templates/wg-easy.env.j2 b/ansible/roles/observability/templates/wg-easy.env.j2
new file mode 100644
index 0000000..4b712f0
--- /dev/null
+++ b/ansible/roles/observability/templates/wg-easy.env.j2
@@ -0,0 +1,25 @@
+# wg-easy environment — rendered from Ansible vault; do not commit plaintext.
+# This file is chmod 0600 and lives under observability_secrets_dir.
+
+# Public hostname (or IP) that WireGuard peers use to reach this server.
+# This is written into generated peer configs, not the bind address.
+WG_HOST={{ wireguard_public_ip | default(ansible_default_ipv4.address) }}
+
+# WireGuard listen port (must match the nftables allow rule)
+WG_PORT={{ wg_easy_wg_port }}
+
+# Web-UI port
+PORT={{ wg_easy_web_port }}
+
+# Bind the web UI only to the WireGuard interface IP so it is never reachable
+# from the public internet (nftables is defense-in-depth on top of this).
+HOST={{ wireguard_server_ip }}
+
+# Default WireGuard subnet assigned to peers (x = incremented per peer)
+WG_DEFAULT_ADDRESS=10.8.0.x
+
+# bcrypt hash of the web-UI password (from vault_wg_easy_password_hash)
+PASSWORD_HASH={{ wg_easy_password_hash }}
+
+# DNS pushed to peers
+WG_DEFAULT_DNS=1.1.1.1,1.0.0.1
diff --git a/ansible/roles/wireguard/handlers/main.yml b/ansible/roles/wireguard/handlers/main.yml
index 1908929..05879c6 100644
--- a/ansible/roles/wireguard/handlers/main.yml
+++ b/ansible/roles/wireguard/handlers/main.yml
@@ -1,5 +1,6 @@
 ---
+# wg-quick is no longer used; wg-easy manages the WireGuard interface.
+# Handler kept as a no-op stub so any lingering notify references don't fail.
 - name: Restart wireguard
-  ansible.builtin.systemd:
-    name: "wg-quick@{{ wireguard_interface }}"
-    state: restarted
+  ansible.builtin.debug:
+    msg: "wg-quick is disabled; wg-easy manages {{ wireguard_interface }}"
diff --git a/ansible/roles/wireguard/tasks/main.yml b/ansible/roles/wireguard/tasks/main.yml
index 1cc18ea..142db5f 100644
--- a/ansible/roles/wireguard/tasks/main.yml
+++ b/ansible/roles/wireguard/tasks/main.yml
@@ -1,4 +1,9 @@
 ---
+- name: Install wireguard-tools
+  ansible.builtin.package:
+    name: wireguard-tools
+    state: present
+
 - name: Ensure /etc/wireguard exists
   ansible.builtin.file:
     path: /etc/wireguard
@@ -7,15 +12,6 @@
     owner: root
     group: root
 
-- name: Render wg0.conf
-  ansible.builtin.template:
-    src: wg0.conf.j2
-    dest: "/etc/wireguard/{{ wireguard_interface }}.conf"
-    mode: "0600"
-    owner: root
-    group: root
-  notify: Restart wireguard
-
 - name: Enable IPv4 forwarding
   ansible.posix.sysctl:
     name: net.ipv4.ip_forward
@@ -24,8 +20,18 @@
     state: present
     reload: true
 
-- name: Enable wg-quick service
+- name: Enable src_valid_mark (required for WireGuard routing)
+  ansible.posix.sysctl:
+    name: net.ipv4.conf.all.src_valid_mark
+    value: "1"
+    sysctl_set: true
+    state: present
+    reload: true
+
+# wg-easy takes over wg0 management; ensure wg-quick is not competing.
+- name: Disable and stop wg-quick
   ansible.builtin.systemd:
     name: "wg-quick@{{ wireguard_interface }}"
-    enabled: true
-    state: started
+    enabled: false
+    state: stopped
+  failed_when: false
diff --git a/build/caddy/Dockerfile b/build/caddy/Dockerfile
new file mode 100644
index 0000000..b2eb2d3
--- /dev/null
+++ b/build/caddy/Dockerfile
@@ -0,0 +1,38 @@
+# syntax=docker/dockerfile:1
+#
+# Multi-stage build: xcaddy compiles Caddy with the Coraza WAF plugin and the
+# OWASP Core Rule Set embedded at build time, then the final image is a minimal
+# debian:bookworm-slim layer that ships only the compiled binary.
+#
+# Rebuild whenever the ARG versions below change; Ansible will detect the
+# Dockerfile checksum change and re-run `podman build`.
+
+FROM golang:1.23-bookworm AS builder
+
+ARG XCADDY_VERSION=v0.3.5
+ARG CADDY_VERSION=v2.9.1
+ARG CORAZA_CADDY_VERSION=v2.5.0
+ARG CORAZA_CRS_VERSION=v4.7.0
+
+RUN go install "github.com/caddyserver/xcaddy/cmd/xcaddy@${XCADDY_VERSION}"
+
+RUN xcaddy build "${CADDY_VERSION}" \
+    --with "github.com/corazawaf/coraza-caddy/v2@${CORAZA_CADDY_VERSION}" \
+    --with "github.com/corazawaf/coraza-coreruleset@${CORAZA_CRS_VERSION}"
+
+# ── Runtime image ──────────────────────────────────────────────────────────────
+FROM debian:bookworm-slim
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /go/caddy /usr/bin/caddy
+
+RUN groupadd --system --gid 1000 caddy \
+    && useradd --system --uid 1000 --gid caddy --no-create-home caddy
+
+EXPOSE 80 3100 3200 4317 4318 8080
+
+ENTRYPOINT ["/usr/bin/caddy"]
+CMD ["run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"]
diff --git a/tofu/buckets.tf b/tofu/buckets.tf
index 7f367db..1d0cdc7 100644
--- a/tofu/buckets.tf
+++ b/tofu/buckets.tf
@@ -10,7 +10,7 @@ locals {
     }
     tempo = {
       name           = "${var.bucket_prefix}-tempo"
-      retention_days = 30
+      retention_days = 90
     }
   }
 }
diff --git a/tofu/outputs.tf b/tofu/outputs.tf
index 51faaee..3d42fda 100644
--- a/tofu/outputs.tf
+++ b/tofu/outputs.tf
@@ -12,6 +12,11 @@ output "server_id" {
   value = hcloud_server.watchtower.id
 }
 
+output "observability_volume_id" {
+  description = "Hetzner Volume ID for the observability data disk. Used by Ansible to construct the stable by-id device path (scsi-0HC_Volume_<id>)."
+  value       = hcloud_volume.observability.id
+}
+
 output "buckets" {
   description = "Telemetry bucket names."
   value       = { for k, v in aws_s3_bucket.telemetry : k => v.bucket }
@@ -23,7 +28,7 @@ output "s3_endpoint" {
 }
 
 output "ansible_inventory" {
-  description = "Drop-in inventory snippet for Ansible."
+  description = "Drop-in inventory snippet for Ansible. Pipe to hosts.yml: tofu output -json | jq -r '.ansible_inventory.value'"
   value = yamlencode({
     all = {
       hosts = {
@@ -31,6 +36,7 @@ output "ansible_inventory" {
           ansible_host               = hcloud_primary_ip.ipv4.ip_address
           ansible_user               = "root"
           ansible_python_interpreter = "/usr/bin/python3"
+          observability_volume_id    = hcloud_volume.observability.id
         }
       }
     }
diff --git a/tofu/server.tf b/tofu/server.tf
index 596fd1e..1f17794 100644
--- a/tofu/server.tf
+++ b/tofu/server.tf
@@ -44,6 +44,28 @@ resource "hcloud_primary_ip" "ipv6" {
   auto_delete     = false
 }
 
+resource "hcloud_volume" "observability" {
+  name     = "${var.server_name}-observability"
+  size     = var.observability_volume_size_gb
+  location = var.location
+
+  labels = {
+    role        = "observability-data"
+    managed_by  = "opentofu"
+    environment = "prod"
+  }
+
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+
+resource "hcloud_volume_attachment" "observability" {
+  volume_id = hcloud_volume.observability.id
+  server_id = hcloud_server.watchtower.id
+  automount = false
+}
+
 resource "hcloud_server" "watchtower" {
   name         = var.server_name
   server_type  = var.server_type
diff --git a/tofu/variables.tf b/tofu/variables.tf
index 16a8c7e..3f58b69 100644
--- a/tofu/variables.tf
+++ b/tofu/variables.tf
@@ -57,6 +57,12 @@ variable "admin_allow_ipv6" {
   default     = ["::/0"]
 }
 
+variable "observability_volume_size_gb" {
+  description = "Size in GB of the Hetzner Volume used for all observability service data (Loki WAL, Tempo WAL, Mimir TSDB, Grafana DB)."
+  type        = number
+  default     = 500
+}
+
 variable "bucket_prefix" {
   description = "Prefix for the three telemetry buckets."
   type        = string
-- 
2.47.3


From 1b3cb87cebd2fc0e2e3c95822f9f9af143ef0783 Mon Sep 17 00:00:00 2001
From: Jason Ross <jason.ross841@outlook.com>
Date: Sun, 10 May 2026 22:05:12 -0500
Subject: [PATCH 2/2] working deployment

---
 .gitignore                                    |  2 +-
 README.md                                     |  6 ++-
 ansible/ansible.cfg                           |  7 +--
 .../{ => inventory}/group_vars/all/main.yml   |  4 +-
 .../group_vars/all/vault.yml.example          |  0
 ansible/inventory/hosts.yml.example           |  4 +-
 ansible/roles/base/tasks/main.yml             |  7 ++-
 ansible/roles/luks/tasks/main.yml             |  2 +-
 ansible/roles/observability/tasks/main.yml    | 28 +++++++++-
 .../templates/mimir-runtime.yaml.j2           |  4 ++
 .../templates/mimir.container.j2              |  1 +
 .../observability/templates/mimir.yaml.j2     |  6 +++
 ansible/roles/wireguard/tasks/main.yml        | 17 ++++++
 build/caddy/Dockerfile                        |  6 +--
 tofu/backend.hcl.example                      | 10 ++++
 tofu/buckets.tf                               | 52 +++----------------
 tofu/server.tf                                | 16 +++---
 17 files changed, 103 insertions(+), 69 deletions(-)
 rename ansible/{ => inventory}/group_vars/all/main.yml (94%)
 rename ansible/{ => inventory}/group_vars/all/vault.yml.example (100%)
 create mode 100644 ansible/roles/observability/templates/mimir-runtime.yaml.j2
 create mode 100644 tofu/backend.hcl.example

diff --git a/.gitignore b/.gitignore
index f69041e..eac96f9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,7 +9,7 @@ secrets/
 .vault_pass
 .vault_password
 ansible/inventory/hosts.yml
-ansible/group_vars/**/vault.yml
+ansible/inventory/group_vars/**/vault.yml
 wireguard/wg0.conf
 wireguard/peers/
 .aws/
diff --git a/README.md b/README.md
index 9007efa..867b99d 100644
--- a/README.md
+++ b/README.md
@@ -14,8 +14,10 @@ single Hetzner CPX42 in Nuremberg (nbg1). Built with Mimir, Loki, Tempo, and Gra
 ```bash
 # 1. Provision infrastructure
 cd tofu
-cp terraform.tfvars.example terraform.tfvars   # fill in your tokens
-tofu init
+cp backend.hcl.example backend.hcl          # set your Cloudflare Account ID
+cp terraform.tfvars.example terraform.tfvars # set Hetzner tokens + SSH key
+set -a && source .env && set +a             # load R2 credentials into env
+tofu init -backend-config=backend.hcl
 tofu apply
 
 # 2. Configure server
diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg
index e1c3e4b..76904c5 100644
--- a/ansible/ansible.cfg
+++ b/ansible/ansible.cfg
@@ -3,11 +3,12 @@ inventory = inventory/hosts.yml
 roles_path = roles
 host_key_checking = False
 retry_files_enabled = False
-stdout_callback = yaml
+stdout_callback = default
+result_format    = yaml
 forks = 5
 interpreter_python = /usr/bin/python3
-vault_password_file = .vault_pass
+vault_password_file = $HOME/.config/watchtower-observe/vault_pass
 
 [ssh_connection]
 pipelining = True
-ssh_args = -o ControlMaster=auto -o ControlPersist=60s
+ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o IdentityFile=~/.ssh/watchtower-observe -o IdentitiesOnly=yes
diff --git a/ansible/group_vars/all/main.yml b/ansible/inventory/group_vars/all/main.yml
similarity index 94%
rename from ansible/group_vars/all/main.yml
rename to ansible/inventory/group_vars/all/main.yml
index e3b2a94..3d5754a 100644
--- a/ansible/group_vars/all/main.yml
+++ b/ansible/inventory/group_vars/all/main.yml
@@ -48,8 +48,8 @@ observability_volume_id: ""
 # When observability_volume_id is set, use the stable Hetzner by-id path.
 # When empty, the LUKS role falls back to a loop-backed image on the root disk (dev/staging only).
 luks_device_source: >-
-  {{ (observability_volume_id | length > 0)
-     | ternary('/dev/disk/by-id/scsi-0HC_Volume_' + observability_volume_id, '/dev/sdb') }}
+  {{ ((observability_volume_id | string) | length > 0)
+     | ternary('/dev/disk/by-id/scsi-0HC_Volume_' + (observability_volume_id | string), '/dev/sdb') }}
 luks_mapper_name: observability_data
 
 # Container images (pin in production)
diff --git a/ansible/group_vars/all/vault.yml.example b/ansible/inventory/group_vars/all/vault.yml.example
similarity index 100%
rename from ansible/group_vars/all/vault.yml.example
rename to ansible/inventory/group_vars/all/vault.yml.example
diff --git a/ansible/inventory/hosts.yml.example b/ansible/inventory/hosts.yml.example
index 73599a2..058744b 100644
--- a/ansible/inventory/hosts.yml.example
+++ b/ansible/inventory/hosts.yml.example
@@ -4,6 +4,6 @@ all:
       ansible_host: REPLACE_WITH_TOFU_OUTPUT_IPV4
       ansible_user: root
       ansible_python_interpreter: /usr/bin/python3
-      # Hetzner Volume ID for the observability data disk.
+      # Hetzner Volume ID for the observability data disk (quote to keep as string).
       # Populate from: tofu output -json | jq -r '.ansible_inventory.value'
-      observability_volume_id: REPLACE_WITH_TOFU_OUTPUT_OBSERVABILITY_VOLUME_ID
+      observability_volume_id: "REPLACE_WITH_TOFU_OUTPUT_OBSERVABILITY_VOLUME_ID"
diff --git a/ansible/roles/base/tasks/main.yml b/ansible/roles/base/tasks/main.yml
index a38f00a..be4bbe9 100644
--- a/ansible/roles/base/tasks/main.yml
+++ b/ansible/roles/base/tasks/main.yml
@@ -9,7 +9,6 @@
   ansible.builtin.dnf:
     name:
       - podman
-      - podman-compose
       - wireguard-tools
       - cryptsetup
       - nftables
@@ -27,6 +26,12 @@
   community.general.timezone:
     name: "{{ timezone }}"
 
+- name: Ensure journald drop-in directory exists
+  ansible.builtin.file:
+    path: /etc/systemd/journald.conf.d
+    state: directory
+    mode: "0755"
+
 - name: Configure persistent journald with size cap
   ansible.builtin.copy:
     dest: /etc/systemd/journald.conf.d/persistent.conf
diff --git a/ansible/roles/luks/tasks/main.yml b/ansible/roles/luks/tasks/main.yml
index 16d4cfe..10bfe85 100644
--- a/ansible/roles/luks/tasks/main.yml
+++ b/ansible/roles/luks/tasks/main.yml
@@ -22,7 +22,7 @@
       to '{{ observability_volume_id }}'. Ensure the Hetzner Volume is attached to the server
       and udev has settled (try: udevadm settle --timeout=30).
   when:
-    - observability_volume_id | length > 0
+    - (observability_volume_id | string) | length > 0
     - not luks_device_stat.stat.exists
 
 - name: Create loop-backed LUKS image (if no real device)
diff --git a/ansible/roles/observability/tasks/main.yml b/ansible/roles/observability/tasks/main.yml
index dc6a3b9..d442bf2 100644
--- a/ansible/roles/observability/tasks/main.yml
+++ b/ansible/roles/observability/tasks/main.yml
@@ -57,6 +57,13 @@
     mode: "0644"
   notify: Restart mimir
 
+- name: Render Mimir runtime overrides
+  ansible.builtin.template:
+    src: mimir-runtime.yaml.j2
+    dest: "{{ observability_config_dir }}/mimir/runtime.yaml"
+    mode: "0644"
+  notify: Restart mimir
+
 - name: Render Loki config
   ansible.builtin.template:
     src: loki.yaml.j2
@@ -75,7 +82,7 @@
   ansible.builtin.template:
     src: grafana.ini.j2
     dest: "{{ observability_config_dir }}/grafana/grafana.ini"
-    mode: "0640"
+    mode: "0644"
   notify: Restart grafana
 
 - name: Render Grafana datasource provisioning
@@ -163,6 +170,25 @@
 - name: Force handlers to flush before health checks
   ansible.builtin.meta: flush_handlers
 
+# Handlers only restart services when the quadlet templates change.  On
+# subsequent runs (or partial first runs) the units exist but are stopped, so
+# explicitly ensure each backend is started before the health checks below.
+# Quadlet-generated units cannot be `enabled` (they're auto-wired by their
+# WantedBy= line), so we only set state=started — systemd treats this as a
+# no-op when the unit is already active.
+- name: Ensure observability services are started
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: started
+    daemon_reload: true
+  loop:
+    - wg-easy.service
+    - mimir.service
+    - loki.service
+    - tempo.service
+    - grafana.service
+    - caddy.service
+
 - name: Deploy disk-guard script
   ansible.builtin.template:
     src: disk-guard.sh.j2
diff --git a/ansible/roles/observability/templates/mimir-runtime.yaml.j2 b/ansible/roles/observability/templates/mimir-runtime.yaml.j2
new file mode 100644
index 0000000..a25839e
--- /dev/null
+++ b/ansible/roles/observability/templates/mimir-runtime.yaml.j2
@@ -0,0 +1,4 @@
+# Mimir runtime overrides (hot-reloaded).  Empty by default; add per-tenant
+# overrides here without restarting Mimir.
+# Reference: https://grafana.com/docs/mimir/latest/configure/about-runtime-configuration/
+overrides: {}
diff --git a/ansible/roles/observability/templates/mimir.container.j2 b/ansible/roles/observability/templates/mimir.container.j2
index b7d67b0..2834388 100644
--- a/ansible/roles/observability/templates/mimir.container.j2
+++ b/ansible/roles/observability/templates/mimir.container.j2
@@ -9,6 +9,7 @@ ContainerName=mimir
 Network=observability.network
 Volume=mimir-data.volume:/var/lib/mimir:Z
 Volume={{ observability_config_dir }}/mimir/mimir.yaml:/etc/mimir/mimir.yaml:ro,Z
+Volume={{ observability_config_dir }}/mimir/runtime.yaml:/etc/mimir/runtime.yaml:ro,Z
 EnvironmentFile={{ observability_secrets_dir }}/s3.env
 
 # -config.expand-env enables ${AWS_*} substitution from EnvironmentFile.
diff --git a/ansible/roles/observability/templates/mimir.yaml.j2 b/ansible/roles/observability/templates/mimir.yaml.j2
index e89212e..19db5ae 100644
--- a/ansible/roles/observability/templates/mimir.yaml.j2
+++ b/ansible/roles/observability/templates/mimir.yaml.j2
@@ -5,6 +5,12 @@ target: all,alertmanager,overrides-exporter
 
 multitenancy_enabled: true
 
+# The activity tracker writes its log to ./metrics-activity.log by default.
+# Inside the container the working directory is "/" which is not writable for
+# the non-root user, so point it at the data volume.
+activity_tracker:
+  filepath: /var/lib/mimir/metrics-activity.log
+
 server:
   http_listen_port: 8080
   grpc_listen_port: 9095
diff --git a/ansible/roles/wireguard/tasks/main.yml b/ansible/roles/wireguard/tasks/main.yml
index 142db5f..9e105c6 100644
--- a/ansible/roles/wireguard/tasks/main.yml
+++ b/ansible/roles/wireguard/tasks/main.yml
@@ -12,6 +12,23 @@
     owner: root
     group: root
 
+# wg-easy runs `wg-quick up wg0` inside its container, which calls
+# iptables-legacy to set up the NAT POSTROUTING rule.  AlmaLinux 10 doesn't
+# auto-load the legacy iptables kernel modules, so the call fails with
+# "can't initialize iptables table 'nat': Table does not exist" and wg0 is
+# torn down.  Load the modules now and persist them across reboots.
+- name: Load iptables kernel modules required by wg-easy
+  community.general.modprobe:
+    name: "{{ item }}"
+    state: present
+    persistent: present
+  loop:
+    - ip_tables
+    - iptable_filter
+    - iptable_nat
+    - nf_nat
+    - nf_conntrack
+
 - name: Enable IPv4 forwarding
   ansible.posix.sysctl:
     name: net.ipv4.ip_forward
diff --git a/build/caddy/Dockerfile b/build/caddy/Dockerfile
index b2eb2d3..89b922a 100644
--- a/build/caddy/Dockerfile
+++ b/build/caddy/Dockerfile
@@ -7,10 +7,10 @@
 # Rebuild whenever the ARG versions below change; Ansible will detect the
 # Dockerfile checksum change and re-run `podman build`.
 
-FROM golang:1.23-bookworm AS builder
+FROM golang:1.26-trixie AS builder
 
 ARG XCADDY_VERSION=v0.3.5
-ARG CADDY_VERSION=v2.9.1
+ARG CADDY_VERSION=v2.11.2
 ARG CORAZA_CADDY_VERSION=v2.5.0
 ARG CORAZA_CRS_VERSION=v4.7.0
 
@@ -21,7 +21,7 @@ RUN xcaddy build "${CADDY_VERSION}" \
     --with "github.com/corazawaf/coraza-coreruleset@${CORAZA_CRS_VERSION}"
 
 # ── Runtime image ──────────────────────────────────────────────────────────────
-FROM debian:bookworm-slim
+FROM debian:trixie-slim
 
 RUN apt-get update \
     && apt-get install -y --no-install-recommends ca-certificates \
diff --git a/tofu/backend.hcl.example b/tofu/backend.hcl.example
new file mode 100644
index 0000000..f5f5af6
--- /dev/null
+++ b/tofu/backend.hcl.example
@@ -0,0 +1,10 @@
+# Copy to backend.hcl (gitignored) and set your Cloudflare Account ID.
+# Pass at init time: tofu init -backend-config=backend.hcl
+#
+# Credentials are read from env vars — set before running tofu:
+#   export AWS_ACCESS_KEY_ID=<r2-access-key-id>
+#   export AWS_SECRET_ACCESS_KEY=<r2-secret-access-key>
+
+endpoints = {
+  s3 = "https://<ACCOUNT_ID>.r2.cloudflarestorage.com"
+}
diff --git a/tofu/buckets.tf b/tofu/buckets.tf
index 1d0cdc7..9d692fa 100644
--- a/tofu/buckets.tf
+++ b/tofu/buckets.tf
@@ -1,17 +1,8 @@
 locals {
   buckets = {
-    mimir = {
-      name           = "${var.bucket_prefix}-mimir"
-      retention_days = 365
-    }
-    loki = {
-      name           = "${var.bucket_prefix}-loki"
-      retention_days = 90
-    }
-    tempo = {
-      name           = "${var.bucket_prefix}-tempo"
-      retention_days = 90
-    }
+    mimir = { name = "${var.bucket_prefix}-mimir" }
+    loki  = { name = "${var.bucket_prefix}-loki" }
+    tempo = { name = "${var.bucket_prefix}-tempo" }
   }
 }
 
@@ -22,40 +13,11 @@ resource "aws_s3_bucket" "telemetry" {
   bucket = each.value.name
 
   # Hetzner does not yet support all S3 ACL operations, so keep this minimal.
+  # Note: Hetzner Object Storage does not support the S3 Lifecycle Configuration
+  # API (PutBucketLifecycleConfiguration / GetBucketLifecycleConfiguration).
+  # Retention is enforced at the application layer: Mimir, Loki, and Tempo each
+  # have native retention settings configured via their respective config files.
   lifecycle {
     prevent_destroy = true
   }
 }
-
-resource "aws_s3_bucket_versioning" "telemetry" {
-  for_each = local.buckets
-  provider = aws.hetzner
-
-  bucket = aws_s3_bucket.telemetry[each.key].id
-  versioning_configuration {
-    status = "Suspended"
-  }
-}
-
-# Lifecycle rule: hard-delete safety net behind application retention.
-resource "aws_s3_bucket_lifecycle_configuration" "telemetry" {
-  for_each = local.buckets
-  provider = aws.hetzner
-
-  bucket = aws_s3_bucket.telemetry[each.key].id
-
-  rule {
-    id     = "hard-delete-after-${each.value.retention_days}d"
-    status = "Enabled"
-
-    filter {}
-
-    expiration {
-      days = each.value.retention_days
-    }
-
-    abort_incomplete_multipart_upload {
-      days_after_initiation = 7
-    }
-  }
-}
diff --git a/tofu/server.tf b/tofu/server.tf
index 1f17794..fa4b234 100644
--- a/tofu/server.tf
+++ b/tofu/server.tf
@@ -31,17 +31,17 @@ resource "hcloud_firewall" "watchtower" {
 }
 
 resource "hcloud_primary_ip" "ipv4" {
-  name            = "${var.server_name}-ipv4"
-  type            = "ipv4"
-  assignee_type   = "server"
-  auto_delete     = false
+  name        = "${var.server_name}-ipv4"
+  type        = "ipv4"
+  location    = var.location
+  auto_delete = false
 }
 
 resource "hcloud_primary_ip" "ipv6" {
-  name            = "${var.server_name}-ipv6"
-  type            = "ipv6"
-  assignee_type   = "server"
-  auto_delete     = false
+  name        = "${var.server_name}-ipv6"
+  type        = "ipv6"
+  location    = var.location
+  auto_delete = false
 }
 
 resource "hcloud_volume" "observability" {
-- 
2.47.3