From 267b6fc39bfeac6bd5f8f239f127e970a3dd4b3d Mon Sep 17 00:00:00 2001 From: Jason Ross Date: Sat, 9 May 2026 20:08:13 -0500 Subject: [PATCH] initial config --- .gitignore | 36 +++ README.md | 61 +++++ ansible/ansible.cfg | 13 ++ ansible/group_vars/all/main.yml | 61 +++++ ansible/group_vars/all/vault.yml.example | 14 ++ ansible/inventory/hosts.yml.example | 6 + ansible/playbooks/deploy.yml | 7 + ansible/playbooks/gdpr-purge.yml | 48 ++++ ansible/playbooks/onboard-tenant.yml | 104 +++++++++ ansible/playbooks/site.yml | 13 ++ ansible/roles/base/handlers/main.yml | 15 ++ ansible/roles/base/tasks/main.yml | 117 ++++++++++ ansible/roles/fail2ban/defaults/main.yml | 12 + ansible/roles/fail2ban/handlers/main.yml | 5 + ansible/roles/fail2ban/tasks/main.yml | 37 +++ .../roles/fail2ban/templates/jail.local.j2 | 43 ++++ ansible/roles/firewall/handlers/main.yml | 5 + ansible/roles/firewall/tasks/main.yml | 46 ++++ .../firewall/templates/watchtower.nft.j2 | 102 +++++++++ ansible/roles/luks/tasks/main.yml | 110 +++++++++ .../observability/files/observability.network | 5 + ansible/roles/observability/handlers/main.yml | 39 ++++ ansible/roles/observability/tasks/main.yml | 158 +++++++++++++ .../templates/datasources.yaml.j2 | 62 +++++ .../templates/grafana-data.volume.j2 | 3 + .../templates/grafana.container.j2 | 34 +++ .../observability/templates/grafana.ini.j2 | 74 ++++++ .../templates/loki-data.volume.j2 | 3 + .../observability/templates/loki.container.j2 | 30 +++ .../observability/templates/loki.yaml.j2 | 90 ++++++++ .../templates/mimir-data.volume.j2 | 3 + .../templates/mimir.container.j2 | 33 +++ .../observability/templates/mimir.yaml.j2 | 97 ++++++++ .../roles/observability/templates/s3.env.j2 | 4 + .../templates/tempo-data.volume.j2 | 3 + .../templates/tempo.container.j2 | 32 +++ .../observability/templates/tempo.yaml.j2 | 73 ++++++ ansible/roles/wireguard/handlers/main.yml | 5 + ansible/roles/wireguard/tasks/main.yml | 31 +++ ansible/roles/wireguard/templates/wg0.conf.j2 | 16 ++ config/alloy/config.alloy.example | 118 ++++++++++ docs/architecture.md | 82 +++++++ docs/gdpr-compliance.md | 137 +++++++++++ docs/multi-tenancy.md | 60 +++++ docs/runbook.md | 213 ++++++++++++++++++ tofu/backend.hcl.example | 12 + tofu/buckets.tf | 61 +++++ tofu/cloud-init.yaml | 27 +++ tofu/outputs.tf | 38 ++++ tofu/providers.tf | 22 ++ tofu/server.tf | 73 ++++++ tofu/terraform.tfvars.example | 8 + tofu/variables.tf | 64 ++++++ tofu/versions.tf | 46 ++++ 54 files changed, 2611 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 ansible/ansible.cfg create mode 100644 ansible/group_vars/all/main.yml create mode 100644 ansible/group_vars/all/vault.yml.example create mode 100644 ansible/inventory/hosts.yml.example create mode 100644 ansible/playbooks/deploy.yml create mode 100644 ansible/playbooks/gdpr-purge.yml create mode 100644 ansible/playbooks/onboard-tenant.yml create mode 100644 ansible/playbooks/site.yml create mode 100644 ansible/roles/base/handlers/main.yml create mode 100644 ansible/roles/base/tasks/main.yml create mode 100644 ansible/roles/fail2ban/defaults/main.yml create mode 100644 ansible/roles/fail2ban/handlers/main.yml create mode 100644 ansible/roles/fail2ban/tasks/main.yml create mode 100644 ansible/roles/fail2ban/templates/jail.local.j2 create mode 100644 ansible/roles/firewall/handlers/main.yml create mode 100644 ansible/roles/firewall/tasks/main.yml create mode 100644 ansible/roles/firewall/templates/watchtower.nft.j2 create mode 100644 ansible/roles/luks/tasks/main.yml create mode 100644 ansible/roles/observability/files/observability.network create mode 100644 ansible/roles/observability/handlers/main.yml create mode 100644 ansible/roles/observability/tasks/main.yml create mode 100644 ansible/roles/observability/templates/datasources.yaml.j2 create mode 100644 ansible/roles/observability/templates/grafana-data.volume.j2 create mode 100644 ansible/roles/observability/templates/grafana.container.j2 create mode 100644 ansible/roles/observability/templates/grafana.ini.j2 create mode 100644 ansible/roles/observability/templates/loki-data.volume.j2 create mode 100644 ansible/roles/observability/templates/loki.container.j2 create mode 100644 ansible/roles/observability/templates/loki.yaml.j2 create mode 100644 ansible/roles/observability/templates/mimir-data.volume.j2 create mode 100644 ansible/roles/observability/templates/mimir.container.j2 create mode 100644 ansible/roles/observability/templates/mimir.yaml.j2 create mode 100644 ansible/roles/observability/templates/s3.env.j2 create mode 100644 ansible/roles/observability/templates/tempo-data.volume.j2 create mode 100644 ansible/roles/observability/templates/tempo.container.j2 create mode 100644 ansible/roles/observability/templates/tempo.yaml.j2 create mode 100644 ansible/roles/wireguard/handlers/main.yml create mode 100644 ansible/roles/wireguard/tasks/main.yml create mode 100644 ansible/roles/wireguard/templates/wg0.conf.j2 create mode 100644 config/alloy/config.alloy.example create mode 100644 docs/architecture.md create mode 100644 docs/gdpr-compliance.md create mode 100644 docs/multi-tenancy.md create mode 100644 docs/runbook.md create mode 100644 tofu/backend.hcl.example create mode 100644 tofu/buckets.tf create mode 100644 tofu/cloud-init.yaml create mode 100644 tofu/outputs.tf create mode 100644 tofu/providers.tf create mode 100644 tofu/server.tf create mode 100644 tofu/terraform.tfvars.example create mode 100644 tofu/variables.tf create mode 100644 tofu/versions.tf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1bd9cf5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,36 @@ +# Secrets — NEVER commit +*.key +*.pem +*.crt +*.priv +secrets/ +**/secrets/ +*.vault +.vault_pass +.vault_password +ansible/inventory/hosts.yml +ansible/group_vars/**/vault.yml +wireguard/wg0.conf +wireguard/peers/ + +# OpenTofu / Terraform +.terraform/ +.terraform.lock.hcl +*.tfstate +*.tfstate.* +*.tfvars +!*.tfvars.example +crash.log +crash.*.log +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Editor / OS +.DS_Store +*.swp +*.swo +.idea/ +.vscode/ +*~ diff --git a/README.md b/README.md new file mode 100644 index 0000000..9007efa --- /dev/null +++ b/README.md @@ -0,0 +1,61 @@ +# Watchtower Observability Stack + +GDPR-compliant, multi-tenant observability stack (metrics, logs, traces) running on a +single Hetzner CPX42 in Nuremberg (nbg1). Built with Mimir, Loki, Tempo, and Grafana. + +- **Infrastructure:** OpenTofu (state in Cloudflare R2) +- **Server config:** Ansible +- **Container runtime:** Podman Quadlets on AlmaLinux 10 +- **Object storage:** Hetzner Object Storage (S3-compatible, EU-resident) +- **Access:** WireGuard VPN only — no public service ports + +## Quick start + +```bash +# 1. Provision infrastructure +cd tofu +cp terraform.tfvars.example terraform.tfvars # fill in your tokens +tofu init +tofu apply + +# 2. Configure server +cd ../ansible +cp inventory/hosts.yml.example inventory/hosts.yml # set host from tofu output +ansible-playbook -i inventory/hosts.yml playbooks/site.yml --ask-vault-pass + +# 3. Onboard your first tenant +ansible-playbook playbooks/onboard-tenant.yml -e tenant_slug=team-alpha +``` + +## Repository layout + +| Path | Purpose | +|------|---------| +| `tofu/` | OpenTofu infrastructure (Hetzner Cloud + Object Storage, Cloudflare R2 state) | +| `ansible/` | Server provisioning roles and playbooks | +| `quadlets/` | Podman Quadlet unit files (deployed by Ansible) | +| `config/` | Mimir, Loki, Tempo, Grafana configuration templates | +| `wireguard/` | WireGuard server template (peers vaulted in Ansible) | +| `scripts/` | Operational scripts (gdpr-purge, onboard-tenant, etc.) | +| `docs/` | Architecture, GDPR, multi-tenancy, runbook | + +## Documentation + +- [Architecture](docs/architecture.md) +- [GDPR compliance](docs/gdpr-compliance.md) +- [Multi-tenancy](docs/multi-tenancy.md) +- [Runbook](docs/runbook.md) + +## Required secrets + +| Secret | Where | Stored as | +|--------|-------|-----------| +| Hetzner Cloud API token | OpenTofu | `terraform.tfvars` (gitignored) | +| Hetzner Object Storage S3 keys | OpenTofu + Ansible | `terraform.tfvars` + Ansible vault | +| Cloudflare R2 access key/secret | OpenTofu backend | env vars `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` | +| WireGuard server private key | Ansible | Ansible vault | +| Grafana admin password | Ansible | Ansible vault | + +## License + +See [LICENSE](LICENSE). diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000..e1c3e4b --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,13 @@ +[defaults] +inventory = inventory/hosts.yml +roles_path = roles +host_key_checking = False +retry_files_enabled = False +stdout_callback = yaml +forks = 5 +interpreter_python = /usr/bin/python3 +vault_password_file = .vault_pass + +[ssh_connection] +pipelining = True +ssh_args = -o ControlMaster=auto -o ControlPersist=60s diff --git a/ansible/group_vars/all/main.yml b/ansible/group_vars/all/main.yml new file mode 100644 index 0000000..54c7c93 --- /dev/null +++ b/ansible/group_vars/all/main.yml @@ -0,0 +1,61 @@ +# Non-secret defaults shared across the stack. +# Secret values live in group_vars/all/vault.yml (ansible-vault encrypted). + +hetzner_location: nbg1 +timezone: Europe/Berlin + +# Object storage +s3_endpoint: "https://{{ hetzner_location }}.your-objectstorage.com" +s3_region: "{{ hetzner_location }}" +s3_bucket_mimir: watchtower-mimir +s3_bucket_loki: watchtower-loki +s3_bucket_tempo: watchtower-tempo + +# Indirection: real values come from vault.yml +hetzner_s3_access_key: "{{ vault_hetzner_s3_access_key }}" +hetzner_s3_secret_key: "{{ vault_hetzner_s3_secret_key }}" +grafana_admin_password: "{{ vault_grafana_admin_password }}" +wireguard_server_private_key: "{{ vault_wireguard_server_private_key }}" +wireguard_server_public_key: "{{ vault_wireguard_server_public_key }}" +wireguard_peers: "{{ vault_wireguard_peers }}" + +# WireGuard +wireguard_interface: wg0 +wireguard_listen_port: 51820 +wireguard_subnet_v4: 10.8.0.0/24 +wireguard_server_address: 10.8.0.1/24 +wireguard_server_ip: 10.8.0.1 + +# Public-facing interface (Hetzner CPX42 default). +public_interface: eth0 + +# CIDRs allowed to reach SSH (22/tcp) and WireGuard (51820/udp) at the host. +# Mirror or tighten what you set in tofu/terraform.tfvars admin_allow_ipv4/v6. +admin_allow_ipv4: + - 0.0.0.0/0 +admin_allow_ipv6: + - "::/0" + +# Local data +observability_data_root: /var/lib/observability +observability_secrets_dir: /etc/observability/secrets +observability_config_dir: /etc/observability/config +luks_device_source: /dev/sdb +luks_mapper_name: observability_data + +# Container images (pin in production) +image_mimir: "docker.io/grafana/mimir:2.14.3" +image_loki: "docker.io/grafana/loki:3.3.2" +image_tempo: "docker.io/grafana/tempo:2.7.1" +image_grafana: "docker.io/grafana/grafana:11.4.0" + +# Retention (per signal) +mimir_retention_days: 90 +loki_retention_days: 30 +tempo_retention_days: 14 + +# Tenants — override in production +tenants: + - slug: default + org_name: Default + grafana_admin_email: admin@example.com diff --git a/ansible/group_vars/all/vault.yml.example b/ansible/group_vars/all/vault.yml.example new file mode 100644 index 0000000..c32e482 --- /dev/null +++ b/ansible/group_vars/all/vault.yml.example @@ -0,0 +1,14 @@ +# Encrypt with: ansible-vault encrypt group_vars/all/vault.yml + +vault_hetzner_s3_access_key: "REPLACE_ME" +vault_hetzner_s3_secret_key: "REPLACE_ME" + +vault_grafana_admin_password: "REPLACE_ME" + +vault_wireguard_server_private_key: "REPLACE_ME_BASE64" +vault_wireguard_server_public_key: "REPLACE_ME_BASE64" + +vault_wireguard_peers: + - name: admin-laptop + public_key: REPLACE_ME + allowed_ips: 10.8.0.10/32 diff --git a/ansible/inventory/hosts.yml.example b/ansible/inventory/hosts.yml.example new file mode 100644 index 0000000..060f88a --- /dev/null +++ b/ansible/inventory/hosts.yml.example @@ -0,0 +1,6 @@ +all: + hosts: + watchtower: + ansible_host: REPLACE_WITH_TOFU_OUTPUT_IPV4 + ansible_user: root + ansible_python_interpreter: /usr/bin/python3 diff --git a/ansible/playbooks/deploy.yml b/ansible/playbooks/deploy.yml new file mode 100644 index 0000000..21f0991 --- /dev/null +++ b/ansible/playbooks/deploy.yml @@ -0,0 +1,7 @@ +--- +# Re-render configs and quadlets only. Use after editing config templates. +- name: Redeploy observability configs and quadlets + hosts: watchtower + become: true + roles: + - observability diff --git a/ansible/playbooks/gdpr-purge.yml b/ansible/playbooks/gdpr-purge.yml new file mode 100644 index 0000000..b3e8d84 --- /dev/null +++ b/ansible/playbooks/gdpr-purge.yml @@ -0,0 +1,48 @@ +--- +# Submit a Loki Delete API request to erase all logs matching a label selector. +# +# Usage: +# ansible-playbook playbooks/gdpr-purge.yml \ +# -e tenant=team-alpha \ +# -e 'selector={user_id="abc-123"}' \ +# -e start=2025-01-01T00:00:00Z \ +# -e end=2026-12-31T23:59:59Z + +- name: GDPR right-to-erasure (Loki) + hosts: watchtower + become: true + gather_facts: true + vars: + tenant: "{{ tenant | mandatory }}" + selector: "{{ selector | mandatory }}" + start: "{{ start | mandatory }}" + end: "{{ end | default(lookup('pipe', 'date -u +%Y-%m-%dT%H:%M:%SZ')) }}" + tasks: + - name: Submit Loki delete request + ansible.builtin.uri: + url: "http://{{ wireguard_server_ip }}:3100/loki/api/v1/delete" + method: POST + headers: + X-Scope-OrgID: "{{ tenant }}" + body_format: form-urlencoded + body: + query: "{{ selector }}" + start: "{{ start }}" + end: "{{ end }}" + status_code: [204] + register: result + + - name: Audit log + ansible.builtin.lineinfile: + path: /var/log/observability/gdpr-audit.log + create: true + mode: "0640" + line: "{{ ansible_date_time.iso8601 }} tenant={{ tenant }} selector='{{ selector }}' start={{ start }} end={{ end }} status=submitted" + + - name: Reminder + ansible.builtin.debug: + msg: | + Loki delete request submitted. Mimir/Tempo do not provide per-record + deletion APIs; the {{ mimir_retention_days }}-day metrics retention + and {{ tempo_retention_days }}-day trace retention satisfy the + GDPR Art. 17 erasure window. diff --git a/ansible/playbooks/onboard-tenant.yml b/ansible/playbooks/onboard-tenant.yml new file mode 100644 index 0000000..193a4c5 --- /dev/null +++ b/ansible/playbooks/onboard-tenant.yml @@ -0,0 +1,104 @@ +--- +# Onboard a new tenant: create Grafana org, admin user, and provisioned datasources. +# +# Usage: +# ansible-playbook playbooks/onboard-tenant.yml \ +# -e tenant_slug=team-alpha \ +# -e tenant_name='Team Alpha' \ +# -e admin_email=alpha-admin@example.com + +- name: Onboard tenant + hosts: watchtower + become: true + gather_facts: false + vars: + tenant_slug: "{{ tenant_slug | mandatory }}" + tenant_name: "{{ tenant_name | default(tenant_slug) }}" + admin_email: "{{ admin_email | mandatory }}" + grafana_url: "http://{{ wireguard_server_ip }}:3000" + tasks: + - name: Create Grafana org + ansible.builtin.uri: + url: "{{ grafana_url }}/api/orgs" + method: POST + user: admin + password: "{{ vault_grafana_admin_password }}" + force_basic_auth: true + body_format: json + body: { name: "{{ tenant_name }}" } + status_code: [200, 409] + register: org_create + + - name: Look up org id + ansible.builtin.uri: + url: "{{ grafana_url }}/api/orgs/name/{{ tenant_name | urlencode }}" + user: admin + password: "{{ vault_grafana_admin_password }}" + force_basic_auth: true + register: org_lookup + + - name: Switch admin context to tenant org + ansible.builtin.uri: + url: "{{ grafana_url }}/api/user/using/{{ org_lookup.json.id }}" + method: POST + user: admin + password: "{{ vault_grafana_admin_password }}" + force_basic_auth: true + status_code: [200] + + - name: Provision tenant-scoped Mimir datasource + ansible.builtin.uri: + url: "{{ grafana_url }}/api/datasources" + method: POST + user: admin + password: "{{ vault_grafana_admin_password }}" + force_basic_auth: true + body_format: json + body: + name: Mimir + type: prometheus + access: proxy + url: "http://mimir:8080/prometheus" + jsonData: + httpHeaderName1: X-Scope-OrgID + secureJsonData: + httpHeaderValue1: "{{ tenant_slug }}" + status_code: [200, 409] + + - name: Provision tenant-scoped Loki datasource + ansible.builtin.uri: + url: "{{ grafana_url }}/api/datasources" + method: POST + user: admin + password: "{{ vault_grafana_admin_password }}" + force_basic_auth: true + body_format: json + body: + name: Loki + type: loki + access: proxy + url: "http://loki:3100" + jsonData: + httpHeaderName1: X-Scope-OrgID + secureJsonData: + httpHeaderValue1: "{{ tenant_slug }}" + status_code: [200, 409] + + - name: Provision tenant-scoped Tempo datasource + ansible.builtin.uri: + url: "{{ grafana_url }}/api/datasources" + method: POST + user: admin + password: "{{ vault_grafana_admin_password }}" + force_basic_auth: true + body_format: json + body: + name: Tempo + type: tempo + access: proxy + url: "http://tempo:3200" + jsonData: + httpHeaderName1: X-Scope-OrgID + secureJsonData: + httpHeaderValue1: "{{ tenant_slug }}" + status_code: [200, 409] diff --git a/ansible/playbooks/site.yml b/ansible/playbooks/site.yml new file mode 100644 index 0000000..2df0a5c --- /dev/null +++ b/ansible/playbooks/site.yml @@ -0,0 +1,13 @@ +--- +- name: Provision Watchtower observability stack + hosts: watchtower + become: true + gather_facts: true + + roles: + - base + - luks + - firewall + - fail2ban + - wireguard + - observability diff --git a/ansible/roles/base/handlers/main.yml b/ansible/roles/base/handlers/main.yml new file mode 100644 index 0000000..c636cab --- /dev/null +++ b/ansible/roles/base/handlers/main.yml @@ -0,0 +1,15 @@ +--- +- name: Restart journald + ansible.builtin.systemd: + name: systemd-journald + state: restarted + +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true + +- name: Restart dnf-automatic timer + ansible.builtin.systemd: + name: dnf-automatic-install.timer + state: restarted + enabled: true diff --git a/ansible/roles/base/tasks/main.yml b/ansible/roles/base/tasks/main.yml new file mode 100644 index 0000000..a38f00a --- /dev/null +++ b/ansible/roles/base/tasks/main.yml @@ -0,0 +1,117 @@ +--- +- name: Update all packages + ansible.builtin.dnf: + name: "*" + state: latest + update_cache: true + +- name: Install required packages + ansible.builtin.dnf: + name: + - podman + - podman-compose + - wireguard-tools + - cryptsetup + - nftables + - python3-libselinux + - policycoreutils-python-utils + - container-selinux + - jq + - bind-utils + - tar + - rsync + - dnf-automatic + state: present + +- name: Set timezone + community.general.timezone: + name: "{{ timezone }}" + +- name: Configure persistent journald with size cap + ansible.builtin.copy: + dest: /etc/systemd/journald.conf.d/persistent.conf + mode: "0644" + content: | + [Journal] + Storage=persistent + SystemMaxUse=2G + SystemMaxFileSize=128M + MaxRetentionSec=30day + notify: Restart journald + +- name: Render dnf-automatic config (security updates, apply, reboot when needed) + ansible.builtin.copy: + dest: /etc/dnf/automatic.conf + mode: "0644" + owner: root + group: root + content: | + # Managed by Ansible — see roles/base/tasks/main.yml. + [commands] + upgrade_type = security + random_sleep = 360 + network_online_timeout = 60 + download_updates = yes + apply_updates = yes + reboot = when-needed + reboot_command = "shutdown -r +5 'Rebooting for security updates'" + + [emitters] + emit_via = stdio,motd + + [base] + # Don't auto-restart the container runtime / observability stack out from + # under us — operator applies these manually during a maintenance window. + exclude = podman* conmon crun containers-common netavark aardvark-dns container-selinux + notify: Restart dnf-automatic timer + +- name: Ensure dnf-automatic-install timer override directory exists + ansible.builtin.file: + path: /etc/systemd/system/dnf-automatic-install.timer.d + state: directory + mode: "0755" + +- name: Pin dnf-automatic to 01:00 Central time (America/Chicago, honours DST) + ansible.builtin.copy: + dest: /etc/systemd/system/dnf-automatic-install.timer.d/override.conf + mode: "0644" + content: | + [Timer] + OnCalendar= + OnCalendar=*-*-* 01:00:00 America/Chicago + RandomizedDelaySec=30m + Persistent=true + notify: + - Reload systemd + - Restart dnf-automatic timer + +- name: Ensure override directory exists + ansible.builtin.file: + path: /etc/systemd/system/dnf-automatic-install.timer.d + state: directory + mode: "0755" + +- name: Disable the download-only timer (we use the install timer) + ansible.builtin.systemd: + name: dnf-automatic.timer + enabled: false + state: stopped + failed_when: false + +- name: Enable dnf-automatic-install.timer (downloads + applies + reboots) + ansible.builtin.systemd: + name: dnf-automatic-install.timer + enabled: true + state: started + daemon_reload: true + +- name: Ensure SELinux enforcing + ansible.posix.selinux: + policy: targeted + state: enforcing + +- name: Enable container manage cgroup boolean (rootful Podman + systemd) + ansible.posix.seboolean: + name: container_manage_cgroup + state: true + persistent: true diff --git a/ansible/roles/fail2ban/defaults/main.yml b/ansible/roles/fail2ban/defaults/main.yml new file mode 100644 index 0000000..ba4a333 --- /dev/null +++ b/ansible/roles/fail2ban/defaults/main.yml @@ -0,0 +1,12 @@ +--- +# Bantime / findtime / maxretry defaults — override in group_vars/all/main.yml. +fail2ban_bantime: 1h +fail2ban_findtime: 10m +fail2ban_maxretry: 5 +fail2ban_destemail: root@localhost +fail2ban_sender: fail2ban@watchtower + +# Recidive: bans repeat offenders for a long time. +fail2ban_recidive_bantime: 1w +fail2ban_recidive_findtime: 1d +fail2ban_recidive_maxretry: 5 diff --git a/ansible/roles/fail2ban/handlers/main.yml b/ansible/roles/fail2ban/handlers/main.yml new file mode 100644 index 0000000..83fad83 --- /dev/null +++ b/ansible/roles/fail2ban/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart fail2ban + ansible.builtin.systemd: + name: fail2ban + state: restarted diff --git a/ansible/roles/fail2ban/tasks/main.yml b/ansible/roles/fail2ban/tasks/main.yml new file mode 100644 index 0000000..d0f943f --- /dev/null +++ b/ansible/roles/fail2ban/tasks/main.yml @@ -0,0 +1,37 @@ +--- +- name: Enable EPEL (provides fail2ban on AlmaLinux 10) + ansible.builtin.dnf: + name: epel-release + state: present + +- name: Install fail2ban + ansible.builtin.dnf: + name: + - fail2ban + - fail2ban-firewalld # provides the firewallcmd actions; harmless even if firewalld is masked + state: present + update_cache: true + +# Default fail2ban-firewalld package drops a 0-firewalld.conf in jail.d that sets +# banaction = firewallcmd-rich-rules — but firewalld is masked. Remove it so our +# nftables banaction wins. +- name: Remove fail2ban-firewalld jail override + ansible.builtin.file: + path: /etc/fail2ban/jail.d/00-firewalld.conf + state: absent + notify: Restart fail2ban + +- name: Render jail.local + ansible.builtin.template: + src: jail.local.j2 + dest: /etc/fail2ban/jail.local + mode: "0644" + owner: root + group: root + notify: Restart fail2ban + +- name: Enable and start fail2ban + ansible.builtin.systemd: + name: fail2ban + enabled: true + state: started diff --git a/ansible/roles/fail2ban/templates/jail.local.j2 b/ansible/roles/fail2ban/templates/jail.local.j2 new file mode 100644 index 0000000..f9a5180 --- /dev/null +++ b/ansible/roles/fail2ban/templates/jail.local.j2 @@ -0,0 +1,43 @@ +# /etc/fail2ban/jail.local — managed by Ansible. +# +# fail2ban manages its own nftables table (inet f2b-table), inserted at higher +# priority than the watchtower ruleset, so bans take effect even when the +# watchtower table accept-list rule has already matched. + +[DEFAULT] +# Backend reads from systemd journal — no /var/log/secure on AlmaLinux 10 by default. +backend = systemd + +bantime = {{ fail2ban_bantime }} +findtime = {{ fail2ban_findtime }} +maxretry = {{ fail2ban_maxretry }} + +# nftables-multiport creates: table inet f2b-table { set f2b- ... } +banaction = nftables-multiport +banaction_allports = nftables-allports + +# Whitelist trusted networks: WireGuard subnet, loopback, and the admin allow list. +ignoreip = 127.0.0.1/8 ::1 {{ wireguard_subnet_v4 }}{% for c in admin_allow_ipv4 %} {{ c }}{% endfor %}{% for c in admin_allow_ipv6 %} {{ c }}{% endfor %} + +destemail = {{ fail2ban_destemail }} +sender = {{ fail2ban_sender }} +mta = sendmail + +# Action: ban only (mailing requires an MTA — disable if you don't run one). +action = %(action_)s + +[sshd] +enabled = true +port = ssh +mode = aggressive +maxretry = {{ fail2ban_maxretry }} +journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd + +# Long-term ban for repeat offenders. Watches fail2ban's own log. +[recidive] +enabled = true +logpath = /var/log/fail2ban.log +banaction = nftables-allports +bantime = {{ fail2ban_recidive_bantime }} +findtime = {{ fail2ban_recidive_findtime }} +maxretry = {{ fail2ban_recidive_maxretry }} diff --git a/ansible/roles/firewall/handlers/main.yml b/ansible/roles/firewall/handlers/main.yml new file mode 100644 index 0000000..69e8f72 --- /dev/null +++ b/ansible/roles/firewall/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Reload nftables + ansible.builtin.systemd: + name: nftables + state: reloaded diff --git a/ansible/roles/firewall/tasks/main.yml b/ansible/roles/firewall/tasks/main.yml new file mode 100644 index 0000000..92193ad --- /dev/null +++ b/ansible/roles/firewall/tasks/main.yml @@ -0,0 +1,46 @@ +--- +- name: Stop and disable firewalld (replaced by native nftables) + ansible.builtin.systemd: + name: firewalld + enabled: false + state: stopped + masked: true + failed_when: false + +- name: Install nftables + ansible.builtin.dnf: + name: nftables + state: present + +- name: Ensure /etc/nftables exists + ansible.builtin.file: + path: /etc/nftables + state: directory + mode: "0755" + +- name: Render watchtower nftables ruleset + ansible.builtin.template: + src: watchtower.nft.j2 + dest: /etc/nftables/watchtower.nft + mode: "0640" + owner: root + group: root + validate: nft --check --file %s + notify: Reload nftables + +- name: Point /etc/sysconfig/nftables.conf at the watchtower ruleset + ansible.builtin.copy: + dest: /etc/sysconfig/nftables.conf + mode: "0640" + owner: root + group: root + content: | + # Loaded at boot by nftables.service. + include "/etc/nftables/watchtower.nft" + notify: Reload nftables + +- name: Enable nftables.service + ansible.builtin.systemd: + name: nftables + enabled: true + state: started diff --git a/ansible/roles/firewall/templates/watchtower.nft.j2 b/ansible/roles/firewall/templates/watchtower.nft.j2 new file mode 100644 index 0000000..347be4f --- /dev/null +++ b/ansible/roles/firewall/templates/watchtower.nft.j2 @@ -0,0 +1,102 @@ +#!/usr/sbin/nft -f +# Watchtower host firewall — explicit nftables ruleset. +# +# Layered defense: +# 1. Hetzner Cloud Firewall (edge) — allows ssh + wg from admin CIDRs +# 2. This nftables ruleset (host) — same policy + state tracking + rate limits +# 3. fail2ban — adds dynamic bans into table inet f2b-table +# 4. WireGuard — only authenticated peers reach services +# 5. Podman PublishPort=10.8.0.1:* — services bound to wg0 only +# +# Reload with: systemctl reload nftables +# Test parse with: nft --check --file /etc/nftables/watchtower.nft + +# Flush only our own table so we don't disturb fail2ban or podman tables. +table inet watchtower +delete table inet watchtower + +table inet watchtower { + + set admin_v4 { + type ipv4_addr + flags interval + elements = { +{% for cidr in admin_allow_ipv4 %} + {{ cidr }}{% if not loop.last %},{% endif %} + +{% endfor %} + } + } + + set admin_v6 { + type ipv6_addr + flags interval + elements = { +{% for cidr in admin_allow_ipv6 %} + {{ cidr }}{% if not loop.last %},{% endif %} + +{% endfor %} + } + } + + chain input { + type filter hook input priority filter; policy drop; + + # Stateful base + ct state established,related accept + ct state invalid drop + + # Loopback + iif "lo" accept + + # ICMP / ICMPv6 (rate-limited) + ip protocol icmp limit rate 20/second accept + ip6 nexthdr icmpv6 limit rate 20/second accept + + # Anything arriving on WireGuard is treated as trusted (only authenticated + # peers can reach this interface). Service ports are bound to 10.8.0.1. + iifname "{{ wireguard_interface }}" accept + + # Podman bridge networks: allow the kernel to deliver packets the + # netavark/aardvark stack creates for inter-container DNS, etc. + # Podman manages its own filter rules in separate tables. + iifname "podman*" accept + iifname "cni-*" accept + + # SSH from admin CIDRs only — short-burst rate limit deters scanners. + # fail2ban will additionally ban repeat offenders. + ip saddr @admin_v4 tcp dport 22 ct state new limit rate 6/minute accept + ip6 saddr @admin_v6 tcp dport 22 ct state new limit rate 6/minute accept + + # WireGuard listen port from admin CIDRs. + ip saddr @admin_v4 udp dport {{ wireguard_listen_port }} accept + ip6 saddr @admin_v6 udp dport {{ wireguard_listen_port }} accept + + # Log a sample of dropped packets so we know if something is mis-routed. + limit rate 5/minute log prefix "nft-drop-input: " level info + counter drop + } + + chain forward { + type filter hook forward priority filter; policy drop; + ct state established,related accept + ct state invalid drop + + # Allow forwarding for Podman-managed bridges. Podman/netavark also + # installs its own rules; this is a safety allow-list. + iifname "podman*" accept + oifname "podman*" accept + iifname "cni-*" accept + oifname "cni-*" accept + + # WireGuard egress to services on the host loopback / podman: + iifname "{{ wireguard_interface }}" accept + oifname "{{ wireguard_interface }}" accept + + counter drop + } + + chain output { + type filter hook output priority filter; policy accept; + } +} diff --git a/ansible/roles/luks/tasks/main.yml b/ansible/roles/luks/tasks/main.yml new file mode 100644 index 0000000..a72016f --- /dev/null +++ b/ansible/roles/luks/tasks/main.yml @@ -0,0 +1,110 @@ +--- +# LUKS2 encrypted volume for Mimir/Loki/Tempo WALs and Grafana DB. +# +# CPX42 ships with a single root disk on /dev/sda. We allocate a sparse-backed +# loop file in /var/lib/observability-luks.img if no second device is present. +# Override luks_device_source and luks_use_loop_file in group_vars to use a real +# block device (e.g. an attached Hetzner Volume). + +- name: Stat configured LUKS source device + ansible.builtin.stat: + path: "{{ luks_device_source }}" + register: luks_device_stat + +- name: Decide whether to use a loop-backed LUKS file + ansible.builtin.set_fact: + luks_use_loop_file: "{{ not luks_device_stat.stat.exists }}" + +- name: Create loop-backed LUKS image (if no real device) + when: luks_use_loop_file + block: + - name: Allocate sparse 80G image + ansible.builtin.command: > + truncate -s 80G /var/lib/observability-luks.img + args: + creates: /var/lib/observability-luks.img + + - name: Set permissions on LUKS image + ansible.builtin.file: + path: /var/lib/observability-luks.img + mode: "0600" + owner: root + group: root + + - name: Set effective LUKS source to loop file + ansible.builtin.set_fact: + luks_effective_source: /var/lib/observability-luks.img + +- name: Set effective LUKS source (real device) + when: not luks_use_loop_file + ansible.builtin.set_fact: + luks_effective_source: "{{ luks_device_source }}" + +- name: Generate LUKS keyfile + ansible.builtin.command: dd if=/dev/urandom of=/root/observability.luks.key bs=4096 count=1 + args: + creates: /root/observability.luks.key + +- name: Secure LUKS keyfile + ansible.builtin.file: + path: /root/observability.luks.key + mode: "0400" + owner: root + group: root + +- name: Check if device is already LUKS-formatted + ansible.builtin.command: cryptsetup isLuks "{{ luks_effective_source }}" + register: is_luks + changed_when: false + failed_when: false + +- name: Format LUKS2 + ansible.builtin.command: > + cryptsetup luksFormat --batch-mode --type luks2 + --key-file /root/observability.luks.key + "{{ luks_effective_source }}" + when: is_luks.rc != 0 + +- name: Add crypttab entry + ansible.builtin.lineinfile: + path: /etc/crypttab + regexp: "^{{ luks_mapper_name }} " + line: "{{ luks_mapper_name }} {{ luks_effective_source }} /root/observability.luks.key luks,nofail" + create: true + mode: "0600" + +- name: Open LUKS device + ansible.builtin.command: > + cryptsetup open --key-file /root/observability.luks.key + "{{ luks_effective_source }}" "{{ luks_mapper_name }}" + args: + creates: "/dev/mapper/{{ luks_mapper_name }}" + +- name: Create filesystem on mapped device + community.general.filesystem: + fstype: xfs + dev: "/dev/mapper/{{ luks_mapper_name }}" + +- name: Ensure data root exists + ansible.builtin.file: + path: "{{ observability_data_root }}" + state: directory + mode: "0755" + +- name: Mount data root + ansible.posix.mount: + path: "{{ observability_data_root }}" + src: "/dev/mapper/{{ luks_mapper_name }}" + fstype: xfs + opts: defaults,nofail,x-systemd.requires=systemd-cryptsetup@{{ luks_mapper_name }}.service + state: mounted + +- name: Set SELinux context for container data + community.general.sefcontext: + target: "{{ observability_data_root }}(/.*)?" + setype: container_file_t + state: present + +- name: Apply SELinux context + ansible.builtin.command: restorecon -R "{{ observability_data_root }}" + changed_when: false diff --git a/ansible/roles/observability/files/observability.network b/ansible/roles/observability/files/observability.network new file mode 100644 index 0000000..112ef3f --- /dev/null +++ b/ansible/roles/observability/files/observability.network @@ -0,0 +1,5 @@ +[Network] +NetworkName=observability +Subnet=172.20.0.0/24 +Gateway=172.20.0.1 +DisableDNS=false diff --git a/ansible/roles/observability/handlers/main.yml b/ansible/roles/observability/handlers/main.yml new file mode 100644 index 0000000..66fe0aa --- /dev/null +++ b/ansible/roles/observability/handlers/main.yml @@ -0,0 +1,39 @@ +--- +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true + +- name: Restart mimir + ansible.builtin.systemd: + name: mimir.service + state: restarted + daemon_reload: true + +- name: Restart loki + ansible.builtin.systemd: + name: loki.service + state: restarted + daemon_reload: true + +- name: Restart tempo + ansible.builtin.systemd: + name: tempo.service + state: restarted + daemon_reload: true + +- name: Restart grafana + ansible.builtin.systemd: + name: grafana.service + state: restarted + daemon_reload: true + +- name: Restart all services + ansible.builtin.systemd: + name: "{{ item }}" + state: restarted + daemon_reload: true + loop: + - mimir.service + - loki.service + - tempo.service + - grafana.service diff --git a/ansible/roles/observability/tasks/main.yml b/ansible/roles/observability/tasks/main.yml new file mode 100644 index 0000000..b5ba214 --- /dev/null +++ b/ansible/roles/observability/tasks/main.yml @@ -0,0 +1,158 @@ +--- +- name: Ensure observability config and secret directories + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + mode: "{{ item.mode }}" + owner: root + group: root + loop: + - { path: "{{ observability_secrets_dir }}", mode: "0700" } + - { path: "{{ observability_config_dir }}", mode: "0755" } + - { path: "{{ observability_config_dir }}/mimir", mode: "0755" } + - { path: "{{ observability_config_dir }}/loki", mode: "0755" } + - { path: "{{ observability_config_dir }}/tempo", mode: "0755" } + - { path: "{{ observability_config_dir }}/grafana", mode: "0755" } + - { path: "{{ observability_config_dir }}/grafana/provisioning", mode: "0755" } + - { path: "{{ observability_config_dir }}/grafana/provisioning/datasources", mode: "0755" } + - { path: "{{ observability_config_dir }}/grafana/provisioning/dashboards", mode: "0755" } + - { path: /var/log/observability, mode: "0750" } + - { path: /etc/containers/systemd, mode: "0755" } + +# Container processes run as non-root UIDs. Bind-mounted data dirs must be +# owned by those UIDs or the services cannot create their WAL/index/DB files. +- name: Ensure data directories with correct UIDs + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + mode: "0750" + owner: "{{ item.uid }}" + group: "{{ item.gid }}" + loop: + - { path: "{{ observability_data_root }}/mimir", uid: 10001, gid: 10001 } + - { path: "{{ observability_data_root }}/loki", uid: 10001, gid: 10001 } + - { path: "{{ observability_data_root }}/tempo", uid: 10001, gid: 10001 } + - { path: "{{ observability_data_root }}/grafana", uid: 472, gid: 472 } + +- name: Write S3 credentials env file + ansible.builtin.template: + src: s3.env.j2 + dest: "{{ observability_secrets_dir }}/s3.env" + mode: "0600" + owner: root + group: root + notify: + - Restart mimir + - Restart loki + - Restart tempo + +- name: Render Mimir config + ansible.builtin.template: + src: mimir.yaml.j2 + dest: "{{ observability_config_dir }}/mimir/mimir.yaml" + mode: "0644" + notify: Restart mimir + +- name: Render Loki config + ansible.builtin.template: + src: loki.yaml.j2 + dest: "{{ observability_config_dir }}/loki/loki.yaml" + mode: "0644" + notify: Restart loki + +- name: Render Tempo config + ansible.builtin.template: + src: tempo.yaml.j2 + dest: "{{ observability_config_dir }}/tempo/tempo.yaml" + mode: "0644" + notify: Restart tempo + +- name: Render Grafana config + ansible.builtin.template: + src: grafana.ini.j2 + dest: "{{ observability_config_dir }}/grafana/grafana.ini" + mode: "0640" + notify: Restart grafana + +- name: Render Grafana datasource provisioning + ansible.builtin.template: + src: datasources.yaml.j2 + dest: "{{ observability_config_dir }}/grafana/provisioning/datasources/datasources.yaml" + mode: "0644" + notify: Restart grafana + +- name: Apply SELinux context to config tree + ansible.builtin.command: restorecon -R /etc/observability + changed_when: false + +- name: Deploy Podman network quadlet + ansible.builtin.copy: + src: observability.network + dest: /etc/containers/systemd/observability.network + mode: "0644" + notify: Reload systemd + +- name: Deploy volume quadlets + ansible.builtin.template: + src: "{{ item }}.volume.j2" + dest: "/etc/containers/systemd/{{ item }}.volume" + mode: "0644" + loop: + - mimir-data + - loki-data + - tempo-data + - grafana-data + notify: Reload systemd + +- name: Deploy container quadlets + ansible.builtin.template: + src: "{{ item }}.container.j2" + dest: "/etc/containers/systemd/{{ item }}.container" + mode: "0644" + loop: + - mimir + - loki + - tempo + - grafana + notify: + - Reload systemd + - Restart all services + +- name: Force handlers to flush before health checks + ansible.builtin.meta: flush_handlers + +- name: Wait for Mimir /ready + ansible.builtin.uri: + url: "http://{{ wireguard_server_ip }}:8080/ready" + status_code: 200 + register: mimir_ready + retries: 30 + delay: 5 + until: mimir_ready.status == 200 + +- name: Wait for Loki /ready + ansible.builtin.uri: + url: "http://{{ wireguard_server_ip }}:3100/ready" + status_code: 200 + register: loki_ready + retries: 30 + delay: 5 + until: loki_ready.status == 200 + +- name: Wait for Tempo /ready + ansible.builtin.uri: + url: "http://{{ wireguard_server_ip }}:3200/ready" + status_code: 200 + register: tempo_ready + retries: 30 + delay: 5 + until: tempo_ready.status == 200 + +- name: Wait for Grafana /api/health + ansible.builtin.uri: + url: "http://{{ wireguard_server_ip }}:3000/api/health" + status_code: 200 + register: grafana_ready + retries: 30 + delay: 5 + until: grafana_ready.status == 200 diff --git a/ansible/roles/observability/templates/datasources.yaml.j2 b/ansible/roles/observability/templates/datasources.yaml.j2 new file mode 100644 index 0000000..a4fe954 --- /dev/null +++ b/ansible/roles/observability/templates/datasources.yaml.j2 @@ -0,0 +1,62 @@ +# Grafana datasource provisioning. +# One datasource set per tenant. The X-Scope-OrgID header is what scopes +# Mimir/Loki/Tempo queries to that tenant. Datasources are created in the +# Default Grafana org; per-tenant Grafana orgs (and their own datasources) +# are created by playbooks/onboard-tenant.yml. + +apiVersion: 1 + +datasources: +{% for tenant in tenants %} + - name: "Mimir ({{ tenant.org_name }})" + uid: "mimir-{{ tenant.slug }}" + type: prometheus + access: proxy + url: http://mimir:8080/prometheus + orgId: 1 + isDefault: {{ 'true' if loop.first else 'false' }} + jsonData: + httpMethod: POST + manageAlerts: true + prometheusType: Mimir + httpHeaderName1: "X-Scope-OrgID" + secureJsonData: + httpHeaderValue1: "{{ tenant.slug }}" + editable: false + + - name: "Loki ({{ tenant.org_name }})" + uid: "loki-{{ tenant.slug }}" + type: loki + access: proxy + url: http://loki:3100 + orgId: 1 + jsonData: + httpHeaderName1: "X-Scope-OrgID" + derivedFields: + - datasourceUid: "tempo-{{ tenant.slug }}" + matcherRegex: "trace_id=(\\w+)" + name: TraceID + url: "$${__value.raw}" + secureJsonData: + httpHeaderValue1: "{{ tenant.slug }}" + editable: false + + - name: "Tempo ({{ tenant.org_name }})" + uid: "tempo-{{ tenant.slug }}" + type: tempo + access: proxy + url: http://tempo:3200 + orgId: 1 + jsonData: + httpHeaderName1: "X-Scope-OrgID" + tracesToLogsV2: + datasourceUid: "loki-{{ tenant.slug }}" + spanStartTimeShift: -1m + spanEndTimeShift: 1m + filterByTraceID: true + serviceMap: + datasourceUid: "mimir-{{ tenant.slug }}" + secureJsonData: + httpHeaderValue1: "{{ tenant.slug }}" + editable: false +{% endfor %} diff --git a/ansible/roles/observability/templates/grafana-data.volume.j2 b/ansible/roles/observability/templates/grafana-data.volume.j2 new file mode 100644 index 0000000..60927c6 --- /dev/null +++ b/ansible/roles/observability/templates/grafana-data.volume.j2 @@ -0,0 +1,3 @@ +[Volume] +Type=bind +Device={{ observability_data_root }}/grafana diff --git a/ansible/roles/observability/templates/grafana.container.j2 b/ansible/roles/observability/templates/grafana.container.j2 new file mode 100644 index 0000000..85728f2 --- /dev/null +++ b/ansible/roles/observability/templates/grafana.container.j2 @@ -0,0 +1,34 @@ +[Unit] +Description=Grafana +Wants=network-online.target observability-network.service mimir.service loki.service tempo.service +After=network-online.target observability-network.service mimir.service loki.service tempo.service wg-quick@{{ wireguard_interface }}.service +Requires=wg-quick@{{ wireguard_interface }}.service + +[Container] +Image={{ image_grafana }} +ContainerName=grafana +Network=observability.network +Volume=grafana-data.volume:/var/lib/grafana:Z +Volume={{ observability_config_dir }}/grafana/grafana.ini:/etc/grafana/grafana.ini:ro,Z +Volume={{ observability_config_dir }}/grafana/provisioning:/etc/grafana/provisioning:ro,Z + +Environment=GF_PATHS_DATA=/var/lib/grafana +Environment=GF_PATHS_LOGS=/var/log/grafana +Environment=GF_PATHS_PLUGINS=/var/lib/grafana/plugins +Environment=GF_PATHS_PROVISIONING=/etc/grafana/provisioning +Environment=GF_SECURITY_ADMIN_PASSWORD={{ grafana_admin_password }} + +PublishPort={{ wireguard_server_ip }}:3000:3000 + +User=472 +Group=472 + +PodmanArgs=--memory=2g --memory-swap=2g + +[Service] +Restart=on-failure +RestartSec=10 +TimeoutStartSec=300 + +[Install] +WantedBy=multi-user.target default.target diff --git a/ansible/roles/observability/templates/grafana.ini.j2 b/ansible/roles/observability/templates/grafana.ini.j2 new file mode 100644 index 0000000..41d0910 --- /dev/null +++ b/ansible/roles/observability/templates/grafana.ini.j2 @@ -0,0 +1,74 @@ +app_mode = production + +[paths] +data = /var/lib/grafana +logs = /var/log/grafana +plugins = /var/lib/grafana/plugins +provisioning = /etc/grafana/provisioning + +[server] +protocol = http +http_addr = 0.0.0.0 +http_port = 3000 +domain = {{ wireguard_server_ip }} +root_url = http://{{ wireguard_server_ip }}:3000/ +enforce_domain = false + +[security] +admin_user = admin +disable_initial_admin_creation = false +cookie_secure = false +cookie_samesite = lax +allow_embedding = false +strict_transport_security = false +content_security_policy = true + +[users] +allow_sign_up = false +allow_org_create = false +auto_assign_org = false +viewers_can_edit = false + +[auth] +disable_login_form = false +disable_signout_menu = false + +[auth.anonymous] +enabled = false + +[auth.basic] +enabled = true + +[analytics] +reporting_enabled = false +check_for_updates = false +check_for_plugin_updates = false +feedback_links_enabled = false + +[snapshots] +external_enabled = false + +[log] +mode = console file +level = info + +[auditing] +enabled = true +loggers = file +log_dashboard_content = false +verbose = false + +[auditing.logs.file] +path = /var/log/grafana/audit.log +max_files = 5 +max_file_size_mb = 256 +max_days = 30 + +[unified_alerting] +enabled = true + +[feature_toggles] +enable = traceqlEditor + +[plugins] +allow_loading_unsigned_plugins = diff --git a/ansible/roles/observability/templates/loki-data.volume.j2 b/ansible/roles/observability/templates/loki-data.volume.j2 new file mode 100644 index 0000000..df3ccc0 --- /dev/null +++ b/ansible/roles/observability/templates/loki-data.volume.j2 @@ -0,0 +1,3 @@ +[Volume] +Type=bind +Device={{ observability_data_root }}/loki diff --git a/ansible/roles/observability/templates/loki.container.j2 b/ansible/roles/observability/templates/loki.container.j2 new file mode 100644 index 0000000..2e13e1d --- /dev/null +++ b/ansible/roles/observability/templates/loki.container.j2 @@ -0,0 +1,30 @@ +[Unit] +Description=Grafana Loki +Wants=network-online.target observability-network.service +After=network-online.target observability-network.service wg-quick@{{ wireguard_interface }}.service +Requires=wg-quick@{{ wireguard_interface }}.service + +[Container] +Image={{ image_loki }} +ContainerName=loki +Network=observability.network +Volume=loki-data.volume:/var/lib/loki:Z +Volume={{ observability_config_dir }}/loki/loki.yaml:/etc/loki/loki.yaml:ro,Z +EnvironmentFile={{ observability_secrets_dir }}/s3.env + +PublishPort={{ wireguard_server_ip }}:3100:3100 + +Exec=-config.file=/etc/loki/loki.yaml -config.expand-env=true + +User=10001 +Group=10001 + +PodmanArgs=--memory=6g --memory-swap=6g + +[Service] +Restart=on-failure +RestartSec=10 +TimeoutStartSec=300 + +[Install] +WantedBy=multi-user.target default.target diff --git a/ansible/roles/observability/templates/loki.yaml.j2 b/ansible/roles/observability/templates/loki.yaml.j2 new file mode 100644 index 0000000..20c835b --- /dev/null +++ b/ansible/roles/observability/templates/loki.yaml.j2 @@ -0,0 +1,90 @@ +# Loki monolithic deployment with multi-tenancy and Hetzner S3 backend. + +auth_enabled: true + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: info + +common: + instance_addr: 127.0.0.1 + path_prefix: /var/lib/loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + s3: + endpoint: "{{ hetzner_location }}.your-objectstorage.com" + region: "{{ s3_region }}" + bucketnames: "{{ s3_bucket_loki }}" + access_key_id: "${AWS_ACCESS_KEY_ID}" + secret_access_key: "${AWS_SECRET_ACCESS_KEY}" + s3forcepathstyle: true + insecure: false + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: loki_index_ + period: 24h + +storage_config: + tsdb_shipper: + active_index_directory: /var/lib/loki/tsdb-index + cache_location: /var/lib/loki/tsdb-cache + aws: + endpoint: "{{ hetzner_location }}.your-objectstorage.com" + region: "{{ s3_region }}" + bucketnames: "{{ s3_bucket_loki }}" + s3forcepathstyle: true + insecure: false + +compactor: + working_directory: /var/lib/loki/compactor + delete_request_store: s3 + retention_enabled: true + retention_delete_delay: 2h + retention_delete_worker_count: 150 + compaction_interval: 10m + +# Required to allow the GDPR right-to-erasure flow via Loki's Delete API. +limits_config: + retention_period: {{ loki_retention_days }}d + max_query_lookback: {{ loki_retention_days }}d + max_query_length: 721h + ingestion_rate_mb: 16 + ingestion_burst_size_mb: 32 + reject_old_samples: true + reject_old_samples_max_age: 168h + allow_structured_metadata: true + volume_enabled: true + deletion_mode: filter-and-delete + +ruler: + storage: + type: s3 + s3: + endpoint: "{{ hetzner_location }}.your-objectstorage.com" + region: "{{ s3_region }}" + bucketnames: "{{ s3_bucket_loki }}" + s3forcepathstyle: true + rule_path: /var/lib/loki/rules + alertmanager_url: http://mimir:8080/alertmanager + +analytics: + reporting_enabled: false + +frontend: + encoding: protobuf + +query_range: + cache_results: true + +querier: + max_concurrent: 8 diff --git a/ansible/roles/observability/templates/mimir-data.volume.j2 b/ansible/roles/observability/templates/mimir-data.volume.j2 new file mode 100644 index 0000000..6be7f13 --- /dev/null +++ b/ansible/roles/observability/templates/mimir-data.volume.j2 @@ -0,0 +1,3 @@ +[Volume] +Type=bind +Device={{ observability_data_root }}/mimir diff --git a/ansible/roles/observability/templates/mimir.container.j2 b/ansible/roles/observability/templates/mimir.container.j2 new file mode 100644 index 0000000..2914f32 --- /dev/null +++ b/ansible/roles/observability/templates/mimir.container.j2 @@ -0,0 +1,33 @@ +[Unit] +Description=Grafana Mimir +Wants=network-online.target observability-network.service +After=network-online.target observability-network.service wg-quick@{{ wireguard_interface }}.service +Requires=wg-quick@{{ wireguard_interface }}.service + +[Container] +Image={{ image_mimir }} +ContainerName=mimir +Network=observability.network +Volume=mimir-data.volume:/var/lib/mimir:Z +Volume={{ observability_config_dir }}/mimir/mimir.yaml:/etc/mimir/mimir.yaml:ro,Z +EnvironmentFile={{ observability_secrets_dir }}/s3.env + +# Bind only to WireGuard interface +PublishPort={{ wireguard_server_ip }}:8080:8080 +PublishPort={{ wireguard_server_ip }}:9095:9095 + +# -config.expand-env enables ${AWS_*} substitution from EnvironmentFile. +Exec=-config.file=/etc/mimir/mimir.yaml -config.expand-env=true + +User=10001 +Group=10001 + +PodmanArgs=--memory=8g --memory-swap=8g + +[Service] +Restart=on-failure +RestartSec=10 +TimeoutStartSec=300 + +[Install] +WantedBy=multi-user.target default.target diff --git a/ansible/roles/observability/templates/mimir.yaml.j2 b/ansible/roles/observability/templates/mimir.yaml.j2 new file mode 100644 index 0000000..e89212e --- /dev/null +++ b/ansible/roles/observability/templates/mimir.yaml.j2 @@ -0,0 +1,97 @@ +# Mimir monolithic deployment with multi-tenancy and Hetzner S3 backend. +# Reference: https://grafana.com/docs/mimir/latest/configure/ + +target: all,alertmanager,overrides-exporter + +multitenancy_enabled: true + +server: + http_listen_port: 8080 + grpc_listen_port: 9095 + log_level: info + +common: + storage: + backend: s3 + s3: + endpoint: "{{ hetzner_location }}.your-objectstorage.com" + region: "{{ s3_region }}" + bucket_name: "{{ s3_bucket_mimir }}" + access_key_id: "${AWS_ACCESS_KEY_ID}" + secret_access_key: "${AWS_SECRET_ACCESS_KEY}" + insecure: false + +blocks_storage: + backend: s3 + s3: + bucket_name: "{{ s3_bucket_mimir }}" + storage_prefix: blocks + tsdb: + dir: /var/lib/mimir/tsdb + retention_period: 24h + bucket_store: + sync_dir: /var/lib/mimir/tsdb-sync + +compactor: + data_dir: /var/lib/mimir/compactor + sharding_ring: + kvstore: + store: memberlist + +distributor: + ring: + instance_addr: 127.0.0.1 + kvstore: + store: memberlist + +ingester: + ring: + instance_addr: 127.0.0.1 + kvstore: + store: memberlist + replication_factor: 1 + +ruler: + rule_path: /var/lib/mimir/rules + alertmanager_url: http://localhost:8080/alertmanager + ring: + instance_addr: 127.0.0.1 + +ruler_storage: + backend: s3 + s3: + bucket_name: "{{ s3_bucket_mimir }}" + storage_prefix: ruler + +alertmanager: + data_dir: /var/lib/mimir/alertmanager + external_url: http://{{ wireguard_server_ip }}:8080/alertmanager + sharding_ring: + replication_factor: 1 + +alertmanager_storage: + backend: s3 + s3: + bucket_name: "{{ s3_bucket_mimir }}" + storage_prefix: alertmanager + +store_gateway: + sharding_ring: + replication_factor: 1 + +memberlist: + bind_port: 7946 + +# Per-tenant retention and limits. +limits: + compactor_blocks_retention_period: {{ mimir_retention_days }}d + ingestion_rate: 100000 + ingestion_burst_size: 1000000 + max_global_series_per_user: 500000 + max_label_names_per_series: 30 + # Reject any incoming labels that match common PII patterns. Defense in depth; + # primary scrubbing happens at the Alloy/Promtail collector. + ingestion_partitions_tenant_shard_size: 0 + +runtime_config: + file: /etc/mimir/runtime.yaml diff --git a/ansible/roles/observability/templates/s3.env.j2 b/ansible/roles/observability/templates/s3.env.j2 new file mode 100644 index 0000000..eebd79d --- /dev/null +++ b/ansible/roles/observability/templates/s3.env.j2 @@ -0,0 +1,4 @@ +AWS_ACCESS_KEY_ID={{ hetzner_s3_access_key }} +AWS_SECRET_ACCESS_KEY={{ hetzner_s3_secret_key }} +S3_ENDPOINT={{ s3_endpoint }} +S3_REGION={{ s3_region }} diff --git a/ansible/roles/observability/templates/tempo-data.volume.j2 b/ansible/roles/observability/templates/tempo-data.volume.j2 new file mode 100644 index 0000000..e3244dd --- /dev/null +++ b/ansible/roles/observability/templates/tempo-data.volume.j2 @@ -0,0 +1,3 @@ +[Volume] +Type=bind +Device={{ observability_data_root }}/tempo diff --git a/ansible/roles/observability/templates/tempo.container.j2 b/ansible/roles/observability/templates/tempo.container.j2 new file mode 100644 index 0000000..1da2b42 --- /dev/null +++ b/ansible/roles/observability/templates/tempo.container.j2 @@ -0,0 +1,32 @@ +[Unit] +Description=Grafana Tempo +Wants=network-online.target observability-network.service +After=network-online.target observability-network.service wg-quick@{{ wireguard_interface }}.service +Requires=wg-quick@{{ wireguard_interface }}.service + +[Container] +Image={{ image_tempo }} +ContainerName=tempo +Network=observability.network +Volume=tempo-data.volume:/var/lib/tempo:Z +Volume={{ observability_config_dir }}/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro,Z +EnvironmentFile={{ observability_secrets_dir }}/s3.env + +PublishPort={{ wireguard_server_ip }}:3200:3200 +PublishPort={{ wireguard_server_ip }}:4317:4317 +PublishPort={{ wireguard_server_ip }}:4318:4318 + +Exec=-config.file=/etc/tempo/tempo.yaml -config.expand-env=true + +User=10001 +Group=10001 + +PodmanArgs=--memory=5g --memory-swap=5g + +[Service] +Restart=on-failure +RestartSec=10 +TimeoutStartSec=300 + +[Install] +WantedBy=multi-user.target default.target diff --git a/ansible/roles/observability/templates/tempo.yaml.j2 b/ansible/roles/observability/templates/tempo.yaml.j2 new file mode 100644 index 0000000..2d5daa7 --- /dev/null +++ b/ansible/roles/observability/templates/tempo.yaml.j2 @@ -0,0 +1,73 @@ +# Tempo single-binary deployment with multi-tenancy and Hetzner S3 backend. + +multitenancy_enabled: true + +server: + http_listen_port: 3200 + grpc_listen_port: 9097 + log_level: info + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + jaeger: + protocols: + thrift_http: + endpoint: 0.0.0.0:14268 + +ingester: + max_block_duration: 5m + trace_idle_period: 10s + +compactor: + compaction: + block_retention: {{ tempo_retention_days * 24 }}h + compacted_block_retention: 1h + compaction_window: 1h + max_block_bytes: 100000000 + retention_concurrency: 16 + +storage: + trace: + backend: s3 + wal: + path: /var/lib/tempo/wal + s3: + endpoint: "{{ hetzner_location }}.your-objectstorage.com" + region: "{{ s3_region }}" + bucket: "{{ s3_bucket_tempo }}" + access_key: "${AWS_ACCESS_KEY_ID}" + secret_key: "${AWS_SECRET_ACCESS_KEY}" + forcepathstyle: true + insecure: false + pool: + max_workers: 100 + queue_depth: 10000 + +metrics_generator: + registry: + external_labels: + source: tempo + cluster: watchtower + storage: + path: /var/lib/tempo/generator/wal + remote_write: + - url: http://mimir:8080/api/v1/push + send_exemplars: true + +overrides: + defaults: + metrics_generator: + processors: [service-graphs, span-metrics] + +usage_report: + reporting_enabled: false + +querier: + frontend_worker: + frontend_address: 127.0.0.1:9097 diff --git a/ansible/roles/wireguard/handlers/main.yml b/ansible/roles/wireguard/handlers/main.yml new file mode 100644 index 0000000..1908929 --- /dev/null +++ b/ansible/roles/wireguard/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart wireguard + ansible.builtin.systemd: + name: "wg-quick@{{ wireguard_interface }}" + state: restarted diff --git a/ansible/roles/wireguard/tasks/main.yml b/ansible/roles/wireguard/tasks/main.yml new file mode 100644 index 0000000..1cc18ea --- /dev/null +++ b/ansible/roles/wireguard/tasks/main.yml @@ -0,0 +1,31 @@ +--- +- name: Ensure /etc/wireguard exists + ansible.builtin.file: + path: /etc/wireguard + state: directory + mode: "0700" + owner: root + group: root + +- name: Render wg0.conf + ansible.builtin.template: + src: wg0.conf.j2 + dest: "/etc/wireguard/{{ wireguard_interface }}.conf" + mode: "0600" + owner: root + group: root + notify: Restart wireguard + +- name: Enable IPv4 forwarding + ansible.posix.sysctl: + name: net.ipv4.ip_forward + value: "1" + sysctl_set: true + state: present + reload: true + +- name: Enable wg-quick service + ansible.builtin.systemd: + name: "wg-quick@{{ wireguard_interface }}" + enabled: true + state: started diff --git a/ansible/roles/wireguard/templates/wg0.conf.j2 b/ansible/roles/wireguard/templates/wg0.conf.j2 new file mode 100644 index 0000000..241fc9d --- /dev/null +++ b/ansible/roles/wireguard/templates/wg0.conf.j2 @@ -0,0 +1,16 @@ +[Interface] +Address = {{ wireguard_server_address }} +ListenPort = {{ wireguard_listen_port }} +PrivateKey = {{ wireguard_server_private_key }} +SaveConfig = false + +{% for peer in wireguard_peers %} +# Peer: {{ peer.name }} +[Peer] +PublicKey = {{ peer.public_key }} +AllowedIPs = {{ peer.allowed_ips }} +{% if peer.preshared_key is defined %} +PresharedKey = {{ peer.preshared_key }} +{% endif %} + +{% endfor %} diff --git a/config/alloy/config.alloy.example b/config/alloy/config.alloy.example new file mode 100644 index 0000000..6c68a99 --- /dev/null +++ b/config/alloy/config.alloy.example @@ -0,0 +1,118 @@ +# Grafana Alloy collector configuration for clients shipping telemetry into +# the watchtower stack. Run this on each client host (over WireGuard). +# +# Defense-in-depth PII redaction before forwarding to Loki/Tempo/Mimir. +# Replace TENANT_SLUG and 10.8.0.1 with your tenant id and watchtower wg IP. + +logging { + level = "info" + format = "logfmt" +} + +// ============================================================ +// LOGS ──► Loki +// ============================================================ + +loki.write "default" { + endpoint { + url = "http://10.8.0.1:3100/loki/api/v1/push" + headers = { + "X-Scope-OrgID" = "TENANT_SLUG", + } + } +} + +loki.source.journal "system" { + forward_to = [loki.process.scrub.receiver] + relabel_rules = loki.relabel.system.rules + labels = { job = "systemd-journal" } +} + +loki.relabel "system" { + forward_to = [] + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + rule { + source_labels = ["__journal__hostname"] + target_label = "host" + } +} + +loki.process "scrub" { + forward_to = [loki.write.default.receiver] + + stage.replace { expression = `\b(?:\d{1,3}\.){3}\d{1,3}\b` replace = "[REDACTED-IP]" } + stage.replace { expression = `\b(?:[0-9a-fA-F]{1,4}:){2,7}[0-9a-fA-F]{1,4}\b` replace = "[REDACTED-IP6]" } + stage.replace { expression = `\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b` replace = "[REDACTED-EMAIL]" } + stage.replace { expression = `\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b` replace = "[REDACTED-JWT]" } + stage.replace { expression = `(?i)(authorization:\s*bearer\s+)[A-Za-z0-9._-]+` replace = "$1[REDACTED-TOKEN]" } + stage.replace { expression = `\b(?:\d[ -]*?){13,19}\b` replace = "[REDACTED-PAN]" } + stage.replace { expression = `\b[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b` replace = "[REDACTED-UUID]" } + + stage.match { + selector = `{job=~".+"} |~ "(?i)(password|ssn|cvv|cvc)\\s*[:=]"` + action = "drop" + } +} + +// ============================================================ +// TRACES ──► Tempo (OTLP) +// ============================================================ + +otelcol.receiver.otlp "default" { + grpc { endpoint = "0.0.0.0:4317" } + http { endpoint = "0.0.0.0:4318" } + output { + traces = [otelcol.processor.attributes.scrub.input] + metrics = [otelcol.processor.attributes.scrub.input] + logs = [otelcol.processor.attributes.scrub.input] + } +} + +otelcol.processor.attributes "scrub" { + action { key = "enduser.id" action = "hash" } + action { key = "user.email" action = "delete" } + action { key = "client.address" action = "delete" } + action { key = "http.request.header.authorization" action = "delete" } + action { key = "http.request.header.cookie" action = "delete" } + action { key = "http.request.body" action = "delete" } + + output { + traces = [otelcol.exporter.otlp.tempo.input] + metrics = [otelcol.exporter.otlphttp.mimir.input] + } +} + +otelcol.exporter.otlp "tempo" { + client { + endpoint = "10.8.0.1:4317" + tls { insecure = true } + headers = { "X-Scope-OrgID" = "TENANT_SLUG" } + } +} + +// ============================================================ +// METRICS ──► Mimir +// ============================================================ + +prometheus.remote_write "mimir" { + endpoint { + url = "http://10.8.0.1:8080/api/v1/push" + headers = { "X-Scope-OrgID" = "TENANT_SLUG" } + } + external_labels = { tenant = "TENANT_SLUG" } +} + +otelcol.exporter.otlphttp "mimir" { + client { + endpoint = "http://10.8.0.1:8080/otlp" + headers = { "X-Scope-OrgID" = "TENANT_SLUG" } + } +} + +prometheus.scrape "node" { + targets = [{"__address__" = "127.0.0.1:9100"}] + forward_to = [prometheus.remote_write.mimir.receiver] +} diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..c27d06b --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,82 @@ +# Architecture + +## Overview + +Watchtower is a single-node observability stack running on a Hetzner CPX42 in +Nuremberg (nbg1). It hosts Mimir (metrics), Loki (logs), Tempo (traces), and +Grafana (UI). Clients ship telemetry over a WireGuard VPN; nothing is exposed +to the public internet. + +## Diagram + +``` + Internet + │ + ┌───────────┴───────────┐ + │ Hetzner Cloud Firewall│ allow: 22/tcp, 51820/udp + └───────────┬───────────┘ + │ + ┌──────┴──────┐ + │ AlmaLinux 10│ + │ CPX42 │ + │ │ + │ nftables │ default-drop input + fail2ban + │ ┌──────────┴──────────┐ + │ │ wg0 (10.8.0.1/24) │ ◄── WireGuard peers + │ └──┬──────────────────┘ + │ │ all service ports bind to 10.8.0.1 + │ ┌──┴──────────────────────┐ + │ │ Podman observability │ + │ │ network (172.20.0.0/24) │ + │ │ mimir :8080 :9095 │ + │ │ loki :3100 │ + │ │ tempo :3200/4317/4318│ + │ │ grafana :3000 │ + │ └──┬──────────────────────┘ + │ │ + │ /var/lib/observability ◄── LUKS2 (xfs) + └─────┘ + │ S3 API (HTTPS) + ▼ + Hetzner Object Storage (nbg1) + - watchtower-mimir + - watchtower-loki + - watchtower-tempo +``` + +## Data flow + +1. **Clients** run Grafana Alloy with PII scrubbing pipelines (see + `config/alloy/config.alloy.example`). +2. Alloy ships **logs → Loki**, **traces → Tempo OTLP**, **metrics → Mimir + remote_write**, all over WireGuard with `X-Scope-OrgID` headers. +3. Mimir/Loki/Tempo write blocks/chunks to Hetzner Object Storage. WALs and + compactor scratch live on the LUKS-encrypted local volume. +4. Grafana queries Mimir/Loki/Tempo via the `observability` Podman bridge + (container DNS), with tenant header injected per Org's datasource config. + +## Components and ports + +| Service | Container | Bind | Public? | +|----------|-----------|------------------|---------| +| Mimir | mimir | 10.8.0.1:8080,9095| no | +| Loki | loki | 10.8.0.1:3100 | no | +| Tempo | tempo | 10.8.0.1:3200, OTLP 4317/4318 | no | +| Grafana | grafana | 10.8.0.1:3000 | no | + +## Storage + +| Layer | Backend | Encryption | +|-------------|----------------|-----------------------------| +| Long-term | Hetzner S3 | Server-side (Hetzner SSE) | +| Local WAL | LUKS2 + xfs | At-rest (LUKS2 on /dev/sdX or loop) | +| Transit | WireGuard | ChaCha20-Poly1305 | + +## Lifecycle + +- **OpenTofu** provisions the server, firewall, and S3 buckets (state in + Cloudflare R2). +- **Ansible** configures AlmaLinux, LUKS, nftables + fail2ban, WireGuard, and deploys + the Podman Quadlets. +- **systemd Quadlet generator** turns `*.container` files into + `mimir.service`, `loki.service`, `tempo.service`, `grafana.service`. diff --git a/docs/gdpr-compliance.md b/docs/gdpr-compliance.md new file mode 100644 index 0000000..0e30da6 --- /dev/null +++ b/docs/gdpr-compliance.md @@ -0,0 +1,137 @@ +# GDPR Compliance + +This stack processes telemetry that is **likely to contain personal data** +(IPs, user IDs, request bodies, emails). The deployment is hosted by Hetzner +in Nuremberg, Germany — all data stays within the EU. + +## Roles + +- **Data Controller:** the operator deploying this stack. +- **Data Processor (infrastructure):** Hetzner Online GmbH (DE). +- **Sub-processors:** none by default. + +## Legal basis (Art. 6) + +Document the basis for each tenant under +`docs/gdpr-compliance.md` (per-tenant section to be filled in). Typical bases: + +- **Art. 6(1)(f)** legitimate interest — operational logging. +- **Art. 6(1)(b)** contract performance — telemetry necessary to deliver service. + +## Required documents + +| Document | Status | Notes | +|----------|--------|-------| +| Data Processing Agreement with Hetzner | sign in Cloud Console | required Art. 28 | +| Record of Processing Activities (RoPA) | template below | required Art. 30 | +| Data Protection Impact Assessment (DPIA) | required | Art. 35 — high-risk processing | +| Breach response runbook | `docs/runbook.md` | Art. 33/34 | + +## Data residency + +All telemetry buckets live in `nbg1` (Hetzner, Nuremberg). The OpenTofu +state lives in Cloudflare R2 — operators must configure R2 to use only EU +jurisdictions (`Jurisdiction: EU`) when creating the state bucket, otherwise +state metadata may be replicated outside the EU. + +## Retention register + +| Signal | Application Retention | Bucket Lifecycle | Effective Erasure Window | +|---------|----------------------|------------------|--------------------------| +| Logs | 30 days (Loki) | 90 days (S3 lifecycle) | 30 days (compactor) | +| Metrics | 90 days (Mimir) | 365 days (S3 lifecycle)| 90 days (compactor) | +| Traces | 14 days (Tempo) | 30 days (S3 lifecycle) | 14 days (compactor) | + +Bucket-level lifecycle is a **safety net** — application-level retention is +the primary erasure mechanism. The wider lifecycle window allows +application-level rollback in case of a misconfiguration. + +## PII minimization + +### Collector-side scrubbing + +Clients run Grafana Alloy with the redaction pipeline at +`config/alloy/config.alloy.example`. Patterns redacted: + +- IPv4 / IPv6 addresses → `[REDACTED-IP*]` +- Email addresses → `[REDACTED-EMAIL]` +- JWT tokens → `[REDACTED-JWT]` +- Bearer tokens in `Authorization` headers → `[REDACTED-TOKEN]` +- Card numbers (PAN, 13–19 digits) → `[REDACTED-PAN]` +- UUIDs → `[REDACTED-UUID]` +- Lines containing `password=`, `ssn=`, `cvv=`, `cvc=` → **dropped** + +### OTLP attribute scrubbing + +The `otelcol.processor.attributes` block hashes/deletes these attributes +before traces reach Tempo: + +| Attribute | Action | +|-----------|--------| +| `enduser.id` | hashed (correlatable but not reversible) | +| `user.email` | deleted | +| `client.address` | deleted | +| `http.request.header.authorization` | deleted | +| `http.request.header.cookie` | deleted | +| `http.request.body` | deleted | + +### Label / cardinality discipline + +Mimir and Tempo do **not** support per-record deletion. To stay compliant, +**no PII may ever appear in metric labels or trace resource attributes**. +Tenants must use opaque identifiers (UUIDs hashed at the collector, opaque +session IDs) — never emails, names, or phone numbers as label values. + +## Right to erasure (Art. 17) + +| Signal | Mechanism | RTO | +|---------|-----------|-----| +| Logs | Loki Delete API via `playbooks/gdpr-purge.yml` | hours | +| Metrics | Retention only — short window + label discipline | ≤90 days | +| Traces | Retention only — 14-day window | ≤14 days | + +Document each erasure in `/var/log/observability/gdpr-audit.log` (the playbook +appends automatically). Notify the data subject within one month per Art. 12(3). + +## Encryption + +| Layer | Mechanism | +|-------|-----------| +| In transit (client → server) | WireGuard (ChaCha20-Poly1305) | +| In transit (server → S3) | TLS 1.2+ | +| At rest (local WAL/Grafana DB) | LUKS2 | +| At rest (S3) | Hetzner SSE | + +## Access control & audit + +- WireGuard: each peer has its own key; revoke by removing from `wg0.conf`. +- Host firewall: nftables with default-drop input policy. Service ports are + bound to `10.8.0.1` (wg0) and unreachable on the public interface. SSH and + WireGuard are restricted to administrator CIDRs (`admin_allow_ipv4` / + `admin_allow_ipv6`) and rate-limited at the kernel. +- fail2ban: brute-force defense for SSH (jail `sshd`) plus a `recidive` jail + that long-bans repeat offenders. Bans are enforced via fail2ban's own + nftables table (`inet f2b-table`). +- Grafana: `[auditing]` enabled — all dashboard / API actions logged to + `/var/log/grafana/audit.log` with rotation. +- SSH: ed25519 keys only, root password auth disabled. +- Tenant isolation: enforced by `X-Scope-OrgID` at Mimir/Loki/Tempo plus + Grafana Org RBAC. + +## Breach response (Art. 33/34) + +72-hour notification to the **Bayerisches Landesamt für Datenschutzaufsicht +(BayLDA)** — Bavaria's supervisory authority. Procedure documented in +`docs/runbook.md` § Breach Response. + +## RoPA template + +For each tenant, fill in: + +- Name & contact of controller +- Categories of data subjects +- Categories of personal data processed in telemetry +- Recipients (none — internal only) +- Transfers outside EEA (none) +- Retention periods (per signal — see above) +- Technical & organisational measures (this document) diff --git a/docs/multi-tenancy.md b/docs/multi-tenancy.md new file mode 100644 index 0000000..bf8e3af --- /dev/null +++ b/docs/multi-tenancy.md @@ -0,0 +1,60 @@ +# Multi-tenancy + +This stack is multi-tenant: multiple teams or projects share the same Mimir, +Loki, Tempo, and Grafana instances. Tenant isolation is enforced at two layers: + +1. **Storage layer** — Mimir/Loki/Tempo route reads and writes by the + `X-Scope-OrgID` HTTP header. Each tenant has a unique slug. +2. **Grafana layer** — one Org per tenant. Each Org has its own datasources + that inject the tenant's `X-Scope-OrgID` automatically. + +## Tenant slug rules + +- Lowercase ASCII letters, digits, and dashes only. +- Max 40 characters. +- Must be unique across the deployment. +- Examples: `team-alpha`, `web-prod`, `customer-acme`. + +## Onboarding + +```bash +ansible-playbook ansible/playbooks/onboard-tenant.yml \ + -e tenant_slug=team-alpha \ + -e tenant_name='Team Alpha' \ + -e admin_email=alpha-admin@example.com +``` + +This creates a Grafana Org named "Team Alpha", switches into it, and +provisions Mimir/Loki/Tempo datasources whose secure HTTP header +`X-Scope-OrgID = team-alpha` is set. + +## Client configuration + +Clients use Grafana Alloy with the tenant slug embedded: + +```alloy +loki.write "default" { + endpoint { + url = "http://10.8.0.1:3100/loki/api/v1/push" + headers = { "X-Scope-OrgID" = "team-alpha" } + } +} +``` + +See `config/alloy/config.alloy.example`. + +## Offboarding + +1. Submit Loki delete requests for the tenant's logs: + `ansible-playbook playbooks/gdpr-purge.yml -e tenant=team-alpha -e selector='{job=~".+"}' -e start=2020-01-01T00:00:00Z` +2. Stop ingesting (revoke the tenant's WireGuard peer + remove their Alloy + instance). +3. Delete the Grafana Org via the Grafana API. +4. Wait for retention to lapse on Mimir/Tempo (90/14 days respectively) — they + have no per-tenant deletion API on this version. + +## Per-tenant overrides + +Mimir limits, Loki retention, etc. can be customized per tenant via +`runtime.yaml` (Mimir) or `limits_config` overrides (Loki). Add per-tenant +sections under `overrides:` in each config — see upstream docs for the schema. diff --git a/docs/runbook.md b/docs/runbook.md new file mode 100644 index 0000000..9dba4e0 --- /dev/null +++ b/docs/runbook.md @@ -0,0 +1,213 @@ +# Runbook + +## Day-to-day operations + +### Start / stop services + +```bash +systemctl start mimir loki tempo grafana +systemctl stop mimir loki tempo grafana +systemctl status mimir +journalctl -u mimir -f +``` + +### Reload config + +After editing a config template: + +```bash +ansible-playbook ansible/playbooks/deploy.yml --ask-vault-pass +``` + +The handler restarts only services whose config changed. + +### View logs + +```bash +journalctl -u loki --since '1 hour ago' +podman logs -f loki +``` + +## WireGuard + +### Add a peer + +1. Peer generates a key pair: `wg genkey | tee priv | wg pubkey > pub`. +2. Add to `ansible/group_vars/all/vault.yml` (vault-encrypted): + ```yaml + vault_wireguard_peers: + - name: alice-laptop + public_key: + allowed_ips: 10.8.0.11/32 + ``` +3. `ansible-playbook playbooks/site.yml --tags wireguard --ask-vault-pass` +4. Send peer their config (including server public key + endpoint). + +### Revoke a peer + +1. Remove the peer block from vault. +2. Re-run the wireguard role. +3. Live revocation: `wg set wg0 peer remove`. + +## GDPR right to erasure + +```bash +ansible-playbook ansible/playbooks/gdpr-purge.yml \ + -e tenant=team-alpha \ + -e 'selector={user_id="abc-123"}' \ + -e start=2025-01-01T00:00:00Z \ + --ask-vault-pass +``` + +The Loki compactor processes deletes after `retention_delete_delay = 2h`. +Confirm completion via: + +```bash +curl -H 'X-Scope-OrgID: team-alpha' \ + 'http://10.8.0.1:3100/loki/api/v1/delete' +``` + +Audit log: `/var/log/observability/gdpr-audit.log`. + +## Breach response + +1. **Detect** — alert from Grafana audit log or Hetzner abuse notice. +2. **Contain** — block the affected WireGuard peer, rotate Hetzner Object + Storage credentials (Cloud Console → Security → Generate new keys), update + `vault.yml`, redeploy. +3. **Assess** — what tenant, what data, what window. Use Grafana audit log + and journald for forensics. +4. **Notify** — within 72 hours notify **BayLDA** + (https://www.lda.bayern.de/). If high risk to data subjects, notify them + directly per Art. 34. +5. **Document** — append to `/var/log/observability/breach-log.md`. + +## Recovery / DR + +- All telemetry is reproducible from clients — no need to back up local WAL. +- Grafana DB (dashboards, users, orgs): nightly export via + `scripts/backup-grafana.sh` to S3 (`watchtower-grafana-backup`). +- OpenTofu state: versioned in Cloudflare R2 with object versioning enabled. +- Server replacement procedure: + 1. `tofu apply -replace=hcloud_server.watchtower` + 2. `ansible-playbook playbooks/site.yml` + 3. Restore Grafana DB from backup. + 4. Buckets are unchanged → telemetry resumes ingesting. + +## Common issues + +| Symptom | Likely cause | Fix | +|---------|--------------|-----| +| Service won't start, `cannot assign requested address` | wg0 not up | `systemctl status wg-quick@wg0`; restart | +| Mimir 403 from S3 | bad keys in `s3.env` | regenerate Hetzner keys, redeploy | +| Loki "no org id" | client missing X-Scope-OrgID header | check Alloy config | +| LUKS volume not mounted after reboot | keyfile missing | check `/root/observability.luks.key` exists, re-run luks role | +| Grafana login redirects loop | cookie_secure=true behind plain HTTP | leave cookie_secure=false (VPN provides encryption) | + +## Host firewall (nftables) + +### Inspect ruleset +```bash +# Watchtower's own table: +nft list table inet watchtower + +# Everything (includes fail2ban's table inet f2b-table and Podman's tables): +nft list ruleset +``` + +### Reload after editing +The ruleset is rendered to `/etc/nftables/watchtower.nft` and loaded via +`/etc/sysconfig/nftables.conf`. + +```bash +# Validate before applying: +nft --check --file /etc/nftables/watchtower.nft + +# Apply: +systemctl reload nftables # or: systemctl restart nftables +``` + +### Drop counters / log +Drops are logged at `info` level (rate-limited 5/min) with prefix `nft-drop-input:`. + +```bash +journalctl -k --since '1 hour ago' | grep nft-drop-input +nft list table inet watchtower | grep counter +``` + +### Adjust admin allow lists +Edit `ansible/group_vars/all/main.yml` (`admin_allow_ipv4`, `admin_allow_ipv6`) and +re-run `ansible-playbook ansible/playbooks/site.yml --tags firewall` (or just rerun +the whole site playbook). + +## fail2ban + +### Status +```bash +fail2ban-client status # list of active jails +fail2ban-client status sshd # per-jail: banned IPs, totals +fail2ban-client status recidive +``` + +### Unban an IP +```bash +fail2ban-client set sshd unbanip 203.0.113.42 +fail2ban-client set recidive unbanip 203.0.113.42 +``` + +### Inspect bans at the kernel level +fail2ban uses its own nftables table — bans are sets inside `inet f2b-table`: + +```bash +nft list table inet f2b-table +``` + +### Logs +```bash +tail -f /var/log/fail2ban.log +journalctl -u fail2ban -f +``` + +### Tuning +Defaults live in `ansible/roles/fail2ban/defaults/main.yml`. Override in +`group_vars/all/main.yml` — e.g. shorten `fail2ban_findtime`, raise +`fail2ban_maxretry`, or extend `fail2ban_recidive_bantime`. + +## Automatic security updates + +Configured via `dnf-automatic` (`roles/base`): + +- **Scope**: `upgrade_type = security` only — feature/major version bumps are not auto-applied. +- **Apply + reboot**: `apply_updates = yes`, `reboot = when-needed` (kernel/glibc/systemd + triggers `shutdown -r +5`). +- **Schedule**: `dnf-automatic-install.timer`, pinned via drop-in to `01:00 America/Chicago` (follows CST/CDT automatically) + with up to 30m random delay. +- **Excluded packages** (operator applies these manually): `podman*`, `conmon`, + `crun`, `containers-common`, `netavark`, `aardvark-dns`, `container-selinux` — + upgrading these mid-day would restart the observability stack. + +### Inspect +```bash +systemctl list-timers dnf-automatic-install.timer +systemctl status dnf-automatic-install.service +journalctl -u dnf-automatic-install.service --since '2 days ago' +dnf updateinfo list security # what's pending +dnf needs-restarting -r # is a reboot needed? +``` + +### Run on demand +```bash +systemctl start dnf-automatic-install.service +``` + +### Update the excluded container packages (manual) +Schedule a maintenance window, then: +```bash +dnf -y upgrade podman conmon crun containers-common netavark aardvark-dns container-selinux +systemctl restart mimir loki tempo grafana +``` + +### Disable temporarily +```bash +systemctl disable --now dnf-automatic-install.timer +``` diff --git a/tofu/backend.hcl.example b/tofu/backend.hcl.example new file mode 100644 index 0000000..b472aed --- /dev/null +++ b/tofu/backend.hcl.example @@ -0,0 +1,12 @@ +# Initialize OpenTofu state in Cloudflare R2: +# +# export AWS_ACCESS_KEY_ID= +# export AWS_SECRET_ACCESS_KEY= +# tofu init -backend-config=backend.hcl +# +# Replace with your Cloudflare account ID. +# Create the R2 bucket "watchtower-tofu-state" in Cloudflare dashboard first. + +endpoints = { + s3 = "https://.r2.cloudflarestorage.com" +} diff --git a/tofu/buckets.tf b/tofu/buckets.tf new file mode 100644 index 0000000..7f367db --- /dev/null +++ b/tofu/buckets.tf @@ -0,0 +1,61 @@ +locals { + buckets = { + mimir = { + name = "${var.bucket_prefix}-mimir" + retention_days = 365 + } + loki = { + name = "${var.bucket_prefix}-loki" + retention_days = 90 + } + tempo = { + name = "${var.bucket_prefix}-tempo" + retention_days = 30 + } + } +} + +resource "aws_s3_bucket" "telemetry" { + for_each = local.buckets + provider = aws.hetzner + + bucket = each.value.name + + # Hetzner does not yet support all S3 ACL operations, so keep this minimal. + lifecycle { + prevent_destroy = true + } +} + +resource "aws_s3_bucket_versioning" "telemetry" { + for_each = local.buckets + provider = aws.hetzner + + bucket = aws_s3_bucket.telemetry[each.key].id + versioning_configuration { + status = "Suspended" + } +} + +# Lifecycle rule: hard-delete safety net behind application retention. +resource "aws_s3_bucket_lifecycle_configuration" "telemetry" { + for_each = local.buckets + provider = aws.hetzner + + bucket = aws_s3_bucket.telemetry[each.key].id + + rule { + id = "hard-delete-after-${each.value.retention_days}d" + status = "Enabled" + + filter {} + + expiration { + days = each.value.retention_days + } + + abort_incomplete_multipart_upload { + days_after_initiation = 7 + } + } +} diff --git a/tofu/cloud-init.yaml b/tofu/cloud-init.yaml new file mode 100644 index 0000000..c892ff8 --- /dev/null +++ b/tofu/cloud-init.yaml @@ -0,0 +1,27 @@ +#cloud-config +# Minimal bootstrap: ensure python3 (for Ansible) and basic hardening. +# Everything else is handled by Ansible. + +package_update: true +package_upgrade: true + +packages: + - python3 + - python3-libselinux + - dnf-plugins-core + +ssh_pwauth: false +disable_root: false + +write_files: + - path: /etc/ssh/sshd_config.d/10-watchtower.conf + permissions: "0644" + content: | + PasswordAuthentication no + PermitRootLogin prohibit-password + PubkeyAuthentication yes + KbdInteractiveAuthentication no + +runcmd: + - systemctl restart sshd + - timedatectl set-timezone Europe/Berlin diff --git a/tofu/outputs.tf b/tofu/outputs.tf new file mode 100644 index 0000000..51faaee --- /dev/null +++ b/tofu/outputs.tf @@ -0,0 +1,38 @@ +output "server_ipv4" { + description = "Public IPv4 of the watchtower server." + value = hcloud_primary_ip.ipv4.ip_address +} + +output "server_ipv6" { + description = "Public IPv6 of the watchtower server." + value = hcloud_primary_ip.ipv6.ip_address +} + +output "server_id" { + value = hcloud_server.watchtower.id +} + +output "buckets" { + description = "Telemetry bucket names." + value = { for k, v in aws_s3_bucket.telemetry : k => v.bucket } +} + +output "s3_endpoint" { + description = "Hetzner Object Storage endpoint URL." + value = "https://${var.location}.your-objectstorage.com" +} + +output "ansible_inventory" { + description = "Drop-in inventory snippet for Ansible." + value = yamlencode({ + all = { + hosts = { + watchtower = { + ansible_host = hcloud_primary_ip.ipv4.ip_address + ansible_user = "root" + ansible_python_interpreter = "/usr/bin/python3" + } + } + } + }) +} diff --git a/tofu/providers.tf b/tofu/providers.tf new file mode 100644 index 0000000..e69e891 --- /dev/null +++ b/tofu/providers.tf @@ -0,0 +1,22 @@ +provider "hcloud" { + token = var.hcloud_token +} + +# AWS provider aliased to Hetzner's S3-compatible Object Storage. +# Used to create the Mimir/Loki/Tempo buckets and lifecycle rules. +provider "aws" { + alias = "hetzner" + region = var.location + access_key = var.hetzner_s3_access_key + secret_key = var.hetzner_s3_secret_key + + endpoints { + s3 = "https://${var.location}.your-objectstorage.com" + } + + skip_credentials_validation = true + skip_region_validation = true + skip_metadata_api_check = true + skip_requesting_account_id = true + s3_use_path_style = true +} diff --git a/tofu/server.tf b/tofu/server.tf new file mode 100644 index 0000000..486745e --- /dev/null +++ b/tofu/server.tf @@ -0,0 +1,73 @@ +resource "hcloud_ssh_key" "admin" { + name = "${var.server_name}-admin" + public_key = var.ssh_public_key +} + +resource "hcloud_firewall" "watchtower" { + name = "${var.server_name}-fw" + + # SSH + rule { + direction = "in" + protocol = "tcp" + port = "22" + source_ips = concat(var.admin_allow_ipv4, var.admin_allow_ipv6) + } + + # WireGuard + rule { + direction = "in" + protocol = "udp" + port = "51820" + source_ips = concat(var.admin_allow_ipv4, var.admin_allow_ipv6) + } + + # ICMP (ping) — useful for ops, optional + rule { + direction = "in" + protocol = "icmp" + source_ips = ["0.0.0.0/0", "::/0"] + } +} + +resource "hcloud_primary_ip" "ipv4" { + name = "${var.server_name}-ipv4" + type = "ipv4" + datacenter = "${var.location}-dc3" + auto_delete = false +} + +resource "hcloud_primary_ip" "ipv6" { + name = "${var.server_name}-ipv6" + type = "ipv6" + datacenter = "${var.location}-dc3" + auto_delete = false +} + +resource "hcloud_server" "watchtower" { + name = var.server_name + server_type = var.server_type + image = var.server_image + location = var.location + ssh_keys = [hcloud_ssh_key.admin.id] + firewall_ids = [hcloud_firewall.watchtower.id] + + public_net { + ipv4_enabled = true + ipv4 = hcloud_primary_ip.ipv4.id + ipv6_enabled = true + ipv6 = hcloud_primary_ip.ipv6.id + } + + user_data = file("${path.module}/cloud-init.yaml") + + labels = { + role = "observability" + managed_by = "opentofu" + environment = "prod" + } + + lifecycle { + ignore_changes = [user_data, ssh_keys] + } +} diff --git a/tofu/terraform.tfvars.example b/tofu/terraform.tfvars.example new file mode 100644 index 0000000..2c87f57 --- /dev/null +++ b/tofu/terraform.tfvars.example @@ -0,0 +1,8 @@ +hcloud_token = "REPLACE_ME_HCLOUD_TOKEN" +hetzner_s3_access_key = "REPLACE_ME_S3_ACCESS_KEY" +hetzner_s3_secret_key = "REPLACE_ME_S3_SECRET_KEY" +ssh_public_key = "ssh-ed25519 AAAA... your-key" + +# Optional: tighten admin access to your VPN egress IPs +# admin_allow_ipv4 = ["198.51.100.42/32"] +# admin_allow_ipv6 = ["2001:db8::/64"] diff --git a/tofu/variables.tf b/tofu/variables.tf new file mode 100644 index 0000000..16a8c7e --- /dev/null +++ b/tofu/variables.tf @@ -0,0 +1,64 @@ +variable "hcloud_token" { + description = "Hetzner Cloud API token (Read & Write)." + type = string + sensitive = true +} + +variable "hetzner_s3_access_key" { + description = "Hetzner Object Storage access key (created in Cloud Console under Security)." + type = string + sensitive = true +} + +variable "hetzner_s3_secret_key" { + description = "Hetzner Object Storage secret key." + type = string + sensitive = true +} + +variable "location" { + description = "Hetzner location code." + type = string + default = "nbg1" +} + +variable "server_type" { + description = "Hetzner server type." + type = string + default = "cpx42" +} + +variable "server_image" { + description = "Hetzner OS image. AlmaLinux 10 — verify the exact slug with `hcloud image list`." + type = string + default = "alma-10" +} + +variable "server_name" { + description = "Server hostname." + type = string + default = "watchtower" +} + +variable "ssh_public_key" { + description = "SSH public key (ed25519) used for initial root access." + type = string +} + +variable "admin_allow_ipv4" { + description = "IPv4 CIDRs allowed to reach SSH (22/tcp) and WireGuard (51820/udp). Use [\"0.0.0.0/0\"] if mobile." + type = list(string) + default = ["0.0.0.0/0"] +} + +variable "admin_allow_ipv6" { + description = "IPv6 CIDRs allowed to reach SSH and WireGuard." + type = list(string) + default = ["::/0"] +} + +variable "bucket_prefix" { + description = "Prefix for the three telemetry buckets." + type = string + default = "watchtower" +} diff --git a/tofu/versions.tf b/tofu/versions.tf new file mode 100644 index 0000000..eb8a864 --- /dev/null +++ b/tofu/versions.tf @@ -0,0 +1,46 @@ +terraform { + required_version = ">= 1.10.0" + + required_providers { + hcloud = { + source = "hetznercloud/hcloud" + version = "~> 1.48" + } + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + tls = { + source = "hashicorp/tls" + version = "~> 4.0" + } + } + + # State backend: Cloudflare R2 (S3-compatible). + # Authenticate using AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars + # set to your Cloudflare R2 access key/secret. Account ID embedded in endpoint. + # + # use_lockfile = true requires OpenTofu >= 1.10 and provides native S3-based + # state locking via conditional PutObject (no DynamoDB needed; R2-compatible). + backend "s3" { + bucket = "watchtower-tofu-state" + key = "watchtower/terraform.tfstate" + region = "auto" + + # Cloudflare R2 endpoint (set via -backend-config or env at init): + # endpoints = { s3 = "https://.r2.cloudflarestorage.com" } + # See backend.hcl.example. + + use_lockfile = true + skip_credentials_validation = true + skip_region_validation = true + skip_metadata_api_check = true + skip_requesting_account_id = true + skip_s3_checksum = true + use_path_style = true + } +}