Akatosh

Vagrantfile

@@ -18,6 +18,17 @@ Vagrant.configure("2") do |config|
     debian.vm.network "private_network", ip: "192.168.121.100"  # Static IP needed for nas
     debian.vm.network "forwarded_port", guest: 22, host: 3023
   end
+  config.vm.define "akatosh" do |debian|
+    debian.vm.box = "debian/bookworm64"
+    debian.vm.hostname = "test-akatosh"
+    debian.vm.network "private_network", type: "dhcp"
+    debian.vm.network "forwarded_port", guest: 22, host: 3029
+    debian.vm.provider "libvirt" do |lv|
+      lv.memory = "6144"
+      lv.cpus = 2
+      lv.machine_virtual_size = 40
+    end
+  end
   config.vm.define "vaermina" do |debian|
     debian.vm.box = "debian/bookworm64"
     debian.vm.hostname = "test-vaermina"

group_vars/backup.yml

@@ -0,0 +1,4 @@
+backup_root: /srv/backups/aurbis
+backup_key_path: /root/.ssh/aurbis_backup_ed25519
+backup_retain_recent: 14
+backup_retain_monthly: 6

group_vars/cloud.yml

@@ -4,3 +4,6 @@ users:
   - jp
   - lourens
   - guus
+# The cloud hosts have no nerevar user;
+ansible_user: mark
+backup_dump_dir: /var/backups/aurbis

handlers/main.yml

@@ -22,3 +22,8 @@
     iptables-save > /etc/iptables/rules.v4
   args:
     executable: /bin/bash
+
+- name: reload nginx
+  service:
+    name: nginx
+    state: reloaded

host_vars/akatosh.yml

@@ -0,0 +1,50 @@
+users:
+  - mark
+  - martijn
+  - jp
+  - lourens
+  - guus
+cloud_nginx_tls: true
+cloud_nginx_sites:
+  - name: nextcloud
+    server_name: cloud.hoekveen.net
+    port: 8081
+    max_body: 10g
+  - name: hdwiki
+    server_name: wiki.hoekveen.net
+    port: 8082
+  - name: gitea
+    server_name: git.hoekveen.net
+    port: 8083
+  - name: authentik
+    server_name: sso.hoekveen.net
+    port: 9000
+  - name: drone
+    server_name: drone.hoekveen.net
+    port: 3496
+  - name: www
+    server_name: www.hoekveen.net
+    root: /opt/hoekveen.net
+  - name: nightingale
+    server_name: karaoke.hoekveen.net
+    port: 8087
+    max_body: 2g # song uploads
+
+backup_dump_dir: /var/backups/aurbis
+backup_paths:
+  - /etc # nginx, certs, systemd units
+  - /opt # all stacks
+  - /var/nextcloud # nextcloud files
+  - /var/lib/docker/volumes # nextcloud_html (apps/config) etc.
+  - /var/backups/aurbis # the database dumps
+backup_excludes:
+  - /var/nextcloud/appdata_*/preview # regenerable thumbnails
+  - /var/nextcloud/nextcloud.log # rotates
+  - /opt/containerd # docker runtime state
+  - /var/lib/docker/volumes/nextcloud_db # raw databases are dumped instead
+  - /var/lib/docker/volumes/gitea_db
+  - /var/lib/docker/volumes/hdwiki_db
+  - /var/lib/docker/volumes/authentik_database
+  - /var/lib/docker/volumes/authentik_redis
+  - /var/lib/docker/volumes/nightingale_data
+  - /var/lib/docker/volumes/backingFsBlockDev

inventory.test.yml

@@ -28,6 +28,11 @@ x86:
       ansible_port: 3022
       ansible_user: vagrant
       ansible_ssh_private_key_file: .vagrant/machines/talos/libvirt/private_key
+    akatosh:
+      # qemu test VM, see docs/migration-survey.md (cloud-init creates nerevar)
+      ansible_host: 127.0.0.1
+      ansible_port: 3029
+      ansible_user: nerevar
     # meridia:
     #   ansible_host: 127.0.0.1
     #   ansible_port: 3025
@@ -44,6 +49,7 @@ debian:
     vaermina:
     talos:
     nocturnal:
+    akatosh:
 # arch:
 #   hosts:
 #     azura:
@@ -61,3 +67,5 @@ home:
 #   hosts:
 #     meridia:
 #     dibella:
+cloud:
+  hosts:

inventory.yml

@@ -16,11 +16,14 @@ x86:
       ansible_host: 62.171.176.43
     dibella:
       ansible_host: 45.88.188.77
+    akatosh:
+      ansible_host: 169.58.40.23
 debian:
   hosts:
     vaermina:
     talos:
     nocturnal:
+    akatosh:
 arch:
   hosts:
     azura:
@@ -38,3 +41,13 @@ cloud:
   hosts:
     meridia:
     dibella:
+# Hosts whose application data talos pulls into hardlinked snapshots.
+# TODO: kill meridia/dibella
+backup_sources:
+  hosts:
+    meridia:
+    dibella:
+    akatosh:
+backup:
+  hosts:
+    talos:

playbook.yml

@@ -1,12 +1,12 @@
 - name: Preprovisioning
-  hosts: all
+  hosts: all:!cloud
   become: true
   tasks:
     - import_tasks: tasks/preprovision.yml
   handlers:
     - import_tasks: handlers/main.yml
 - name: Provision all hosts
-  hosts: all
+  hosts: all:!cloud
   become: true
   remote_user: nerevar
   tasks:
@@ -47,7 +47,7 @@
     - name: Update
       include_tasks: tasks/update_debian.yml
 - name: Update Arch hosts
-  hosts: arch
+  hosts: arch:!cloud
   become: true
   remote_user: nerevar
   tasks:
@@ -60,7 +60,7 @@
     - name: Update
       include_tasks: tasks/install_docker_debian.yml
 - name: Add utility scripts
-  hosts: all
+  hosts: all:!cloud
   become: true
   remote_user: nerevar
   tasks:
@@ -79,7 +79,7 @@
   tasks:
     - import_tasks: tasks/zsh.yml
 - name: Install tmux
-  hosts: all
+  hosts: all:!cloud
   become: true
   remote_user: nerevar
   tasks:
@@ -97,6 +97,54 @@
   handlers:
     - import_tasks: handlers/main.yml
   tags: leaf
+- name: Backup host (pull snapshots from cloud)
+  hosts: backup
+  become: true
+  remote_user: nerevar
+  tasks:
+    - import_tasks: tasks/backup_pull.yml
+  tags:
+    - leaf
+    - backup
+- name: Backup sources (cloud hosts)
+  hosts: cloud
+  become: true
+  tasks:
+    - import_tasks: tasks/backup_source.yml
+  tags:
+    - leaf
+    - backup
+- name: Akatosh install (replacement cloud server)
+  hosts: akatosh
+  become: true
+  remote_user: nerevar
+  tasks:
+    - name: Install base tools (minimal cloud image lacks them)
+      package:
+        name: [rsync, curl, sqlite3]
+        state: present
+    - name: Normalise /opt app ownership to root:aurbis (top dir only, not recursive)
+      file:
+        path: "/opt/{{ item }}"
+        state: directory
+        owner: root
+        group: aurbis
+        mode: "0775"
+      loop: [nextcloud, hdwiki, gitea, authentik, drone, "hoekveen.net"]
+    - import_tasks: tasks/nextcloud.yml
+    - import_tasks: tasks/hdwiki.yml
+    - import_tasks: tasks/gitea.yml
+    - import_tasks: tasks/nightingale.yml
+    - import_tasks: tasks/cloud_nginx.yml
+    - name: Ensure the backup user can run the forced-command pull
+      user:
+        name: backup
+        shell: /bin/bash
+        create_home: false
+    - import_tasks: tasks/backup_source.yml
+  handlers:
+    - import_tasks: handlers/main.yml
+  tags: leaf
 - name: Vaermina install
   hosts: vaermina
   become: true

tasks/backup_pull.yml

@@ -0,0 +1,48 @@
+- name: Ensure rsync is installed
+  package:
+    name: rsync
+    state: present
+
+- name: Create backup root
+  file:
+    path: "{{ backup_root }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+
+- name: Generate pull ssh key
+  command:
+    cmd: ssh-keygen -t ed25519 -N "" -C aurbis-backup-pull -f {{ backup_key_path }}
+    creates: "{{ backup_key_path }}"
+
+- name: Read pull public key
+  slurp:
+    src: "{{ backup_key_path }}.pub"
+  register: backup_pubkey_slurp
+
+- name: Export public key as fact for the source hosts
+  set_fact:
+    aurbis_backup_pubkey: "{{ backup_pubkey_slurp.content | b64decode | trim }}"
+
+- name: Install pull script
+  template:
+    src: backup/aurbis-backup-pull.sh.j2
+    dest: /usr/local/bin/aurbis-backup-pull
+    mode: "0750"
+
+- name: Install pull service and timer
+  copy:
+    src: "backup/{{ item }}"
+    dest: /etc/systemd/system/
+    mode: "0644"
+  with_items:
+    - aurbis-backup-pull.service
+    - aurbis-backup-pull.timer
+
+- name: Enable pull timer
+  systemd:
+    name: aurbis-backup-pull.timer
+    enabled: yes
+    state: started
+    daemon_reload: yes

tasks/backup_source.yml

@@ -0,0 +1,55 @@
+# daily DB dumps into {{ backup_dump_dir }}
+# read-only rsync access for the backup host's pull key (user `backup`)
+- name: Resolve the backup pull public key from the backup host
+  when: aurbis_backup_pubkey is not defined
+  block:
+    - name: Read the pull public key
+      slurp:
+        src: "{{ hostvars[groups['backup'][0]].backup_key_path }}.pub"
+      delegate_to: "{{ groups['backup'][0] }}"
+      register: _pull_pubkey
+    - name: Set the pull pubkey fact
+      set_fact:
+        aurbis_backup_pubkey: "{{ _pull_pubkey.content | b64decode | trim }}"
+
+- name: Install database dump script
+  template:
+    src: "backup/aurbis-dump-{{ inventory_hostname }}.sh.j2"
+    dest: /usr/local/bin/aurbis-dump
+    mode: "0750"
+
+- name: Install dump service and timer
+  copy:
+    src: "backup/{{ item }}"
+    dest: /etc/systemd/system/
+    mode: "0644"
+  with_items:
+    - aurbis-dump.service
+    - aurbis-dump.timer
+
+- name: Enable dump timer
+  systemd:
+    name: aurbis-dump.timer
+    enabled: yes
+    state: started
+    daemon_reload: yes
+
+- name: Install read-only rsync wrapper for the backup user
+  copy:
+    src: backup/backup-rsync-shell
+    dest: /usr/local/bin/backup-rsync-shell
+    mode: "0755"
+
+- name: Allow backup user to run read-only rsync as root
+  community.general.sudoers:
+    name: backup-rsync
+    state: present
+    user: backup
+    commands: /usr/bin/rsync --server --sender *
+
+- name: Authorize the backup host's pull key (forced read-only rsync)
+  ansible.posix.authorized_key:
+    user: backup
+    state: present
+    key: "{{ aurbis_backup_pubkey }}"
+    key_options: 'command="/usr/local/bin/backup-rsync-shell",restrict'

tasks/cloud_nginx.yml

@@ -0,0 +1,41 @@
+# Reverse proxy for all cloud services.
+# the vhosts serve https using certs in /etc/letsencrypt
+- name: Install nginx (+certbot when TLS is on)
+  package:
+    name: "{{ ['nginx', 'certbot'] if cloud_nginx_tls | default(false) else ['nginx'] }}"
+    state: present
+
+- name: Create ACME webroot
+  file:
+    path: /var/www/letsencrypt
+    state: directory
+    mode: "0755"
+  when: cloud_nginx_tls | default(false)
+
+- name: Template vhosts
+  template:
+    src: cloud_nginx/vhost.conf.j2
+    dest: "/etc/nginx/sites-available/{{ item.name }}"
+    mode: "0644"
+  loop: "{{ cloud_nginx_sites }}"
+  notify: reload nginx
+
+- name: Enable vhosts
+  file:
+    src: "/etc/nginx/sites-available/{{ item.name }}"
+    dest: "/etc/nginx/sites-enabled/{{ item.name }}"
+    state: link
+  loop: "{{ cloud_nginx_sites }}"
+  notify: reload nginx
+
+- name: Disable default site
+  file:
+    path: /etc/nginx/sites-enabled/default
+    state: absent
+  notify: reload nginx
+
+- name: Enable nginx
+  systemd:
+    name: nginx
+    enabled: yes
+    state: started

tasks/cloud_nginx/vhost.conf.j2

@@ -0,0 +1,51 @@
+# {{ ansible_managed }}
+{% if cloud_nginx_tls | default(false) %}
+server {
+    listen 80;
+    listen [::]:80;
+    server_name {{ item.server_name }};
+    # ACME http-01 renewals keep working over plain http
+    location /.well-known/acme-challenge/ { root /var/www/letsencrypt; }
+    location / { return 301 https://$host$request_uri; }
+}
+{% endif %}
+
+server {
+{% if cloud_nginx_tls | default(false) %}
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    ssl_certificate /etc/letsencrypt/live/{{ item.server_name }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ item.server_name }}/privkey.pem;
+{% else %}
+    listen 80;
+    listen [::]:80;
+{% endif %}
+    server_name {{ item.server_name }};
+
+    client_max_body_size {{ item.max_body | default('512m') }};
+
+{% if item.root is defined %}
+    root {{ item.root }};
+    index index.html index.htm;
+{% else %}
+    location / {
+        proxy_pass http://127.0.0.1:{{ item.port }};
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host;
+        proxy_read_timeout 300s;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+    }
+{% endif %}
+{% if item.name == 'nextcloud' %}
+
+    # Nextcloud service discovery
+    location = /.well-known/carddav { return 301 /remote.php/dav/; }
+    location = /.well-known/caldav  { return 301 /remote.php/dav/; }
+{% endif %}
+}

tasks/gitea.yml

@@ -0,0 +1,32 @@
+# Git-over-ssh stays on the host sshd port 22
+# ("SSH container passthrough" in gitea docs)
+- name: Create gitea group (gid must match compose USER_GID)
+  group:
+    name: gitea
+    gid: 2222
+- name: Create gitea passthrough user (uid must match compose USER_UID)
+  user:
+    name: gitea
+    uid: 2222
+    group: gitea
+    groups: docker # for the docker exec
+    append: true
+    home: /opt/gitea/data/git # holds .ssh/authorized_keys
+    create_home: false
+    shell: /bin/bash # standard sh doesn't work
+    password_lock: true
+- name: Install gitea ssh passthrough shim
+  copy:
+    dest: /usr/local/bin/gitea
+    mode: "0755"
+    content: |
+      #!/bin/sh
+      # Managed by ansible, see tasks/gitea.yml.
+      exec /usr/bin/docker exec -i -u gitea \
+        -e SSH_ORIGINAL_COMMAND="$SSH_ORIGINAL_COMMAND" \
+        -e GITEA_CUSTOM=/data/gitea \
+        gitea-gitea-1 /usr/local/bin/gitea "$@"
+- name: Deploy Gitea
+  include_tasks: tasks/setup_docker_service.yml
+  vars:
+    service_name: gitea

tasks/gitea/docker-compose.yml.j2

@@ -0,0 +1,39 @@
+# Git-over-ssh stays on the host sshd port 22 via passthrough
+services:
+  db:
+    image: mariadb:11.8
+    restart: unless-stopped
+    environment:
+      MARIADB_DATABASE: gitea
+      MARIADB_USER: gitea
+      MARIADB_PASSWORD: "{{ gitea_db_password }}"
+      MARIADB_RANDOM_ROOT_PASSWORD: 1
+    volumes:
+      - db:/var/lib/mysql
+    healthcheck:
+      test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
+      interval: 10s
+      retries: 12
+  gitea:
+    image: docker.gitea.com/gitea:1.26.4
+    restart: unless-stopped
+    depends_on:
+      db:
+        condition: service_healthy
+    environment:
+      # Match the host "gitea" passthrough user (tasks/gitea.yml)
+      # for access to authorized_keys
+      USER_UID: 2222
+      USER_GID: 2222
+      # Must match RUN_USER in data/gitea/conf/app.ini.
+      USER: gitea
+    ports:
+      - "127.0.0.1:8083:3000"
+      - "127.0.0.1:2222:22"
+    volumes:
+      - ./data:/data
+      - /etc/timezone:/etc/timezone:ro
+      - /etc/localtime:/etc/localtime:ro
+
+volumes:
+  db:

tasks/gitea/vault.yml

@@ -0,0 +1,8 @@
+$ANSIBLE_VAULT;1.1;AES256
+38636231623835373231353663356664626532653632623064323361316330653838373637646261
+6164313264623363626236346138303639613666363466310a633563353032646531333339666563
+36353563313934623637646131383231396436393138333537396138393834376532336630326366
+3633396139613661380a393431356635663136633131633836616630306131333231316436663432
+39326232383262363936333134313862393265383266303137616537623032616437623735643734
+36646231343438383566366230616365623638643132383533643731633535326535623765336664
+303661616561333336306165633932353765

tasks/hdwiki.yml

@@ -0,0 +1,4 @@
+- name: Deploy HD MediaWiki
+  include_tasks: tasks/setup_docker_service.yml
+  vars:
+    service_name: hdwiki

tasks/hdwiki/docker-compose.yml.j2

@@ -0,0 +1,42 @@
+services:
+  db:
+    image: mariadb:11.8
+    restart: unless-stopped
+    environment:
+      MARIADB_DATABASE: mediawikidb
+      MARIADB_USER: mediawiki
+      MARIADB_PASSWORD: "{{ hdwiki_db_password }}"
+      MARIADB_RANDOM_ROOT_PASSWORD: 1
+    volumes:
+      - db:/var/lib/mysql
+    healthcheck:
+      test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
+      interval: 10s
+      retries: 12
+  mediawiki:
+    # ffmpeg added for TimedMediaHandler
+    build:
+      context: .
+      dockerfile_inline: |
+        FROM mediawiki:1.45.4
+        RUN apt-get update \
+          && apt-get install -y --no-install-recommends ffmpeg \
+          && rm -rf /var/lib/apt/lists/*
+        # mediawiki image ships no php.ini, leaving display_errors=On
+        RUN cp /usr/local/etc/php/php.ini-production /usr/local/etc/php/php.ini
+    restart: unless-stopped
+    depends_on:
+      db:
+        condition: service_healthy
+    ports:
+      - "127.0.0.1:8082:80"
+    volumes:
+      - ./images:/var/www/html/images
+      - ./LocalSettings.php:/var/www/html/LocalSettings.php:ro
+      # Extension volumes:
+      - ./extensions-1.45/TimedMediaHandler:/var/www/html/extensions/TimedMediaHandler
+      - ./extensions-1.45/PluggableAuth:/var/www/html/extensions/PluggableAuth
+      - ./extensions-1.45/OpenIDConnect:/var/www/html/extensions/OpenIDConnect
+
+volumes:
+  db:

tasks/hdwiki/vault.yml

@@ -0,0 +1,8 @@
+$ANSIBLE_VAULT;1.1;AES256
+61376366643562623839396662336136653036353930323864646330633535633935393936366133
+3935633032356364376335363865353662303561626362300a656362323464636331666164333037
+34313065633861653036613265336464333935613164356462336533373534386234653765393932
+6536303464313235340a336138613863666662316231306365366131326339646466626631613430
+36643564333135353834346534313533373566623838363538306437643766626630633461313136
+32346132306165353134356439306232643436623333613533633765633039333764383331383530
+663431346261346363323132356165656462

tasks/install_docker_debian.yml

@@ -3,6 +3,7 @@
     name:
       - ca-certificates
       - curl
+      - gnupg # not present on minimal Debian 13
     state: latest
     update_cache: true
 - name: Create /etc/apt/keyrings directory

tasks/nextcloud.yml

@@ -0,0 +1,4 @@
+- name: Deploy Nextcloud
+  include_tasks: tasks/setup_docker_service.yml
+  vars:
+    service_name: nextcloud

tasks/nextcloud/docker-compose.yml.j2

@@ -0,0 +1,52 @@
+services:
+  db:
+    image: mariadb:11.8
+    restart: unless-stopped
+    command: --transaction-isolation=READ-COMMITTED
+    environment:
+      MARIADB_DATABASE: nextcloud
+      MARIADB_USER: nextcloud
+      MARIADB_PASSWORD: "{{ nextcloud_db_password }}"
+      MARIADB_RANDOM_ROOT_PASSWORD: 1
+    volumes:
+      - db:/var/lib/mysql
+    healthcheck:
+      test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
+      interval: 10s
+      retries: 12
+  redis:
+    image: redis:alpine
+    restart: unless-stopped
+  nextcloud:
+    image: nextcloud:34.0.1-apache
+    restart: unless-stopped
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_started
+    ports:
+      - "127.0.0.1:8081:80"
+    volumes:
+      - html:/var/www/html
+      - /var/nextcloud:/var/nextcloud
+      - /opt/nightingale/songs:/karaoke
+    environment:
+      TRUSTED_PROXIES: "172.16.0.0/12"
+      APACHE_DISABLE_REWRITE_IP: 1
+  cron:
+    image: nextcloud:34.0.1-apache
+    restart: unless-stopped
+    entrypoint: /cron.sh
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_started
+    volumes:
+      - html:/var/www/html
+      - /var/nextcloud:/var/nextcloud
+
+volumes:
+  db:
+  html:

tasks/nextcloud/vault.yml

@@ -0,0 +1,8 @@
+$ANSIBLE_VAULT;1.1;AES256
+32386635343938666234396638643362313132333261366139326466633332386336656232353434
+3631396261346166373435616436303738323563616265610a353064643636363161306565393566
+63336231376663303437303565383463323463376563356533303333663734653730373731393932
+3139346639613732630a376463343931363330316430313833316137383066343831323564333638
+30636130353266306438333538623161306432613631313035376537623961316433363333623335
+31663238326666326630313562343062313166643535303736343335623031303635623636613366
+646463656333366161626336386466613465

tasks/nightingale.yml

@@ -0,0 +1,29 @@
+- name: Create nightingale directory
+  file:
+    path: /opt/nightingale
+    state: directory
+    owner: root
+    group: aurbis
+    mode: "0775"
+
+# Song library. Read-only for nightingale.
+# Keep this in sync with the /karaoke mount in nextcloud
+# TODO: Maybe make this a configurable non-dependency
+- name: Create nightingale songs dir (writable by nextcloud's www-data)
+  file:
+    path: /opt/nightingale/songs
+    state: directory
+    owner: root
+    group: www-data
+    mode: "02775"
+
+- name: Template nightingale compose
+  template:
+    src: nightingale/docker-compose.yml.j2
+    dest: /opt/nightingale/docker-compose.yml
+    group: aurbis
+    mode: "0664"
+
+- name: Start nightingale
+  community.docker.docker_compose_v2:
+    project_src: /opt/nightingale

tasks/nightingale/docker-compose.yml.j2

@@ -0,0 +1,16 @@
+# {{ ansible_managed }}
+# CPU-only prebuilt image
+services:
+  nightingale:
+    image: razzaru/nightingale:latest
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:8087:8080"
+    volumes:
+      - data:/data
+      - ./songs:/songs:ro
+    environment:
+      NIGHTINGALE_LIBRARY_PATH: /songs
+
+volumes:
+  data: