backups

tasks/backup/aurbis-backup-pull.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Pull application-data backup from the cloud hosts
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/aurbis-backup-pull
+Nice=10
+IOSchedulingClass=idle

tasks/backup/aurbis-backup-pull.sh.j2

@@ -0,0 +1,80 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# Pull application data from the cloud hosts into hardlinked snapshots
+# rsnapshot-style: rsync --link-dest
+set -uo pipefail
+
+ROOT={{ backup_root }}
+KEY={{ backup_key_path }}
+RETAIN={{ backup_retain_recent | default(14) }}
+RETAIN_MONTHLY={{ backup_retain_monthly | default(6) }}
+SSH="ssh -i $KEY -o StrictHostKeyChecking=accept-new -o BatchMode=yes"
+STAMP=$(date +%Y-%m-%d_%H%M)
+
+exec 9> "$ROOT/.lock" || exit 1
+flock -n 9 || { echo "another backup run is active, aborting"; exit 1; }
+
+overall_rc=0
+
+{% raw %}
+prune() {
+    local host_root=$1
+    local -a snaps
+    mapfile -t snaps < <(find "$host_root" -mindepth 1 -maxdepth 1 -type d -name '20*' | sort)
+    ((${#snaps[@]})) || return 0
+    declare -A keep_set month_first
+    # newest RETAIN snapshots
+    local n=${#snaps[@]} start=0
+    ((n > RETAIN)) && start=$((n - RETAIN))
+    for s in "${snaps[@]:$start}"; do keep_set[$s]=1; done
+    # oldest snapshot of each month, for the RETAIN_MONTHLY most recent months
+    for s in "${snaps[@]}"; do
+        local m; m=$(basename "$s" | cut -c1-7)
+        [[ -z ${month_first[$m]:-} ]] && month_first[$m]=$s
+    done
+    mapfile -t months < <(printf '%s\n' "${!month_first[@]}" | sort | tail -n "$RETAIN_MONTHLY")
+    for m in "${months[@]}"; do keep_set[${month_first[$m]}]=1; done
+    for s in "${snaps[@]}"; do
+        if [[ -z ${keep_set[$s]:-} ]]; then
+            echo "pruning $s"
+            rm -rf "$s"
+        fi
+    done
+}
+{% endraw %}
+
+{% for host in groups['backup_sources'] %}
+##### {{ host }} #####
+host={{ host }}
+addr={{ hostvars[host].ansible_host }}
+host_root=$ROOT/$host
+new=$host_root/incomplete
+mkdir -p "$host_root"
+rm -rf "$new"
+mkdir "$new"
+link_dest=()
+[[ -d $host_root/latest/. ]] && link_dest=(--link-dest="$host_root/latest/")
+host_rc=0
+{% for path in hostvars[host].backup_paths %}
+rsync -aR --numeric-ids --delete --delete-excluded \
+    {% for ex in hostvars[host].backup_excludes | default([]) %}--exclude='{{ ex }}' {% endfor %}\
+    -e "$SSH" "${link_dest[@]}" \
+    "backup@$addr:{{ path }}" "$new/"
+rc=$?
+if ((rc != 0 && rc != 24)); then
+    echo "ERROR: $host:{{ path }} failed with rsync exit $rc" >&2
+    host_rc=1
+fi
+{% endfor %}
+if ((host_rc == 0)); then
+    mv "$new" "$host_root/$STAMP"
+    ln -sfn "$STAMP" "$host_root/latest"
+    prune "$host_root"
+    echo "$host: snapshot $STAMP complete"
+else
+    echo "$host: snapshot FAILED, keeping previous 'latest'" >&2
+    overall_rc=1
+fi
+
+{% endfor %}
+exit $overall_rc

tasks/backup/aurbis-backup-pull.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Daily pull of the aurbis application-data backup
+
+[Timer]
+OnCalendar=*-*-* 04:30
+RandomizedDelaySec=15m
+Persistent=true
+
+[Install]
+WantedBy=timers.target

tasks/backup/aurbis-dump-akatosh.sh.j2

@@ -0,0 +1,31 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# Dump application databases to {{ backup_dump_dir }}
+# for pickup by the backup host
+set -euo pipefail
+umask 077
+DEST={{ backup_dump_dir }}
+mkdir -p "$DEST"
+
+# MariaDB stacks: dump via the stack's `db` service using its own env creds.
+dump_mariadb() { # <stack dir under /opt> <output file>
+    docker compose -f "/opt/$1/docker-compose.yml" exec -T db sh -c \
+        'exec mariadb-dump --single-transaction --quick --routines --events \
+            -u"$MARIADB_USER" -p"$MARIADB_PASSWORD" "$MARIADB_DATABASE"' \
+        | gzip > "$DEST/$2.tmp"
+    mv "$DEST/$2.tmp" "$DEST/$2"
+}
+dump_mariadb nextcloud nextcloud.sql.gz
+dump_mariadb gitea     gitea.sql.gz
+dump_mariadb hdwiki    mediawikidb.sql.gz
+
+# Authentik PostgreSQL (docker compose)
+docker compose -f /opt/authentik/docker-compose.yml exec -T postgresql \
+    pg_dump -U authentik -d authentik | gzip > "$DEST/authentik.sql.gz.tmp"
+mv "$DEST/authentik.sql.gz.tmp" "$DEST/authentik.sql.gz"
+
+# Drone CI sqlite (online-safe consistent copy)
+sqlite3 /opt/drone/drone/database.sqlite ".backup '$DEST/drone.sqlite.tmp'"
+mv "$DEST/drone.sqlite.tmp" "$DEST/drone.sqlite"
+
+date --iso-8601=seconds > "$DEST/LAST_DUMP"

tasks/backup/aurbis-dump.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=Dump application databases for the aurbis backup
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/aurbis-dump
+Nice=10
+IOSchedulingClass=idle

tasks/backup/aurbis-dump.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Daily application database dump for the aurbis backup
+
+[Timer]
+OnCalendar=*-*-* 03:00
+RandomizedDelaySec=15m
+Persistent=true
+
+[Install]
+WantedBy=timers.target

tasks/backup/backup-rsync-shell

@@ -0,0 +1,11 @@
+#!/bin/bash
+# only allow read-only rsync, BUT elevated so all application data is readable.
+case "$SSH_ORIGINAL_COMMAND" in
+    "rsync --server --sender "*)
+        exec sudo /usr/bin/rsync ${SSH_ORIGINAL_COMMAND#rsync }
+        ;;
+    *)
+        echo "backup key only permits read-only rsync" >&2
+        exit 1
+        ;;
+esac