diff --git a/tasks/backup/aurbis-backup-pull.service b/tasks/backup/aurbis-backup-pull.service new file mode 100644 index 0000000..fb62da1 --- /dev/null +++ b/tasks/backup/aurbis-backup-pull.service @@ -0,0 +1,10 @@ +[Unit] +Description=Pull application-data backup from the cloud hosts +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/aurbis-backup-pull +Nice=10 +IOSchedulingClass=idle diff --git a/tasks/backup/aurbis-backup-pull.sh.j2 b/tasks/backup/aurbis-backup-pull.sh.j2 new file mode 100644 index 0000000..d01b32d --- /dev/null +++ b/tasks/backup/aurbis-backup-pull.sh.j2 @@ -0,0 +1,80 @@ +#!/bin/bash +# {{ ansible_managed }} +# Pull application data from the cloud hosts into hardlinked snapshots +# rsnapshot-style: rsync --link-dest +set -uo pipefail + +ROOT={{ backup_root }} +KEY={{ backup_key_path }} +RETAIN={{ backup_retain_recent | default(14) }} +RETAIN_MONTHLY={{ backup_retain_monthly | default(6) }} +SSH="ssh -i $KEY -o StrictHostKeyChecking=accept-new -o BatchMode=yes" +STAMP=$(date +%Y-%m-%d_%H%M) + +exec 9> "$ROOT/.lock" || exit 1 +flock -n 9 || { echo "another backup run is active, aborting"; exit 1; } + +overall_rc=0 + +{% raw %} +prune() { + local host_root=$1 + local -a snaps + mapfile -t snaps < <(find "$host_root" -mindepth 1 -maxdepth 1 -type d -name '20*' | sort) + ((${#snaps[@]})) || return 0 + declare -A keep_set month_first + # newest RETAIN snapshots + local n=${#snaps[@]} start=0 + ((n > RETAIN)) && start=$((n - RETAIN)) + for s in "${snaps[@]:$start}"; do keep_set[$s]=1; done + # oldest snapshot of each month, for the RETAIN_MONTHLY most recent months + for s in "${snaps[@]}"; do + local m; m=$(basename "$s" | cut -c1-7) + [[ -z ${month_first[$m]:-} ]] && month_first[$m]=$s + done + mapfile -t months < <(printf '%s\n' "${!month_first[@]}" | sort | tail -n "$RETAIN_MONTHLY") + for m in "${months[@]}"; do keep_set[${month_first[$m]}]=1; done + for s in "${snaps[@]}"; do + if [[ -z ${keep_set[$s]:-} ]]; then + echo "pruning $s" + rm -rf "$s" + fi + done +} +{% endraw %} + +{% for host in groups['backup_sources'] %} +##### {{ host }} ##### +host={{ host }} +addr={{ hostvars[host].ansible_host }} +host_root=$ROOT/$host +new=$host_root/incomplete +mkdir -p "$host_root" +rm -rf "$new" +mkdir "$new" +link_dest=() +[[ -d $host_root/latest/. ]] && link_dest=(--link-dest="$host_root/latest/") +host_rc=0 +{% for path in hostvars[host].backup_paths %} +rsync -aR --numeric-ids --delete --delete-excluded \ + {% for ex in hostvars[host].backup_excludes | default([]) %}--exclude='{{ ex }}' {% endfor %}\ + -e "$SSH" "${link_dest[@]}" \ + "backup@$addr:{{ path }}" "$new/" +rc=$? +if ((rc != 0 && rc != 24)); then + echo "ERROR: $host:{{ path }} failed with rsync exit $rc" >&2 + host_rc=1 +fi +{% endfor %} +if ((host_rc == 0)); then + mv "$new" "$host_root/$STAMP" + ln -sfn "$STAMP" "$host_root/latest" + prune "$host_root" + echo "$host: snapshot $STAMP complete" +else + echo "$host: snapshot FAILED, keeping previous 'latest'" >&2 + overall_rc=1 +fi + +{% endfor %} +exit $overall_rc diff --git a/tasks/backup/aurbis-backup-pull.timer b/tasks/backup/aurbis-backup-pull.timer new file mode 100644 index 0000000..f057a83 --- /dev/null +++ b/tasks/backup/aurbis-backup-pull.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Daily pull of the aurbis application-data backup + +[Timer] +OnCalendar=*-*-* 04:30 +RandomizedDelaySec=15m +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/tasks/backup/aurbis-dump-akatosh.sh.j2 b/tasks/backup/aurbis-dump-akatosh.sh.j2 new file mode 100644 index 0000000..e5b0ebb --- /dev/null +++ b/tasks/backup/aurbis-dump-akatosh.sh.j2 @@ -0,0 +1,31 @@ +#!/bin/bash +# {{ ansible_managed }} +# Dump application databases to {{ backup_dump_dir }} +# for pickup by the backup host +set -euo pipefail +umask 077 +DEST={{ backup_dump_dir }} +mkdir -p "$DEST" + +# MariaDB stacks: dump via the stack's `db` service using its own env creds. +dump_mariadb() { # + docker compose -f "/opt/$1/docker-compose.yml" exec -T db sh -c \ + 'exec mariadb-dump --single-transaction --quick --routines --events \ + -u"$MARIADB_USER" -p"$MARIADB_PASSWORD" "$MARIADB_DATABASE"' \ + | gzip > "$DEST/$2.tmp" + mv "$DEST/$2.tmp" "$DEST/$2" +} +dump_mariadb nextcloud nextcloud.sql.gz +dump_mariadb gitea gitea.sql.gz +dump_mariadb hdwiki mediawikidb.sql.gz + +# Authentik PostgreSQL (docker compose) +docker compose -f /opt/authentik/docker-compose.yml exec -T postgresql \ + pg_dump -U authentik -d authentik | gzip > "$DEST/authentik.sql.gz.tmp" +mv "$DEST/authentik.sql.gz.tmp" "$DEST/authentik.sql.gz" + +# Drone CI sqlite (online-safe consistent copy) +sqlite3 /opt/drone/drone/database.sqlite ".backup '$DEST/drone.sqlite.tmp'" +mv "$DEST/drone.sqlite.tmp" "$DEST/drone.sqlite" + +date --iso-8601=seconds > "$DEST/LAST_DUMP" diff --git a/tasks/backup/aurbis-dump.service b/tasks/backup/aurbis-dump.service new file mode 100644 index 0000000..eb3b5f3 --- /dev/null +++ b/tasks/backup/aurbis-dump.service @@ -0,0 +1,8 @@ +[Unit] +Description=Dump application databases for the aurbis backup + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/aurbis-dump +Nice=10 +IOSchedulingClass=idle diff --git a/tasks/backup/aurbis-dump.timer b/tasks/backup/aurbis-dump.timer new file mode 100644 index 0000000..ef02fc0 --- /dev/null +++ b/tasks/backup/aurbis-dump.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Daily application database dump for the aurbis backup + +[Timer] +OnCalendar=*-*-* 03:00 +RandomizedDelaySec=15m +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/tasks/backup/backup-rsync-shell b/tasks/backup/backup-rsync-shell new file mode 100644 index 0000000..a83fdd2 --- /dev/null +++ b/tasks/backup/backup-rsync-shell @@ -0,0 +1,11 @@ +#!/bin/bash +# only allow read-only rsync, BUT elevated so all application data is readable. +case "$SSH_ORIGINAL_COMMAND" in + "rsync --server --sender "*) + exec sudo /usr/bin/rsync ${SSH_ORIGINAL_COMMAND#rsync } + ;; + *) + echo "backup key only permits read-only rsync" >&2 + exit 1 + ;; +esac