nchsdhg-agathe-backup/scripts/agathe_backup.sh

#!/usr/bin/env bash
# MAIN-HINWEIS: Diese Version ist nun inkl. Tomedo-Backup.
# Robust: WireGuard hoch, CIFS mounten (inkl. Stale-Handle-Fix), rsync,
# nur erlaubte Dateitypen zusätzlich nach paperless-consume (flach, OHNE GIF),
# Paperless-Backup rsync als echtes Sync (mit --delete),
# Anmeldung ebenfalls sync + consume,
# zusätzlich: Borg lokal (inkrementell, versioniert) + Mirror per rsync auf CIFS (mit delete),
# zusätzlich: Tomedo-Backup per rsync,
# robust gegen CIFS-Aussetzer: ensure_cifs + rsync temp-dir lokal + Mount-Fallback.
# Zusätzlich: Pushover-Benachrichtigung bei Erfolg/Fehler.
# Fix: Wenn wg0 schon aktiv ist, wird es nicht erneut gestartet und beim Cleanup
# nur dann beendet, wenn dieses Skript es selbst hochgebracht hat.
# Zusätzlich: Schutz gegen parallele Ausführung per flock.

set -Eeuo pipefail

# -----------------------------
# Konfiguration
# -----------------------------
WG_IF="wg0"
VPN_TEST_IP="10.202.101.10"
WG_WAS_STARTED_BY_SCRIPT=0

LOCK_FILE="/var/lock/agathe_backup.lock"

CIFS_SHARE="//10.202.101.10/nchsdhg"
CIFS_MOUNTPOINT="/mnt/nchsdhg_agathe"

DEST_GROOT="${CIFS_MOUNTPOINT}/groot_mobile/"
DEST_ANMELDUNG="${CIFS_MOUNTPOINT}/Anmeldung/"
DEST_PAPERLESS_BACKUP="${CIFS_MOUNTPOINT}/paperless-backup/"

SRC_GROOT="/mnt/groot/"
SRC_ANMELDUNG="/mnt/Anmeldung/"
SRC_PAPERLESS_BACKUP="/mnt/paperless-backup/daily/"

PAPERLESS_CONSUME="/mnt/paperless-consume"

CIFS_USER="nchsdhg"
CIFS_PASS="mugkeN-zexdab-9gyfky"
CIFS_CRED_FILE=""  # z.B. "/etc/samba/cred_nchsdhg"

# -----------------------------
# Pushover
# -----------------------------
PUSHOVER_USER_TOKEN="uFBKJ1LmL3eMUbHZs2ktLjSH8RyJ2Z"
PUSHOVER_API_TOKEN="a3xzevfk8vwpbj6wp6duzbwy43pcmx"
PUSHOVER_TITLE="nchsdhg"

# -----------------------------
# Borg (lokal + Mirror auf CIFS)
# -----------------------------
LOCAL_BORG_REPO="/var/backups/borg/agathe"
LOCAL_BORG_BASE="/var/backups/borg"
REMOTE_BORG_MIRROR="${CIFS_MOUNTPOINT}/borg-mirror/agathe"

BORG_PASSPHRASE_FILE="/root/.config/borg/passphrase"
BORG_LOCK_WAIT="120"

KEEP_DAILY=7
KEEP_WEEKLY=4
KEEP_MONTHLY=6

# Mirror: Ziel exakt wie Quelle (löscht auf Ziel, was lokal nicht mehr existiert)
MIRROR_DELETE="1"

# Optional: nach jedem Lauf borg check machen (kostet Zeit)
BORG_CHECK="0"

# rsync Optionen (CIFS-freundlich: kein owner/group/perms setzen)
# Zusätzlich: --temp-dir=/tmp -> rsync mkstemp NICHT auf CIFS (hilft bei CIFS-Reconnect/Timeouts)
RSYNC_OPTS=(-rltvh --info=progress2 --no-owner --no-group --no-perms --temp-dir=/tmp)
RSYNC_EXCLUDES=(--exclude="#recycle/" --exclude=".DS_Store" --exclude="._*" --exclude="Thumbs.db")

# Paperless Backup: echtes Sync (LÖSCHT auf Ziel, was in Quelle nicht mehr existiert)
RSYNC_DELETE_OPTS=(--delete --delete-delay)

# Optional: Testlauf für delete (auf "1" setzen um erstmal nur zu sehen, was gelöscht würde)
DRY_RUN_DELETE="0"

# Welche Dateien nach paperless-consume? (OHNE GIF)
CONSUME_EXT_REGEX='\.([Pp][Dd][Ff]|[Dd][Oo][Cc][Xx]?|[Xx][Ll][Ss][Xx]?|[Pp][Pp][Tt][Xx]?|[Oo][Dd][Tt]|[Oo][Dd][Ss]|[Oo][Dd][Pp]|[Rr][Tt][Ff]|[Tt][Xx][Tt]|[Cc][Ss][Vv]|[Mm][Dd]|[Hh][Tt][Mm][Ll]?|[Jj][Pp][Ee]?[Gg]|[Pp][Nn][Gg]|[Tt][Ii][Ff][Ff]?|[Ww][Ee][Bb][Pp]|[Hh][Ee][Ii][Cc])$'

# -----------------------------
# Tomedo Backup
# -----------------------------
TOMEDO_SRC_ROOT="/mnt/TomedoBackup"
TOMEDO_LAST_FILE="${TOMEDO_SRC_ROOT}/lastFilesBackup"
TOMEDO_DEST_ROOT="${CIFS_MOUNTPOINT}/TomedoBackup"
TOMEDO_MACOS_EXCLUDES="${TOMEDO_DEST_ROOT}/macos.excludes"
TOMEDO_FILES_EXCLUDES="${TOMEDO_DEST_ROOT}/files.excludes"

# -----------------------------
# Hilfsfunktionen
# -----------------------------
log() { echo "[$(date +'%F %T')] $*"; }
ts_now() { date +'%F %T'; }

send_pushover() {
  local message="$1"
  local priority="${2:-0}"

  if command -v curl >/dev/null 2>&1; then
    curl -s \
      --form-string "token=${PUSHOVER_API_TOKEN}" \
      --form-string "user=${PUSHOVER_USER_TOKEN}" \
      --form-string "title=${PUSHOVER_TITLE}" \
      --form-string "message=${message}" \
      --form-string "priority=${priority}" \
      https://api.pushover.net/1/messages.json >/dev/null || true
  else
    log "WARNUNG: curl fehlt, Pushover konnte nicht gesendet werden."
  fi
}

need_root() {
  if [[ "${EUID}" -ne 0 ]]; then
    echo "Bitte als root ausführen (z.B. sudo $0)."
    exit 1
  fi
}

acquire_lock() {
  exec 200>"$LOCK_FILE"

  if ! flock -n 200; then
    log "Eine andere Instanz läuft bereits. Abbruch."
    send_pushover "Backup wurde nicht gestartet, weil bereits eine Instanz läuft."
    exit 0
  fi

  echo "$$" 1>&200 || true
  log "Lock gesetzt: $LOCK_FILE"
}

cleanup() {
  set +e
  log "== Cleanup =="

  if mountpoint -q "$CIFS_MOUNTPOINT"; then
    log "Unmount: $CIFS_MOUNTPOINT"
    umount "$CIFS_MOUNTPOINT" 2>/dev/null || umount -l "$CIFS_MOUNTPOINT" || true
  else
    log "Mountpoint nicht gemountet: $CIFS_MOUNTPOINT"
  fi

  if [[ "$WG_WAS_STARTED_BY_SCRIPT" -eq 1 ]]; then
    if wg show "$WG_IF" &>/dev/null; then
      log "WireGuard down: $WG_IF"
      wg-quick down "$WG_IF" >/dev/null 2>&1 || true
    else
      log "WireGuard war bereits nicht mehr aktiv: $WG_IF"
    fi
  else
    log "WireGuard bleibt aktiv (nicht von diesem Skript gestartet)."
  fi
}

on_error() {
  local exit_code=$?
  local line_no="${1:-unknown}"
  log "FEHLER in Zeile ${line_no}, Exit-Code ${exit_code}"
  send_pushover "Backup FEHLER auf $(hostname): Zeile ${line_no}, Exit-Code ${exit_code}" 1
  trap - EXIT
  cleanup
  exit "$exit_code"
}

on_exit() {
  cleanup
}

trap 'on_error $LINENO' ERR
trap on_exit EXIT

check_deps() {
  local deps=(wg-quick wg mount rsync ping mountpoint umount awk cp date basename tee mktemp sleep id mkdir rm dirname borg df curl hostname flock cat)
  for d in "${deps[@]}"; do
    command -v "$d" >/dev/null 2>&1 || { echo "Fehlt: $d"; exit 1; }
  done
}

bringup_wg() {
  log "== WireGuard prüfen: $WG_IF =="

  if wg show "$WG_IF" &>/dev/null; then
    log "WireGuard ist bereits aktiv: $WG_IF"
  else
    log "== WireGuard up: $WG_IF =="
    wg-quick up "$WG_IF"
    WG_WAS_STARTED_BY_SCRIPT=1
  fi

  log "== Prüfe VPN/Konnektivität zu $VPN_TEST_IP =="
  for i in {1..10}; do
    if ping -c1 -W1 "$VPN_TEST_IP" >/dev/null 2>&1; then
      log "OK: $VPN_TEST_IP erreichbar."
      return 0
    fi
    sleep 1
  done

  log "FEHLER: $VPN_TEST_IP nicht erreichbar."
  exit 1
}

_mount_cifs_with_opts() {
  local opts="$1"

  if [[ -n "$CIFS_CRED_FILE" ]]; then
    log "Mount mit credentials file: $CIFS_CRED_FILE"
    mount -t cifs "$CIFS_SHARE" "$CIFS_MOUNTPOINT" -o "credentials=${CIFS_CRED_FILE},${opts}"
  else
    log "Mount mit username/pass (klartext) + opts: ${opts}"
    mount -t cifs "$CIFS_SHARE" "$CIFS_MOUNTPOINT" -o "username=${CIFS_USER},password=${CIFS_PASS},${opts}"
  fi
}

mount_cifs() {
  log "== CIFS mount: $CIFS_SHARE -> $CIFS_MOUNTPOINT =="

  if mountpoint -q "$CIFS_MOUNTPOINT"; then
    log "Mountpoint ist gemountet -> versuche umount"
    umount "$CIFS_MOUNTPOINT" 2>/dev/null || umount -l "$CIFS_MOUNTPOINT" || true
  fi

  if ! mkdir -p "$CIFS_MOUNTPOINT" 2>/dev/null; then
    log "WARNUNG: Mountpoint kaputt (stale handle). Versuche umount -l und neu anlegen."
    umount -l "$CIFS_MOUNTPOINT" 2>/dev/null || true
    mkdir -p "$(dirname "$CIFS_MOUNTPOINT")"
    rm -rf "$CIFS_MOUNTPOINT" 2>/dev/null || true
    mkdir -p "$CIFS_MOUNTPOINT"
  fi

  local opts_base="uid=$(id -u),gid=$(id -g),iocharset=utf8,vers=3.0"
  local opts_robust="${opts_base},cache=strict,actimeo=30,echo_interval=30"

  set +e
  _mount_cifs_with_opts "$opts_robust"
  local rc=$?
  set -e
  if [[ $rc -eq 0 ]]; then
    log "OK gemountet (robust)."
    return 0
  fi

  log "WARNUNG: CIFS mount (robust) fehlgeschlagen (rc=$rc). Fallback auf Basis-Optionen."
  set +e
  _mount_cifs_with_opts "$opts_base"
  rc=$?
  set -e
  if [[ $rc -eq 0 ]]; then
    log "OK gemountet (base)."
    return 0
  fi

  log "FEHLER: CIFS mount fehlgeschlagen (rc=$rc)."
  exit 1
}

ensure_cifs() {
  if ! mountpoint -q "$CIFS_MOUNTPOINT"; then
    log "WARNUNG: CIFS nicht gemountet -> remount"
    mount_cifs
  fi
  if ! mountpoint -q "$CIFS_MOUNTPOINT"; then
    log "FEHLER: CIFS Remount fehlgeschlagen."
    exit 1
  fi
}

copy_to_paperless_flat_if_allowed_ext() {
  local src="$1"
  local rel="$2"
  local base
  base="$(basename "$src")"

  [[ "$base" == ._* ]] && return 0
  [[ ! "$base" =~ $CONSUME_EXT_REGEX ]] && return 0

  mkdir -p "$PAPERLESS_CONSUME"
  local dst="$PAPERLESS_CONSUME/$base"

  if [[ -e "$dst" ]]; then
    local ts name ext
    ts="$(date +%Y%m%d-%H%M%S)"
    name="${base%.*}"
    ext="${base##*.}"
    if [[ "$base" == *.* && "$ext" != "$base" ]]; then
      dst="$PAPERLESS_CONSUME/${name}_${ts}.${ext}"
    else
      dst="$PAPERLESS_CONSUME/${base}_${ts}"
    fi
  fi

  log "Paperless copy: $rel -> $(basename "$dst")"
  cp --preserve=mode,timestamps -- "$src" "$dst" || {
    log "WARNUNG: cp fehlgeschlagen: '$src' -> '$dst' (mache weiter)"
    return 0
  }
}

rsync_and_copy_to_consume_flat() {
  local SRC="$1"
  local DEST="$2"
  local LABEL="$3"

  ensure_cifs

  log "== rsync ${LABEL}: $SRC -> $DEST =="
  mkdir -p "$DEST"

  local tmpfile rc
  tmpfile="$(mktemp)"

  set +e
  rsync "${RSYNC_OPTS[@]}" "${RSYNC_EXCLUDES[@]}" --out-format='%i %n' "$SRC" "$DEST" | tee "$tmpfile"
  rc=${PIPESTATUS[0]}
  set -e

  log "rsync(${LABEL}) Exit-Code: $rc"

  if [[ "$rc" -ne 0 && "$rc" -ne 23 && "$rc" -ne 24 ]]; then
    log "FEHLER: rsync (${LABEL}) Exit-Code $rc"
    exit "$rc"
  fi
  if [[ "$rc" -ne 0 ]]; then
    log "WARNUNG: rsync (${LABEL}) Exit-Code $rc (mache trotzdem weiter)"
  fi

  log "== Kopiere zusätzlich (${LABEL}) (NUR erlaubte Endungen, OHNE GIF) flach nach: $PAPERLESS_CONSUME =="

  awk '
    $1 !~ /^\./ && $1 ~ /^.[f]/ {
      $1=""; sub(/^ /,""); print
    }
  ' "$tmpfile" | while IFS= read -r rel; do
      local local_src="${SRC}${rel}"

      if [[ ! -f "$local_src" ]]; then
        log "WARNUNG: Quelle nicht gefunden (überspringe): $local_src"
        continue
      fi

      copy_to_paperless_flat_if_allowed_ext "$local_src" "$rel"
    done

  rm -f "$tmpfile"
}

rsync_paperless_backup() {
  ensure_cifs

  log "== rsync Paperless Backup (SYNC mit delete): $SRC_PAPERLESS_BACKUP -> $DEST_PAPERLESS_BACKUP =="

  if [[ -z "$DEST_PAPERLESS_BACKUP" || "$DEST_PAPERLESS_BACKUP" == "/" ]]; then
    log "FEHLER: DEST_PAPERLESS_BACKUP ist unsicher gesetzt."
    exit 1
  fi

  mkdir -p "$DEST_PAPERLESS_BACKUP"

  local extra=()
  if [[ "$DRY_RUN_DELETE" == "1" ]]; then
    extra+=(--dry-run)
    log "HINWEIS: DRY RUN aktiv (es wird nichts wirklich gelöscht/kopiert)."
  fi

  rsync "${RSYNC_OPTS[@]}" "${RSYNC_EXCLUDES[@]}" "${RSYNC_DELETE_OPTS[@]}" "${extra[@]}" \
    "$SRC_PAPERLESS_BACKUP" "$DEST_PAPERLESS_BACKUP"
}

rsync_tomedo_backup() {
  ensure_cifs

  log "== Tomedo Backup starten =="

  if [[ ! -f "$TOMEDO_LAST_FILE" ]]; then
    log "FEHLER: Datei fehlt: $TOMEDO_LAST_FILE"
    exit 1
  fi

  local last_backup
  last_backup="$(cat "$TOMEDO_LAST_FILE")"

  if [[ -z "$last_backup" ]]; then
    log "FEHLER: $TOMEDO_LAST_FILE ist leer"
    exit 1
  fi

  local src_snapshot="${TOMEDO_SRC_ROOT}/${last_backup}/"
  local dst_snapshot="${TOMEDO_DEST_ROOT}/${last_backup}"
  local src_files="${TOMEDO_SRC_ROOT}/files/"
  local dst_files="${TOMEDO_DEST_ROOT}/files"

  if [[ ! -d "$src_snapshot" ]]; then
    log "FEHLER: Tomedo Snapshot-Ordner fehlt: $src_snapshot"
    exit 1
  fi

  if [[ ! -d "$src_files" ]]; then
    log "FEHLER: Tomedo files-Ordner fehlt: $src_files"
    exit 1
  fi

  if [[ ! -f "$TOMEDO_MACOS_EXCLUDES" ]]; then
    log "FEHLER: Exclude-Datei fehlt: $TOMEDO_MACOS_EXCLUDES"
    exit 1
  fi

  if [[ ! -f "$TOMEDO_FILES_EXCLUDES" ]]; then
    log "FEHLER: Exclude-Datei fehlt: $TOMEDO_FILES_EXCLUDES"
    exit 1
  fi

  mkdir -p "$dst_snapshot" "$dst_files"

  log "== rsync Tomedo Snapshot: $src_snapshot -> $dst_snapshot =="
  rsync -r -l -t -O --info=progress2 \
    --exclude-from "$TOMEDO_MACOS_EXCLUDES" \
    "$src_snapshot" \
    "$dst_snapshot"

  log "== rsync Tomedo Files: $src_files -> $dst_files =="
  rsync -r -l -t -O --info=progress2 \
    --exclude-from "$TOMEDO_MACOS_EXCLUDES" \
    --exclude-from "$TOMEDO_FILES_EXCLUDES" \
    "$src_files" \
    "$dst_files"

  log "== Tomedo Backup fertig: $last_backup =="
}

# -----------------------------
# Borg: lokal sichern + Mirror auf CIFS
# -----------------------------
run_borg_allow_warning() {
  local label="$1"
  shift

  local rc
  set +e
  "$@"
  rc=$?
  set -e

  if [[ "$rc" -ge 2 ]]; then
    log "FEHLER: ${label} fehlgeschlagen (Exit-Code ${rc})."
    return "$rc"
  fi

  if [[ "$rc" -eq 1 ]]; then
    log "WARNUNG: ${label} mit Warnungen beendet (Exit-Code 1), mache weiter."
  fi
}

borg_local_backup() {
  log "== Borg lokal Backup -> $LOCAL_BORG_REPO =="

  if [[ ! -d "$LOCAL_BORG_REPO" ]]; then
    log "FEHLER: Lokales Borg Repo fehlt: $LOCAL_BORG_REPO (einmal borg init ausführen)"
    exit 1
  fi
  if [[ ! -f "$BORG_PASSPHRASE_FILE" ]]; then
    log "FEHLER: Passphrase-Datei fehlt: $BORG_PASSPHRASE_FILE"
    exit 1
  fi

  export BORG_PASSPHRASE
  BORG_PASSPHRASE="$(<"$BORG_PASSPHRASE_FILE")"
  export BORG_LOCK_WAIT="$BORG_LOCK_WAIT"

  local archive="agathe-$(hostname)-$(date +%F_%H%M%S)"
  log "Platz lokal: $(df -h "$LOCAL_BORG_BASE" | tail -1)"

  run_borg_allow_warning "borg create" borg create --stats --compression zstd,6 \
    "$LOCAL_BORG_REPO::$archive" \
    / \
    /boot \
    /boot/efi \
    --numeric-ids \
    --one-file-system \
    --exclude /proc \
    --exclude /sys \
    --exclude /dev \
    --exclude /run \
    --exclude /tmp \
    --exclude /var/tmp \
    --exclude /mnt \
    --exclude /media \
    --exclude /var/cache \
    --exclude /var/lib/docker \
    --exclude /var/lib/containerd \
    --exclude /swapfile \
    --exclude /mnt/paperless \
    --exclude /mnt/paperless-consume \
    --exclude /mnt/paperless-backup \
    --exclude /mnt/nchsdhg_agathe \
    --exclude /mnt/groot \
    --exclude-caches

  run_borg_allow_warning "borg prune" borg prune -v --list "$LOCAL_BORG_REPO" \
    --keep-daily="$KEEP_DAILY" \
    --keep-weekly="$KEEP_WEEKLY" \
    --keep-monthly="$KEEP_MONTHLY" \
    --prefix "agathe-$(hostname)-"

  if [[ "$BORG_CHECK" == "1" ]]; then
    run_borg_allow_warning "borg check" borg check -v --verify-data "$LOCAL_BORG_REPO"
  fi

  log "Lokales Borg Backup fertig: $archive"
}

rsync_borg_mirror_to_cifs() {
  log "== Spiegel Borg Repo per rsync -> $REMOTE_BORG_MIRROR =="

  ensure_cifs
  mkdir -p "$REMOTE_BORG_MIRROR"

  local extra=()
  if [[ "$MIRROR_DELETE" == "1" ]]; then
    extra+=(--delete --delete-delay)
    log "Mirror: delete aktiv (Ziel wird an Quelle angepasst)."
  else
    log "Mirror: delete NICHT aktiv."
  fi

  rsync -rltvh --info=progress2 --no-owner --no-group --no-perms --temp-dir=/tmp \
    "${extra[@]}" \
    "$LOCAL_BORG_REPO/" \
    "$REMOTE_BORG_MIRROR/"
}

# -----------------------------
# Main
# -----------------------------
need_root
check_deps
acquire_lock

send_pushover "Backup START auf $(hostname) um $(ts_now)."

bringup_wg
mount_cifs

rsync_and_copy_to_consume_flat "$SRC_GROOT" "$DEST_GROOT" "GROOT"
rsync_and_copy_to_consume_flat "$SRC_ANMELDUNG" "$DEST_ANMELDUNG" "ANMELDUNG"

rsync_paperless_backup
rsync_tomedo_backup

borg_local_backup
rsync_borg_mirror_to_cifs

log "== Fertig. =="
send_pushover "Backup ENDE auf $(hostname) um $(ts_now)."