Harden CIFS mount detection and remount retry

scripts/agathe_backup.sh

@@ -1,16 +1,18 @@
 #!/usr/bin/env bash
+# MAIN-HINWEIS: Diese Version ist nun inkl. Tomedo-Backup.
 # Robust: WireGuard hoch, CIFS mounten (inkl. Stale-Handle-Fix), rsync,
 # nur erlaubte Dateitypen zusätzlich nach paperless-consume (flach, OHNE GIF),
 # Paperless-Backup rsync als echtes Sync (mit --delete),
 # Anmeldung ebenfalls sync + consume,
 # zusätzlich: Borg lokal (inkrementell, versioniert) + Mirror per rsync auf CIFS (mit delete),
+# zusätzlich: Tomedo-Backup per rsync,
 # robust gegen CIFS-Aussetzer: ensure_cifs + rsync temp-dir lokal + Mount-Fallback.
 # Zusätzlich: Pushover-Benachrichtigung bei Erfolg/Fehler.
 # Fix: Wenn wg0 schon aktiv ist, wird es nicht erneut gestartet und beim Cleanup
 # nur dann beendet, wenn dieses Skript es selbst hochgebracht hat.
 # Zusätzlich: Schutz gegen parallele Ausführung per flock.
 
-set -euo pipefail
+set -Eeuo pipefail
 
 # -----------------------------
 # Konfiguration
@@ -79,10 +81,22 @@ DRY_RUN_DELETE="0"
 # Welche Dateien nach paperless-consume? (OHNE GIF)
 CONSUME_EXT_REGEX='\.([Pp][Dd][Ff]|[Dd][Oo][Cc][Xx]?|[Xx][Ll][Ss][Xx]?|[Pp][Pp][Tt][Xx]?|[Oo][Dd][Tt]|[Oo][Dd][Ss]|[Oo][Dd][Pp]|[Rr][Tt][Ff]|[Tt][Xx][Tt]|[Cc][Ss][Vv]|[Mm][Dd]|[Hh][Tt][Mm][Ll]?|[Jj][Pp][Ee]?[Gg]|[Pp][Nn][Gg]|[Tt][Ii][Ff][Ff]?|[Ww][Ee][Bb][Pp]|[Hh][Ee][Ii][Cc])$'
 
+# -----------------------------
+# Tomedo Backup
+# -----------------------------
+TOMEDO_SRC_ROOT="/mnt/TomedoBackup"
+TOMEDO_LAST_FILE="${TOMEDO_SRC_ROOT}/lastFilesBackup"
+TOMEDO_DEST_ROOT="${CIFS_MOUNTPOINT}/TomedoBackup"
+TOMEDO_MACOS_EXCLUDES="${TOMEDO_DEST_ROOT}/macos.excludes"
+TOMEDO_FILES_EXCLUDES="${TOMEDO_DEST_ROOT}/files.excludes"
+TOMEDO_RSYNC_MAX_RETRIES=3
+TOMEDO_RSYNC_RETRY_SLEEP=5
+
 # -----------------------------
 # Hilfsfunktionen
 # -----------------------------
 log() { echo "[$(date +'%F %T')] $*"; }
+ts_now() { date +'%F %T'; }
 
 send_pushover() {
   local message="$1"
@@ -125,7 +139,7 @@ cleanup() {
   set +e
   log "== Cleanup =="
 
-  if mountpoint -q "$CIFS_MOUNTPOINT"; then
+  if is_cifs_mounted; then
     log "Unmount: $CIFS_MOUNTPOINT"
     umount "$CIFS_MOUNTPOINT" 2>/dev/null || umount -l "$CIFS_MOUNTPOINT" || true
   else
@@ -149,6 +163,7 @@ on_error() {
   local line_no="${1:-unknown}"
   log "FEHLER in Zeile ${line_no}, Exit-Code ${exit_code}"
   send_pushover "Backup FEHLER auf $(hostname): Zeile ${line_no}, Exit-Code ${exit_code}" 1
+  trap - EXIT
   cleanup
   exit "$exit_code"
 }
@@ -161,7 +176,7 @@ trap 'on_error $LINENO' ERR
 trap on_exit EXIT
 
 check_deps() {
-  local deps=(wg-quick wg mount rsync ping mountpoint umount awk cp date basename tee mktemp sleep id mkdir rm dirname borg df curl hostname flock)
+  local deps=(wg-quick wg mount rsync ping mountpoint umount awk cp date basename tee mktemp sleep id mkdir rm dirname borg df curl hostname flock cat)
   for d in "${deps[@]}"; do
     command -v "$d" >/dev/null 2>&1 || { echo "Fehlt: $d"; exit 1; }
   done
@@ -203,10 +218,14 @@ _mount_cifs_with_opts() {
   fi
 }
 
+is_cifs_mounted() {
+  awk -v mp="$CIFS_MOUNTPOINT" '$2 == mp && $3 == "cifs" { found=1 } END { exit(found ? 0 : 1) }' /proc/mounts
+}
+
 mount_cifs() {
   log "== CIFS mount: $CIFS_SHARE -> $CIFS_MOUNTPOINT =="
 
-  if mountpoint -q "$CIFS_MOUNTPOINT"; then
+  if is_cifs_mounted; then
     log "Mountpoint ist gemountet -> versuche umount"
     umount "$CIFS_MOUNTPOINT" 2>/dev/null || umount -l "$CIFS_MOUNTPOINT" || true
   fi
@@ -226,34 +245,88 @@ mount_cifs() {
   _mount_cifs_with_opts "$opts_robust"
   local rc=$?
   set -e
-  if [[ $rc -eq 0 ]]; then
+  if [[ $rc -eq 0 ]] && is_cifs_mounted; then
     log "OK gemountet (robust)."
     return 0
   fi
+  if [[ $rc -eq 0 ]]; then
+    log "WARNUNG: mount meldet Erfolg, aber CIFS ist nicht stabil eingehängt."
+  fi
 
   log "WARNUNG: CIFS mount (robust) fehlgeschlagen (rc=$rc). Fallback auf Basis-Optionen."
   set +e
   _mount_cifs_with_opts "$opts_base"
   rc=$?
   set -e
-  if [[ $rc -eq 0 ]]; then
+  if [[ $rc -eq 0 ]] && is_cifs_mounted; then
     log "OK gemountet (base)."
     return 0
   fi
+  if [[ $rc -eq 0 ]]; then
+    log "WARNUNG: mount (base) meldet Erfolg, aber CIFS ist nicht stabil eingehängt."
+  fi
 
   log "FEHLER: CIFS mount fehlgeschlagen (rc=$rc)."
   exit 1
 }
 
 ensure_cifs() {
-  if ! mountpoint -q "$CIFS_MOUNTPOINT"; then
-    log "WARNUNG: CIFS nicht gemountet -> remount"
+  local attempt
+  for ((attempt=1; attempt<=3; attempt++)); do
+    if is_cifs_mounted; then
+      return 0
+    fi
+
+    log "WARNUNG: CIFS nicht gemountet -> remount (Versuch ${attempt}/3)"
     mount_cifs
-  fi
-  if ! mountpoint -q "$CIFS_MOUNTPOINT"; then
-    log "FEHLER: CIFS Remount fehlgeschlagen."
-    exit 1
-  fi
+    sleep 1
+
+    if is_cifs_mounted; then
+      return 0
+    fi
+  done
+
+  log "FEHLER: CIFS Remount fehlgeschlagen."
+  exit 1
+}
+
+run_rsync_with_cifs_retry() {
+  local label="$1"
+  shift
+
+  local attempt rc=0
+  for ((attempt=1; attempt<=TOMEDO_RSYNC_MAX_RETRIES; attempt++)); do
+    ensure_cifs
+    log "rsync ${label}: Versuch ${attempt}/${TOMEDO_RSYNC_MAX_RETRIES}"
+
+    set +e
+    rsync "$@"
+    rc=$?
+    set -e
+
+    if [[ "$rc" -eq 0 ]]; then
+      return 0
+    fi
+
+    if [[ "$rc" -eq 11 || "$rc" -eq 12 || "$rc" -eq 30 || "$rc" -eq 35 ]]; then
+      if [[ "$attempt" -lt "$TOMEDO_RSYNC_MAX_RETRIES" ]]; then
+        log "WARNUNG: rsync ${label} fehlgeschlagen (rc=$rc). Remount und Retry in ${TOMEDO_RSYNC_RETRY_SLEEP}s."
+        if is_cifs_mounted; then
+          umount "$CIFS_MOUNTPOINT" 2>/dev/null || umount -l "$CIFS_MOUNTPOINT" || true
+        fi
+        sleep "$TOMEDO_RSYNC_RETRY_SLEEP"
+        mount_cifs
+        continue
+      fi
+      log "FEHLER: rsync ${label} nach ${TOMEDO_RSYNC_MAX_RETRIES} Versuchen fehlgeschlagen (rc=$rc)."
+      return "$rc"
+    fi
+
+    log "FEHLER: rsync ${label} fehlgeschlagen (rc=$rc)."
+    return "$rc"
+  done
+
+  return "$rc"
 }
 
 copy_to_paperless_flat_if_allowed_ext() {
@@ -357,9 +430,91 @@ rsync_paperless_backup() {
     "$SRC_PAPERLESS_BACKUP" "$DEST_PAPERLESS_BACKUP"
 }
 
+rsync_tomedo_backup() {
+  ensure_cifs
+
+  log "== Tomedo Backup starten =="
+
+  if [[ ! -f "$TOMEDO_LAST_FILE" ]]; then
+    log "FEHLER: Datei fehlt: $TOMEDO_LAST_FILE"
+    exit 1
+  fi
+
+  local last_backup
+  last_backup="$(cat "$TOMEDO_LAST_FILE")"
+
+  if [[ -z "$last_backup" ]]; then
+    log "FEHLER: $TOMEDO_LAST_FILE ist leer"
+    exit 1
+  fi
+
+  local src_snapshot="${TOMEDO_SRC_ROOT}/${last_backup}/"
+  local dst_snapshot="${TOMEDO_DEST_ROOT}/${last_backup}"
+  local src_files="${TOMEDO_SRC_ROOT}/files/"
+  local dst_files="${TOMEDO_DEST_ROOT}/files"
+
+  if [[ ! -d "$src_snapshot" ]]; then
+    log "FEHLER: Tomedo Snapshot-Ordner fehlt: $src_snapshot"
+    exit 1
+  fi
+
+  if [[ ! -d "$src_files" ]]; then
+    log "FEHLER: Tomedo files-Ordner fehlt: $src_files"
+    exit 1
+  fi
+
+  if [[ ! -f "$TOMEDO_MACOS_EXCLUDES" ]]; then
+    log "FEHLER: Exclude-Datei fehlt: $TOMEDO_MACOS_EXCLUDES"
+    exit 1
+  fi
+
+  if [[ ! -f "$TOMEDO_FILES_EXCLUDES" ]]; then
+    log "FEHLER: Exclude-Datei fehlt: $TOMEDO_FILES_EXCLUDES"
+    exit 1
+  fi
+
+  mkdir -p "$dst_snapshot" "$dst_files"
+  local tomedo_rsync_opts=(-r -l -t -O --info=progress2 --temp-dir=/tmp)
+
+  log "== rsync Tomedo Snapshot: $src_snapshot -> $dst_snapshot =="
+  run_rsync_with_cifs_retry "Tomedo Snapshot" "${tomedo_rsync_opts[@]}" \
+    --exclude-from "$TOMEDO_MACOS_EXCLUDES" \
+    "$src_snapshot" \
+    "$dst_snapshot"
+
+  log "== rsync Tomedo Files: $src_files -> $dst_files =="
+  run_rsync_with_cifs_retry "Tomedo Files" "${tomedo_rsync_opts[@]}" \
+    --exclude-from "$TOMEDO_MACOS_EXCLUDES" \
+    --exclude-from "$TOMEDO_FILES_EXCLUDES" \
+    "$src_files" \
+    "$dst_files"
+
+  log "== Tomedo Backup fertig: $last_backup =="
+}
+
 # -----------------------------
 # Borg: lokal sichern + Mirror auf CIFS
 # -----------------------------
+run_borg_allow_warning() {
+  local label="$1"
+  shift
+
+  local rc
+  set +e
+  "$@"
+  rc=$?
+  set -e
+
+  if [[ "$rc" -ge 2 ]]; then
+    log "FEHLER: ${label} fehlgeschlagen (Exit-Code ${rc})."
+    return "$rc"
+  fi
+
+  if [[ "$rc" -eq 1 ]]; then
+    log "WARNUNG: ${label} mit Warnungen beendet (Exit-Code 1), mache weiter."
+  fi
+}
+
 borg_local_backup() {
   log "== Borg lokal Backup -> $LOCAL_BORG_REPO =="
 
@@ -379,7 +534,7 @@ borg_local_backup() {
   local archive="agathe-$(hostname)-$(date +%F_%H%M%S)"
   log "Platz lokal: $(df -h "$LOCAL_BORG_BASE" | tail -1)"
 
-  borg create --stats --compression zstd,6 \
+  run_borg_allow_warning "borg create" borg create --stats --compression zstd,6 \
     "$LOCAL_BORG_REPO::$archive" \
     / \
     /boot \
@@ -405,14 +560,14 @@ borg_local_backup() {
     --exclude /mnt/groot \
     --exclude-caches
 
-  borg prune -v --list "$LOCAL_BORG_REPO" \
+  run_borg_allow_warning "borg prune" borg prune -v --list "$LOCAL_BORG_REPO" \
     --keep-daily="$KEEP_DAILY" \
     --keep-weekly="$KEEP_WEEKLY" \
     --keep-monthly="$KEEP_MONTHLY" \
     --prefix "agathe-$(hostname)-"
 
   if [[ "$BORG_CHECK" == "1" ]]; then
-    borg check -v --verify-data "$LOCAL_BORG_REPO"
+    run_borg_allow_warning "borg check" borg check -v --verify-data "$LOCAL_BORG_REPO"
   fi
 
   log "Lokales Borg Backup fertig: $archive"
@@ -445,6 +600,8 @@ need_root
 check_deps
 acquire_lock
 
+send_pushover "Backup START auf $(hostname) um $(ts_now)."
+
 bringup_wg
 mount_cifs
 
@@ -452,9 +609,10 @@ rsync_and_copy_to_consume_flat "$SRC_GROOT" "$DEST_GROOT" "GROOT"
 rsync_and_copy_to_consume_flat "$SRC_ANMELDUNG" "$DEST_ANMELDUNG" "ANMELDUNG"
 
 rsync_paperless_backup
+rsync_tomedo_backup
 
 borg_local_backup
 rsync_borg_mirror_to_cifs
 
 log "== Fertig. =="
-send_pushover "Backup erfolgreich auf $(hostname) abgeschlossen."
+send_pushover "Backup ENDE auf $(hostname) um $(ts_now)."