Harden CIFS mount detection and remount retry

scripts/agathe_backup.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# MAIN-HINWEIS: Diese Version ist nun inkl. Tomedo-Backup.
 # Robust: WireGuard hoch, CIFS mounten (inkl. Stale-Handle-Fix), rsync,
 # nur erlaubte Dateitypen zusätzlich nach paperless-consume (flach, OHNE GIF),
 # Paperless-Backup rsync als echtes Sync (mit --delete),
@@ -88,6 +89,8 @@ TOMEDO_LAST_FILE="${TOMEDO_SRC_ROOT}/lastFilesBackup"
 TOMEDO_DEST_ROOT="${CIFS_MOUNTPOINT}/TomedoBackup"
 TOMEDO_MACOS_EXCLUDES="${TOMEDO_DEST_ROOT}/macos.excludes"
 TOMEDO_FILES_EXCLUDES="${TOMEDO_DEST_ROOT}/files.excludes"
+TOMEDO_RSYNC_MAX_RETRIES=3
+TOMEDO_RSYNC_RETRY_SLEEP=5
 
 # -----------------------------
 # Hilfsfunktionen
@@ -136,7 +139,7 @@ cleanup() {
   set +e
   log "== Cleanup =="
 
-  if mountpoint -q "$CIFS_MOUNTPOINT"; then
+  if is_cifs_mounted; then
     log "Unmount: $CIFS_MOUNTPOINT"
     umount "$CIFS_MOUNTPOINT" 2>/dev/null || umount -l "$CIFS_MOUNTPOINT" || true
   else
@@ -215,10 +218,14 @@ _mount_cifs_with_opts() {
   fi
 }
 
+is_cifs_mounted() {
+  awk -v mp="$CIFS_MOUNTPOINT" '$2 == mp && $3 == "cifs" { found=1 } END { exit(found ? 0 : 1) }' /proc/mounts
+}
+
 mount_cifs() {
   log "== CIFS mount: $CIFS_SHARE -> $CIFS_MOUNTPOINT =="
 
-  if mountpoint -q "$CIFS_MOUNTPOINT"; then
+  if is_cifs_mounted; then
     log "Mountpoint ist gemountet -> versuche umount"
     umount "$CIFS_MOUNTPOINT" 2>/dev/null || umount -l "$CIFS_MOUNTPOINT" || true
   fi
@@ -238,34 +245,88 @@ mount_cifs() {
   _mount_cifs_with_opts "$opts_robust"
   local rc=$?
   set -e
-  if [[ $rc -eq 0 ]]; then
+  if [[ $rc -eq 0 ]] && is_cifs_mounted; then
     log "OK gemountet (robust)."
     return 0
   fi
+  if [[ $rc -eq 0 ]]; then
+    log "WARNUNG: mount meldet Erfolg, aber CIFS ist nicht stabil eingehängt."
+  fi
 
   log "WARNUNG: CIFS mount (robust) fehlgeschlagen (rc=$rc). Fallback auf Basis-Optionen."
   set +e
   _mount_cifs_with_opts "$opts_base"
   rc=$?
   set -e
-  if [[ $rc -eq 0 ]]; then
+  if [[ $rc -eq 0 ]] && is_cifs_mounted; then
     log "OK gemountet (base)."
     return 0
   fi
+  if [[ $rc -eq 0 ]]; then
+    log "WARNUNG: mount (base) meldet Erfolg, aber CIFS ist nicht stabil eingehängt."
+  fi
 
   log "FEHLER: CIFS mount fehlgeschlagen (rc=$rc)."
   exit 1
 }
 
 ensure_cifs() {
-  if ! mountpoint -q "$CIFS_MOUNTPOINT"; then
-    log "WARNUNG: CIFS nicht gemountet -> remount"
-    mount_cifs
+  local attempt
+  for ((attempt=1; attempt<=3; attempt++)); do
+    if is_cifs_mounted; then
+      return 0
     fi
-  if ! mountpoint -q "$CIFS_MOUNTPOINT"; then
+
+    log "WARNUNG: CIFS nicht gemountet -> remount (Versuch ${attempt}/3)"
+    mount_cifs
+    sleep 1
+
+    if is_cifs_mounted; then
+      return 0
+    fi
+  done
+
   log "FEHLER: CIFS Remount fehlgeschlagen."
   exit 1
+}
+
+run_rsync_with_cifs_retry() {
+  local label="$1"
+  shift
+
+  local attempt rc=0
+  for ((attempt=1; attempt<=TOMEDO_RSYNC_MAX_RETRIES; attempt++)); do
+    ensure_cifs
+    log "rsync ${label}: Versuch ${attempt}/${TOMEDO_RSYNC_MAX_RETRIES}"
+
+    set +e
+    rsync "$@"
+    rc=$?
+    set -e
+
+    if [[ "$rc" -eq 0 ]]; then
+      return 0
     fi
+
+    if [[ "$rc" -eq 11 || "$rc" -eq 12 || "$rc" -eq 30 || "$rc" -eq 35 ]]; then
+      if [[ "$attempt" -lt "$TOMEDO_RSYNC_MAX_RETRIES" ]]; then
+        log "WARNUNG: rsync ${label} fehlgeschlagen (rc=$rc). Remount und Retry in ${TOMEDO_RSYNC_RETRY_SLEEP}s."
+        if is_cifs_mounted; then
+          umount "$CIFS_MOUNTPOINT" 2>/dev/null || umount -l "$CIFS_MOUNTPOINT" || true
+        fi
+        sleep "$TOMEDO_RSYNC_RETRY_SLEEP"
+        mount_cifs
+        continue
+      fi
+      log "FEHLER: rsync ${label} nach ${TOMEDO_RSYNC_MAX_RETRIES} Versuchen fehlgeschlagen (rc=$rc)."
+      return "$rc"
+    fi
+
+    log "FEHLER: rsync ${label} fehlgeschlagen (rc=$rc)."
+    return "$rc"
+  done
+
+  return "$rc"
 }
 
 copy_to_paperless_flat_if_allowed_ext() {
@@ -413,15 +474,16 @@ rsync_tomedo_backup() {
   fi
 
   mkdir -p "$dst_snapshot" "$dst_files"
+  local tomedo_rsync_opts=(-r -l -t -O --info=progress2 --temp-dir=/tmp)
 
   log "== rsync Tomedo Snapshot: $src_snapshot -> $dst_snapshot =="
-  rsync -r -l -t -O --info=progress2 \
+  run_rsync_with_cifs_retry "Tomedo Snapshot" "${tomedo_rsync_opts[@]}" \
     --exclude-from "$TOMEDO_MACOS_EXCLUDES" \
     "$src_snapshot" \
     "$dst_snapshot"
 
   log "== rsync Tomedo Files: $src_files -> $dst_files =="
-  rsync -r -l -t -O --info=progress2 \
+  run_rsync_with_cifs_retry "Tomedo Files" "${tomedo_rsync_opts[@]}" \
     --exclude-from "$TOMEDO_MACOS_EXCLUDES" \
     --exclude-from "$TOMEDO_FILES_EXCLUDES" \
     "$src_files" \