Harden CIFS mount detection and remount retry

scripts/agathe_backup.sh

@@ -1,9 +1,11 @@
 #!/usr/bin/env bash
+# MAIN-HINWEIS: Diese Version ist nun inkl. Tomedo-Backup.
 # Robust: WireGuard hoch, CIFS mounten (inkl. Stale-Handle-Fix), rsync,
 # nur erlaubte Dateitypen zusätzlich nach paperless-consume (flach, OHNE GIF),
 # Paperless-Backup rsync als echtes Sync (mit --delete),
 # Anmeldung ebenfalls sync + consume,
 # zusätzlich: Borg lokal (inkrementell, versioniert) + Mirror per rsync auf CIFS (mit delete),
+# zusätzlich: Tomedo-Backup per rsync,
 # robust gegen CIFS-Aussetzer: ensure_cifs + rsync temp-dir lokal + Mount-Fallback.
 # Zusätzlich: Pushover-Benachrichtigung bei Erfolg/Fehler.
 # Fix: Wenn wg0 schon aktiv ist, wird es nicht erneut gestartet und beim Cleanup
@@ -18,7 +20,6 @@ set -Eeuo pipefail
 WG_IF="wg0"
 VPN_TEST_IP="10.202.101.10"
 WG_WAS_STARTED_BY_SCRIPT=0
-PUSHOVER_FAILURE_SENT=0
 
 LOCK_FILE="/var/lock/agathe_backup.lock"
 
@@ -80,37 +81,38 @@ DRY_RUN_DELETE="0"
 # Welche Dateien nach paperless-consume? (OHNE GIF)
 CONSUME_EXT_REGEX='\.([Pp][Dd][Ff]|[Dd][Oo][Cc][Xx]?|[Xx][Ll][Ss][Xx]?|[Pp][Pp][Tt][Xx]?|[Oo][Dd][Tt]|[Oo][Dd][Ss]|[Oo][Dd][Pp]|[Rr][Tt][Ff]|[Tt][Xx][Tt]|[Cc][Ss][Vv]|[Mm][Dd]|[Hh][Tt][Mm][Ll]?|[Jj][Pp][Ee]?[Gg]|[Pp][Nn][Gg]|[Tt][Ii][Ff][Ff]?|[Ww][Ee][Bb][Pp]|[Hh][Ee][Ii][Cc])$'
 
+# -----------------------------
+# Tomedo Backup
+# -----------------------------
+TOMEDO_SRC_ROOT="/mnt/TomedoBackup"
+TOMEDO_LAST_FILE="${TOMEDO_SRC_ROOT}/lastFilesBackup"
+TOMEDO_DEST_ROOT="${CIFS_MOUNTPOINT}/TomedoBackup"
+TOMEDO_MACOS_EXCLUDES="${TOMEDO_DEST_ROOT}/macos.excludes"
+TOMEDO_FILES_EXCLUDES="${TOMEDO_DEST_ROOT}/files.excludes"
+TOMEDO_RSYNC_MAX_RETRIES=3
+TOMEDO_RSYNC_RETRY_SLEEP=5
+
 # -----------------------------
 # Hilfsfunktionen
 # -----------------------------
 log() { echo "[$(date +'%F %T')] $*"; }
+ts_now() { date +'%F %T'; }
 
 send_pushover() {
   local message="$1"
   local priority="${2:-0}"
 
   if command -v curl >/dev/null 2>&1; then
-    local rc
+    curl -s \
-    set +e
-    curl -fsS --max-time 20 --retry 2 --retry-delay 2 \
       --form-string "token=${PUSHOVER_API_TOKEN}" \
       --form-string "user=${PUSHOVER_USER_TOKEN}" \
       --form-string "title=${PUSHOVER_TITLE}" \
       --form-string "message=${message}" \
       --form-string "priority=${priority}" \
-      https://api.pushover.net/1/messages.json >/dev/null
+      https://api.pushover.net/1/messages.json >/dev/null || true
-    rc=$?
-    set -e
-    if [[ "$rc" -ne 0 ]]; then
-      log "WARNUNG: Pushover konnte nicht gesendet werden (curl Exit-Code: $rc)."
-      return 1
-    fi
   else
     log "WARNUNG: curl fehlt, Pushover konnte nicht gesendet werden."
-    return 1
   fi
-
-  return 0
 }
 
 need_root() {
@@ -137,7 +139,7 @@ cleanup() {
   set +e
   log "== Cleanup =="
 
-  if mountpoint -q "$CIFS_MOUNTPOINT"; then
+  if is_cifs_mounted; then
     log "Unmount: $CIFS_MOUNTPOINT"
     umount "$CIFS_MOUNTPOINT" 2>/dev/null || umount -l "$CIFS_MOUNTPOINT" || true
   else
@@ -160,22 +162,13 @@ on_error() {
   local exit_code=$?
   local line_no="${1:-unknown}"
   log "FEHLER in Zeile ${line_no}, Exit-Code ${exit_code}"
-  if send_pushover "Backup FEHLER auf $(hostname): Zeile ${line_no}, Exit-Code ${exit_code}" 1; then
+  send_pushover "Backup FEHLER auf $(hostname): Zeile ${line_no}, Exit-Code ${exit_code}" 1
-    PUSHOVER_FAILURE_SENT=1
-  fi
   trap - EXIT
   cleanup
   exit "$exit_code"
 }
 
 on_exit() {
-  local exit_code=$?
-  if [[ "$exit_code" -ne 0 && "$PUSHOVER_FAILURE_SENT" -eq 0 ]]; then
-    log "WARNUNG: Abbruch ohne ERR-Notification erkannt (Exit-Code ${exit_code}), sende Fallback-Pushover."
-    if send_pushover "Backup FEHLER auf $(hostname): Exit-Code ${exit_code} (EXIT trap)." 1; then
-      PUSHOVER_FAILURE_SENT=1
-    fi
-  fi
   cleanup
 }
 
@@ -183,7 +176,7 @@ trap 'on_error $LINENO' ERR
 trap on_exit EXIT
 
 check_deps() {
-  local deps=(wg-quick wg mount rsync ping mountpoint umount awk cp date basename tee mktemp sleep id mkdir rm dirname borg df curl hostname flock)
+  local deps=(wg-quick wg mount rsync ping mountpoint umount awk cp date basename tee mktemp sleep id mkdir rm dirname borg df curl hostname flock cat)
   for d in "${deps[@]}"; do
     command -v "$d" >/dev/null 2>&1 || { echo "Fehlt: $d"; exit 1; }
   done
@@ -225,10 +218,14 @@ _mount_cifs_with_opts() {
   fi
 }
 
+is_cifs_mounted() {
+  awk -v mp="$CIFS_MOUNTPOINT" '$2 == mp && $3 == "cifs" { found=1 } END { exit(found ? 0 : 1) }' /proc/mounts
+}
+
 mount_cifs() {
   log "== CIFS mount: $CIFS_SHARE -> $CIFS_MOUNTPOINT =="
 
-  if mountpoint -q "$CIFS_MOUNTPOINT"; then
+  if is_cifs_mounted; then
     log "Mountpoint ist gemountet -> versuche umount"
     umount "$CIFS_MOUNTPOINT" 2>/dev/null || umount -l "$CIFS_MOUNTPOINT" || true
   fi
@@ -248,34 +245,88 @@ mount_cifs() {
   _mount_cifs_with_opts "$opts_robust"
   local rc=$?
   set -e
-  if [[ $rc -eq 0 ]]; then
+  if [[ $rc -eq 0 ]] && is_cifs_mounted; then
     log "OK gemountet (robust)."
     return 0
   fi
+  if [[ $rc -eq 0 ]]; then
+    log "WARNUNG: mount meldet Erfolg, aber CIFS ist nicht stabil eingehängt."
+  fi
 
   log "WARNUNG: CIFS mount (robust) fehlgeschlagen (rc=$rc). Fallback auf Basis-Optionen."
   set +e
   _mount_cifs_with_opts "$opts_base"
   rc=$?
   set -e
-  if [[ $rc -eq 0 ]]; then
+  if [[ $rc -eq 0 ]] && is_cifs_mounted; then
     log "OK gemountet (base)."
     return 0
   fi
+  if [[ $rc -eq 0 ]]; then
+    log "WARNUNG: mount (base) meldet Erfolg, aber CIFS ist nicht stabil eingehängt."
+  fi
 
   log "FEHLER: CIFS mount fehlgeschlagen (rc=$rc)."
   exit 1
 }
 
 ensure_cifs() {
-  if ! mountpoint -q "$CIFS_MOUNTPOINT"; then
+  local attempt
-    log "WARNUNG: CIFS nicht gemountet -> remount"
+  for ((attempt=1; attempt<=3; attempt++)); do
+    if is_cifs_mounted; then
+      return 0
+    fi
+
+    log "WARNUNG: CIFS nicht gemountet -> remount (Versuch ${attempt}/3)"
     mount_cifs
-  fi
+    sleep 1
-  if ! mountpoint -q "$CIFS_MOUNTPOINT"; then
+
-    log "FEHLER: CIFS Remount fehlgeschlagen."
+    if is_cifs_mounted; then
-    exit 1
+      return 0
-  fi
+    fi
+  done
+
+  log "FEHLER: CIFS Remount fehlgeschlagen."
+  exit 1
+}
+
+run_rsync_with_cifs_retry() {
+  local label="$1"
+  shift
+
+  local attempt rc=0
+  for ((attempt=1; attempt<=TOMEDO_RSYNC_MAX_RETRIES; attempt++)); do
+    ensure_cifs
+    log "rsync ${label}: Versuch ${attempt}/${TOMEDO_RSYNC_MAX_RETRIES}"
+
+    set +e
+    rsync "$@"
+    rc=$?
+    set -e
+
+    if [[ "$rc" -eq 0 ]]; then
+      return 0
+    fi
+
+    if [[ "$rc" -eq 11 || "$rc" -eq 12 || "$rc" -eq 30 || "$rc" -eq 35 ]]; then
+      if [[ "$attempt" -lt "$TOMEDO_RSYNC_MAX_RETRIES" ]]; then
+        log "WARNUNG: rsync ${label} fehlgeschlagen (rc=$rc). Remount und Retry in ${TOMEDO_RSYNC_RETRY_SLEEP}s."
+        if is_cifs_mounted; then
+          umount "$CIFS_MOUNTPOINT" 2>/dev/null || umount -l "$CIFS_MOUNTPOINT" || true
+        fi
+        sleep "$TOMEDO_RSYNC_RETRY_SLEEP"
+        mount_cifs
+        continue
+      fi
+      log "FEHLER: rsync ${label} nach ${TOMEDO_RSYNC_MAX_RETRIES} Versuchen fehlgeschlagen (rc=$rc)."
+      return "$rc"
+    fi
+
+    log "FEHLER: rsync ${label} fehlgeschlagen (rc=$rc)."
+    return "$rc"
+  done
+
+  return "$rc"
 }
 
 copy_to_paperless_flat_if_allowed_ext() {
@@ -379,6 +430,68 @@ rsync_paperless_backup() {
     "$SRC_PAPERLESS_BACKUP" "$DEST_PAPERLESS_BACKUP"
 }
 
+rsync_tomedo_backup() {
+  ensure_cifs
+
+  log "== Tomedo Backup starten =="
+
+  if [[ ! -f "$TOMEDO_LAST_FILE" ]]; then
+    log "FEHLER: Datei fehlt: $TOMEDO_LAST_FILE"
+    exit 1
+  fi
+
+  local last_backup
+  last_backup="$(cat "$TOMEDO_LAST_FILE")"
+
+  if [[ -z "$last_backup" ]]; then
+    log "FEHLER: $TOMEDO_LAST_FILE ist leer"
+    exit 1
+  fi
+
+  local src_snapshot="${TOMEDO_SRC_ROOT}/${last_backup}/"
+  local dst_snapshot="${TOMEDO_DEST_ROOT}/${last_backup}"
+  local src_files="${TOMEDO_SRC_ROOT}/files/"
+  local dst_files="${TOMEDO_DEST_ROOT}/files"
+
+  if [[ ! -d "$src_snapshot" ]]; then
+    log "FEHLER: Tomedo Snapshot-Ordner fehlt: $src_snapshot"
+    exit 1
+  fi
+
+  if [[ ! -d "$src_files" ]]; then
+    log "FEHLER: Tomedo files-Ordner fehlt: $src_files"
+    exit 1
+  fi
+
+  if [[ ! -f "$TOMEDO_MACOS_EXCLUDES" ]]; then
+    log "FEHLER: Exclude-Datei fehlt: $TOMEDO_MACOS_EXCLUDES"
+    exit 1
+  fi
+
+  if [[ ! -f "$TOMEDO_FILES_EXCLUDES" ]]; then
+    log "FEHLER: Exclude-Datei fehlt: $TOMEDO_FILES_EXCLUDES"
+    exit 1
+  fi
+
+  mkdir -p "$dst_snapshot" "$dst_files"
+  local tomedo_rsync_opts=(-r -l -t -O --info=progress2 --temp-dir=/tmp)
+
+  log "== rsync Tomedo Snapshot: $src_snapshot -> $dst_snapshot =="
+  run_rsync_with_cifs_retry "Tomedo Snapshot" "${tomedo_rsync_opts[@]}" \
+    --exclude-from "$TOMEDO_MACOS_EXCLUDES" \
+    "$src_snapshot" \
+    "$dst_snapshot"
+
+  log "== rsync Tomedo Files: $src_files -> $dst_files =="
+  run_rsync_with_cifs_retry "Tomedo Files" "${tomedo_rsync_opts[@]}" \
+    --exclude-from "$TOMEDO_MACOS_EXCLUDES" \
+    --exclude-from "$TOMEDO_FILES_EXCLUDES" \
+    "$src_files" \
+    "$dst_files"
+
+  log "== Tomedo Backup fertig: $last_backup =="
+}
+
 # -----------------------------
 # Borg: lokal sichern + Mirror auf CIFS
 # -----------------------------
@@ -487,6 +600,8 @@ need_root
 check_deps
 acquire_lock
 
+send_pushover "Backup START auf $(hostname) um $(ts_now)."
+
 bringup_wg
 mount_cifs
 
@@ -494,9 +609,10 @@ rsync_and_copy_to_consume_flat "$SRC_GROOT" "$DEST_GROOT" "GROOT"
 rsync_and_copy_to_consume_flat "$SRC_ANMELDUNG" "$DEST_ANMELDUNG" "ANMELDUNG"
 
 rsync_paperless_backup
+rsync_tomedo_backup
 
 borg_local_backup
 rsync_borg_mirror_to_cifs
 
 log "== Fertig. =="
-send_pushover "Backup erfolgreich auf $(hostname) abgeschlossen."
+send_pushover "Backup ENDE auf $(hostname) um $(ts_now)."