From 5d356e0698ae5d9f308886657a90f3ce7869fcb5 Mon Sep 17 00:00:00 2001 From: "sebastian.serfling" Date: Wed, 18 Mar 2026 12:57:27 +0000 Subject: [PATCH] restore.sh aktualisiert --- restore.sh | 156 ++++++++++++++++++++++++++--------------------------- 1 file changed, 76 insertions(+), 80 deletions(-) diff --git a/restore.sh b/restore.sh index ed8c25d..7835e94 100644 --- a/restore.sh +++ b/restore.sh @@ -41,21 +41,22 @@ RSYNC_TARGET="" PBS_STORAGE="" WEBHOOK_URL="" WEBHOOK_TOKEN="" +SERVER_HOSTNAME="" BACKUP_SIZE_BYTES=0 while [[ $# -gt 0 ]]; do case $1 in - --job-uuid) JOB_UUID="$2"; shift 2 ;; - --backup-path) BACKUP_PATH="$2"; shift 2 ;; - --client) CLIENT_NAME="$2"; shift 2 ;; - --restore-mount) RESTORE_MOUNT="$2"; shift 2 ;; - --restore-path) RESTORE_PATH="$2"; shift 2 ;; - --rsync-target) RSYNC_TARGET="$2"; shift 2 ;; - --pbs-storage) PBS_STORAGE="$2"; shift 2 ;; - --webhook-url) WEBHOOK_URL="$2"; shift 2 ;; - --webhook-token) WEBHOOK_TOKEN="$2"; shift 2 ;; - --server-hostname) SERVER_HOSTNAME="$2"; shift 2 ;; - --backup-size) BACKUP_SIZE_BYTES="$2"; shift 2 ;; + --job-uuid) JOB_UUID="$2"; shift 2 ;; + --backup-path) BACKUP_PATH="$2"; shift 2 ;; + --client) CLIENT_NAME="$2"; shift 2 ;; + --restore-mount) RESTORE_MOUNT="$2"; shift 2 ;; + --restore-path) RESTORE_PATH="$2"; shift 2 ;; + --rsync-target) RSYNC_TARGET="$2"; shift 2 ;; + --pbs-storage) PBS_STORAGE="$2"; shift 2 ;; + --webhook-url) WEBHOOK_URL="$2"; shift 2 ;; + --webhook-token) WEBHOOK_TOKEN="$2"; shift 2 ;; + --server-hostname) SERVER_HOSTNAME="$2"; shift 2 ;; + --backup-size) BACKUP_SIZE_BYTES="$2"; shift 2 ;; *) echo "Unbekannter Parameter: $1" >&2; exit 1 ;; esac done @@ -69,27 +70,27 @@ done echo "FEHLER: Restore-Mount '$RESTORE_MOUNT' existiert nicht!" >&2; exit 1 } +# Fallback SERVER_HOSTNAME +SERVER_HOSTNAME="${SERVER_HOSTNAME:-$(hostname -f 2>/dev/null || hostname)}" + # ── Logging ─────────────────────────────────────────────────────────────────── LOG_DIR="/opt/windmill-restore/logs" mkdir -p "$LOG_DIR" SAFE_CLIENT="${CLIENT_NAME//\//_}" SAFE_CLIENT="${SAFE_CLIENT//:/_}" -# LOG_FILE = gleicher Name wie nohup-Redirect aus Step D/E -# So landen alle Ausgaben in derselben Datei LOG_FILE="$LOG_DIR/${SAFE_CLIENT}.log" exec >> "$LOG_FILE" 2>&1 # ── Backup-Pfad zerlegen ────────────────────────────────────────────────────── -# Format: "tnp-Invest-GmbH:vm/100/2024-01-15T02:00:00Z" -# oder "Jaehler-GmbH:ct/105/2024-01-15T06:00:00Z" DATASTORE=$(echo "$BACKUP_PATH" | cut -d: -f1) SNAPSHOT_PATH=$(echo "$BACKUP_PATH" | cut -d: -f2-) -BACKUP_TYPE=$(echo "$SNAPSHOT_PATH" | cut -d/ -f1) # "vm" oder "ct" +BACKUP_TYPE=$(echo "$SNAPSHOT_PATH" | cut -d/ -f1) PVE_BACKUP_REF="${PBS_STORAGE}:backup/${SNAPSHOT_PATH}" # ── Messvariablen ───────────────────────────────────────────────────────────── LAST_DATE=$(date +"%Y-%m-%d" -d "1 day ago") -# STI-BAC01: rsync_target ist lokal gemountet → ZIP direkt dorthin, kein Rsync + +# STI-BAC01: rsync_target lokal gemountet → ZIP direkt dorthin, kein Rsync if [[ "$SERVER_HOSTNAME" == "STI-BAC01" ]]; then ZIP_DIR="${RSYNC_TARGET}/${LAST_DATE}" SKIP_RSYNC=1 @@ -97,11 +98,10 @@ else ZIP_DIR="${RESTORE_MOUNT}/zips/${LAST_DATE}" SKIP_RSYNC=0 fi + BACKUP_SERVER_HOST=$(cat /opt/windmill-restore/backup_server_host 2>/dev/null \ || echo "backup-server") -# SERVER_HOSTNAME kommt als Parameter --server-hostname -# Fallback auf hostname -f falls nicht gesetzt -SERVER_HOSTNAME="${SERVER_HOSTNAME:-$(hostname -f 2>/dev/null || hostname)}" +KEY_DIR="/opt/windmill-restore/keys" RESTORE_START=$(date +%s) STATUS="success" @@ -119,9 +119,10 @@ RSYNC_RETRIES=0 QM_AGENT_OK="false" ZIP_FILE="" ZIP_PASSWORD="" +FREE_GB=0 echo "============================================================" -echo " Windmill Restore Worker v1.0.11" +echo " Windmill Restore Worker" echo " Client: $CLIENT_NAME" echo " Typ: $BACKUP_TYPE" echo " Datastore: $DATASTORE" @@ -130,14 +131,30 @@ echo " PBS-Storage: $PBS_STORAGE" echo " Restore-Mount: $RESTORE_MOUNT" echo " Restore-Path: $RESTORE_PATH" echo " Rsync-Target: $RSYNC_TARGET" +echo " Server: $SERVER_HOSTNAME" +echo " Skip-Rsync: $SKIP_RSYNC" echo " Job-UUID: $JOB_UUID" echo " Start: $(date '+%Y-%m-%d %H:%M:%S')" echo "============================================================" +# ── JSON Escape Funktion ────────────────────────────────────────────────────── +escape_json() { + local input="$1" + input="${input//\\/\\\\}" + input="${input//\"/\\\"}" + input="${input//$'\n'/\\n}" + input="${input//$'\r'/\\r}" + input="${input//$'\t'/\\t}" + echo "$input" +} + # ── Webhook-Funktion ────────────────────────────────────────────────────────── send_webhook() { local wh_status="$1" - local wh_error="${2:-}" + local wh_error + wh_error=$(escape_json "${2:-}") + local wh_vm_name + wh_vm_name=$(escape_json "${VM_NAME:-$SAFE_CLIENT}") local duration=$(( $(date +%s) - RESTORE_START )) local payload payload=$(printf '{ @@ -161,7 +178,7 @@ send_webhook() { "log_file": "%s" }' \ "$JOB_UUID" "$CLIENT_NAME" "$wh_status" "$wh_error" \ - "$SERVER_HOSTNAME" "$FREE_GB" "${VM_NAME:-$SAFE_CLIENT}" \ + "$SERVER_HOSTNAME" "$FREE_GB" "$wh_vm_name" \ "$VM_ID_ORIGINAL" "$VM_ID_RESTORED" \ "$duration" "$ACTUAL_DISK_BYTES" \ "$ZIP_SIZE_BYTES" "$ZIP_DURATION" \ @@ -170,18 +187,24 @@ send_webhook() { echo "" echo "==> Sende Webhook..." - local http_code - http_code=$(curl -s -o /dev/null -w "%{http_code}" \ + echo " Payload: $payload" + local http_response + http_response=$(curl -s -w "\n%{http_code}" \ -X POST "$WEBHOOK_URL" \ -H "Content-Type: application/json" \ ${WEBHOOK_TOKEN:+-H "Authorization: Bearer ${WEBHOOK_TOKEN}"} \ -d "$payload") + local http_code + http_code=$(echo "$http_response" | tail -1) + local http_body + http_body=$(echo "$http_response" | head -n -1) echo " HTTP: $http_code" + echo " Response: $http_body" [[ "$http_code" =~ ^2 ]] && echo " Webhook OK." \ || echo " WARNUNG: HTTP $http_code" } -# ── ERR-Trap: aufräumen und Webhook senden ──────────────────────────────────── +# ── ERR-Trap ────────────────────────────────────────────────────────────────── trap 'STATUS="failed" ERROR_LINE=$LINENO echo "" @@ -208,7 +231,6 @@ trap 'STATUS="failed" # ═════════════════════════════════════════════════════════════════════════════ echo "" echo "==> [0/13] 7z-Passwort vom PBS-Server holen ($PBS_HOST)..." -KEY_DIR="/opt/windmill-restore/keys" mkdir -p "$KEY_DIR" chmod 700 "$KEY_DIR" @@ -236,9 +258,6 @@ echo " 7z-Passwort geladen ✓" # ═════════════════════════════════════════════════════════════════════════════ # [1/13] SPACE-CHECK -# Prüft ob genug Platz für Restore + 50% für ZIP vorhanden ist. -# Benötigter Platz: backup_size_bytes * 1.5 aus dem Webhook-Parameter -# Falls nicht genug → Webhook mit failed senden und Exit # ═════════════════════════════════════════════════════════════════════════════ echo "" echo "==> [1/13] Prüfe freien Speicherplatz auf $RESTORE_MOUNT..." @@ -248,8 +267,6 @@ FREE_GB=$(( FREE_KB / 1024 / 1024 )) FREE_BYTES=$(( FREE_KB * 1024 )) echo " Frei: ${FREE_GB} GB" -# Benötigten Platz berechnen: Backup-Größe * 1.5 -# BACKUP_SIZE_BYTES kommt als Parameter --backup-size REQUIRED_BYTES=$(( BACKUP_SIZE_BYTES * 3 / 2 )) REQUIRED_GB=$(( REQUIRED_BYTES / 1024 / 1024 / 1024 )) echo " Benötigt: ~${REQUIRED_GB} GB (Restore + 50% für ZIP)" @@ -293,8 +310,8 @@ echo " Restore-ID: $VM_ID_RESTORED" # ═════════════════════════════════════════════════════════════════════════════ # [2.5/13] CONFIG-CHECK -# Config direkt aus PBS-Backup lesen (kein Restore nötig) um VM-Name zu -# ermitteln und zu prüfen ob ZIP bereits auf dem Backup-Server existiert. +# Config direkt aus PBS-Backup lesen um VM-Name zu ermitteln und zu prüfen +# ob ZIP bereits auf dem Backup-Server existiert → Restore überspringen # ═════════════════════════════════════════════════════════════════════════════ echo "" echo "==> [2.5/13] Config aus PBS-Backup lesen..." @@ -302,9 +319,6 @@ echo "==> [2.5/13] Config aus PBS-Backup lesen..." CONFIG_VM_NAME="" CONFIG_TMP="/tmp/pbs_config_${VM_ID_ORIGINAL}_$$.conf" -# proxmox-backup-client restore holt nur die Config-Datei aus dem Backup -# SNAPSHOT_PATH Format: "vm/100/2024-01-15T02:00:00Z" -# Config-Datei im Backup heißt "vm.conf" oder "pct.conf" if [[ "$BACKUP_TYPE" == "ct" ]]; then CONF_FILE_IN_BACKUP="pct.conf" NAME_KEY="^hostname:" @@ -317,7 +331,11 @@ export PBS_PASSWORD export PBS_REPOSITORY="${PBS_USER}@${PBS_HOST}:${DATASTORE}" SNAP_ID=$(echo "$SNAPSHOT_PATH" | cut -d/ -f3) -echo " Befehl: proxmox-backup-client restore --keyfile ${KEY_DIR}/${DATASTORE}.keyfile ${BACKUP_TYPE}/${VM_ID_ORIGINAL}/${SNAP_ID} ${CONF_FILE_IN_BACKUP} ${CONFIG_TMP}" +echo " Repository: $PBS_REPOSITORY" +echo " Snapshot: ${BACKUP_TYPE}/${VM_ID_ORIGINAL}/${SNAP_ID}" +echo " Config: $CONF_FILE_IN_BACKUP" +echo " Keyfile: ${KEY_DIR}/${DATASTORE}.keyfile" + proxmox-backup-client restore \ --keyfile "${KEY_DIR}/${DATASTORE}.keyfile" \ "${BACKUP_TYPE}/${VM_ID_ORIGINAL}/${SNAP_ID}" \ @@ -326,23 +344,20 @@ proxmox-backup-client restore \ 2>&1 || echo " WARNUNG: proxmox-backup-client restore fehlgeschlagen (exit $?)" if [[ -f "$CONFIG_TMP" ]]; then - CONFIG_VM_NAME=$(grep -m1 "$NAME_KEY" "$CONFIG_TMP" 2>/dev/null | awk -F': ' '{print $2}' | tr -d '[:space:]' || echo "") + CONFIG_VM_NAME=$(grep -m1 "$NAME_KEY" "$CONFIG_TMP" 2>/dev/null \ + | awk -F': ' '{print $2}' | tr -d '[:space:]' || echo "") rm -f "$CONFIG_TMP" - echo " VM-Name aus Config: ${CONFIG_VM_NAME:-unbekannt}" + echo " VM-Name: ${CONFIG_VM_NAME:-unbekannt}" else echo " Config nicht lesbar – überspringe ZIP-Check." fi -echo " VM-Name aus Config: ${CONFIG_VM_NAME:-unbekannt}" - -# Prüfen ob ZIP bereits auf Backup-Server vorhanden +# Prüfen ob ZIP bereits vorhanden if [[ -n "$CONFIG_VM_NAME" ]]; then + ZIP_CHECK="${RSYNC_TARGET}/${LAST_DATE}/${CONFIG_VM_NAME}-${VM_ID_ORIGINAL}.7z" if [[ "$SKIP_RSYNC" == "1" ]]; then - # Lokaler Modus: direkt prüfen - ZIP_CHECK="${RSYNC_TARGET}/${LAST_DATE}/${CONFIG_VM_NAME}-${VM_ID_ORIGINAL}.7z" if [[ -f "$ZIP_CHECK" ]]; then echo " ZIP bereits vorhanden (lokal): $ZIP_CHECK" - echo " Überspringe Restore – sende success Webhook." VM_NAME="$CONFIG_VM_NAME" ZIP_SIZE_BYTES=$(stat -c%s "$ZIP_CHECK" 2>/dev/null || echo "0") RSYNC_OK="true" @@ -353,12 +368,11 @@ if [[ -n "$CONFIG_VM_NAME" ]]; then exit 0 fi else - ZIP_CHECK="${RSYNC_TARGET}/${LAST_DATE}/${CONFIG_VM_NAME}-${VM_ID_ORIGINAL}.7z" if ssh "$BACKUP_SERVER_HOST" "test -f '$ZIP_CHECK'" 2>/dev/null; then echo " ZIP bereits vorhanden (remote): $ZIP_CHECK" - echo " Überspringe Restore – sende success Webhook." VM_NAME="$CONFIG_VM_NAME" - ZIP_SIZE_BYTES=$(ssh "$BACKUP_SERVER_HOST" "stat -c%s '$ZIP_CHECK'" 2>/dev/null || echo "0") + ZIP_SIZE_BYTES=$(ssh "$BACKUP_SERVER_HOST" \ + "stat -c%s '$ZIP_CHECK'" 2>/dev/null || echo "0") RSYNC_OK="true" RSYNC_SIZE_BYTES=$ZIP_SIZE_BYTES QM_AGENT_OK="skipped" @@ -367,13 +381,11 @@ if [[ -n "$CONFIG_VM_NAME" ]]; then exit 0 fi fi - echo " Kein vorhandenes ZIP gefunden – starte vollständigen Restore." + echo " Kein vorhandenes ZIP – starte vollständigen Restore." fi # ═════════════════════════════════════════════════════════════════════════════ # [3/13] RESTORE -# VM → qmrestore -# CT → pct restore # ═════════════════════════════════════════════════════════════════════════════ echo "" echo "==> [3/13] Restore vom PBS-Storage ($BACKUP_TYPE)..." @@ -400,9 +412,6 @@ echo " Restore abgeschlossen in ${RESTORE_DURATION}s" # ═════════════════════════════════════════════════════════════════════════════ # [4/13] IMAGE_DIR DYNAMISCH ERMITTELN -# VM → /mnt/HDD_5TB.1/images/${VM_ID_RESTORED} -# CT → /mnt/HDD_5TB.1/private/${VM_ID_RESTORED} -# oder /mnt/HDD_5TB.1/rootdir/${VM_ID_RESTORED} # ═════════════════════════════════════════════════════════════════════════════ echo "" echo "==> [4/13] Ermittle Image-Verzeichnis..." @@ -415,44 +424,43 @@ print(cfg.get('path', '')) if [[ -n "$STORAGE_BASE" ]]; then if [[ "$BACKUP_TYPE" == "ct" ]]; then - # CT auf dir-Storage: alle möglichen Pfade durchsuchen - # Reihenfolge: images/ → private/ → rootdir/ (je nach Storage-Konfiguration) IMAGE_DIR="" - for candidate in "${STORAGE_BASE}/images/${VM_ID_RESTORED}" "${STORAGE_BASE}/private/${VM_ID_RESTORED}" "${STORAGE_BASE}/rootdir/${VM_ID_RESTORED}"; do + for candidate in \ + "${STORAGE_BASE}/images/${VM_ID_RESTORED}" \ + "${STORAGE_BASE}/private/${VM_ID_RESTORED}" \ + "${STORAGE_BASE}/rootdir/${VM_ID_RESTORED}"; do if [[ -d "$candidate" ]] && [[ -n "$(ls -A "$candidate" 2>/dev/null)" ]]; then IMAGE_DIR="$candidate" - echo " CT-Image gefunden unter: $IMAGE_DIR" + echo " CT-Image gefunden: $IMAGE_DIR" break else echo " Nicht gefunden: $candidate" fi done - # Falls kein Verzeichnis gefunden → find als letzter Versuch if [[ -z "$IMAGE_DIR" ]]; then - IMAGE_DIR=$(find "$STORAGE_BASE" -maxdepth 2 -type d -name "$VM_ID_RESTORED" 2>/dev/null | head -1 || echo "") + IMAGE_DIR=$(find "$STORAGE_BASE" -maxdepth 2 -type d \ + -name "$VM_ID_RESTORED" 2>/dev/null | head -1 || echo "") [[ -n "$IMAGE_DIR" ]] && echo " CT-Image via find: $IMAGE_DIR" fi else IMAGE_DIR="${STORAGE_BASE}/images/${VM_ID_RESTORED}" fi else - # Fallback if [[ "$BACKUP_TYPE" == "ct" ]]; then IMAGE_DIR="/var/lib/vz/private/${VM_ID_RESTORED}" else IMAGE_DIR="/var/lib/vz/images/${VM_ID_RESTORED}" fi - echo " WARNUNG: Storage-Pfad nicht ermittelt, nutze Fallback: $IMAGE_DIR" + echo " WARNUNG: Storage-Pfad nicht ermittelt, Fallback: $IMAGE_DIR" fi -# Letzter Fallback falls IMAGE_DIR noch leer if [[ -z "$IMAGE_DIR" ]]; then if [[ "$BACKUP_TYPE" == "ct" ]]; then IMAGE_DIR="/var/lib/vz/private/${VM_ID_RESTORED}" else IMAGE_DIR="/var/lib/vz/images/${VM_ID_RESTORED}" fi - echo " WARNUNG: Kein Image-Verzeichnis gefunden, nutze Fallback: $IMAGE_DIR" + echo " WARNUNG: Fallback: $IMAGE_DIR" fi echo " Image-Dir: $IMAGE_DIR" @@ -481,8 +489,6 @@ echo " Images vorhanden ✓" # ═════════════════════════════════════════════════════════════════════════════ # [6/13] VORBEREITEN -# VM: unlock → stop → cdrom/ide0/net entfernen → Agent aktivieren -# CT: unlock → stop → net entfernen # ═════════════════════════════════════════════════════════════════════════════ echo "" echo "==> [6/13] Vorbereiten ($BACKUP_TYPE)..." @@ -491,7 +497,6 @@ if [[ "$BACKUP_TYPE" == "ct" ]]; then pct unlock "$VM_ID_RESTORED" 2>/dev/null || true pct stop "$VM_ID_RESTORED" 2>/dev/null || true sleep 3 - # Alle Netzwerkinterfaces entfernen for ((net=0; net<=10; net++)); do pct set "$VM_ID_RESTORED" --delete "net${net}" 2>/dev/null || true done @@ -511,8 +516,6 @@ fi # ═════════════════════════════════════════════════════════════════════════════ # [7/13] STARTEN & PRÜFEN -# VM: qm-Agent 120s warten -# CT: pct exec ping (vereinfacht) # ═════════════════════════════════════════════════════════════════════════════ echo "" echo "==> [7/13] Starte & prüfe ($BACKUP_TYPE)..." @@ -520,16 +523,14 @@ echo "==> [7/13] Starte & prüfe ($BACKUP_TYPE)..." if [[ "$BACKUP_TYPE" == "ct" ]]; then pct start "$VM_ID_RESTORED" 2>/dev/null || true sleep 10 - # CT: prüfen ob gestartet if pct status "$VM_ID_RESTORED" 2>/dev/null | grep -q "running"; then QM_AGENT_OK="true" echo " CT läuft ✓" - # Hostname aus CT holen CT_HOSTNAME=$(pct exec "$VM_ID_RESTORED" -- hostname 2>/dev/null || echo "unbekannt") echo " Hostname: $CT_HOSTNAME" else QM_AGENT_OK="false" - echo " CT nicht gestartet – läuft weiter." + echo " CT nicht gestartet." fi else qm start "$VM_ID_RESTORED" 2>/dev/null || true @@ -573,9 +574,6 @@ echo " Gestoppt." # ═════════════════════════════════════════════════════════════════════════════ # [9/13] CONFIG SICHERN -# VM: qemu-server.conf aus /etc/pve/qemu-server/ -# CT: lxc.conf aus /etc/pve/lxc/ -# Originale Config (mit Netzwerk) ins ZIP-Verzeichnis legen. # ═════════════════════════════════════════════════════════════════════════════ echo "" echo "==> [9/13] Config sichern..." @@ -583,16 +581,15 @@ echo "==> [9/13] Config sichern..." if [[ "$BACKUP_TYPE" == "ct" ]]; then PVE_CONF="/etc/pve/lxc/${VM_ID_RESTORED}.conf" CONF_FILENAME="lxc.conf" - # CT-Name aus Config lesen VM_NAME=$(grep -m1 "^hostname:" "$PVE_CONF" 2>/dev/null \ | awk -F': ' '{print $2}' | tr -d '[:space:]' \ - || echo "$SAFE_CLIENT") + || echo "${CONFIG_VM_NAME:-$SAFE_CLIENT}") else PVE_CONF="/etc/pve/qemu-server/${VM_ID_RESTORED}.conf" CONF_FILENAME="qemu-server.conf" VM_NAME=$(grep -m1 "^name:" "$PVE_CONF" 2>/dev/null \ | awk -F': ' '{print $2}' | tr -d '[:space:]' \ - || echo "$SAFE_CLIENT") + || echo "${CONFIG_VM_NAME:-$SAFE_CLIENT}") fi if [[ -f "$PVE_CONF" ]]; then @@ -634,7 +631,6 @@ RSYNC_TARGET_DATE="${RSYNC_TARGET}/${LAST_DATE}" echo "==> [11/13] Rsync / Datei-Transfer..." if [[ "$SKIP_RSYNC" == "1" ]]; then - # STI-BAC01: ZIP liegt bereits im Zielverzeichnis – kein Rsync nötig echo " Lokaler Modus: ZIP bereits in ${RSYNC_TARGET_DATE} – kein Rsync." RSYNC_OK="true" RSYNC_SIZE_BYTES=$ZIP_SIZE_BYTES