511 lines
23 KiB
Bash
511 lines
23 KiB
Bash
#!/usr/bin/env bash
|
||
# =============================================================================
|
||
# /opt/windmill-restore/restore.sh
|
||
# Windmill Backup Restore Worker
|
||
# Version: 1.0.8
|
||
#
|
||
# WAS DIESES SCRIPT MACHT:
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Läuft direkt auf dem Proxmox Restore-Server, gestartet von Windmill per SSH
|
||
# non-blocking (nohup ... &). Komplett eigenständig, keine offene Verbindung.
|
||
#
|
||
# Alle Pfade kommen aus der Datenbank via Windmill:
|
||
# --restore-mount Mountpoint des Restore-Volumes z.B. /mnt/4TB
|
||
# --restore-path PVE-Storage-Name für qmrestore z.B. local-lvm
|
||
# --rsync-target Rsync-Ziel auf Backup-Server z.B. /backup/incoming/TNP
|
||
# --pbs-storage PVE-Storage-ID des PBS-Datastores z.B. pbs-tnp-invest-gmbh
|
||
#
|
||
# KEYS – beide werden per Rsync vom PBS-Server geholt (PBS_HOST aus pbs.conf):
|
||
# PBS Encrypt-Key: /root/Scripte/${DATASTORE}.keyfile → für qmrestore
|
||
# 7z-Passwort: /root/Scripte/password_7z.txt → für 7z-Archiv
|
||
# Format: "tnp-Invest-GmbH: Passwort123"
|
||
# Beide werden lokal gecacht – bei mehreren VMs desselben Kunden
|
||
# nur einmal vom PBS-Server geholt.
|
||
#
|
||
# ABLAUF:
|
||
# [0] Keys holen – Keyfile + 7z-Passwort per Rsync vom PBS-Server
|
||
# [1] Space-Check – Freier Platz auf restore-mount prüfen
|
||
# [2] VM-ID ermitteln – Original aus Backup-Pfad, Restore-ID ab 1000
|
||
# [3] qmrestore – Direkt vom PBS-Storage mit --keyfile
|
||
# [4] VM vorbereiten – unlock, cdrom/ide0 entfernen, alle Netzwerkkarten
|
||
# löschen, Agent aktivieren
|
||
# [5] VM starten & – 120s auf qm-Agent warten (10s Schritte)
|
||
# Agent prüfen Kein Agent = qm_agent_ok=false, KEIN Abbruch
|
||
# [6] VM stoppen – Sauberes Shutdown, nach 30s force-stop
|
||
# [7] 7z-Archiv – VM-Images verschlüsselt zippen (7z-Passwort)
|
||
# [8] Rsync – ZIP zum Backup-Server (Firmen-Zielverzeichnis)
|
||
# 3 Versuche + Größenvergleich
|
||
# [9] Aufräumen – VM destroy, ZIP löschen
|
||
# [10] Webhook – JSON → Windmill schreibt DB & startet nächsten
|
||
# =============================================================================
|
||
set -euo pipefail
|
||
|
||
# ── Konfigdatei laden (PBS-Credentials) ──────────────────────────────────────
|
||
# pbs.conf wird von Windmill Step C per SFTP deployt (chmod 600)
|
||
# Enthält: PBS_HOST, PBS_PORT, PBS_USER, PBS_PASSWORD, PBS_FINGERPRINT
|
||
CONF_FILE="/opt/windmill-restore/pbs.conf"
|
||
[[ ! -f "$CONF_FILE" ]] && { echo "FEHLER: $CONF_FILE fehlt!" >&2; exit 1; }
|
||
# shellcheck source=/dev/null
|
||
source "$CONF_FILE"
|
||
|
||
# ── Argument-Parser ───────────────────────────────────────────────────────────
|
||
# Kein --encrypt-key – Keys werden per Rsync vom PBS-Server geholt
|
||
JOB_UUID=""
|
||
BACKUP_PATH=""
|
||
CLIENT_NAME=""
|
||
RESTORE_MOUNT=""
|
||
RESTORE_PATH=""
|
||
RSYNC_TARGET=""
|
||
PBS_STORAGE=""
|
||
WEBHOOK_URL=""
|
||
WEBHOOK_TOKEN=""
|
||
|
||
while [[ $# -gt 0 ]]; do
|
||
case $1 in
|
||
--job-uuid) JOB_UUID="$2"; shift 2 ;;
|
||
--backup-path) BACKUP_PATH="$2"; shift 2 ;;
|
||
--client) CLIENT_NAME="$2"; shift 2 ;;
|
||
--restore-mount) RESTORE_MOUNT="$2"; shift 2 ;;
|
||
--restore-path) RESTORE_PATH="$2"; shift 2 ;;
|
||
--rsync-target) RSYNC_TARGET="$2"; shift 2 ;;
|
||
--pbs-storage) PBS_STORAGE="$2"; shift 2 ;;
|
||
--webhook-url) WEBHOOK_URL="$2"; shift 2 ;;
|
||
--webhook-token) WEBHOOK_TOKEN="$2"; shift 2 ;;
|
||
*) echo "Unbekannter Parameter: $1" >&2; exit 1 ;;
|
||
esac
|
||
done
|
||
|
||
# Pflichtparameter prüfen – kein Fallback
|
||
for var in JOB_UUID BACKUP_PATH CLIENT_NAME \
|
||
RESTORE_MOUNT RESTORE_PATH RSYNC_TARGET PBS_STORAGE WEBHOOK_URL; do
|
||
[[ -z "${!var}" ]] && { echo "FEHLER: --${var//_/-} fehlt" >&2; exit 1; }
|
||
done
|
||
|
||
[[ ! -d "$RESTORE_MOUNT" ]] && {
|
||
echo "FEHLER: Restore-Mount '$RESTORE_MOUNT' existiert nicht!" >&2
|
||
exit 1
|
||
}
|
||
|
||
# ── Logging ───────────────────────────────────────────────────────────────────
|
||
LOG_DIR="/opt/windmill-restore/logs"
|
||
mkdir -p "$LOG_DIR"
|
||
SAFE_CLIENT="${CLIENT_NAME//\//_}" # "vm/100" → "vm_100"
|
||
LOG_FILE="$LOG_DIR/${SAFE_CLIENT}_$(date +%Y%m%d_%H%M%S).log"
|
||
exec > >(tee -a "$LOG_FILE") 2>&1
|
||
|
||
# ── Backup-Pfad zerlegen ──────────────────────────────────────────────────────
|
||
# BACKUP_PATH Format: "tnp-Invest-GmbH:vm/100/2024-01-15T02:00:00Z"
|
||
# DATASTORE = "tnp-Invest-GmbH"
|
||
# SNAPSHOT_PATH = "vm/100/2024-01-15T02:00:00Z"
|
||
# PVE_BACKUP_REF = "pbs-tnp-invest-gmbh:backup/vm/100/2024-01-15T02:00:00Z"
|
||
DATASTORE=$(echo "$BACKUP_PATH" | cut -d: -f1)
|
||
SNAPSHOT_PATH=$(echo "$BACKUP_PATH" | cut -d: -f2)
|
||
PVE_BACKUP_REF="${PBS_STORAGE}:backup/${SNAPSHOT_PATH}"
|
||
|
||
# ── Pfade & Messvariablen ─────────────────────────────────────────────────────
|
||
ZIP_DIR="${RESTORE_MOUNT}/zips"
|
||
BACKUP_SERVER_HOST=$(cat /opt/windmill-restore/backup_server_host 2>/dev/null \
|
||
|| echo "backup-server")
|
||
|
||
RESTORE_START=$(date +%s)
|
||
STATUS="success"
|
||
ERROR_MSG=""
|
||
VM_ID_ORIGINAL=0
|
||
VM_ID_RESTORED=0
|
||
VM_NAME=""
|
||
ACTUAL_DISK_BYTES=0
|
||
ZIP_SIZE_BYTES=0
|
||
ZIP_DURATION=0
|
||
RSYNC_SIZE_BYTES=0
|
||
RSYNC_OK="true"
|
||
RSYNC_RETRIES=0
|
||
QM_AGENT_OK="false"
|
||
RESTORE_DURATION=0
|
||
ZIP_PASSWORD=""
|
||
KEYFILE_LOCAL=""
|
||
|
||
echo "============================================================"
|
||
echo " Windmill Restore Worker v1.0.8"
|
||
echo " Client: $CLIENT_NAME"
|
||
echo " Datastore: $DATASTORE"
|
||
echo " Backup: $BACKUP_PATH"
|
||
echo " PBS-Storage: $PBS_STORAGE"
|
||
echo " Restore-Mount: $RESTORE_MOUNT"
|
||
echo " Restore-Path: $RESTORE_PATH"
|
||
echo " Rsync-Target: $RSYNC_TARGET"
|
||
echo " Job-UUID: $JOB_UUID"
|
||
echo " Start: $(date '+%Y-%m-%d %H:%M:%S')"
|
||
echo "============================================================"
|
||
|
||
# ── Webhook-Funktion ──────────────────────────────────────────────────────────
|
||
send_webhook() {
|
||
local wh_status="$1"
|
||
local wh_error="${2:-}"
|
||
local duration=$(( $(date +%s) - RESTORE_START ))
|
||
local payload
|
||
payload=$(printf '{
|
||
"job_uuid": "%s",
|
||
"client_name": "%s",
|
||
"status": "%s",
|
||
"error_message": "%s",
|
||
"vm_id_original": %d,
|
||
"vm_id_restored": %d,
|
||
"restore_duration_sec": %d,
|
||
"actual_disk_used_bytes": %d,
|
||
"zip_size_bytes": %d,
|
||
"zip_duration_sec": %d,
|
||
"rsync_size_bytes": %d,
|
||
"rsync_ok": %s,
|
||
"rsync_retries": %d,
|
||
"qm_agent_ok": %s,
|
||
"log_file": "%s"
|
||
}' \
|
||
"$JOB_UUID" "$CLIENT_NAME" "$wh_status" "$wh_error" \
|
||
"$VM_ID_ORIGINAL" "$VM_ID_RESTORED" \
|
||
"$duration" "$ACTUAL_DISK_BYTES" \
|
||
"$ZIP_SIZE_BYTES" "$ZIP_DURATION" \
|
||
"$RSYNC_SIZE_BYTES" "$RSYNC_OK" "$RSYNC_RETRIES" \
|
||
"$QM_AGENT_OK" "$LOG_FILE")
|
||
|
||
echo ""
|
||
echo "==> Sende Webhook..."
|
||
local http_code
|
||
http_code=$(curl -s -o /dev/null -w "%{http_code}" \
|
||
-X POST "$WEBHOOK_URL" \
|
||
-H "Content-Type: application/json" \
|
||
${WEBHOOK_TOKEN:+-H "Authorization: Bearer ${WEBHOOK_TOKEN}"} \
|
||
-d "$payload")
|
||
echo " HTTP: $http_code"
|
||
[[ "$http_code" =~ ^2 ]] && echo " Webhook OK." \
|
||
|| echo " WARNUNG: HTTP $http_code"
|
||
}
|
||
|
||
trap 'STATUS="failed"
|
||
send_webhook "failed" "Abgebrochen in Zeile $LINENO – $LOG_FILE"' ERR
|
||
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
# [0/10] KEYS VOM PBS-SERVER HOLEN
|
||
# PBS_HOST kommt aus pbs.conf (bereits gesourct).
|
||
# Beide Dateien werden lokal gecacht – bei mehreren VMs desselben Datastores
|
||
# werden sie nur einmal geholt, nicht bei jeder VM erneut.
|
||
#
|
||
# PBS Encrypt-Keyfile: /root/Scripte/${DATASTORE}.keyfile
|
||
# → wird an qmrestore --keyfile übergeben
|
||
# → entschlüsselt die PBS-VM-Images beim Restore
|
||
#
|
||
# 7z-Passwort: /root/Scripte/password_7z.txt
|
||
# → Format: "tnp-Invest-GmbH: Passwort123"
|
||
# → wird an 7z -p übergeben zum Verschlüsseln des ZIP-Archivs
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
echo ""
|
||
echo "==> [0/10] Keys vom PBS-Server holen ($PBS_HOST)..."
|
||
KEY_DIR="/opt/windmill-restore/keys"
|
||
mkdir -p "$KEY_DIR"
|
||
chmod 700 "$KEY_DIR"
|
||
|
||
# ── PBS Encrypt-Keyfile ───────────────────────────────────────────────────────
|
||
KEYFILE_LOCAL="${KEY_DIR}/${DATASTORE}.keyfile"
|
||
if [[ -f "$KEYFILE_LOCAL" && -s "$KEYFILE_LOCAL" ]]; then
|
||
echo " Keyfile bereits vorhanden: $KEYFILE_LOCAL"
|
||
else
|
||
echo " Hole Keyfile: root@${PBS_HOST}:/root/Scripte/${DATASTORE}.keyfile"
|
||
rsync -az \
|
||
"root@${PBS_HOST}:/root/Scripte/${DATASTORE}.keyfile" \
|
||
"$KEYFILE_LOCAL" \
|
||
2>&1
|
||
chmod 600 "$KEYFILE_LOCAL"
|
||
echo " Keyfile gespeichert ✓"
|
||
fi
|
||
|
||
[[ ! -f "$KEYFILE_LOCAL" || ! -s "$KEYFILE_LOCAL" ]] && {
|
||
echo "FEHLER: Keyfile fehlt oder leer für Datastore '$DATASTORE'" >&2
|
||
exit 1
|
||
}
|
||
|
||
# ── 7z-Passwort aus password_7z.txt ──────────────────────────────────────────
|
||
PW_FILE_LOCAL="${KEY_DIR}/password_7z.txt"
|
||
if [[ ! -f "$PW_FILE_LOCAL" || ! -s "$PW_FILE_LOCAL" ]]; then
|
||
echo " Hole password_7z.txt: root@${PBS_HOST}:/root/Scripte/password_7z.txt"
|
||
rsync -az \
|
||
"root@${PBS_HOST}:/root/Scripte/password_7z.txt" \
|
||
"$PW_FILE_LOCAL" \
|
||
2>&1
|
||
chmod 600 "$PW_FILE_LOCAL"
|
||
echo " password_7z.txt gespeichert ✓"
|
||
else
|
||
echo " password_7z.txt bereits vorhanden."
|
||
fi
|
||
|
||
# Passwort für diesen Datastore extrahieren
|
||
# Format der Datei: "tnp-Invest-GmbH: Passwort123"
|
||
ZIP_PASSWORD=$(grep -m1 "^${DATASTORE}:" "$PW_FILE_LOCAL" \
|
||
| awk -F': ' '{print $2}' | tr -d '[:space:]')
|
||
|
||
[[ -z "$ZIP_PASSWORD" ]] && {
|
||
echo "FEHLER: Kein 7z-Passwort für '$DATASTORE' in password_7z.txt" >&2
|
||
exit 1
|
||
}
|
||
echo " 7z-Passwort geladen ✓"
|
||
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
# [1/10] SPACE-CHECK
|
||
# Nur Warnung – kein Abbruch. PBS-Restore bricht selbst ab bei vollem Disk.
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
echo ""
|
||
echo "==> [1/10] Prüfe freien Speicherplatz auf $RESTORE_MOUNT..."
|
||
mkdir -p "$ZIP_DIR"
|
||
FREE_KB=$(df "$RESTORE_MOUNT" 2>/dev/null | awk 'NR==2{print $4}' || echo "0")
|
||
FREE_GB=$(( FREE_KB / 1024 / 1024 ))
|
||
echo " Frei: ${FREE_GB} GB"
|
||
[[ $FREE_GB -lt 30 ]] && echo " WARNUNG: Weniger als 30 GB frei!"
|
||
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
# [2/10] VM-ID ERMITTELN
|
||
# Original aus Snapshot-Pfad (vm/100/... → 100)
|
||
# Restore-ID: erste freie ab 1000, prüft QEMU + LXC
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
echo ""
|
||
echo "==> [2/10] Ermittle VM-IDs..."
|
||
VM_ID_ORIGINAL=$(echo "$SNAPSHOT_PATH" | grep -oP '\d+' | head -1 || echo "0")
|
||
echo " Original VM-ID: $VM_ID_ORIGINAL"
|
||
|
||
VM_ID_RESTORED=$(
|
||
{
|
||
pvesh get /nodes/localhost/qemu --output-format json 2>/dev/null || echo "[]"
|
||
pvesh get /nodes/localhost/lxc --output-format json 2>/dev/null || echo "[]"
|
||
} | python3 -c "
|
||
import json, sys
|
||
data = []
|
||
for line in sys.stdin:
|
||
line = line.strip()
|
||
if line:
|
||
try: data.extend(json.loads(line))
|
||
except: pass
|
||
existing = {int(v.get('vmid', 0)) for v in data}
|
||
for i in range(1000, 2000):
|
||
if i not in existing:
|
||
print(i); break
|
||
" 2>/dev/null || echo "1000"
|
||
)
|
||
echo " Restore VM-ID: $VM_ID_RESTORED"
|
||
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
# [3/10] QMRESTORE VOM PBS-STORAGE
|
||
# Restore direkt vom registrierten PBS-PVE-Storage.
|
||
# --storage aus DB (restore_path), z.B. "local-lvm", "local"
|
||
# --keyfile lokal gecachtes Keyfile vom PBS-Server
|
||
# --unique 1 verhindert Konflikte mit bestehenden VM-Configs
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
echo ""
|
||
echo "==> [3/10] qmrestore vom PBS-Storage..."
|
||
echo " Backup-Ref: $PVE_BACKUP_REF"
|
||
echo " Storage: $RESTORE_PATH"
|
||
echo " Keyfile: $KEYFILE_LOCAL"
|
||
echo " VM-ID: $VM_ID_RESTORED"
|
||
|
||
RESTORE_START_INNER=$(date +%s)
|
||
|
||
qmrestore "$PVE_BACKUP_REF" "$VM_ID_RESTORED" \
|
||
--storage "$RESTORE_PATH" \
|
||
--keyfile "$KEYFILE_LOCAL" \
|
||
--unique 1 \
|
||
2>&1
|
||
|
||
RESTORE_DURATION=$(( $(date +%s) - RESTORE_START_INNER ))
|
||
echo " Restore abgeschlossen in ${RESTORE_DURATION}s"
|
||
|
||
VM_IMAGE_DIR="/var/lib/vz/images/${VM_ID_RESTORED}"
|
||
ACTUAL_DISK_BYTES=$(du -sb "$VM_IMAGE_DIR" 2>/dev/null | awk '{print $1}' || echo "0")
|
||
echo " Image-Größe: $(( ACTUAL_DISK_BYTES / 1024 / 1024 / 1024 )) GB"
|
||
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
# [4/10] VM VORBEREITEN
|
||
# unlock → stop → cdrom/ide0 entfernen → alle Netzwerkkarten (net0-net10)
|
||
# löschen → Agent aktivieren.
|
||
# Kein Netzwerk = kein versehentlicher Zugang zum Produktivnetz beim Test.
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
echo ""
|
||
echo "==> [4/10] VM vorbereiten..."
|
||
qm unlock "$VM_ID_RESTORED" 2>/dev/null || true
|
||
qm stop "$VM_ID_RESTORED" 2>/dev/null || true
|
||
sleep 3
|
||
|
||
qm set "$VM_ID_RESTORED" -delete cdrom 2>/dev/null || true
|
||
qm set "$VM_ID_RESTORED" -delete ide0 2>/dev/null || true
|
||
|
||
for ((net=0; net<=10; net++)); do
|
||
qm set "$VM_ID_RESTORED" -delete "net${net}" 2>/dev/null || true
|
||
done
|
||
|
||
qm set "$VM_ID_RESTORED" --agent "enabled=1,type=virtio" 2>/dev/null || true
|
||
echo " VM vorbereitet (Netzwerkkarten entfernt, Agent aktiviert)."
|
||
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
# [5/10] VM STARTEN & QM-AGENT PRÜFEN
|
||
# 120s in 10s-Schritten. Agent läuft über QEMU Guest Agent Channel –
|
||
# kein Netzwerk nötig. Kein Agent nach 120s = KEIN Abbruch,
|
||
# qm_agent_ok=false wird im Webhook an Windmill gemeldet → DB-Eintrag.
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
echo ""
|
||
echo "==> [5/10] Starte VM & warte auf qm-Agent (max. 120s)..."
|
||
qm start "$VM_ID_RESTORED" 2>/dev/null || true
|
||
|
||
AGENT_WAIT=0
|
||
AGENT_MAX=120
|
||
AGENT_INTERVAL=10
|
||
QM_AGENT_OK="false"
|
||
|
||
while [[ $AGENT_WAIT -lt $AGENT_MAX ]]; do
|
||
sleep $AGENT_INTERVAL
|
||
AGENT_WAIT=$(( AGENT_WAIT + AGENT_INTERVAL ))
|
||
echo -n " [${AGENT_WAIT}s/${AGENT_MAX}s] qm-Agent... "
|
||
if qm agent "$VM_ID_RESTORED" ping 2>/dev/null | grep -qi "pong\|ping"; then
|
||
QM_AGENT_OK="true"
|
||
echo "ONLINE ✓"
|
||
hostname_info=$(qm agent "$VM_ID_RESTORED" get-host-name 2>/dev/null \
|
||
| grep host-name | tr -d '"' || true)
|
||
fs_info=$(qm agent "$VM_ID_RESTORED" get-fsinfo 2>/dev/null \
|
||
| grep mountpoint | tr -d '"' || true)
|
||
echo " Hostname: ${hostname_info:-unbekannt}"
|
||
echo " Mountpoints: ${fs_info:-unbekannt}"
|
||
break
|
||
else
|
||
echo "nicht erreichbar."
|
||
fi
|
||
done
|
||
|
||
if [[ "$QM_AGENT_OK" == "false" ]]; then
|
||
echo " qm-Agent nach ${AGENT_MAX}s nicht erreichbar."
|
||
echo " → qm_agent_ok=false in DB, Restore läuft weiter."
|
||
fi
|
||
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
# [6/10] VM STOPPEN
|
||
# Sauberes Shutdown, nach 30s force-stop falls VM noch läuft.
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
echo ""
|
||
echo "==> [6/10] Stoppe VM..."
|
||
qm shutdown "$VM_ID_RESTORED" 2>/dev/null || true
|
||
sleep 30
|
||
qm stop "$VM_ID_RESTORED" --skiplock 1 2>/dev/null || true
|
||
echo " VM gestoppt."
|
||
sleep 5
|
||
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
# [7/10] 7Z-ARCHIV ERSTELLEN
|
||
# VM-Images aus /var/lib/vz/images/$VM_ID_RESTORED/* zippen.
|
||
# VM-Name aus qemu-server.conf lesen → ZIP bekommt echten Namen.
|
||
# Passwort kommt aus password_7z.txt (Schritt [0], pro Datastore).
|
||
# -mx=1 → schnellste Kompression (Images meist schon komprimiert)
|
||
# -mhe=on → Header-Verschlüsselung (Dateinamen ohne Key unsichtbar)
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
echo ""
|
||
echo "==> [7/10] Erstelle verschlüsseltes 7z-Archiv..."
|
||
|
||
VM_CONF="${VM_IMAGE_DIR}/qemu-server.conf"
|
||
VM_NAME=$(grep -m1 "^name:" "$VM_CONF" 2>/dev/null \
|
||
| awk -F': ' '{print $2}' | tr -d '[:space:]' \
|
||
|| echo "$SAFE_CLIENT")
|
||
echo " VM-Name: $VM_NAME"
|
||
|
||
ZIP_FILE="${ZIP_DIR}/${VM_NAME}_$(date +%Y%m%d).7z"
|
||
ZIP_START=$(date +%s)
|
||
|
||
7z a -t7z \
|
||
-mmt=4 \
|
||
-mx=1 \
|
||
-md=16M \
|
||
-p"${ZIP_PASSWORD}" \
|
||
-mhe=on \
|
||
"$ZIP_FILE" \
|
||
"${VM_IMAGE_DIR}/"* \
|
||
2>&1 | tail -5
|
||
|
||
ZIP_DURATION=$(( $(date +%s) - ZIP_START ))
|
||
ZIP_SIZE_BYTES=$(stat -c%s "$ZIP_FILE" 2>/dev/null || echo "0")
|
||
echo " ZIP: $(( ZIP_SIZE_BYTES / 1024 / 1024 )) MB in ${ZIP_DURATION}s"
|
||
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
# [8/10] RSYNC ZUM BACKUP-SERVER
|
||
# Zielverzeichnis aus DB (rsync_target) = Firmen-spezifischer Pfad.
|
||
# 3 Versuche mit 60s Pause + Größenvergleich lokal vs. remote.
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
echo ""
|
||
echo "==> [8/10] Rsync → ${BACKUP_SERVER_HOST}:${RSYNC_TARGET}..."
|
||
MAX_RETRIES=3
|
||
|
||
rsync_transfer() {
|
||
rsync -avz --progress --timeout=300 \
|
||
"$ZIP_FILE" \
|
||
"${BACKUP_SERVER_HOST}:${RSYNC_TARGET}/" \
|
||
2>&1
|
||
}
|
||
|
||
ssh "$BACKUP_SERVER_HOST" "mkdir -p '${RSYNC_TARGET}'" 2>/dev/null || true
|
||
|
||
while [[ $RSYNC_RETRIES -lt $MAX_RETRIES ]]; do
|
||
if rsync_transfer; then
|
||
RSYNC_OK="true"
|
||
RSYNC_SIZE_BYTES=$ZIP_SIZE_BYTES
|
||
echo " Rsync OK: $(( RSYNC_SIZE_BYTES / 1024 / 1024 )) MB"
|
||
break
|
||
else
|
||
RSYNC_RETRIES=$(( RSYNC_RETRIES + 1 ))
|
||
if [[ $RSYNC_RETRIES -lt $MAX_RETRIES ]]; then
|
||
echo " Fehlgeschlagen ($RSYNC_RETRIES/$MAX_RETRIES). Warte 60s..."
|
||
sleep 60
|
||
else
|
||
RSYNC_OK="false"
|
||
STATUS="failed"
|
||
ERROR_MSG="Rsync fehlgeschlagen nach ${RSYNC_RETRIES} Versuchen"
|
||
echo " FEHLER: $ERROR_MSG"
|
||
fi
|
||
fi
|
||
done
|
||
|
||
if [[ "$RSYNC_OK" == "true" ]]; then
|
||
REMOTE_SIZE=$(ssh "$BACKUP_SERVER_HOST" \
|
||
"stat -c%s '${RSYNC_TARGET}/$(basename "$ZIP_FILE")'" \
|
||
2>/dev/null || echo "0")
|
||
if [[ "$REMOTE_SIZE" != "$ZIP_SIZE_BYTES" ]]; then
|
||
echo " WARNUNG: Remote ${REMOTE_SIZE}B != lokal ${ZIP_SIZE_BYTES}B"
|
||
RSYNC_OK="false"
|
||
STATUS="failed"
|
||
ERROR_MSG="Größenabweichung: lokal=${ZIP_SIZE_BYTES} remote=${REMOTE_SIZE}"
|
||
else
|
||
echo " Größenprüfung OK: ${REMOTE_SIZE} Bytes."
|
||
fi
|
||
fi
|
||
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
# [9/10] AUFRÄUMEN
|
||
# VM destroy inkl. Disks (/var/lib/vz/images/$VM_ID wird durch destroy entfernt).
|
||
# Lokale ZIP löschen.
|
||
# Keys bleiben gecacht für nächste VM desselben Datastores/Kunden.
|
||
# ═════════════════════════════════════════════════════════════════════════════
|
||
echo ""
|
||
echo "==> [9/10] Aufräumen..."
|
||
qm destroy "$VM_ID_RESTORED" \
|
||
--destroy-unreferenced-disks 1 \
|
||
--purge 1 \
|
||
2>/dev/null || echo " VM $VM_ID_RESTORED nicht mehr vorhanden."
|
||
rm -f "$ZIP_FILE"
|
||
echo " VM ${VM_ID_RESTORED} entfernt, ZIP gelöscht."
|
||
echo " Keys gecacht in $KEY_DIR (für nächste VM desselben Kunden)."
|
||
|
||
# ── Zusammenfassung & Webhook ─────────────────────────────────────────────────
|
||
TOTAL=$(( $(date +%s) - RESTORE_START ))
|
||
echo ""
|
||
echo "============================================================"
|
||
echo " Status: $STATUS"
|
||
echo " Gesamtdauer: ${TOTAL}s"
|
||
echo " VM-Name: ${VM_NAME:-$SAFE_CLIENT}"
|
||
echo " Datastore: $DATASTORE"
|
||
echo " qm-Agent: $QM_AGENT_OK"
|
||
echo " Rsync: $RSYNC_OK (Versuche: $RSYNC_RETRIES)"
|
||
echo " ZIP: $(( ZIP_SIZE_BYTES / 1024 / 1024 )) MB"
|
||
[[ -n "$ERROR_MSG" ]] && echo " Fehler: $ERROR_MSG"
|
||
echo "============================================================"
|
||
|
||
trap - ERR
|
||
send_webhook "$STATUS" "$ERROR_MSG" |