#!/usr/bin/env bash
# =============================================================================
# /opt/windmill-restore/restore.sh
# Windmill Backup Restore Worker
# Version: 1.0.11
#
# Unterstützt sowohl VM (qm) als auch CT (pct) Backups.
# Backup-Typ wird automatisch aus dem Backup-Pfad erkannt (vm/ oder ct/).
#
# ABLAUF:
#   [0]  7z-Passwort holen  – password_7z.txt per Rsync vom PBS-Server
#   [1]  Space-Check        – Freier Platz auf restore-mount prüfen
#   [2]  ID ermitteln       – Original aus Backup-Pfad, Restore-ID ab 1000
#   [3]  Restore            – qmrestore (VM) oder pct restore (CT)
#   [4]  IMAGE_DIR          – Dynamisch aus PVE-Storage-Pfad ermitteln
#   [5]  Images prüfen      – Abbruch wenn leer/nicht vorhanden
#   [6]  Vorbereiten        – VM: unlock/cdrom/net entfernen/Agent
#                             CT: unlock/net entfernen
#   [7]  Starten & prüfen   – VM: qm-Agent 120s | CT: pct exec ping
#   [8]  Stoppen            – VM: qm shutdown | CT: pct stop
#   [9]  Config sichern     – Originale Config ins ZIP-Verzeichnis
#   [10] 7z-Archiv          – Images verschlüsselt zippen
#   [11] Rsync              – ZIP zum Backup-Server
#   [12] Aufräumen          – destroy, ZIP löschen
#   [13] Webhook            – JSON → Windmill
# =============================================================================
set -euo pipefail

# ── Konfigdatei laden ─────────────────────────────────────────────────────────
CONF_FILE="/opt/windmill-restore/pbs.conf"
[[ ! -f "$CONF_FILE" ]] && { echo "FEHLER: $CONF_FILE fehlt!" >&2; exit 1; }
source "$CONF_FILE"

# ── Argument-Parser ───────────────────────────────────────────────────────────
JOB_UUID=""
BACKUP_PATH=""
CLIENT_NAME=""
RESTORE_MOUNT=""
RESTORE_PATH=""
RSYNC_TARGET=""
PBS_STORAGE=""
WEBHOOK_URL=""
WEBHOOK_TOKEN=""
BACKUP_SIZE_BYTES=0

while [[ $# -gt 0 ]]; do
    case $1 in
        --job-uuid)      JOB_UUID="$2";      shift 2 ;;
        --backup-path)   BACKUP_PATH="$2";   shift 2 ;;
        --client)        CLIENT_NAME="$2";   shift 2 ;;
        --restore-mount) RESTORE_MOUNT="$2"; shift 2 ;;
        --restore-path)  RESTORE_PATH="$2";  shift 2 ;;
        --rsync-target)  RSYNC_TARGET="$2";  shift 2 ;;
        --pbs-storage)   PBS_STORAGE="$2";   shift 2 ;;
        --webhook-url)   WEBHOOK_URL="$2";   shift 2 ;;
        --webhook-token)    WEBHOOK_TOKEN="$2";    shift 2 ;;
        --server-hostname) SERVER_HOSTNAME="$2"; shift 2 ;;
        --backup-size)     BACKUP_SIZE_BYTES="$2"; shift 2 ;;
        *) echo "Unbekannter Parameter: $1" >&2; exit 1 ;;
    esac
done

for var in JOB_UUID BACKUP_PATH CLIENT_NAME \
           RESTORE_MOUNT RESTORE_PATH RSYNC_TARGET PBS_STORAGE WEBHOOK_URL; do
    [[ -z "${!var}" ]] && { echo "FEHLER: --${var//_/-} fehlt" >&2; exit 1; }
done

[[ ! -d "$RESTORE_MOUNT" ]] && {
    echo "FEHLER: Restore-Mount '$RESTORE_MOUNT' existiert nicht!" >&2; exit 1
}

# ── Logging ───────────────────────────────────────────────────────────────────
LOG_DIR="/opt/windmill-restore/logs"
mkdir -p "$LOG_DIR"
SAFE_CLIENT="${CLIENT_NAME//\//_}"
LOG_FILE="$LOG_DIR/${SAFE_CLIENT}_$(date +%Y%m%d_%H%M%S).log"
exec > >(tee -a "$LOG_FILE") 2>&1

# ── Backup-Pfad zerlegen ──────────────────────────────────────────────────────
# Format: "tnp-Invest-GmbH:vm/100/2024-01-15T02:00:00Z"
#      oder "Jaehler-GmbH:ct/105/2024-01-15T06:00:00Z"
DATASTORE=$(echo "$BACKUP_PATH" | cut -d: -f1)
SNAPSHOT_PATH=$(echo "$BACKUP_PATH" | cut -d: -f2-)
BACKUP_TYPE=$(echo "$SNAPSHOT_PATH" | cut -d/ -f1)   # "vm" oder "ct"
PVE_BACKUP_REF="${PBS_STORAGE}:backup/${SNAPSHOT_PATH}"

# ── Messvariablen ─────────────────────────────────────────────────────────────
LAST_DATE=$(date +"%Y-%m-%d" -d "1 day ago")
# STI-BAC01: rsync_target ist lokal gemountet → ZIP direkt dorthin, kein Rsync
if [[ "$SERVER_HOSTNAME" == "STI-BAC01" ]]; then
    ZIP_DIR="${RSYNC_TARGET}/${LAST_DATE}"
    SKIP_RSYNC=1
else
    ZIP_DIR="${RESTORE_MOUNT}/zips/${LAST_DATE}"
    SKIP_RSYNC=0
fi
BACKUP_SERVER_HOST=$(cat /opt/windmill-restore/backup_server_host 2>/dev/null \
    || echo "backup-server")
# SERVER_HOSTNAME kommt als Parameter --server-hostname
# Fallback auf hostname -f falls nicht gesetzt
SERVER_HOSTNAME="${SERVER_HOSTNAME:-$(hostname -f 2>/dev/null || hostname)}"

RESTORE_START=$(date +%s)
STATUS="success"
ERROR_MSG=""
VM_ID_ORIGINAL=0
VM_ID_RESTORED=0
VM_NAME=""
IMAGE_DIR=""
ACTUAL_DISK_BYTES=0
ZIP_SIZE_BYTES=0
ZIP_DURATION=0
RSYNC_SIZE_BYTES=0
RSYNC_OK="true"
RSYNC_RETRIES=0
QM_AGENT_OK="false"
ZIP_FILE=""
ZIP_PASSWORD=""

echo "============================================================"
echo " Windmill Restore Worker v1.0.11"
echo " Client:        $CLIENT_NAME"
echo " Typ:           $BACKUP_TYPE"
echo " Datastore:     $DATASTORE"
echo " Backup:        $BACKUP_PATH"
echo " PBS-Storage:   $PBS_STORAGE"
echo " Restore-Mount: $RESTORE_MOUNT"
echo " Restore-Path:  $RESTORE_PATH"
echo " Rsync-Target:  $RSYNC_TARGET"
echo " Job-UUID:      $JOB_UUID"
echo " Start:         $(date '+%Y-%m-%d %H:%M:%S')"
echo "============================================================"

# ── Webhook-Funktion ──────────────────────────────────────────────────────────
send_webhook() {
    local wh_status="$1"
    local wh_error="${2:-}"
    local duration=$(( $(date +%s) - RESTORE_START ))
    local payload
    payload=$(printf '{
  "job_uuid":               "%s",
  "client_name":            "%s",
  "status":                 "%s",
  "error_message":          "%s",
  "server_hostname":        "%s",
  "free_space_gb":          %d,
  "vm_name":                "%s",
  "vm_id_original":         %d,
  "vm_id_restored":         %d,
  "restore_duration_sec":   %d,
  "actual_disk_used_bytes": %d,
  "zip_size_bytes":         %d,
  "zip_duration_sec":       %d,
  "rsync_size_bytes":       %d,
  "rsync_ok":               %s,
  "rsync_retries":          %d,
  "qm_agent_ok":            %s,
  "log_file":               "%s"
}' \
        "$JOB_UUID" "$CLIENT_NAME" "$wh_status" "$wh_error" \
        "$SERVER_HOSTNAME" "$FREE_GB" "${VM_NAME:-$SAFE_CLIENT}" \
        "$VM_ID_ORIGINAL" "$VM_ID_RESTORED" \
        "$duration" "$ACTUAL_DISK_BYTES" \
        "$ZIP_SIZE_BYTES" "$ZIP_DURATION" \
        "$RSYNC_SIZE_BYTES" "$RSYNC_OK" "$RSYNC_RETRIES" \
        "$QM_AGENT_OK" "$LOG_FILE")

    echo ""
    echo "==> Sende Webhook..."
    local http_code
    http_code=$(curl -s -o /dev/null -w "%{http_code}" \
        -X POST "$WEBHOOK_URL" \
        -H "Content-Type: application/json" \
        ${WEBHOOK_TOKEN:+-H "Authorization: Bearer ${WEBHOOK_TOKEN}"} \
        -d "$payload")
    echo "    HTTP: $http_code"
    [[ "$http_code" =~ ^2 ]] && echo "    Webhook OK." \
                              || echo "    WARNUNG: HTTP $http_code"
}

# ── ERR-Trap: aufräumen und Webhook senden ────────────────────────────────────
trap 'STATUS="failed"
      ERROR_LINE=$LINENO
      echo ""
      echo "FEHLER in Zeile ${ERROR_LINE} – räume auf..."
      if [[ ${VM_ID_RESTORED:-0} -gt 0 ]]; then
          if [[ "$BACKUP_TYPE" == "ct" ]]; then
              pct stop    "$VM_ID_RESTORED" 2>/dev/null || true
              sleep 3
              pct destroy "$VM_ID_RESTORED" --purge 1 2>/dev/null || true
          else
              qm stop     "$VM_ID_RESTORED" --skiplock 1 2>/dev/null || true
              sleep 5
              qm destroy  "$VM_ID_RESTORED" \
                  --destroy-unreferenced-disks 1 --purge 1 2>/dev/null || true
          fi
          echo "    ${BACKUP_TYPE^^} ${VM_ID_RESTORED} entfernt."
      fi
      [[ -n "${ZIP_FILE:-}" && -f "$ZIP_FILE" ]] && rm -f "$ZIP_FILE"
      [[ -n "${IMAGE_DIR:-}" && -d "$IMAGE_DIR" ]] && rm -rf "$IMAGE_DIR"
      send_webhook "failed" "Abgebrochen in Zeile ${ERROR_LINE} – $LOG_FILE"' ERR

# ═════════════════════════════════════════════════════════════════════════════
# [0/13] 7Z-PASSWORT VOM PBS-SERVER HOLEN
# ═════════════════════════════════════════════════════════════════════════════
echo ""
echo "==> [0/13] 7z-Passwort vom PBS-Server holen ($PBS_HOST)..."
KEY_DIR="/opt/windmill-restore/keys"
mkdir -p "$KEY_DIR"
chmod 700 "$KEY_DIR"

PW_FILE_LOCAL="${KEY_DIR}/password_7z.txt"
if [[ ! -f "$PW_FILE_LOCAL" || ! -s "$PW_FILE_LOCAL" ]]; then
    echo "    Hole password_7z.txt..."
    rsync -az \
        -e "ssh -o StrictHostKeyChecking=no" \
        "root@${PBS_HOST}:/root/Scripte/password_7z.txt" \
        "$PW_FILE_LOCAL" \
        2>&1
    chmod 600 "$PW_FILE_LOCAL"
    echo "    password_7z.txt gespeichert ✓"
else
    echo "    password_7z.txt bereits vorhanden."
fi

ZIP_PASSWORD=$(grep -m1 "^${DATASTORE}:" "$PW_FILE_LOCAL" \
    | awk -F': ' '{print $2}' | tr -d '[:space:]')

[[ -z "$ZIP_PASSWORD" ]] && {
    echo "FEHLER: Kein 7z-Passwort für '$DATASTORE' in password_7z.txt" >&2; exit 1
}
echo "    7z-Passwort geladen ✓"

# ═════════════════════════════════════════════════════════════════════════════
# [1/13] SPACE-CHECK
# Prüft ob genug Platz für Restore + 50% für ZIP vorhanden ist.
# Benötigter Platz: backup_size_bytes * 1.5 aus dem Webhook-Parameter
# Falls nicht genug → Webhook mit failed senden und Exit
# ═════════════════════════════════════════════════════════════════════════════
echo ""
echo "==> [1/13] Prüfe freien Speicherplatz auf $RESTORE_MOUNT..."
mkdir -p "$ZIP_DIR"
FREE_KB=$(df "$RESTORE_MOUNT" 2>/dev/null | awk 'NR==2{print $4}' || echo "0")
FREE_GB=$(( FREE_KB / 1024 / 1024 ))
FREE_BYTES=$(( FREE_KB * 1024 ))
echo "    Frei: ${FREE_GB} GB"

# Benötigten Platz berechnen: Backup-Größe * 1.5
# BACKUP_SIZE_BYTES kommt als Parameter --backup-size
REQUIRED_BYTES=$(( BACKUP_SIZE_BYTES * 3 / 2 ))
REQUIRED_GB=$(( REQUIRED_BYTES / 1024 / 1024 / 1024 ))
echo "    Benötigt: ~${REQUIRED_GB} GB (Restore + 50% für ZIP)"

if [[ $BACKUP_SIZE_BYTES -gt 0 ]] && [[ $FREE_BYTES -lt $REQUIRED_BYTES ]]; then
    ERROR_MSG="Nicht genug Speicherplatz: ${FREE_GB}GB frei, ~${REQUIRED_GB}GB benötigt"
    echo "    FEHLER: $ERROR_MSG"
    trap - ERR
    send_webhook "failed" "$ERROR_MSG"
    exit 0
fi
[[ $FREE_GB -lt 50 ]] && echo "    WARNUNG: Weniger als 50 GB frei!"

# ═════════════════════════════════════════════════════════════════════════════
# [2/13] ID ERMITTELN
# ═════════════════════════════════════════════════════════════════════════════
echo ""
echo "==> [2/13] Ermittle IDs..."
VM_ID_ORIGINAL=$(echo "$SNAPSHOT_PATH" | grep -oP '\d+' | head -1 || echo "0")
echo "    Original-ID: $VM_ID_ORIGINAL (Typ: $BACKUP_TYPE)"

VM_ID_RESTORED=$(
    {
        pvesh get /nodes/localhost/qemu --output-format json 2>/dev/null || echo "[]"
        pvesh get /nodes/localhost/lxc  --output-format json 2>/dev/null || echo "[]"
    } | python3 -c "
import json, sys
data = []
for line in sys.stdin:
    line = line.strip()
    if line:
        try: data.extend(json.loads(line))
        except: pass
existing = {int(v.get('vmid', 0)) for v in data}
for i in range(1000, 2000):
    if i not in existing:
        print(i); break
" 2>/dev/null || echo "1000"
)
echo "    Restore-ID: $VM_ID_RESTORED"

# ═════════════════════════════════════════════════════════════════════════════
# [2.5/13] CONFIG-ONLY CHECK
# Config-only Restore um VM-Name zu ermitteln und zu prüfen ob ZIP bereits
# auf dem Backup-Server existiert → bei Fund: Restore überspringen
# ═════════════════════════════════════════════════════════════════════════════
echo ""
echo "==> [2.5/13] Config-only Restore zum Prüfen..."

CONFIG_VM_NAME=""
if [[ "$BACKUP_TYPE" == "ct" ]]; then
    pct restore "$VM_ID_RESTORED" "$PVE_BACKUP_REF"         --storage "$RESTORE_PATH"         --config-only 1         2>/dev/null || true
    CONFIG_VM_NAME=$(grep -m1 "^hostname:"         /etc/pve/lxc/${VM_ID_RESTORED}.conf 2>/dev/null         | awk -F': ' '{print $2}' | tr -d '[:space:]' || echo "")
    pct destroy "$VM_ID_RESTORED" --purge 1 2>/dev/null || true
else
    qmrestore "$PVE_BACKUP_REF" "$VM_ID_RESTORED"         --storage "$RESTORE_PATH"         --config-only 1         2>/dev/null || true
    CONFIG_VM_NAME=$(grep -m1 "^name:"         /etc/pve/qemu-server/${VM_ID_RESTORED}.conf 2>/dev/null         | awk -F': ' '{print $2}' | tr -d '[:space:]' || echo "")
    qm destroy "$VM_ID_RESTORED" --purge 1 2>/dev/null || true
fi

echo "    VM-Name aus Config: ${CONFIG_VM_NAME:-unbekannt}"

# Prüfen ob ZIP bereits auf Backup-Server vorhanden
if [[ -n "$CONFIG_VM_NAME" ]]; then
    if [[ "$SKIP_RSYNC" == "1" ]]; then
        # Lokaler Modus: direkt prüfen
        ZIP_CHECK="${RSYNC_TARGET}/${LAST_DATE}/${CONFIG_VM_NAME}-${VM_ID_ORIGINAL}.7z"
        if [[ -f "$ZIP_CHECK" ]]; then
            echo "    ZIP bereits vorhanden (lokal): $ZIP_CHECK"
            echo "    Überspringe Restore – sende success Webhook."
            VM_NAME="$CONFIG_VM_NAME"
            ZIP_SIZE_BYTES=$(stat -c%s "$ZIP_CHECK" 2>/dev/null || echo "0")
            RSYNC_OK="true"
            RSYNC_SIZE_BYTES=$ZIP_SIZE_BYTES
            QM_AGENT_OK="skipped"
            trap - ERR
            send_webhook "success" ""
            exit 0
        fi
    else
        ZIP_CHECK="${RSYNC_TARGET}/${LAST_DATE}/${CONFIG_VM_NAME}-${VM_ID_ORIGINAL}.7z"
        if ssh "$BACKUP_SERVER_HOST" "test -f '$ZIP_CHECK'" 2>/dev/null; then
            echo "    ZIP bereits vorhanden (remote): $ZIP_CHECK"
            echo "    Überspringe Restore – sende success Webhook."
            VM_NAME="$CONFIG_VM_NAME"
            ZIP_SIZE_BYTES=$(ssh "$BACKUP_SERVER_HOST"                 "stat -c%s '$ZIP_CHECK'" 2>/dev/null || echo "0")
            RSYNC_OK="true"
            RSYNC_SIZE_BYTES=$ZIP_SIZE_BYTES
            QM_AGENT_OK="skipped"
            trap - ERR
            send_webhook "success" ""
            exit 0
        fi
    fi
    echo "    Kein vorhandenes ZIP gefunden – starte vollständigen Restore."
fi

# ═════════════════════════════════════════════════════════════════════════════
# [3/13] RESTORE
# VM  → qmrestore
# CT  → pct restore
# ═════════════════════════════════════════════════════════════════════════════
echo ""
echo "==> [3/13] Restore vom PBS-Storage ($BACKUP_TYPE)..."
echo "    Backup-Ref: $PVE_BACKUP_REF"
echo "    Storage:    $RESTORE_PATH"
echo "    ID:         $VM_ID_RESTORED"

RESTORE_START_INNER=$(date +%s)

if [[ "$BACKUP_TYPE" == "ct" ]]; then
    pct restore "$VM_ID_RESTORED" "$PVE_BACKUP_REF" \
        --storage "$RESTORE_PATH" \
        --unique 1 \
        2>&1
else
    qmrestore "$PVE_BACKUP_REF" "$VM_ID_RESTORED" \
        --storage "$RESTORE_PATH" \
        --unique 1 \
        2>&1
fi

RESTORE_DURATION=$(( $(date +%s) - RESTORE_START_INNER ))
echo "    Restore abgeschlossen in ${RESTORE_DURATION}s"

# ═════════════════════════════════════════════════════════════════════════════
# [4/13] IMAGE_DIR DYNAMISCH ERMITTELN
# VM  → /mnt/HDD_5TB.1/images/${VM_ID_RESTORED}
# CT  → /mnt/HDD_5TB.1/private/${VM_ID_RESTORED}
#        oder /mnt/HDD_5TB.1/rootdir/${VM_ID_RESTORED}
# ═════════════════════════════════════════════════════════════════════════════
echo ""
echo "==> [4/13] Ermittle Image-Verzeichnis..."
STORAGE_BASE=$(pvesh get "/storage/${RESTORE_PATH}" --output-format json \
    2>/dev/null | python3 -c "
import json, sys
cfg = json.load(sys.stdin)
print(cfg.get('path', ''))
" 2>/dev/null || echo "")

if [[ -n "$STORAGE_BASE" ]]; then
    if [[ "$BACKUP_TYPE" == "ct" ]]; then
        # CT auf dir-Storage: alle möglichen Pfade durchsuchen
        # Reihenfolge: images/ → private/ → rootdir/ (je nach Storage-Konfiguration)
        IMAGE_DIR=""
        for candidate in             "${STORAGE_BASE}/images/${VM_ID_RESTORED}"             "${STORAGE_BASE}/private/${VM_ID_RESTORED}"             "${STORAGE_BASE}/rootdir/${VM_ID_RESTORED}"; do
            if [[ -d "$candidate" ]] && [[ -n "$(ls -A "$candidate" 2>/dev/null)" ]]; then
                IMAGE_DIR="$candidate"
                echo "    CT-Image gefunden unter: $IMAGE_DIR"
                break
            else
                echo "    Nicht gefunden: $candidate"
            fi
        done
        # Falls kein Verzeichnis gefunden → find als letzter Versuch
        if [[ -z "$IMAGE_DIR" ]]; then
            IMAGE_DIR=$(find "$STORAGE_BASE" -maxdepth 2 -type d                 -name "$VM_ID_RESTORED" 2>/dev/null | head -1 || echo "")
            [[ -n "$IMAGE_DIR" ]] && echo "    CT-Image via find: $IMAGE_DIR"
        fi
    else
        IMAGE_DIR="${STORAGE_BASE}/images/${VM_ID_RESTORED}"
    fi
else
    # Fallback
    if [[ "$BACKUP_TYPE" == "ct" ]]; then
        IMAGE_DIR="/var/lib/vz/private/${VM_ID_RESTORED}"
    else
        IMAGE_DIR="/var/lib/vz/images/${VM_ID_RESTORED}"
    fi
    echo "    WARNUNG: Storage-Pfad nicht ermittelt, nutze Fallback: $IMAGE_DIR"
fi

# Letzter Fallback falls IMAGE_DIR noch leer
if [[ -z "$IMAGE_DIR" ]]; then
    if [[ "$BACKUP_TYPE" == "ct" ]]; then
        IMAGE_DIR="/var/lib/vz/private/${VM_ID_RESTORED}"
    else
        IMAGE_DIR="/var/lib/vz/images/${VM_ID_RESTORED}"
    fi
    echo "    WARNUNG: Kein Image-Verzeichnis gefunden, nutze Fallback: $IMAGE_DIR"
fi
echo "    Image-Dir: $IMAGE_DIR"

ACTUAL_DISK_BYTES=$(du -sb "$IMAGE_DIR" 2>/dev/null | awk '{print $1}' || echo "0")
echo "    Image-Größe: $(( ACTUAL_DISK_BYTES / 1024 / 1024 / 1024 )) GB"

# ═════════════════════════════════════════════════════════════════════════════
# [5/13] IMAGES PRÜFEN
# ═════════════════════════════════════════════════════════════════════════════
echo ""
echo "==> [5/13] Prüfe Images..."
if [[ ! -d "$IMAGE_DIR" ]] || [[ -z "$(ls -A "$IMAGE_DIR" 2>/dev/null)" ]]; then
    ERROR_MSG="IMAGE_DIR leer oder nicht vorhanden: $IMAGE_DIR"
    echo "    FEHLER: $ERROR_MSG"
    if [[ "$BACKUP_TYPE" == "ct" ]]; then
        pct destroy "$VM_ID_RESTORED" --purge 1 2>/dev/null || true
    else
        qm destroy "$VM_ID_RESTORED" \
            --destroy-unreferenced-disks 1 --purge 1 2>/dev/null || true
    fi
    trap - ERR
    send_webhook "failed" "$ERROR_MSG"
    exit 0
fi
echo "    Images vorhanden ✓"

# ═════════════════════════════════════════════════════════════════════════════
# [6/13] VORBEREITEN
# VM: unlock → stop → cdrom/ide0/net entfernen → Agent aktivieren
# CT: unlock → stop → net entfernen
# ═════════════════════════════════════════════════════════════════════════════
echo ""
echo "==> [6/13] Vorbereiten ($BACKUP_TYPE)..."

if [[ "$BACKUP_TYPE" == "ct" ]]; then
    pct unlock "$VM_ID_RESTORED" 2>/dev/null || true
    pct stop   "$VM_ID_RESTORED" 2>/dev/null || true
    sleep 3
    # Alle Netzwerkinterfaces entfernen
    for ((net=0; net<=10; net++)); do
        pct set "$VM_ID_RESTORED" --delete "net${net}" 2>/dev/null || true
    done
    echo "    CT vorbereitet (Netzwerkkarten entfernt)."
else
    qm unlock "$VM_ID_RESTORED" 2>/dev/null || true
    qm stop   "$VM_ID_RESTORED" 2>/dev/null || true
    sleep 3
    qm set "$VM_ID_RESTORED" -delete cdrom 2>/dev/null || true
    qm set "$VM_ID_RESTORED" -delete ide0  2>/dev/null || true
    for ((net=0; net<=10; net++)); do
        qm set "$VM_ID_RESTORED" -delete "net${net}" 2>/dev/null || true
    done
    qm set "$VM_ID_RESTORED" --agent "enabled=1,type=virtio" 2>/dev/null || true
    echo "    VM vorbereitet (Netzwerkkarten entfernt, Agent aktiviert)."
fi

# ═════════════════════════════════════════════════════════════════════════════
# [7/13] STARTEN & PRÜFEN
# VM: qm-Agent 120s warten
# CT: pct exec ping (vereinfacht)
# ═════════════════════════════════════════════════════════════════════════════
echo ""
echo "==> [7/13] Starte & prüfe ($BACKUP_TYPE)..."

if [[ "$BACKUP_TYPE" == "ct" ]]; then
    pct start "$VM_ID_RESTORED" 2>/dev/null || true
    sleep 10
    # CT: prüfen ob gestartet
    if pct status "$VM_ID_RESTORED" 2>/dev/null | grep -q "running"; then
        QM_AGENT_OK="true"
        echo "    CT läuft ✓"
        # Hostname aus CT holen
        CT_HOSTNAME=$(pct exec "$VM_ID_RESTORED" -- hostname 2>/dev/null || echo "unbekannt")
        echo "    Hostname: $CT_HOSTNAME"
    else
        QM_AGENT_OK="false"
        echo "    CT nicht gestartet – läuft weiter."
    fi
else
    qm start "$VM_ID_RESTORED" 2>/dev/null || true
    AGENT_WAIT=0
    AGENT_MAX=120
    AGENT_INTERVAL=10
    while [[ $AGENT_WAIT -lt $AGENT_MAX ]]; do
        sleep $AGENT_INTERVAL
        AGENT_WAIT=$(( AGENT_WAIT + AGENT_INTERVAL ))
        echo -n "    [${AGENT_WAIT}s/${AGENT_MAX}s] qm-Agent... "
        if qm agent "$VM_ID_RESTORED" ping 2>/dev/null | grep -qi "pong\|ping"; then
            QM_AGENT_OK="true"
            echo "ONLINE ✓"
            hostname_info=$(qm agent "$VM_ID_RESTORED" get-host-name 2>/dev/null \
                | grep host-name | tr -d '"' || true)
            echo "    Hostname: ${hostname_info:-unbekannt}"
            break
        else
            echo "nicht erreichbar."
        fi
    done
    [[ "$QM_AGENT_OK" == "false" ]] && \
        echo "    qm-Agent nicht erreichbar – qm_agent_ok=false in DB."
fi

# ═════════════════════════════════════════════════════════════════════════════
# [8/13] STOPPEN
# ═════════════════════════════════════════════════════════════════════════════
echo ""
echo "==> [8/13] Stoppe $BACKUP_TYPE..."
if [[ "$BACKUP_TYPE" == "ct" ]]; then
    pct stop "$VM_ID_RESTORED" 2>/dev/null || true
    sleep 10
else
    qm shutdown "$VM_ID_RESTORED" 2>/dev/null || true
    sleep 30
    qm stop "$VM_ID_RESTORED" --skiplock 1 2>/dev/null || true
    sleep 5
fi
echo "    Gestoppt."

# ═════════════════════════════════════════════════════════════════════════════
# [9/13] CONFIG SICHERN
# VM: qemu-server.conf aus /etc/pve/qemu-server/
# CT: lxc.conf aus /etc/pve/lxc/
# Originale Config (mit Netzwerk) ins ZIP-Verzeichnis legen.
# ═════════════════════════════════════════════════════════════════════════════
echo ""
echo "==> [9/13] Config sichern..."

if [[ "$BACKUP_TYPE" == "ct" ]]; then
    PVE_CONF="/etc/pve/lxc/${VM_ID_RESTORED}.conf"
    CONF_FILENAME="lxc.conf"
    # CT-Name aus Config lesen
    VM_NAME=$(grep -m1 "^hostname:" "$PVE_CONF" 2>/dev/null \
        | awk -F': ' '{print $2}' | tr -d '[:space:]' \
        || echo "$SAFE_CLIENT")
else
    PVE_CONF="/etc/pve/qemu-server/${VM_ID_RESTORED}.conf"
    CONF_FILENAME="qemu-server.conf"
    VM_NAME=$(grep -m1 "^name:" "$PVE_CONF" 2>/dev/null \
        | awk -F': ' '{print $2}' | tr -d '[:space:]' \
        || echo "$SAFE_CLIENT")
fi

if [[ -f "$PVE_CONF" ]]; then
    cp "$PVE_CONF" "${IMAGE_DIR}/${CONF_FILENAME}"
    echo "    Config gesichert: ${IMAGE_DIR}/${CONF_FILENAME}"
else
    echo "    WARNUNG: Config nicht gefunden: $PVE_CONF"
fi
echo "    Name: $VM_NAME"

# ═════════════════════════════════════════════════════════════════════════════
# [10/13] 7Z-ARCHIV
# ═════════════════════════════════════════════════════════════════════════════
echo ""
echo "==> [10/13] Erstelle verschlüsseltes 7z-Archiv..."

ZIP_FILE="${ZIP_DIR}/${VM_NAME}-${VM_ID_ORIGINAL}.7z"
ZIP_START=$(date +%s)

7z a -t7z \
    -mmt=4 \
    -mx=1 \
    -md=16M \
    -p"${ZIP_PASSWORD}" \
    -mhe=on \
    "$ZIP_FILE" \
    "${IMAGE_DIR}/"* \
    2>&1 | tail -5

ZIP_DURATION=$(( $(date +%s) - ZIP_START ))
ZIP_SIZE_BYTES=$(stat -c%s "$ZIP_FILE" 2>/dev/null || echo "0")
echo "    ZIP: $(( ZIP_SIZE_BYTES / 1024 / 1024 )) MB in ${ZIP_DURATION}s"

# ═════════════════════════════════════════════════════════════════════════════
# [11/13] RSYNC ZUM BACKUP-SERVER
# ═════════════════════════════════════════════════════════════════════════════
echo ""
RSYNC_TARGET_DATE="${RSYNC_TARGET}/${LAST_DATE}"
echo "==> [11/13] Rsync / Datei-Transfer..."

if [[ "$SKIP_RSYNC" == "1" ]]; then
    # STI-BAC01: ZIP liegt bereits im Zielverzeichnis – kein Rsync nötig
    echo "    Lokaler Modus: ZIP bereits in ${RSYNC_TARGET_DATE} – kein Rsync."
    RSYNC_OK="true"
    RSYNC_SIZE_BYTES=$ZIP_SIZE_BYTES
    echo "    Groesse: $(( RSYNC_SIZE_BYTES / 1024 / 1024 )) MB"
else
    MAX_RETRIES=3

    rsync_transfer() {
        rsync -avz --progress --timeout=300 \
            "$ZIP_FILE" \
            "${BACKUP_SERVER_HOST}:${RSYNC_TARGET_DATE}/" \
            2>&1
    }

    ssh "$BACKUP_SERVER_HOST" "mkdir -p '${RSYNC_TARGET_DATE}'" 2>/dev/null || true

    while [[ $RSYNC_RETRIES -lt $MAX_RETRIES ]]; do
        if rsync_transfer; then
            RSYNC_OK="true"
            RSYNC_SIZE_BYTES=$ZIP_SIZE_BYTES
            echo "    Rsync OK: $(( RSYNC_SIZE_BYTES / 1024 / 1024 )) MB"
            break
        else
            RSYNC_RETRIES=$(( RSYNC_RETRIES + 1 ))
            if [[ $RSYNC_RETRIES -lt $MAX_RETRIES ]]; then
                echo "    Fehlgeschlagen ($RSYNC_RETRIES/$MAX_RETRIES). Warte 60s..."
                sleep 60
            else
                RSYNC_OK="false"
                STATUS="failed"
                ERROR_MSG="Rsync fehlgeschlagen nach ${RSYNC_RETRIES} Versuchen"
                echo "    FEHLER: $ERROR_MSG"
            fi
        fi
    done

    if [[ "$RSYNC_OK" == "true" ]]; then
        REMOTE_SIZE=$(ssh "$BACKUP_SERVER_HOST" \
            "stat -c%s '${RSYNC_TARGET_DATE}/$(basename "$ZIP_FILE")'" \
            2>/dev/null || echo "0")
        if [[ "$REMOTE_SIZE" != "$ZIP_SIZE_BYTES" ]]; then
            echo "    WARNUNG: Remote ${REMOTE_SIZE}B != lokal ${ZIP_SIZE_BYTES}B"
            RSYNC_OK="false"
            STATUS="failed"
            ERROR_MSG="Groessenabweichung: lokal=${ZIP_SIZE_BYTES} remote=${REMOTE_SIZE}"
        else
            echo "    Groessenprüfung OK: ${REMOTE_SIZE} Bytes."
        fi
    fi
fi

# ═════════════════════════════════════════════════════════════════════════════
# [12/13] AUFRÄUMEN
# ═════════════════════════════════════════════════════════════════════════════
echo ""
echo "==> [12/13] Aufräumen..."
if [[ "$BACKUP_TYPE" == "ct" ]]; then
    pct destroy "$VM_ID_RESTORED" --purge 1 \
        2>/dev/null || echo "    CT $VM_ID_RESTORED nicht mehr vorhanden."
else
    qm destroy "$VM_ID_RESTORED" \
        --destroy-unreferenced-disks 1 \
        --purge 1 \
        2>/dev/null || echo "    VM $VM_ID_RESTORED nicht mehr vorhanden."
fi
if [[ "$SKIP_RSYNC" == "1" ]]; then
    echo "    ${BACKUP_TYPE^^} ${VM_ID_RESTORED} entfernt. ZIP bleibt am Zielort."
else
    rm -f "$ZIP_FILE"
    echo "    ${BACKUP_TYPE^^} ${VM_ID_RESTORED} entfernt, ZIP gelöscht."
fi

# ── Zusammenfassung & Webhook ─────────────────────────────────────────────────
TOTAL=$(( $(date +%s) - RESTORE_START ))
echo ""
echo "============================================================"
echo " Status:        $STATUS"
echo " Typ:           $BACKUP_TYPE"
echo " Gesamtdauer:   ${TOTAL}s"
echo " Name:          ${VM_NAME:-$SAFE_CLIENT}"
echo " Image-Dir:     $IMAGE_DIR"
echo " qm-Agent/CT:   $QM_AGENT_OK"
echo " Rsync:         $RSYNC_OK (Versuche: $RSYNC_RETRIES)"
echo " ZIP:           $(( ZIP_SIZE_BYTES / 1024 / 1024 )) MB"
[[ -n "$ERROR_MSG" ]] && echo " Fehler:        $ERROR_MSG"
echo "============================================================"

trap - ERR
send_webhook "$STATUS" "$ERROR_MSG"