From b078fe66e2d44c98cd5f19128973816cc5db178b Mon Sep 17 00:00:00 2001 From: Jeremy McClure Date: Mon, 13 Jul 2026 00:48:28 -0400 Subject: [PATCH] added proxmox backup script --- proxmox-backup/README.md | 286 ++++++++++++++++++ proxmox-backup/pve-backup.sh | 541 +++++++++++++++++++++++++++++++++++ 2 files changed, 827 insertions(+) create mode 100644 proxmox-backup/README.md create mode 100755 proxmox-backup/pve-backup.sh diff --git a/proxmox-backup/README.md b/proxmox-backup/README.md new file mode 100644 index 0000000..540812b --- /dev/null +++ b/proxmox-backup/README.md @@ -0,0 +1,286 @@ +# Proxmox VE Backup & Restore Tool + +A self-contained bash script for backing up and restoring all critical Proxmox VE configuration files. Archives are stored locally (never on the remote host) and can be used to rebuild a PVE node after boot drive failure. + +## Overview + +The script connects to a Proxmox node (locally or over SSH), collects configuration files from across the filesystem, packages them into a timestamped `.tar.gz` archive, and stores it on the machine running the script. Restore extracts a backup archive and pushes files back. + +## Requirements + +- **Source node:** Proxmox VE (any supported version) +- **Machine running the script:** Linux with `bash`, `tar`, `gzip` +- **Remote access:** SSH access to the Proxmox node (passwordless key recommended) +- **Permissions:** Root or passwordless sudo on the Proxmox node +- **rsync (optional):** Installed on both machines for incremental transfers — falls back to tar/cat/pipe if absent + +## Usage + +``` +Usage: pve-backup.sh [options] + +Commands: + backup Create a backup of Proxmox configuration + restore Restore Proxmox configuration from a backup + list List available backups + verify Verify a backup archive's integrity (SHA256) + +Backup options: + -d, --dir DIR Local backup directory (default: ./pve-backups) + -h, --host HOST Remote Proxmox host (SSH) + -u, --user USER SSH user (default: root) + -i, --identity FILE SSH identity file + +Restore / Verify options: + -d, --dir DIR Directory containing backup archive + -f, --file FILE Specific backup archive to restore/verify + -y, --yes Skip confirmation prompts (DANGEROUS) + +List options: + -d, --dir DIR Directory to list backups from +``` + +### Examples + +```bash +# Local backup (must be root on the Proxmox node) +./pve-backup.sh backup + +# Remote backup with custom directory +./pve-backup.sh backup -h 192.168.1.100 -d /mnt/backups + +# Remote backup as non-root user (uses sudo -n automatically) +./pve-backup.sh backup -h 192.168.1.100 -u admin -i ~/.ssh/id_rsa + +# Verify a backup archive +./pve-backup.sh verify -f pve-backups/pve-backup-prox1-20250101_120000.tar.gz + +# Restore the latest backup +./pve-backup.sh restore + +# Restore a specific backup archive +./pve-backup.sh restore -f /path/to/pve-backup-prox1-20250101_120000.tar.gz + +# Non-interactive restore (for automation) +./pve-backup.sh restore -f /path/to/backup.tar.gz -y + +# List all local backups +./pve-backup.sh list +``` + +### Defaults + +| Setting | Default | +|---|---| +| Backup directory | `./pve-backups` (relative to script location) | +| SSH user | `root` | +| Archive name format | `pve-backup-{hostname}-{YYYYMMDD}_{HHMMSS}.tar.gz` | + +## Transfer Method + +The script uses one of two transfer methods depending on what's available: + +### rsync (preferred) + +When connecting to a remote host, the script checks if `rsync` is installed on the remote side. If yes: + +- File-based configs (`/etc/pve/`, `/etc/network/interfaces`, APT sources, etc.) are transferred via `rsync -a` +- Subsequent backups only transfer changed files, reducing bandwidth and time +- Permissions and ownership are preserved natively by rsync +- If sudo is needed, `--rsync-path="sudo -n rsync"` is used automatically + +### tar/cat/pipe (fallback) + +If rsync is not available on the remote host, the script falls back to the original method: + +- `/etc/pve/` is tarballed on the remote, piped through SSH, and extracted locally +- Individual files are fetched with `cat > file` over SSH +- Directories (APT sources, sysctl, cron) are tarballed with `cp -a`, then piped through SSH +- Slightly slower on repeated runs since everything is transferred every time + +## What Gets Backed Up + +### /etc/pve/ (pmxcfs snapshot) + +The core Proxmox cluster filesystem. Transferred via `rsync -a` when available, otherwise tarballed and piped over SSH. Either way, runtime-only data is excluded: + +| Included | Description | +|---|---| +| `datacenter.cfg` | Datacenter-wide settings | +| `storage.cfg` | Storage pool definitions | +| `user.cfg` | Users, groups, and permissions | +| `domains.cfg` | Authentication realms (LDAP, AD, etc.) | +| `vzdump.cfg` | Default backup job settings | +| `nodes/{node}/qemu-server/*.conf` | VM configuration files | +| `nodes/{node}/lxc/*.conf` | LXC container configuration files | +| `nodes/{node}/pve-ssl.pem` | Web UI SSL certificate | +| `nodes/{node}/pve-ssl.key` | Web UI SSL private key | +| `nodes/{node}/pveproxy-ssl.pem` | Custom SSL certificate (if set) | +| `nodes/{node}/pveproxy-ssl.key` | Custom SSL private key (if set) | +| `sdn/` | Software Defined Network configs (VLANs, VNets, subnets) | +| `ha/` | High Availability resources and groups | +| `firewall/` | Datacenter and node-level firewall rules | +| `priv/tfa.cfg` | TFA tokens (restore with caution) | +| `priv/shadow` | User password hashes | + +| Excluded | Reason | +|---|---| +| `.rrd/` | Round-robin database (runtime stats, regenerated) | +| `priv/known_hosts` | SSH host keys (regenerated on boot) | +| `*.lock` | Runtime locks (ephemeral) | + +### Network Configuration + +| File | Description | +|---|---| +| `/etc/network/interfaces` | Bridge, bond, VLAN, and NIC definitions | +| `/etc/hostname` | System hostname | +| `/etc/hosts` | Static hostname resolution | +| `/etc/fstab` | Filesystem mount table (ZFS, NFS, CIFS mounts, etc.) | +| `/etc/resolv.conf` | DNS nameservers and search domains | + +### APT Sources + +| Path | Description | +|---|---| +| `/etc/apt/sources.list` | Main APT repository list | +| `/etc/apt/sources.list.d/` | Third-party and Proxmox enterprise/no-subscription repos | +| `/etc/apt/auth.conf` | APT authentication credentials | + +### Cluster Configuration + +| File | Description | +|---|---| +| `/etc/corosync/corosync.conf` | Corosync cluster ring and membership config | + +### Ceph Storage + +| File | Description | +|---|---| +| `/etc/ceph/ceph.conf` | Ceph cluster connection settings | + +### System Tuning + +| Path | Description | +|---|---| +| `/etc/sysctl.d/` | Kernel parameter overrides (network buffers, etc.) | +| `/etc/cron.d/` | System cron jobs | +| `/etc/cron.daily/` | Daily cron scripts | + +### Package Manifest + +| File | Contents | +|---|---| +| `packages.txt` | Full list of installed packages (`dpkg-query -W`) | +| `packages-manual.txt` | Manually installed packages (`apt-mark showmanual`) | +| `pveversion.txt` | Proxmox VE version string | + +### Firewall Rules + +| File | Description | +|---|---| +| `iptables.rules` | IPv4 iptables ruleset (via `iptables-save`) | +| `ip6tables.rules` | IPv6 ip6tables ruleset (via `ip6tables-save`) | + +### Metadata (META file inside archive) + +``` +hostname: proxmox1 +created: 2025-01-01T12:00:00Z +pveversion: pve-manager/8.2.4/123456789abc +pve_nodes: proxmox1 proxmox2 proxmox3 +``` + +## Restore Process + +1. Extracts the backup archive to a temporary directory +2. **Creates a rollback snapshot** of the current `/etc/pve` at `/var/tmp/pve-restore-rollback-{timestamp}/` (and also a tarball at `/etc/pve-rollback.tar.gz` on the remote node) +3. Restores `/etc/pve` by piping the archived tarball into `tar -x -C /etc/pve` on the target node +4. Restores network configs (`/etc/network/interfaces`, `hostname`, `hosts`, `resolv.conf`) by piping each file through SSH +5. Restores APT sources, sysctl, and cron overrides +6. Does **not** restore the package manifest — this is for reference only + +After restore: +- **Reboot recommended** — network and hostname changes apply at boot +- **Restart `pveproxy`** if the web UI behaves oddly after `/etc/pve` restore + +## SSH Non-Root Access (sudo) + +If you connect with a non-root SSH user (`-u admin`), the script automatically detects this and prefixes all remote commands with `sudo -n`. The remote user must have passwordless sudo configured: + +```bash +echo "admin ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/admin +``` + +If passwordless sudo is not available, the script exits with a clear error. + +## Archive Format + +Each backup is a gzipped tarball with this internal structure. The `/etc/pve/` data and sysconfig may be stored either as raw directories (when rsync was used) or as nested tarballs (when the fallback method was used): + +``` +pve-backup-proxmox1-20250101_120000.tar.gz +├── META # Metadata (hostname, date, version, nodes) +└── etc/ + ├── pve/ + │ ├── datacenter.cfg # (raw files — rsync) or + │ └── pve-config.tar.gz # (nested tarball — fallback) + ├── apt/ + │ ├── sources.list # (raw files — rsync) or + │ └── apt-sources.tar.gz # (nested tarball — fallback) + └── system/ + ├── interfaces # /etc/network/interfaces + ├── hostname # /etc/hostname + ├── hosts # /etc/hosts + ├── resolv.conf # /etc/resolv.conf + ├── fstab # /etc/fstab + ├── corosync.conf # /etc/corosync/corosync.conf + ├── ceph.conf # /etc/ceph/ceph.conf + ├── sysctl.d/ # (raw dir — rsync) or + ├── sysconfig.tar.gz # (nested tarball — fallback) + ├── packages.txt # Full package manifest + ├── packages-manual.txt # Manually installed packages + ├── pveversion.txt # pveversion output + ├── iptables.rules # iptables-save output + └── ip6tables.rules # ip6tables-save output +``` + +The restore logic handles both layouts transparently. + +## Verification + +Every backup automatically generates a `.sha256` checksum file alongside the archive. You can verify integrity at any time: + +```bash +# Verify the latest backup +./pve-backup.sh verify -f pve-backups/pve-backup-prox1-20250101_120000.tar.gz +``` + +The verify command checks: +1. The archive is a valid gzip'd tar file (no corruption) +2. The SHA256 checksum matches the stored `.sha256` file (no tampering) + +If no `.sha256` file exists (e.g., from older backups), one is generated on the spot. + +## Security Notes + +- **Backup over SSH:** All data is transferred over the encrypted SSH channel. No data is stored on the remote host. +- **Sensitive data:** The archive contains SSL private keys, password hashes (`/etc/pve/priv/shadow`), and TFA tokens. Treat backup archives as sensitive credentials. +- **SSH keys:** Use `-i` to specify an SSH identity file. Agent forwarding is not required. + +## Limitations + +- Package manifest is for reference only — the script does not install/restore packages +- Firewall rules are exported via `iptables-save` — restoration requires manual `iptables-restore` (PVE firewall is handled via `/etc/pve/firewall/` in the pve-config archive) +- Ceph config is backed up, but the Ceph monitor maps and OSD tree are not +- VM/CT disk images and backups are **not** included — use the built-in Proxmox Backup Server or `vzdump` for those + +## Future Enhancements (not yet implemented) + +These are left as exercises for the reader or future development: + +- **Retention / rotation** — `--keep N` flag to auto-prune backups older than N days (or keep only N most recent) +- **Cron / systemd timer** — drop-in unit for daily automated backups +- **Encryption** — GPG or `age` encrypt the archive before writing it to disk, decrypt on restore +- **Email / notification** — alert on backup failure or successful completion +- **Restore dry-run** — show what would be restored without actually pushing files diff --git a/proxmox-backup/pve-backup.sh b/proxmox-backup/pve-backup.sh new file mode 100755 index 0000000..11e6f26 --- /dev/null +++ b/proxmox-backup/pve-backup.sh @@ -0,0 +1,541 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BACKUP_DIR="$SCRIPT_DIR/pve-backups" +SSH_KEY="" +REMOTE_HOST="" +REMOTE_USER="root" +SUDO_CMD="" +HAS_RSYNC=0 +SCRIPT_NAME=$(basename "$0") + +usage() { + cat < [options] + +Commands: + backup Create a backup of Proxmox configuration + restore Restore Proxmox configuration from a backup + verify Verify a backup archive's integrity (SHA256) + list List available backups + +Backup options: + -d, --dir DIR Local backup directory (default: $SCRIPT_DIR/pve-backups) + -h, --host HOST Remote Proxmox host (SSH) + -u, --user USER SSH user (default: root) + -i, --identity FILE SSH identity file + +Restore / Verify options: + -d, --dir DIR Directory containing backup archive + -f, --file FILE Specific backup archive to restore/verify + -y, --yes Skip confirmation prompts (DANGEROUS) + +List options: + -d, --dir DIR Directory to list backups from (default: $BACKUP_DIR) + +Examples: + $SCRIPT_NAME backup + $SCRIPT_NAME backup -d /mnt/backup + $SCRIPT_NAME backup -h 192.168.1.100 -i ~/.ssh/id_rsa + $SCRIPT_NAME restore -f /mnt/backup/pve-backup-20250101_120000.tar.gz + $SCRIPT_NAME verify -f /mnt/backup/pve-backup-20250101_120000.tar.gz + $SCRIPT_NAME list +EOF + exit 1 +} + +log() { echo "[*] $*"; } +warn() { echo "[!] $*" >&2; } +die() { echo "[FATAL] $*" >&2; exit 1; } + +# Parse common args (consume them, return remaining via ${PARSE_REMAINING[@]}) +parse_common_args() { + local parsed=() + PARSE_REMAINING=() + while [[ $# -gt 0 ]]; do + case "$1" in + -d|--dir) BACKUP_DIR="$2"; shift 2 ;; + -h|--host) REMOTE_HOST="$2"; shift 2 ;; + -u|--user) REMOTE_USER="$2"; shift 2 ;; + -i|--identity) SSH_KEY="$2"; shift 2 ;; + -y|--yes) FORCE_YES=1; shift ;; + -f|--file) RESTORE_FILE="$2"; shift 2 ;; + --) shift; parsed+=("$@"); break ;; + -*) + PARSE_REMAINING=("$@") + return + ;; + *) parsed+=("$1"); shift ;; + esac + done + PARSE_REMAINING=("${parsed[@]}") +} + +build_ssh_cmd() { + local cmd=() + if [[ -n "$REMOTE_HOST" ]]; then + cmd=(ssh) + [[ -n "$SSH_KEY" ]] && cmd+=(-i "$SSH_KEY") + cmd+=("${REMOTE_USER}@${REMOTE_HOST}") + fi + echo "${cmd[@]}" +} + +build_ssh_rsync_cmd() { + local cmd="ssh" + [[ -n "$SSH_KEY" ]] && cmd+=" -i $SSH_KEY" + echo "$cmd" +} + +run_remote() { + local ssh_cmd + ssh_cmd=$(build_ssh_cmd) + if [[ -n "$ssh_cmd" ]]; then + $ssh_cmd $SUDO_CMD "$@" + else + ${SUDO_CMD}bash -c "$*" + fi +} + +run_remote_raw() { + local ssh_cmd + ssh_cmd=$(build_ssh_cmd) + if [[ -n "$ssh_cmd" ]]; then + $ssh_cmd "${SUDO_CMD}bash -s" <<< "$1" + else + ${SUDO_CMD}bash -c "$1" + fi +} + +check_proxmox() { + if [[ -z "$REMOTE_HOST" ]]; then + if [[ "$(id -u)" -ne 0 ]]; then + SUDO_CMD="sudo -n " + fi + elif [[ "$REMOTE_USER" != "root" ]]; then + SUDO_CMD="sudo -n " + fi + + if ! run_remote "test -f /etc/pve/datacenter.cfg" 2>/dev/null; then + local where="${REMOTE_HOST:-locally}" + local who="${REMOTE_USER:-$(whoami)}" + if [[ -n "$SUDO_CMD" ]]; then + die "Cannot read /etc/pve $where — user '$who' needs passwordless sudo" + else + die "Cannot read /etc/pve $where — this script must be run as root" + fi + fi +} + +check_rsync() { + HAS_RSYNC=0 + if [[ -z "$REMOTE_HOST" ]]; then + return + fi + if run_remote "command -v rsync" &>/dev/null; then + HAS_RSYNC=1 + log "rsync available on remote — using for incremental transfers" + else + log "rsync not found on remote — falling back to tar/cat transfer" + fi +} + +# rsync wrapper: copies remote path(s) into dest dir. +# Handles SSH, sudo (via --rsync-path), and both file and directory sources. +# Prefix src with 'dir:' to treat it as a directory (--delete). +rsync_pull() { + local src="$1" + local dest="$2" + local is_dir=0 + + if [[ "$HAS_RSYNC" -ne 1 || -z "$REMOTE_HOST" ]]; then + return 1 + fi + + if [[ "$src" == dir:* ]]; then + is_dir=1 + src="${src:4}" + fi + + local rsync_ssh + rsync_ssh=$(build_ssh_rsync_cmd) + + local rsync_opts=(-a) + [[ "$is_dir" -eq 1 ]] && rsync_opts+=(--delete) + [[ -n "$SUDO_CMD" ]] && rsync_opts+=(--rsync-path="${SUDO_CMD}rsync") + + mkdir -p "$dest" + rsync "${rsync_opts[@]}" -e "$rsync_ssh" \ + "${REMOTE_USER}@${REMOTE_HOST}:$src" "$dest" &>/dev/null +} + +collect_pve_configs() { + local temp_dir="$1" + local dest="$temp_dir/etc" + + mkdir -p "$dest/pve" "$dest/apt" "$dest/system" + + # ---- /etc/pve ---- + log "Backing up /etc/pve configuration..." + if rsync_pull "dir:/etc/pve/" "$dest/pve/" && [[ -f "$dest/pve/datacenter.cfg" ]]; then + # Clean up rsync'd noise + rm -rf "$dest/pve/.rrd" "$dest/pve/nodes/"*/qemu-server/*.lock \ + "$dest/pve/nodes/"*/lxc/*.lock "$dest/pve/priv/known_hosts" 2>/dev/null || true + else + # Fallback: tar-based snapshot via SSH + rm -rf "$dest/pve" 2>/dev/null; mkdir "$dest/pve" + run_remote_raw ' + set -euo pipefail + tmp=$(mktemp -d) + mkdir "$tmp/pve" + cd /etc/pve + tar cf - \ + --exclude="./nodes/*/qemu-server/*.lock" \ + --exclude="./nodes/*/lxc/*.lock" \ + --exclude="./priv/known_hosts" \ + --exclude=".rrd" \ + . 2>/dev/null | tar xf - -C "$tmp/pve" + tar czf - -C "$tmp" pve + rm -rf "$tmp" + ' > "$dest/pve/pve-config.tar.gz" 2>/dev/null + fi + + # ---- Network & system files ---- + log "Backing up network and system configs..." + local sysfiles=( + /etc/network/interfaces /etc/hostname /etc/hosts + /etc/resolv.conf /etc/mailname /etc/fstab + ) + local rsync_ok=1 + for f in "${sysfiles[@]}"; do + if ! rsync_pull "$f" "$dest/system/"; then + rsync_ok=0 + break + fi + done + if [[ "$rsync_ok" -eq 0 ]]; then + for f in "${sysfiles[@]}"; do + run_remote "cat $f 2>/dev/null" > "$dest/system/$(basename "$f")" 2>/dev/null || true + done + fi + + # ---- APT sources ---- + log "Backing up APT sources..." + if ! rsync_pull "dir:/etc/apt/sources.list.d/" "$dest/apt/" || \ + ! rsync_pull "/etc/apt/sources.list" "$dest/apt/" || \ + ! rsync_pull "/etc/apt/auth.conf" "$dest/apt/"; then + # Fallback + rm -rf "$dest/apt" 2>/dev/null; mkdir "$dest/apt" + run_remote_raw ' + set -euo pipefail + tmp=$(mktemp -d) + mkdir "$tmp/apt" + cp -a /etc/apt/sources.list "$tmp/apt/" 2>/dev/null || true + cp -a /etc/apt/sources.list.d "$tmp/apt/" 2>/dev/null || true + cp -a /etc/apt/auth.conf "$tmp/apt/" 2>/dev/null || true + tar czf - -C "$tmp" apt + rm -rf "$tmp" + ' > "$dest/apt/apt-sources.tar.gz" 2>/dev/null + fi + + # ---- Corosync ---- + log "Backing up corosync config..." + if ! rsync_pull "/etc/corosync/corosync.conf" "$dest/system/"; then + run_remote "cat /etc/corosync/corosync.conf 2>/dev/null" > "$dest/system/corosync.conf" 2>/dev/null || true + fi + + # ---- Ceph ---- + log "Backing up Ceph config..." + if ! rsync_pull "/etc/ceph/ceph.conf" "$dest/system/"; then + run_remote "cat /etc/ceph/ceph.conf 2>/dev/null" > "$dest/system/ceph.conf" 2>/dev/null || true + fi + + # ---- sysctl / cron ---- + log "Backing up sysctl and cron..." + if ! rsync_pull "dir:/etc/sysctl.d/" "$dest/system/sysctl.d/" || \ + ! rsync_pull "dir:/etc/cron.d/" "$dest/system/cron.d/" || \ + ! rsync_pull "dir:/etc/cron.daily/" "$dest/system/cron.daily/"; then + # Fallback + rm -rf "$dest/system/sysctl.d" "$dest/system/cron.d" "$dest/system/cron.daily" 2>/dev/null + run_remote_raw ' + tmp=$(mktemp -d) + mkdir "$tmp/etc" + cp -a /etc/sysctl.d "$tmp/etc/" 2>/dev/null || true + cp -a /etc/cron.d "$tmp/etc/" 2>/dev/null || true + cp -a /etc/cron.daily "$tmp/etc/" 2>/dev/null || true + tar czf - -C "$tmp" etc + rm -rf "$tmp" + ' > "$dest/system/sysconfig.tar.gz" 2>/dev/null + fi + + # ---- Package manifest (always via command) ---- + log "Recording package manifest..." + run_remote "dpkg-query -W 2>/dev/null || true" > "$dest/system/packages.txt" 2>/dev/null || true + run_remote "apt-mark showmanual 2>/dev/null || true" > "$dest/system/packages-manual.txt" 2>/dev/null || true + run_remote "pveversion 2>/dev/null || true" > "$dest/system/pveversion.txt" 2>/dev/null || true + + # ---- Firewall rules (always via command) ---- + log "Backing up firewall rules..." + run_remote "iptables-save 2>/dev/null || true" > "$dest/system/iptables.rules" 2>/dev/null || true + run_remote "ip6tables-save 2>/dev/null || true" > "$dest/system/ip6tables.rules" 2>/dev/null || true + + log "Config collection complete." +} + +do_backup() { + check_proxmox + check_rsync + + local stamp=$(run_remote "date +%Y%m%d_%H%M%S 2>/dev/null || date +%Y%m%d_%H%M%S") + local hostname=$(run_remote "hostname -s 2>/dev/null || echo unknown") + + local temp_dir=$(mktemp -d) + local archive_name="pve-backup-${hostname}-${stamp}.tar.gz" + local archive_path="$BACKUP_DIR/$archive_name" + + mkdir -p "$BACKUP_DIR" "$temp_dir" + + collect_pve_configs "$temp_dir" + + # Add metadata + cat > "$temp_dir/META" </dev/null || echo unknown") +pve_nodes: $(run_remote 'ls /etc/pve/nodes/ 2>/dev/null | tr "\n" " "' 2>/dev/null) +EOM + + tar czf "$archive_path" -C "$temp_dir" $(ls -A "$temp_dir") + rm -rf "$temp_dir" + + log "Backup written to: $archive_path" + echo " Size: $(du -h "$archive_path" | cut -f1)" + + # Generate and store SHA256 checksum + (cd "$BACKUP_DIR" && sha256sum "$archive_name" > "$archive_path.sha256") + log "Checksum: $(cat "$archive_path.sha256")" +} + +# Remote rollback + restore of /etc/pve via rsync or tar pipe. +# Handles both rsync'd directories and tarball archives. +_restore_pve() { + local src_dir="$1" + local ssh_cmd + ssh_cmd=$(build_ssh_cmd) + + run_remote "tar czf /etc/pve-rollback.tar.gz -C /etc/pve . 2>/dev/null" 2>/dev/null || true + + if [[ -f "$src_dir/pve-config.tar.gz" ]]; then + log " Restoring /etc/pve (from tarball)..." + run_remote "tar xzf - -C /etc/pve" < "$src_dir/pve-config.tar.gz" 2>/dev/null || \ + warn "Failed to restore some /etc/pve files" + elif [[ -f "$src_dir/datacenter.cfg" ]]; then + log " Restoring /etc/pve (from directory)..." + local tried_rsync=0 + if [[ -n "$ssh_cmd" ]] && command -v rsync &>/dev/null && run_remote "command -v rsync" &>/dev/null; then + local rsync_ssh + rsync_ssh=$(build_ssh_rsync_cmd) + local rsync_opts=(-a --delete) + [[ -n "$SUDO_CMD" ]] && rsync_opts+=(--rsync-path="${SUDO_CMD}rsync") + rsync "${rsync_opts[@]}" -e "$rsync_ssh" "$src_dir/" "${REMOTE_USER}@${REMOTE_HOST}:/etc/pve/" &>/dev/null && \ + tried_rsync=1 + fi + if [[ "$tried_rsync" -eq 0 ]]; then + tar czf - -C "$src_dir" . | run_remote "tar xzf - -C /etc/pve" 2>/dev/null || \ + warn "Failed to restore /etc/pve files" + fi + fi +} + +_restore_dir() { + local src_dir="$1" remote_path="$2" label="$3" + if [[ ! -d "$src_dir" ]] || ! ls -A "$src_dir" &>/dev/null; then + return + fi + log " Restoring $label..." + local ssh_cmd + ssh_cmd=$(build_ssh_cmd) + local tried_rsync=0 + if [[ -n "$ssh_cmd" ]] && command -v rsync &>/dev/null && run_remote "command -v rsync" &>/dev/null; then + local rsync_ssh + rsync_ssh=$(build_ssh_rsync_cmd) + local rsync_opts=(-a) + [[ -n "$SUDO_CMD" ]] && rsync_opts+=(--rsync-path="${SUDO_CMD}rsync") + rsync "${rsync_opts[@]}" -e "$rsync_ssh" "$src_dir/" "${REMOTE_USER}@${REMOTE_HOST}:$remote_path" &>/dev/null && \ + tried_rsync=1 + fi + if [[ "$tried_rsync" -eq 0 ]]; then + tar czf - -C "$src_dir" . | run_remote "tar xzf - -C $remote_path" 2>/dev/null || \ + warn "Failed to restore $label" + fi +} + +do_restore() { + check_proxmox + + local archive_file="" + + if [[ -n "${RESTORE_FILE:-}" ]]; then + archive_file="$RESTORE_FILE" + else + local latest + latest=$(ls -t "$BACKUP_DIR"/pve-backup-*.tar.gz 2>/dev/null | head -1) || true + if [[ -z "$latest" ]]; then + die "No backup archives found in $BACKUP_DIR and no --file specified" + fi + archive_file="$latest" + fi + + if [[ ! -f "$archive_file" ]]; then + die "Backup file not found: $archive_file" + fi + + log "Restoring from: $archive_file" + + if [[ "${FORCE_YES:-0}" -ne 1 ]]; then + echo "" + warn "WARNING: This will overwrite Proxmox configuration files!" + warn "Your current configs will be backed up to /var/tmp/pve-restore-rollback/" + echo "" + read -r -p "Are you sure you want to proceed? [y/N] " reply + if [[ ! "$reply" =~ ^[Yy]$ ]]; then + die "Restore cancelled." + fi + fi + + local rollback_dir="/var/tmp/pve-restore-rollback-$(date +%Y%m%d_%H%M%S)" + log "Creating rollback snapshot at $rollback_dir..." + mkdir -p "$rollback_dir" + + local temp_dir + temp_dir=$(mktemp -d) + tar xzf "$archive_file" -C "$temp_dir" + + # Restore /etc/pve + _restore_pve "$temp_dir/etc/pve" + + # Restore individual system files + for f in "$temp_dir"/etc/system/*; do + local basename_f + basename_f=$(basename "$f") + [[ -f "$f" ]] || continue + local target="" + case "$basename_f" in + interfaces) target="/etc/network/interfaces" ;; + hostname) target="/etc/hostname" ;; + hosts) target="/etc/hosts" ;; + resolv.conf) target="/etc/resolv.conf" ;; + fstab) target="/etc/fstab" ;; + corosync.conf) target="/etc/corosync/corosync.conf" ;; + ceph.conf) target="/etc/ceph/ceph.conf" ;; + mailname) target="/etc/mailname" ;; + *) continue ;; + esac + if [[ -s "$f" ]]; then + run_remote "mkdir -p $(dirname "$target") && cat > $target" < "$f" 2>/dev/null || warn "Failed to restore $target" + fi + done + + # Restore APT sources + if [[ -f "$temp_dir/etc/apt/apt-sources.tar.gz" ]]; then + log "Restoring APT sources..." + run_remote "tar xzf - -C /" < "$temp_dir/etc/apt/apt-sources.tar.gz" 2>/dev/null || warn "Failed to restore APT sources" + elif [[ -d "$temp_dir/etc/apt" ]]; then + _restore_dir "$temp_dir/etc/apt" "/etc/apt" "APT sources" + fi + + # Restore sysconfig + if [[ -f "$temp_dir/etc/system/sysconfig.tar.gz" ]]; then + log "Restoring sysctl/cron configs..." + run_remote "tar xzf - -C /" < "$temp_dir/etc/system/sysconfig.tar.gz" 2>/dev/null || warn "Failed to restore sysconfig" + elif [[ -d "$temp_dir/etc/system/sysctl.d" ]]; then + _restore_dir "$temp_dir/etc/system/sysctl.d" "/etc/sysctl.d" "sysctl" + _restore_dir "$temp_dir/etc/system/cron.d" "/etc/cron.d" "cron" + _restore_dir "$temp_dir/etc/system/cron.daily" "/etc/cron.daily" "cron.daily" + fi + + rm -rf "$temp_dir" + + log "Restore complete!" + log "Rollback snapshot: $rollback_dir" + echo "" + warn "NOTE: Some changes (e.g., network, hostname) require a reboot to take full effect." + warn "NOTE: Restored /etc/pve files sync via pmxcfs — restart pveproxy if web UI acts odd." +} + +do_verify() { + local archive_file="" + + if [[ -n "${RESTORE_FILE:-}" ]]; then + archive_file="$RESTORE_FILE" + else + die "Specify an archive with -f/--file" + fi + + if [[ ! -f "$archive_file" ]]; then + die "File not found: $archive_file" + fi + + local checksum_file="${archive_file}.sha256" + + log "Verifying: $archive_file" + + # Check tar integrity + if tar tzf "$archive_file" &>/dev/null; then + log " Archive integrity: OK (valid tar.gz)" + else + die " Archive integrity: FAILED — corrupt or truncated" + fi + + # Check SHA256 if available + local archive_dir archive_basename + archive_dir=$(dirname "$archive_file") + archive_basename=$(basename "$archive_file") + if [[ -f "$checksum_file" ]]; then + if cd "$archive_dir" && sha256sum -c "$archive_basename.sha256" &>/dev/null; then + cd "$OLDPWD" + log " SHA256 checksum: OK (matches stored checksum)" + else + die " SHA256 checksum: FAILED — archive modified or corrupted" + fi + else + # Generate and save checksum if none exists + warn " SHA256 checksum: none found — generating now" + (cd "$archive_dir" && sha256sum "$archive_basename" > "$archive_basename.sha256") + log " Checksum saved to: $checksum_file" + fi +} + +do_list() { + ls -lh "$BACKUP_DIR"/pve-backup-*.tar.gz 2>/dev/null || echo "No backups found in $BACKUP_DIR" +} + +# ---- Main ---- +COMMAND="${1:-}" +[[ -z "$COMMAND" ]] && usage +shift + +FORCE_YES=0 +RESTORE_FILE="" + +parse_common_args "$@" +set -- "${PARSE_REMAINING[@]}" + +case "$COMMAND" in + backup) + do_backup + ;; + restore) + do_restore + ;; + verify) + do_verify + ;; + list) + do_list + ;; + *) + usage + ;; +esac