#!/bin/bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" BACKUP_DIR="$SCRIPT_DIR/pve-backups" SSH_KEY="" REMOTE_HOST="" REMOTE_USER="root" SUDO_CMD="" HAS_RSYNC=0 SCRIPT_NAME=$(basename "$0") usage() { cat < [options] Commands: backup Create a backup of Proxmox configuration restore Restore Proxmox configuration from a backup verify Verify a backup archive's integrity (SHA256) list List available backups Backup options: -d, --dir DIR Local backup directory (default: $SCRIPT_DIR/pve-backups) -h, --host HOST Remote Proxmox host (SSH) -u, --user USER SSH user (default: root) -i, --identity FILE SSH identity file Restore / Verify options: -d, --dir DIR Directory containing backup archive -f, --file FILE Specific backup archive to restore/verify -y, --yes Skip confirmation prompts (DANGEROUS) List options: -d, --dir DIR Directory to list backups from (default: $BACKUP_DIR) Examples: $SCRIPT_NAME backup $SCRIPT_NAME backup -d /mnt/backup $SCRIPT_NAME backup -h 192.168.1.100 -i ~/.ssh/id_rsa $SCRIPT_NAME restore -f /mnt/backup/pve-backup-20250101_120000.tar.gz $SCRIPT_NAME verify -f /mnt/backup/pve-backup-20250101_120000.tar.gz $SCRIPT_NAME list EOF exit 1 } log() { echo "[*] $*"; } warn() { echo "[!] $*" >&2; } die() { echo "[FATAL] $*" >&2; exit 1; } # Parse common args (consume them, return remaining via ${PARSE_REMAINING[@]}) parse_common_args() { local parsed=() PARSE_REMAINING=() while [[ $# -gt 0 ]]; do case "$1" in -d|--dir) BACKUP_DIR="$2"; shift 2 ;; -h|--host) REMOTE_HOST="$2"; shift 2 ;; -u|--user) REMOTE_USER="$2"; shift 2 ;; -i|--identity) SSH_KEY="$2"; shift 2 ;; -y|--yes) FORCE_YES=1; shift ;; -f|--file) RESTORE_FILE="$2"; shift 2 ;; --) shift; parsed+=("$@"); break ;; -*) PARSE_REMAINING=("$@") return ;; *) parsed+=("$1"); shift ;; esac done PARSE_REMAINING=("${parsed[@]}") } build_ssh_cmd() { local cmd=() if [[ -n "$REMOTE_HOST" ]]; then cmd=(ssh) [[ -n "$SSH_KEY" ]] && cmd+=(-i "$SSH_KEY") cmd+=("${REMOTE_USER}@${REMOTE_HOST}") fi echo "${cmd[@]}" } build_ssh_rsync_cmd() { local cmd="ssh" [[ -n "$SSH_KEY" ]] && cmd+=" -i $SSH_KEY" echo "$cmd" } run_remote() { local ssh_cmd ssh_cmd=$(build_ssh_cmd) if [[ -n "$ssh_cmd" ]]; then $ssh_cmd $SUDO_CMD "$@" else ${SUDO_CMD}bash -c "$*" fi } run_remote_raw() { local ssh_cmd ssh_cmd=$(build_ssh_cmd) if [[ -n "$ssh_cmd" ]]; then $ssh_cmd "${SUDO_CMD}bash -s" <<< "$1" else ${SUDO_CMD}bash -c "$1" fi } check_proxmox() { if [[ -z "$REMOTE_HOST" ]]; then if [[ "$(id -u)" -ne 0 ]]; then SUDO_CMD="sudo -n " fi elif [[ "$REMOTE_USER" != "root" ]]; then SUDO_CMD="sudo -n " fi if ! run_remote "test -f /etc/pve/datacenter.cfg" 2>/dev/null; then local where="${REMOTE_HOST:-locally}" local who="${REMOTE_USER:-$(whoami)}" if [[ -n "$SUDO_CMD" ]]; then die "Cannot read /etc/pve $where — user '$who' needs passwordless sudo" else die "Cannot read /etc/pve $where — this script must be run as root" fi fi } check_rsync() { HAS_RSYNC=0 if [[ -z "$REMOTE_HOST" ]]; then return fi if run_remote "command -v rsync" &>/dev/null; then HAS_RSYNC=1 log "rsync available on remote — using for incremental transfers" else log "rsync not found on remote — falling back to tar/cat transfer" fi } # rsync wrapper: copies remote path(s) into dest dir. # Handles SSH, sudo (via --rsync-path), and both file and directory sources. # Prefix src with 'dir:' to treat it as a directory (--delete). rsync_pull() { local src="$1" local dest="$2" local is_dir=0 if [[ "$HAS_RSYNC" -ne 1 || -z "$REMOTE_HOST" ]]; then return 1 fi if [[ "$src" == dir:* ]]; then is_dir=1 src="${src:4}" fi local rsync_ssh rsync_ssh=$(build_ssh_rsync_cmd) local rsync_opts=(-a) [[ "$is_dir" -eq 1 ]] && rsync_opts+=(--delete) [[ -n "$SUDO_CMD" ]] && rsync_opts+=(--rsync-path="${SUDO_CMD}rsync") mkdir -p "$dest" rsync "${rsync_opts[@]}" -e "$rsync_ssh" \ "${REMOTE_USER}@${REMOTE_HOST}:$src" "$dest" &>/dev/null } collect_pve_configs() { local temp_dir="$1" local dest="$temp_dir/etc" mkdir -p "$dest/pve" "$dest/apt" "$dest/system" # ---- /etc/pve ---- log "Backing up /etc/pve configuration..." if rsync_pull "dir:/etc/pve/" "$dest/pve/" && [[ -f "$dest/pve/datacenter.cfg" ]]; then # Clean up rsync'd noise rm -rf "$dest/pve/.rrd" "$dest/pve/nodes/"*/qemu-server/*.lock \ "$dest/pve/nodes/"*/lxc/*.lock "$dest/pve/priv/known_hosts" 2>/dev/null || true else # Fallback: tar-based snapshot via SSH rm -rf "$dest/pve" 2>/dev/null; mkdir "$dest/pve" run_remote_raw ' set -euo pipefail tmp=$(mktemp -d) mkdir "$tmp/pve" cd /etc/pve tar cf - \ --exclude="./nodes/*/qemu-server/*.lock" \ --exclude="./nodes/*/lxc/*.lock" \ --exclude="./priv/known_hosts" \ --exclude=".rrd" \ . 2>/dev/null | tar xf - -C "$tmp/pve" tar czf - -C "$tmp" pve rm -rf "$tmp" ' > "$dest/pve/pve-config.tar.gz" 2>/dev/null fi # ---- Network & system files ---- log "Backing up network and system configs..." local sysfiles=( /etc/network/interfaces /etc/hostname /etc/hosts /etc/resolv.conf /etc/mailname /etc/fstab ) local rsync_ok=1 for f in "${sysfiles[@]}"; do if ! rsync_pull "$f" "$dest/system/"; then rsync_ok=0 break fi done if [[ "$rsync_ok" -eq 0 ]]; then for f in "${sysfiles[@]}"; do run_remote "cat $f 2>/dev/null" > "$dest/system/$(basename "$f")" 2>/dev/null || true done fi # ---- APT sources ---- log "Backing up APT sources..." if ! rsync_pull "dir:/etc/apt/sources.list.d/" "$dest/apt/" || \ ! rsync_pull "/etc/apt/sources.list" "$dest/apt/" || \ ! rsync_pull "/etc/apt/auth.conf" "$dest/apt/"; then # Fallback rm -rf "$dest/apt" 2>/dev/null; mkdir "$dest/apt" run_remote_raw ' set -euo pipefail tmp=$(mktemp -d) mkdir "$tmp/apt" cp -a /etc/apt/sources.list "$tmp/apt/" 2>/dev/null || true cp -a /etc/apt/sources.list.d "$tmp/apt/" 2>/dev/null || true cp -a /etc/apt/auth.conf "$tmp/apt/" 2>/dev/null || true tar czf - -C "$tmp" apt rm -rf "$tmp" ' > "$dest/apt/apt-sources.tar.gz" 2>/dev/null fi # ---- Corosync ---- log "Backing up corosync config..." if ! rsync_pull "/etc/corosync/corosync.conf" "$dest/system/"; then run_remote "cat /etc/corosync/corosync.conf 2>/dev/null" > "$dest/system/corosync.conf" 2>/dev/null || true fi # ---- Ceph ---- log "Backing up Ceph config..." if ! rsync_pull "/etc/ceph/ceph.conf" "$dest/system/"; then run_remote "cat /etc/ceph/ceph.conf 2>/dev/null" > "$dest/system/ceph.conf" 2>/dev/null || true fi # ---- sysctl / cron ---- log "Backing up sysctl and cron..." if ! rsync_pull "dir:/etc/sysctl.d/" "$dest/system/sysctl.d/" || \ ! rsync_pull "dir:/etc/cron.d/" "$dest/system/cron.d/" || \ ! rsync_pull "dir:/etc/cron.daily/" "$dest/system/cron.daily/"; then # Fallback rm -rf "$dest/system/sysctl.d" "$dest/system/cron.d" "$dest/system/cron.daily" 2>/dev/null run_remote_raw ' tmp=$(mktemp -d) mkdir "$tmp/etc" cp -a /etc/sysctl.d "$tmp/etc/" 2>/dev/null || true cp -a /etc/cron.d "$tmp/etc/" 2>/dev/null || true cp -a /etc/cron.daily "$tmp/etc/" 2>/dev/null || true tar czf - -C "$tmp" etc rm -rf "$tmp" ' > "$dest/system/sysconfig.tar.gz" 2>/dev/null fi # ---- Package manifest (always via command) ---- log "Recording package manifest..." run_remote "dpkg-query -W 2>/dev/null || true" > "$dest/system/packages.txt" 2>/dev/null || true run_remote "apt-mark showmanual 2>/dev/null || true" > "$dest/system/packages-manual.txt" 2>/dev/null || true run_remote "pveversion 2>/dev/null || true" > "$dest/system/pveversion.txt" 2>/dev/null || true # ---- Firewall rules (always via command) ---- log "Backing up firewall rules..." run_remote "iptables-save 2>/dev/null || true" > "$dest/system/iptables.rules" 2>/dev/null || true run_remote "ip6tables-save 2>/dev/null || true" > "$dest/system/ip6tables.rules" 2>/dev/null || true log "Config collection complete." } do_backup() { check_proxmox check_rsync local stamp=$(run_remote "date +%Y%m%d_%H%M%S 2>/dev/null || date +%Y%m%d_%H%M%S") local hostname=$(run_remote "hostname -s 2>/dev/null || echo unknown") local temp_dir=$(mktemp -d) local archive_name="pve-backup-${hostname}-${stamp}.tar.gz" local archive_path="$BACKUP_DIR/$archive_name" mkdir -p "$BACKUP_DIR" "$temp_dir" collect_pve_configs "$temp_dir" # Add metadata cat > "$temp_dir/META" </dev/null || echo unknown") pve_nodes: $(run_remote 'ls /etc/pve/nodes/ 2>/dev/null | tr "\n" " "' 2>/dev/null) EOM tar czf "$archive_path" -C "$temp_dir" $(ls -A "$temp_dir") rm -rf "$temp_dir" log "Backup written to: $archive_path" echo " Size: $(du -h "$archive_path" | cut -f1)" # Generate and store SHA256 checksum (cd "$BACKUP_DIR" && sha256sum "$archive_name" > "$archive_path.sha256") log "Checksum: $(cat "$archive_path.sha256")" } # Remote rollback + restore of /etc/pve via rsync or tar pipe. # Handles both rsync'd directories and tarball archives. _restore_pve() { local src_dir="$1" local ssh_cmd ssh_cmd=$(build_ssh_cmd) run_remote "tar czf /etc/pve-rollback.tar.gz -C /etc/pve . 2>/dev/null" 2>/dev/null || true if [[ -f "$src_dir/pve-config.tar.gz" ]]; then log " Restoring /etc/pve (from tarball)..." run_remote "tar xzf - -C /etc/pve" < "$src_dir/pve-config.tar.gz" 2>/dev/null || \ warn "Failed to restore some /etc/pve files" elif [[ -f "$src_dir/datacenter.cfg" ]]; then log " Restoring /etc/pve (from directory)..." local tried_rsync=0 if [[ -n "$ssh_cmd" ]] && command -v rsync &>/dev/null && run_remote "command -v rsync" &>/dev/null; then local rsync_ssh rsync_ssh=$(build_ssh_rsync_cmd) local rsync_opts=(-a --delete) [[ -n "$SUDO_CMD" ]] && rsync_opts+=(--rsync-path="${SUDO_CMD}rsync") rsync "${rsync_opts[@]}" -e "$rsync_ssh" "$src_dir/" "${REMOTE_USER}@${REMOTE_HOST}:/etc/pve/" &>/dev/null && \ tried_rsync=1 fi if [[ "$tried_rsync" -eq 0 ]]; then tar czf - -C "$src_dir" . | run_remote "tar xzf - -C /etc/pve" 2>/dev/null || \ warn "Failed to restore /etc/pve files" fi fi } _restore_dir() { local src_dir="$1" remote_path="$2" label="$3" if [[ ! -d "$src_dir" ]] || ! ls -A "$src_dir" &>/dev/null; then return fi log " Restoring $label..." local ssh_cmd ssh_cmd=$(build_ssh_cmd) local tried_rsync=0 if [[ -n "$ssh_cmd" ]] && command -v rsync &>/dev/null && run_remote "command -v rsync" &>/dev/null; then local rsync_ssh rsync_ssh=$(build_ssh_rsync_cmd) local rsync_opts=(-a) [[ -n "$SUDO_CMD" ]] && rsync_opts+=(--rsync-path="${SUDO_CMD}rsync") rsync "${rsync_opts[@]}" -e "$rsync_ssh" "$src_dir/" "${REMOTE_USER}@${REMOTE_HOST}:$remote_path" &>/dev/null && \ tried_rsync=1 fi if [[ "$tried_rsync" -eq 0 ]]; then tar czf - -C "$src_dir" . | run_remote "tar xzf - -C $remote_path" 2>/dev/null || \ warn "Failed to restore $label" fi } do_restore() { check_proxmox local archive_file="" if [[ -n "${RESTORE_FILE:-}" ]]; then archive_file="$RESTORE_FILE" else local latest latest=$(ls -t "$BACKUP_DIR"/pve-backup-*.tar.gz 2>/dev/null | head -1) || true if [[ -z "$latest" ]]; then die "No backup archives found in $BACKUP_DIR and no --file specified" fi archive_file="$latest" fi if [[ ! -f "$archive_file" ]]; then die "Backup file not found: $archive_file" fi log "Restoring from: $archive_file" if [[ "${FORCE_YES:-0}" -ne 1 ]]; then echo "" warn "WARNING: This will overwrite Proxmox configuration files!" warn "Your current configs will be backed up to /var/tmp/pve-restore-rollback/" echo "" read -r -p "Are you sure you want to proceed? [y/N] " reply if [[ ! "$reply" =~ ^[Yy]$ ]]; then die "Restore cancelled." fi fi local rollback_dir="/var/tmp/pve-restore-rollback-$(date +%Y%m%d_%H%M%S)" log "Creating rollback snapshot at $rollback_dir..." mkdir -p "$rollback_dir" local temp_dir temp_dir=$(mktemp -d) tar xzf "$archive_file" -C "$temp_dir" # Restore /etc/pve _restore_pve "$temp_dir/etc/pve" # Restore individual system files for f in "$temp_dir"/etc/system/*; do local basename_f basename_f=$(basename "$f") [[ -f "$f" ]] || continue local target="" case "$basename_f" in interfaces) target="/etc/network/interfaces" ;; hostname) target="/etc/hostname" ;; hosts) target="/etc/hosts" ;; resolv.conf) target="/etc/resolv.conf" ;; fstab) target="/etc/fstab" ;; corosync.conf) target="/etc/corosync/corosync.conf" ;; ceph.conf) target="/etc/ceph/ceph.conf" ;; mailname) target="/etc/mailname" ;; *) continue ;; esac if [[ -s "$f" ]]; then run_remote "mkdir -p $(dirname "$target") && cat > $target" < "$f" 2>/dev/null || warn "Failed to restore $target" fi done # Restore APT sources if [[ -f "$temp_dir/etc/apt/apt-sources.tar.gz" ]]; then log "Restoring APT sources..." run_remote "tar xzf - -C /" < "$temp_dir/etc/apt/apt-sources.tar.gz" 2>/dev/null || warn "Failed to restore APT sources" elif [[ -d "$temp_dir/etc/apt" ]]; then _restore_dir "$temp_dir/etc/apt" "/etc/apt" "APT sources" fi # Restore sysconfig if [[ -f "$temp_dir/etc/system/sysconfig.tar.gz" ]]; then log "Restoring sysctl/cron configs..." run_remote "tar xzf - -C /" < "$temp_dir/etc/system/sysconfig.tar.gz" 2>/dev/null || warn "Failed to restore sysconfig" elif [[ -d "$temp_dir/etc/system/sysctl.d" ]]; then _restore_dir "$temp_dir/etc/system/sysctl.d" "/etc/sysctl.d" "sysctl" _restore_dir "$temp_dir/etc/system/cron.d" "/etc/cron.d" "cron" _restore_dir "$temp_dir/etc/system/cron.daily" "/etc/cron.daily" "cron.daily" fi rm -rf "$temp_dir" log "Restore complete!" log "Rollback snapshot: $rollback_dir" echo "" warn "NOTE: Some changes (e.g., network, hostname) require a reboot to take full effect." warn "NOTE: Restored /etc/pve files sync via pmxcfs — restart pveproxy if web UI acts odd." } do_verify() { local archive_file="" if [[ -n "${RESTORE_FILE:-}" ]]; then archive_file="$RESTORE_FILE" else die "Specify an archive with -f/--file" fi if [[ ! -f "$archive_file" ]]; then die "File not found: $archive_file" fi local checksum_file="${archive_file}.sha256" log "Verifying: $archive_file" # Check tar integrity if tar tzf "$archive_file" &>/dev/null; then log " Archive integrity: OK (valid tar.gz)" else die " Archive integrity: FAILED — corrupt or truncated" fi # Check SHA256 if available local archive_dir archive_basename archive_dir=$(dirname "$archive_file") archive_basename=$(basename "$archive_file") if [[ -f "$checksum_file" ]]; then if cd "$archive_dir" && sha256sum -c "$archive_basename.sha256" &>/dev/null; then cd "$OLDPWD" log " SHA256 checksum: OK (matches stored checksum)" else die " SHA256 checksum: FAILED — archive modified or corrupted" fi else # Generate and save checksum if none exists warn " SHA256 checksum: none found — generating now" (cd "$archive_dir" && sha256sum "$archive_basename" > "$archive_basename.sha256") log " Checksum saved to: $checksum_file" fi } do_list() { ls -lh "$BACKUP_DIR"/pve-backup-*.tar.gz 2>/dev/null || echo "No backups found in $BACKUP_DIR" } # ---- Main ---- COMMAND="${1:-}" [[ -z "$COMMAND" ]] && usage shift FORCE_YES=0 RESTORE_FILE="" parse_common_args "$@" set -- "${PARSE_REMAINING[@]}" case "$COMMAND" in backup) do_backup ;; restore) do_restore ;; verify) do_verify ;; list) do_list ;; *) usage ;; esac