scripts/docker/docker-stack-backup/docker-stack-backup.sh

#!/usr/bin/env bash
set -euo pipefail

# --- Configuration -----------------------------------------------------------
# NOTE: INITIAL_DIR is re-set inside main() after exec sudo, since sudo may
# change the working directory.  The top-level value here is a fallback for
# the usage text.
_DEFAULT_BACKUP_DIR="${PWD}/backups"
VERBOSE=false
DRY_RUN=false
ACTION=""
TARGET_STACKS=()
MODE="all"  # all|running
HAS_FLAGS=false
ORIG_ARGS=()

# --- Helper Functions --------------------------------------------------------

log()   { if $VERBOSE; then echo "[INFO]  $*"; fi; }
warn()  { echo "[WARN]  $*" >&2; }
err()   { echo "[ERROR] $*" >&2; exit 1; }
dry()   { if $DRY_RUN; then echo "[DRY-RUN] $*"; else "$@"; fi; }

# When run via sudo, chown files back to the original user
fix_owner() {
  [ -n "${SUDO_USER:-}" ] && chown "$SUDO_USER:" "$@" 2>/dev/null || true
}

# --- Stack Spec Resolution ----------------------------------------------------
# A stack spec can be:
#   name              – resolved as <stacks_dir>/<name>
#   /absolute/path    – used directly, name = basename

stack_spec_to_path() {
  local stacks_dir="$1"
  local spec="$2"
  if [[ "$spec" == /* ]]; then
    echo "$spec"
  else
    echo "$stacks_dir/$spec"
  fi
}

stack_spec_to_name() {
  local spec="$1"
  basename "$spec"
}

usage() {
  cat <<EOF
Usage: $(basename "$0") <command> [options]

Commands:
  backup                  Backup one or more docker compose stacks.
                          With no flags, shows an interactive menu.
  restore                 Restore one or more stacks from a backup.
                          With no --stack flag, shows an interactive menu.
  list                    List available stacks and backups
  verify                  Verify backup integrity (checks sha256 hash)

Options:
  -s, --stack <name>      Target a specific stack (repeatable, or comma-separated).
                          Can be a name (relative to stacks dir) or absolute path.
  -a, --all               Target all stacks found in stacks directory
  -r, --running           Target only currently running compose projects wherever
                          they live (discovers compose file paths automatically)
  -d, --backup-dir <dir>  Backup directory (default: ${_DEFAULT_BACKUP_DIR})
  -n, --dry-run           Show what would be done without making changes
  -v, --verbose           Verbose output
  -h, --help              Show this help message

Must be run as root (preserves file ownership in backups and restores).

Compose x-backup extensions (add to compose.yaml):
  x-backup:
    stop: false                # Skip container stop/start
    exclude:
      - path/to/exclude        # Paths to exclude from the archive
    pre-hook: "command"        # Run before backup
    post-hook: "command"       # Run after backup
    pre-restore-hook: "cmd"    # Run before restore
    post-restore-hook: "cmd"   # Run after restore
    retention: 7               # Keep only N latest backups
    backup-volumes: false      # Skip named Docker volume dumps

Examples:
  $(basename "$0") backup --all
  $(basename "$0") backup --running
  $(basename "$0") backup --stack random_stack --stack another_stack
  $(basename "$0") backup -s random_stack -v
  $(basename "$0") restore --stack random_stack -d /path/to/backups
  $(basename "$0") list
EOF
  exit 0
}

# --- Dependency Checks -------------------------------------------------------

check_deps() {
  for cmd in docker tar; do
    command -v "$cmd" >/dev/null 2>&1 || err "Required command '$cmd' is not installed"
  done
}

require_root() {
  if [ "$(id -u)" -eq 0 ]; then
    return
  fi
  echo "Elevating to root via sudo..."
  exec sudo "$0" "${ORIG_ARGS[@]}"
}

# --- Stack Discovery ---------------------------------------------------------

# Get all directories in stacks dir that contain a compose file
discover_all_stacks() {
  local stacks_dir="$1"
  local stack
  local result=()
  for dir in "$stacks_dir"/*/; do
    stack="$(basename "$dir")"
    if [ -f "$dir/compose.yaml" ] || [ -f "$dir/compose.yml" ] || [ -f "$dir/docker-compose.yaml" ] || [ -f "$dir/docker-compose.yml" ]; then
      result+=("$stack")
    fi
  done
  echo "${result[@]}"
}

_RUNNING_CACHE=""

# Get currently running compose projects — returns absolute directory paths
# Results are cached for the lifetime of the process since running compose
# projects don't change during a single backup/restore/list invocation.
discover_running_stacks() {
  [ -n "$_RUNNING_CACHE" ] && echo "$_RUNNING_CACHE" && return
  local result
  if result="$(docker compose ls --format json 2>/dev/null | python3 -c "
import sys, json, os

raw = sys.stdin.read().strip()
if not raw:
    sys.exit(0)

try:
    data = json.loads(raw)
except json.JSONDecodeError:
    sys.exit(0)

if isinstance(data, list):
    items = data
else:
    items = [data]

for obj in items:
    if not isinstance(obj, dict):
        continue
    name = obj.get('Name', '') or ''
    config_files = obj.get('ConfigFiles', '') or ''
    if config_files:
        path = config_files.split(',')[0].strip()
        path = os.path.dirname(path)
        print(path)
    else:
        wd = obj.get('WorkingDirectory', '') or ''
        if wd:
            print(os.path.join(wd, name))
        else:
            print(name)
" 2>/dev/null)"; then
    _RUNNING_CACHE="$result"
  else
    result="$(docker compose ls 2>/dev/null | tail -n +2 | awk '{print $1}' || true)"
    _RUNNING_CACHE="$result"
  fi
  echo "$_RUNNING_CACHE"
}

resolve_target_stacks() {
  local stacks_dir="$1"
  local mode="$2"
  shift 2
  local explicit=("$@")

  if [ ${#explicit[@]} -gt 0 ]; then
    for spec in "${explicit[@]}"; do
      stack_spec_to_path "$stacks_dir" "$spec"
    done
    return
  fi

  case "$mode" in
    running)
      discover_running_stacks
      ;;
    all|*)
      discover_all_stacks "$stacks_dir"
      ;;
  esac
}

# --- Compose File Helpers ----------------------------------------------------

# Resolve the compose file path for a stack
resolve_compose_file() {
  local stack_path="$1"
  for name in compose.yaml compose.yml docker-compose.yaml docker-compose.yml; do
    [ -f "$stack_path/$name" ] && echo "$stack_path/$name" && return
  done
  echo ""
}

# Extract the x-backup block (lines between "x-backup:" and the next top-level key)
extract_x_backup_block() {
  local compose_file="$1"
  [ -z "$compose_file" ] && return 1
  awk '
    /^x-backup:/ { found=1; next }
    found && /^[a-zA-Z_-][a-zA-Z0-9_-]*:/ { found=0; next }
    found { print }
  ' "$compose_file" 2>/dev/null || true
}

# Get a boolean x-backup setting (default: "true")
x_backup_bool() {
  local block="$1"
  local key="$2"
  local default="${3:-true}"
  local val
  val="$(echo "$block" | grep -E "^\s+${key}:" | sed "s/.*${key}:[[:space:]]*//" | head -1 | tr -d ' \t' | tr '[:upper:]' '[:lower:]')"
  # Strip inline YAML comments
  val="${val%%#*}"
  [ -z "$val" ] && echo "$default" && return
  [ "$val" = "false" ] || [ "$val" = "0" ] || [ "$val" = "no" ] && echo "false" && return
  echo "true"
}

# Get a string x-backup setting
x_backup_str() {
  local block="$1"
  local key="$2"
  local default="${3:-}"
  local val
  val="$(echo "$block" | grep -E "^\s+${key}:" | sed "s/.*${key}:[[:space:]]*//" | head -1)"
  # Strip inline YAML comments and trailing whitespace
  val="${val%%#*}"
  val="${val%% }"
  # Strip YAML quotes only when the entire value is wrapped
  if [[ "$val" =~ ^\"(.*)\"$ ]] || [[ "$val" =~ ^\'(.*)\'$ ]]; then
    val="${BASH_REMATCH[1]}"
  fi
  echo "${val:-$default}"
}

# Get a list x-backup setting (items starting with "- ")
x_backup_list() {
  local block="$1"
  local key="$2"
  local indent
  indent="$(echo "$block" | grep -n "^\s\+${key}:" | head -1 | sed 's/\([0-9]*\):.*/\1/')"
  if [ -n "$indent" ]; then
    echo "$block" | awk -v start="$indent" 'NR > start && /^\s+- / { gsub(/^\s+-[[:space:]]*/, ""); gsub(/"/, ""); gsub(/#.*/, ""); gsub(/[[:space:]]+$/, ""); print }'
  fi
}

# Load x-backup config for a stack into global vars
load_x_backup_config() {
  local stack_path="$1"
  local compose_file
  compose_file="$(resolve_compose_file "$stack_path")"

  XB_STOP=true
  XB_EXCLUDE=""
  XB_PRE_HOOK=""
  XB_POST_HOOK=""
  XB_PRE_RESTORE_HOOK=""
  XB_POST_RESTORE_HOOK=""
  XB_RETENTION=""
  XB_BACKUP_VOLUMES="true"

  [ -z "$compose_file" ] && return

  local block
  block="$(extract_x_backup_block "$compose_file")"
  [ -z "$block" ] && return

  XB_STOP="$(x_backup_bool "$block" "stop" "true")"
  XB_PRE_HOOK="$(x_backup_str "$block" "pre-hook")"
  XB_POST_HOOK="$(x_backup_str "$block" "post-hook")"
  XB_PRE_RESTORE_HOOK="$(x_backup_str "$block" "pre-restore-hook")"
  XB_POST_RESTORE_HOOK="$(x_backup_str "$block" "post-restore-hook")"
  XB_RETENTION="$(x_backup_str "$block" "retention")"

  local excludes
  excludes="$(x_backup_list "$block" "exclude")"
  if [ -n "$excludes" ]; then
    XB_EXCLUDE=""
    while IFS= read -r item; do
      [ -n "$item" ] && XB_EXCLUDE="$XB_EXCLUDE --exclude=$(basename "$stack_path")/$item"
    done <<< "$excludes"
  fi

  XB_BACKUP_VOLUMES="$(x_backup_bool "$block" "backup-volumes" "true")"
}

# Build the x-backup config tag string for list display (reads global XB_ vars)
format_xb_tags() {
  local tags=""
  [ "$XB_STOP" = "false" ]      && tags="$tags no-stop"
  [ -n "$XB_PRE_HOOK" ]         && tags="$tags pre-hook"
  [ -n "$XB_POST_HOOK" ]        && tags="$tags post-hook"
  [ -n "$XB_EXCLUDE" ]          && tags="$tags exclude"
  [ -n "$XB_RETENTION" ]        && tags="$tags retention=$XB_RETENTION"
  [ -n "$tags" ]                && tags=" [x-backup:$tags]"
  echo "$tags"
}

# Apply retention: keep only the N most recent backups
apply_retention() {
  local backup_dir="$1"
  local stack_name="$2"
  local retention="$3"

  [ -z "$retention" ] && return
  [ "$retention" -le 0 ] 2>/dev/null && return

  local count
  count="$(find "$backup_dir" -maxdepth 1 -name "${stack_name}_*.tar.gz" -type f 2>/dev/null | wc -l)"
  if [ "$count" -gt "$retention" ]; then
    local to_delete
    to_delete="$((count - retention))"
    log "Retention=$retention, pruning $to_delete old backup(s) for '$stack_name'"
    find "$backup_dir" -maxdepth 1 -name "${stack_name}_*.tar.gz" -type f 2>/dev/null | sort | head -n "$to_delete" | while IFS= read -r f; do
      log "  Pruning: $(basename "$f")"
      dry rm -f "$f" "${f}.meta"
    done
  fi
}

# --- Named Volume Detection & Backup -----------------------------------------

# Parse the compose file and return named (non-bind-mount) volume names.
# These are listed under a top-level "volumes:" key or referenced under services.
detect_named_volumes() {
  local compose_file="$1"
  [ -z "$compose_file" ] && return

  python3 -c '
import sys, re

with open(sys.argv[1]) as f:
    content = f.read()

service_vols = set()
in_vol_list = False
for line in content.split("\n"):
    s = line.strip()
    if re.match(r"^volumes:", s):
        in_vol_list = True
        continue
    if in_vol_list:
        if not s or not line[0].isspace():
            in_vol_list = False
            continue
        # Short syntax: - source:target
        if s.startswith("- ") and ":" in s:
            parts = s.split(":", 1)
            src = parts[0].replace("- ", "", 1).strip()
            # Skip long-syntax keys (type, source, target, content)
            if src in ("type", "source", "target", "content", "consistency"):
                continue
            if src and not src.startswith("/") and not src.startswith("./") and not src.startswith("../") and not src.startswith("${"):
                service_vols.add(src)
            continue
        # Long syntax: source: value
        m = re.search(r"source:\s*(\S+)", s)
        if m:
            src = m.group(1)
            if src and not src.startswith("/") and not src.startswith("./") and not src.startswith("../") and not src.startswith("${"):
                service_vols.add(src)

for v in sorted(service_vols):
    print(v)
' "$compose_file" 2>/dev/null || true
}

# Warn about named volumes that won't be included in the file backup
warn_named_volumes() {
  local compose_file="$1"
  local stack_name="$2"

  [ -z "$compose_file" ] && return

  local vols
  vols="$(detect_named_volumes "$compose_file")"
  [ -z "$vols" ] && return

  echo "  [!] Named volumes detected (not backed up by default):"
  for v in $vols; do
    echo "      - $v"
  done
  echo "      These store data inside Docker's storage area, not on the host filesystem."
  echo "      Set 'x-backup.backup-volumes: false' to skip them."
}

# Resolve the actual Docker volume name for a compose volume reference.
# Compose v2 names volumes as <project>_<name> where project = stack dir name.
resolve_docker_volume() {
  local stack_name="$1"
  local vol_name="$2"
  echo "${stack_name}_${vol_name}"
}

# Back up named Docker volumes into a temporary directory.
# Each volume is dumped as <stack_path>/_volumes/<name>.tar.gz
backup_named_volumes() {
  local stack_path="$1"
  local stack_name="$2"
  local compose_file="$3"

  [ -z "$compose_file" ] && return

  local vols
  vols="$(detect_named_volumes "$compose_file")"
  [ -z "$vols" ] && return

  local volumes_dir="$stack_path/_volumes"
  mkdir -p "$volumes_dir"

  # shellcheck disable=SC2066
  while IFS= read -r vol; do
    [ -z "$vol" ] && continue
    # Skip anything that looks like a shell variable — it's a parametrised
    # bind-mount path, not a named Docker volume.
    [[ "$vol" == *'$'* ]] && continue
    local docker_vol
    docker_vol="$(resolve_docker_volume "$stack_name" "$vol")"

    if docker volume inspect "$docker_vol" >/dev/null 2>&1; then
      log "Dumping named volume '$docker_vol'..."
      if $DRY_RUN; then
        echo "[DRY-RUN] docker run --rm -v ${docker_vol}:/_src alpine tar -czf - -C /_src . > $volumes_dir/${vol}.tar.gz"
      else
        docker run --rm -v "${docker_vol}:/_src" alpine tar -czf - -C /_src . > "$volumes_dir/${vol}.tar.gz" 2>/dev/null || \
          warn "Failed to dump named volume '$docker_vol'"
      fi
    else
      warn "Named volume '$docker_vol' not found, skipping"
    fi
  done <<< "$vols"
}

# Restore named Docker volumes from a backup archive (the _volumes/ dir)
restore_named_volumes() {
  local stack_path="$1"
  local stack_name="$2"

  local volumes_dir="$stack_path/_volumes"
  [ ! -d "$volumes_dir" ] && return

  local any_restored=false
  for dump in "$volumes_dir"/*.tar.gz; do
    [ -f "$dump" ] || continue
    local vol_name
    vol_name="$(basename "$dump" .tar.gz)"
    local docker_vol
    docker_vol="$(resolve_docker_volume "$stack_name" "$vol_name")"

    log "Restoring named volume '$docker_vol'..."
    if $DRY_RUN; then
      echo "[DRY-RUN] docker volume create $docker_vol"
      echo "[DRY-RUN] cat $dump | docker run -i --rm -v ${docker_vol}:/_dst alpine tar -xzf - -C /_dst"
    else
      docker volume rm "$docker_vol" 2>/dev/null || true
      docker volume create "$docker_vol" >/dev/null || { warn "Failed to create volume '$docker_vol'"; continue; }
      # Pipe via cat + -i flag to ensure stdin works inside the container
      cat "$dump" | docker run -i --rm -v "${docker_vol}:/_dst" alpine tar -xzf - -C /_dst 2>/dev/null || \
        warn "Failed to restore named volume '$docker_vol'"
    fi
    any_restored=true
  done

  # Clean up the extracted volume dumps
  rm -rf "$volumes_dir"

  if $any_restored; then
    echo "  -> Named volumes restored"
  fi
}

run_hook() {
  local stage="$1"
  local stack_name="$2"
  local command="$3"

  [ -z "$command" ] && return
  log "Running $stage hook for '$stack_name': $command"
  if $DRY_RUN; then
    echo "[DRY-RUN] $command"
  else
    eval "$command" || warn "Hook '$stage' for '$stack_name' exited with status $?"
  fi
}

# --- Docker Operations -------------------------------------------------------

stack_is_running() {
  local stack_path="$1"
  local running
  running="$(discover_running_stacks)"
  # Match against either the full path or the basename (name-only fallback)
  echo "$running" | grep -qx "$stack_path" || echo "$running" | grep -qx "$(basename "$stack_path")"
}

stop_stack() {
  local stack_path="$1"
  local stack_name
  stack_name="$(basename "$stack_path")"

  # Find compose file in the stack directory
  local compose_file
  compose_file="$(resolve_compose_file "$stack_path")"

  if stack_is_running "$stack_path"; then
    log "Stopping stack '$stack_name' ($stack_path)..."
    if [ -n "$compose_file" ]; then
      dry docker compose -f "$compose_file" --project-directory "$stack_path" down 2>/dev/null || \
      dry docker compose -p "$stack_name" down 2>/dev/null || \
      warn "Failed to stop stack '$stack_name'"
    else
      dry docker compose -p "$stack_name" down 2>/dev/null || \
      warn "Failed to stop stack '$stack_name'"
    fi
  else
    log "Stack '$stack_name' is not running, skipping stop"
  fi
}

start_stack() {
  local stack_path="$1"
  local stack_name
  stack_name="$(basename "$stack_path")"

  local compose_file
  compose_file="$(resolve_compose_file "$stack_path")"

  if [ -n "$compose_file" ]; then
    log "Starting stack '$stack_name' ($stack_path)..."
    dry docker compose -f "$compose_file" --project-directory "$stack_path" up -d 2>/dev/null || \
    warn "Failed to start stack '$stack_name'"
  fi
}

# --- Backup ------------------------------------------------------------------

backup_stack() {
  require_root
  local TIMESTAMP
  TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
  local stacks_dir="$1"
  local backup_dir="$2"
  local stack_path="$3"
  local stack_name
  stack_name="$(stack_spec_to_name "$stack_path")"
  local backup_file="$backup_dir/${stack_name}_${TIMESTAMP}.tar.gz"
  local archive_root
  archive_root="$(dirname "$stack_path")"
  local archive_basename
  archive_basename="$(basename "$stack_path")"

  echo "Backing up stack '$stack_name' ($stack_path) -> $backup_file"

  # Load x-backup config
  load_x_backup_config "$stack_path"

  if [ "$XB_STOP" = "false" ]; then
    log "x-backup.stop=false, skipping container stop"
  else
    stop_stack "$stack_path"
  fi

  # Dump named Docker volumes (opt-out via x-backup.backup-volumes: false)
  local compose_file
  compose_file="$(resolve_compose_file "$stack_path")"
  local volumes_dumped=false
  if [ "$XB_BACKUP_VOLUMES" = "false" ]; then
    warn_named_volumes "$compose_file" "$stack_name"
  else
    log "Dumping named volumes..."
    backup_named_volumes "$stack_path" "$stack_name" "$compose_file"
    volumes_dumped=true
  fi

  # Pre-backup hook
  run_hook "pre-backup" "$stack_name" "$XB_PRE_HOOK"

  # Create the backup: tar from the parent directory so the stack dir
  # appears as itself in the archive, preserving relative exclude paths.
  # If named volumes were dumped, they live under _volumes/ inside stack_path
  # and are included in the archive automatically.
  dry mkdir -p "$backup_dir"
  fix_owner "$backup_dir" 2>/dev/null || true
  if $DRY_RUN; then
    echo "[DRY-RUN] tar -czf $backup_file --same-permissions --preserve-permissions $XB_EXCLUDE -C $archive_root $archive_basename"
  else
    # shellcheck disable=SC2086
    tar -czf "$backup_file" --same-permissions --preserve-permissions $XB_EXCLUDE -C "$archive_root" "$archive_basename"
    fix_owner "$backup_file"
    echo "  -> Backup complete: $(du -h "$backup_file" | cut -f1)"
  fi

  # Clean up temporary volume dumps
  if $volumes_dumped && [ -d "$stack_path/_volumes" ]; then
    dry rm -rf "$stack_path/_volumes"
  fi

  # Write metadata — used by the interactive restore menu and for
  # identifying backups without extracting the archive.
  if ! $DRY_RUN; then
    local has_volumes="false"
    [ -d "$stack_path/_volumes" ] && has_volumes="true"
    local archive_sha256 archive_size
    archive_sha256="$(sha256sum "$backup_file" | cut -d' ' -f1)"
    archive_size="$(stat -c%s "$backup_file" 2>/dev/null || stat -f%z "$backup_file" 2>/dev/null || echo "0")"
    cat > "${backup_file}.meta" <<EOF
stack=${stack_name}
path=${stack_path}
date=${TIMESTAMP}
compose=$(basename "${compose_file:-compose.yaml}")
volumes=${has_volumes}
sha256=${archive_sha256}
size=${archive_size}
EOF
    fix_owner "${backup_file}.meta"
  fi

  # Post-backup hook
  run_hook "post-backup" "$stack_name" "$XB_POST_HOOK"

  # Retention
  apply_retention "$backup_dir" "$stack_name" "$XB_RETENTION"

  # Restart the stack
  if [ "$XB_STOP" = "false" ]; then
    log "x-backup.stop=false, skipping container start"
  else
    start_stack "$stack_path"
  fi
}

# --- Interactive Restore Menu ------------------------------------------------

# Parse a backup filename and return a readable timestamp
format_backup_date() {
  local filename="$1"  # e.g. random_stack_20260711_193956.tar.gz
  local ts
  ts="$(echo "$filename" | sed 's/.*_\([0-9]\{8\}\)_\([0-9]\{6\}\)\.tar\.gz$/\1 \2/')"
  [ "$ts" = "$filename" ] && echo "unknown" && return
  local date_part="${ts% *}"
  local time_part="${ts#* }"
  local y="${date_part:0:4}"
  local m="${date_part:4:2}"
  local d="${date_part:6:2}"
  local hr="${time_part:0:2}"
  local min="${time_part:2:2}"
  local sec="${time_part:4:2}"
  echo "${y}-${m}-${d} ${hr}:${min}:${sec}"
}

interactive_restore_menu() {
  local stacks_dir="$1"
  local backup_dir="$2"

  if [ ! -d "$backup_dir" ]; then
    err "No backups directory found at $backup_dir"
  fi

  # Collect all backup files, deduplicate by stack (keep latest)
  local -A stack_latest
  local -A stack_date
  local f
  for f in "$backup_dir"/*.tar.gz; do
    [ -f "$f" ] || continue
    local basename_f
    basename_f="$(basename "$f")"
    local sname
    sname="$(echo "$basename_f" | sed 's/_[0-9]\{8\}_[0-9]\{6\}\.tar\.gz$//')"
    [ -z "$sname" ] && continue
    # Keep the latest (sorted by filename which is timestamped)
    if [ -z "${stack_latest[$sname]:-}" ] || [ "$basename_f" > "$(basename "${stack_latest[$sname]}")" ]; then
      stack_latest["$sname"]="$f"
      stack_date["$sname"]="$(format_backup_date "$basename_f")"
      local sname_meta="${f}.meta"
      if [ -f "$sname_meta" ]; then
        local sv
        sv="$(grep '^volumes=' "$sname_meta" | head -1 | sed 's/^volumes=//')"
        [ "$sv" = "true" ] && stack_date["$sname"]="${stack_date[$sname]} [volumes]" || true
      fi
    fi
  done

  local names=()
  for key in "${!stack_latest[@]}"; do
    names+=("$key")
  done

  if [ ${#names[@]} -eq 0 ]; then
    echo "No backups found in $backup_dir"
    exit 0
  fi

  # Sort alphabetically
  IFS=$'\n' names=($(sort <<<"${names[*]}")); unset IFS

  echo ""
  echo "Available backups:"
  echo "------------------"
  local i
  for i in "${!names[@]}"; do
    local idx=$((i + 1))
    printf "  %2d) %-20s  %s\n" "$idx" "${names[$i]}" "${stack_date[${names[$i]}]:-}"
  done
  echo "  all) Restore all stacks"
  echo "   q)  Cancel"
  echo ""

  read -r -p "Select stacks to restore (numbers, 'all', or 'q'): " selection

  [ -z "$selection" ] && echo "Cancelled." && exit 0

  # Normalise: handle "all"
  if [ "$selection" = "all" ] || [ "$selection" = "a" ]; then
    selection=""
    for i in "${!names[@]}"; do
      selection="$selection $((i + 1))"
    done
  fi

  if [ "$selection" = "q" ]; then
    echo "Cancelled."
    exit 0
  fi

  # Convert space-separated indices to stack names
  local selected_stacks=()
  local seen_idx
  for seen_idx in $selection; do
    seen_idx="$(echo "$seen_idx" | xargs)"  # trim
    # Allow ranges like 1-3
    if [[ "$seen_idx" =~ ^([0-9]+)-([0-9]+)$ ]]; then
      for ((j = ${BASH_REMATCH[1]}; j <= ${BASH_REMATCH[2]}; j++)); do
        if [ "$j" -ge 1 ] && [ "$j" -le "${#names[@]}" ]; then
          selected_stacks+=("${names[$((j - 1))]}")
        fi
      done
    elif [[ "$seen_idx" =~ ^[0-9]+$ ]]; then
      if [ "$seen_idx" -ge 1 ] && [ "$seen_idx" -le "${#names[@]}" ]; then
        selected_stacks+=("${names[$((seen_idx - 1))]}")
      else
        echo "Invalid selection: $seen_idx (out of range)"
      fi
    fi
  done

  if [ ${#selected_stacks[@]} -eq 0 ]; then
    echo "No valid stacks selected."
    exit 0
  fi

  # Resolve paths and re-exec via sudo with explicit --stack flags
  local -a stack_flags=()
  for s in "${selected_stacks[@]}"; do
    local spath
    # Read original path from .meta file if available
    local meta_file
    meta_file="$(find "$backup_dir" -maxdepth 1 -name "${s}_*.tar.gz.meta" -type f 2>/dev/null | sort | tail -1)"
    if [ -n "$meta_file" ]; then
      spath="$(grep '^path=' "$meta_file" | head -1 | sed 's/^path=//')"
      [ -z "$spath" ] && spath="$(cat "$meta_file" | head -1)"  # legacy format
    elif [ -d "$stacks_dir/$s" ]; then
      spath="$stacks_dir/$s"
    else
      warn "Cannot determine where to restore '$s': no metadata file and no directory under '$stacks_dir'.  Use '--stack /original/path'."
      continue
    fi
    stack_flags+=("--stack" "$spath")
  done

  if [ ${#stack_flags[@]} -eq 0 ]; then
    echo "No valid stacks selected."
    exit 0
  fi

  # Elevate and re-exec with the selected stacks
  if [ "$(id -u)" -ne 0 ]; then
    echo "Elevating to root via sudo..."
    exec sudo "$0" restore --backup-dir "$backup_dir" "${stack_flags[@]}"
  fi

  echo ""
  for ((i = 0; i < ${#stack_flags[@]}; i += 2)); do
    local spath="${stack_flags[$((i + 1))]}"
    restore_stack "$stacks_dir" "$backup_dir" "$spath"
  done
}

# --- Verify -------------------------------------------------------------------

verify_backup() {
  local backup_dir="$1"
  local stack_name="$2"

  if [ -n "$stack_name" ]; then
    local files
    files="$(find "$backup_dir" -maxdepth 1 -name "${stack_name}_*.tar.gz" -type f 2>/dev/null | sort)"
  else
    local files
    files="$(find "$backup_dir" -maxdepth 1 -name '*.tar.gz' -type f 2>/dev/null | sort)"
  fi

  if [ -z "$files" ]; then
    echo "No backups found."
    return
  fi

  local exit_code=0
  for f in $files; do
    local basename_f
    basename_f="$(basename "$f")"
    local meta_file="${f}.meta"

    if [ ! -f "$meta_file" ]; then
      echo "  SKIP  $basename_f  (no .meta file)"
      continue
    fi

    local stored_hash stored_size
    stored_hash="$(grep '^sha256=' "$meta_file" | head -1 | sed 's/^sha256=//')"
    stored_size="$(grep '^size=' "$meta_file" | head -1 | sed 's/^size=//')"

    if [ -z "$stored_hash" ]; then
      echo "  SKIP  $basename_f  (no sha256 in .meta — legacy backup)"
      continue
    fi

    # Check file still exists
    if [ ! -f "$f" ]; then
      echo "  FAIL  $basename_f  (file missing)"
      exit_code=1
      continue
    fi

    # Check size
    local actual_size
    actual_size="$(stat -c%s "$f" 2>/dev/null || stat -f%z "$f" 2>/dev/null || echo "0")"
    if [ "$actual_size" != "$stored_size" ]; then
      echo "  FAIL  $basename_f  (size mismatch: $actual_size vs $stored_size)"
      exit_code=1
      continue
    fi

    # Check hash
    local actual_hash
    actual_hash="$(sha256sum "$f" | cut -d' ' -f1)"
    if [ "$actual_hash" != "$stored_hash" ]; then
      echo "  FAIL  $basename_f  (sha256 mismatch)"
      exit_code=1
    else
      local human_size
      human_size="$(echo "$stored_size" | numfmt --to=iec 2>/dev/null || echo "${stored_size}B")"
      echo "  OK    $basename_f  ($human_size)"
    fi
  done

  return $exit_code
}

# --- Interactive Backup Menu ------------------------------------------------

interactive_backup_menu() {
  local stacks_dir="$1"
  local backup_dir="$2"

  # Collect stacks: local + running, deduped by canonical path
  local -A seen
  local -a names
  local -a paths
  local -a labels

  # Running stacks (absolute paths)
  local running
  running="$(discover_running_stacks 2>/dev/null || true)"
  if [ -n "$running" ]; then
    while IFS= read -r run_path; do
      [ -z "$run_path" ] && continue
      [[ "$run_path" != /* ]] && continue
      local canon
      canon="$(cd "$run_path" 2>/dev/null && pwd || echo "$run_path")"
      [ -n "${seen[$canon]:-}" ] && continue
      seen["$canon"]=1
      local run_name
      run_name="$(basename "$run_path")"
      names+=("$run_name")
      if [ "$canon" != "$stacks_dir/$run_name" ]; then
        paths+=("$canon")
        labels+=("(running, $canon)")
      else
        paths+=("$canon")
        labels+=("(running)")
      fi
    done <<< "$running"
  fi

  # Local stacks (not already seen)
  for dir in "$stacks_dir"/*/; do
    [ -d "$dir" ] || continue
    if [ -f "$dir/compose.yaml" ] || [ -f "$dir/compose.yml" ] || [ -f "$dir/docker-compose.yaml" ] || [ -f "$dir/docker-compose.yml" ]; then
      local canon
      canon="$(cd "$dir" 2>/dev/null && pwd || echo "$dir")"
      [ -n "${seen[$canon]:-}" ] && continue
      seen["$canon"]=1
      local stack
      stack="$(basename "$dir")"
      names+=("$stack")
      paths+=("$canon")
      local running_label=""
      stack_is_running "$canon" 2>/dev/null && running_label=" (running)"
      labels+=("$running_label")
    fi
  done

  if [ ${#names[@]} -eq 0 ]; then
    echo "No stacks found."
    exit 0
  fi

  echo ""
  echo "Available stacks:"
  echo "-----------------"
  local i
  for i in "${!names[@]}"; do
    local idx=$((i + 1))
    printf "  %2d) %-20s %s\n" "$idx" "${names[$i]}" "${labels[$i]}"
  done
  echo "  all) Back up all stacks"
  echo "   q)  Cancel"
  echo ""

  read -r -p "Select stacks to back up (numbers, 'all', or 'q'): " selection

  [ -z "$selection" ] && echo "Cancelled." && exit 0

  if [ "$selection" = "all" ] || [ "$selection" = "a" ]; then
    selection=""
    for i in "${!names[@]}"; do
      selection="$selection $((i + 1))"
    done
  fi

  if [ "$selection" = "q" ]; then
    echo "Cancelled."
    exit 0
  fi

  local selected_paths=()
  local seen_idx
  for seen_idx in $selection; do
    seen_idx="$(echo "$seen_idx" | xargs)"
    if [[ "$seen_idx" =~ ^([0-9]+)-([0-9]+)$ ]]; then
      for ((j = ${BASH_REMATCH[1]}; j <= ${BASH_REMATCH[2]}; j++)); do
        if [ "$j" -ge 1 ] && [ "$j" -le "${#names[@]}" ]; then
          selected_paths+=("${paths[$((j - 1))]}")
        fi
      done
    elif [[ "$seen_idx" =~ ^[0-9]+$ ]]; then
      if [ "$seen_idx" -ge 1 ] && [ "$seen_idx" -le "${#names[@]}" ]; then
        selected_paths+=("${paths[$((seen_idx - 1))]}")
      else
        echo "Invalid selection: $seen_idx (out of range)"
      fi
    fi
  done

  if [ ${#selected_paths[@]} -eq 0 ]; then
    echo "No valid stacks selected."
    exit 0
  fi

  # Elevate and re-exec with the selected stacks
  if [ "$(id -u)" -ne 0 ]; then
    local -a stack_flags=()
    for spath in "${selected_paths[@]}"; do
      stack_flags+=("--stack" "$spath")
    done
    echo "Elevating to root via sudo..."
    exec sudo "$0" backup --backup-dir "$backup_dir" "${stack_flags[@]}"
  fi

  echo ""
  for spath in "${selected_paths[@]}"; do
    backup_stack "$stacks_dir" "$backup_dir" "$spath"
  done
}

# --- Restore -----------------------------------------------------------------

restore_stack() {
  require_root
  local TIMESTAMP
  TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
  local stacks_dir="$1"
  local backup_dir="$2"
  local stack_path="$3"
  local stack_name
  stack_name="$(stack_spec_to_name "$stack_path")"
  local archive_root
  archive_root="$(dirname "$stack_path")"
  local archive_basename
  archive_basename="$(basename "$stack_path")"

  # Find the latest backup for this stack
  local backup_file
  backup_file="$(find "$backup_dir" -maxdepth 1 -name "${stack_name}_*.tar.gz" -type f 2>/dev/null | sort | tail -1)"

  if [ -z "$backup_file" ]; then
    err "No backup found for stack '$stack_name' in $backup_dir"
  fi

  echo "Restoring stack '$stack_name' ($stack_path) from $backup_file"

  # Get list of files in backup to confirm it's the right one
  if $VERBOSE; then
    tar -tzf "$backup_file" | head -20
  fi

  # Load x-backup config from within the backup (extract to temp)
  local temp_dir
  temp_dir="$(mktemp -d)"
  local parsed_compose=""
  tar -xzf "$backup_file" -C "$temp_dir" --wildcards "*/compose.y*" "*/compose.yml*" "*/docker-compose.y*" "*/docker-compose.yml*" 2>/dev/null || true
  local stack_dir_in_backup
  stack_dir_in_backup="$(ls "$temp_dir" 2>/dev/null | head -1)"
  [ -n "$stack_dir_in_backup" ] && parsed_compose="$temp_dir/$stack_dir_in_backup"

  if [ -n "$parsed_compose" ]; then
    load_x_backup_config "$parsed_compose"
  else
    # Fallback: load from the live dir if it still exists
    load_x_backup_config "$stack_path"
  fi

  # Track whether the stack was running before we stop it, so we can
  # restart it afterwards only if it was originally running.
  local was_running=false
  stack_is_running "$stack_path" && was_running=true

  # Always stop during restore — we're about to rename/move the entire
  # stack directory out from under any running containers.  The
  # x-backup.stop flag only applies to backup, where data is read-only.
  stop_stack "$stack_path"

  # Pre-restore hook
  run_hook "pre-restore" "$stack_name" "$XB_PRE_RESTORE_HOOK"

  # Clean up any stale pre-restore dirs from previous failed restores
  for old in "$stack_path".pre-restore-*; do
    [ -d "$old" ] || continue
    log "Removing stale pre-restore backup: $(basename "$old")"
    dry rm -rf "$old"
  done

  # Restore
  if $DRY_RUN; then
    echo "[DRY-RUN] tar -xzf $backup_file --same-permissions --preserve-permissions -C $archive_root"
  else
    if [ -d "$stack_path" ]; then
      local existing_backup="${stack_path}.pre-restore-${TIMESTAMP}"
      log "Moving existing '$stack_name' to $existing_backup"
      mv "$stack_path" "$existing_backup"
    fi
    tar -xzf "$backup_file" --same-permissions --preserve-permissions -C "$archive_root"
    # Restore succeeded — remove the pre-restore safety copy since the
    # backup archive is the authoritative restore point.
    if [ -n "${existing_backup:-}" ] && [ -d "$existing_backup" ]; then
      dry rm -rf "$existing_backup"
    fi
    echo "  -> Restore complete"
    # Restore named Docker volumes from the _volumes/ dir inside the archive
    restore_named_volumes "$stack_path" "$stack_name"
  fi

  # Post-restore hook
  run_hook "post-restore" "$stack_name" "$XB_POST_RESTORE_HOOK"

  # Cleanup temp
  rm -rf "$temp_dir"

  # Only restart if it was running before we stopped it
  if $was_running; then
    start_stack "$stack_path"
  else
    log "Stack was not running before restore, skipping start"
  fi
}

# --- List --------------------------------------------------------------------

list_stacks_and_backups() {
  local stacks_dir="$1"
  local backup_dir="$2"

  echo "=== Stacks ==="
  # Collect all seen stack dirs to avoid duplicates
  local -A seen

  # 1) Running stacks (discovered wherever they live)
  local running
  running="$(discover_running_stacks 2>/dev/null || true)"
  local has_running=false
  if [ -n "$running" ]; then
    while IFS= read -r run_path; do
      [ -z "$run_path" ] && continue
      # skip bare names (fallback from no-Docker mode — they'll be in local)
      [[ "$run_path" != /* ]] && continue
      # canonical key: realpath or full path
      local canon
      canon="$(cd "$run_path" 2>/dev/null && pwd || echo "$run_path")"
      seen["$canon"]=1
      local run_name
      run_name="$(basename "$run_path")"
      local label="(running)"
      load_x_backup_config "$run_path"
      local config_tags
      config_tags="$(format_xb_tags)"
      if [ "$canon" != "$stacks_dir/$run_name" ]; then
        echo "  - $run_name ($run_path) $label$config_tags"
      else
        echo "  - $run_name $label$config_tags"
      fi
      has_running=true
    done <<< "$running"
  fi

  # 2) Local stacks (not already listed as running)
  local has_local=false
  for dir in "$stacks_dir"/*/; do
    [ -d "$dir" ] || continue
    local stack
    stack="$(basename "$dir")"
    if [ -f "$dir/compose.yaml" ] || [ -f "$dir/compose.yml" ] || [ -f "$dir/docker-compose.yaml" ] || [ -f "$dir/docker-compose.yml" ]; then
      local canon
      canon="$(cd "$dir" 2>/dev/null && pwd || echo "$dir")"
      if [ -z "${seen["$canon"]:-}" ]; then
        seen["$canon"]=1
        local label=""
        stack_is_running "$dir" 2>/dev/null && label=" (running)"
        load_x_backup_config "$dir"
        local config_tags
        config_tags="$(format_xb_tags)"
        echo "  - $stack$label$config_tags"
        has_local=true
      fi
    fi
  done

  if ! $has_running && ! $has_local; then
    echo "  No stacks found."
  fi

  echo ""
  echo "=== Backups ==="
  if [ -d "$backup_dir" ]; then
    local backup_count=0
    for f in "$backup_dir"/*.tar.gz; do
      [ -f "$f" ] || continue
      local sname
      sname="$(basename "$f" | sed 's/_[0-9]\{8\}_[0-9]\{6\}\.tar\.gz$//')"
      local meta_info=""
      if [ -f "${f}.meta" ]; then
        local mpath mvols mhash
        mpath="$(grep '^path=' "${f}.meta" | head -1 | sed 's/^path=//')"
        mvols="$(grep '^volumes=' "${f}.meta" | head -1 | sed 's/^volumes=//')"
        mhash="$(grep '^sha256=' "${f}.meta" | head -1 | sed 's/^sha256=//')"
        [ -n "$mhash" ] && meta_info=" sha256:${mhash:0:12}.."
        [ -n "$mpath" ] && [ "$mpath" != "$(dirname "$f")/$sname" ] && meta_info="$meta_info ($mpath)"
        [ "$mvols" = "true" ] && meta_info="$meta_info [volumes]"
      fi
      echo "  $(basename "$f")  ($(du -h "$f" | cut -f1))  -> $sname$meta_info"
      backup_count=$((backup_count + 1))
    done
    if [ "$backup_count" -eq 0 ]; then
      echo "  No backups found in $backup_dir"
    fi
  else
    echo "  No backups directory found at $backup_dir"
  fi
}

# --- Main --------------------------------------------------------------------

main() {
  check_deps
  ORIG_ARGS=("$@")

  local SCRIPT_DIR
  SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
  local INITIAL_DIR
  INITIAL_DIR="$(pwd -P)"
  local stacks_dir="$SCRIPT_DIR"
  local backup_dir="${INITIAL_DIR}/backups"

  # Parse arguments
  while [ $# -gt 0 ]; do
    case "$1" in
      backup|restore|list|verify)
        ACTION="$1"
        shift
        ;;
      -s|--stack)
        if [ $# -lt 2 ]; then err "Option $1 requires an argument"; fi
        IFS=',' read -ra ADDR <<< "$2"
        for s in "${ADDR[@]}"; do
          TARGET_STACKS+=("$(echo "$s" | xargs)")
        done
        HAS_FLAGS=true
        shift 2
        ;;
      -a|--all)
        MODE="all"
        HAS_FLAGS=true
        shift
        ;;
      -r|--running)
        MODE="running"
        HAS_FLAGS=true
        shift
        ;;
      -d|--backup-dir)
        if [ $# -lt 2 ]; then err "Option $1 requires an argument"; fi
        backup_dir="$2"
        shift 2
        ;;
      -n|--dry-run)
        DRY_RUN=true
        shift
        ;;
      -v|--verbose)
        VERBOSE=true
        shift
        ;;
      -h|--help)
        usage
        ;;
      *)
        err "Unknown option: $1. Use -h for help."
        ;;
    esac
  done

  if [ -z "$ACTION" ]; then
    usage
  fi

  if [ ! -d "$stacks_dir" ]; then
    err "Stacks directory not found: $stacks_dir"
  fi

  case "$ACTION" in
    list)
      list_stacks_and_backups "$stacks_dir" "$backup_dir"
      ;;
    backup)
      mkdir -p "$backup_dir"
      if ! $HAS_FLAGS; then
        # No flags at all — show interactive menu (display doesn't need root)
        interactive_backup_menu "$stacks_dir" "$backup_dir"
        exit 0
      fi
      local stacks
      stacks="$(resolve_target_stacks "$stacks_dir" "$MODE" "${TARGET_STACKS[@]}")"
      if [ -z "$stacks" ]; then
        echo "No stacks found in '$stacks_dir'.  Either cd there, pass --stack /path, or use -d to set the backup directory."
        exit 0
      fi
      local stack
      for stack in $stacks; do
        local spath
        spath="$(stack_spec_to_path "$stacks_dir" "$stack")"
        backup_stack "$stacks_dir" "$backup_dir" "$spath" || warn "Backup failed for '$spath'"
      done
      ;;
    restore)
      if [ ! -d "$backup_dir" ]; then
        err "Backup directory not found: $backup_dir"
      fi
      if [ ${#TARGET_STACKS[@]} -eq 0 ]; then
        # No --stack flag — show interactive menu (display doesn't need root)
        interactive_restore_menu "$stacks_dir" "$backup_dir"
        exit 0
      fi
      for spec in "${TARGET_STACKS[@]}"; do
        local spath
        spath="$(stack_spec_to_path "$stacks_dir" "$spec")"
        restore_stack "$stacks_dir" "$backup_dir" "$spath" || warn "Restore failed for '$spath'"
      done
      ;;
    verify)
      if [ ! -d "$backup_dir" ]; then
        err "Backup directory not found: $backup_dir"
      fi
      local vtarget=""
      [ ${#TARGET_STACKS[@]} -gt 0 ] && vtarget="${TARGET_STACKS[0]}"
      verify_backup "$backup_dir" "$vtarget"
      ;;
  esac
}

main "$@"