CR-2026-04-18: Rack Relocation — Implementation

# Mount Synology NFS
sudo mount -t nfs nas-01.inside.domusdigitalis.dev:/volume1/borg_backups /mnt/synology

# Load credentials
eval "$(dsec source d000 dev/storage)"

# Run backup
sudo -E BORG_PASSPHRASE="$BORG_PASSPHRASE" ~/.local/bin/borg-backup-synology.sh

# Verify latest archive
sudo -E BORG_PASSPHRASE="$BORG_PASSPHRASE" borg list /mnt/synology/borg-repo | tail -5

# Clear credentials and unmount
eval "$(dsec unsource)"
sudo umount /mnt/synology

ssh kvm-01 'BACKUP_DIR="/mnt/onboard-ssd/backup-$(date +%Y%m%d)" && \
  sudo mkdir -p "$BACKUP_DIR" && \
  for vm in $(sudo virsh list --all --name); do
    sudo virsh dumpxml "$vm" > "$BACKUP_DIR/$vm.xml"
  done && \
  ls -lh "$BACKUP_DIR"'

ssh kvm-02 'BACKUP_DIR="/var/lib/libvirt/backup-$(date +%Y%m%d)" && \
  sudo mkdir -p "$BACKUP_DIR" && \
  for vm in $(sudo virsh list --all --name); do
    sudo virsh dumpxml "$vm" > "$BACKUP_DIR/$vm.xml"
  done && \
  ls -lh "$BACKUP_DIR"'

mkdir -p ~/Documents/rack-backup/{kvm-01,kvm-02}
scp kvm-01:/mnt/onboard-ssd/backup-$(date +%Y%m%d)/*.xml ~/Documents/rack-backup/kvm-01/
scp kvm-02:/var/lib/libvirt/backup-$(date +%Y%m%d)/*.xml ~/Documents/rack-backup/kvm-02/
ls -lh ~/Documents/rack-backup/kvm-01/ ~/Documents/rack-backup/kvm-02/

enable
write mem
show running-config | redirect flash:pre-relocation-2026-04-18.txt

# Application VMs (no dependents)
for vm in 9800-WLC-02 ise-02; do
  sudo virsh shutdown $vm
done

# Wait for application VMs
while sudo virsh list | grep -E 'WLC-02|ise-02' | grep -q running; do
  sleep 3; echo "waiting for app VMs..."
done

# Network
sudo virsh shutdown vyos-02

# Infrastructure (DNS, secrets) — last
for vm in vault-03 vault-02 bind-02; do
  sudo virsh shutdown $vm
done

# Wait for all
while sudo virsh list | grep -q running; do
  sleep 3; echo "waiting..."
done
echo "kvm-02: all VMs off"

# Verify
sudo virsh list --all | awk 'NR>2 {print $2": "$3}'

# Application VMs
for vm in 9800-WLC-01 k3s-master-01 ipsk-mgr-01 ipa-01; do
  sudo virsh shutdown $vm
done

# Wait for application VMs
while sudo virsh list | grep -E 'WLC-01|k3s|ipsk|ipa' | grep -q running; do
  sleep 3; echo "waiting for app VMs..."
done

# Network
sudo virsh shutdown vyos-01

# Infrastructure — AD before DNS
for vm in vault-01 home-dc01 bind-01; do
  sudo virsh shutdown $vm
done

# Wait for all
while sudo virsh list | grep -q running; do
  sleep 3; echo "waiting..."
done
echo "kvm-01: all VMs off"

# Verify
sudo virsh list --all | awk 'NR>2 {print $2": "$3}'

# kvm-02 first (secondary)
ssh kvm-02 'sudo shutdown -h now'

# kvm-01 second (primary)
ssh kvm-01 'sudo shutdown -h now'

ssh admin@nas-01 'sudo shutdown -h now'

enable
show startup-config | include hostname
show vlan brief

# From workstation — poll until NAS responds
while ! ping -c1 -W2 nas-01.inside.domusdigitalis.dev &>/dev/null; do
  sleep 5; echo "waiting for NAS..."
done
echo "NAS is up"

# Poll until hypervisors respond
for host in kvm-01 kvm-02; do
  while ! ssh -o ConnectTimeout=3 $host 'echo up' &>/dev/null; do
    sleep 5; echo "waiting for $host..."
  done
  echo "$host is up"
done

# DNS first — everything depends on resolution
sudo virsh start bind-01

# Wait for DNS to respond
while ! dig @bind-01.inside.domusdigitalis.dev inside.domusdigitalis.dev SOA +short &>/dev/null; do
  sleep 3; echo "waiting for bind-01..."
done
echo "bind-01 DNS responding"

# Active Directory — depends on DNS
sudo virsh start home-dc01

# Vault — depends on DNS
sudo virsh start vault-01

sudo virsh start bind-02
sudo virsh start vault-02
sudo virsh start vault-03

# kvm-01
ssh kvm-01 'sudo virsh start vyos-01'

# kvm-02
ssh kvm-02 'sudo virsh start vyos-02'

for vm in ipa-01 9800-WLC-01 k3s-master-01 ipsk-mgr-01; do
  sudo virsh start $vm
done

for vm in ise-02 9800-WLC-02; do
  sudo virsh start $vm
done