Runbook: Backup All Infrastructure

#!/bin/bash
# Vault Backup - Verification & Manual trigger
# Runbook: backup-all-infrastructure.adoc
# Detailed: vault-backup.adoc

# Prerequisites
# - SSH access to vault-01 (or run directly on vault-01)
# - systemd timer configured (see vault-backup.adoc)

# =============================================================================
# 1. Check timer status
# =============================================================================
systemctl list-timers vault-backup.timer --no-pager

# Expected output:
# NEXT                        LEFT   LAST                        PASSED       UNIT               ACTIVATES
# Tue 2026-02-24 02:28:20 UTC 23h    Mon 2026-02-23 02:13:58 UTC 38min ago    vault-backup.timer vault-backup.service

# =============================================================================
# 2. Check last backup log
# =============================================================================
sudo journalctl -u vault-backup.service -n 10 --no-pager

# Expected output (successful):
# Feb 23 02:13:58 vault-01... systemd[1]: Starting vault-backup.service - Vault Backup to NAS...
# Feb 23 02:13:59 vault-01... bash[55856]: sending incremental file list
# Feb 23 02:13:59 vault-01... bash[55856]: vault-backup-20260223-021358.tar.gz
# Feb 23 02:13:59 vault-01... bash[55856]: sent 117,703 bytes  received 35 bytes  235,476.00 bytes/sec
# Feb 23 02:13:59 vault-01... bash[55856]: total size is 149,853  speedup is 1.27
# Feb 23 02:13:59 vault-01... systemd[1]: vault-backup.service: Deactivated successfully.
# Feb 23 02:13:59 vault-01... systemd[1]: Finished vault-backup.service - Vault Backup to NAS.

# =============================================================================
# 3. Verify backups on NAS
# =============================================================================
ssh nas-01 "ls -lh /volume1/vault_backups/*.tar.gz | tail -3"

# Expected output:
# -rw-r--r-- 1 hashi-backups users 137K Feb 20 18:04 vault-backup-20260221-020458.tar.gz
# -rw-r--r-- 1 hashi-backups users 143K Feb 21 18:18 vault-backup-20260222-021858.tar.gz
# -rw-r--r-- 1 hashi-backups users 147K Feb 22 18:13 vault-backup-20260223-021358.tar.gz

# =============================================================================
# 4. Manual backup (if needed)
# =============================================================================
# sudo systemctl start vault-backup.service
# sudo systemctl status vault-backup.service

# =============================================================================
# TROUBLESHOOTING: SSH falls back to YubiKey instead of Vault cert
# =============================================================================
# If prompted for YubiKey passphrase, your Vault cert may be missing a principal.
#
# CRITICAL: Vault SSH roles do NOT support 'default_principals' - it's silently
# ignored. You MUST specify valid_principals on EVERY sign request.
#
# Check current principals:
#   ssh-keygen -Lf ~/.ssh/id_ed25519_vault-cert.pub | grep -A6 "Principals:"
#
# Required principals by host:
#   - adminerosado: Synology NAS
#   - admin: VyOS, network devices
#   - ansible: automation
#   - evanusmodestus: Linux workstations
#   - root: emergency access
#
# Fix: Update vault-ssh-sign script then re-sign:
#   sed -i 's/valid_principals="\([^"]*\)"/valid_principals="adminerosado,admin,ansible,evanusmodestus,root"/' ~/.local/bin/vault-ssh-sign
#   vault-ssh-sign
#
# Or manual re-sign (not recommended):
#   dsource d000 dev/vault
#   vault write -field=signed_key ssh/sign/domus-client \
#     public_key=@$HOME/.ssh/id_ed25519_vault.pub \
#     valid_principals="adminerosado,admin,ansible,evanusmodestus,root" \
#     >| ~/.ssh/id_ed25519_vault-cert.pub
#   ssh-add -d ~/.ssh/id_ed25519_vault 2>/dev/null
#   ssh-add ~/.ssh/id_ed25519_vault

#!/bin/bash
# ISE Backup
# Runbook: backup-all-infrastructure.adoc

# Prerequisites
dsource d000 dev/network

# Trigger ISE config backup (async - takes several minutes)
netapi ise backup --upload-nas

# Check backup status
netapi ise api-call openapi GET '/api/v1/backup-restore/config/last-backup-status' | jq .

# Verify on NAS (after backup completes)
ssh nas-01 "ls -lh /volume1/ise_backups/*.tar.gpg | tail -3"

# Manual: Check ISE repository
netapi ise api-call openapi GET '/api/v1/repository' | jq -r '.response[].name'

#!/bin/bash
# VyOS Backup
# Runbook: backup-all-infrastructure.adoc

# Prerequisites
dsource d000 dev/network

# Backup VyOS configuration (both nodes for HA)
for node in vyos-01 vyos-02; do
    echo "=== Backing up $node ==="
    ssh $node "source /opt/vyatta/etc/functions/script-template && run show configuration commands" > /tmp/${node}-$(date +%Y%m%d).cfg
done

# Upload to NAS
scp /tmp/vyos-*.cfg nas-01:/volume1/firewall_backups/

# Verify on NAS
ssh nas-01 "ls -lh /volume1/firewall_backups/vyos*.cfg | tail -5"

# Cleanup local files
rm -f /tmp/vyos-*.cfg

# Alternative: netapi (when implemented)
# netapi vyos backup --upload-nas

#!/bin/bash
# 9800-CL WLC Backup
# Runbook: backup-all-infrastructure.adoc

# Prerequisites
dsource d000 dev/network

# Backup (exports running config and uploads to NAS)
netapi wlc backup --upload-nas

# Verify on NAS
ssh nas-01 "ls -lh /volume1/wlc_backups/*.json | tail -3"

# Manual alternative (no netapi)
# ssh wlc-01 "show running-config" > /tmp/wlc-$(date +%Y%m%d).cfg
# scp /tmp/wlc-*.cfg nas-01:/volume1/wlc_backups/

#!/bin/bash
# IOS Switch Backup
# Runbook: backup-all-infrastructure.adoc

# Prerequisites
dsource d000 dev/network

# Backup all switches
netapi ios backup --all --upload-nas

# Backup single switch
# netapi ios backup --host 10.50.1.10 --upload-nas

# Verify on NAS
ssh nas-01 "ls -lh /volume1/switch_backups/*.cfg | tail -5"

# Manual alternative (no netapi)
# ssh admin@10.50.1.10 "show running-config" > /tmp/switch-$(date +%Y%m%d).cfg
# scp /tmp/switch-*.cfg nas-01:/volume1/switch_backups/

#!/bin/bash
# KVM VM Definitions Backup
# Runbook: backup-all-infrastructure.adoc
# Note: This backs up XML definitions, NOT disk images

# Prerequisites
dsource d000 dev/network

# Backup all VM definitions
netapi kvm backup --all --upload-nas

# Verify on NAS
ssh nas-01 "ls -lh /volume1/kvm_backups/*.xml | tail -5"

# Manual alternative (no netapi)
ssh kvm-01 'for vm in $(virsh list --all --name); do
  virsh dumpxml "$vm" > /tmp/"$vm".xml
done'
scp kvm-01:/tmp/*.xml nas-01:/volume1/kvm_backups/

# Note: Disk images backed up separately via Borg

#!/bin/bash
# k3s Backup (Single-node SQLite)
# Runbook: backup-all-infrastructure.adoc, k3s-operations.adoc
# Note: Single-node k3s uses SQLite, NOT etcd

# Run on k3s-master-01
ssh k3s-master-01 << 'EOF'
sudo mkdir -p /mnt/k3s_backups
sudo mount -t nfs nas-01:/volume1/k3s_backups /mnt/k3s_backups

# Backup SQLite database
sudo cp -v /var/lib/rancher/k3s/server/db/state.db \
  /mnt/k3s_backups/etcd/state.db.$(date +%Y%m%d)

# Backup manifests, TLS, token (use -rv directory, NOT *)
sudo cp -rv /var/lib/rancher/k3s/server/manifests /mnt/k3s_backups/
sudo cp -rv /var/lib/rancher/k3s/server/tls /mnt/k3s_backups/
sudo cp -v /var/lib/rancher/k3s/server/token /mnt/k3s_backups/

# Verify
ls -la /mnt/k3s_backups/

sudo umount /mnt/k3s_backups
EOF

#!/bin/bash
# Keycloak Backup
# Runbook: backup-all-infrastructure.adoc
# Status: PENDING - needs keycloak_backups share on NAS

# Prerequisites
dsource d000 dev/identity

# Backup realm export
netapi keycloak backup --upload-nas

# Manual alternative
ssh keycloak-01 "/opt/keycloak/bin/kc.sh export \
  --dir /tmp/keycloak-export \
  --realm domus"
scp -r keycloak-01:/tmp/keycloak-export/* nas-01:/volume1/keycloak_backups/

# Verify on NAS
ssh nas-01 "ls -lh /volume1/keycloak_backups/*.json | tail -3"

#!/bin/bash
# BIND DNS Backup
# Runbook: backup-all-infrastructure.adoc
# Status: PENDING - needs bind_backups share on NAS

# Run on bind-01
ssh bind-01 << 'EOF'
DATE=$(date +%Y%m%d)
BACKUP_DIR="/tmp/bind-backup-$DATE"
mkdir -p "$BACKUP_DIR"

# Backup zone files
cp -rv /var/named/*.zone "$BACKUP_DIR/"
cp -rv /var/named/*.db "$BACKUP_DIR/"

# Backup config
cp -v /etc/named.conf "$BACKUP_DIR/"
cp -rv /etc/named/ "$BACKUP_DIR/named-conf.d/"

# Create tarball
tar -czf "/tmp/bind-backup-$DATE.tar.gz" -C /tmp "bind-backup-$DATE"
rm -rf "$BACKUP_DIR"
EOF

# Copy to NAS
scp bind-01:/tmp/bind-backup-*.tar.gz nas-01:/volume1/bind_backups/

# Verify
ssh nas-01 "ls -lh /volume1/bind_backups/*.tar.gz | tail -3"

#!/bin/bash
# FreeIPA Backup
# Runbook: backup-all-infrastructure.adoc
# Status: PENDING - needs ipa_backups share on NAS

# Run on ipa-01 (as root)
ssh ipa-01 << 'EOF'
# Full backup (LDAP + files)
sudo ipa-backup

# Or data-only backup (faster)
# sudo ipa-backup --data

# Find latest backup
LATEST=$(ls -td /var/lib/ipa/backup/ipa-full-* | head -1)
echo "Latest backup: $LATEST"
EOF

# Copy to NAS
LATEST=$(ssh ipa-01 "ls -td /var/lib/ipa/backup/ipa-full-* | head -1")
scp -r ipa-01:"$LATEST" nas-01:/volume1/ipa_backups/

# Verify
ssh nas-01 "ls -lh /volume1/ipa_backups/"