kvm-01 Rocky Linux Rebuild

# Set hostname to match kvm-02 naming convention
hostnamectl set-hostname kvm-01.inside.domusdigitalis.dev

# Verify
hostnamectl --static

# If user doesn't exist (check first with: id evanusmodestus)
useradd -m -G wheel evanusmodestus
passwd evanusmodestus

# Add to libvirt group (required for VM management)
usermod -aG libvirt evanusmodestus

# Verify groups match kvm-02
groups evanusmodestus

# Create .ssh directory
mkdir -p /home/evanusmodestus/.ssh
chmod 700 /home/evanusmodestus/.ssh

# Add authorized keys (Vault-signed + YubiKeys + ed25519)
cat > /home/evanusmodestus/.ssh/authorized_keys << 'EOF'
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIrgE9z8gkQVRVkkdbc1ejdth7vJkqpY35FrIUv8L6JB vault-signed-20260219
sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIG/EGu00HuV3jnisul7DUBuk9jLtrE3yR4BZCwGb2YpCAAAABHNzaDo= d000-nano-35641207
sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIFHfsGSAFAkqwYj6EGS9sA2MROjs28zM6LJds3gagsCkAAAACHNzaDpkMDAw evanusmodestus@d000-yubikey
sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIEBZ+kus4aTHzQt1zNnEnGxJs+Lf56vrCdcyvqLhpp9hAAAACHNzaDpkMDAw ssh:d000
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL3vaIABqHOwy88p/5GcX3ZNU044GAz/3T5dH8GIU7DS evanusmodestus@d000
EOF

# Set permissions
chmod 600 /home/evanusmodestus/.ssh/authorized_keys
chown -R evanusmodestus:evanusmodestus /home/evanusmodestus/.ssh

# Download Vault SSH CA public key
curl -sk https://vault-01.inside.domusdigitalis.dev:8200/v1/ssh/public_key > /etc/ssh/vault-ca.pub

# Verify CA key downloaded
cat /etc/ssh/vault-ca.pub

# Add to sshd_config
echo "TrustedUserCAKeys /etc/ssh/vault-ca.pub" >> /etc/ssh/sshd_config

# Restart sshd
systemctl restart sshd

# Exit root shell
exit

# Reconnect as regular user (Vault cert or YubiKey)
ssh evanusmodestus@10.50.1.110

# Create mount points
sudo mkdir -p /mnt/nas/{vms,isos,backups}

# Mount NFS shares
sudo mount -t nfs nas-01:/volume1/vms /mnt/nas/vms

sudo mount -t nfs nas-01:/volume1/isos /mnt/nas/isos

sudo mount -t nfs nas-01:/volume1/backups /mnt/nas/backups

# Verify mounts
mount | grep nfs | awk '{print $1, $3}'

nas-01:/volume1/vms /mnt/nas/vms
nas-01:/volume1/isos /mnt/nas/isos
nas-01:/volume1/backups /mnt/nas/backups

# Add persistent mounts (matching kvm-02 fstab)
cat << 'EOF' | sudo tee -a /etc/fstab
nas-01:/volume1/vms      /mnt/nas/vms     nfs  defaults,_netdev  0 0
nas-01:/volume1/isos     /mnt/nas/isos    nfs  defaults,_netdev  0 0
nas-01:/volume1/backups  /mnt/nas/backups nfs  defaults,_netdev  0 0
EOF

# Verify fstab
grep nfs /etc/fstab

# Reload systemd to pick up fstab changes
sudo systemctl daemon-reload

# Check backup directory exists
ls -la /mnt/nas/backups/kvm-01-rebuild/

sudo tee /etc/libvirt/hooks/qemu << 'EOF'
#!/bin/bash
# /etc/libvirt/hooks/qemu
# Configures 802.1Q VLANs and PVID on br-mgmt vnet interfaces at VM start
# NOTE: Never use virsh here — deadlocks libvirtd

GUEST_NAME="$1"
OPERATION="$2"

BRIDGE="br-mgmt"
VLANS="10 20 30 40 100 110 120"
VNET_WAIT_SECS=15   # max seconds to wait for vnets to attach

PVID100_VMS="vyos-01 vyos-02
9800-WLC-01 9800-WLC-02
ise-01 ise-02
bind-01 bind-02
home-dc01 home-dc02
keycloak-01 keycloak-02
ipsk-manager ipsk-manager-01 ipsk-manager-02
vault-01 vault-02 vault-03
ipa-01 ipa-02
k3s-master-01 k3s-master-02 k3s-master-03
k3s-worker-01 k3s-worker-02 k3s-worker-03"

log()  { logger -t "libvirt-hook[${GUEST_NAME}]" "$*"; }
warn() { logger -t "libvirt-hook[${GUEST_NAME}]" "WARN: $*"; }
err()  { logger -t "libvirt-hook[${GUEST_NAME}]" "ERROR: $*"; }

needs_pvid100() {
    echo "$PVID100_VMS" | tr ' \n' '\n' | grep -qx "$1"
}

get_vm_vnets() {
    local guest="$1"
    local xml="/etc/libvirt/qemu/${guest}.xml"

    if [[ ! -f "$xml" ]]; then
        err "VM XML not found: $xml"
        return 1
    fi

    local macs
    macs=$(grep -oP "(?<=<mac address=[\"'])[0-9a-f:]+" "$xml")

    if [[ -z "$macs" ]]; then
        warn "No MAC addresses found in $xml"
        return 1
    fi

    local found=0
    for mac in $macs; do
        local suffix="${mac:3}"
        for vnet in $(ip link show master "$BRIDGE" 2>/dev/null \
                      | awk -F'[ :]+' '/vnet/{print $2}'); do
            local vnet_mac
            vnet_mac=$(cat /sys/class/net/"$vnet"/address 2>/dev/null)
            if [[ "${vnet_mac:3}" == "$suffix" ]]; then
                echo "$vnet"
                (( found++ ))
            fi
        done
    done

    [[ $found -eq 0 ]] && return 1
    return 0
}

configure_vnet() {
    local vnet="$1"
    local pvid="$2"
    local errors=0

    log "Configuring $vnet — bridge: $BRIDGE — VLANs: $VLANS — PVID: $pvid"

    # Verify interface still exists
    if ! ip link show "$vnet" &>/dev/null; then
        err "$vnet: interface disappeared before configuration"
        return 1
    fi

    # Remove default PVID 1
    bridge vlan del vid 1 dev "$vnet" pvid untagged 2>/dev/null

    # Add all tagged VLANs
    for vid in $VLANS; do
        if bridge vlan add vid "$vid" dev "$vnet" 2>/dev/null; then
            log "$vnet: added VLAN $vid"
        else
            warn "$vnet: failed to add VLAN $vid"
            (( errors++ ))
        fi
    done

    # Set PVID — must come after tagged VLANs are added
    if bridge vlan add vid "$pvid" dev "$vnet" pvid untagged 2>/dev/null; then
        log "$vnet: PVID set to $pvid"
    else
        err "$vnet: failed to set PVID $pvid"
        (( errors++ ))
    fi

    # Verify final state (single line for syslog)
    local vlan_state
    vlan_state=$(bridge vlan show dev "$vnet" 2>/dev/null | tr '\n' ' ')
    log "$vnet: final state — $vlan_state"

    return $errors
}

case "$OPERATION" in
    started)
        (
            log "started — waiting for vnets on $BRIDGE"

            # Poll until vnets appear — replaces fragile sleep 3
            vnets=()
            for i in $(seq 1 "$VNET_WAIT_SECS"); do
                mapfile -t vnets < <(get_vm_vnets "$GUEST_NAME")
                [[ ${#vnets[@]} -gt 0 ]] && break
                log "Waiting for vnets... attempt $i/${VNET_WAIT_SECS}"
                sleep 1
            done

            if [[ ${#vnets[@]} -eq 0 ]]; then
                err "No vnets found for $GUEST_NAME on $BRIDGE after ${VNET_WAIT_SECS}s — giving up"
                exit 1
            fi

            log "Found ${#vnets[@]} vnet(s): ${vnets[*]}"

            if needs_pvid100 "$GUEST_NAME"; then
                pvid=100
            else
                pvid=1
                warn "$GUEST_NAME not in PVID100 list — using PVID 1, verify this is correct"
            fi

            total_errors=0
            for vnet in "${vnets[@]}"; do
                configure_vnet "$vnet" "$pvid"
                (( total_errors += $? ))
            done

            if [[ $total_errors -eq 0 ]]; then
                log "Complete — ${#vnets[@]} interface(s) configured successfully"
            else
                err "Complete with $total_errors error(s) — check VLAN state manually"
                exit 1
            fi
        ) &
        ;;
    begin)
        log "begin — no action"
        ;;
    stopped|reconnect)
        log "$OPERATION — no action (bridge cleans up vnets automatically)"
        ;;
    *)
        log "unhandled operation: $OPERATION"
        ;;
esac

exit 0
EOF

sudo chmod 755 /etc/libvirt/hooks/qemu
sudo systemctl restart libvirtd

# Watch hook logs during VM start
journalctl -t "libvirt-hook[k3s-master-01]" -f

# Test by restarting a VM
sudo virsh shutdown k3s-master-01 && sleep 5 && sudo virsh start k3s-master-01

# Verify correct VLAN assignment
VNET=$(grep -oP "(?<=<mac address=[\"'])[0-9a-f:]+" /etc/libvirt/qemu/k3s-master-01.xml | head -1)
echo "VM MAC: $VNET"
bridge vlan show | grep -A10 vnet

vnet0             10
                  20
                  30
                  40
                  100 PVID Egress Untagged
                  110
                  120

# Get vnet interface for the VM
VNET=$(sudo virsh domiflist <vm-name> | awk '/br-mgmt/{print $1}')
echo "vnet: $VNET"

# Add PVID 100 and all VLANs, remove VLAN 1
sudo bridge vlan add vid 100 dev $VNET pvid untagged
sudo bridge vlan add vid 10 dev $VNET
sudo bridge vlan add vid 20 dev $VNET
sudo bridge vlan add vid 30 dev $VNET
sudo bridge vlan add vid 40 dev $VNET
sudo bridge vlan add vid 110 dev $VNET
sudo bridge vlan add vid 120 dev $VNET
sudo bridge vlan del vid 1 dev $VNET

bridge vlan show dev $VNET