kvm-02 Deployment: Supermicro E300-9D-8CN8TP with Rocky Linux

# Load network device credentials (includes IPMI)
dsource d000 dev/network

# Verify variables are set
echo "IPMI_USER: $IPMI_USER"
echo "IPMI_PASS is set: $([ -n "$IPMI_PASS" ] && echo yes || echo no)"

# Generate and store password (16 chars = safe for IPMI)
# Structure: ipmi-02/ADMIN allows per-user credentials
gopass generate v3/domains/d000/servers/ipmi/ipmi-02/ADMIN 16

# Add metadata
gopass edit v3/domains/d000/servers/ipmi/ipmi-02/ADMIN

<generated-password>
---
user_id: 2
ip: 10.50.1.201
hostname: ipmi-02.inside.domusdigitalis.dev
hardware: Supermicro E300-9D-8CN8TP
server: kvm-02
description: kvm-02 BMC/IPMI ADMIN account
created: 2026-03-01

# List IPMI users (find user ID for ADMIN)
sudo ipmitool user list 1

# Disable extra users (ID 3 and above)
# Note: Cannot set empty name, but disabled users cannot authenticate
sudo ipmitool user disable 3

# Verify only ADMIN remains active
sudo ipmitool user list 1

# On WORKSTATION: Copy password to clipboard
gopass show -c v3/domains/d000/servers/ipmi/ipmi-02/ADMIN
# Copied to clipboard, clears in 45 seconds

# On SERVER (SSH session): Set password interactively
# Paste from clipboard when prompted
sudo ipmitool user set password 2
# Enter Password: <paste>
# Confirm Password: <paste>

# Verify new password works (paste password when prompted)
sudo ipmitool user test 2 20

# On WORKSTATION: Verify remote IPMI access with new credentials
ipmitool -I lanplus -H 10.50.1.201 -U ADMIN \
  -P "$(gopass show -o v3/domains/d000/servers/ipmi/ipmi-02/ADMIN)" power status
# Expected: Chassis Power is on

# Check if ipmitool is installed
rpm -q ipmitool

# Install if not present
sudo dnf install -y ipmitool

# Verify installation
rpm -q ipmitool && ipmitool -V

# Check if IPMI modules are loaded
lsmod | grep ipmi

# Load IPMI modules if missing
sudo modprobe ipmi_devintf
sudo modprobe ipmi_si

# Verify modules loaded
lsmod | grep ipmi

# Local IPMI commands (no network needed)
sudo ipmitool lan print 1

# Set static IP for IPMI (no -I lanplus = local)
sudo ipmitool lan set 1 ipsrc static
sudo ipmitool lan set 1 ipaddr 10.50.1.201
sudo ipmitool lan set 1 netmask 255.255.255.0
sudo ipmitool lan set 1 defgw ipaddr 10.50.1.1

# Verify settings before applying
sudo ipmitool lan print 1 | grep -E "IP Address|Subnet Mask|Default Gateway"

# Apply changes (BMC reboots)
sudo ipmitool mc reset cold

# Check current LAN mode
sudo ipmitool raw 0x30 0x70 0x0c 0

# Set LAN mode to Dedicated (0x00)
sudo ipmitool raw 0x30 0x70 0x0c 1 0

# Reset BMC to apply LAN mode change
sudo ipmitool mc reset cold

# From workstation, verify IPMI is reachable
ping 10.50.1.201

# Test IPMI commands remotely
dsource d000 dev/network
ipmitool -I lanplus -H 10.50.1.201 -U $IPMI_USER -P $IPMI_PASS power status

# Power operations
ipmitool -I lanplus -H 10.50.1.201 -U $IPMI_USER -P $IPMI_PASS power on
ipmitool -I lanplus -H 10.50.1.201 -U $IPMI_USER -P $IPMI_PASS power off
ipmitool -I lanplus -H 10.50.1.201 -U $IPMI_USER -P $IPMI_PASS power cycle

# SOL (Serial Over LAN) console
ipmitool -I lanplus -H 10.50.1.201 -U $IPMI_USER -P $IPMI_PASS sol activate
# Exit SOL: ~.

# Boot to BIOS
ipmitool -I lanplus -H 10.50.1.201 -U $IPMI_USER -P $IPMI_PASS chassis bootdev bios

# Sensor check
ipmitool -I lanplus -H 10.50.1.201 -U $IPMI_USER -P $IPMI_PASS sdr type temperature

# Issue certificate from Vault
vault write -format=json pki_int/issue/domus-server \
    common_name="ipmi-02.inside.domusdigitalis.dev" \
    ttl="8760h" > /tmp/ipmi-cert.json

# Extract cert and key
jq -r '.data.certificate' /tmp/ipmi-cert.json > /tmp/ipmi-02.crt
jq -r '.data.private_key' /tmp/ipmi-cert.json > /tmp/ipmi-02.key
jq -r '.data.ca_chain[]' /tmp/ipmi-cert.json >> /tmp/ipmi-02.crt

# Upload via IPMI web interface:
# Configuration → SSL Certification → Upload
# Or via ipmitool (Supermicro specific):
ipmitool -I lanplus -H 10.50.1.201 -U $IPMI_USER -P $IPMI_PASS \
    raw 0x30 0x9d 0x01 0x00 0x00 < /tmp/ipmi-02.crt

# On workstation or nas-01
# Rocky 9.7 (current as of 2025-11) - 2.5GB minimal
wget https://download.rockylinux.org/pub/rocky/9/isos/x86_64/Rocky-9.7-x86_64-minimal.iso

# Alternative: "latest" symlink (always points to current minor release)
# wget https://download.rockylinux.org/pub/rocky/9/isos/x86_64/Rocky-9-latest-x86_64-minimal.iso

# Verify checksum
wget https://download.rockylinux.org/pub/rocky/9/isos/x86_64/Rocky-9.7-x86_64-minimal.iso.CHECKSUM
sha256sum -c Rocky-9.7-x86_64-minimal.iso.CHECKSUM 2>&1 | grep -E 'OK|FAILED'

# Transfer to USB or mount via IPMI virtual media

# Check current repos
dnf repolist

# Enable EPEL (Extra Packages for Enterprise Linux)
sudo dnf install -y epel-release

# Enable CRB (CodeReady Builder - replacement for PowerTools)
sudo dnf config-manager --set-enabled crb

# Verify repos enabled
dnf repolist | grep -E "epel|crb"

# Update system
sudo dnf update -y

# Verify kernel version
uname -r

# System utilities
sudo dnf install -y \
    vim-enhanced git tmux htop tree wget curl \
    bash-completion NetworkManager-tui \
    openssh-server rsync tar unzip \
    firewalld chrony ipmitool \
    policycoreutils-python-utils \
    nmap-ncat

# Verify critical packages installed
rpm -q vim-enhanced tmux htop ipmitool chrony policycoreutils-python-utils

# Check SELinux status
getenforce

# Check SELinux config file
grep "^SELINUX=" /etc/selinux/config

# If permissive or disabled, set to enforcing
sudo sed -i 's/^SELINUX=.*/SELINUX=enforcing/' /etc/selinux/config

# Verify change
grep "^SELINUX=" /etc/selinux/config

# Check for SELinux denials (should be empty on fresh install)
sudo ausearch -m avc -ts recent 2>/dev/null | head -20

# Enable serial console on GRUB
sudo grubby --update-kernel=ALL --args="console=tty0 console=ttyS0,115200n8"

# Enable getty on serial console
sudo systemctl enable serial-getty@ttyS0.service
sudo systemctl start serial-getty@ttyS0.service

# Verify
cat /proc/cmdline | grep console

# Enable firewall
sudo systemctl enable --now firewalld

# Allow SSH and libvirt
sudo firewall-cmd --permanent --add-service=ssh
sudo firewall-cmd --permanent --add-service=cockpit
sudo firewall-cmd --permanent --add-service=libvirt
sudo firewall-cmd --permanent --add-service=nfs

# Allow VNC for VM consoles
sudo firewall-cmd --permanent --add-port=5900-5910/tcp

# Allow NFS (if not covered by service)
sudo firewall-cmd --permanent --add-port=2049/tcp
sudo firewall-cmd --permanent --add-port=111/tcp
sudo firewall-cmd --permanent --add-port=111/udp

# Allow libvirt migration (if doing live migration later)
sudo firewall-cmd --permanent --add-port=49152-49215/tcp

# Allow Prometheus node exporter (for monitoring)
sudo firewall-cmd --permanent --add-port=9100/tcp

# Reload
sudo firewall-cmd --reload

# Verify
sudo firewall-cmd --list-all

# Check current NTP sources
chronyc sources

# Backup current config
sudo cp /etc/chrony.conf /etc/chrony.conf.bak

# Configure chrony for internal NTP (VyOS gateway)
cat << 'EOF' | sudo tee /etc/chrony.conf
# Internal NTP source (VyOS)
server 10.50.1.1 iburst

# Drift file
driftfile /var/lib/chrony/drift

# Allow large time jumps on first 3 updates
makestep 1.0 3

# Sync RTC
rtcsync
EOF

# Restart chronyd to apply
sudo systemctl restart chronyd

# Verify VyOS is the NTP source
chronyc sources

# Check sync status (should show "Leap status: Normal")
chronyc tracking

# Import Wazuh GPG key
sudo rpm --import https://packages.wazuh.com/key/GPG-KEY-WAZUH

# Add Wazuh repository
cat << 'EOF' | sudo tee /etc/yum.repos.d/wazuh.repo
[wazuh]
gpgcheck=1
gpgkey=https://packages.wazuh.com/key/GPG-KEY-WAZUH
enabled=1
name=EL-$releasever - Wazuh
baseurl=https://packages.wazuh.com/4.x/yum/
protect=1
EOF

# Install Wazuh agent
sudo dnf install -y wazuh-agent

# Configure agent
sudo sed -i 's/MANAGER_IP/10.50.1.134/' /var/ossec/etc/ossec.conf

# Start and enable
sudo systemctl daemon-reload
sudo systemctl enable --now wazuh-agent

# Verify agent processes running
sudo /var/ossec/bin/wazuh-control status

# Check agent logs for manager connection
sudo tail -20 /var/ossec/logs/ossec.log | grep -i manager

# Install KVM, QEMU, libvirt
sudo dnf install -y \
    qemu-kvm libvirt libvirt-daemon-kvm \
    virt-install virt-manager virt-viewer \
    bridge-utils virt-top libguestfs-tools

# Install Cockpit for web management
sudo dnf install -y cockpit cockpit-machines cockpit-storaged

# Start and enable libvirt
sudo systemctl enable --now libvirtd

# Enable Cockpit
sudo systemctl enable --now cockpit.socket

# Add user to libvirt group
sudo usermod -aG libvirt evanusmodestus

# Check virtualization extensions
egrep -c '(vmx|svm)' /proc/cpuinfo
# Should return > 0

# Check KVM modules
lsmod | grep kvm

# Verify libvirt
sudo virsh list --all

# Enable nested virtualization (Intel)
cat << 'EOF' | sudo tee /etc/modprobe.d/kvm.conf
options kvm_intel nested=1
EOF

# Reload KVM module (requires reboot for full effect)
sudo modprobe -r kvm_intel
sudo modprobe kvm_intel

# Verify
cat /sys/module/kvm_intel/parameters/nested
# Should show Y or 1

# Create local storage pool for performance-critical VMs
sudo virsh pool-define-as local-vms dir - - - - /var/lib/libvirt/images
sudo virsh pool-build local-vms
sudo virsh pool-start local-vms
sudo virsh pool-autostart local-vms

# Install NFS utilities
sudo dnf install -y nfs-utils

# Create mount points (bash brace expansion)
sudo mkdir -p /mnt/nas/vms /mnt/nas/isos /mnt/nas/backups /mnt/nas/k3s

# Add fstab entries
cat << 'EOF' | sudo tee -a /etc/fstab
# Synology NAS mounts
10.50.1.70:/volume1/VMs /mnt/nas/vms nfs defaults,_netdev,nofail 0 0
10.50.1.70:/volume1/ISOs /mnt/nas/isos nfs defaults,_netdev,nofail 0 0
10.50.1.70:/volume1/Backups /mnt/nas/backups nfs defaults,_netdev,nofail 0 0
10.50.1.70:/volume1/k3s /mnt/nas/k3s nfs defaults,_netdev,nofail 0 0
EOF

# Reload systemd and mount
sudo systemctl daemon-reload
sudo mount -a

# Verify mounts
df -h | grep nas

# ISOs pool
sudo virsh pool-define-as nas-isos dir - - - - /mnt/nas/isos
sudo virsh pool-build nas-isos
sudo virsh pool-start nas-isos
sudo virsh pool-autostart nas-isos

# VMs pool (for non-critical VMs)
sudo virsh pool-define-as nas-vms dir - - - - /mnt/nas/vms
sudo virsh pool-build nas-vms
sudo virsh pool-start nas-vms
sudo virsh pool-autostart nas-vms

# List all interfaces
ip -o link show | awk -F': ' '{print $2}'

# Expected on Supermicro E300-9D-8CN8TP:
# eno1-eno4: 1GbE (Intel I210-AT)
# eno5np0-eno8np3: 10GbE SFP+ (Intel X550)

cat << 'EOF' > /tmp/br-mgmt-network.xml
<network>
  <name>br-mgmt</name>
  <forward mode="bridge"/>
  <bridge name="br-mgmt"/>
</network>
EOF

sudo virsh net-define /tmp/br-mgmt-network.xml
sudo virsh net-start br-mgmt
sudo virsh net-autostart br-mgmt

# Allocate 16GB huge pages (8192 x 2MB)
echo "vm.nr_hugepages = 8192" | sudo tee -a /etc/sysctl.conf
sudo sysctl -p

# Verify
grep HugePages /proc/meminfo

# Install cpupower
sudo dnf install -y kernel-tools

# Set performance governor
sudo cpupower frequency-set -g performance

# Make persistent (CPUPOWER_START_OPTS is the correct variable)
echo 'CPUPOWER_START_OPTS="frequency-set -g performance"' | sudo tee /etc/sysconfig/cpupower
sudo systemctl enable --now cpupower

# Verify service started
systemctl status cpupower.service --no-pager | grep -E "Active:|loaded"

# Check current swappiness (default 60)
cat /proc/sys/vm/swappiness

# Set swappiness to 1 (only swap when critical)
# Keeps swap as safety valve without impacting VM performance
# Huge pages (16GB) are non-swappable regardless
echo "vm.swappiness = 1" | sudo tee -a /etc/sysctl.conf
sudo sysctl -p | grep swappiness
# Expected: vm.swappiness = 1

dig +short kvm-02.inside.domusdigitalis.dev @10.50.1.90

cat << 'HEALTHEOF' | sudo tee /usr/local/bin/kvm-health-check > /dev/null
#!/bin/bash
# kvm-02 Health Check Script
# Run after deployment or anytime to verify system state

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
NC='\033[0m'

echo "================================================"
echo "       kvm-02 Health Check"
echo "================================================"
echo ""

PASSED=0
FAILED=0

check() {
    local name="$1"
    local cmd="$2"
    if eval "$cmd" &>/dev/null; then
        printf "  ${GREEN}✓${NC} %-35s ${GREEN}[OK]${NC}\n" "$name"
        ((PASSED++))
    else
        printf "  ${RED}✗${NC} %-35s ${RED}[FAIL]${NC}\n" "$name"
        ((FAILED++))
    fi
}

echo "== Services =="
check "libvirtd running" "systemctl is-active libvirtd"
check "cockpit.socket running" "systemctl is-active cockpit.socket"
check "chronyd running" "systemctl is-active chronyd"
check "firewalld running" "systemctl is-active firewalld"
check "sshd running" "systemctl is-active sshd"
check "wazuh-agent running" "systemctl is-active wazuh-agent"

echo ""
echo "== Virtualization =="
check "KVM modules loaded" "lsmod | grep -q kvm_intel"
check "Nested virtualization" "cat /sys/module/kvm_intel/parameters/nested | grep -q Y"
check "VT-d (IOMMU) enabled" "dmesg | grep -q 'IOMMU enabled'"
check "libvirt connection" "sudo virsh list --all"

echo ""
echo "== Storage =="
check "local-vms pool active" "sudo virsh pool-info local-vms | grep -q 'State:.*running'"
check "Huge pages allocated" "grep -q 'HugePages_Total:.*[1-9]' /proc/meminfo"
# NAS pools (optional - uncomment when NAS NFS permissions configured)
# check "nas-isos pool active" "sudo virsh pool-info nas-isos | grep -q 'State:.*running'"
# check "nas-vms pool active" "sudo virsh pool-info nas-vms | grep -q 'State:.*running'"
# check "NFS mounts present" "df -h | grep -q nas"

echo ""
echo "== Performance =="
check "CPU governor performance" "cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor | grep -q performance"
check "Swappiness <= 10" "test \$(cat /proc/sys/vm/swappiness) -le 10"
check "cpupower service active" "systemctl is-active cpupower"

echo ""
echo "== Network =="
check "br-mgmt bridge exists" "ip link show br-mgmt"
check "br-mgmt network active" "sudo virsh net-info br-mgmt | grep -q 'Active:.*yes'"
check "Gateway reachable" "ping -c1 -W2 10.50.1.1"
check "DNS resolution" "host kvm-01.inside.domusdigitalis.dev"
check "Vault reachable" "curl -sk https://vault-01.inside.domusdigitalis.dev:8200/v1/sys/health | grep -q initialized"

echo ""
echo "== Security =="
check "SELinux enforcing" "getenforce | grep -q Enforcing"
check "Vault SSH CA trusted" "test -f /etc/ssh/vault-ca.pub"
check "Password auth disabled" "sshd -T | grep -q 'passwordauthentication no'"

echo ""
echo "================================================"
printf "Results: ${GREEN}%d passed${NC}, ${RED}%d failed${NC}\n" "$PASSED" "$FAILED"
echo "================================================"

[[ $FAILED -gt 0 ]] && exit 1
exit 0
HEALTHEOF

sudo chmod +x /usr/local/bin/kvm-health-check

# Show first 5 and last 5 lines with line numbers
awk 'NR<=5 {print NR": "$0; next} {buf[NR]=$0} END {print "..."; for(i=NR-4;i<=NR;i++) print i": "buf[i]}' /usr/local/bin/kvm-health-check

# Services
systemctl is-active libvirtd cockpit.socket chronyd firewalld sshd wazuh-agent

# Storage pools
sudo virsh pool-list --all

# Networks
sudo virsh net-list --all

# Bridge
ip -4 addr show br-mgmt | awk '/inet/{print $2}'

# NFS mounts
df -h | awk '/nas/{print $1, $6}'

# Huge pages
awk '/HugePages_Total/{print "Huge Pages:", $2}' /proc/meminfo

# Reset BMC from local console
ipmitool mc reset cold

# Check IPMI over USB (if supported)
ipmitool -I usb mc info

# Test NFS connectivity
showmount -e 10.50.1.70

# Check firewall on NAS side
# DSM → Control Panel → Security → Firewall

# Check libvirt errors
sudo journalctl -u libvirtd -f

# Verify storage pool
sudo virsh pool-refresh local-vms

# Check disk permissions
ls -la /var/lib/libvirt/images/

# List all attributes
awk '/^    [a-z].*:/' docs/asciidoc/antora.yml

# Search for specific attribute
grep -i "ipmi" docs/asciidoc/antora.yml

# Search multiple attributes
grep -E "ipmi-02-ip|netmask-24|vyos-vip" docs/asciidoc/antora.yml