DNS Outage — DHCP Misconfiguration Across VLAN Interfaces

# What nameservers did DHCP assign?
cat /etc/resolv.conf

# What does NetworkManager report for DNS?
nmcli dev show | grep -i dns

# Confirm BIND is alive
ping -c1 10.50.1.90

# Is BIND responding to queries?
dig @10.50.1.90 google.com +short

# Is pfSense DNS responding?
dig @10.50.1.1 google.com +short

# TEST: does the wrong DNS server (10.50.10.1) actually do DNS?
dig @10.50.10.1 google.com +short

# Bounce WiFi to get new lease attempt (connection was Domus-Secure-802.1X at the time)
sudo nmcli con down "Domus-Secure-802.1X" && sudo nmcli con up "Domus-Secure-802.1X"

# Override DNS on wired profile -- ignore what DHCP hands out
sudo nmcli con mod "Wired-802.1X-Vault" ipv4.dns "10.50.1.90 10.50.1.1"
sudo nmcli con mod "Wired-802.1X-Vault" ipv4.ignore-auto-dns yes
sudo nmcli con down "Wired-802.1X-Vault" && sudo nmcli con up "Wired-802.1X-Vault"

# Quick port connectivity check (separate test)
timeout 3 nc -zv 10.50.1.200 80 2>&1 || echo "http closed"

# Show DHCP config for DATA VLAN
netapi pfsense dhcp show opt1

# Check pfSense DHCP leases for the DATA subnet
netapi pfsense dhcp-leases | grep -i "10.50.10"

# Or SSH to pfSense directly and check dhcpd config
ssh pfsense-01 "cat /var/dhcpd/etc/dhcpd.conf" | grep -A10 "subnet 10.50.10"

# Full audit across all DHCP interfaces
for iface in lan opt1 opt2 opt3 opt4 opt5; do
  echo "=== $iface ==="
  netapi pfsense dhcp show $iface 2>/dev/null | awk '/DNS|Enabled/'
done

# Detailed check for IoT and MGMT ranges/gateways
netapi pfsense dhcp show opt4 | awk '/DNS|Gateway|Range/'
netapi pfsense dhcp show opt5 | awk '/DNS|Gateway|Range/'

netapi pfsense dhcp update --help   # will fail
netapi pfsense dhcp --help          # correct

netapi pfsense dhcp set-dns --help

# FAILS -- comma-separated
netapi pfsense dhcp set-dns opt1 "10.50.1.90,10.50.1.1"
# Error: [400] Field `dnsserver` must be a valid IPv4 address, received `10.50.1.90,10.50.1.1`

# FAILS -- comma with space
netapi pfsense dhcp set-dns opt1 "10.50.1.90, 10.50.1.1"
# Error: [400] Field `dnsserver` must be a valid IPv4 address, received `10.50.1.90, 10.50.1.1`

# CORRECT -- space-separated positional args, no quotes
netapi pfsense dhcp set-dns opt1 10.50.1.90 10.50.1.1

# Fix opt1 (DATA VLAN)
netapi pfsense dhcp set-dns opt1 10.50.1.90 10.50.1.1

# Verify
netapi pfsense dhcp show opt1

# Extend to opt2 and opt3
# NOTE: later identified as incorrect -- should use per-VLAN pfSense IPs
# See rollback in Follow-ups
netapi pfsense dhcp set-dns opt2 10.50.1.90 10.50.1.1
netapi pfsense dhcp set-dns opt3 10.50.1.90 10.50.1.1

# Verify opt1-3
for iface in opt1 opt2 opt3; do
  netapi pfsense dhcp show $iface | awk -v i="$iface" '/DNS/ {print i": "$0}'
done

# List active connections (non-infrastructure)
nmcli con show | awk '/wired|ethernet|802.1X/ {print $1, $3}'

# Or active only
nmcli con show --active | awk 'NR>1 {print $1, $3}'

# Bounce wired to renew DHCP lease
sudo nmcli con down "Wired-802.1X-Vault" && sudo nmcli con up "Wired-802.1X-Vault"

# Verify nameservers
awk '/nameserver/ {print $2}' /etc/resolv.conf

# Resolution test loop
for d in google.com github.com cloudflare.com inside.domusdigitalis.dev; do
  printf "%-30s %s\n" "$d" "$(dig +short $d | head -1)"
done

# Extended -- internal hosts
echo "google.com github.com vault-01.inside.domusdigitalis.dev ise-01.inside.domusdigitalis.dev" | \
  tr ' ' '\n' | while read d; do
    dig +short "$d" | awk -v dom="$d" 'NR==1 {printf "%-40s %s\n", dom, $0}'
  done

# List non-infrastructure connections
nmcli con show | awk 'NR>1 && !/bridge|docker|virbr|lo/ {print NR": "$1, "("$3")"}'

# Rename connection IDs (persistent -- NM writes id= field to .nmconnection)
# Syntax: nmcli con mod "OLD-NAME" connection.id "NEW-NAME"
nmcli con mod "Wired-802.1X-Vault"  connection.id "Domus-Wired-EAP-TLS"
nmcli con mod "Domus-Secure-802.1X" connection.id "Domus-WiFi-EAP-TLS"

nmcli con show | awk '/Domus/ {print $1, $3}'

# Confirm internal IDs in backing files
# NOTE: zsh glob fails on /etc/NetworkManager without sudo -- use sudo bash -c
sudo bash -c "awk -F= '/^id=/ {print FILENAME\": \"\$2}' \
  /etc/NetworkManager/system-connections/*.nmconnection"

# Alternative -- find avoids glob entirely
sudo find /etc/NetworkManager/system-connections -name "*.nmconnection" \
  -exec awk -F= '/^id=/ {print FILENAME": "$2}' {} \;

# Rename backing files to match (cosmetic, but clean)
sudo mv /etc/NetworkManager/system-connections/Wired-802.1X-Vault.nmconnection \
        /etc/NetworkManager/system-connections/Domus-Wired-EAP-TLS.nmconnection
sudo mv /etc/NetworkManager/system-connections/Domus-Secure-802.1X.nmconnection \
        /etc/NetworkManager/system-connections/Domus-WiFi-EAP-TLS.nmconnection
sudo nmcli con reload
sudo ls /etc/NetworkManager/system-connections/ | awk '/Domus/'

# Quick cert/key path dump
sudo bash -c 'grep -h "cert=\|key=" \
  /etc/NetworkManager/system-connections/Domus-*.nmconnection' \
  | awk -F= '{print $1": "$2}'

# Full structured extraction -- all 802.1X fields
sudo bash -c 'awk -F= "
  /^\[/ {section=\$0}
  /^id=|^type=|^interface-name=/ {print \$2}
  /eap=|identity=|ca-cert=|client-cert=|private-key=/ {print \"  \"\$1\": \"\$2}
" /etc/NetworkManager/system-connections/Domus-*.nmconnection'

# Formatted loop -- human-readable per-profile summary
sudo bash -c '
for f in /etc/NetworkManager/system-connections/Domus-*.nmconnection; do
  echo "=== $(basename $f .nmconnection) ==="
  awk -F= "
    /^id=/          {print \"  ID:          \"\$2}
    /^type=/        {print \"  Type:        \"\$2}
    /^eap=/         {print \"  EAP Method:  \"\$2}
    /^identity=/    {print \"  Identity:    \"\$2}
    /^ca-cert=/     {print \"  CA Cert:     \"\$2}
    /^client-cert=/ {print \"  Client Cert: \"\$2}
    /^private-key=/ {print \"  Private Key: \"\$2}
  " "$f"
  echo
done
'

# Side-by-side diff comparison (wired vs wifi)
sudo bash -c 'paste \
  <(cat /etc/NetworkManager/system-connections/Domus-Wired-EAP-TLS.nmconnection) \
  <(cat /etc/NetworkManager/system-connections/Domus-WiFi-EAP-TLS.nmconnection)' \
| awk -F'\t' '
  BEGIN {
    printf "\033[1;36m%-25s %-35s %-35s\033[0m\n", "FIELD", "WIRED", "WIFI"
  }
  /^id=|^type=|^eap=|^identity=|ca-cert=|client-cert=|private-key=/ {
    split($1,a,"="); split($2,b,"=")
    printf "\033[33m%-25s\033[0m \033[32m%-35s\033[0m \033[34m%-35s\033[0m\n", a[1], a[2], b[2]
  }'

# ANSI box display -- single awk pass over both files
sudo bash -c 'awk -F= "
  BEGIN {
    printf \"\033[1;37m┌─────────────────────────────────────────────────────────────┐\n│  🔐 DOMUS 802.1X PROFILES                                     │\n└─────────────────────────────────────────────────────────────┘\033[0m\n\"
  }
  FNR==1        {printf \"\n\033[1;36m◆ %s\033[0m\n\", FILENAME}
  /^id=/        {printf \"  \033[33m⎔\033[0m ID:       \033[1;32m%s\033[0m\n\",\$2}
  /^type=/      {printf \"  \033[33m⎔\033[0m Type:     \033[34m%s\033[0m\n\",\$2}
  /^eap=/       {printf \"  \033[33m⎔\033[0m EAP:      \033[35m%s\033[0m\n\",\$2}
  /^identity=/  {printf \"  \033[33m⎔\033[0m Identity: \033[32m%s\033[0m\n\",\$2}
  /client-cert=/{printf \"  \033[33m⎔\033[0m Cert:     \033[36m%s\033[0m\n\",\$2}
  /private-key=/{printf \"  \033[33m⎔\033[0m Key:      \033[31m%s\033[0m\n\",\$2}
" /etc/NetworkManager/system-connections/Domus-*.nmconnection'

# Certificate expiry -- date string
sudo bash -c 'for f in /etc/NetworkManager/system-connections/Domus-*.nmconnection; do
  name=$(basename $f .nmconnection)
  cert=$(awk -F= "/client-cert=/{print \$2}" $f)
  exp=$(openssl x509 -enddate -noout -in "$cert" 2>/dev/null | cut -d= -f2)
  printf "⚡ %s\n  Cert: %s\n  Expires: %s\n\n" "$name" "$cert" "$exp"
done'

# Certificate expiry -- days remaining (add to dotfiles as cert monitor alias)
sudo bash -c 'for f in /etc/NetworkManager/system-connections/Domus-*.nmconnection; do
  cert=$(awk -F= "/client-cert=/{print \$2}" "$f")
  days=$(( ($(openssl x509 -enddate -noout -in "$cert" | cut -d= -f2 \
    | xargs -I{} date -d "{}" +%s) - $(date +%s)) / 86400 ))
  awk -F= -v d="$days" "
    FNR==1         {printf \"\n\033[1;36m◆ %s\033[0m\n\", FILENAME}
    /^id=/         {printf \"  ⎔ ID:       \033[32m%s\033[0m\n\",\$2}
    /^type=/       {printf \"  ⎔ Type:     \033[34m%s\033[0m\n\",\$2}
    /client-cert=/ {printf \"  ⎔ Cert:     %s\n  ⎔ Expires:  \033[33m%d days\033[0m\n\",\$2,d}
  " "$f"
done'

# List all endpoint groups
netapi ise get-endpoint-groups

# Attempt create -- endpoint already existed from prior MAB auth
netapi ise create-endpoint 80:3F:5D:08:37:B8 \
  --group "BYOD-Registered" \
  --description "Work MacBook - MAB"

# Update existing endpoint
netapi ise update-endpoint 80:3F:5D:08:37:B8 \
  --group "BYOD-Registered" \
  --description "Work MacBook - MAB"

# Send CoA to apply new policy if device is currently connected
netapi ise coa reauth 80:3F:5D:08:37:B8

# Check pfSense firewall rules for IoT -- is DNS to 10.50.1.90 allowed?
netapi pfsense rules list opt4 | grep -i "dns\|53\|1.90"

# Windows client on IoT VLAN -- run on the affected host
ipconfig
ping 10.50.40.1           # gateway -- must respond
ping 10.50.1.1            # MGMT pfSense -- likely blocked
ping 8.8.8.8              # internet -- was failing (routing/firewall, not DNS)
nslookup google.com
ipconfig /release && ipconfig /renew
nslookup google.com

# Live supplicant log
journalctl -u wpa_supplicant --since "10 min ago" | tail -20

# Active connection check
nmcli con show --active | awk '/Wired|EAP/'

# List all physical MACs (exclude virtual interfaces)
ip link show | awk '/ether/ && !/docker|virbr|br-/ {print $2}'

# ISE auth status for a MAC
netapi ise mnt auth-status DC:8C:37:96:20:A6

# Diagnostic — confirmed BIND not forwarding external, pfSense works
dig @10.50.1.90 google.com +short   # returns nothing
dig @10.50.10.1 google.com +short   # returns 142.251.214.110

# Rollback — restore per-VLAN DNS
netapi pfsense dhcp set-dns opt1 10.50.10.1
netapi pfsense dhcp set-dns opt2 10.50.20.1
netapi pfsense dhcp set-dns opt3 10.50.30.1
netapi pfsense dhcp set-dns opt4 10.50.40.1

# Verify all VLANs
for iface in opt1 opt2 opt3 opt4; do
  netapi pfsense dhcp show $iface | awk -v i="$iface" '/DNS/ {print i": "$0}'
done

# Renew workstation lease
sudo nmcli con down "Domus-Wired-EAP-TLS" && sudo nmcli con up "Domus-Wired-EAP-TLS"

# What interface am I on? What DNS am I using?
nmcli dev status | awk '/connected/'
awk '/nameserver/ {print "DNS: "$2}' /etc/resolv.conf

# Which connection is active?
nmcli con show --active | awk 'NR>1 && !/bridge|docker|virbr/ {print $1, $3}'

# Basic resolution
dig google.com +short

# With timing
dig google.com | awk '/Query time|SERVER/'

# Specific server test (pfSense gateway)
dig @10.50.10.1 google.com +short

# Internal domain (should go to BIND via pfSense)
dig vault-01.inside.domusdigitalis.dev +short

# Full trace — see every hop
dig google.com +trace

# Verbose with all sections
dig google.com +noall +answer +authority +additional

# Show which server answered
dig google.com +noall +answer +comments | awk '/SERVER|;.*IN/'

# Get DNS from each profile
for prof in "Domus-Wired-EAP-TLS" "Domus-WiFi-EAP-TLS"; do
  echo "=== $prof ==="
  nmcli con show "$prof" | awk '/ipv4.dns:|IP4.DNS/'
done

# What DNS did DHCP actually assign? (active connection)
nmcli dev show | awk '/IP4.DNS/ {print}'

# 10 queries, 1 second apart — watch for failures
for i in {1..10}; do
  result=$(dig +short google.com 2>&1)
  ts=$(date +%H:%M:%S)
  if [ -z "$result" ]; then
    echo "$ts FAIL"
  else
    echo "$ts OK: $result"
  fi
  sleep 1
done

# Continuous monitor (Ctrl+C to stop)
while true; do
  ts=$(date +%H:%M:%S)
  iface=$(ip route get 8.8.8.8 2>/dev/null | awk '/dev/ {print $5}')
  dns=$(awk '/nameserver/ {print $2; exit}' /etc/resolv.conf)
  result=$(timeout 2 dig +short google.com 2>&1)
  if [ -z "$result" ]; then
    printf "%s [%s] DNS=%s \033[31mFAIL\033[0m\n" "$ts" "$iface" "$dns"
  else
    printf "%s [%s] DNS=%s \033[32mOK\033[0m %s\n" "$ts" "$iface" "$dns" "$result"
  fi
  sleep 2
done

# Is NetworkManager ignoring DHCP DNS?
nmcli con show "Domus-Wired-EAP-TLS" | awk '/ignore-auto-dns/'
nmcli con show "Domus-WiFi-EAP-TLS" | awk '/ignore-auto-dns/'

# Hardcoded DNS in profiles? (should be empty if using DHCP)
nmcli con show "Domus-Wired-EAP-TLS" | awk '/ipv4.dns:/'
nmcli con show "Domus-WiFi-EAP-TLS" | awk '/ipv4.dns:/'

# systemd-resolved status (if active)
resolvectl status 2>/dev/null | awk '/DNS Server|Current Scopes/'

# Can I reach the DNS server?
ping -c1 -W2 $(awk '/nameserver/ {print $2; exit}' /etc/resolv.conf)

# Is port 53 open to DNS?
dns=$(awk '/nameserver/ {print $2; exit}' /etc/resolv.conf)
timeout 2 nc -zvu $dns 53 2>&1

# Default route — which interface?
ip route get 8.8.8.8 | awk '{print "via", $3, "dev", $5}'

# Watch for interface changes (run in background)
ip monitor link | awk '/state UP|state DOWN/ {print strftime("%H:%M:%S"), $0}'

# Current connection priority (lower = preferred)
nmcli con show "Domus-Wired-EAP-TLS" | awk '/connection.autoconnect-priority/'
nmcli con show "Domus-WiFi-EAP-TLS" | awk '/connection.autoconnect-priority/'

# One-shot diagnostic
echo "=== Active Connections ===" && \
nmcli con show --active | awk 'NR>1 && !/bridge|docker|virbr/' && \
echo -e "\n=== DNS Servers ===" && \
awk '/nameserver/ {print $2}' /etc/resolv.conf && \
echo -e "\n=== Default Route ===" && \
ip route get 8.8.8.8 | awk '{print "via", $3, "dev", $5}' && \
echo -e "\n=== Resolution Test ===" && \
dig +short google.com && \
dig +short vault-01.inside.domusdigitalis.dev

# Check wired profile DNS settings
nmcli con show "Domus-Wired-EAP-TLS" | awk '/ipv4.dns:|ignore-auto-dns/'

# Check WiFi profile DNS settings
nmcli con show "Domus-WiFi-EAP-TLS" | awk '/ipv4.dns:|ignore-auto-dns/'

# Remove hardcoded DNS from wired profile
nmcli con mod "Domus-Wired-EAP-TLS" ipv4.dns ""
nmcli con mod "Domus-Wired-EAP-TLS" ipv4.ignore-auto-dns no

# Remove hardcoded DNS from WiFi profile
nmcli con mod "Domus-WiFi-EAP-TLS" ipv4.dns ""
nmcli con mod "Domus-WiFi-EAP-TLS" ipv4.ignore-auto-dns no

# Bounce connections to apply
sudo nmcli con down "Domus-Wired-EAP-TLS" && sudo nmcli con up "Domus-Wired-EAP-TLS"

# Both profiles should show empty DNS and ignore-auto-dns: no
nmcli con show "Domus-Wired-EAP-TLS" | awk '/ipv4.dns:|ignore-auto-dns/'
nmcli con show "Domus-WiFi-EAP-TLS" | awk '/ipv4.dns:|ignore-auto-dns/'

ip route | awk '/default/{print "Gateway:", $3}'

# Test connectivity to vyos-02
ping -c2 10.50.1.3

# Test kvm-02 (hypervisor)
ping -c1 10.50.1.111

dig @10.50.1.90 google.com +short +time=2

# Check if vyos-02 is conflicting with pfSense gateway
ip neigh show 10.50.10.1

while true; do
  ts=$(date +%H:%M:%S)
  result=$(timeout 2 dig @10.50.1.90 google.com +short 2>&1)
  if [ -n "$result" ] && [[ ! "$result" =~ "timed out" ]]; then
    printf "%s BIND \033[32mOK\033[0m %s\n" "$ts" "$result"
  else
    printf "%s BIND \033[31mFAIL\033[0m\n" "$ts"
  fi
  sleep 3
done

nmcli dev show enp130s0 | awk '/IP4.DNS/'

while true; do
  ts=$(date +%H:%M:%S)
  result=$(timeout 2 dig @10.50.10.1 google.com +short 2>&1)
  if [ -n "$result" ] && [[ ! "$result" =~ "timed out" ]]; then
    printf "%s pfSense \033[32mOK\033[0m %s\n" "$ts" "$result"
  else
    printf "%s pfSense \033[31mFAIL\033[0m\n" "$ts"
  fi
  sleep 3
done

! Te1/0/1 → kvm-02 (vyos-02)
interface TenGigabitEthernet1/0/1
 description TRUNK-TO-SUPERMICRO-KVM-02
 switchport trunk allowed vlan 10,20,30,40,100,999
 switchport trunk native vlan 100
 switchport mode trunk
 ip arp inspection trust
 spanning-tree portfast edge trunk
 ip dhcp snooping trust

! Te1/0/2 → kvm-01 (vyos-01)
interface TenGigabitEthernet1/0/2
 description TRUNK-TO-SUPERMICRO-KVM-01
 switchport trunk allowed vlan 10,20,30,40,100,999
 switchport trunk native vlan 100
 switchport mode trunk
 ip arp inspection trust
 spanning-tree portfast edge trunk
 ip dhcp snooping trust