WRKLOG-2026-03-11

Node        Address                                    State       Voter
vault-01    vault-01.inside.domusdigitalis.dev:8201    leader      true
vault-02    vault-02.inside.domusdigitalis.dev:8201    follower    true
vault-03    vault-03.inside.domusdigitalis.dev:8201    follower    true

# One MAC per line, uppercase with colons
cat > /tmp/vocera-macs.txt << 'EOF'
00:09:EF:AA:BB:CC
00:09:EF:DD:EE:FF
18:B4:30:11:22:33
EOF

# Get endpoint details for each MAC
while read -r mac; do
    echo "=== $mac ==="
    netapi ise endpoint get "$mac" -f json | jq '{mac: .mac, profile: .profileId, group: .groupId, staticGroupAssignment: .staticGroupAssignment}'
done < /tmp/vocera-macs.txt

# Build SQL IN clause from file
MACS=$(awk '{printf "\x27%s\x27,", $1}' /tmp/vocera-macs.txt | sed 's/,$//')

netapi ise dc query "SELECT
    acs_timestamp,
    calling_station_id AS mac,
    passed,
    failure_reason,
    selected_azn_profiles
FROM mnt.radius_auth_48_live
WHERE calling_station_id IN ($MACS)
ORDER BY acs_timestamp DESC
LIMIT 100" -f json | jq '.'

while read -r mac; do
    echo "=== $mac failures ==="
    netapi ise dc query "SELECT acs_timestamp, failure_reason
        FROM mnt.radius_auth_48_live
        WHERE calling_station_id = '$mac' AND passed = 0
        ORDER BY acs_timestamp DESC LIMIT 5" -f json | jq -r '.[] | "\(.acs_timestamp): \(.failure_reason)"'
done < /tmp/vocera-macs.txt

# Reauth all devices after supplicant is configured
while read -r mac; do
    echo "Reauth: $mac"
    netapi ise mnt coa reauth --mac "$mac"
    sleep 1  # Rate limit
done < /tmp/vocera-macs.txt

export VAULT_ADDR="https://vault-01.inside.domusdigitalis.dev:8200"
vault operator raft list-peers

# Stop the leader
ssh vault-01 "sudo systemctl stop vault"

# Wait for election
sleep 10

# Check new leader (from vault-02)
export VAULT_ADDR="https://vault-02.inside.domusdigitalis.dev:8200"
vault operator raft list-peers

vault list pki_int/certs | head -3

ssh vault-01 "sudo systemctl start vault"

# Unseal (3 keys required)
ssh -t vault-01 "VAULT_ADDR=https://127.0.0.1:8200 VAULT_SKIP_VERIFY=1 vault operator unseal"
ssh -t vault-01 "VAULT_ADDR=https://127.0.0.1:8200 VAULT_SKIP_VERIFY=1 vault operator unseal"
ssh -t vault-01 "VAULT_ADDR=https://127.0.0.1:8200 VAULT_SKIP_VERIFY=1 vault operator unseal"

# Verify rejoin
vault operator raft list-peers

# /var/log/vault MUST exist on all nodes before failover works
ssh vault-02 "sudo mkdir -p /var/log/vault && sudo chown vault:vault /var/log/vault"
ssh vault-03 "sudo mkdir -p /var/log/vault && sudo chown vault:vault /var/log/vault"

rsync: [sender] Failed to exec ssh: Permission denied (13)
rsync error: error in IPC code (code 14)

# Capture ALL denials at once
sudo semanage permissive -a rsync_t
sudo systemctl start vault-backup.service

# Generate comprehensive policy
sudo ausearch -m avc --start today | grep rsync | audit2allow -M vault-backup

# Install and re-enable enforcing
sudo semodule -i vault-backup.pp
sudo semanage permissive -d rsync_t

# Test
sudo systemctl start vault-backup.service && systemctl status vault-backup.service

# Health check endpoint
# 200 = active leader, 429 = standby (both can serve)
curl -sk --max-time 3 -o /dev/null -w "%{http_code}" \
    "https://vault-01.inside.domusdigitalis.dev:8200/v1/sys/health"

VAULT_NODES=(
    "https://vault-01.inside.domusdigitalis.dev:8200"
    "https://vault-02.inside.domusdigitalis.dev:8200"
    "https://vault-03.inside.domusdigitalis.dev:8200"
)

# Run the updated script
vault-ssh-sign

# Verify SSH cert works
vault-ssh-test

# List all repo names
netapi github repos -f json | jq -r '.[].full_name'

# Get open MR titles from GitLab
netapi gitlab mrs mygroup/myproject -f json | jq -r '.[].title'

# Search Gitea and get clone URLs
netapi gitea search netapi -f json | jq -r '.[].ssh_url'

# GitHub PR files with stats
netapi github pr-files owner/repo 42 -f json | jq '.[] | "\(.filename): +\(.additions) -\(.deletions)"'

ISE Error: 5411 - "Supplicant stopped responding to ISE"
Failure Reason: 12935 - "Supplicant stopped responding to ISE during EAP-TLS certificate exchange"
Step 12935 latency: 120001 ms (2 minute timeout)

EAP-TLS: Certificate chain validated ✓ (ROOT → ISSUING → ise-02)
Selected EAP-TLS
CTRL-EVENT-DISCONNECTED reason=3 locally_generated=1

Secrets were required, but not provided

# Create chain file
cat /etc/ssl/certs/modestus-razer-eaptls.pem \
    /etc/ssl/certs/DOMUS-ISSUING-CA.pem \
    > /tmp/modestus-razer-chain.pem

# Verify chain
openssl crl2pkcs7 -nocrl -certfile /tmp/modestus-razer-chain.pem | \
    openssl pkcs7 -print_certs -noout | grep -c "subject="
# Should be 2 (leaf + intermediate)

# Update NM connection
sudo cp /tmp/modestus-razer-chain.pem /etc/ssl/certs/modestus-razer-eaptls-chain.pem
sudo nmcli con modify "Domus-WiFi-EAP-TLS" \
    802-1x.client-cert /etc/ssl/certs/modestus-razer-eaptls-chain.pem

# Reconnect
nmcli con down "Domus-WiFi-EAP-TLS" && nmcli con up "Domus-WiFi-EAP-TLS"

# From kvm-01/kvm-02
sudo virsh console vault-01

# Login as ansible (has NOPASSWD sudo)
# Reset user password
sudo passwd evanusmodestus

# Exit console: Ctrl+]