Runbook: ISE 3.5 Deployment

# ISE admin password (for setup wizard)
gopass show -o v2/DOMUS/servers/ise-01/admin

# AD administrator (for domain join)
gopass show -o v2/DOMUS/ad/administrator

# ISE ERS API password
gopass show -o v2/DOMUS/servers/ise-01/ers

dsource d000 dev/storage
netapi ise backup --repo nas-01 --name "pre-ise34-migration" --wait

Repository: nas-01 (NFS)
Server: 10.50.1.70:/volume1/ise_backups
Name: pre-ise34-migration

✓ Backup initiated successfully
Waiting for backup to complete...
✓ Backup completed: pre-ise34-migration

# Only if you want JSON exports for documentation
netapi ise get-policy-sets --format json > /tmp/ise-02-policy-sets.json
netapi ise get-network-devices --format json > /tmp/ise-02-network-devices.json

# Remove VM definition only (keep qcow2)
sudo virsh undefine ise-01

# Or remove VM definition AND storage
sudo virsh undefine ise-01 --remove-all-storage

# Check current boot order
sudo virsh dumpxml ise-01 | grep -A5 "<os>"

sudo virsh destroy ise-01
sudo virsh dumpxml ise-01 > /tmp/ise-01.xml
sudo sed -i "s|<boot dev='hd'/>|<boot dev='cdrom'/>\n    <boot dev='hd'/>|" /tmp/ise-01.xml
sudo virsh define /tmp/ise-01.xml
sudo virsh start ise-01

sudo virsh change-media ise-02 sda /mnt/nas/isos/Cisco-ISE-3.5.0.527.SPA.x86_64.iso --insert --config

sudo virsh edit ise-01

<cputune>
  <vcpupin vcpu='0' cpuset='6'/>
  <vcpupin vcpu='1' cpuset='7'/>
  <vcpupin vcpu='2' cpuset='8'/>
  <vcpupin vcpu='3' cpuset='9'/>
</cputune>

sudo virsh vcpuinfo ise-01

VCPU:           0
CPU:            6
State:          running
CPU Affinity:   ------y---------

VCPU:           1
CPU:            7
State:          running
CPU Affinity:   -------y--------
...

# Check actual QEMU process memory
ps aux --sort=-%mem | grep qemu | awk '{print $6/1024/1024 " GB", $NF}'

# Get VNC display number
sudo virsh vncdisplay ise-01
# Output: :4  (means port 5904)

# Connect with vncviewer
vncviewer 192.168.1.181:5904

Enter hostname[]: ise-02
Enter default DNS domain[]: inside.domusdigitalis.dev
Enter IPv4/IPv6 address[]: 10.50.1.21
Enter IP netmask[]: 255.255.255.0
Enter IP default gateway[]: 10.50.1.1
Do you want to configure IPv6 address? Y/N [N]: N
Enter primary nameserver[]: 10.50.1.90
Add secondary nameserver? Y/N [N]: Y
Enter secondary nameserver[]: 10.50.1.91
Add tertiary nameserver? Y/N [N]: N
Enter NTP server[time.nist.gov]: 10.50.1.1
Add another NTP server? Y/N [N]: N
Enter system timezone[UTC]: US/Pacific
Enable SSH service? Y/N [N]: Y
Enter username[admin]: admin
Enter password: <gopass show -o v3/domains/d000/identity/ise/ise-02/admin>
Enter password again: <same>

# SSH to ISE CLI
ssh admin@10.50.1.20

# Check application status
show application status ise

ISE PROCESS NAME                       STATE            PROCESS ID
-----------------------------------------------------------------
Database Listener                      running          xxxx
Database Server                        running          xxxx
Application Server                     running          xxxx
...

dsource d000 dev/storage
echo $ISE_BACKUP_KEY

dsource d000 dev/storage
netapi synology list-files /ise_backups

scheduled-CFG10-260222-1948.tar.gpg      318.6 MB   (most recent)
pre-ise34-migration-CFG10-260209-2251.t  335.8 MB   (pre-3.4 migration)

gopass show -c v3/domains/d000/identity/ise/ise-02/cli

sudo ssh admin@ise-02.{domain}

configure terminal
repository nas-01
  url nfs://{nas-ip}:/volume1/ise_backups
exit

show repository nas-01

scheduled-CFG10-260222-1948.tar.gpg
pre-ise34-migration-CFG10-260209-2251.tar.gpg
...

restore <backup-filename> repository nas-01 encryption-key plain <YOUR_KEY>

restore scheduled-CFG10-260222-1948.tar.gpg repository nas-01 encryption-key plain MySecretKey123

show application status ise

ISE PROCESS NAME                       STATE            PROCESS ID
--------------------------------------------------------------------
Database Listener                      running          xxxx
Database Server                        running          xxxx
Application Server                     running          xxxx
Profiler Database                      running          xxxx
...

dsource d000 dev/network

netapi ise api-call ers GET '/ers/config/activedirectory' --target ise-02 | jq '.SearchResult.total'

1

netapi ise api-call ers GET '/ers/config/networkdevice?size=100' --target ise-02 | jq '.SearchResult.total'

5

netapi ise get-policy-sets --target ise-02 | head -20

netapi ise api-call ers GET '/ers/config/downloadableacl?size=100' --target ise-02 | jq '.SearchResult.resources[].name'

ise-patchbundle-3.5.0.527-Patch2-26022210.SPA.x86_64.tar.gz

rsync -avP ~/Downloads/ise-patchbundle-3.5.0.527-Patch2-26022210.SPA.x86_64.tar.gz nas-01:/volume1/ise_backups/

sending incremental file list
ise-patchbundle-3.5.0.527-Patch2-26022210.SPA.x86_64.tar.gz
    831,286,426 100%   36.42MB/s    0:00:21 (xfr#1, to-chk=0/1)

ssh nas-01 "ls -lh /volume1/ise_backups/*patch*"

ssh admin@10.50.1.21

show repository nas-01

configure terminal
repository nas-01
  url nfs://10.50.1.70:/volume1/ise_backups
exit

patch install ise-patchbundle-3.5.0.527-Patch2-26022210.SPA.x86_64.tar.gz repository nas-01

show application status ise

show version

dsource d000 dev/vault

vault write pki_int/issue/domus-server \
  common_name="ise-02.inside.domusdigitalis.dev" \
  ip_sans="10.50.1.21" \
  ttl="8760h" \
  -format=json > /dev/shm/ise-02-cert.json

jq -r '.data.certificate' /dev/shm/ise-02-cert.json > /dev/shm/ise-02.crt

jq -r '.data.private_key' /dev/shm/ise-02-cert.json > /dev/shm/ise-02.key

jq -r '.data.ca_chain[]' /dev/shm/ise-02-cert.json > /dev/shm/ise-02-chain.pem

openssl x509 -in /dev/shm/ise-02.crt -noout -subject -issuer -dates

subject=CN=ise-02.inside.domusdigitalis.dev
issuer=CN=DOMUS-ISSUING-CA
notBefore=Mar  5 21:57:05 2026 GMT
notAfter=Mar  5 21:57:35 2027 GMT

cd /dev/shm

awk '/-----BEGIN CERTIFICATE-----/{n++} n==1' ise-02-chain.pem > DOMUS-ISSUING-CA.pem

awk '/-----BEGIN CERTIFICATE-----/{n++} n==2' ise-02-chain.pem > DOMUS-ROOT-CA.pem

openssl x509 -in DOMUS-ISSUING-CA.pem -noout -subject
openssl x509 -in DOMUS-ROOT-CA.pem -noout -subject

subject=CN=DOMUS-ISSUING-CA
subject=C=US, O=Domus Digitalis, OU=Enterprise PKI, CN=DOMUS-ROOT-CA

ssh vault-01

export VAULT_ADDR='http://127.0.0.1:8200'

vault status

vault operator unseal
# Paste VAULT_UNSEAL_KEY_1, press Enter

vault operator unseal
# Paste VAULT_UNSEAL_KEY_2, press Enter

export VAULT_TOKEN='<paste-VAULT_ROOT_TOKEN-from-dsec>'

vault secrets list | grep pki

pki/        pki         ...
pki_int/    pki         ...

vault write pki_int/issue/domus-server \
  common_name="ise-01.inside.domusdigitalis.dev" \
  ip_sans="10.50.1.20" \
  ttl="8760h" \
  -format=json > /tmp/ise-01-cert.json

jq -r '.data.certificate' /tmp/ise-01-cert.json | openssl x509 -noout -subject -issuer

subject=CN = ise-01.inside.domusdigitalis.dev
issuer=CN = DOMUS-ISSUING-CA

jq -r '.data.certificate' /tmp/ise-01-cert.json > /tmp/ise-01.crt

jq -r '.data.private_key' /tmp/ise-01-cert.json > /tmp/ise-01.key

jq -r '.data.ca_chain[]' /tmp/ise-01-cert.json > /tmp/ise-01-chain.pem

ls -la /tmp/ise-01*

-rw-r--r--. 1 ansible ansible 5765 Feb 10 18:47 /tmp/ise-01-cert.json
-rw-r--r--. 1 ansible ansible 4248 Feb 10 18:47 /tmp/ise-01-chain.pem
-rw-r--r--. 1 ansible ansible 1850 Feb 10 18:47 /tmp/ise-01.crt
-rw-r--r--. 1 ansible ansible 1675 Feb 10 18:47 /tmp/ise-01.key

openssl x509 -in /tmp/ise-01.crt -noout -subject -issuer -dates

subject=CN=ise-01.inside.domusdigitalis.dev
issuer=CN=DOMUS-ISSUING-CA
notBefore=Feb 10 18:41:01 2026 GMT
notAfter=Feb 10 18:41:31 2027 GMT

grep -c "BEGIN CERTIFICATE" /tmp/ise-01-chain.pem

2

exit

scp vault-01:/tmp/ise-01.crt /dev/shm/

scp vault-01:/tmp/ise-01.key /dev/shm/

scp vault-01:/tmp/ise-01-chain.pem /dev/shm/

ssh vault-01

export VAULT_ADDR='http://127.0.0.1:8200'
vault operator seal

Success! Vault is sealed.

exit

Get-ADOrganizationalUnit -Filter * | Select-Object Name, DistinguishedName

New-ADOrganizationalUnit -Name "ISE" -Path "OU=Tier 1 - Servers,DC=inside,DC=domusdigitalis,DC=dev"

New-ADUser -Name "svc-ise-adjoin" -SamAccountName "svc-ise-adjoin" -UserPrincipalName "svc-ise-adjoin@inside.domusdigitalis.dev" -Path "OU=Service Accounts,OU=Tier 0 - Identity,DC=inside,DC=domusdigitalis,DC=dev" -AccountPassword (Read-Host -AsSecureString "Password") -Enabled $true -PasswordNeverExpires $true -Description "ISE AD join service account"

$ou = "OU=ISE,OU=Tier 1 - Servers,DC=inside,DC=domusdigitalis,DC=dev"
dsacls $ou /G "INSIDE\svc-ise-adjoin:CC;computer"
dsacls $ou /G "INSIDE\svc-ise-adjoin:WP;servicePrincipalName;computer"
dsacls $ou /G "INSIDE\svc-ise-adjoin:WP;userAccountControl;computer"

gopass generate v2/DOMUS/ad/svc-ise-adjoin 32
gopass edit v2/DOMUS/ad/svc-ise-adjoin

<generated-password>
username: svc-ise-adjoin
upn: svc-ise-adjoin@inside.domusdigitalis.dev
description: ISE AD join service account - delegated permissions to join computers to ISE OU
ou: OU=Service Accounts,OU=Tier 0 - Identity,DC=inside,DC=domusdigitalis,DC=dev
created: 2026-02-10

# From ISE CLI
ssh admin@ise-01.inside.domusdigitalis.dev
show run | include active-directory

netapi ise get-ad-join-points

# Using netapi (after updating credentials for ise-01)
netapi ise add-ad-groups "inside.domusdigitalis.dev" "GRP-Computers-Linux-Workstations"
netapi ise add-ad-groups "inside.domusdigitalis.dev" "GRP-Computers-Linux-Servers"
netapi ise add-ad-groups "inside.domusdigitalis.dev" "GRP-Users-802.1X-Wired"
netapi ise add-ad-groups "inside.domusdigitalis.dev" "GRP-Users-802.1X-Wireless"

Did Not Connect: Potential Security Issue
Firefox detected a potential security threat and did not continue to ise-02.inside.domusdigitalis.dev:8443

https://ise-01.inside.domusdigitalis.dev/admin/LoginAction.do?local=true

ssh admin@ise-01.inside.domusdigitalis.dev
show logging application ise-psc.log tail | include SAML

netapi ise create-network-device \
  --name "pfSense-FW01" \
  --ip "10.50.1.1" \
  --radius-secret "$(gopass show -o d000/radius/pfsense)" \
  --device-type "Firewall" \
  --location "Home Enterprise"

netapi ise create-network-device \
  --name "9800-CL-WLC" \
  --ip "10.50.1.40" \
  --radius-secret "$(gopass show -o d000/radius/wlc)" \
  --device-type "Wireless LAN Controller#Cisco#C9800-CL" \
  --location "Home Enterprise"

netapi ise create-network-device \
  --name "SWITCH_9300" \
  --ip "10.50.1.11" \
  --radius-secret "$(gopass show -o d000/radius/switch)" \
  --device-type "Switch#Cisco#Catalyst 9300" \
  --location "Home Enterprise"

# /etc/wpa_supplicant/wpa_supplicant-wired.conf
network={
    key_mgmt=IEEE8021X
    eap=TLS
    identity="modestus-razer.inside.domusdigitalis.dev"
    ca_cert="/etc/ssl/certs/DOMUS-CA-CHAIN.pem"
    client_cert="/etc/ssl/certs/modestus-razer-eaptls.pem"
    private_key="/etc/ssl/private/modestus-razer-eaptls.key"
    eapol_flags=0
}

# Check DOMUS CA chain is present
openssl verify -CAfile /etc/ssl/certs/DOMUS-CA-CHAIN.pem \
  /etc/ssl/certs/$(hostname)-eaptls.pem

# On a Linux workstation
sudo systemctl restart wpa_supplicant@eth0

# Check authentication
journalctl -u wpa_supplicant@eth0 -f

# Update environment for ise-01
export ISE_PAN_FQDN=ise-01.inside.domusdigitalis.dev

# Check live sessions
netapi ise mnt sessions

# Check authentication log
netapi ise dc query "SELECT USERNAME, POLICY_SET_NAME, PASSED, FAILURE_REASON
  FROM RADIUS_AUTHENTICATIONS
  WHERE TIMESTAMP_TIMEZONE > SYSDATE - INTERVAL '1' HOUR
  ORDER BY TIMESTAMP_TIMEZONE DESC
  FETCH FIRST 20 ROWS ONLY"

# Check active sessions
netapi ise mnt sessions

# Verify EAP-TLS authentications
netapi ise dc query "SELECT USERNAME, AUTHENTICATION_PROTOCOL, POLICY_SET_NAME, PASSED
  FROM RADIUS_AUTHENTICATIONS
  WHERE AUTHENTICATION_PROTOCOL = 'EAP-TLS'
    AND TIMESTAMP_TIMEZONE > SYSDATE - INTERVAL '1' HOUR
  ORDER BY TIMESTAMP_TIMEZONE DESC
  FETCH FIRST 10 ROWS ONLY"

gopass edit d000/ise/admin
gopass edit d000/ise/api-token

# Update ISE_PAN_FQDN in dsec env file
vim ~/.secrets/env/d000/dev/network.env

ISE_PAN_FQDN=ise-01.inside.domusdigitalis.dev

# Ensure ise-02 is stopped
sudo virsh destroy ise-02

# Remove from libvirt
sudo virsh undefine ise-02 --remove-all-storage

# Or keep storage for rollback
sudo virsh undefine ise-02

# Remove the broken VM
sudo virsh destroy ise-02
sudo virsh undefine ise-02 --remove-all-storage

# Download ISE ISO (not OVA) to kvm-02
rsync -avP ~/Downloads/Cisco-ISE-3.5.0.527.SPA.x86_64.iso kvm-02:/mnt/nas/isos/

# Create VM with ISO installation (see Phase 2)

# Verify pinning
sudo virsh vcpuinfo ise-01

# Apply live if missed
sudo virsh vcpupin ise-01 0 6
sudo virsh vcpupin ise-01 1 7
sudo virsh vcpupin ise-01 2 8
sudo virsh vcpupin ise-01 3 9

# Verify chain has 2 certificates
grep -c "BEGIN CERTIFICATE" /etc/ssl/certs/DOMUS-CA-CHAIN.pem
# Should return: 2

# If only 1, concatenate:
cat DOMUS-ROOT-CA.pem DOMUS-ISSUING-CA.pem > DOMUS-CA-CHAIN.pem

# Quick check for recent failures
netapi ise dc query "SELECT USERNAME, FAILURE_REASON
  FROM RADIUS_AUTHENTICATIONS
  WHERE PASSED = 0
    AND TIMESTAMP_TIMEZONE > SYSDATE - INTERVAL '1' HOUR
  ORDER BY TIMESTAMP_TIMEZONE DESC
  FETCH FIRST 10 ROWS ONLY"

dsec edit d000 dev/network

# Change active target from ISE_HQ_01 to ISE_HQ_02
ISE_DEPLOYMENT_TYPE={{ISE_HQ_02_DEPLOYMENT}}
ISE_PAN_FQDN={{ISE_HQ_02_HOST}}
ISE_PAN_IP={{ISE_HQ_02_IP}}
ISE_PORT={{ISE_HQ_02_PORT}}

# DataConnect must use FQDN (not IP) for SSL hostname verification
ISE_DATACONNECT_HOST={{ISE_PAN_FQDN}}

dsource d000 dev/network

netapi ise get-cert-profiles

ISE_preloaded_Cisco_Certificate_Profile_For_RA
ISE_BYOD_Certificate_Profile

netapi ise api-call openapi GET '/api/v1/deployment/node' | jq -r '.response[] | "\(.hostname) (\(.fqdn))"'

ise-01 (ise-01.inside.domusdigitalis.dev)

netapi ise mnt count

Total Active Sessions: 3

netapi ise dc test

✓ Connected successfully!
Host: ise-01.inside.domusdigitalis.dev
Port: 2484
Service: cpm10
User: dataconnect

netapi ise pxgrid test

✓ pxGrid connection successful
Host: ise-01.inside.domusdigitalis.dev:8910
Client: netapi-pxgrid

sudo nmcli connection up "Wired-802.1X-Vault"

netapi ise dc query "SELECT USERNAME, ISE_NODE, POLICY_SET_NAME, AUTHORIZATION_RULE, PASSED
FROM RADIUS_AUTHENTICATIONS
WHERE AUTHENTICATION_PROTOCOL = 'EAP-TLS'
  AND TIMESTAMP_TIMEZONE > SYSDATE - INTERVAL '1' HOUR
ORDER BY TIMESTAMP_TIMEZONE DESC
FETCH FIRST 5 ROWS ONLY"

USERNAME                                 ISE_NODE  POLICY_SET_NAME      AUTHORIZATION_RULE      PASSED
modestus-razer.inside.domusdigitalis.dev ise-01    Domus-Wired 802.1X   Linux_EAP-TLS_Permit    Pass

netapi ise dc query "SELECT USERNAME, ISE_NODE, NAS_IP_ADDRESS, FRAMED_IP_ADDRESS, PASSED
FROM RADIUS_AUTHENTICATIONS
WHERE PASSED = 1 AND TIMESTAMP_TIMEZONE > SYSDATE - 1
ORDER BY TIMESTAMP_TIMEZONE DESC
FETCH FIRST 20 ROWS ONLY"

netapi ise dc query "SELECT USERNAME, ISE_NODE, FAILURE_REASON, TIMESTAMP_TIMEZONE
FROM RADIUS_AUTHENTICATIONS
WHERE PASSED = 0 AND TIMESTAMP_TIMEZONE > SYSDATE - 1
ORDER BY TIMESTAMP_TIMEZONE DESC
FETCH FIRST 20 ROWS ONLY"

netapi ise dc query "SELECT POLICY_SET_NAME, COUNT(*) AS AUTH_COUNT
FROM RADIUS_AUTHENTICATIONS
WHERE AUTHENTICATION_PROTOCOL = 'EAP-TLS'
  AND TIMESTAMP_TIMEZONE > SYSDATE - 1
GROUP BY POLICY_SET_NAME
ORDER BY AUTH_COUNT DESC"

blk_update_request: I/O error, dev vda, sector 93521729 op 0x0:(READ)
EXT4-fs (vda3): error loading journal
mount: /sysroot: can't read superblock on /dev/vda3

# Check disk integrity (container may be OK even if filesystem corrupt)
sudo qemu-img check /mnt/nas/vms/ise-02.qcow2

# Delete corrupted VM
sudo virsh destroy ise-02
sudo virsh undefine ise-02 --remove-all-storage

# Redeploy from ISO (see Phase 2)

sudo virsh domblklist ise-02

 Target   Source
-------------------------------------
 vda      /mnt/nas/vms/ise-02.qcow2
 sda      -

# Attach ISO
sudo virsh change-media ise-02 sda /mnt/nas/isos/Cisco-ISE-3.5.0.527.SPA.x86_64.iso --insert --config

# Must destroy/start for --config to take effect
sudo virsh destroy ise-02
sudo virsh start ise-02

sudo virsh domblklist ise-02

 Target   Source
------------------------------------------------------------
 vda      /mnt/nas/vms/ise-02.qcow2
 sda      /mnt/nas/isos/Cisco-ISE-3.5.0.527.SPA.x86_64.iso

sudo virt-xml ise-02 --edit --boot cdrom,hd
sudo virsh destroy ise-02
sudo virsh start ise-02

***** Auto install configured
***** The ZTP configuration image is missing or improper. Automatic installation flow exited.
***** Power off and attach the proper ZTP configuration image or choose manual boot to proceed.
[   44.710453] reboot: System halted

# WRONG - fails on Synology
scp file.tar.gz nas-01:/volume1/ise_backups/

# CORRECT - works on Synology
rsync -avP file.tar.gz nas-01:/volume1/ise_backups/