Vault SSH Certificate Authority

ssh-keygen -Lf ~/.ssh/id_ed25519_vault-cert.pub | awk '/Type:|Valid:|Principals:/'

        Type: ssh-ed25519-cert-v01@openssh.com user certificate
        Valid: from 2026-02-22T12:30:41 to 2026-02-22T20:31:11
        Principals:
                admin
                adminerosado
                ansible
                evanusmodestus
                root

vault-ssh-sign

# Load Vault credentials
dsource d000 dev/vault

# Sign with ALL principals (never omit any)
vault write -field=signed_key ssh/sign/domus-client \
  public_key=@$HOME/.ssh/id_ed25519_vault.pub \
  valid_principals="admin,adminerosado,ansible,evanusmodestus,gabriel,root" \
  >| ~/.ssh/id_ed25519_vault-cert.pub

# Reload SSH agent
ssh-add -d ~/.ssh/id_ed25519_vault
ssh-add ~/.ssh/id_ed25519_vault

# 1. Copy CA public key to host
scp /tmp/vault-ca.pub <new-host>:/tmp/

# 2. On target host
ssh <new-host>
sudo mv /tmp/vault-ca.pub /etc/ssh/vault-ca.pub
sudo chmod 644 /etc/ssh/vault-ca.pub

# 3. Add to sshd_config (BEFORE any Match blocks)
echo "TrustedUserCAKeys /etc/ssh/vault-ca.pub" | sudo tee -a /etc/ssh/sshd_config

# 4. Verify and restart
sudo sshd -t && sudo systemctl restart sshd

vault-ssh-sign

Enter passphrase for /home/evanusmodestus/.ssh/id_ed25519_vault:
Identity added: /home/evanusmodestus/.ssh/id_ed25519_vault (vault-signed-20260219)
Certificate added: /home/evanusmodestus/.ssh/id_ed25519_vault-cert.pub (vault-root-4baadeb7...)
Lifetime set to 08:04:53
Certificate signed successfully:
        Valid: from 2026-02-23T06:47:54 to 2026-02-23T14:48:24
        Extensions:

ssh k3s-master-01

cp docs/asciidoc/modules/ROOT/examples/vault-ssh-sign ~/.local/bin/
chmod +x ~/.local/bin/vault-ssh-sign

vault-ssh-sign                     # Sign default key
vault-ssh-sign ~/.ssh/other_key    # Sign specific key

cp docs/asciidoc/modules/ROOT/examples/vault-ssh-test.sh ~/.local/bin/vault-ssh-test
chmod +x ~/.local/bin/vault-ssh-test

vault-ssh-test           # Test all hosts
vault-ssh-test --quick   # Skip slow hosts (Windows DC)

dsource d000 dev/vault

 echo $VAULT_TOKEN
 echo $VAULT_UNSEAL_KEY_1
 echo $VAULT_UNSEAL_KEY_2

ssh vault-01

export VAULT_ADDR='http://127.0.0.1:8200'

vault status

vault status | awk 'NR <= 5'

Key             Value
---             -----
Seal Type       shamir
Initialized     true
Sealed          false

vault operator unseal

vault operator unseal

vault status | awk '/Sealed/ {print $1, $2}'

export VAULT_TOKEN='<paste-VAULT_TOKEN-from-dsec>'

vault secrets enable -path=ssh ssh

Success! Enabled the ssh secrets engine at: ssh/

vault secrets list | awk '/ssh/ {print $1, $2}'

ssh/ ssh

vault write ssh/config/ca generate_signing_key=true

Key             Value
---             -----
public_key      ssh-rsa AAAAB3NzaC1yc2EAAAA...

vault read -field=public_key ssh/config/ca > /tmp/vault-ssh-ca.pub

awk '{print NR": "$0}' /tmp/vault-ssh-ca.pub

awk '{print "Type:", $1, "Length:", length($2)}' /tmp/vault-ssh-ca.pub

Type: ssh-rsa Length: 716

vault write ssh/roles/domus-client - <<EOF
{
  "key_type": "ca",
  "allow_user_certificates": true,
  "default_user": "evanusmodestus",
  "allowed_users": "evanusmodestus,adminerosado,ansible,root",
  "allowed_extensions": "permit-pty,permit-port-forwarding",
  "default_extensions": {
    "permit-pty": ""
  },
  "ttl": "8h",
  "max_ttl": "24h"
}
EOF

vault write ssh/roles/domus-host - <<EOF
{
  "key_type": "ca",
  "allow_host_certificates": true,
  "allowed_domains": "inside.domusdigitalis.dev",
  "allow_subdomains": true,
  "ttl": "87600h",
  "max_ttl": "175200h"
}
EOF

vault list ssh/roles

Keys
domus-client
domus-host

vault read ssh/roles/domus-client | awk 'NR <= 15'

scp vault-01:/tmp/vault-ssh-ca.pub /tmp/

scp /tmp/vault-ssh-ca.pub <target-host>:/tmp/

scp /tmp/vault-ssh-ca.pub modestus-aw:/tmp/

for host in modestus-aw modestus-p50 nas-01; do
  scp /tmp/vault-ssh-ca.pub ${host}:/tmp/
done

ssh <target-host>

ssh modestus-aw

sudo mv /tmp/vault-ssh-ca.pub /etc/ssh/vault-ca.pub

sudo chmod 644 /etc/ssh/vault-ca.pub

ls -la /etc/ssh/vault-ca.pub && awk '{print "Installed:", $1}' /etc/ssh/vault-ca.pub

-rw-r--r--. 1 root root 742 Feb 19 21:00 /etc/ssh/vault-ca.pub
Installed: ssh-rsa

sudo tee /etc/ssh/sshd_config.d/vault-ca.conf << 'EOF'
# Vault SSH CA - Trust certificates signed by this CA
TrustedUserCAKeys /etc/ssh/vault-ca.pub

# AuthorizedPrincipalsFile: Optional - restrict which principals can login
# AuthorizedPrincipalsFile /etc/ssh/auth_principals/%u
EOF

sudo sshd -t

sudo systemctl restart sshd

systemctl is-active sshd && echo "sshd running"

[ -f ~/.ssh/id_ed25519_vault ] && echo "Key exists - skip generation" || echo "No key - generate one"

[ -f ~/.ssh/id_ed25519_vault ] || ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519_vault -C "vault-signed-$(date +%Y%m%d)"

ssh-keygen -Lf ~/.ssh/id_ed25519_vault-cert.pub

ssh-keygen -Lf ~/.ssh/id_ed25519_vault-cert.pub | awk '/Type:|Valid:|Principals:/'

        Type: ssh-ed25519-cert-v01@openssh.com user certificate
        Valid: from 2026-02-19T21:00:00 to 2026-02-20T05:00:00
        Principals:

ssh <target-host> 'hostname'

ssh -o ControlPath=none -o IdentitiesOnly=yes -i ~/.ssh/id_ed25519_vault <target-host> 'hostnamectl' | awk '/Static hostname|Operating System|Kernel|Architecture/ {gsub(/^[[:space:]]+/, ""); print}'

ssh -o ControlPath=none -o IdentitiesOnly=yes -i ~/.ssh/id_ed25519_vault kvm-01 'hostnamectl' | awk '/Static hostname|Operating System|Kernel|Architecture/ {gsub(/^[[:space:]]+/, ""); print}'

Static hostname: supermicro300-9d1
Operating System: Arch Linux
Kernel: Linux 6.17.5-arch1-1
Architecture: x86-64

sudo journalctl -u sshd -n 20 | awk '/Accepted.*CERT/'

Accepted publickey for evanusmodestus from 10.x.x.x port xxxxx ssh2: ED25519-CERT ID vault-userkey-evanusmodestus

vault policy write ssh-client - << 'EOF'
# Allow signing SSH keys with the client role
path "ssh/sign/domus-client" {
  capabilities = ["create", "update"]
}

# Allow reading CA public key (for host trust setup)
path "ssh/config/ca" {
  capabilities = ["read"]
}
EOF

vault policy read ssh-client

vault auth enable approle

Success! Enabled approle auth method at: approle/

vault write auth/approle/role/ssh-user \
  token_policies="ssh-client" \
  token_ttl="1h" \
  token_max_ttl="4h"

Success! Data written to: auth/approle/role/ssh-user

mkdir -p ~/.local/bin

cat > ~/.local/bin/vault-ssh-sign << 'SCRIPT'
#!/bin/bash
set -euo pipefail

KEY="${1:-$HOME/.ssh/id_ed25519_vault}"
CERT="${KEY}-cert.pub"

# ALL principals needed across infrastructure
# - adminerosado: Synology NAS
# - vyos: VyOS routers
# - ansible: automation
# - evanusmodestus: Linux workstations/servers
# - gabriel: modestus-p50 workstation
# - root: emergency access
PRINCIPALS="adminerosado,vyos,ansible,evanusmodestus,gabriel,root"

# Load Vault credentials from dsec
eval "$(dsource d000 dev/vault 2>/dev/null)" || {
    echo "ERROR: Failed to load Vault credentials from dsec" >&2
    exit 1
}

# Verify required environment variables
[[ -z "${VAULT_ADDR:-}" ]] && { echo "ERROR: VAULT_ADDR not set" >&2; exit 1; }
[[ -z "${VAULT_TOKEN:-}" ]] && { echo "ERROR: VAULT_TOKEN not set" >&2; exit 1; }

# Verify public key exists
[[ ! -f "${KEY}.pub" ]] && { echo "ERROR: Public key not found: ${KEY}.pub" >&2; exit 1; }

# Sign the key with ALL principals
vault write -field=signed_key ssh/sign/domus-client \
  public_key=@"${KEY}.pub" \
  valid_principals="$PRINCIPALS" > "$CERT"

# Reload agent with new cert
ssh-add -d "$KEY" 2>/dev/null || true
ssh-add "$KEY"

# Display validity and principals
echo "Certificate signed successfully:"
ssh-keygen -Lf "$CERT" | awk '/Valid:|Principals:|Extensions:/' | head -20
SCRIPT

chmod +x ~/.local/bin/vault-ssh-sign

vault read ssh/roles/domus-client | grep allowed_users

vault write ssh/roles/domus-client - <<'EOF'
{
  "key_type": "ca",
  "allow_user_certificates": true,
  "default_user": "evanusmodestus",
  "allowed_users": "adminerosado,admin,ansible,evanusmodestus,gabriel,root",
  "allowed_extensions": "permit-pty,permit-port-forwarding",
  "default_extensions": {
    "permit-pty": "",
    "permit-port-forwarding": ""
  },
  "ttl": "8h",
  "max_ttl": "24h"
}
EOF

vault-ssh-sign
ssh <new-host>

# Check current principals in script
grep PRINCIPALS ~/.local/bin/vault-ssh-sign

# Add missing principal (example: adding 'admin')
sed -i 's/PRINCIPALS="adminerosado,ansible,evanusmodestus,root"/PRINCIPALS="adminerosado,vyos,ansible,evanusmodestus,gabriel,root"/' ~/.local/bin/vault-ssh-sign

# Verify
grep PRINCIPALS ~/.local/bin/vault-ssh-sign

sed -i 's/valid_principals="ansible,evanusmodestus,root"/valid_principals="adminerosado,admin,ansible,evanusmodestus,root"/' ~/.local/bin/vault-ssh-sign

vault-ssh-sign                              # Default key
vault-ssh-sign ~/.ssh/id_ed25519_custom     # Custom key

Enter passphrase for /home/evanusmodestus/.ssh/id_ed25519_vault:
Identity added: /home/evanusmodestus/.ssh/id_ed25519_vault (vault-signed-20260219)
Certificate added: /home/evanusmodestus/.ssh/id_ed25519_vault-cert.pub (...)
Certificate signed successfully:
        Valid: from 2026-02-21T00:53:42 to 2026-02-21T08:54:12
        Extensions:

crontab -e

0 */6 * * * ~/.local/bin/vault-ssh-sign 2>/dev/null

SSH_AUTH_SOCK=/run/user/1000/ssh-agent.socket ssh-add ~/.ssh/id_ed25519_vault

Enter passphrase for /home/evanusmodestus/.ssh/id_ed25519_vault:
Identity added: /home/evanusmodestus/.ssh/id_ed25519_vault (vault-signed-20260219)
Certificate added: /home/evanusmodestus/.ssh/id_ed25519_vault-cert.pub (...)
Lifetime set to 08:03:18

vault write -field=signed_key ssh/sign/domus-host \
  cert_type=host \
  public_key=@/etc/ssh/ssh_host_ed25519_key.pub \
  valid_principals="$(hostname -f)" \
  > /etc/ssh/ssh_host_ed25519_key-cert.pub

echo "HostCertificate /etc/ssh/ssh_host_ed25519_key-cert.pub" | sudo tee -a /etc/ssh/sshd_config
sudo systemctl restart sshd

echo "@cert-authority *.inside.domusdigitalis.dev $(< /tmp/vault-ssh-ca.pub)" >> ~/.ssh/known_hosts

vault secrets disable ssh

sudo rm -f /etc/ssh/sshd_config.d/vault-ca.conf
sudo rm -f /etc/ssh/vault-ca.pub
sudo systemctl restart sshd

sudo sed -i 's/^PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config
sudo systemctl restart sshd

ssh-keygen -Lf ~/.ssh/id_ed25519_vault-cert.pub | grep -A3 Extensions

        Extensions: (none)

        Extensions:
                permit-port-forwarding
                permit-pty

vault write ssh/roles/domus-client - <<'EOF'
{
  "key_type": "ca",
  "allow_user_certificates": true,
  "default_user": "evanusmodestus",
  "allowed_users": "evanusmodestus,adminerosado,admin,administrator,domus\\administrator,ansible,root",
  "allowed_extensions": "permit-pty,permit-port-forwarding",
  "default_extensions": {
    "permit-pty": "",
    "permit-port-forwarding": ""
  },
  "ttl": "8h",
  "max_ttl": "24h"
}
EOF

vault write -field=signed_key ssh/sign/domus-client \
  public_key=@$HOME/.ssh/id_ed25519_vault.pub \
  valid_principals="admin,adminerosado,ansible,evanusmodestus,gabriel,root" \
  >| ~/.ssh/id_ed25519_vault-cert.pub

ssh -O exit <target-host> 2>/dev/null
ssh-add -d ~/.ssh/id_ed25519_vault 2>/dev/null
ssh-add ~/.ssh/id_ed25519_vault

ssh <target-host>

ssh-keygen -Lf ~/.ssh/id_ed25519_vault-cert.pub | awk '/Valid:/'

vault-ssh-sign

Enter passphrase for key '/home/evanusmodestus/.ssh/id_ed25519_sk_rk_d000_nano':

# Check what principals your cert has
ssh-keygen -Lf ~/.ssh/id_ed25519_vault-cert.pub | grep -A6 "Principals:"

# Check what user the SSH config uses for this host
grep -A5 "Host <hostname>" ~/.ssh/config | grep -i user

# Example: add 'vyos' for VyOS routers
sed -i 's/valid_principals="\([^"]*\)"/valid_principals="\1,vyos"/' ~/.local/bin/vault-ssh-sign

# Re-sign with updated principals
vault-ssh-sign

# Verify principal is now included
ssh-keygen -Lf ~/.ssh/id_ed25519_vault-cert.pub | grep -A6 "Principals:"

Received disconnect from 10.50.1.70 port 22:2: Too many authentication failures

ssh-keygen -Lf ~/.ssh/id_ed25519_vault-cert.pub | grep -A1 "Principals:"

        Principals:
                (none)

vault-ssh-sign

dsource d000 dev/vault

vault write -field=signed_key ssh/sign/domus-client \
  public_key=@$HOME/.ssh/id_ed25519_vault.pub \
  valid_principals="adminerosado,admin,ansible,evanusmodestus,root" \
  >| ~/.ssh/id_ed25519_vault-cert.pub

ssh-add -d ~/.ssh/id_ed25519_vault
ssh-add ~/.ssh/id_ed25519_vault

ssh-keygen -Lf ~/.ssh/id_ed25519_vault-cert.pub | awk '/Principals:/{getline; print}'

sudo journalctl -u sshd --since "5 minutes ago" | awk '/error|refused|denied/i'

ssh <target-host> "ls -la /etc/ssh/vault-ca.pub && awk '{print \$1}' /etc/ssh/vault-ca.pub"

ssh <target-host> "sudo sshd -T | awk '/trustedusercakeys/'"

ssh -vvv <target-host> 2>&1 | awk '/Offering|Server accepts|certificate/'

ssh -o ControlPath=none -vvv <target-host> 2>&1 | awk '/Offering|Server accepts|cert/'

ssh -o ControlPath=none <target-host> 'echo "Certificate auth verified: $(date)"'

ssh -vvv <target-host> 2>&1 | awk '/Remote: cert:/'

debug1: Remote: cert: key options: pty

ps aux | awk '/ssh.*<hostname>/ {print $2, $11, $12}'

3236333 | |
3245292 | |
3206218 | |

pkill -f "ssh.*nas-01"
pkill -f "ssh.*kvm-01"

~.