k3s Kubernetes Deployment

# ============================================================
# K3S DEPLOYMENT SESSION VARIABLES
# Adjust K3S_NODE_* for each node deployment
# ============================================================

# Target Node (change for each deployment)
K3S_NODE_NAME="k3s-master-01"
K3S_NODE_IP="10.50.1.120"
K3S_NODE_ROLE="server"        # server=master, agent=worker

# Hypervisor (kvm-01 for master-01/worker-01, kvm-02 for others)
KVM_HOST="kvm-01"

# Cluster Configuration
K3S_TOKEN="<generated-token>"  # Generate once, use for all nodes
K3S_SERVER_URL="https://10.50.1.120:6443"  # For joining nodes

# Network Configuration
DOMAIN="inside.domusdigitalis.dev"
GATEWAY="10.50.1.1"           # VyOS HA VIP
DNS_PRIMARY="10.50.1.90"
DNS_SECONDARY="10.50.1.91"

# Storage
NAS_IP="10.50.1.70"
NFS_SHARE="/volume1/k3s"

# Vault Integration
VAULT_ADDR="https://vault-01.inside.domusdigitalis.dev:8200"

echo "=== k3s Node Deployment ==="
echo "Node:       $K3S_NODE_NAME"
echo "IP:         $K3S_NODE_IP"
echo "Role:       $K3S_NODE_ROLE"
echo "Hypervisor: $KVM_HOST"
echo "Gateway:    $GATEWAY"
echo "DNS:        $DNS_PRIMARY"

cd /var/lib/libvirt/images
sudo cp Rocky-9-GenericCloud-Base.latest.x86_64.qcow2 k3s-master-01.qcow2
sudo qemu-img resize k3s-master-01.qcow2 50G

cat > /tmp/k3s-cloud-init.yml << 'EOF'
#cloud-config
hostname: k3s-master-01
fqdn: k3s-master-01.inside.domusdigitalis.dev
manage_etc_hosts: true

users:
  - name: evanusmodestus
    groups: wheel
    sudo: ALL=(ALL) NOPASSWD:ALL
    shell: /bin/bash
    lock_passwd: false
    plain_text_passwd: changeme123
    ssh_authorized_keys:
      - ssh-ed25519 AAAAC3... # Add your SSH public key
      - sk-ssh-ed25519@openssh.com AAAAG... # YubiKey resident key (optional)

write_files:
  - path: /etc/NetworkManager/system-connections/eth0.nmconnection
    permissions: '0600'
    content: |
      [connection]
      id=eth0
      type=ethernet
      interface-name=eth0
      autoconnect=true

      [ipv4]
      method=manual
      addresses=10.50.1.120/24
      gateway=10.50.1.1
      dns=10.50.1.90;10.50.1.91

      [ipv6]
      method=disabled

runcmd:
  - nmcli connection reload
  - nmcli connection up eth0
  - growpart /dev/vda 4
  - xfs_growfs /
EOF

awk 'NR<=5 {print NR": ["$0"]"}' /tmp/k3s-cloud-init.yml

1: [#cloud-config]
2: [hostname: k3s-master-01]
3: [fqdn: k3s-master-01.inside.domusdigitalis.dev]
4: [manage_etc_hosts: true]
5: []

sed -i 's/^  //' /tmp/k3s-cloud-init.yml

sudo cloud-localds /var/lib/libvirt/images/k3s-cloud-init.iso /tmp/k3s-cloud-init.yml

sudo virt-install \
  --name k3s-master-01 \
  --memory 4096 \
  --vcpus 2 \
  --disk path=/var/lib/libvirt/images/k3s-master-01.qcow2,format=qcow2 \
  --disk path=/var/lib/libvirt/images/k3s-cloud-init.iso,device=cdrom \
  --os-variant rocky9 \
  --network bridge=virbr0,model=virtio \
  --graphics none \
  --console pty,target_type=serial \
  --import \
  --noautoconsole

sudo virsh list | grep k3s

virt-install \
  --name k3s-master-01 \
  --memory 4096 \
  --vcpus 2 \
  --disk path=/var/lib/libvirt/images/k3s-master-01.qcow2,size=50,format=qcow2 \
  --os-variant rocky9 \
  --network bridge=virbr0,model=virtio \
  --graphics none \
  --console pty,target_type=serial \
  --location /var/lib/libvirt/iso/Rocky-9-latest-x86_64-minimal.iso \
  --extra-args "inst.ks=http://10.50.1.110/ks/rocky9-minimal.cfg ip=10.50.1.120::10.50.1.1:255.255.255.0:k3s-master-01.inside.domusdigitalis.dev:enp1s0:none nameserver=10.50.1.90"

hostnamectl | awk '/Static hostname|Operating System|Kernel|Architecture/ {gsub(/^[[:space:]]+/, ""); print}'

Static hostname: k3s-master-01
Operating System: Rocky Linux 9.7 (Blue Onyx)
Kernel: Linux 5.14.0-611.5.1.el9_7.x86_64
Architecture: x86-64

ip -4 -o addr show | awk '{split($4,a,"/"); print $2": "$4" (scope:"$NF")"}' | grep -v "^lo:"

eth0: 10.50.1.120/24 (scope:global)

awk '/^nameserver/{print "DNS: "$2} /^search/{print "Search: "$2}' /etc/resolv.conf

for target in 10.50.1.1 10.50.1.60 10.50.1.90; do
  ping -c1 -W1 $target &>/dev/null && echo "$target: OK" || echo "$target: FAIL"
done

{vyos-vip}: OK     # VyOS HA VIP gateway
{vault-ip}: OK     # Vault
{bind-ip}: OK      # bind-01 DNS

for host in vault.inside.domusdigitalis.dev k3s-master-01.inside.domusdigitalis.dev; do
  dig +short $host | awk -v h="$host" 'NR==1 {print h": "$0}'
done

nmcli -t -f NAME,DEVICE conn show --active | awk -F: '{print $1": "$2}'

sudo nmcli conn delete "System eth0" 2>/dev/null
sudo nmcli conn delete "Wired connection 1" 2>/dev/null
sudo nmcli conn delete "cloud-init eth0" 2>/dev/null

sudo nmcli conn add con-name eth0 type ethernet ifname eth0 \
  ipv4.method manual \
  ipv4.addresses 10.50.1.120/24 \
  ipv4.gateway 10.50.1.1 \
  ipv4.dns "10.50.1.90,10.50.1.91" \
  autoconnect yes

sudo nmcli conn up eth0

ssh evanusmodestus@10.50.1.120

ip -4 -o addr show eth0 | awk '{print $4}'

10.50.1.120/24

sudo dnf update -y

sudo reboot

getenforce

sestatus | awk '/Current mode/{print $3}'

sudo dnf install -y firewalld

sudo systemctl enable --now firewalld

systemctl status firewalld | awk '/Active:/{print $2, $3}'

sudo firewall-cmd --permanent --add-port=6443/tcp

sudo firewall-cmd --permanent --add-port=80/tcp
sudo firewall-cmd --permanent --add-port=443/tcp

sudo firewall-cmd --permanent --add-port=4240/tcp
sudo firewall-cmd --permanent --add-port=4244/tcp

sudo firewall-cmd --permanent --add-port=8472/udp

sudo firewall-cmd --permanent --add-port=10250/tcp

sudo firewall-cmd --reload

sudo firewall-cmd --list-ports | awk '{gsub(/ /,"\n"); print}' | sort

10250/tcp
4240/tcp
4244/tcp
443/tcp
6443/tcp
80/tcp
8472/udp

sudo dnf install -y curl wget tar git jq bash-completion

cat >> ~/.bashrc << 'EOF'

# History configuration
HISTSIZE=10000
HISTFILESIZE=20000
HISTCONTROL=ignoreboth:erasedups
shopt -s histappend
EOF

source ~/.bashrc

curl -sSk https://10.50.1.60:8200/v1/ssh/public_key | sudo tee /etc/ssh/trusted-user-ca-keys.pem

sudo awk '{print substr($0,1,50); exit}' /etc/ssh/trusted-user-ca-keys.pem

ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCdUEnm7yqL

echo 'TrustedUserCAKeys /etc/ssh/trusted-user-ca-keys.pem' | sudo tee -a /etc/ssh/sshd_config

sudo awk '/TrustedUserCAKeys|^Match/' /etc/ssh/sshd_config

sudo systemctl restart sshd

ssh k3s-master-01 hostname

curl -sfL https://get.k3s.io | sudo INSTALL_K3S_EXEC="server \
  --selinux \
  --flannel-backend=none \
  --disable-network-policy" sh -

curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

helm version

CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)
CLI_ARCH=amd64
curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}

helm repo add cilium https://helm.cilium.io/
helm repo update

cat > /tmp/cilium-values.yaml << 'EOF'
# Cilium Helm Values - Production
# k3s single-node deployment

cluster:
  name: domus-k3s

# k3s API server (internal)
k8sServiceHost: 127.0.0.1
k8sServicePort: 6443

# Replace kube-proxy entirely (eBPF-native)
kubeProxyReplacement: true

# Network mode
routingMode: tunnel
tunnelProtocol: vxlan

# Operator HA (single node = 1, multi-node = 2)
operator:
  replicas: 1

# Hubble observability (headless - CLI only)
hubble:
  enabled: true
  relay:
    enabled: true
  # UI disabled - headless server, use `hubble observe` CLI
  ui:
    enabled: false

# Security hardening
securityContext:
  capabilities:
    ciliumAgent:
      - CHOWN
      - KILL
      - NET_ADMIN
      - NET_RAW
      - IPC_LOCK
      - SYS_ADMIN
      - SYS_RESOURCE
      - DAC_OVERRIDE
      - FOWNER
      - SETGID
      - SETUID
    cleanCiliumState:
      - NET_ADMIN
      - SYS_ADMIN
      - SYS_RESOURCE
EOF

helm install cilium cilium/cilium -n kube-system -f /tmp/cilium-values.yaml

kubectl rollout status daemonset/cilium -n kube-system --timeout=300s

cilium status

    /¯¯\
 /¯¯\__/¯¯\    Cilium:             OK
 \__/¯¯\__/    Operator:           OK
 /¯¯\__/¯¯\    Envoy DaemonSet:    OK
 \__/¯¯\__/    Hubble Relay:       OK
    \__/       ClusterMesh:        disabled

kubectl get pods -n kube-system -l k8s-app=cilium -o custom-columns='NAME:.metadata.name,STATUS:.status.phase,READY:.status.containerStatuses[0].ready'

NAME           STATUS    READY
cilium-xxxxx   Running   true

kubectl get nodes -o custom-columns='NAME:.metadata.name,STATUS:.status.conditions[-1].type,VERSION:.status.nodeInfo.kubeletVersion'

NAME                                        STATUS   VERSION
k3s-master-01.inside.domusdigitalis.dev     Ready    v1.34.x+k3s1

kubectl get pods -n kube-system -l app.kubernetes.io/name=hubble-relay -o custom-columns='NAME:.metadata.name,STATUS:.status.phase,READY:.status.containerStatuses[0].ready'

NAME                  STATUS    READY
hubble-relay-xxxxx    Running   true

# Real-time flow visibility
hubble observe

# Filter by namespace
hubble observe -n kube-system

# Show only policy denials
hubble observe --verdict DROPPED

# DNS queries
hubble observe --protocol DNS

cilium connectivity test --single-node

kubectl delete namespace cilium-test-1 cilium-test-ccnp1 cilium-test-ccnp2 --wait=false 2>/dev/null

cilium completion bash | sudo tee /etc/bash_completion.d/cilium > /dev/null
kubectl completion bash | sudo tee /etc/bash_completion.d/kubectl > /dev/null

source /etc/bash_completion.d/cilium
source /etc/bash_completion.d/kubectl

systemctl show k3s --property=ActiveState,SubState,MainPID | awk -F= '{print $1": "$2}'

ActiveState: active
SubState: running
MainPID: 1234

kubectl get nodes -o json | jq -r '.items[] | "\(.metadata.name): \(.status.conditions[] | select(.type=="Ready") | .status) (\(.status.nodeInfo.kubeletVersion))"'

k3s-master-01: True (v1.31.4+k3s1)

kubectl get nodes -o json | jq -r '.items[] | "CPU: \(.status.capacity.cpu) | Memory: \(.status.capacity.memory) | Pods: \(.status.capacity.pods)"'

kubectl get pods -A --no-headers | awk '
{
  ns[$1]++
  status[$4]++
  combo[$1","$4]++
}
END {
  print "=== By Namespace ==="
  for(n in ns) printf "%-20s %d\n", n, ns[n]
  print "\n=== By Status ==="
  for(s in status) printf "%-15s %d\n", s, status[s]
}'

kubectl get pods -A -o json | jq -r '.items[] | select(.status.phase != "Running" and .status.phase != "Succeeded") | "\(.metadata.namespace)/\(.metadata.name): \(.status.phase)"'

echo "=== k3s ===" && systemctl is-active k3s && \
echo -e "\n=== Cilium ===" && cilium status --output json 2>/dev/null | jq -r '"Cilium: \(.cilium.state)\nOperator: \(.operator.state)\nHubble: \(.hubble.state // "disabled")"' && \
echo -e "\n=== Nodes ===" && kubectl get nodes --no-headers | awk '{printf "%-25s %s\n", $1, $2}'

mkdir -p ~/.kube
sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
sudo chown $(id -u):$(id -g) ~/.kube/config
chmod 600 ~/.kube/config

export KUBECONFIG=~/.kube/config

echo 'export KUBECONFIG=~/.kube/config' >> ~/.bashrc

kubectl get nodes

echo 'source <(kubectl completion bash)' >> ~/.bashrc

source ~/.bashrc

sudo awk '{print NR": "$0}' /var/lib/rancher/k3s/server/node-token

sudo awk '{print "Length:", length($0)}' /var/lib/rancher/k3s/server/node-token

gopass edit v3/domains/d000/k3s/node-token

scp k3s-master-01:~/.kube/config /tmp/k3s-kubeconfig

awk 'NR <= 20 {print NR": "$0}' /tmp/k3s-kubeconfig

gopass edit v3/domains/d000/k3s/kubeconfig

vault auth enable kubernetes

vault auth list | grep kubernetes

kubernetes/    kubernetes    auth_kubernetes_xxxx    n/a

kubectl config view --raw --minify --flatten -o jsonpath='{.clusters[].cluster.certificate-authority-data}' | base64 -d > /tmp/k3s-ca.crt

openssl x509 -in /tmp/k3s-ca.crt -noout -subject -issuer -dates | awk -F'=' '
/Subject:/ {sub(/.*CN ?= ?/, ""); print "Subject: "$0}
/Issuer:/  {sub(/.*CN ?= ?/, ""); print "Issuer:  "$0}
/notBefore/ {print "Valid From: "$2}
/notAfter/  {print "Valid Until: "$2}
'

Subject: k3s-master-01
Issuer:  k3s-master-01
Valid From: Feb 21 00:00:00 2026 GMT
Valid Until: Feb 19 00:00:00 2036 GMT

kubectl create serviceaccount vault-auth -n kube-system

kubectl create clusterrolebinding vault-auth-binding \
  --clusterrole=system:auth-delegator \
  --serviceaccount=kube-system:vault-auth

kubectl create token vault-auth -n kube-system --duration=8760h > /tmp/vault-auth-token

awk '{print "Token length:", length($0)}' /tmp/vault-auth-token

vault write auth/kubernetes/config \
  kubernetes_host="https://10.50.1.120:6443" \
  kubernetes_ca_cert=@/tmp/k3s-ca.crt \
  token_reviewer_jwt=@/tmp/vault-auth-token

vault policy write k3s-secrets - <<'EOF'
# Allow k3s pods to read secrets
path "kv/data/k3s/*" {
  capabilities = ["read"]
}

# Allow PKI certificate issuance
path "pki_int/issue/domus-workstation" {
  capabilities = ["create", "update"]
}
EOF

vault write auth/kubernetes/role/k3s-app \
  bound_service_account_names="*" \
  bound_service_account_namespaces="default,production" \
  policies="k3s-secrets" \
  ttl="1h"

scp /etc/ssl/certs/DOMUS-CA-CHAIN.pem k3s-master-01:/tmp/

helm repo add hashicorp https://helm.releases.hashicorp.com
helm repo update

helm install vault hashicorp/vault \
  --set "injector.enabled=true" \
  --set "injector.externalVaultAddr=https://vault-01.inside.domusdigitalis.dev:8200" \
  --set "server.enabled=false" \
  --set "injector.certs.caBundle=$(base64 -w0 /tmp/DOMUS-CA-CHAIN.pem)"

kubectl create secret generic vault-tls --from-file=ca.crt=/tmp/DOMUS-CA-CHAIN.pem

kubectl get pods -l app.kubernetes.io/name=vault-agent-injector -o json | jq -r '
.items[] | "\(.metadata.name): \(.status.phase) (Ready: \(.status.containerStatuses[0].ready))"
'

vault-agent-injector-xxxxx: Running (Ready: true)

kubectl logs -l app.kubernetes.io/name=vault-agent-injector --tail=50 | awk '/level=error|level=warn/ {print}'

vault kv put kv/k3s/test username="testuser" password="testpass123"

vault kv get -format=json kv/k3s/test | jq -r '
"Path: \(.request_id)",
"Version: \(.data.metadata.version)",
"Created: \(.data.metadata.created_time)",
"Keys: \(.data.data | keys | join(", "))"
'

vault kv get -format=json kv/k3s/test | jq -r '.data.data | to_entries[] | "\(.key)=\(.value)"'

password=testpass123
username=testuser

kubectl apply -f - <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  name: test-app
---
apiVersion: v1
kind: Pod
metadata:
  name: vault-test
  annotations:
    vault.hashicorp.com/agent-inject: "true"
    vault.hashicorp.com/role: "k3s-app"
    vault.hashicorp.com/agent-inject-secret-credentials.txt: "kv/data/k3s/test"
    vault.hashicorp.com/tls-secret: "vault-tls"
    vault.hashicorp.com/ca-cert: "/vault/tls/ca.crt"
spec:
  serviceAccountName: test-app
  containers:
  - name: app
    image: busybox
    command: ["sh", "-c", "while true; do sleep 3600; done"]
EOF

kubectl wait --for=condition=Ready pod/vault-test --timeout=120s && echo "Pod ready" || echo "Timeout - check events"

kubectl get pod vault-test -o json | jq -r '
"Phase: \(.status.phase)",
"Containers: \(.spec.containers | length) app + \(.spec.initContainers // [] | length) init",
"Vault Injected: \(.metadata.annotations["vault.hashicorp.com/agent-inject"] // "false")",
(.status.containerStatuses[]? | "  \(.name): \(.ready) (restarts: \(.restartCount))")
'

kubectl exec vault-test -- sh -c 'cat /vault/secrets/credentials.txt' | awk -F'=' '{printf "%-15s %s\n", $1":", $2}'

username:       testuser
password:       testpass123

kubectl logs vault-test -c vault-agent-init 2>/dev/null | awk '
/level=info.*secret.*rendered/ {print "✓ Secret rendered"}
/level=error/ {print "✗ ERROR: "$0}
END {if(NR==0) print "No init container logs (may have completed)"}
'

kubectl exec vault-test -- sh -c '
echo "=== Secret File ===" && ls -la /vault/secrets/ && \
echo -e "\n=== Contents ===" && cat /vault/secrets/credentials.txt && \
echo -e "\n=== Permissions ===" && stat -c "%a %U:%G" /vault/secrets/credentials.txt
'

kubectl delete pod vault-test

kubectl delete sa test-app

ssh bind-01 "sudo nsupdate -l << 'EOF'
zone inside.domusdigitalis.dev
update add k3s-master-01.inside.domusdigitalis.dev. 3600 A 10.50.1.120
send
EOF"

dig k3s-master-01.inside.domusdigitalis.dev

dig +short k3s-master-01.inside.domusdigitalis.dev | awk 'NR==1'

echo "=== k3s Cluster Health Check ===" && echo

# k3s service
printf "%-20s" "k3s service:"
systemctl is-active k3s 2>/dev/null | awk '{print ($1=="active") ? "✓ "$1 : "✗ "$1}'

# SELinux
printf "%-20s" "SELinux:"
getenforce | awk '{print ($1=="Enforcing") ? "✓ "$1 : "✗ "$1}'

# Firewall
printf "%-20s" "firewalld:"
systemctl is-active firewalld 2>/dev/null | awk '{print ($1=="active") ? "✓ "$1 : "✗ "$1}'

# Node status
printf "%-20s" "Node Ready:"
kubectl get nodes -o json 2>/dev/null | jq -r '.items[0].status.conditions[] | select(.type=="Ready") | if .status=="True" then "✓ Ready" else "✗ NotReady" end'

# Cilium
printf "%-20s" "Cilium:"
cilium status --output json 2>/dev/null | jq -r 'if .cilium.state=="Ok" then "✓ Ok" else "✗ \(.cilium.state)" end' || echo "✗ not installed"

# Hubble
printf "%-20s" "Hubble:"
cilium status --output json 2>/dev/null | jq -r 'if .hubble.state then "✓ \(.hubble.state)" else "○ disabled" end' || echo "○ N/A"

# Vault injector
printf "%-20s" "Vault Injector:"
kubectl get pods -l app.kubernetes.io/name=vault-agent-injector -o json 2>/dev/null | jq -r '.items[0].status.phase // "not found"' | awk '{print ($1=="Running") ? "✓ "$1 : "✗ "$1}'

# Pod summary
echo -e "\n=== Pod Summary ==="
kubectl get pods -A --no-headers 2>/dev/null | awk '
{status[$4]++; total++}
END {
  for(s in status) printf "%-15s %d\n", s":", status[s]
  print "---"
  printf "%-15s %d\n", "Total:", total
}'

# Unhealthy pods
echo -e "\n=== Unhealthy Pods ==="
kubectl get pods -A --no-headers 2>/dev/null | awk '$4!="Running" && $4!="Completed" {print $1"/"$2": "$4}' || echo "None"

systemctl show k3s --property=ActiveState,SubState,MainPID,MemoryCurrent | awk -F= '{
  if($1=="MemoryCurrent") printf "%-15s %.1f MB\n", $1":", $2/1024/1024
  else printf "%-15s %s\n", $1":", $2
}'

sudo journalctl -u k3s --since "24 hours ago" -p err --no-pager | awk '
!seen[$0]++ {print NR": "$0}
END {if(NR==0) print "No errors in last 24h"}
' | head -20

cilium status --output json 2>/dev/null | jq -r '
"Cilium:     \(.cilium.state)",
"Operator:   \(.operator.state)",
"Hubble:     \(.hubble.state // "disabled")",
"ClusterMesh: \(.cluster_mesh.state // "disabled")"
'

kubectl get pods -A | awk 'NR==1 || /Running|Pending|Error/'

kubectl get pods -A --no-headers | awk '{count[$4]++} END {for(s in count) print s, count[s]}'

Running 8
Completed 2

kubectl get pods -A | awk 'NR==1 || !/Running/'

kubectl top nodes 2>/dev/null | awk '{print $1, $3, $5}'

kubectl get pods -A -o wide --no-headers | awk '{count[$8]++} END {for(n in count) print n, count[n]}'

# 1. Backup any persistent data
kubectl get pv -o yaml > /tmp/pv-backup.yaml
kubectl get pvc -A -o yaml > /tmp/pvc-backup.yaml

# 2. Uninstall k3s completely
/usr/local/bin/k3s-uninstall.sh

# 3. Reinstall with Cilium flags (Phase 3.1)
curl -sfL https://get.k3s.io | sudo INSTALL_K3S_EXEC="server \
  --selinux \
  --flannel-backend=none \
  --disable-network-policy" sh -

# 4. Install Cilium (Phase 3.2)
# ... follow Phase 3.2 steps

# 1. Install Cilium alongside Flannel
cilium install --version 1.16.5

# 2. Wait for Cilium to be ready
cilium status --wait

# 3. Verify pods have Cilium networking
kubectl get pods -A -o wide

# 4. Delete Flannel (after Cilium verified working)
kubectl delete -n kube-system daemonset kube-flannel-ds 2>/dev/null || true

# 5. Remove Flannel CNI config
sudo rm -f /etc/cni/net.d/10-flannel.conflist

# 6. Restart k3s
sudo systemctl restart k3s

sudo ausearch -m avc -ts recent

sudo ausearch -m avc -ts recent | awk '/denied/{print}'

sudo ausearch -c 'k3s' --raw | audit2allow -M k3s-selinux

sudo semodule -i k3s-selinux.pp

sudo journalctl -xeu k3s --no-pager -p err | awk 'NR <= 30'

sudo journalctl -xeu k3s --no-pager | awk 'NR > (NR-100)'

kubectl logs -l app.kubernetes.io/name=vault-agent-injector

kubectl logs -l app.kubernetes.io/name=vault-agent-injector | awk '/error|Error|ERROR/'

kubectl describe pod <pod-name> | awk '/Events:/,0'

kubectl run vault-test --rm -it --image=alpine -- wget --ca-certificate=/etc/ssl/certs/ca-certificates.crt -qO- https://vault-01.inside.domusdigitalis.dev:8200/v1/sys/health

kubectl run vault-test --rm -it --image=alpine -- wget --ca-certificate=/etc/ssl/certs/ca-certificates.crt -qO- https://vault-01.inside.domusdigitalis.dev:8200/v1/sys/health 2>/dev/null | jq -r '.sealed, .initialized'

# Check what Cilium thinks the node IP is
kubectl get ciliumnode

# Compare to actual node IP
ip -4 addr show eth0 | awk '/inet/{print $2}'

# Delete stale CiliumNode (will recreate with correct IP)
kubectl delete ciliumnode $(hostname -f)

# Restart Cilium to pick up correct IP
kubectl rollout restart daemonset/cilium -n kube-system
kubectl rollout status daemonset/cilium -n kube-system

# Verify correct IP
kubectl get ciliumnode

# Check binary inside image
sudo mkdir -p /mnt/test-image
sudo /usr/local/bin/k3s ctr images mount <image>@<digest> /mnt/test-image
file /mnt/test-image/usr/bin/<binary>
sudo umount /mnt/test-image

# Stop k3s
sudo systemctl stop k3s

# Remove corrupted snapshotter state
sudo rm -rf /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/*
sudo rm -f /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs/metadata.db

# Start k3s (will re-pull all images fresh)
sudo systemctl start k3s

# Wait for cluster to stabilize
sleep 60
kubectl get pods -A

sudo ctr images ls
# Error: sudo: ctr: command not found

sudo /usr/local/bin/k3s ctr images ls

# List images
sudo /usr/local/bin/k3s ctr images ls

# Remove image
sudo /usr/local/bin/k3s ctr images rm <image>

# Pull image with specific platform
sudo /usr/local/bin/k3s ctr images pull --platform linux/amd64 <image>

# List containers
sudo /usr/local/bin/k3s ctr containers ls

# Check content store
sudo /usr/local/bin/k3s ctr content ls

Error: UPGRADE FAILED: template: cilium/templates/xxx.yaml:1:14: nil pointer evaluating interface \{}.enabled

# Export current values
helm get values cilium -n kube-system -o yaml > /tmp/cilium-values.yaml

# Review and add any missing required fields
cat /tmp/cilium-values.yaml

# Upgrade with explicit values
helm upgrade cilium cilium/cilium -n kube-system -f /tmp/cilium-values.yaml

/usr/local/bin/k3s-uninstall.sh

LAST DEPLOYED: Sun Feb 22 10:35:29 2026
NAMESPACE: default
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
Thank you for installing HashiCorp Vault!

Now that you have deployed Vault, you should look over the docs on using
Vault with Kubernetes available here:

https://developer.hashicorp.com/vault/docs

Your release is named vault. To learn more about the release, try:

  $ helm status vault
  $ helm get manifest vault

helm install vault hashicorp/vault \
  --set "injector.enabled=true" \
  --set "injector.externalVaultAddr=https://vault-01.inside.domusdigitalis.dev:8200" \
  --set "server.enabled=false"

kubectl exec vault-test -c app -- cat /vault/secrets/credentials.txt
data: map[password:testpass123 username:testuser]
metadata: map[created_time:2026-02-22T10:38:41.572336046Z ...]