CR-2026-02-25 - Wazuh Credential Rotation (Emergency)

v3/domains/d000/k3s/wazuh/     # Single entry with mixed metadata

v3/domains/d000/k3s/wazuh/indexer    # OpenSearch admin
v3/domains/d000/k3s/wazuh/dashboard  # Dashboard backend
v3/domains/d000/k3s/wazuh/api        # Manager API

# Store current password for comparison
CURRENT_PASS=$(ssh k3s-master-01 "kubectl -n wazuh get secret indexer-cred -o jsonpath='{.data.password}' | base64 -d")
echo "Current password: $CURRENT_PASS"

# TEST 1: Indexer API health (from inside cluster)
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- \
  curl -sk -u admin:$CURRENT_PASS https://localhost:9200/_cluster/health" | \
  jq -C 'if .status then "✓ PRE-CHECK: Indexer API accessible (status: \(.status))" else "✗ PRE-CHECK FAILED" end'

# TEST 2: Indexer node info
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- \
  curl -sk -u admin:$CURRENT_PASS https://localhost:9200/_nodes/_local/info" | \
  jq -C '.nodes | to_entries[0].value | "✓ PRE-CHECK: Node \(.name) version \(.version)"'

# TEST 3: Dashboard backend can reach indexer (check logs for auth errors)
ssh k3s-master-01 'kubectl -n wazuh logs deployment/wazuh-dashboard --tail=20 | \
  grep -E "authentication|unauthorized|error" | head -5 || echo "✓ PRE-CHECK: No auth errors in dashboard logs"'

# TEST 4: Manager can write to indexer
ssh k3s-master-01 'kubectl -n wazuh logs wazuh-manager-master-0 -c wazuh-manager --tail=50 | \
  grep -iE "indexer.*error|opensearch.*fail|connection.*refused" | head -5 || echo "✓ PRE-CHECK: No indexer errors in manager logs"'

# Capture pod state for comparison (tee runs locally)
ssh k3s-master-01 'kubectl -n wazuh get pods -o json | \
  jq -r ".items[] | \"\(.metadata.name),\(.status.phase),\(.status.containerStatuses[0].restartCount)\""' | \
  tee /tmp/wazuh-pods-before.csv

# Display captured state
echo "=== POD STATE BEFORE ==="
column -t -s',' /tmp/wazuh-pods-before.csv

# Generate cryptographically secure 32-character password
NEW_PASS=$(tr -dc 'A-Za-z0-9' </dev/urandom | head -c 32)
echo "New password: $NEW_PASS"
echo "Length: $(echo -n "$NEW_PASS" | wc -c)"

# Save for later phases (do NOT persist to disk)
export NEW_PASS

# Relocate old flat entry (keep until validation passes)
gopass mv v3/domains/d000/k3s/wazuh v3/domains/d000/k3s/wazuh-pre-rotation 2>/dev/null || true

# Create indexer entry with metadata (heredoc)
cat << EOF | gopass insert -m v3/domains/d000/k3s/wazuh/indexer
$NEW_PASS
---
username: admin
service: wazuh-indexer (OpenSearch)
k8s_secret: indexer-cred
k8s_namespace: wazuh
url: https://wazuh.inside.domusdigitalis.dev:9200
rotated: 2026-02-25
EOF

# Get existing dashboard password from K8s
DASHBOARD_PASS=$(ssh k3s-master-01 "kubectl -n wazuh get secret dashboard-cred -o jsonpath='{.data.password}' | base64 -d")

# Create dashboard entry with metadata
cat << EOF | gopass insert -m v3/domains/d000/k3s/wazuh/dashboard
$DASHBOARD_PASS
---
username: kibanaserver
service: wazuh-dashboard
k8s_secret: dashboard-cred
k8s_namespace: wazuh
url: https://wazuh.inside.domusdigitalis.dev:8443
rotated: 2026-02-25
EOF

# Get existing API password from K8s
API_PASS=$(ssh k3s-master-01 "kubectl -n wazuh get secret wazuh-api-cred -o jsonpath='{.data.password}' | base64 -d")

# Create api entry with metadata
cat << EOF | gopass insert -m v3/domains/d000/k3s/wazuh/api
$API_PASS
---
username: wazuh-wui
service: wazuh-manager API
k8s_secret: wazuh-api-cred
k8s_namespace: wazuh
url: https://wazuh.inside.domusdigitalis.dev:55000
rotated: 2026-02-25
EOF

# Verify structure
gopass ls v3/domains/d000/k3s/wazuh/

v3/domains/d000/k3s/wazuh/api
v3/domains/d000/k3s/wazuh/dashboard
v3/domains/d000/k3s/wazuh/indexer

# Verify structure
gopass ls v3/domains/d000/k3s/wazuh/
v3/domains/d000/k3s/wazuh/
├── api
├── dashboard
└── indexer

# Patch secret with new password (atomic operation)
ssh k3s-master-01 "kubectl -n wazuh patch secret indexer-cred \
  --type='json' \
  -p='[{\"op\": \"replace\", \"path\": \"/data/password\", \"value\": \"'$(echo -n "$NEW_PASS" | base64)'\"}]'"

# Verify secret updated (resourceVersion should change)
ssh k3s-master-01 'kubectl -n wazuh get secret indexer-cred -o json | \
  jq -r "\"Secret updated: resourceVersion=\(.metadata.resourceVersion)\""'

# 1. Indexer first (database layer) - check if recently restarted
POD_AGE=$(ssh k3s-master-01 "kubectl -n wazuh get pod wazuh-indexer-0 -o jsonpath='{.metadata.creationTimestamp}'" | \
  xargs -I{} date -d {} +%s)
NOW=$(date +%s)
AGE_MIN=$(( (NOW - POD_AGE) / 60 ))

[[ $AGE_MIN -lt 10 ]] && echo "✓ Indexer restarted ${AGE_MIN}m ago, skipping" || \
  { echo "Restarting indexer (last restart: ${AGE_MIN}m ago)..."; \
    ssh k3s-master-01 "kubectl -n wazuh rollout restart statefulset wazuh-indexer && \
      kubectl -n wazuh rollout status statefulset wazuh-indexer --timeout=180s"; }

❯ # Verify secret updated (resourceVersion should change)
ssh k3s-master-01 'kubectl -n wazuh get secret indexer-cred -o json | \
  jq -r "\"Secret updated: resourceVersion=\(.metadata.resourceVersion)\""'
Secret updated: resourceVersion=224900
❯ # 1. Indexer first (database layer) - must be healthy before others
ssh k3s-master-01 "kubectl -n wazuh rollout restart statefulset wazuh-indexer && \
  kubectl -n wazuh rollout status statefulset wazuh-indexer --timeout=180s"
statefulset.apps/wazuh-indexer restarted
Waiting for partitioned roll out to finish: 0 out of 1 new pods have been updated...
Waiting for 1 pods to be ready...
Waiting for 1 pods to be ready...
Waiting for 1 pods to be ready...
partitioned roll out complete: 1 new pods have been updated...

for i in {1..30}; do
  if ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- curl -sk -u admin:$NEW_PASS https://localhost:9200/_cluster/health" 2>/dev/null | jq -e '.status' &>/dev/null; then
    echo "✓ Indexer API ready after $i attempts"
    break
  fi
  echo "Waiting for indexer API... ($i/30)"
  sleep 5
done

# 3. Manager master (connects to indexer) - with health check
ssh k3s-master-01 "kubectl -n wazuh rollout restart statefulset wazuh-manager-master && \
  kubectl -n wazuh rollout status statefulset wazuh-manager-master --timeout=120s" && \
  echo "✓ Manager master restarted" || echo "✗ Manager master restart failed"

# 4. Manager worker - with health check
ssh k3s-master-01 "kubectl -n wazuh rollout restart statefulset wazuh-manager-worker && \
  kubectl -n wazuh rollout status statefulset wazuh-manager-worker --timeout=120s" && \
  echo "✓ Manager worker restarted" || echo "✗ Manager worker restart failed"

# 5. Dashboard last (UI layer) - with health check
ssh k3s-master-01 "kubectl -n wazuh rollout restart deployment wazuh-dashboard && \
  kubectl -n wazuh rollout status deployment wazuh-dashboard --timeout=120s" && \
  echo "✓ Dashboard restarted" || echo "✗ Dashboard restart failed"

# 6. Final pod status check - all should be Running with 0 restarts since restart
ssh k3s-master-01 'kubectl -n wazuh get pods -o custom-columns="NAME:.metadata.name,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount,AGE:.metadata.creationTimestamp"' | \
  awk 'NR==1 {print; next} {
    cmd="date -d " $4 " +%s"; cmd | getline ts; close(cmd);
    age=systime()-ts;
    status=($2=="Running" && $3==0) ? "✓" : "⚠";
    printf "%s %-35s %-10s %-10s %dm\n", status, $1, $2, $3, age/60
  }'

# Set NEW_PASS for tests (retrieve from gopass if shell restarted)
NEW_PASS=$(gopass show -o v3/domains/d000/k3s/wazuh/indexer)
echo "NEW_PASS: ${NEW_PASS:0:5}..."

# TEST 1: Indexer API health with NEW password
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- \
  curl -sk -u admin:$NEW_PASS https://localhost:9200/_cluster/health" | \
  jq -C 'if .status == "green" then "✓ POST-CHECK: Indexer healthy (green)" elif .status then "⚠ POST-CHECK: Indexer status \(.status)" else "✗ POST-CHECK FAILED: No response" end'

# TEST 2: Verify OLD password NO LONGER works (uses known old password)
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- curl -sk -u admin:SecretPassword https://localhost:9200/_cluster/health" 2>&1 | grep -qE 'Unauthorized|401' && echo "✓ POST-CHECK: Old password rejected" || echo "✗ SECURITY ISSUE: Old password still works"

# TEST 3: Index count (verify data accessible)
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- \
  curl -sk -u admin:$NEW_PASS https://localhost:9200/_cat/indices?v" | \
  awk 'NR>1 {count++} END {print "✓ POST-CHECK: " count " indices accessible"}'

# TEST 4: Dashboard login page (via Traefik on 443)
curl -sk --connect-timeout 5 -o /dev/null -w "%{http_code}" https://wazuh.inside.domusdigitalis.dev/app/login | \
  awk '{print ($1 == 200 || $1 == 302) ? "✓ POST-CHECK: Dashboard accessible (HTTP " $1 ")" : "✗ POST-CHECK: Dashboard returned HTTP " $1}'

# TEST 5: Manager API responding (401 = API is up, just needs auth)
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-manager-master-0 -c wazuh-manager -- \
  curl -sk -o /dev/null -w '%{http_code}' https://localhost:55000/" | \
  awk '{print ($1 == 401 || $1 == 200) ? "✓ POST-CHECK: Manager API responding (HTTP " $1 ")" : "✗ POST-CHECK: Manager API returned HTTP " $1}'

# TEST 6: No auth errors in logs (last 2 minutes)
ssh k3s-master-01 'kubectl -n wazuh logs wazuh-indexer-0 -c wazuh-indexer --since=2m | \
  grep -ciE "authentication.*fail|unauthorized|401" | \
  awk "{print (\$1 == 0) ? \"✓ POST-CHECK: No auth failures in indexer logs\" : \"✗ POST-CHECK: \" \$1 \" auth failures detected\"}"'

# Capture AFTER pod state (tee runs locally)
ssh k3s-master-01 'kubectl -n wazuh get pods -o json | \
  jq -r ".items[] | \"\(.metadata.name),\(.status.phase),\(.status.containerStatuses[0].restartCount)\""' | \
  tee /tmp/wazuh-pods-after.csv

# Display BEFORE state
echo "=== POD STATE COMPARISON ==="
echo "BEFORE:"
column -t -s',' /tmp/wazuh-pods-before.csv

# Display AFTER state (restart counts should be +1, no CrashLoopBackOff)
echo "AFTER:"
column -t -s',' /tmp/wazuh-pods-after.csv

# Find and remove hardcoded Wazuh passwords
grep -rn "WAZUH.*PASSWORD.*=" ~/.claude/ 2>/dev/null | \
  awk -F: '{print $1 ":" $2}' | sort -u

// REMOVE:
"Bash(WAZUH_INDEXER_PASSWORD=\"xRPz...\" ...)"

// REPLACE WITH:
"Bash(WAZUH_INDEXER_PASSWORD=$WAZUH_INDEXER_PASSWORD ...)"

# Delete pre-rotation backup (ONLY after all validation passes)
gopass rm v3/domains/d000/k3s/wazuh-pre-rotation

# Find file by name
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- \
  find /usr/share/wazuh-indexer -name 'internal_users.yml' 2>/dev/null"

# Find all certs (.pem files)
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- \
  find /usr/share/wazuh-indexer -name '*.pem' 2>/dev/null"

# Find all scripts (.sh files)
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- \
  find /usr/share/wazuh-indexer -name '*.sh' 2>/dev/null"

# Find directories only (understand structure)
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- \
  find /usr/share/wazuh-indexer -type d 2>/dev/null" | head -20

# Extract hash value from internal_users.yml for admin user
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- \
  awk '/^admin:/,/^[a-z]/' /usr/share/wazuh-indexer/config/opensearch-security/internal_users.yml" | \
  awk '/hash:/ {print $2}'

# List all users in internal_users.yml (lines starting with word, ending with colon)
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- \
  awk '/^[a-z]+:$/ {print}' /usr/share/wazuh-indexer/config/opensearch-security/internal_users.yml"

# Show file permissions in columnar format
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- \
  ls -la /usr/share/wazuh-indexer/config/certs/" | \
  awk '{printf "%-20s %-10s %-10s %s\n", $9, $1, $3, $4}'

# Step 1: Find all config directories
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- \
  find /usr/share/wazuh-indexer -type d -name 'config' -o -name '*security*' 2>/dev/null"

# Step 2: List contents of discovered directory
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- \
  ls -la /usr/share/wazuh-indexer/config/"

# Step 3: Read specific file
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- \
  head -30 /usr/share/wazuh-indexer/config/opensearch-security/internal_users.yml"

# Copy password to clipboard
gopass show -c v3/domains/d000/k3s/wazuh/indexer

# Open dashboard
echo "https://wazuh.inside.domusdigitalis.dev"
# Login: admin / <password from clipboard>

# Set credential
NEW_PASS=$(gopass show -o v3/domains/d000/k3s/wazuh/indexer)

# Indexer cluster health (expect: green)
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- curl -sk -u admin:$NEW_PASS https://localhost:9200/_cluster/health" | jq -r '.status'

# Index count (expect: 25-30)
ssh k3s-master-01 "kubectl -n wazuh exec wazuh-indexer-0 -c wazuh-indexer -- curl -sk -u admin:$NEW_PASS https://localhost:9200/_cat/indices" | wc -l

# No auth failures (expect: 0)
ssh k3s-master-01 'kubectl -n wazuh logs wazuh-indexer-0 -c wazuh-indexer --since=5m' | grep -ci "authentication.*fail"

# All pods running
ssh k3s-master-01 "kubectl -n wazuh get pods -o wide"