k3s Prometheus + Grafana

ssh k3s-master-01.inside.domusdigitalis.dev
kubectl get nodes -o wide
kubectl get pods -A | grep -v Running

helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
helm repo update

helm install nfs-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \
  --namespace kube-system \
  --set nfs.server=10.50.1.70 \
  --set nfs.path=/volume1/k3s \
  --set storageClass.name=nfs-client \
  --set storageClass.defaultClass=false \
  --set storageClass.archiveOnDelete=true

kubectl get storageclass

NAME                   PROVISIONER                                     RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION
local-path (default)   rancher.io/local-path                           Delete          WaitForFirstConsumer   false
nfs-client             cluster.local/nfs-provisioner                   Delete          Immediate              true

cat <<'EOF' | kubectl apply -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: test-nfs-claim
spec:
  storageClassName: nfs-client
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
EOF

# Verify
kubectl get pvc test-nfs-claim

# Cleanup
kubectl delete pvc test-nfs-claim

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

helm search repo prometheus-community/kube-prometheus-stack --versions | head -5

cat > /tmp/prometheus-values.yaml << 'EOF'
# =============================================================================
# kube-prometheus-stack values
# Runbook: k3s-monitoring.adoc
# =============================================================================

# -----------------------------------------------------------------------------
# Global Settings
# -----------------------------------------------------------------------------
defaultRules:
  create: true
  rules:
    alertmanager: true
    etcd: false  # k3s uses SQLite, not etcd
    kubeApiserver: true
    kubeScheduler: true
    kubeControllerManager: true
    kubeProxy: true
    node: true

# -----------------------------------------------------------------------------
# Prometheus
# -----------------------------------------------------------------------------
prometheus:
  prometheusSpec:
    replicas: 1
    retention: 30d
    retentionSize: "45GB"

    # NFS Storage
    storageSpec:
      volumeClaimTemplate:
        spec:
          storageClassName: nfs-client
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 50Gi

    # Resource limits
    resources:
      requests:
        cpu: 200m
        memory: 512Mi
      limits:
        cpu: 1000m
        memory: 2Gi

    # Scrape all namespaces
    podMonitorNamespaceSelector: {}
    podMonitorSelector: {}
    serviceMonitorNamespaceSelector: {}
    serviceMonitorSelector: {}

# -----------------------------------------------------------------------------
# Grafana
# -----------------------------------------------------------------------------
grafana:
  enabled: true
  adminPassword: "CHANGE_ME_BEFORE_INSTALL"

  # NFS Persistence
  persistence:
    enabled: true
    storageClassName: nfs-client
    size: 10Gi

  # Resource limits
  resources:
    requests:
      cpu: 100m
      memory: 128Mi
    limits:
      cpu: 500m
      memory: 512Mi

  # Default dashboards
  defaultDashboardsEnabled: true
  defaultDashboardsTimezone: America/Los_Angeles

  # Sidecar for dashboard provisioning
  sidecar:
    dashboards:
      enabled: true
      searchNamespace: ALL
    datasources:
      enabled: true

  # Grafana.ini settings
  # IMPORTANT: For port-forward access, use explicit IP and serve_from_sub_path: false
  # The %(domain)s variable resolves to 'localhost' inside the container, breaking redirects
  grafana.ini:
    server:
      root_url: "http://10.50.1.120:3000"
      serve_from_sub_path: false
    security:
      admin_user: admin
      cookie_secure: false  # Set true when using HTTPS/Traefik
    users:
      auto_assign_org_role: Viewer
    auth.anonymous:
      enabled: false

# -----------------------------------------------------------------------------
# AlertManager
# -----------------------------------------------------------------------------
alertmanager:
  enabled: true
  alertmanagerSpec:
    replicas: 1

    # NFS Storage
    storage:
      volumeClaimTemplate:
        spec:
          storageClassName: nfs-client
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 5Gi

    # Resource limits
    resources:
      requests:
        cpu: 50m
        memory: 64Mi
      limits:
        cpu: 200m
        memory: 256Mi

# -----------------------------------------------------------------------------
# Node Exporter (DaemonSet)
# -----------------------------------------------------------------------------
nodeExporter:
  enabled: true

# -----------------------------------------------------------------------------
# kube-state-metrics
# -----------------------------------------------------------------------------
kubeStateMetrics:
  enabled: true

# -----------------------------------------------------------------------------
# Prometheus Operator
# -----------------------------------------------------------------------------
prometheusOperator:
  enabled: true
  resources:
    requests:
      cpu: 100m
      memory: 100Mi
    limits:
      cpu: 200m
      memory: 200Mi

# -----------------------------------------------------------------------------
# k3s Specific: Disable components not present
# -----------------------------------------------------------------------------
kubeEtcd:
  enabled: false

kubeScheduler:
  enabled: false

kubeControllerManager:
  enabled: false

kubeProxy:
  enabled: false
EOF

helm install prometheus prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --values /tmp/prometheus-values.yaml \
  --dry-run

Error: INSTALLATION FAILED: unable to build kubernetes objects from release manifest:
resource mapping not found for name: "prometheus-kube-prometheus-alertmanager"
no matches for kind "Alertmanager" in version "monitoring.coreos.com/v1"
ensure CRDs are installed first

helm install prometheus prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --values /tmp/prometheus-values.yaml

kubectl -n monitoring get pods -w

NAME                                                     READY   STATUS    RESTARTS   AGE
alertmanager-prometheus-kube-prometheus-alertmanager-0   2/2     Running   0          67s
prometheus-grafana-6dcfb98c6-rjxsm                       3/3     Running   0          83s
prometheus-kube-prometheus-operator-664f9898f9-txw9b     1/1     Running   0          77s
prometheus-kube-state-metrics-7dfddfdf48-5cwlc           1/1     Running   0          77s
prometheus-prometheus-kube-prometheus-prometheus-0       2/2     Running   0          66s
prometheus-prometheus-node-exporter-kp67p                1/1     Running   0          78s

kubectl -n monitoring get secrets prometheus-grafana -o jsonpath="{.data.admin-password}" | base64 -d ; echo

kubectl -n monitoring get pvc

NAME                                             STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS
prometheus-prometheus-kube-prometheus-prom...    Bound    pvc-xxxx                                   50Gi       RWO            nfs-client
alertmanager-prometheus-kube-prometheus-al...    Bound    pvc-xxxx                                   5Gi        RWO            nfs-client
prometheus-grafana                               Bound    pvc-xxxx                                   10Gi       RWO            nfs-client

# Use short hostname (SSH config handles the rest)
ssh nas-01 "ls -la /volume1/k3s/"

drwxrwxrwx+ 1 root root 1012 Feb 22 21:56 .
drwxrwxrwx  1 root root    0 Feb 22 21:56 monitoring-alertmanager-prometheus-kube-prometheus-...
drwxrwxrwx  1  472  472   52 Feb 22 22:01 monitoring-prometheus-grafana-pvc-...
drwxrwxrwx  1 root root   26 Feb 22 21:56 monitoring-prometheus-prometheus-kube-prometheus-...

kubectl -n monitoring get svc | awk '{print $1, $3, $5}'

NAME CLUSTER-IP PORT(S)
alertmanager-operated None 9093/TCP,9094/TCP,9094/UDP
prometheus-grafana 10.43.18.82 80/TCP
prometheus-kube-prometheus-alertmanager 10.43.69.110 9093/TCP,8080/TCP
prometheus-kube-prometheus-operator 10.43.162.126 443/TCP
prometheus-kube-prometheus-prometheus 10.43.205.102 9090/TCP,8080/TCP
prometheus-kube-state-metrics 10.43.147.44 8080/TCP
prometheus-operated None 9090/TCP
prometheus-prometheus-node-exporter 10.43.228.32 9100/TCP

sudo firewall-cmd --add-port=3000/tcp --permanent   # Grafana
sudo firewall-cmd --add-port=9090/tcp --permanent   # Prometheus
sudo firewall-cmd --add-port=9093/tcp --permanent   # AlertManager
sudo firewall-cmd --add-port=9100/tcp --permanent   # Node Exporter (metrics scraping)
sudo firewall-cmd --reload

success
success
success
success
success

nc -zv 10.50.1.120 3000
# Expected: Connection to 10.50.1.120 3000 port [tcp/hbci] succeeded!

# Grafana (localhost:3000)
kubectl -n monitoring port-forward svc/prometheus-grafana 3000:80 --address 0.0.0.0 &

# Prometheus (localhost:9090)
kubectl -n monitoring port-forward svc/prometheus-kube-prometheus-prometheus 9090:9090 --address 0.0.0.0 &

# AlertManager (localhost:9093)
kubectl -n monitoring port-forward svc/prometheus-kube-prometheus-alertmanager 9093:9093 --address 0.0.0.0 &

# Kill old port-forwards
kill %1 %2 %3

# Restart all
kubectl -n monitoring port-forward svc/prometheus-grafana 3000:80 --address 0.0.0.0 &
kubectl -n monitoring port-forward svc/prometheus-kube-prometheus-prometheus 9090:9090 --address 0.0.0.0 &
kubectl -n monitoring port-forward svc/prometheus-kube-prometheus-alertmanager 9093:9093 --address 0.0.0.0 &

scp /tmp/grafana.crt /tmp/grafana.key k3s-master-01.inside.domusdigitalis.dev:/tmp/

scp /tmp/prometheus.crt /tmp/prometheus.key k3s-master-01.inside.domusdigitalis.dev:/tmp/

scp /tmp/alertmanager.crt /tmp/alertmanager.key k3s-master-01.inside.domusdigitalis.dev:/tmp/

kubectl -n monitoring create secret tls grafana-tls \
  --cert=/tmp/grafana.crt \
  --key=/tmp/grafana.key

kubectl -n monitoring create secret tls prometheus-tls \
  --cert=/tmp/prometheus.crt \
  --key=/tmp/prometheus.key

kubectl -n monitoring create secret tls alertmanager-tls \
  --cert=/tmp/alertmanager.crt \
  --key=/tmp/alertmanager.key

kubectl -n monitoring get secrets | grep -E "tls"

alertmanager-tls   kubernetes.io/tls   2      10s
grafana-tls        kubernetes.io/tls   2      15s
prometheus-tls     kubernetes.io/tls   2      12s

cat <<'EOF' | kubectl apply -f -
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
  name: grafana
  namespace: monitoring
spec:
  entryPoints:
    - websecure
  routes:
    - match: Host(`grafana.inside.domusdigitalis.dev`)
      kind: Rule
      services:
        - name: prometheus-grafana
          port: 80
  tls:
    secretName: grafana-tls
EOF

cat <<'EOF' | kubectl apply -f -
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
  name: prometheus
  namespace: monitoring
spec:
  entryPoints:
    - websecure
  routes:
    - match: Host(`prometheus.inside.domusdigitalis.dev`)
      kind: Rule
      services:
        - name: prometheus-kube-prometheus-prometheus
          port: 9090
  tls:
    secretName: prometheus-tls
EOF

cat <<'EOF' | kubectl apply -f -
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
  name: alertmanager
  namespace: monitoring
spec:
  entryPoints:
    - websecure
  routes:
    - match: Host(`alertmanager.inside.domusdigitalis.dev`)
      kind: Rule
      services:
        - name: prometheus-kube-prometheus-alertmanager
          port: 9093
  tls:
    secretName: alertmanager-tls
EOF

kubectl get ingressroute -n monitoring

NAME           AGE
alertmanager   5s
grafana        15s
prometheus     10s

kubectl get ingressroute -n monitoring -o custom-columns=NAME:.metadata.name,HOST:.spec.routes[0].match

NAME           HOST
alertmanager   Host(`alertmanager.inside.domusdigitalis.dev`)
grafana        Host(`grafana.inside.domusdigitalis.dev`)
prometheus     Host(`prometheus.inside.domusdigitalis.dev`)

curl -sS -o /dev/null -w "HTTP: %{http_code}\n" https://grafana.inside.domusdigitalis.dev

HTTP: 200

echo | openssl s_client -connect grafana.inside.domusdigitalis.dev:443 2>/dev/null | \
  openssl x509 -noout -subject -issuer

subject=CN=grafana.inside.domusdigitalis.dev
issuer=CN=DOMUS-ISSUING-CA

for h in grafana prometheus alertmanager; do
  host ${h}.inside.domusdigitalis.dev | awk '\{print $1, $NF\}'
done

grafana.inside.domusdigitalis.dev 10.50.1.120
prometheus.inside.domusdigitalis.dev 10.50.1.120
alertmanager.inside.domusdigitalis.dev 10.50.1.120

ssh bind-01 "sudo nsupdate -l << 'EOF'
zone inside.domusdigitalis.dev
update add grafana.inside.domusdigitalis.dev. 3600 A 10.50.1.120
update add prometheus.inside.domusdigitalis.dev. 3600 A 10.50.1.120
update add alertmanager.inside.domusdigitalis.dev. 3600 A 10.50.1.120
send
EOF"

cat <<'EOF' | kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
  name: custom-dashboard-example
  namespace: monitoring
  labels:
    grafana_dashboard: "1"
data:
  my-dashboard.json: |
    {
      "title": "Custom Dashboard",
      "uid": "custom-example",
      "version": 1,
      "panels": []
    }
EOF

cat <<'EOF' | kubectl apply -f -
apiVersion: v1
kind: Secret
metadata:
  name: alertmanager-prometheus-kube-prometheus-alertmanager
  namespace: monitoring
stringData:
  alertmanager.yaml: |
    global:
      resolve_timeout: 5m
    route:
      group_by: ['alertname', 'namespace']
      group_wait: 30s
      group_interval: 5m
      repeat_interval: 12h
      receiver: 'default'
      routes:
        - match:
            severity: critical
          receiver: 'critical-alerts'
    receivers:
      - name: 'default'
        # No action for default
      - name: 'critical-alerts'
        slack_configs:
          - api_url: 'https://hooks.slack.com/services/YOUR/WEBHOOK/URL'
            channel: '#alerts'
            send_resolved: true
EOF

echo "=== Monitoring Stack Health Check ==="

echo -e "\n[1] Pods Status:"
kubectl -n monitoring get pods -o custom-columns='NAME:.metadata.name,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount'

echo -e "\n[2] PVCs:"
kubectl -n monitoring get pvc -o custom-columns='NAME:.metadata.name,STATUS:.status.phase,CAPACITY:.status.capacity.storage'

echo -e "\n[3] Services:"
kubectl -n monitoring get svc -o custom-columns='NAME:.metadata.name,TYPE:.spec.type,CLUSTER-IP:.spec.clusterIP,PORT:.spec.ports[0].port'

echo -e "\n[4] Prometheus Targets:"
kubectl -n monitoring exec -it prometheus-prometheus-kube-prometheus-prometheus-0 -- wget -qO- http://localhost:9090/api/v1/targets | jq -r '.data.activeTargets | length'

echo -e "\n[5] Grafana Datasources:"
kubectl -n monitoring exec -it deploy/prometheus-grafana -c grafana -- curl -s http://localhost:3000/api/datasources | jq -r '.[].name'

# Query node CPU usage
kubectl -n monitoring exec -it prometheus-prometheus-kube-prometheus-prometheus-0 -- \
  wget -qO- 'http://localhost:9090/api/v1/query?query=node_cpu_seconds_total' | jq -r '.data.result | length'

# Check events
kubectl -n monitoring describe pod <pod-name> | grep -A10 Events

# Common cause: PVC not binding
kubectl -n monitoring get pvc
kubectl describe pvc <pvc-name>

# Test NFS from node
ssh k3s-master-01.inside.domusdigitalis.dev
sudo mount -t nfs 10.50.1.70:/volume1/k3s /mnt/test
ls /mnt/test
sudo umount /mnt/test

# Copy password to clipboard (secure - no screen exposure)
gopass show -c v3/domains/d000/k3s/grafana

# Or show password (use -o only in scripts, not interactively)
gopass show -o v3/domains/d000/k3s/grafana

# Get password from dsec
eval "$(dsec source d000 dev/app)" && echo $K3S_GRAFANA_ADMIN_PASS

# Get current password from k8s secret (what Helm deployed)
kubectl -n monitoring get secret prometheus-grafana -o jsonpath="{.data.admin-password}" | base64 -d; echo

# Reset password (updates k8s only - also update gopass/dsec!)
kubectl -n monitoring exec -it deploy/prometheus-grafana -c grafana -- grafana-cli admin reset-admin-password <new-password>

< Location: http://localhost:3000/grafana/

vi /tmp/prometheus-values.yaml

# Change grafana.ini section to:
  grafana.ini:
    server:
      root_url: "http://{k3s-master-01-ip}:3000"
      serve_from_sub_path: false
    security:
      cookie_secure: false

# Upgrade
helm upgrade prometheus prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --values /tmp/prometheus-values.yaml

# Restart port-forwards (pods restarted)
kill %1 %2 %3
kubectl -n monitoring port-forward svc/prometheus-grafana 3000:80 --address 0.0.0.0 &

# Check ServiceMonitor resources
kubectl get servicemonitors -A

# Check Prometheus config
kubectl -n monitoring exec -it prometheus-prometheus-kube-prometheus-prometheus-0 -- cat /etc/prometheus/prometheus.yml | head -50

vault write -format=json pki_int/issue/domus-client \
  common_name="grafana.inside.domusdigitalis.dev" \
  ttl="8760h" > /tmp/grafana-cert.json

jq -r '.data.certificate' /tmp/grafana-cert.json > /tmp/grafana.crt

jq -r '.data.private_key' /tmp/grafana-cert.json > /tmp/grafana.key

jq -r '.data.ca_chain[]' /tmp/grafana-cert.json > /tmp/grafana-ca.crt

openssl x509 -in /tmp/grafana.crt -noout -subject -issuer -dates

subject=CN=grafana.inside.domusdigitalis.dev
issuer=CN=DOMUS-ISSUING-CA
notBefore=...
notAfter=... (1 year from now)

scp /tmp/grafana.crt /tmp/grafana.key /tmp/grafana-ca.crt k3s-master-01.inside.domusdigitalis.dev:/tmp/

kubectl -n monitoring create secret tls grafana-tls-vault \
  --cert=/tmp/grafana.crt \
  --key=/tmp/grafana.key \
  --dry-run=client -o yaml | kubectl apply -f -

grafana:
  # ... existing settings ...

  # Enable HTTPS with Vault certificate
  grafana.ini:
    server:
      protocol: https
      cert_file: /etc/grafana/tls/tls.crt
      cert_key: /etc/grafana/tls/tls.key
      root_url: "https://grafana.inside.domusdigitalis.dev"
    security:
      cookie_secure: true

  # Mount TLS secret
  extraSecretMounts:
    - name: grafana-tls
      secretName: grafana-tls-vault
      mountPath: /etc/grafana/tls
      readOnly: true

helm upgrade prometheus prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --values /tmp/prometheus-values.yaml

cat << 'EOF' | sudo tee /etc/systemd/system/grafana-pf.service
[Unit]
Description=Grafana Port Forward (HTTPS)
After=network.target k3s.service
Wants=k3s.service

[Service]
Type=simple
Environment="KUBECONFIG=/etc/rancher/k3s/k3s.yaml"
ExecStart=/usr/local/bin/kubectl port-forward -n monitoring svc/prometheus-grafana 3000:443 --address=0.0.0.0
Restart=always
RestartSec=10
User=root

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload
sudo systemctl restart grafana-pf

echo | openssl s_client -connect grafana.inside.domusdigitalis.dev:3000 2>/dev/null | openssl x509 -noout -subject -issuer

subject=CN=grafana.inside.domusdigitalis.dev
issuer=CN=DOMUS-ISSUING-CA