observability: fix dropped kubelet/cadvisor metrics, add KSM + vmalert k8s.rules

The node scrape jobs labelmapped every node label (~50+ NFD labels) onto each series, exceeding vminsert's 40-label default so 100% of kubelet and cadvisor series were silently dropped (vm_rows_ignored_total reason=too_many_labels). kube-state-metrics, VMAlert and the kube-prometheus k8s.rules recording rules were also absent, so dashboard queries like node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate had no producer. - replaces the blanket node labelmap with a single node-name relabel - renames node jobs so series carry job=kubelet (+ metrics_path) matching kube-prometheus selectors - adds kube-state-metrics (v2.20.0 via artifactapi k8s-registry remote) with RBAC, Service and VMServiceScrape - adds VMAlert wired to vmselect/vminsert, recording rules only - adds VMRule with the kube-prometheus k8s.rules recording group
2026-08-23 12:37:04 +10:00
parent 32611bfd16
commit 74354e847d
5 changed files with 376 additions and 6 deletions
@@ -0,0 +1,201 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-state-metrics
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: kube-state-metrics
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics
+  labels:
+    app.kubernetes.io/name: kube-state-metrics
+rules:
+  - apiGroups: [""]
+    resources:
+      - configmaps
+      - secrets
+      - nodes
+      - pods
+      - services
+      - serviceaccounts
+      - resourcequotas
+      - replicationcontrollers
+      - limitranges
+      - persistentvolumeclaims
+      - persistentvolumes
+      - namespaces
+      - endpoints
+    verbs: [list, watch]
+  - apiGroups: [apps]
+    resources:
+      - statefulsets
+      - daemonsets
+      - deployments
+      - replicasets
+    verbs: [list, watch]
+  - apiGroups: [batch]
+    resources:
+      - cronjobs
+      - jobs
+    verbs: [list, watch]
+  - apiGroups: [autoscaling]
+    resources:
+      - horizontalpodautoscalers
+    verbs: [list, watch]
+  - apiGroups: [authentication.k8s.io]
+    resources:
+      - tokenreviews
+    verbs: [create]
+  - apiGroups: [authorization.k8s.io]
+    resources:
+      - subjectaccessreviews
+    verbs: [create]
+  - apiGroups: [policy]
+    resources:
+      - poddisruptionbudgets
+    verbs: [list, watch]
+  - apiGroups: [certificates.k8s.io]
+    resources:
+      - certificatesigningrequests
+    verbs: [list, watch]
+  - apiGroups: [discovery.k8s.io]
+    resources:
+      - endpointslices
+    verbs: [list, watch]
+  - apiGroups: [storage.k8s.io]
+    resources:
+      - storageclasses
+      - volumeattachments
+    verbs: [list, watch]
+  - apiGroups: [admissionregistration.k8s.io]
+    resources:
+      - mutatingwebhookconfigurations
+      - validatingwebhookconfigurations
+    verbs: [list, watch]
+  - apiGroups: [networking.k8s.io]
+    resources:
+      - networkpolicies
+      - ingressclasses
+      - ingresses
+    verbs: [list, watch]
+  - apiGroups: [coordination.k8s.io]
+    resources:
+      - leases
+    verbs: [list, watch]
+  - apiGroups: [rbac.authorization.k8s.io]
+    resources:
+      - clusterrolebindings
+      - clusterroles
+      - rolebindings
+      - roles
+    verbs: [list, watch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-state-metrics
+  labels:
+    app.kubernetes.io/name: kube-state-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-state-metrics
+subjects:
+  - kind: ServiceAccount
+    name: kube-state-metrics
+    namespace: observability
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-state-metrics
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: kube-state-metrics
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kube-state-metrics
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kube-state-metrics
+    spec:
+      serviceAccountName: kube-state-metrics
+      automountServiceAccountToken: true
+      containers:
+        - name: kube-state-metrics
+          image: artifactapi.k8s.syd1.au.unkin.net/k8s-registry/kube-state-metrics/kube-state-metrics:v2.20.0
+          ports:
+            - name: http-metrics
+              containerPort: 8080
+            - name: telemetry
+              containerPort: 8081
+          livenessProbe:
+            httpGet:
+              path: /livez
+              port: http-metrics
+            initialDelaySeconds: 5
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: telemetry
+            initialDelaySeconds: 5
+            timeoutSeconds: 5
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 65534
+            seccompProfile:
+              type: RuntimeDefault
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-state-metrics
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: kube-state-metrics
+spec:
+  clusterIP: None
+  selector:
+    app.kubernetes.io/name: kube-state-metrics
+  ports:
+    - name: http-metrics
+      port: 8080
+      targetPort: http-metrics
+    - name: telemetry
+      port: 8081
+      targetPort: telemetry
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMServiceScrape
+metadata:
+  name: kube-state-metrics
+  namespace: observability
+spec:
+  # jobLabel yields job="kube-state-metrics" as kube-prometheus selectors expect.
+  jobLabel: app.kubernetes.io/name
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kube-state-metrics
+  endpoints:
+    - port: http-metrics
+      honorLabels: true
+    - port: telemetry
@@ -6,6 +6,9 @@ resources:
  - namespace.yaml
  - vmcluster.yaml
  - vmagent.yaml
+  - vmalert.yaml
+  - vmrule-k8s.yaml
+  - kube-state-metrics.yaml
  - gateway.yaml
  - httproute.yaml
  - vmpodscrape-cnpg.yaml
@@ -49,7 +49,8 @@ spec:
            - __meta_kubernetes_endpoint_port_name
          action: keep
          regex: default;kubernetes;https
-    - job_name: "kubernetes-nodes"
+    # Only the node name goes onto series: a blanket node-label labelmap pushes past vminsert's 40-label limit and drops everything.
+    - job_name: "kubelet"
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
@@ -58,9 +59,11 @@ spec:
      kubernetes_sd_configs:
        - role: node
      relabel_configs:
-        - action: labelmap
-          regex: __meta_kubernetes_node_label_(.+)
-    - job_name: "kubernetes-nodes-cadvisor"
+        - source_labels: [__meta_kubernetes_node_name]
+          target_label: node
+        - source_labels: [__metrics_path__]
+          target_label: metrics_path
+    - job_name: "kubelet-cadvisor"
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
@@ -70,10 +73,13 @@ spec:
        - role: node
      metrics_path: /metrics/cadvisor
      relabel_configs:
-        - action: labelmap
-          regex: __meta_kubernetes_node_label_(.+)
+        - source_labels: [__meta_kubernetes_node_name]
+          target_label: node
        - source_labels: [__metrics_path__]
          target_label: metrics_path
+        # job_name must be unique; kube-prometheus selectors expect job="kubelet"
+        - target_label: job
+          replacement: kubelet
      metric_relabel_configs:
        - action: replace
          source_labels: [pod]
@@ -0,0 +1,27 @@
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMAlert
+metadata:
+  name: main
+  namespace: observability
+spec:
+  replicaCount: 1
+  selectAllByDefault: true
+  evaluationInterval: 30s
+  datasource:
+    url: http://vmselect-main.observability.svc.cluster.local:8481/select/0/prometheus/
+  remoteWrite:
+    url: http://vminsert-main.observability.svc.cluster.local:8480/insert/0/prometheus/
+  remoteRead:
+    url: http://vmselect-main.observability.svc.cluster.local:8481/select/0/prometheus/
+  extraArgs:
+    loggerFormat: json
+    # recording rules only for now; no alertmanager deployed
+    notifier.blackhole: "true"
+  resources:
+    requests:
+      cpu: 50m
+      memory: 128Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
@@ -0,0 +1,133 @@
+---
+# kube-prometheus k8s.rules recording rules (dashboard queries depend on these).
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  name: k8s-rules
+  namespace: observability
+spec:
+  groups:
+    - name: k8s.rules
+      rules:
+        - record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate
+          expr: |
+            sum by (cluster, namespace, pod, container) (
+              irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m])
+            ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (
+              1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
+            )
+        - record: node_namespace_pod_container:container_memory_working_set_bytes
+          expr: |
+            container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
+            * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (
+              1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
+            )
+        - record: node_namespace_pod_container:container_memory_rss
+          expr: |
+            container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
+            * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (
+              1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
+            )
+        - record: node_namespace_pod_container:container_memory_cache
+          expr: |
+            container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
+            * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (
+              1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
+            )
+        - record: node_namespace_pod_container:container_memory_swap
+          expr: |
+            container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
+            * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (
+              1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
+            )
+        - record: namespace_memory:kube_pod_container_resource_requests:sum
+          expr: |
+            sum by (namespace, cluster) (
+              sum by (namespace, pod, cluster) (
+                max by (namespace, pod, container, cluster) (
+                  kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"}
+                ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
+                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
+                )
+              )
+            )
+        - record: namespace_cpu:kube_pod_container_resource_requests:sum
+          expr: |
+            sum by (namespace, cluster) (
+              sum by (namespace, pod, cluster) (
+                max by (namespace, pod, container, cluster) (
+                  kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"}
+                ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
+                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
+                )
+              )
+            )
+        - record: namespace_memory:kube_pod_container_resource_limits:sum
+          expr: |
+            sum by (namespace, cluster) (
+              sum by (namespace, pod, cluster) (
+                max by (namespace, pod, container, cluster) (
+                  kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"}
+                ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
+                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
+                )
+              )
+            )
+        - record: namespace_cpu:kube_pod_container_resource_limits:sum
+          expr: |
+            sum by (namespace, cluster) (
+              sum by (namespace, pod, cluster) (
+                max by (namespace, pod, container, cluster) (
+                  kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"}
+                ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
+                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
+                )
+              )
+            )
+        - record: namespace_workload_pod:kube_pod_owner:relabel
+          labels:
+            workload_type: deployment
+          expr: |
+            max by (cluster, namespace, workload, pod) (
+              label_replace(
+                label_replace(
+                  kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"},
+                  "replicaset", "$1", "owner_name", "(.*)"
+                ) * on (replicaset, namespace) group_left(owner_name) topk by (replicaset, namespace) (
+                  1, max by (replicaset, namespace, owner_name) (
+                    kube_replicaset_owner{job="kube-state-metrics"}
+                  )
+                ),
+                "workload", "$1", "owner_name", "(.*)"
+              )
+            )
+        - record: namespace_workload_pod:kube_pod_owner:relabel
+          labels:
+            workload_type: daemonset
+          expr: |
+            max by (cluster, namespace, workload, pod) (
+              label_replace(
+                kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"},
+                "workload", "$1", "owner_name", "(.*)"
+              )
+            )
+        - record: namespace_workload_pod:kube_pod_owner:relabel
+          labels:
+            workload_type: statefulset
+          expr: |
+            max by (cluster, namespace, workload, pod) (
+              label_replace(
+                kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"},
+                "workload", "$1", "owner_name", "(.*)"
+              )
+            )
+        - record: namespace_workload_pod:kube_pod_owner:relabel
+          labels:
+            workload_type: job
+          expr: |
+            max by (cluster, namespace, workload, pod) (
+              label_replace(
+                kube_pod_owner{job="kube-state-metrics", owner_kind="Job"},
+                "workload", "$1", "owner_name", "(.*)"
+              )
+            )