From e608dadae74b4ce8c5825b8c58fb7392bf51724a Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 23 Aug 2026 21:38:06 +1000 Subject: [PATCH] observability: fix dropped kubelet/cadvisor metrics, add kube-state-metrics + VMAlert k8s.rules (#399) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why Kubernetes container metrics are entirely missing from VictoriaMetrics, leaving the CNPG dashboard CPU panels empty. The `kubernetes-nodes`/`kubernetes-nodes-cadvisor` scrape jobs use a blanket `labelmap` on `__meta_kubernetes_node_label_(.+)`, copying ~50+ node-feature-discovery labels onto every series; every kubelet/cadvisor series then exceeds vminsert's 40-label default and 100% are silently dropped (`vm_rows_ignored_total{reason="too_many_labels"}` ~38B). On top of that, kube-state-metrics, VMAlert and the kube-prometheus `k8s.rules` recording rules were never deployed, so `node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate` (and companions) have no producer. ## Changes - replaces the blanket node-label `labelmap` in both node scrape jobs with a single `__meta_kubernetes_node_name -> node` relabel - renames the node jobs so series carry `job="kubelet"` plus `metrics_path` (`/metrics` and `/metrics/cadvisor`), matching kube-prometheus selectors; the cadvisor job keeps a distinct `job_name` (uniqueness requirement) and forces `job=kubelet` via relabel - adds kube-state-metrics v2.20.0 (Deployment, standard ClusterRole/Binding, headless Service, VMServiceScrape with `jobLabel` yielding `job="kube-state-metrics"`), image routed through the artifactapi `k8s-registry` remote - adds a VMAlert CR (datasource vmselect-main, remoteWrite/remoteRead vminsert/vmselect-main, `notifier.blackhole` since it only evaluates recording rules) - adds a VMRule with the kube-prometheus `k8s.rules` recording group (cpu sum_irate, memory working_set/rss/cache/swap, namespace requests/limits sums, `kube_pod_owner` workload relabel rules) ## Prerequisite The artifactapi `k8s-registry` remote exists but its `patterns` allowlist (`^autoscaling/vpa-`, `^external-dns/`, `^sig-storage/`) does not cover `kube-state-metrics/` — terraform-artifactapi needs `^kube-state-metrics/` added to `config/remote_docker/k8s-registry.yaml` before/with this merge, or the KSM pod will ImagePullBackOff. ## Notes - validated with `kustomize build` (base + au-syd1 overlay) and kubeconform (VM CRD schemas skipped) - the grafana CNPG dashboard also has stale datasource uids — separate follow-up, deliberately not touched here Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/399 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- .../observability/kube-state-metrics.yaml | 201 ++++++++++++++++++ apps/base/observability/kustomization.yaml | 3 + apps/base/observability/vmagent.yaml | 18 +- apps/base/observability/vmalert.yaml | 27 +++ apps/base/observability/vmrule-k8s.yaml | 133 ++++++++++++ 5 files changed, 376 insertions(+), 6 deletions(-) create mode 100644 apps/base/observability/kube-state-metrics.yaml create mode 100644 apps/base/observability/vmalert.yaml create mode 100644 apps/base/observability/vmrule-k8s.yaml diff --git a/apps/base/observability/kube-state-metrics.yaml b/apps/base/observability/kube-state-metrics.yaml new file mode 100644 index 0000000..7f7db9d --- /dev/null +++ b/apps/base/observability/kube-state-metrics.yaml @@ -0,0 +1,201 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-state-metrics + namespace: observability + labels: + app.kubernetes.io/name: kube-state-metrics +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics + labels: + app.kubernetes.io/name: kube-state-metrics +rules: + - apiGroups: [""] + resources: + - configmaps + - secrets + - nodes + - pods + - services + - serviceaccounts + - resourcequotas + - replicationcontrollers + - limitranges + - persistentvolumeclaims + - persistentvolumes + - namespaces + - endpoints + verbs: [list, watch] + - apiGroups: [apps] + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: [list, watch] + - apiGroups: [batch] + resources: + - cronjobs + - jobs + verbs: [list, watch] + - apiGroups: [autoscaling] + resources: + - horizontalpodautoscalers + verbs: [list, watch] + - apiGroups: [authentication.k8s.io] + resources: + - tokenreviews + verbs: [create] + - apiGroups: [authorization.k8s.io] + resources: + - subjectaccessreviews + verbs: [create] + - apiGroups: [policy] + resources: + - poddisruptionbudgets + verbs: [list, watch] + - apiGroups: [certificates.k8s.io] + resources: + - certificatesigningrequests + verbs: [list, watch] + - apiGroups: [discovery.k8s.io] + resources: + - endpointslices + verbs: [list, watch] + - apiGroups: [storage.k8s.io] + resources: + - storageclasses + - volumeattachments + verbs: [list, watch] + - apiGroups: [admissionregistration.k8s.io] + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: [list, watch] + - apiGroups: [networking.k8s.io] + resources: + - networkpolicies + - ingressclasses + - ingresses + verbs: [list, watch] + - apiGroups: [coordination.k8s.io] + resources: + - leases + verbs: [list, watch] + - apiGroups: [rbac.authorization.k8s.io] + resources: + - clusterrolebindings + - clusterroles + - rolebindings + - roles + verbs: [list, watch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-state-metrics + labels: + app.kubernetes.io/name: kube-state-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-state-metrics +subjects: + - kind: ServiceAccount + name: kube-state-metrics + namespace: observability +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-state-metrics + namespace: observability + labels: + app.kubernetes.io/name: kube-state-metrics +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + template: + metadata: + labels: + app.kubernetes.io/name: kube-state-metrics + spec: + serviceAccountName: kube-state-metrics + automountServiceAccountToken: true + containers: + - name: kube-state-metrics + image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.20.0 + ports: + - name: http-metrics + containerPort: 8080 + - name: telemetry + containerPort: 8081 + livenessProbe: + httpGet: + path: /livez + port: http-metrics + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /readyz + port: telemetry + initialDelaySeconds: 5 + timeoutSeconds: 5 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-state-metrics + namespace: observability + labels: + app.kubernetes.io/name: kube-state-metrics +spec: + clusterIP: None + selector: + app.kubernetes.io/name: kube-state-metrics + ports: + - name: http-metrics + port: 8080 + targetPort: http-metrics + - name: telemetry + port: 8081 + targetPort: telemetry +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMServiceScrape +metadata: + name: kube-state-metrics + namespace: observability +spec: + # jobLabel yields job="kube-state-metrics" as kube-prometheus selectors expect. + jobLabel: app.kubernetes.io/name + selector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + endpoints: + - port: http-metrics + honorLabels: true + - port: telemetry diff --git a/apps/base/observability/kustomization.yaml b/apps/base/observability/kustomization.yaml index 53d138e..93d5367 100644 --- a/apps/base/observability/kustomization.yaml +++ b/apps/base/observability/kustomization.yaml @@ -6,6 +6,9 @@ resources: - namespace.yaml - vmcluster.yaml - vmagent.yaml + - vmalert.yaml + - vmrule-k8s.yaml + - kube-state-metrics.yaml - gateway.yaml - httproute.yaml - vmpodscrape-cnpg.yaml diff --git a/apps/base/observability/vmagent.yaml b/apps/base/observability/vmagent.yaml index 7027954..0a2d2b5 100644 --- a/apps/base/observability/vmagent.yaml +++ b/apps/base/observability/vmagent.yaml @@ -49,7 +49,8 @@ spec: - __meta_kubernetes_endpoint_port_name action: keep regex: default;kubernetes;https - - job_name: "kubernetes-nodes" + # Only the node name goes onto series: a blanket node-label labelmap pushes past vminsert's 40-label limit and drops everything. + - job_name: "kubelet" scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt @@ -58,9 +59,11 @@ spec: kubernetes_sd_configs: - role: node relabel_configs: - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) - - job_name: "kubernetes-nodes-cadvisor" + - source_labels: [__meta_kubernetes_node_name] + target_label: node + - source_labels: [__metrics_path__] + target_label: metrics_path + - job_name: "kubelet-cadvisor" scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt @@ -70,10 +73,13 @@ spec: - role: node metrics_path: /metrics/cadvisor relabel_configs: - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) + - source_labels: [__meta_kubernetes_node_name] + target_label: node - source_labels: [__metrics_path__] target_label: metrics_path + # job_name must be unique; kube-prometheus selectors expect job="kubelet" + - target_label: job + replacement: kubelet metric_relabel_configs: - action: replace source_labels: [pod] diff --git a/apps/base/observability/vmalert.yaml b/apps/base/observability/vmalert.yaml new file mode 100644 index 0000000..930210a --- /dev/null +++ b/apps/base/observability/vmalert.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMAlert +metadata: + name: main + namespace: observability +spec: + replicaCount: 1 + selectAllByDefault: true + evaluationInterval: 30s + datasource: + url: http://vmselect-main.observability.svc.cluster.local:8481/select/0/prometheus/ + remoteWrite: + url: http://vminsert-main.observability.svc.cluster.local:8480/insert/0/prometheus/ + remoteRead: + url: http://vmselect-main.observability.svc.cluster.local:8481/select/0/prometheus/ + extraArgs: + loggerFormat: json + # recording rules only for now; no alertmanager deployed + notifier.blackhole: "true" + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi diff --git a/apps/base/observability/vmrule-k8s.yaml b/apps/base/observability/vmrule-k8s.yaml new file mode 100644 index 0000000..623e45f --- /dev/null +++ b/apps/base/observability/vmrule-k8s.yaml @@ -0,0 +1,133 @@ +--- +# kube-prometheus k8s.rules recording rules (dashboard queries depend on these). +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMRule +metadata: + name: k8s-rules + namespace: observability +spec: + groups: + - name: k8s.rules + rules: + - record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + expr: | + sum by (cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + - record: node_namespace_pod_container:container_memory_working_set_bytes + expr: | + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + - record: node_namespace_pod_container:container_memory_rss + expr: | + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + - record: node_namespace_pod_container:container_memory_cache + expr: | + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + - record: node_namespace_pod_container:container_memory_swap + expr: | + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + - record: namespace_memory:kube_pod_container_resource_requests:sum + expr: | + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} + ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + - record: namespace_cpu:kube_pod_container_resource_requests:sum + expr: | + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} + ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + - record: namespace_memory:kube_pod_container_resource_limits:sum + expr: | + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} + ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + - record: namespace_cpu:kube_pod_container_resource_limits:sum + expr: | + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} + ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + - record: namespace_workload_pod:kube_pod_owner:relabel + labels: + workload_type: deployment + expr: | + max by (cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on (replicaset, namespace) group_left(owner_name) topk by (replicaset, namespace) ( + 1, max by (replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="kube-state-metrics"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + - record: namespace_workload_pod:kube_pod_owner:relabel + labels: + workload_type: daemonset + expr: | + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + - record: namespace_workload_pod:kube_pod_owner:relabel + labels: + workload_type: statefulset + expr: | + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + - record: namespace_workload_pod:kube_pod_owner:relabel + labels: + workload_type: job + expr: | + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + )