From 8636d16773b61c9015d442767223b4d9a0b43ca0 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 23 Aug 2026 12:35:55 +1000 Subject: [PATCH] Export and scrape metrics from all redis/valkey instances No redis/valkey instance in the estate exports metrics today. Adds a pinned redis_exporter (v1.89.0 via the artifactapi dockerhub remote) next to every instance and two cluster-wide VMPodScrapes so the observability VMAgent picks them all up. - enables the valkey-operator exporter sidecar on the jellyfin ValkeyCluster, overriding the operator's bare-dockerhub default image with the artifactapi-proxied pin - adds a metrics-exporter sidecar (:9121, port name metrics) to the artifactapi/authentik/litellm redis and gitea/netbox valkey Deployments, with the monitoring.unkin.net/redis-exporter opt-in pod label - adds VMPodScrape redis-exporters selecting that label in any namespace, and VMPodScrape valkey-operator-clusters for operator-managed pods whose labels are fixed (the ValkeyCluster CR has no pod-label passthrough) --- apps/base/artifactapi/redis-deployment.yaml | 15 +++++++ apps/base/authentik/redis-deployment.yaml | 15 +++++++ apps/base/gitea/valkey-deployment.yaml | 21 ++++++++++ apps/base/jellyfin/valkey.yaml | 9 ++++- apps/base/litellm/redis-deployment.yaml | 15 +++++++ apps/base/netbox/valkey-deployment.yaml | 21 ++++++++++ apps/base/observability/kustomization.yaml | 1 + .../base/observability/vmpodscrape-redis.yaml | 40 +++++++++++++++++++ 8 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 apps/base/observability/vmpodscrape-redis.yaml diff --git a/apps/base/artifactapi/redis-deployment.yaml b/apps/base/artifactapi/redis-deployment.yaml index c14a555..623baf3 100644 --- a/apps/base/artifactapi/redis-deployment.yaml +++ b/apps/base/artifactapi/redis-deployment.yaml @@ -13,6 +13,7 @@ spec: metadata: labels: app: redis + monitoring.unkin.net/redis-exporter: "true" spec: containers: - name: redis @@ -53,4 +54,18 @@ spec: periodSeconds: 10 successThreshold: 1 timeoutSeconds: 5 + - name: metrics-exporter + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/oliver006/redis_exporter:v1.89.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9121 + name: metrics + protocol: TCP + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 25m + memory: 32Mi restartPolicy: Always diff --git a/apps/base/authentik/redis-deployment.yaml b/apps/base/authentik/redis-deployment.yaml index 2c1d251..777bf02 100644 --- a/apps/base/authentik/redis-deployment.yaml +++ b/apps/base/authentik/redis-deployment.yaml @@ -15,6 +15,7 @@ spec: metadata: labels: app: redis + monitoring.unkin.net/redis-exporter: "true" spec: containers: - name: redis @@ -52,6 +53,20 @@ spec: volumeMounts: - mountPath: /data name: redis-data + - name: metrics-exporter + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/oliver006/redis_exporter:v1.89.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9121 + name: metrics + protocol: TCP + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 25m + memory: 32Mi volumes: - name: redis-data persistentVolumeClaim: diff --git a/apps/base/gitea/valkey-deployment.yaml b/apps/base/gitea/valkey-deployment.yaml index 22725dd..7cb1647 100644 --- a/apps/base/gitea/valkey-deployment.yaml +++ b/apps/base/gitea/valkey-deployment.yaml @@ -27,6 +27,7 @@ spec: app: gitea-valkey app.kubernetes.io/name: gitea app.kubernetes.io/component: valkey + monitoring.unkin.net/redis-exporter: "true" spec: securityContext: fsGroup: 999 @@ -82,6 +83,26 @@ spec: volumeMounts: - mountPath: /data name: data + - name: metrics-exporter + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/oliver006/redis_exporter:v1.89.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9121 + name: metrics + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: + - ALL + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 25m + memory: 32Mi restartPolicy: Always volumes: - name: data diff --git a/apps/base/jellyfin/valkey.yaml b/apps/base/jellyfin/valkey.yaml index 147895b..23ad378 100644 --- a/apps/base/jellyfin/valkey.yaml +++ b/apps/base/jellyfin/valkey.yaml @@ -25,8 +25,15 @@ spec: shards: 1 replicas: 2 image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/valkey/valkey:9.0.0 + # redis_exporter sidecar (:9121, port name `metrics`) on every ValkeyNode pod; + # the operator manages a dedicated _exporter ACL user for it. Image overridden + # from the operator default (bare dockerhub oliver006/redis_exporter:v1.80.0) + # to the artifactapi-proxied pin. Scraped by the valkey-operator VMPodScrape in + # apps/base/observability (operator pods get fixed labels only, so they need + # their own selector rather than monitoring.unkin.net/redis-exporter). exporter: - enabled: false + enabled: true + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/oliver006/redis_exporter:v1.89.0 scheduling: node: spread: diff --git a/apps/base/litellm/redis-deployment.yaml b/apps/base/litellm/redis-deployment.yaml index cf97be0..3ca1a14 100644 --- a/apps/base/litellm/redis-deployment.yaml +++ b/apps/base/litellm/redis-deployment.yaml @@ -15,6 +15,7 @@ spec: metadata: labels: app: redis + monitoring.unkin.net/redis-exporter: "true" spec: containers: - name: redis @@ -60,6 +61,20 @@ spec: - mountPath: /data mountPropagation: None name: data + - name: metrics-exporter + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/oliver006/redis_exporter:v1.89.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9121 + name: metrics + protocol: TCP + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 25m + memory: 32Mi restartPolicy: Always volumes: - name: data diff --git a/apps/base/netbox/valkey-deployment.yaml b/apps/base/netbox/valkey-deployment.yaml index 52f04cf..1efbd71 100644 --- a/apps/base/netbox/valkey-deployment.yaml +++ b/apps/base/netbox/valkey-deployment.yaml @@ -27,6 +27,7 @@ spec: app: netbox-valkey app.kubernetes.io/name: netbox app.kubernetes.io/component: valkey + monitoring.unkin.net/redis-exporter: "true" spec: securityContext: fsGroup: 999 @@ -82,6 +83,26 @@ spec: volumeMounts: - mountPath: /data name: data + - name: metrics-exporter + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/oliver006/redis_exporter:v1.89.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9121 + name: metrics + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: + - ALL + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 25m + memory: 32Mi restartPolicy: Always volumes: - name: data diff --git a/apps/base/observability/kustomization.yaml b/apps/base/observability/kustomization.yaml index f848c26..dfca04f 100644 --- a/apps/base/observability/kustomization.yaml +++ b/apps/base/observability/kustomization.yaml @@ -9,3 +9,4 @@ resources: - gateway.yaml - httproute.yaml - vmpodscrape-cnpg.yaml + - vmpodscrape-redis.yaml diff --git a/apps/base/observability/vmpodscrape-redis.yaml b/apps/base/observability/vmpodscrape-redis.yaml new file mode 100644 index 0000000..80735c6 --- /dev/null +++ b/apps/base/observability/vmpodscrape-redis.yaml @@ -0,0 +1,40 @@ +--- +# Scrape metrics (:9121) from every redis/valkey pod carrying a redis_exporter +# sidecar in any namespace. Raw redis/valkey Deployments opt in via the +# monitoring.unkin.net/redis-exporter pod label. Picked up by the observability +# VMAgent (selectAllByDefault). +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: redis-exporters + namespace: observability +spec: + namespaceSelector: + any: true + selector: + matchLabels: + monitoring.unkin.net/redis-exporter: "true" + podMetricsEndpoints: + - port: metrics + path: /metrics +--- +# valkey-operator-managed ValkeyCluster pods (e.g. jellyfin-valkey) get fixed +# operator labels only -- the ValkeyCluster CR has no pod-label passthrough, so +# they cannot carry monitoring.unkin.net/redis-exporter and need their own +# selector. The operator's exporter sidecar also listens on :9121 with a port +# named `metrics`. +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: valkey-operator-clusters + namespace: observability +spec: + namespaceSelector: + any: true + selector: + matchLabels: + app.kubernetes.io/managed-by: valkey-operator + app.kubernetes.io/component: valkey-node + podMetricsEndpoints: + - port: metrics + path: /metrics