From b3d202b8fac545fba6960a7c13075c784ce5bc03 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 23 Aug 2026 12:35:56 +1000 Subject: [PATCH 1/4] arrstack: add exportarr metrics sidecars to sonarr and radarr (#395) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sonarr and radarr currently expose no Prometheus metrics, so queue depth, indexer health, and library stats are invisible to the observability stack. This adds the standard exportarr exporter as a sidecar to both Deployments. - Adds an exportarr sidecar (ghcr.io/onedr0p/exportarr v2.3.0, pulled through the artifactapi `ghcr` remote, which already whitelists `^onedr0p/`) to the sonarr and radarr Deployments - Points each exporter at its local replica via env (`URL=http://localhost:8989/sonarr` / `:7878/radarr`, matching the apps' UrlBase) and reuses the existing VSO-synced API-key Secrets (`sonarr-apikey`/`radarr-apikey`, key `apitoken`) via `APIKEY` secretKeyRef — no key in argv - Serves metrics on 9707 (sonarr) / 9708 (radarr) with `/healthz` liveness+readiness probes and small resources (25m/32Mi requests, 100m/128Mi limits) - Adds per-app VMPodScrape objects (picked up by the observability VMAgent, selectAllByDefault); pod-level rather than VMServiceScrape because the app Services don't expose the metrics port Validated with `kustomize build apps/base/arrstack` + kubeconform (0 invalid). Note: with 3 replicas per app, each pod's exporter reports the same shared-Postgres-backed stats, so series are duplicated across the `pod` label; dashboards should aggregate with `max` or filter to one pod. Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/395 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- apps/base/arrstack/radarr/deployment.yaml | 45 ++++++++++++++++++++ apps/base/arrstack/radarr/kustomization.yaml | 1 + apps/base/arrstack/radarr/vmpodscrape.yaml | 16 +++++++ apps/base/arrstack/sonarr/deployment.yaml | 45 ++++++++++++++++++++ apps/base/arrstack/sonarr/kustomization.yaml | 1 + apps/base/arrstack/sonarr/vmpodscrape.yaml | 16 +++++++ 6 files changed, 124 insertions(+) create mode 100644 apps/base/arrstack/radarr/vmpodscrape.yaml create mode 100644 apps/base/arrstack/sonarr/vmpodscrape.yaml diff --git a/apps/base/arrstack/radarr/deployment.yaml b/apps/base/arrstack/radarr/deployment.yaml index 39e8dbb..b64079b 100644 --- a/apps/base/arrstack/radarr/deployment.yaml +++ b/apps/base/arrstack/radarr/deployment.yaml @@ -182,6 +182,51 @@ spec: - name: vault-ca mountPath: /etc/ssl/vault-ca readOnly: true + # exportarr sidecar: polls the local replica's API and exposes Prometheus + # metrics on :9708 (scraped by the radarr-exportarr VMPodScrape). + - name: exportarr + image: artifactapi.k8s.syd1.au.unkin.net/ghcr/onedr0p/exportarr:v2.3.0 + imagePullPolicy: IfNotPresent + args: + - radarr + env: + - name: PORT + value: "9708" + # URL includes the /radarr UrlBase (Radarr__Server__UrlBase). + - name: URL + value: http://localhost:7878/radarr + - name: APIKEY + valueFrom: + secretKeyRef: + name: radarr-apikey + key: apitoken + ports: + - name: metrics + containerPort: 9708 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: metrics + initialDelaySeconds: 15 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthz + port: metrics + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi volumes: - name: config emptyDir: {} diff --git a/apps/base/arrstack/radarr/kustomization.yaml b/apps/base/arrstack/radarr/kustomization.yaml index faed5e3..8789f80 100644 --- a/apps/base/arrstack/radarr/kustomization.yaml +++ b/apps/base/arrstack/radarr/kustomization.yaml @@ -9,3 +9,4 @@ resources: - service.yaml - gateway.yaml - httproute.yaml + - vmpodscrape.yaml diff --git a/apps/base/arrstack/radarr/vmpodscrape.yaml b/apps/base/arrstack/radarr/vmpodscrape.yaml new file mode 100644 index 0000000..097f571 --- /dev/null +++ b/apps/base/arrstack/radarr/vmpodscrape.yaml @@ -0,0 +1,16 @@ +--- +# Scrape the exportarr sidecar (:9708) on every radarr pod. Picked up by the +# observability VMAgent (selectAllByDefault). Pod-level rather than +# VMServiceScrape because the radarr Service doesn't expose the metrics port. +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: radarr-exportarr + namespace: arrstack +spec: + selector: + matchLabels: + app: radarr + podMetricsEndpoints: + - port: metrics + path: /metrics diff --git a/apps/base/arrstack/sonarr/deployment.yaml b/apps/base/arrstack/sonarr/deployment.yaml index 05d0551..f2331bc 100644 --- a/apps/base/arrstack/sonarr/deployment.yaml +++ b/apps/base/arrstack/sonarr/deployment.yaml @@ -184,6 +184,51 @@ spec: - name: vault-ca mountPath: /etc/ssl/vault-ca readOnly: true + # exportarr sidecar: polls the local replica's API and exposes Prometheus + # metrics on :9707 (scraped by the sonarr-exportarr VMPodScrape). + - name: exportarr + image: artifactapi.k8s.syd1.au.unkin.net/ghcr/onedr0p/exportarr:v2.3.0 + imagePullPolicy: IfNotPresent + args: + - sonarr + env: + - name: PORT + value: "9707" + # URL includes the /sonarr UrlBase (Sonarr__Server__UrlBase). + - name: URL + value: http://localhost:8989/sonarr + - name: APIKEY + valueFrom: + secretKeyRef: + name: sonarr-apikey + key: apitoken + ports: + - name: metrics + containerPort: 9707 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: metrics + initialDelaySeconds: 15 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthz + port: metrics + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi volumes: - name: config emptyDir: {} diff --git a/apps/base/arrstack/sonarr/kustomization.yaml b/apps/base/arrstack/sonarr/kustomization.yaml index faed5e3..8789f80 100644 --- a/apps/base/arrstack/sonarr/kustomization.yaml +++ b/apps/base/arrstack/sonarr/kustomization.yaml @@ -9,3 +9,4 @@ resources: - service.yaml - gateway.yaml - httproute.yaml + - vmpodscrape.yaml diff --git a/apps/base/arrstack/sonarr/vmpodscrape.yaml b/apps/base/arrstack/sonarr/vmpodscrape.yaml new file mode 100644 index 0000000..82f61f4 --- /dev/null +++ b/apps/base/arrstack/sonarr/vmpodscrape.yaml @@ -0,0 +1,16 @@ +--- +# Scrape the exportarr sidecar (:9707) on every sonarr pod. Picked up by the +# observability VMAgent (selectAllByDefault). Pod-level rather than +# VMServiceScrape because the sonarr Service doesn't expose the metrics port. +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: sonarr-exportarr + namespace: arrstack +spec: + selector: + matchLabels: + app: sonarr + podMetricsEndpoints: + - port: metrics + path: /metrics From 19f7afac92e791d06b4c6f221061f3f52e552f38 Mon Sep 17 00:00:00 2001 From: Unkin Agent Date: Sun, 23 Aug 2026 12:37:42 +1000 Subject: [PATCH 2/4] Add shared arrstack Valkey and wire arr apps' Redis features (#394) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Activates the arr forks (sonarr/radarr/prowlarr) #14 Redis features — SignalR backplane, cross-replica cache-invalidation bus, and distributed rate limiter — which ship deployed but inert because no Valkey exists in arrstack and nothing is wired to it. Ben wants ONE Valkey shared by all three apps. ## Changes - Adds a single shared `ValkeyCluster` (`arrstack-valkey`) under `apps/base/arrstack/valkey/`, modeled on `jellyfin-valkey`: `shards:1`/`replicas:2` HA (one primary + two replicas, automatic failover), image via artifactapi (`artifactapi.k8s.syd1.au.unkin.net/dockerhub/valkey/valkey:9.0.0`), operator-default passwordless `default` user, node spread across hosts, cluster-aware PDB, ephemeral `/data`. - Registers the `valkey` component in the arrstack base kustomization. - Wires `__Redis__Host` + `__Redis__Port` into the `sonarr-env` / `radarr-env` / `prowlarr-env` ConfigMaps, all pointing at the shared service `valkey-arrstack-valkey.arrstack.svc.cluster.local:6379`. ## Notes - Setting `Host` is the activation switch: the fork's `RedisOptions.IsConfigured` gates purely on a non-empty `Host`, so there is no separate Enabled flag. - Passwordless (jellyfin parity — the operator leaves the `default` user without auth), so no `Password`/`Ssl` is wired and there is no secret to reference. - One cluster is safe for all three: each fork namespaces its keys and pub/sub channels by a per-app prefix (`sonarr:ratelimit:` / `radarr:ratelimit:` / `prowlarr:ratelimit:`), so their state never collides. - App `image:` (-unkin5), the waitfordb initContainer, and S3 buckets are untouched. - Validated: `kubectl kustomize` renders clean for both `apps/base/arrstack` and `apps/overlays/au-syd1/arrstack`. Follow-up: after merge, Valkey must come up and the three apps must roll (pick up the new env) before the #14 features can be validated live. --------- Co-authored-by: Ben Vincent Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/394 Co-authored-by: Unkin Agent Co-committed-by: Unkin Agent --- apps/base/arrstack/kustomization.yaml | 1 + apps/base/arrstack/prowlarr/configmap.yaml | 9 ++++ apps/base/arrstack/prowlarr/deployment.yaml | 7 +++ apps/base/arrstack/radarr/configmap.yaml | 9 ++++ apps/base/arrstack/radarr/deployment.yaml | 7 +++ apps/base/arrstack/sonarr/configmap.yaml | 9 ++++ apps/base/arrstack/sonarr/deployment.yaml | 7 +++ apps/base/arrstack/valkey/kustomization.yaml | 6 +++ apps/base/arrstack/valkey/valkeycluster.yaml | 47 ++++++++++++++++++++ 9 files changed, 102 insertions(+) create mode 100644 apps/base/arrstack/valkey/kustomization.yaml create mode 100644 apps/base/arrstack/valkey/valkeycluster.yaml diff --git a/apps/base/arrstack/kustomization.yaml b/apps/base/arrstack/kustomization.yaml index 762a342..2798a6b 100644 --- a/apps/base/arrstack/kustomization.yaml +++ b/apps/base/arrstack/kustomization.yaml @@ -14,6 +14,7 @@ resources: - media-bucket.yaml - backups-bucket.yaml - postgres + - valkey - sonarr - radarr - prowlarr diff --git a/apps/base/arrstack/prowlarr/configmap.yaml b/apps/base/arrstack/prowlarr/configmap.yaml index 353c02b..18b943d 100644 --- a/apps/base/arrstack/prowlarr/configmap.yaml +++ b/apps/base/arrstack/prowlarr/configmap.yaml @@ -22,3 +22,12 @@ data: Prowlarr__Server__Port: "9696" Prowlarr__Server__UrlBase: /prowlarr Prowlarr__Update__Mechanism: External + # Shared arrstack Valkey (valkey-operator). Setting Host is what activates the + # fork's #14 Redis features (SignalR backplane, cross-replica cache-invalidation + # bus, distributed rate limiter): RedisOptions.IsConfigured gates purely on a + # non-empty Host, so there is no separate Enabled flag. The operator leaves the + # default user passwordless (jellyfin parity), so no Password/Ssl is wired. + # Channels/keys are namespaced by this fork's prowlarr:ratelimit: prefix, so the + # one cluster is safe to share with sonarr/radarr. + Prowlarr__Redis__Host: valkey-arrstack-valkey.arrstack.svc.cluster.local + Prowlarr__Redis__Port: "6379" diff --git a/apps/base/arrstack/prowlarr/deployment.yaml b/apps/base/arrstack/prowlarr/deployment.yaml index bdc3499..6e41040 100644 --- a/apps/base/arrstack/prowlarr/deployment.yaml +++ b/apps/base/arrstack/prowlarr/deployment.yaml @@ -4,6 +4,13 @@ kind: Deployment metadata: name: prowlarr namespace: arrstack + annotations: + # prowlarr-env is a plain (unhashed) ConfigMap consumed by fixed-name envFrom, + # so editing it does not roll the Deployment on its own. Reloader watches the + # referenced ConfigMap and triggers a rolling restart on change, so adding the + # Redis env activates the #14 features on the next ArgoCD sync without a manual + # `rollout restart`. + configmap.reloader.stakater.com/auto: "true" spec: # Active-active: the -unkin2 fork keeps all state in the shared Postgres # (arrstack-postgres) and coordinates via Postgres advisory locks, so N diff --git a/apps/base/arrstack/radarr/configmap.yaml b/apps/base/arrstack/radarr/configmap.yaml index 8678ced..17eedce 100644 --- a/apps/base/arrstack/radarr/configmap.yaml +++ b/apps/base/arrstack/radarr/configmap.yaml @@ -22,3 +22,12 @@ data: Radarr__Server__Port: "7878" Radarr__Server__UrlBase: /radarr Radarr__Update__Mechanism: External + # Shared arrstack Valkey (valkey-operator). Setting Host is what activates the + # fork's #14 Redis features (SignalR backplane, cross-replica cache-invalidation + # bus, distributed rate limiter): RedisOptions.IsConfigured gates purely on a + # non-empty Host, so there is no separate Enabled flag. The operator leaves the + # default user passwordless (jellyfin parity), so no Password/Ssl is wired. + # Channels/keys are namespaced by this fork's radarr:ratelimit: prefix, so the + # one cluster is safe to share with sonarr/prowlarr. + Radarr__Redis__Host: valkey-arrstack-valkey.arrstack.svc.cluster.local + Radarr__Redis__Port: "6379" diff --git a/apps/base/arrstack/radarr/deployment.yaml b/apps/base/arrstack/radarr/deployment.yaml index b64079b..e3120bb 100644 --- a/apps/base/arrstack/radarr/deployment.yaml +++ b/apps/base/arrstack/radarr/deployment.yaml @@ -4,6 +4,13 @@ kind: Deployment metadata: name: radarr namespace: arrstack + annotations: + # radarr-env is a plain (unhashed) ConfigMap consumed by fixed-name envFrom, + # so editing it does not roll the Deployment on its own. Reloader watches the + # referenced ConfigMap and triggers a rolling restart on change, so adding the + # Redis env activates the #14 features on the next ArgoCD sync without a manual + # `rollout restart`. + configmap.reloader.stakater.com/auto: "true" spec: # Active-active: the -unkin2 fork keeps all state in the shared Postgres # (arrstack-postgres) and coordinates via Postgres advisory locks, so N diff --git a/apps/base/arrstack/sonarr/configmap.yaml b/apps/base/arrstack/sonarr/configmap.yaml index e837b0e..dd2242c 100644 --- a/apps/base/arrstack/sonarr/configmap.yaml +++ b/apps/base/arrstack/sonarr/configmap.yaml @@ -22,3 +22,12 @@ data: Sonarr__Server__Port: "8989" Sonarr__Server__UrlBase: /sonarr Sonarr__Update__Mechanism: External + # Shared arrstack Valkey (valkey-operator). Setting Host is what activates the + # fork's #14 Redis features (SignalR backplane, cross-replica cache-invalidation + # bus, distributed rate limiter): RedisOptions.IsConfigured gates purely on a + # non-empty Host, so there is no separate Enabled flag. The operator leaves the + # default user passwordless (jellyfin parity), so no Password/Ssl is wired. + # Channels/keys are namespaced by this fork's sonarr:ratelimit: prefix, so the + # one cluster is safe to share with radarr/prowlarr. + Sonarr__Redis__Host: valkey-arrstack-valkey.arrstack.svc.cluster.local + Sonarr__Redis__Port: "6379" diff --git a/apps/base/arrstack/sonarr/deployment.yaml b/apps/base/arrstack/sonarr/deployment.yaml index f2331bc..18cd15f 100644 --- a/apps/base/arrstack/sonarr/deployment.yaml +++ b/apps/base/arrstack/sonarr/deployment.yaml @@ -4,6 +4,13 @@ kind: Deployment metadata: name: sonarr namespace: arrstack + annotations: + # sonarr-env is a plain (unhashed) ConfigMap consumed by fixed-name envFrom, + # so editing it does not roll the Deployment on its own. Reloader watches the + # referenced ConfigMap and triggers a rolling restart on change, so adding the + # Redis env activates the #14 features on the next ArgoCD sync without a manual + # `rollout restart`. + configmap.reloader.stakater.com/auto: "true" spec: # Active-active: the -unkin2 fork keeps all state in the shared Postgres # (arrstack-postgres) and coordinates via Postgres advisory locks, so N diff --git a/apps/base/arrstack/valkey/kustomization.yaml b/apps/base/arrstack/valkey/kustomization.yaml new file mode 100644 index 0000000..f4160f4 --- /dev/null +++ b/apps/base/arrstack/valkey/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - valkeycluster.yaml diff --git a/apps/base/arrstack/valkey/valkeycluster.yaml b/apps/base/arrstack/valkey/valkeycluster.yaml new file mode 100644 index 0000000..2c107df --- /dev/null +++ b/apps/base/arrstack/valkey/valkeycluster.yaml @@ -0,0 +1,47 @@ +--- +# Single shared HA Valkey for the arr apps (sonarr/radarr/prowlarr), managed by +# valkey-operator. It activates the fork's #14 Redis features: the SignalR +# backplane, the cross-replica cache-invalidation bus, and the distributed rate +# limiter. One cluster is safe for all three because each fork namespaces its +# keys and pub/sub channels by a per-app prefix (sonarr:ratelimit: / +# radarr:ratelimit: / prowlarr:ratelimit:), so their state never collides. +# +# Modeled on jellyfin-valkey: shards:1 + replicas:2 is one primary with two +# replicas in a single shard group (three ValkeyNodes total); losing the primary +# triggers an automatic failover so a node/pod loss no longer drops the shared +# state the app replicas coordinate through. The operator runs Valkey +# cluster-mode-enabled with protected-mode off and leaves the built-in `default` +# user passwordless, so clients connect with no auth/TLS; StackExchange.Redis +# seeds off the single service and auto-discovers topology plus failovers. +# scheduling.node.spread.shard:Required keeps the three nodes on distinct hosts, +# so one host loss removes at most one node; podDisruptionBudget.mode:Cluster +# lets the operator manage a quorum-aware PDB. Persistence is omitted (/data is an +# emptyDir): the coordination state is ephemeral (short TTLs / transient pub/sub), +# replication+failover already provide redundancy, and an operator-managed PVC +# cannot carry the k8up.io/backup:"false" annotation the namespace k8up Schedule +# needs to skip in-use RWO volumes. +apiVersion: valkey.io/v1alpha1 +kind: ValkeyCluster +metadata: + name: arrstack-valkey + namespace: arrstack +spec: + shards: 1 + replicas: 2 + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/valkey/valkey:9.0.0 + exporter: + enabled: false + scheduling: + node: + spread: + shard: + mode: Required + podDisruptionBudget: + mode: Cluster + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi From 8fd1d83b1bca201fea8d650d4ced2900d40107b3 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 23 Aug 2026 12:38:55 +1000 Subject: [PATCH 3/4] Add Redis/Valkey Grafana dashboard (#396) ## Why The grafana-operator estate has no dashboard for Redis/Valkey; redis_exporter instances are being added in a parallel PR and need visibility. ## Changes - Adds `apps/base/grafana/dashboards/redis.yaml`, a GrafanaDashboard CR embedding grafana.com dashboard 763 ("Redis Dashboard for Prometheus Redis Exporter 1.x") as gzipJson like the sibling dashboards, with namespace/instance template variables over `redis_up` for multi-instance use and all datasource refs pinned to the VictoriaMetrics datasource uid (`det2y55dac4jkc`), registered in the grafana kustomization Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/396 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- apps/base/grafana/dashboards/redis.yaml | 13 +++++++++++++ apps/base/grafana/kustomization.yaml | 1 + 2 files changed, 14 insertions(+) create mode 100644 apps/base/grafana/dashboards/redis.yaml diff --git a/apps/base/grafana/dashboards/redis.yaml b/apps/base/grafana/dashboards/redis.yaml new file mode 100644 index 0000000..b2d7f72 --- /dev/null +++ b/apps/base/grafana/dashboards/redis.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: redis + namespace: grafana +spec: + instanceSelector: + matchLabels: + dashboards: "grafana" + resyncPeriod: 5m + allowCrossNamespaceImport: false + gzipJson: H4sIAAAAAAAC/+1cbVPbOhb+K17vnZ2yY6jtkCYw0w9Ay1xmC+1d2u7sQiej2EqiYlteWYaETO5vv+dIsmPzmgQItM0HTCzr5eg55zxHL7bGdqdDIxrTRGb29nji2CRJuCSS8QQT7Ihl0t4+GdvdnEXyILG3PccOiSQZz0VAMYscpfC/mujYOQshqS9IjyTEhmppQroRZJMip449YGH5mwU82eMRF1BA9LvkletYvufBpdl0LG8NaktIjC3sTEWz/mHtRFSA0M60/WzQ5USE9uQbNBjSLBAsxdzw8N80ZJn1rshi9biwPgkeUzmgeWbpx++HKReSCsvbGEK9kCarQvdYFpDov5SIY0mEPOSJHNjbrmP3EyoPoL+tNw0H+5wOPnMeSZYqsBCJJI8iB7BMzgDUk2/485we8Qt7u0eiDOpOSUKjTAF9I7hpKWsJbkilP2o2QxJsfj8LEOMeo1G4x5Me62PRkPZIHim92oEGeAx5hjSswP2qAUh7PmDubTUQ65ijanQ+W+EYsJigbNDTmKQpS/paUJ6WVhITGQAWNvYT6hA0g4aV+HQI/+2j1zv2ZFKqKkuhUhKBoiBpALkHPAp1Rbp10s14lEu0pEzSVLdn+gAIU5rAk3MS5VRjO3GmjwXIXT5su8oY8oShGAo8+HkMejRqhYf8nAoBBqlUM0ENsvATV+JAn1qODWoCxQ4VAiO4TrRSt2oajcnwHWjuE2fKlTwXMlcQUtId6t4lPMGeKUsxSURQAknf80yy3qhIzCWHRC4YuCcxljyA20swPaKBDvOAfqw0Q6IABbIjkskjLo9QId+MacADu4AmM6YHfQH0Lz5RsLdE7g1I0qelVaL26rJcAE4fyIjnsoQvjfI+S75SkWkBPXejsdFAvySiT+VDjZoOU9QqwPsK/jqorI5kMX0l0Gc7eYo3HZZ0MgpEEmZjlmSSJAF9++ep/Vtxc2pPTn7rQDZwb+j/tzU0dSABMFyoXFWRUcAZJSkyabSKu30SSLQvH7RO+zQJ94vS6DRUChbo34L2kAzsHWO8YAtt11WmziSanX1IhtYXJfeUvUBMaaMZr9z/ydzf+N1iDOArBmiUDIAh7DOo8KMpW4YzUI7nr6hhedSQ5bEhAyCAhAaShp0gQmBuJ4MZ3P8BDu9rb4/pPnQD+x3bU+/f06It3fWNb1UcbgnuDaZvDD5miWpkfodXg8ImUlTLd6xNGBS6G1uttTsoAAv4jRYy2xaUcLFEe2utxgvX8mPFzU39Bw1Ucm81qyySan944FCiORuRePMQCUD8lV3+Tll/AIK2mo5J+Q8LcaSKCcsgjM+Fhj+QrhrUGsqoPTsk4gx4oehsxi7B8goyWSp1AIrWPy3DIDGNuRh18gw4pDuS9HYGsazXVq0Mjk7uLrL2jKxzqIS0vmSkXxl19EkOt0vjnhR8RUq6HoBFZUyVDiCsobRjmwxZtstFSMXxoDIzwuQ9iujQ8H9U8PqDSuhUZKSTleFpmPD2U0QCNcWdBqsuETsR6yc6FdwoFOTiWI4UVuBxiFGPRdHHlARMjpAxVFQO0X3qsRr9VyM+NiqbBsliGmjuz9nl1FXAMGACi041dRFs+QD7mvKocFJMI8LWD40rAy2kSALH7JIqLsGZKX0Hc3XBurkuWGrRVFCEc0MedkKBcNB4YO5ppDC+KElwprxxDF3meWqsLC66DQYZYXVTKjfIlYrmvZ6KCDXuKoPLsmd9QHOL03Vb0XW7NvPz2rWuVSi4MIApe8JzIIk0IsXQTa2oIMFNjbLLJTqBVtCHqg1NKkZU4hSDfzHMjf3aVisdCu0lDreIpOWYK44JzLs6EtZmotvp8sSLYdZldUfWqyAOH3H6NR5bUKE1mVQJsiZbB3QRGQFvZs7N2vzsM2a09kzHgOdhajnlTJTWCLsizvuJ03tRxOk/K3Em9IG0ufgg+nnp03tT58+XRJ9aLUujT1bhzjM6AiMKaGfA5H0EetIE/rydNfWKuvGGeSkUm3csINKivStsehtpFtBglodx4V3gxCzL6HPCowWYEaDdWwGqRJjfAXCIK4eqYgtmlNbxKsSsQsxdIaaYRP+UMUbNm++NMWYliiqzVRbfHR0RtYpexhC14oRkLji4lWSmoMqtBSlFuLIa/vfdfW/Xda+tgYOctwU3z69sC7VMcGstIbgZcF5abJt3DeUhyyFY/YuIWXMuAs0Zo+6ISmR4Q/9vDEPXJzc3Lwet4s4q7jzX1OblhZt7WB//VWgfjeFX5f0rS0LwOgzstKe51FQ4y8j9QSvjigShuToD1pd3nKfpJux2PnY/51j30q3PSPxHVF5wcWYdvP64YvwFGb+1YvwfdRdgppc/bpphfFW11RidIxh9JaLeNcW2YY35IFO2WUrgTm6biSh73yiNpjIniSpUX7EdfVsxnVmnJd6mCVDN2ePT1R3fny1YFbu9YVetMt2+Yau2K8KuUw6yH5e/wy5w9x2D+J1Z1pL0gP5A0livJL3bXdH7it5Xm7yLD+lLyvQav/SYfj6aLCnSWreuF+5AtUyAbcxay+MRLXw0YBWtL0S2zpPA+YyI3IHGvVsa1dDz3tRjnWcWDBfWy/t/Qf9WUWi1rPTjLis9atiZaeOCnrNAZgvuXbS33H3Xv2nvwplt14TGAYFMYYfhQLKjhIG7K0sapza8g7vhtTbcDW97y2u6p7bznXchfeq/uMb9PJ0QFLydxYti2Nh/0240Ft7/8T0zamivFgJNpFNxBqxIhbtZVsdmDXgP2s43Ur2IzaOrgBm3e3TA5sZIC7LIBtN7De/r97qK1Vjg5xkLeD/4qw0vYjAw2xy0jCb46twvHU6e7ssu7xrtlc3UAsDjMb3qSzfiwdlT98Q0UiPsKkvvFT19vatzWte+S1tR9VxUvfpA5Mch6l9xI8hvm4jiv6Q3FZ7h4xV209crYS6UPxXHCFwZfVtv/7SujL/r37OcJvBB4Glya/1zV/eSP5DZAU+BF8cs/GzVOk7BDFBw88HM6nuZ1YeGP/4e1JOfKHNluF+Qs7e5Iud5yfna0kidS62/vbXcl/+94dxk+k1VjeatRc9gvBKTUkmNLVSP8oO6FpSs+FoHrPSm6P3aA8uTvoJcCGU8eBzY4koFfmbgGYZbkCg7+rSA8uQex8KDvdRHPmvlop6LOgiiPKQ78PJJwVfaMIs7cx5YWbo6KEOv+H9OxWjGVksIffzdp0MD5hlLv4joeJQE0wUK5RWeglUNBrM/TEO2hrp6q5HSkjj4tvpnpQtDyktEeVx2V/lJeQeOUn+vZQb8NW8b+Iuyc6E/szTPohcka3UuE24O6FMUEn6x7m8OsCjXt7bJlDKYN6tRh5G1PF5K+VwT1ea5eG2oqzqPoYkXT10brvqNVft48ULbHODQmeJZLaCyvlFZVQEtlo/BqRWqCqEGI9wljhmA/gW/yFT0LchmkZPwtPFR1213g0Zvve0Rf33T7W2td0nPXyeNRi9sw0kyLcVU5wUBgeIuKD1Th+WpBfO/AMd2wkZtUAAA diff --git a/apps/base/grafana/kustomization.yaml b/apps/base/grafana/kustomization.yaml index 08f0a2f..a0aa661 100644 --- a/apps/base/grafana/kustomization.yaml +++ b/apps/base/grafana/kustomization.yaml @@ -23,6 +23,7 @@ resources: - dashboards/nzbget.yaml - dashboards/postgresql-database.yaml - dashboards/puppet-report.yaml + - dashboards/redis.yaml - dashboards/victorialogs-cluster.yaml - dashboards/victoriametrics-cluster.yaml - dashboards/victoriametrics-vmagent.yaml From ba71bd1a35cb0fb72f5cf8b808314339f8c24d48 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 23 Aug 2026 12:41:58 +1000 Subject: [PATCH 4/4] Scrape traefik gateway metrics into VictoriaMetrics (#397) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why Both traefik instances (traefik-internal/traefik-external, chart 40.0.0, image v3.7.0) already expose prometheus metrics on container port `metrics` (9100), but nothing scrapes them — VictoriaMetrics has no traefik series (verified empty). The observability VMAgent runs with selectAllByDefault and consumes VM*Scrape CRs only, so a VMPodScrape is all that is needed. ## Changes - Adds `apps/base/observability/vmpodscrape-traefik.yaml`: scrapes port `metrics` /metrics on `app.kubernetes.io/name: traefik` pods in traefik-system, relabeling `app.kubernetes.io/instance` to a stable `traefik_instance` label to split internal vs external series - Registers the new CR in the observability base kustomization - Enables `metrics.prometheus.addEntryPointsLabels` and `addRoutersLabels` in both au-syd1 traefik values files for per-entrypoint/per-route dashboards (pods restart on rollout — acceptable) Validated with `kustomize build` (observability base + overlay, traefik-system overlay with --enable-helm) and kubeconform (unknown CRDs skipped); rendered deployments carry the new metric flags and the VMPodScrape renders in the overlay. Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/397 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- apps/base/observability/kustomization.yaml | 1 + .../observability/vmpodscrape-traefik.yaml | 23 +++++++++++++++++++ .../traefik-system/values-external.yaml | 5 ++++ .../traefik-system/values-internal.yaml | 5 ++++ 4 files changed, 34 insertions(+) create mode 100644 apps/base/observability/vmpodscrape-traefik.yaml diff --git a/apps/base/observability/kustomization.yaml b/apps/base/observability/kustomization.yaml index f848c26..53d138e 100644 --- a/apps/base/observability/kustomization.yaml +++ b/apps/base/observability/kustomization.yaml @@ -9,3 +9,4 @@ resources: - gateway.yaml - httproute.yaml - vmpodscrape-cnpg.yaml + - vmpodscrape-traefik.yaml diff --git a/apps/base/observability/vmpodscrape-traefik.yaml b/apps/base/observability/vmpodscrape-traefik.yaml new file mode 100644 index 0000000..369966b --- /dev/null +++ b/apps/base/observability/vmpodscrape-traefik.yaml @@ -0,0 +1,23 @@ +--- +# Scrape metrics (:9100) from the traefik-internal and traefik-external gateway +# pods. Picked up by the observability VMAgent (selectAllByDefault). The +# app.kubernetes.io/instance pod label is kept as traefik_instance so series +# from the two releases stay distinguishable. +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: traefik + namespace: observability +spec: + namespaceSelector: + matchNames: + - traefik-system + selector: + matchLabels: + app.kubernetes.io/name: traefik + podMetricsEndpoints: + - port: metrics + path: /metrics + relabelConfigs: + - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance] + targetLabel: traefik_instance diff --git a/apps/overlays/au-syd1/traefik-system/values-external.yaml b/apps/overlays/au-syd1/traefik-system/values-external.yaml index ec830e3..42cab53 100644 --- a/apps/overlays/au-syd1/traefik-system/values-external.yaml +++ b/apps/overlays/au-syd1/traefik-system/values-external.yaml @@ -29,6 +29,11 @@ providers: nativeLBByDefault: false labelSelector: "traefik.io/instance=external" +metrics: + prometheus: + addEntryPointsLabels: true + addRoutersLabels: true + logs: access: enabled: true diff --git a/apps/overlays/au-syd1/traefik-system/values-internal.yaml b/apps/overlays/au-syd1/traefik-system/values-internal.yaml index 26ff362..8779a41 100644 --- a/apps/overlays/au-syd1/traefik-system/values-internal.yaml +++ b/apps/overlays/au-syd1/traefik-system/values-internal.yaml @@ -29,6 +29,11 @@ providers: nativeLBByDefault: false labelSelector: "traefik.io/instance=internal" +metrics: + prometheus: + addEntryPointsLabels: true + addRoutersLabels: true + logs: access: enabled: true