From ba71bd1a35cb0fb72f5cf8b808314339f8c24d48 Mon Sep 17 00:00:00 2001
From: unkin-agent <unkin-agent@unkin.net>
Date: Sun, 23 Aug 2026 12:41:58 +1000
Subject: [PATCH] Scrape traefik gateway metrics into VictoriaMetrics (#397)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Why

Both traefik instances (traefik-internal/traefik-external, chart 40.0.0, image v3.7.0) already expose prometheus metrics on container port `metrics` (9100), but nothing scrapes them — VictoriaMetrics has no traefik series (verified empty). The observability VMAgent runs with selectAllByDefault and consumes VM*Scrape CRs only, so a VMPodScrape is all that is needed.

## Changes

- Adds `apps/base/observability/vmpodscrape-traefik.yaml`: scrapes port `metrics` /metrics on `app.kubernetes.io/name: traefik` pods in traefik-system, relabeling `app.kubernetes.io/instance` to a stable `traefik_instance` label to split internal vs external series
- Registers the new CR in the observability base kustomization
- Enables `metrics.prometheus.addEntryPointsLabels` and `addRoutersLabels` in both au-syd1 traefik values files for per-entrypoint/per-route dashboards (pods restart on rollout — acceptable)

Validated with `kustomize build` (observability base + overlay, traefik-system overlay with --enable-helm) and kubeconform (unknown CRDs skipped); rendered deployments carry the new metric flags and the VMPodScrape renders in the overlay.

Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/397
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
---
 apps/base/observability/kustomization.yaml    |  1 +
 .../observability/vmpodscrape-traefik.yaml    | 23 +++++++++++++++++++
 .../traefik-system/values-external.yaml       |  5 ++++
 .../traefik-system/values-internal.yaml       |  5 ++++
 4 files changed, 34 insertions(+)
 create mode 100644 apps/base/observability/vmpodscrape-traefik.yaml

diff --git a/apps/base/observability/kustomization.yaml b/apps/base/observability/kustomization.yaml
index f848c26..53d138e 100644
--- a/apps/base/observability/kustomization.yaml
+++ b/apps/base/observability/kustomization.yaml
@@ -9,3 +9,4 @@ resources:
   - gateway.yaml
   - httproute.yaml
   - vmpodscrape-cnpg.yaml
+  - vmpodscrape-traefik.yaml
diff --git a/apps/base/observability/vmpodscrape-traefik.yaml b/apps/base/observability/vmpodscrape-traefik.yaml
new file mode 100644
index 0000000..369966b
--- /dev/null
+++ b/apps/base/observability/vmpodscrape-traefik.yaml
@@ -0,0 +1,23 @@
+---
+# Scrape metrics (:9100) from the traefik-internal and traefik-external gateway
+# pods. Picked up by the observability VMAgent (selectAllByDefault). The
+# app.kubernetes.io/instance pod label is kept as traefik_instance so series
+# from the two releases stay distinguishable.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: traefik
+  namespace: observability
+spec:
+  namespaceSelector:
+    matchNames:
+      - traefik-system
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: traefik
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
+      relabelConfigs:
+        - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
+          targetLabel: traefik_instance
diff --git a/apps/overlays/au-syd1/traefik-system/values-external.yaml b/apps/overlays/au-syd1/traefik-system/values-external.yaml
index ec830e3..42cab53 100644
--- a/apps/overlays/au-syd1/traefik-system/values-external.yaml
+++ b/apps/overlays/au-syd1/traefik-system/values-external.yaml
@@ -29,6 +29,11 @@ providers:
     nativeLBByDefault: false
     labelSelector: "traefik.io/instance=external"
 
+metrics:
+  prometheus:
+    addEntryPointsLabels: true
+    addRoutersLabels: true
+
 logs:
   access:
     enabled: true
diff --git a/apps/overlays/au-syd1/traefik-system/values-internal.yaml b/apps/overlays/au-syd1/traefik-system/values-internal.yaml
index 26ff362..8779a41 100644
--- a/apps/overlays/au-syd1/traefik-system/values-internal.yaml
+++ b/apps/overlays/au-syd1/traefik-system/values-internal.yaml
@@ -29,6 +29,11 @@ providers:
     nativeLBByDefault: false
     labelSelector: "traefik.io/instance=internal"
 
+metrics:
+  prometheus:
+    addEntryPointsLabels: true
+    addRoutersLabels: true
+
 logs:
   access:
     enabled: true