Enable ARC metrics with VictoriaMetrics scrapes and dashboard (#580)

ARC runs with metrics disabled, so scale-set behaviour (assigned jobs, desired/idle runners, job startup time) and controller health are invisible.

- enable controller and listener metrics on :8080 via chart `metrics:` values
- add VMPodScrapes for the controller and listener pods in arc-system
- keep runner-namespace labels on listener/ephemeral-runner series so the dashboard filters match
- add the upstream ARC Autoscaling Runner Set dashboard (gha-runner-scale-set-0.15.1) resolved to VictoriaMetrics

Reviewed-on: #580
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

apps/base/arc-system/kustomization.yaml

@@ -5,3 +5,4 @@ kind: Kustomization
 resources:
   - namespace.yaml
   - pdb.yaml
+  - vmpodscrape.yaml

apps/base/arc-system/vmpodscrape.yaml

@@ -0,0 +1,40 @@
+---
+# Controller metrics. gha_controller_*_ephemeral_runners carry the runner
+# namespace, which the dashboard filters on, so restore it over the target's.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: arc-controller
+  namespace: arc-system
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: gha-rs-controller
+      app.kubernetes.io/instance: arc
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
+      metricRelabelConfigs:
+        - sourceLabels: [__name__, exported_namespace]
+          regex: gha_controller_.+_ephemeral_runners;(.+)
+          targetLabel: namespace
+---
+# Listener metrics carry their own name/namespace (the runner scale set), and
+# the dashboard also expects the scale-set pod labels.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: arc-listener
+  namespace: arc-system
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: runner-scale-set-listener
+      app.kubernetes.io/part-of: gha-runner-scale-set
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
+      honorLabels: true
+      relabelConfigs:
+        - action: labelmap
+          regex: __meta_kubernetes_pod_label_(actions_github_com_scale_set_.+)

apps/base/grafana/kustomization.yaml

@@ -30,3 +30,4 @@ resources:
   - dashboards/victoriametrics-cluster.yaml
   - dashboards/victoriametrics-vmagent.yaml
   - dashboards/cloudnativepg.yaml
+  - dashboards/arc-runner-scale-set.yaml

apps/overlays/au-syd1/arc-system/values.yaml

@@ -32,3 +32,8 @@ topologySpreadConstraints:
 flags:
   logLevel: info
   logFormat: json
+
+metrics:
+  controllerManagerAddr: ":8080"
+  listenerAddr: ":8080"
+  listenerEndpoint: "/metrics"