From ec60a8c9d78be531fb30ee11e6c28ee6ff128f06 Mon Sep 17 00:00:00 2001
From: unkin-agent <unkin-agent@unkin.net>
Date: Fri, 2 Oct 2026 22:37:19 +1000
Subject: [PATCH] grafana: enable unified alerting HA gossip (#509)

Grafana runs 3 replicas with no alerting HA config, so every replica sends every notification.

- add headless `grafana-alerting` Service exposing gossip on 9094 TCP/UDP
- inject `POD_IP` from `status.podIP` into the grafana container
- set `unified_alerting` `ha_peers`, `ha_listen_address` and `ha_advertise_address` so replicas form one Alertmanager cluster

Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/509
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
---
 apps/base/grafana/grafana.yaml          |  9 +++++++++
 apps/base/grafana/kustomization.yaml    |  1 +
 apps/base/grafana/service-alerting.yaml | 20 ++++++++++++++++++++
 3 files changed, 30 insertions(+)
 create mode 100644 apps/base/grafana/service-alerting.yaml

diff --git a/apps/base/grafana/grafana.yaml b/apps/base/grafana/grafana.yaml
index ff40d7a..123636d 100644
--- a/apps/base/grafana/grafana.yaml
+++ b/apps/base/grafana/grafana.yaml
@@ -22,6 +22,10 @@ spec:
           containers:
             - name: grafana
               env:
+                - name: POD_IP
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: status.podIP
                 # DB password + OAuth client secret injected from the
                 # Vault-synced secrets (GF_ env overrides grafana.ini).
                 - name: GF_DATABASE_PASSWORD
@@ -69,3 +73,8 @@ spec:
       # to akR-global-admin members (and direct members) via terraform-authentik.
       role_attribute_path: "contains(ak_groups[*], 'akP-grafana-admin') && 'Admin' || 'Viewer'"
       role_attribute_strict: "false"
+    # Gossip between replicas so each alert notification is sent once.
+    unified_alerting:
+      ha_peers: "grafana-alerting.grafana.svc.cluster.local:9094"
+      ha_listen_address: "${POD_IP}:9094"
+      ha_advertise_address: "${POD_IP}:9094"
diff --git a/apps/base/grafana/kustomization.yaml b/apps/base/grafana/kustomization.yaml
index 1214b1a..ccdb11d 100644
--- a/apps/base/grafana/kustomization.yaml
+++ b/apps/base/grafana/kustomization.yaml
@@ -10,6 +10,7 @@ resources:
   - vaultauth.yaml
   - vaultstaticsecret.yaml
   - grafana.yaml
+  - service-alerting.yaml
   - grafanadatasource.yaml
   - gateway.yaml
   - httproute.yaml
diff --git a/apps/base/grafana/service-alerting.yaml b/apps/base/grafana/service-alerting.yaml
new file mode 100644
index 0000000..c8d8dd2
--- /dev/null
+++ b/apps/base/grafana/service-alerting.yaml
@@ -0,0 +1,20 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana-alerting
+  namespace: grafana
+spec:
+  clusterIP: None
+  publishNotReadyAddresses: true
+  selector:
+    app: grafana
+  ports:
+    - name: gossip-tcp
+      port: 9094
+      targetPort: 9094
+      protocol: TCP
+    - name: gossip-udp
+      port: 9094
+      targetPort: 9094
+      protocol: UDP