grafana: enable unified alerting HA gossip (#509)

Grafana runs 3 replicas with no alerting HA config, so every replica sends every notification.

- add headless `grafana-alerting` Service exposing gossip on 9094 TCP/UDP
- inject `POD_IP` from `status.podIP` into the grafana container
- set `unified_alerting` `ha_peers`, `ha_listen_address` and `ha_advertise_address` so replicas form one Alertmanager cluster

Reviewed-on: #509
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

apps/base/grafana/grafana.yaml

@@ -22,6 +22,10 @@ spec:
           containers:
             - name: grafana
               env:
+                - name: POD_IP
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: status.podIP
                 # DB password + OAuth client secret injected from the
                 # Vault-synced secrets (GF_ env overrides grafana.ini).
                 - name: GF_DATABASE_PASSWORD
@@ -69,3 +73,8 @@ spec:
       # to akR-global-admin members (and direct members) via terraform-authentik.
       role_attribute_path: "contains(ak_groups[*], 'akP-grafana-admin') && 'Admin' || 'Viewer'"
       role_attribute_strict: "false"
+    # Gossip between replicas so each alert notification is sent once.
+    unified_alerting:
+      ha_peers: "grafana-alerting.grafana.svc.cluster.local:9094"
+      ha_listen_address: "${POD_IP}:9094"
+      ha_advertise_address: "${POD_IP}:9094"

apps/base/grafana/kustomization.yaml

@@ -10,6 +10,7 @@ resources:
   - vaultauth.yaml
   - vaultstaticsecret.yaml
   - grafana.yaml
+  - service-alerting.yaml
   - grafanadatasource.yaml
   - gateway.yaml
   - httproute.yaml

apps/base/grafana/service-alerting.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana-alerting
+  namespace: grafana
+spec:
+  clusterIP: None
+  publishNotReadyAddresses: true
+  selector:
+    app: grafana
+  ports:
+    - name: gossip-tcp
+      port: 9094
+      targetPort: 9094
+      protocol: TCP
+    - name: gossip-udp
+      port: 9094
+      targetPort: 9094
+      protocol: UDP