Run Stalwart as a 3-node StatefulSet coordinated through Valkey

apps/base/mail/kustomization.yaml

@@ -12,8 +12,10 @@ resources:
   - stalwart-apply-rbac.yaml
   - stalwart-blobs-bucket.yaml
   - stalwart-configmap.yaml
-  - stalwart-deployment.yaml
+  - stalwart-pdb.yaml
   - stalwart-service.yaml
+  - stalwart-statefulset.yaml
+  - valkey.yaml
   - vaultauth.yaml
   - vaultstaticsecret.yaml
   - vmpodscrape.yaml

apps/base/mail/stalwart-apply-job.yaml

@@ -1,7 +1,8 @@
 ---
 # Converges Stalwart's database-held settings to plan.ndjson after every sync,
-# then restarts the pod: listener and store changes only bind on startup.
-# ponytail: restarts on every sync since neither image has a shell to diff apply output.
+# then rolls the StatefulSet: listener, store and coordinator changes only bind
+# on startup. It rolls on every sync because neither image has a shell to diff the
+# apply output.
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -68,7 +69,7 @@ spec:
           args:
             - rollout
             - restart
-            - deployment/stalwart
+            - statefulset/stalwart
             - --namespace=mail
           securityContext:
             runAsUser: 65532

apps/base/mail/stalwart-apply-rbac.yaml

@@ -5,7 +5,7 @@ metadata:
   name: stalwart-apply
   namespace: mail
 ---
-# Lets the stalwart-apply Job roll the Stalwart pod after applying the plan.
+# Lets the stalwart-apply Job roll the Stalwart pods after applying the plan.
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
@@ -15,7 +15,7 @@ rules:
   - apiGroups:
       - apps
     resources:
-      - deployments
+      - statefulsets
     resourceNames:
       - stalwart
     verbs:

apps/base/mail/stalwart-configmap.yaml

@@ -1,6 +1,8 @@
 ---
 # config.json only names the datastore; everything else lives in Postgres and
 # is converged from plan.ndjson by the stalwart-apply PostSync Job.
+# The Coordinator uses the plain Redis client: its cluster client never
+# resubscribes once the Valkey node it subscribed on is replaced.
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -19,7 +21,8 @@ data:
     {"@type":"reconcile","object":"Tracer","matchOn":"*","value":{"stdout":{"@type":"Stdout","enable":true,"level":"info","ansi":false,"multiline":false,"buffered":false,"lossy":false,"eventsPolicy":"exclude","events":{}}}}
     {"@type":"update","object":"BlobStore","value":{"@type":"S3","bucket":"stalwart-blobs","region":{"@type":"Custom","customEndpoint":"https://s3.ceph.unkin.net","customRegion":"us-east-1"},"accessKey":{"@type":"EnvironmentVariable","variableName":"AWS_ACCESS_KEY_ID"},"secretKey":{"@type":"EnvironmentVariable","variableName":"AWS_SECRET_ACCESS_KEY"},"securityToken":{"@type":"None"},"sessionToken":{"@type":"None"}}}
     {"@type":"update","object":"SearchStore","value":{"@type":"Default"}}
-    {"@type":"update","object":"InMemoryStore","value":{"@type":"Default"}}
+    {"@type":"update","object":"InMemoryStore","value":{"@type":"RedisCluster","urls":{"redis://valkey-mail-valkey.mail.svc:6379":true},"authUsername":null,"authSecret":{"@type":"None"},"readFromReplicas":false,"protocolVersion":"resp3"}}
+    {"@type":"update","object":"Coordinator","value":{"@type":"Redis","url":"redis://valkey-mail-valkey.mail.svc:6379"}}
     {"@type":"update","object":"Authentication","value":{"directoryId":"#authentik"}}
     {"@type":"update","object":"MtaOutboundStrategy","value":{"route":{"match":{"0":{"if":"is_local_domain(rcpt_domain)","then":"'local'"}},"else":"'smarthost'"}}}
     {"@type":"update","object":"MtaStageAuth","value":{"require":{"match":{},"else":"local_port != 25 && local_port != 24"}}}

apps/base/mail/stalwart-pdb.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: stalwart
+  namespace: mail
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: stalwart

apps/base/mail/stalwart-service.yaml

@@ -16,6 +16,21 @@ spec:
       port: 24
       targetPort: lmtp
 ---
+# Governing Service for the StatefulSet's per-pod DNS.
+apiVersion: v1
+kind: Service
+metadata:
+  name: stalwart-headless
+  namespace: mail
+spec:
+  clusterIP: None
+  selector:
+    app.kubernetes.io/name: stalwart
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+---
 # Client-facing mail ports on the lab network. 443 is here too so the single
 # internal mail.unkin.net record serves web as well as IMAP/submission.
 apiVersion: v1
@@ -24,7 +39,7 @@ metadata:
   name: stalwart-lb
   namespace: mail
   annotations:
-    purelb.io/addresses: 198.18.200.11
+    purelb.io/addresses: 198.18.200.18
     purelb.io/service-group: common
 spec:
   type: LoadBalancer

apps/base/mail/stalwart-statefulset.yaml

@@ -1,16 +1,19 @@
 ---
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: stalwart
   namespace: mail
   annotations:
     secret.reloader.stakater.com/reload: "stalwart-admin,stalwart-db-app,stalwart-blobs-s3,wildcard-unkin-net-tls,vault-ca-cert,stalwart-db-ca"
 spec:
-  # ponytail: single replica, no cluster coordinator; add one before scaling out.
-  replicas: 1
-  strategy:
-    type: Recreate
+  # Nodes coordinate through mail-valkey; each leases its cluster node id by
+  # hostname, so stable pod names keep a stable id across restarts.
+  replicas: 3
+  serviceName: stalwart-headless
+  podManagementPolicy: Parallel
+  updateStrategy:
+    type: RollingUpdate
   selector:
     matchLabels:
       app.kubernetes.io/name: stalwart
@@ -26,6 +29,13 @@ spec:
         fsGroup: 2000
         seccompProfile:
           type: RuntimeDefault
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: stalwart
       # rustls loads SSL_CERT_FILE, so public roots, the internal CA (RGW) and the CNPG CA go in one bundle.
       initContainers:
         - name: combine-certs
@@ -61,6 +71,10 @@ spec:
           image: docker.io/stalwartlabs/stalwart:v0.16.25
           imagePullPolicy: IfNotPresent
           env:
+            - name: STALWART_HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
             - name: SSL_CERT_FILE
               value: /etc/ssl/combined/ca-certificates.crt
             - name: DB_PASSWORD

apps/base/mail/valkey.yaml

@@ -0,0 +1,29 @@
+---
+# Stalwart's in-memory store (locks, rate limits, queue leases) and cluster
+# coordinator pub/sub. Ephemeral state, so no persistence; the operator runs it
+# cluster-mode, passwordless and without TLS behind valkey-mail-valkey.
+apiVersion: valkey.io/v1alpha1
+kind: ValkeyCluster
+metadata:
+  name: mail-valkey
+  namespace: mail
+spec:
+  shards: 1
+  replicas: 2
+  image: docker.io/valkey/valkey:9.0.0
+  exporter:
+    enabled: false
+  scheduling:
+    node:
+      spread:
+        shard:
+          mode: Required
+  podDisruptionBudget:
+    mode: Cluster
+  resources:
+    requests:
+      cpu: 50m
+      memory: 128Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi