From eb7dbca947fe02defaad1db8683f72e84c5e4d2f Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Mon, 12 Oct 2026 21:48:56 +1100 Subject: [PATCH] Run Stalwart as a 3-node StatefulSet coordinated through Valkey --- apps/base/mail/kustomization.yaml | 4 ++- apps/base/mail/stalwart-apply-job.yaml | 7 +++-- apps/base/mail/stalwart-apply-rbac.yaml | 4 +-- apps/base/mail/stalwart-configmap.yaml | 5 +++- apps/base/mail/stalwart-pdb.yaml | 11 +++++++ apps/base/mail/stalwart-service.yaml | 17 ++++++++++- ...loyment.yaml => stalwart-statefulset.yaml} | 24 +++++++++++---- apps/base/mail/valkey.yaml | 29 +++++++++++++++++++ 8 files changed, 88 insertions(+), 13 deletions(-) create mode 100644 apps/base/mail/stalwart-pdb.yaml rename apps/base/mail/{stalwart-deployment.yaml => stalwart-statefulset.yaml} (89%) create mode 100644 apps/base/mail/valkey.yaml diff --git a/apps/base/mail/kustomization.yaml b/apps/base/mail/kustomization.yaml index e131c9e..bdeda69 100644 --- a/apps/base/mail/kustomization.yaml +++ b/apps/base/mail/kustomization.yaml @@ -12,8 +12,10 @@ resources: - stalwart-apply-rbac.yaml - stalwart-blobs-bucket.yaml - stalwart-configmap.yaml - - stalwart-deployment.yaml + - stalwart-pdb.yaml - stalwart-service.yaml + - stalwart-statefulset.yaml + - valkey.yaml - vaultauth.yaml - vaultstaticsecret.yaml - vmpodscrape.yaml diff --git a/apps/base/mail/stalwart-apply-job.yaml b/apps/base/mail/stalwart-apply-job.yaml index 9483cc9..70059a3 100644 --- a/apps/base/mail/stalwart-apply-job.yaml +++ b/apps/base/mail/stalwart-apply-job.yaml @@ -1,7 +1,8 @@ --- # Converges Stalwart's database-held settings to plan.ndjson after every sync, -# then restarts the pod: listener and store changes only bind on startup. -# ponytail: restarts on every sync since neither image has a shell to diff apply output. +# then rolls the StatefulSet: listener, store and coordinator changes only bind +# on startup. It rolls on every sync because neither image has a shell to diff the +# apply output. apiVersion: batch/v1 kind: Job metadata: @@ -68,7 +69,7 @@ spec: args: - rollout - restart - - deployment/stalwart + - statefulset/stalwart - --namespace=mail securityContext: runAsUser: 65532 diff --git a/apps/base/mail/stalwart-apply-rbac.yaml b/apps/base/mail/stalwart-apply-rbac.yaml index 65c642c..bb47d35 100644 --- a/apps/base/mail/stalwart-apply-rbac.yaml +++ b/apps/base/mail/stalwart-apply-rbac.yaml @@ -5,7 +5,7 @@ metadata: name: stalwart-apply namespace: mail --- -# Lets the stalwart-apply Job roll the Stalwart pod after applying the plan. +# Lets the stalwart-apply Job roll the Stalwart pods after applying the plan. apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -15,7 +15,7 @@ rules: - apiGroups: - apps resources: - - deployments + - statefulsets resourceNames: - stalwart verbs: diff --git a/apps/base/mail/stalwart-configmap.yaml b/apps/base/mail/stalwart-configmap.yaml index 4211da1..9a2fc57 100644 --- a/apps/base/mail/stalwart-configmap.yaml +++ b/apps/base/mail/stalwart-configmap.yaml @@ -1,6 +1,8 @@ --- # config.json only names the datastore; everything else lives in Postgres and # is converged from plan.ndjson by the stalwart-apply PostSync Job. +# The Coordinator uses the plain Redis client: its cluster client never +# resubscribes once the Valkey node it subscribed on is replaced. apiVersion: v1 kind: ConfigMap metadata: @@ -19,7 +21,8 @@ data: {"@type":"reconcile","object":"Tracer","matchOn":"*","value":{"stdout":{"@type":"Stdout","enable":true,"level":"info","ansi":false,"multiline":false,"buffered":false,"lossy":false,"eventsPolicy":"exclude","events":{}}}} {"@type":"update","object":"BlobStore","value":{"@type":"S3","bucket":"stalwart-blobs","region":{"@type":"Custom","customEndpoint":"https://s3.ceph.unkin.net","customRegion":"us-east-1"},"accessKey":{"@type":"EnvironmentVariable","variableName":"AWS_ACCESS_KEY_ID"},"secretKey":{"@type":"EnvironmentVariable","variableName":"AWS_SECRET_ACCESS_KEY"},"securityToken":{"@type":"None"},"sessionToken":{"@type":"None"}}} {"@type":"update","object":"SearchStore","value":{"@type":"Default"}} - {"@type":"update","object":"InMemoryStore","value":{"@type":"Default"}} + {"@type":"update","object":"InMemoryStore","value":{"@type":"RedisCluster","urls":{"redis://valkey-mail-valkey.mail.svc:6379":true},"authUsername":null,"authSecret":{"@type":"None"},"readFromReplicas":false,"protocolVersion":"resp3"}} + {"@type":"update","object":"Coordinator","value":{"@type":"Redis","url":"redis://valkey-mail-valkey.mail.svc:6379"}} {"@type":"update","object":"Authentication","value":{"directoryId":"#authentik"}} {"@type":"update","object":"MtaOutboundStrategy","value":{"route":{"match":{"0":{"if":"is_local_domain(rcpt_domain)","then":"'local'"}},"else":"'smarthost'"}}} {"@type":"update","object":"MtaStageAuth","value":{"require":{"match":{},"else":"local_port != 25 && local_port != 24"}}} diff --git a/apps/base/mail/stalwart-pdb.yaml b/apps/base/mail/stalwart-pdb.yaml new file mode 100644 index 0000000..6d43f7a --- /dev/null +++ b/apps/base/mail/stalwart-pdb.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: stalwart + namespace: mail +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: stalwart diff --git a/apps/base/mail/stalwart-service.yaml b/apps/base/mail/stalwart-service.yaml index fd4b350..b8837c7 100644 --- a/apps/base/mail/stalwart-service.yaml +++ b/apps/base/mail/stalwart-service.yaml @@ -16,6 +16,21 @@ spec: port: 24 targetPort: lmtp --- +# Governing Service for the StatefulSet's per-pod DNS. +apiVersion: v1 +kind: Service +metadata: + name: stalwart-headless + namespace: mail +spec: + clusterIP: None + selector: + app.kubernetes.io/name: stalwart + ports: + - name: http + port: 8080 + targetPort: http +--- # Client-facing mail ports on the lab network. 443 is here too so the single # internal mail.unkin.net record serves web as well as IMAP/submission. apiVersion: v1 @@ -24,7 +39,7 @@ metadata: name: stalwart-lb namespace: mail annotations: - purelb.io/addresses: 198.18.200.11 + purelb.io/addresses: 198.18.200.18 purelb.io/service-group: common spec: type: LoadBalancer diff --git a/apps/base/mail/stalwart-deployment.yaml b/apps/base/mail/stalwart-statefulset.yaml similarity index 89% rename from apps/base/mail/stalwart-deployment.yaml rename to apps/base/mail/stalwart-statefulset.yaml index 2675d8d..f0ec3a1 100644 --- a/apps/base/mail/stalwart-deployment.yaml +++ b/apps/base/mail/stalwart-statefulset.yaml @@ -1,16 +1,19 @@ --- apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: stalwart namespace: mail annotations: secret.reloader.stakater.com/reload: "stalwart-admin,stalwart-db-app,stalwart-blobs-s3,wildcard-unkin-net-tls,vault-ca-cert,stalwart-db-ca" spec: - # ponytail: single replica, no cluster coordinator; add one before scaling out. - replicas: 1 - strategy: - type: Recreate + # Nodes coordinate through mail-valkey; each leases its cluster node id by + # hostname, so stable pod names keep a stable id across restarts. + replicas: 3 + serviceName: stalwart-headless + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate selector: matchLabels: app.kubernetes.io/name: stalwart @@ -26,6 +29,13 @@ spec: fsGroup: 2000 seccompProfile: type: RuntimeDefault + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: stalwart # rustls loads SSL_CERT_FILE, so public roots, the internal CA (RGW) and the CNPG CA go in one bundle. initContainers: - name: combine-certs @@ -61,6 +71,10 @@ spec: image: docker.io/stalwartlabs/stalwart:v0.16.25 imagePullPolicy: IfNotPresent env: + - name: STALWART_HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name - name: SSL_CERT_FILE value: /etc/ssl/combined/ca-certificates.crt - name: DB_PASSWORD diff --git a/apps/base/mail/valkey.yaml b/apps/base/mail/valkey.yaml new file mode 100644 index 0000000..a495c71 --- /dev/null +++ b/apps/base/mail/valkey.yaml @@ -0,0 +1,29 @@ +--- +# Stalwart's in-memory store (locks, rate limits, queue leases) and cluster +# coordinator pub/sub. Ephemeral state, so no persistence; the operator runs it +# cluster-mode, passwordless and without TLS behind valkey-mail-valkey. +apiVersion: valkey.io/v1alpha1 +kind: ValkeyCluster +metadata: + name: mail-valkey + namespace: mail +spec: + shards: 1 + replicas: 2 + image: docker.io/valkey/valkey:9.0.0 + exporter: + enabled: false + scheduling: + node: + spread: + shard: + mode: Required + podDisruptionBudget: + mode: Cluster + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi