From f627902dd6aba843356c2565c79ebed6f613ce98 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Mon, 12 Oct 2026 21:47:39 +1100 Subject: [PATCH 1/2] Add rspamd, ClamAV and sentinel Valkey to the mail namespace --- apps/base/mail/clamav.yaml | 87 ++++++++++ apps/base/mail/kustomization.yaml | 31 ++++ apps/base/mail/rspamd-deployment.yaml | 158 +++++++++++++++++ apps/base/mail/rspamd-sa-rules.yaml | 8 + apps/base/mail/rspamd-valkey.yaml | 177 ++++++++++++++++++++ apps/base/mail/rspamd-valkey/start.sh | 34 ++++ apps/base/mail/rspamd-vault.yaml | 79 +++++++++ apps/base/mail/rspamd/actions.conf | 3 + apps/base/mail/rspamd/antivirus.conf | 8 + apps/base/mail/rspamd/dkim_signing.conf | 8 + apps/base/mail/rspamd/gpt.conf | 25 +++ apps/base/mail/rspamd/greylist.conf | 1 + apps/base/mail/rspamd/history_redis.conf | 1 + apps/base/mail/rspamd/options.inc | 3 + apps/base/mail/rspamd/redis.conf | 4 + apps/base/mail/rspamd/spamassassin.conf | 1 + apps/base/mail/rspamd/worker-controller.inc | 3 + apps/base/mail/rspamd/worker-normal.inc | 2 + apps/base/mail/rspamd/worker-proxy.inc | 1 + apps/base/mail/vaultauth.yaml | 18 ++ 20 files changed, 652 insertions(+) create mode 100644 apps/base/mail/clamav.yaml create mode 100644 apps/base/mail/rspamd-deployment.yaml create mode 100644 apps/base/mail/rspamd-sa-rules.yaml create mode 100644 apps/base/mail/rspamd-valkey.yaml create mode 100755 apps/base/mail/rspamd-valkey/start.sh create mode 100644 apps/base/mail/rspamd-vault.yaml create mode 100644 apps/base/mail/rspamd/actions.conf create mode 100644 apps/base/mail/rspamd/antivirus.conf create mode 100644 apps/base/mail/rspamd/dkim_signing.conf create mode 100644 apps/base/mail/rspamd/gpt.conf create mode 100644 apps/base/mail/rspamd/greylist.conf create mode 100644 apps/base/mail/rspamd/history_redis.conf create mode 100644 apps/base/mail/rspamd/options.inc create mode 100644 apps/base/mail/rspamd/redis.conf create mode 100644 apps/base/mail/rspamd/spamassassin.conf create mode 100644 apps/base/mail/rspamd/worker-controller.inc create mode 100644 apps/base/mail/rspamd/worker-normal.inc create mode 100644 apps/base/mail/rspamd/worker-proxy.inc create mode 100644 apps/base/mail/vaultauth.yaml diff --git a/apps/base/mail/clamav.yaml b/apps/base/mail/clamav.yaml new file mode 100644 index 0000000..09ad431 --- /dev/null +++ b/apps/base/mail/clamav.yaml @@ -0,0 +1,87 @@ +--- +# clamd + freshclam (the image's init runs both); signatures come from database.clamav.net. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clamav + namespace: mail +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: clamav + template: + metadata: + labels: + app.kubernetes.io/name: clamav + spec: + automountServiceAccountToken: false + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: clamav + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: clamav + image: docker.io/clamav/clamav:1.5.4 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3310 + name: clamd + protocol: TCP + startupProbe: + tcpSocket: + port: clamd + periodSeconds: 10 + failureThreshold: 60 + readinessProbe: + tcpSocket: + port: clamd + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: clamd + periodSeconds: 30 + resources: + requests: + cpu: 100m + memory: 1536Mi + limits: + cpu: "2" + memory: 3Gi + volumeMounts: + - name: db + mountPath: /var/lib/clamav + volumes: + - name: db + emptyDir: {} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: clamav + namespace: mail +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: clamav +--- +apiVersion: v1 +kind: Service +metadata: + name: clamav + namespace: mail +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: clamav + ports: + - name: clamd + port: 3310 + targetPort: clamd diff --git a/apps/base/mail/kustomization.yaml b/apps/base/mail/kustomization.yaml index ebe64ef..fd9d126 100644 --- a/apps/base/mail/kustomization.yaml +++ b/apps/base/mail/kustomization.yaml @@ -3,7 +3,38 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - clamav.yaml - cnpg_backup.yaml - cnpg_cluster.yaml - namespace.yaml + - rspamd-deployment.yaml + - rspamd-sa-rules.yaml + - rspamd-valkey.yaml + - rspamd-vault.yaml - stalwart-blobs-bucket.yaml + - vaultauth.yaml + +configMapGenerator: + - name: rspamd-config + namespace: mail + options: + disableNameSuffixHash: true + files: + - rspamd/actions.conf + - rspamd/antivirus.conf + - rspamd/dkim_signing.conf + - rspamd/gpt.conf + - rspamd/greylist.conf + - rspamd/history_redis.conf + - rspamd/options.inc + - rspamd/redis.conf + - rspamd/spamassassin.conf + - rspamd/worker-controller.inc + - rspamd/worker-normal.inc + - rspamd/worker-proxy.inc + - name: rspamd-valkey-scripts + namespace: mail + options: + disableNameSuffixHash: true + files: + - rspamd-valkey/start.sh diff --git a/apps/base/mail/rspamd-deployment.yaml b/apps/base/mail/rspamd-deployment.yaml new file mode 100644 index 0000000..dd0c933 --- /dev/null +++ b/apps/base/mail/rspamd-deployment.yaml @@ -0,0 +1,158 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rspamd + namespace: mail + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: rspamd + template: + metadata: + labels: + app.kubernetes.io/name: rspamd + spec: + serviceAccountName: rspamd + automountServiceAccountToken: false + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: rspamd + securityContext: + runAsNonRoot: true + runAsUser: 11333 + runAsGroup: 11333 + fsGroup: 11333 + seccompProfile: + type: RuntimeDefault + containers: + - name: rspamd + image: docker.io/rspamd/rspamd:4.2.2 + imagePullPolicy: IfNotPresent + env: + - name: RSPAMD_LOCAL_ADDR + value: "*" + - name: RSPAMD_LOG_TYPE + value: console + - name: RSPAMD_LITELLM_API_KEY + valueFrom: + secretKeyRef: + name: rspamd-litellm + key: key + - name: RSPAMD_CONTROLLER_PASSWORD + valueFrom: + secretKeyRef: + name: rspamd-controller + key: password + ports: + - containerPort: 11332 + name: milter + protocol: TCP + - containerPort: 11334 + name: controller + protocol: TCP + readinessProbe: + tcpSocket: + port: milter + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /ping + port: controller + initialDelaySeconds: 60 + periodSeconds: 30 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 1536Mi + volumeMounts: + - name: config + mountPath: /etc/rspamd/local.d + readOnly: true + - name: sa-rules + mountPath: /etc/rspamd/sa-rules + readOnly: true + - name: dkim + mountPath: /etc/rspamd/dkim + readOnly: true + - name: data + mountPath: /var/lib/rspamd + volumes: + - name: config + configMap: + name: rspamd-config + - name: sa-rules + configMap: + name: rspamd-sa-rules + - name: dkim + secret: + secretName: dkim-unkin-net + defaultMode: 0440 + items: + - key: private_key + path: unkin.net.mail.key + - name: data + emptyDir: {} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: rspamd + namespace: mail +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: rspamd +--- +# Milter for the smarthost (rspamd.mail.svc:11332); controller/web UI is in-cluster only. +apiVersion: v1 +kind: Service +metadata: + name: rspamd + namespace: mail +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: rspamd + ports: + - name: milter + port: 11332 + targetPort: milter + - name: controller + port: 11334 + targetPort: controller +--- +# Milter for the DMZ gateways over WireGuard. +apiVersion: v1 +kind: Service +metadata: + name: rspamd-milter + namespace: mail + annotations: + purelb.io/addresses: 198.18.200.17 + purelb.io/service-group: common +spec: + type: LoadBalancer + selector: + app.kubernetes.io/name: rspamd + ports: + - name: milter + port: 11332 + targetPort: milter diff --git a/apps/base/mail/rspamd-sa-rules.yaml b/apps/base/mail/rspamd-sa-rules.yaml new file mode 100644 index 0000000..9789497 --- /dev/null +++ b/apps/base/mail/rspamd-sa-rules.yaml @@ -0,0 +1,8 @@ +--- +# SpamAssassin rules loaded by the rspamd spamassassin module (*.cf). +apiVersion: v1 +kind: ConfigMap +metadata: + name: rspamd-sa-rules + namespace: mail +data: {} diff --git a/apps/base/mail/rspamd-valkey.yaml b/apps/base/mail/rspamd-valkey.yaml new file mode 100644 index 0000000..ecf49e1 --- /dev/null +++ b/apps/base/mail/rspamd-valkey.yaml @@ -0,0 +1,177 @@ +--- +# Sentinel-managed Valkey for rspamd (bayes, history, ratelimit, gpt cache). +# rspamd has no Redis Cluster support, so the valkey-operator (cluster mode) +# cannot be used. Pod 0 seeds a fresh set; afterwards start.sh follows the +# master the sentinels report. Hostnames carry a trailing dot so lookups of a +# deleted pod fail fast instead of walking the search list (sentinel TILT). +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: rspamd-valkey + namespace: mail + annotations: + configmap.reloader.stakater.com/reload: "rspamd-valkey-scripts" +spec: + serviceName: rspamd-valkey + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/name: rspamd-valkey + template: + metadata: + labels: + app.kubernetes.io/name: rspamd-valkey + spec: + automountServiceAccountToken: false + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: rspamd-valkey + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + seccompProfile: + type: RuntimeDefault + containers: + - name: valkey + image: docker.io/valkey/valkey:9.0.0 + imagePullPolicy: IfNotPresent + command: ["sh", "/scripts/start.sh", "valkey"] + env: + - name: DOMAIN_SUFFIX + value: .rspamd-valkey.mail.svc.cluster.local. + ports: + - containerPort: 6379 + name: valkey + protocol: TCP + readinessProbe: + exec: + command: ["valkey-cli", "ping"] + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + exec: + command: ["valkey-cli", "ping"] + initialDelaySeconds: 30 + periodSeconds: 30 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + volumeMounts: + - name: data + mountPath: /data + - name: scripts + mountPath: /scripts + readOnly: true + - name: sentinel + image: docker.io/valkey/valkey:9.0.0 + imagePullPolicy: IfNotPresent + command: ["sh", "/scripts/start.sh", "sentinel"] + env: + - name: DOMAIN_SUFFIX + value: .rspamd-valkey.mail.svc.cluster.local. + ports: + - containerPort: 26379 + name: sentinel + protocol: TCP + readinessProbe: + exec: + command: ["valkey-cli", "-p", "26379", "ping"] + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + exec: + command: ["valkey-cli", "-p", "26379", "ping"] + initialDelaySeconds: 30 + periodSeconds: 30 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + volumeMounts: + - name: sentinel + mountPath: /sentinel + - name: scripts + mountPath: /scripts + readOnly: true + volumes: + - name: scripts + configMap: + name: rspamd-valkey-scripts + - name: sentinel + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ReadWriteOnce] + storageClassName: cephrbd-fast-delete + resources: + requests: + storage: 5Gi +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: rspamd-valkey + namespace: mail +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: rspamd-valkey +--- +# Stable per-pod DNS, published before readiness so replicas can find the seed. +apiVersion: v1 +kind: Service +metadata: + name: rspamd-valkey + namespace: mail +spec: + clusterIP: None + publishNotReadyAddresses: true + selector: + app.kubernetes.io/name: rspamd-valkey + ports: + - name: valkey + port: 6379 + targetPort: valkey + - name: sentinel + port: 26379 + targetPort: sentinel +--- +apiVersion: v1 +kind: Service +metadata: + name: rspamd-valkey-sentinel + namespace: mail +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: rspamd-valkey + ports: + - name: sentinel + port: 26379 + targetPort: sentinel diff --git a/apps/base/mail/rspamd-valkey/start.sh b/apps/base/mail/rspamd-valkey/start.sh new file mode 100755 index 0000000..9609465 --- /dev/null +++ b/apps/base/mail/rspamd-valkey/start.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Usage: start.sh valkey|sentinel. Ask peer sentinels for the current master; +# with no answer (fresh cluster) pod 0 is the master. +set -eu +name="$(hostname)" +sts="${name%-*}" +self="${name}${DOMAIN_SUFFIX}" +master="" +for i in 0 1 2; do + peer="${sts}-${i}${DOMAIN_SUFFIX}" + [ "$peer" = "$self" ] && continue + master="$(timeout 3 valkey-cli -h "$peer" -p 26379 --raw sentinel get-master-addr-by-name rspamd 2>/dev/null | head -n1 || true)" + [ -n "$master" ] && break +done +[ -n "$master" ] || master="${sts}-0${DOMAIN_SUFFIX}" + +if [ "$1" = sentinel ]; then + cat > /sentinel/sentinel.conf <= 12 then return false, 'score not uncertain' end + return true +end +EOD diff --git a/apps/base/mail/rspamd/greylist.conf b/apps/base/mail/rspamd/greylist.conf new file mode 100644 index 0000000..a6ee831 --- /dev/null +++ b/apps/base/mail/rspamd/greylist.conf @@ -0,0 +1 @@ +enabled = false; diff --git a/apps/base/mail/rspamd/history_redis.conf b/apps/base/mail/rspamd/history_redis.conf new file mode 100644 index 0000000..f3507c7 --- /dev/null +++ b/apps/base/mail/rspamd/history_redis.conf @@ -0,0 +1 @@ +nrows = 5000; diff --git a/apps/base/mail/rspamd/options.inc b/apps/base/mail/rspamd/options.inc new file mode 100644 index 0000000..311b6f3 --- /dev/null +++ b/apps/base/mail/rspamd/options.inc @@ -0,0 +1,3 @@ +# Pod network (Stalwart via the smarthost) and lab networks: local, so outbound mail from them is DKIM-signed. +local_addrs = ["127.0.0.0/8", "::1", "10.42.0.0/16", "198.18.0.0/16"]; +task_timeout = 20s; diff --git a/apps/base/mail/rspamd/redis.conf b/apps/base/mail/rspamd/redis.conf new file mode 100644 index 0000000..1f94eb5 --- /dev/null +++ b/apps/base/mail/rspamd/redis.conf @@ -0,0 +1,4 @@ +sentinels = "rspamd-valkey-sentinel.mail.svc.cluster.local:26379"; +sentinel_masters_pattern = "^rspamd$"; +sentinel_watch_time = 10s; +servers = "rspamd-valkey-0.rspamd-valkey.mail.svc.cluster.local.:6379"; diff --git a/apps/base/mail/rspamd/spamassassin.conf b/apps/base/mail/rspamd/spamassassin.conf new file mode 100644 index 0000000..6377a4d --- /dev/null +++ b/apps/base/mail/rspamd/spamassassin.conf @@ -0,0 +1 @@ +ruleset = "/etc/rspamd/sa-rules/*.cf"; diff --git a/apps/base/mail/rspamd/worker-controller.inc b/apps/base/mail/rspamd/worker-controller.inc new file mode 100644 index 0000000..2894951 --- /dev/null +++ b/apps/base/mail/rspamd/worker-controller.inc @@ -0,0 +1,3 @@ +password = "{= env.CONTROLLER_PASSWORD =}"; +enable_password = "{= env.CONTROLLER_PASSWORD =}"; +allow_file_and_shm_inputs = false; diff --git a/apps/base/mail/rspamd/worker-normal.inc b/apps/base/mail/rspamd/worker-normal.inc new file mode 100644 index 0000000..2a16e07 --- /dev/null +++ b/apps/base/mail/rspamd/worker-normal.inc @@ -0,0 +1,2 @@ +count = 2; +allow_file_and_shm_inputs = false; diff --git a/apps/base/mail/rspamd/worker-proxy.inc b/apps/base/mail/rspamd/worker-proxy.inc new file mode 100644 index 0000000..897afe8 --- /dev/null +++ b/apps/base/mail/rspamd/worker-proxy.inc @@ -0,0 +1 @@ +allow_file_and_shm_inputs = false; diff --git a/apps/base/mail/vaultauth.yaml b/apps/base/mail/vaultauth.yaml new file mode 100644 index 0000000..8f0b14a --- /dev/null +++ b/apps/base/mail/vaultauth.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: default + namespace: mail +spec: + allowedNamespaces: + - mail + kubernetes: + audiences: + - vault + role: default + serviceAccount: default + tokenExpirationSeconds: 600 + method: kubernetes + mount: k8s/au/syd1 + vaultConnectionRef: vso-system/default -- 2.47.3 From ecede40f7f64aa457833ff61ced3d7cfc154d27a Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Mon, 12 Oct 2026 22:14:55 +1100 Subject: [PATCH 2/2] Scan inbound and outbound via the smarthost only, harden rspamd and Valkey - tag inbound instead of rejecting, keep virus reject and tempfail when ClamAV fails - score inbound against the first hop before the edge relays - sign and skip GPT only for smarthost mynetworks sources - elect the highest-offset Valkey on full restart, cap memory at 384mb - read a hashed controller password --- apps/base/mail/kustomization.yaml | 5 ++ apps/base/mail/rspamd-deployment.yaml | 24 +------ apps/base/mail/rspamd-valkey.yaml | 9 +-- apps/base/mail/rspamd-valkey/start.sh | 74 +++++++++++++++++---- apps/base/mail/rspamd-vault.yaml | 2 +- apps/base/mail/rspamd/dkim_signing.conf | 3 +- apps/base/mail/rspamd/external_relay.conf | 8 +++ apps/base/mail/rspamd/force_actions.conf | 7 ++ apps/base/mail/rspamd/gpt.conf | 5 +- apps/base/mail/rspamd/local_networks.map | 30 +++++++++ apps/base/mail/rspamd/milter_headers.conf | 2 + apps/base/mail/rspamd/options.inc | 3 +- apps/base/mail/rspamd/settings.conf | 19 ++++++ apps/base/mail/rspamd/worker-controller.inc | 4 +- 14 files changed, 146 insertions(+), 49 deletions(-) create mode 100644 apps/base/mail/rspamd/external_relay.conf create mode 100644 apps/base/mail/rspamd/force_actions.conf create mode 100644 apps/base/mail/rspamd/local_networks.map create mode 100644 apps/base/mail/rspamd/milter_headers.conf create mode 100644 apps/base/mail/rspamd/settings.conf diff --git a/apps/base/mail/kustomization.yaml b/apps/base/mail/kustomization.yaml index fd9d126..0a73449 100644 --- a/apps/base/mail/kustomization.yaml +++ b/apps/base/mail/kustomization.yaml @@ -23,11 +23,16 @@ configMapGenerator: - rspamd/actions.conf - rspamd/antivirus.conf - rspamd/dkim_signing.conf + - rspamd/external_relay.conf + - rspamd/force_actions.conf - rspamd/gpt.conf - rspamd/greylist.conf - rspamd/history_redis.conf + - rspamd/local_networks.map + - rspamd/milter_headers.conf - rspamd/options.inc - rspamd/redis.conf + - rspamd/settings.conf - rspamd/spamassassin.conf - rspamd/worker-controller.inc - rspamd/worker-normal.inc diff --git a/apps/base/mail/rspamd-deployment.yaml b/apps/base/mail/rspamd-deployment.yaml index dd0c933..2764651 100644 --- a/apps/base/mail/rspamd-deployment.yaml +++ b/apps/base/mail/rspamd-deployment.yaml @@ -46,11 +46,11 @@ spec: secretKeyRef: name: rspamd-litellm key: key - - name: RSPAMD_CONTROLLER_PASSWORD + - name: RSPAMD_CONTROLLER_PASSWORD_HASH valueFrom: secretKeyRef: name: rspamd-controller - key: password + key: password_hash ports: - containerPort: 11332 name: milter @@ -121,7 +121,7 @@ spec: matchLabels: app.kubernetes.io/name: rspamd --- -# Milter for the smarthost (rspamd.mail.svc:11332); controller/web UI is in-cluster only. +# Milter for the smarthost only (rspamd.mail.svc:11332); controller/web UI is in-cluster only. apiVersion: v1 kind: Service metadata: @@ -138,21 +138,3 @@ spec: - name: controller port: 11334 targetPort: controller ---- -# Milter for the DMZ gateways over WireGuard. -apiVersion: v1 -kind: Service -metadata: - name: rspamd-milter - namespace: mail - annotations: - purelb.io/addresses: 198.18.200.17 - purelb.io/service-group: common -spec: - type: LoadBalancer - selector: - app.kubernetes.io/name: rspamd - ports: - - name: milter - port: 11332 - targetPort: milter diff --git a/apps/base/mail/rspamd-valkey.yaml b/apps/base/mail/rspamd-valkey.yaml index ecf49e1..4d9cd67 100644 --- a/apps/base/mail/rspamd-valkey.yaml +++ b/apps/base/mail/rspamd-valkey.yaml @@ -1,9 +1,9 @@ --- # Sentinel-managed Valkey for rspamd (bayes, history, ratelimit, gpt cache). # rspamd has no Redis Cluster support, so the valkey-operator (cluster mode) -# cannot be used. Pod 0 seeds a fresh set; afterwards start.sh follows the -# master the sentinels report. Hostnames carry a trailing dot so lookups of a -# deleted pod fail fast instead of walking the search list (sentinel TILT). +# cannot be used. start.sh follows the master the sentinels report; on a full +# restart the highest replication offset wins. Hostnames carry a trailing dot so +# lookups of a deleted pod fail fast instead of walking the search list (sentinel TILT). apiVersion: apps/v1 kind: StatefulSet metadata: @@ -13,6 +13,7 @@ metadata: configmap.reloader.stakater.com/reload: "rspamd-valkey-scripts" spec: serviceName: rspamd-valkey + podManagementPolicy: Parallel replicas: 3 selector: matchLabels: @@ -51,7 +52,7 @@ spec: protocol: TCP readinessProbe: exec: - command: ["valkey-cli", "ping"] + command: ["sh", "-c", "valkey-cli info replication | grep -q '^master_host:127.0.0.1' && exit 1; valkey-cli ping | grep -q PONG"] initialDelaySeconds: 5 periodSeconds: 10 livenessProbe: diff --git a/apps/base/mail/rspamd-valkey/start.sh b/apps/base/mail/rspamd-valkey/start.sh index 9609465..b30cdce 100755 --- a/apps/base/mail/rspamd-valkey/start.sh +++ b/apps/base/mail/rspamd-valkey/start.sh @@ -1,20 +1,66 @@ #!/bin/sh -# Usage: start.sh valkey|sentinel. Ask peer sentinels for the current master; -# with no answer (fresh cluster) pod 0 is the master. +# Usage: start.sh valkey|sentinel. valkey boots read-only as a replica of +# 127.0.0.1:1 ("undecided") so peers can compare offsets; decide() then follows +# the sentinels' master, a live peer master, or the highest-offset node. set -eu name="$(hostname)" sts="${name%-*}" self="${name}${DOMAIN_SUFFIX}" -master="" -for i in 0 1 2; do - peer="${sts}-${i}${DOMAIN_SUFFIX}" - [ "$peer" = "$self" ] && continue - master="$(timeout 3 valkey-cli -h "$peer" -p 26379 --raw sentinel get-master-addr-by-name rspamd 2>/dev/null | head -n1 || true)" - [ -n "$master" ] && break -done -[ -n "$master" ] || master="${sts}-0${DOMAIN_SUFFIX}" + +info() { + timeout 3 valkey-cli -h "$1" info replication 2>/dev/null | tr -d '\r' || true +} + +field() { + printf '%s\n' "$1" | sed -n "s/^$2://p" +} + +undecided() { + [ "$(field "$1" master_host)" = 127.0.0.1 ] +} + +choose_master() { + deadline=$(($(date +%s) + 60)) + while :; do + best="" + best_off=-1 + answered=0 + for i in 0 1 2; do + peer="${sts}-${i}${DOMAIN_SUFFIX}" + if [ "$peer" = "$self" ]; then + r="$(info 127.0.0.1)" + else + m="$(timeout 3 valkey-cli -h "$peer" -p 26379 --raw sentinel get-master-addr-by-name rspamd 2>/dev/null | head -n1 || true)" + [ -n "$m" ] && { echo "$m"; return; } + r="$(info "$peer")" + [ -n "$r" ] || continue + answered=$((answered + 1)) + [ "$(field "$r" role)" = master ] && { echo "$peer"; return; } + fi + undecided "$r" || continue + off="$(field "$r" master_repl_offset)" + [ "${off:-0}" -gt "$best_off" ] && best="$peer" best_off="${off:-0}" + done + if [ "$best" = "$self" ] && { [ "$answered" -eq 2 ] || [ "$(date +%s)" -ge "$deadline" ]; }; then + echo "$self" + return + fi + sleep 3 + done +} + +decide() { + until valkey-cli ping >/dev/null 2>&1; do sleep 1; done + master="$(choose_master)" + target="$master 6379" + [ "$master" = "$self" ] && target="no one" + until valkey-cli replicaof $target; do sleep 1; done +} if [ "$1" = sentinel ]; then + until r="$(info 127.0.0.1)" && [ -n "$(field "$r" role)" ] && ! undecided "$r"; do sleep 2; done + master="$self" + [ "$(field "$r" role)" = master ] || master="$(field "$r" master_host)" cat > /sentinel/sentinel.conf <