Scan inbound and outbound via the smarthost only, harden rspamd and Valkey

- tag inbound instead of rejecting, keep virus reject and tempfail when ClamAV fails
- score inbound against the first hop before the edge relays
- sign and skip GPT only for smarthost mynetworks sources
- elect the highest-offset Valkey on full restart, cap memory at 384mb
- read a hashed controller password

apps/base/mail/kustomization.yaml

@@ -23,11 +23,16 @@ configMapGenerator:
       - rspamd/actions.conf
       - rspamd/antivirus.conf
       - rspamd/dkim_signing.conf
+      - rspamd/external_relay.conf
+      - rspamd/force_actions.conf
       - rspamd/gpt.conf
       - rspamd/greylist.conf
       - rspamd/history_redis.conf
+      - rspamd/local_networks.map
+      - rspamd/milter_headers.conf
       - rspamd/options.inc
       - rspamd/redis.conf
+      - rspamd/settings.conf
       - rspamd/spamassassin.conf
       - rspamd/worker-controller.inc
       - rspamd/worker-normal.inc

apps/base/mail/rspamd-deployment.yaml

@@ -46,11 +46,11 @@ spec:
                 secretKeyRef:
                   name: rspamd-litellm
                   key: key
-            - name: RSPAMD_CONTROLLER_PASSWORD
+            - name: RSPAMD_CONTROLLER_PASSWORD_HASH
               valueFrom:
                 secretKeyRef:
                   name: rspamd-controller
-                  key: password
+                  key: password_hash
           ports:
             - containerPort: 11332
               name: milter
@@ -121,7 +121,7 @@ spec:
     matchLabels:
       app.kubernetes.io/name: rspamd
 ---
-# Milter for the smarthost (rspamd.mail.svc:11332); controller/web UI is in-cluster only.
+# Milter for the smarthost only (rspamd.mail.svc:11332); controller/web UI is in-cluster only.
 apiVersion: v1
 kind: Service
 metadata:
@@ -138,21 +138,3 @@ spec:
     - name: controller
       port: 11334
       targetPort: controller
----
-# Milter for the DMZ gateways over WireGuard.
-apiVersion: v1
-kind: Service
-metadata:
-  name: rspamd-milter
-  namespace: mail
-  annotations:
-    purelb.io/addresses: 198.18.200.17
-    purelb.io/service-group: common
-spec:
-  type: LoadBalancer
-  selector:
-    app.kubernetes.io/name: rspamd
-  ports:
-    - name: milter
-      port: 11332
-      targetPort: milter

apps/base/mail/rspamd-valkey.yaml

@@ -1,9 +1,9 @@
 ---
 # Sentinel-managed Valkey for rspamd (bayes, history, ratelimit, gpt cache).
 # rspamd has no Redis Cluster support, so the valkey-operator (cluster mode)
-# cannot be used. Pod 0 seeds a fresh set; afterwards start.sh follows the
-# master the sentinels report. Hostnames carry a trailing dot so lookups of a
-# deleted pod fail fast instead of walking the search list (sentinel TILT).
+# cannot be used. start.sh follows the master the sentinels report; on a full
+# restart the highest replication offset wins. Hostnames carry a trailing dot so
+# lookups of a deleted pod fail fast instead of walking the search list (sentinel TILT).
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -13,6 +13,7 @@ metadata:
     configmap.reloader.stakater.com/reload: "rspamd-valkey-scripts"
 spec:
   serviceName: rspamd-valkey
+  podManagementPolicy: Parallel
   replicas: 3
   selector:
     matchLabels:
@@ -51,7 +52,7 @@ spec:
               protocol: TCP
           readinessProbe:
             exec:
-              command: ["valkey-cli", "ping"]
+              command: ["sh", "-c", "valkey-cli info replication | grep -q '^master_host:127.0.0.1' && exit 1; valkey-cli ping | grep -q PONG"]
             initialDelaySeconds: 5
             periodSeconds: 10
           livenessProbe:

apps/base/mail/rspamd-valkey/start.sh

@@ -1,20 +1,66 @@
 #!/bin/sh
-# Usage: start.sh valkey|sentinel. Ask peer sentinels for the current master;
-# with no answer (fresh cluster) pod 0 is the master.
+# Usage: start.sh valkey|sentinel. valkey boots read-only as a replica of
+# 127.0.0.1:1 ("undecided") so peers can compare offsets; decide() then follows
+# the sentinels' master, a live peer master, or the highest-offset node.
 set -eu
 name="$(hostname)"
 sts="${name%-*}"
 self="${name}${DOMAIN_SUFFIX}"
-master=""
-for i in 0 1 2; do
-  peer="${sts}-${i}${DOMAIN_SUFFIX}"
-  [ "$peer" = "$self" ] && continue
-  master="$(timeout 3 valkey-cli -h "$peer" -p 26379 --raw sentinel get-master-addr-by-name rspamd 2>/dev/null | head -n1 || true)"
-  [ -n "$master" ] && break
-done
-[ -n "$master" ] || master="${sts}-0${DOMAIN_SUFFIX}"
+
+info() {
+  timeout 3 valkey-cli -h "$1" info replication 2>/dev/null | tr -d '\r' || true
+}
+
+field() {
+  printf '%s\n' "$1" | sed -n "s/^$2://p"
+}
+
+undecided() {
+  [ "$(field "$1" master_host)" = 127.0.0.1 ]
+}
+
+choose_master() {
+  deadline=$(($(date +%s) + 60))
+  while :; do
+    best=""
+    best_off=-1
+    answered=0
+    for i in 0 1 2; do
+      peer="${sts}-${i}${DOMAIN_SUFFIX}"
+      if [ "$peer" = "$self" ]; then
+        r="$(info 127.0.0.1)"
+      else
+        m="$(timeout 3 valkey-cli -h "$peer" -p 26379 --raw sentinel get-master-addr-by-name rspamd 2>/dev/null | head -n1 || true)"
+        [ -n "$m" ] && { echo "$m"; return; }
+        r="$(info "$peer")"
+        [ -n "$r" ] || continue
+        answered=$((answered + 1))
+        [ "$(field "$r" role)" = master ] && { echo "$peer"; return; }
+      fi
+      undecided "$r" || continue
+      off="$(field "$r" master_repl_offset)"
+      [ "${off:-0}" -gt "$best_off" ] && best="$peer" best_off="${off:-0}"
+    done
+    if [ "$best" = "$self" ] && { [ "$answered" -eq 2 ] || [ "$(date +%s)" -ge "$deadline" ]; }; then
+      echo "$self"
+      return
+    fi
+    sleep 3
+  done
+}
+
+decide() {
+  until valkey-cli ping >/dev/null 2>&1; do sleep 1; done
+  master="$(choose_master)"
+  target="$master 6379"
+  [ "$master" = "$self" ] && target="no one"
+  until valkey-cli replicaof $target; do sleep 1; done
+}
 
 if [ "$1" = sentinel ]; then
+  until r="$(info 127.0.0.1)" && [ -n "$(field "$r" role)" ] && ! undecided "$r"; do sleep 2; done
+  master="$self"
+  [ "$(field "$r" role)" = master ] || master="$(field "$r" master_host)"
   cat > /sentinel/sentinel.conf <<CONF
 port 26379
 protected-mode no
@@ -29,6 +75,8 @@ CONF
   exec valkey-sentinel /sentinel/sentinel.conf
 fi
 
-set -- --dir /data --appendonly yes --protected-mode no --replica-announce-ip "$self"
-[ "$master" = "$self" ] || set -- "$@" --replicaof "$master" 6379
-exec valkey-server "$@"
+decide &
+# RDB only: an AOF reload resets the replication offset that choose_master compares.
+exec valkey-server --dir /data --appendonly no --save "60 1" \
+  --maxmemory 384mb --maxmemory-policy volatile-ttl \
+  --protected-mode no --replica-announce-ip "$self" --replicaof 127.0.0.1 1

apps/base/mail/rspamd-vault.yaml

@@ -60,7 +60,7 @@ spec:
   type: kv-v2
   vaultAuthRef: default
 ---
-# Controller (web UI) password (field `password`).
+# Controller (web UI) password; config reads the `rspamadm pw` hash (field `password_hash`).
 apiVersion: secrets.hashicorp.com/v1beta1
 kind: VaultStaticSecret
 metadata:

apps/base/mail/rspamd/dkim_signing.conf

@@ -1,5 +1,4 @@
-# Sign outbound only: authenticated or local_addrs senders, never inbound.
-sign_authenticated = true;
+sign_authenticated = false;
 sign_local = true;
 sign_inbound = false;
 use_domain = "header";

apps/base/mail/rspamd/external_relay.conf

@@ -0,0 +1,8 @@
+# Edge gateway public IPs (smarthost gateways.cidr): score the first hop before them.
+enabled = true;
+rules {
+  EDGE_RELAY {
+    strategy = "ip_map";
+    ip_map = ["192.0.2.1/32", "192.0.2.2/32"];
+  }
+}

apps/base/mail/rspamd/force_actions.conf

@@ -0,0 +1,7 @@
+rules {
+  CLAMAV_UNAVAILABLE {
+    action = "soft reject";
+    expression = "CLAM_VIRUS_FAIL";
+    message = "Virus scanner unavailable, try again later";
+  }
+}

apps/base/mail/rspamd/gpt.conf

@@ -10,12 +10,9 @@ model_parameters = {
   }
 };
 timeout = 10s;
-# Only ask the model about inbound mail whose score is uncertain (2 <= score < 12).
+# Inbound only (settings.conf disables it outbound); only uncertain scores (2 <= score < 12).
 condition = <<EOD
 return function(task)
-  if task:get_user() then return false, 'authenticated' end
-  local ip = task:get_from_ip()
-  if ip and ip:is_local() then return false, 'local sender' end
   local res = task:get_metric_result()
   if not res then return false, 'no result' end
   if res.passthrough then return false, 'passthrough' end

apps/base/mail/rspamd/local_networks.map

@@ -0,0 +1,30 @@
+# Outbound sources (smarthost mynetworks): DKIM-signed, scored with reject, no GPT.
+127.0.0.0/8
+::1
+10.42.0.0/16
+10.10.12.200/32
+198.18.1.10/32
+198.18.2.160/27
+198.18.2.192/27
+198.18.13.0/24
+198.18.14.0/24
+198.18.15.0/24
+198.18.16.0/24
+198.18.17.0/24
+198.18.18.0/24
+198.18.19.0/24
+198.18.20.0/24
+198.18.21.0/24
+198.18.22.0/24
+198.18.23.0/24
+198.18.25.0/24
+198.18.26.0/24
+198.18.27.0/24
+198.18.28.0/24
+198.18.29.0/24
+198.18.64.0/24
+198.18.65.0/24
+198.18.66.0/24
+198.18.67.0/24
+198.18.68.0/24
+198.18.69.0/24

apps/base/mail/rspamd/milter_headers.conf

@@ -0,0 +1,2 @@
+use = ["x-spamd-result", "x-spam-status", "x-spam-level", "authentication-results"];
+skip_local = true;

apps/base/mail/rspamd/options.inc

@@ -1,3 +1,2 @@
-# Pod network (Stalwart via the smarthost) and lab networks: local, so outbound mail from them is DKIM-signed.
-local_addrs = ["127.0.0.0/8", "::1", "10.42.0.0/16", "198.18.0.0/16"];
+local_addrs = "/etc/rspamd/local.d/local_networks.map";
 task_timeout = 20s;

apps/base/mail/rspamd/settings.conf

@@ -0,0 +1,19 @@
+outbound {
+  priority = high;
+  ip_map = "/etc/rspamd/local.d/local_networks.map";
+  apply {
+    symbols_disabled = ["GPT_CHECK"];
+  }
+}
+inbound {
+  priority = high;
+  ip_map = "/etc/rspamd/local.d/local_networks.map";
+  inverse = true;
+  apply {
+    symbols_disabled = ["DKIM_SIGNED"];
+    # Tag only; reject stays defined (unreachable by score) so ClamAV's virus reject still applies.
+    actions {
+      reject = 1000;
+    }
+  }
+}

apps/base/mail/rspamd/worker-controller.inc

@@ -1,3 +1,3 @@
-password = "{= env.CONTROLLER_PASSWORD =}";
-enable_password = "{= env.CONTROLLER_PASSWORD =}";
+password = "{= env.CONTROLLER_PASSWORD_HASH =}";
+enable_password = "{= env.CONTROLLER_PASSWORD_HASH =}";
 allow_file_and_shm_inputs = false;