Merge remote-tracking branch 'origin/benvin/mail-stalwart' into benvin/mail-sieve-reconcile

2026-10-12 22:43:56 +11:00
parent b9624da169 869dfb7ccc
commit eabeb00cfa
24 changed files with 728 additions and 0 deletions
@@ -0,0 +1,87 @@
+---
+# clamd + freshclam (the image's init runs both); signatures come from database.clamav.net.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: clamav
+  namespace: mail
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: clamav
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: clamav
+    spec:
+      automountServiceAccountToken: false
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: clamav
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: clamav
+          image: docker.io/clamav/clamav:1.5.4
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 3310
+              name: clamd
+              protocol: TCP
+          startupProbe:
+            tcpSocket:
+              port: clamd
+            periodSeconds: 10
+            failureThreshold: 60
+          readinessProbe:
+            tcpSocket:
+              port: clamd
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket:
+              port: clamd
+            periodSeconds: 30
+          resources:
+            requests:
+              cpu: 100m
+              memory: 1536Mi
+            limits:
+              cpu: "2"
+              memory: 3Gi
+          volumeMounts:
+            - name: db
+              mountPath: /var/lib/clamav
+      volumes:
+        - name: db
+          emptyDir: {}
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: clamav
+  namespace: mail
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: clamav
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: clamav
+  namespace: mail
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: clamav
+  ports:
+    - name: clamd
+      port: 3310
+      targetPort: clamd
@@ -3,11 +3,16 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization

 resources:
+  - clamav.yaml
  - cnpg_backup.yaml
  - cnpg_cluster.yaml
  - gateway.yaml
  - httproute.yaml
  - namespace.yaml
+  - rspamd-deployment.yaml
+  - rspamd-sa-rules.yaml
+  - rspamd-valkey.yaml
+  - rspamd-vault.yaml
  - smarthost-networkpolicy.yaml
  - smarthost-pdb.yaml
  - smarthost-service.yaml
@@ -26,6 +31,34 @@ resources:
  - vmpodscrape.yaml

 configMapGenerator:
+  - name: rspamd-config
+    namespace: mail
+    options:
+      disableNameSuffixHash: true
+    files:
+      - rspamd/actions.conf
+      - rspamd/antivirus.conf
+      - rspamd/dkim_signing.conf
+      - rspamd/external_relay.conf
+      - rspamd/force_actions.conf
+      - rspamd/gpt.conf
+      - rspamd/greylist.conf
+      - rspamd/history_redis.conf
+      - rspamd/local_networks.map
+      - rspamd/milter_headers.conf
+      - rspamd/options.inc
+      - rspamd/redis.conf
+      - rspamd/settings.conf
+      - rspamd/spamassassin.conf
+      - rspamd/worker-controller.inc
+      - rspamd/worker-normal.inc
+      - rspamd/worker-proxy.inc
+  - name: rspamd-valkey-scripts
+    namespace: mail
+    options:
+      disableNameSuffixHash: true
+    files:
+      - rspamd-valkey/start.sh
  - name: smarthost-config
    namespace: mail
    options:
@@ -0,0 +1,140 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rspamd
+  namespace: mail
+  annotations:
+    reloader.stakater.com/auto: "true"
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: rspamd
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: rspamd
+    spec:
+      serviceAccountName: rspamd
+      automountServiceAccountToken: false
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: rspamd
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 11333
+        runAsGroup: 11333
+        fsGroup: 11333
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: rspamd
+          image: docker.io/rspamd/rspamd:4.2.2
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: RSPAMD_LOCAL_ADDR
+              value: "*"
+            - name: RSPAMD_LOG_TYPE
+              value: console
+            - name: RSPAMD_LITELLM_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: rspamd-litellm
+                  key: key
+            - name: RSPAMD_CONTROLLER_PASSWORD_HASH
+              valueFrom:
+                secretKeyRef:
+                  name: rspamd-controller
+                  key: password_hash
+          ports:
+            - containerPort: 11332
+              name: milter
+              protocol: TCP
+            - containerPort: 11334
+              name: controller
+              protocol: TCP
+          readinessProbe:
+            tcpSocket:
+              port: milter
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: controller
+            initialDelaySeconds: 60
+            periodSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 250m
+              memory: 512Mi
+            limits:
+              cpu: "2"
+              memory: 1536Mi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/rspamd/local.d
+              readOnly: true
+            - name: sa-rules
+              mountPath: /etc/rspamd/sa-rules
+              readOnly: true
+            - name: dkim
+              mountPath: /etc/rspamd/dkim
+              readOnly: true
+            - name: data
+              mountPath: /var/lib/rspamd
+      volumes:
+        - name: config
+          configMap:
+            name: rspamd-config
+        - name: sa-rules
+          configMap:
+            name: rspamd-sa-rules
+        - name: dkim
+          secret:
+            secretName: dkim-unkin-net
+            defaultMode: 0440
+            items:
+              - key: private_key
+                path: unkin.net.mail.key
+        - name: data
+          emptyDir: {}
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: rspamd
+  namespace: mail
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: rspamd
+---
+# Milter for the smarthost only (rspamd.mail.svc:11332); controller/web UI is in-cluster only.
+apiVersion: v1
+kind: Service
+metadata:
+  name: rspamd
+  namespace: mail
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: rspamd
+  ports:
+    - name: milter
+      port: 11332
+      targetPort: milter
+    - name: controller
+      port: 11334
+      targetPort: controller
@@ -0,0 +1,8 @@
+---
+# SpamAssassin rules loaded by the rspamd spamassassin module (*.cf).
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: rspamd-sa-rules
+  namespace: mail
+data: {}
@@ -0,0 +1,178 @@
+---
+# Sentinel-managed Valkey for rspamd (bayes, history, ratelimit, gpt cache).
+# rspamd has no Redis Cluster support, so the valkey-operator (cluster mode)
+# cannot be used. start.sh follows the master the sentinels report; on a full
+# restart the highest replication offset wins. Hostnames carry a trailing dot so
+# lookups of a deleted pod fail fast instead of walking the search list (sentinel TILT).
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: rspamd-valkey
+  namespace: mail
+  annotations:
+    configmap.reloader.stakater.com/reload: "rspamd-valkey-scripts"
+spec:
+  serviceName: rspamd-valkey
+  podManagementPolicy: Parallel
+  replicas: 3
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: rspamd-valkey
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: rspamd-valkey
+    spec:
+      automountServiceAccountToken: false
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: rspamd-valkey
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 999
+        runAsGroup: 999
+        fsGroup: 999
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: valkey
+          image: docker.io/valkey/valkey:9.0.0
+          imagePullPolicy: IfNotPresent
+          command: ["sh", "/scripts/start.sh", "valkey"]
+          env:
+            - name: DOMAIN_SUFFIX
+              value: .rspamd-valkey.mail.svc.cluster.local.
+          ports:
+            - containerPort: 6379
+              name: valkey
+              protocol: TCP
+          readinessProbe:
+            exec:
+              command: ["sh", "-c", "valkey-cli info replication | grep -q '^master_host:127.0.0.1' && exit 1; valkey-cli ping | grep -q PONG"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            exec:
+              command: ["valkey-cli", "ping"]
+            initialDelaySeconds: 30
+            periodSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+          volumeMounts:
+            - name: data
+              mountPath: /data
+            - name: scripts
+              mountPath: /scripts
+              readOnly: true
+        - name: sentinel
+          image: docker.io/valkey/valkey:9.0.0
+          imagePullPolicy: IfNotPresent
+          command: ["sh", "/scripts/start.sh", "sentinel"]
+          env:
+            - name: DOMAIN_SUFFIX
+              value: .rspamd-valkey.mail.svc.cluster.local.
+          ports:
+            - containerPort: 26379
+              name: sentinel
+              protocol: TCP
+          readinessProbe:
+            exec:
+              command: ["valkey-cli", "-p", "26379", "ping"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            exec:
+              command: ["valkey-cli", "-p", "26379", "ping"]
+            initialDelaySeconds: 30
+            periodSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 64Mi
+          volumeMounts:
+            - name: sentinel
+              mountPath: /sentinel
+            - name: scripts
+              mountPath: /scripts
+              readOnly: true
+      volumes:
+        - name: scripts
+          configMap:
+            name: rspamd-valkey-scripts
+        - name: sentinel
+          emptyDir: {}
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: [ReadWriteOnce]
+        storageClassName: cephrbd-fast-delete
+        resources:
+          requests:
+            storage: 5Gi
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: rspamd-valkey
+  namespace: mail
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: rspamd-valkey
+---
+# Stable per-pod DNS, published before readiness so replicas can find the seed.
+apiVersion: v1
+kind: Service
+metadata:
+  name: rspamd-valkey
+  namespace: mail
+spec:
+  clusterIP: None
+  publishNotReadyAddresses: true
+  selector:
+    app.kubernetes.io/name: rspamd-valkey
+  ports:
+    - name: valkey
+      port: 6379
+      targetPort: valkey
+    - name: sentinel
+      port: 26379
+      targetPort: sentinel
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: rspamd-valkey-sentinel
+  namespace: mail
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: rspamd-valkey
+  ports:
+    - name: sentinel
+      port: 26379
+      targetPort: sentinel
@@ -0,0 +1,82 @@
+#!/bin/sh
+# Usage: start.sh valkey|sentinel. valkey boots read-only as a replica of
+# 127.0.0.1:1 ("undecided") so peers can compare offsets; decide() then follows
+# the sentinels' master, a live peer master, or the highest-offset node.
+set -eu
+name="$(hostname)"
+sts="${name%-*}"
+self="${name}${DOMAIN_SUFFIX}"
+
+info() {
+  timeout 3 valkey-cli -h "$1" info replication 2>/dev/null | tr -d '\r' || true
+}
+
+field() {
+  printf '%s\n' "$1" | sed -n "s/^$2://p"
+}
+
+undecided() {
+  [ "$(field "$1" master_host)" = 127.0.0.1 ]
+}
+
+choose_master() {
+  deadline=$(($(date +%s) + 60))
+  while :; do
+    best=""
+    best_off=-1
+    answered=0
+    for i in 0 1 2; do
+      peer="${sts}-${i}${DOMAIN_SUFFIX}"
+      if [ "$peer" = "$self" ]; then
+        r="$(info 127.0.0.1)"
+      else
+        m="$(timeout 3 valkey-cli -h "$peer" -p 26379 --raw sentinel get-master-addr-by-name rspamd 2>/dev/null | head -n1 || true)"
+        [ -n "$m" ] && { echo "$m"; return; }
+        r="$(info "$peer")"
+        [ -n "$r" ] || continue
+        answered=$((answered + 1))
+        [ "$(field "$r" role)" = master ] && { echo "$peer"; return; }
+      fi
+      undecided "$r" || continue
+      off="$(field "$r" master_repl_offset)"
+      [ "${off:-0}" -gt "$best_off" ] && best="$peer" best_off="${off:-0}"
+    done
+    if [ "$best" = "$self" ] && { [ "$answered" -eq 2 ] || [ "$(date +%s)" -ge "$deadline" ]; }; then
+      echo "$self"
+      return
+    fi
+    sleep 3
+  done
+}
+
+decide() {
+  until valkey-cli ping >/dev/null 2>&1; do sleep 1; done
+  master="$(choose_master)"
+  target="$master 6379"
+  [ "$master" = "$self" ] && target="no one"
+  until valkey-cli replicaof $target; do sleep 1; done
+}
+
+if [ "$1" = sentinel ]; then
+  until r="$(info 127.0.0.1)" && [ -n "$(field "$r" role)" ] && ! undecided "$r"; do sleep 2; done
+  master="$self"
+  [ "$(field "$r" role)" = master ] || master="$(field "$r" master_host)"
+  cat > /sentinel/sentinel.conf <<CONF
+port 26379
+protected-mode no
+sentinel resolve-hostnames yes
+sentinel announce-hostnames yes
+sentinel announce-ip $self
+sentinel monitor rspamd $master 6379 2
+sentinel down-after-milliseconds rspamd 5000
+sentinel failover-timeout rspamd 60000
+sentinel parallel-syncs rspamd 1
+CONF
+  exec valkey-sentinel /sentinel/sentinel.conf
+fi
+
+decide &
+# RDB only: an AOF reload resets the replication offset that choose_master compares.
+exec valkey-server --dir /data --appendonly no --save "60 1" \
+  --maxmemory 384mb --maxmemory-policy volatile-ttl \
+  --protected-mode no --replica-announce-ip "$self" --replicaof 127.0.0.1 1
@@ -0,0 +1,79 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: rspamd
+  namespace: mail
+---
+# Separate auth for the litellm engine: the `rspamd` k8s role is the only one
+# whose policy grants litellm/creds/rspamd.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: rspamd
+  namespace: mail
+spec:
+  allowedNamespaces:
+    - mail
+  kubernetes:
+    audiences:
+      - vault
+    role: rspamd
+    serviceAccount: rspamd
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default
+---
+# Haiku-scoped LiteLLM virtual key for the gpt module (field `key`).
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultDynamicSecret
+metadata:
+  name: rspamd-litellm
+  namespace: mail
+spec:
+  allowStaticCreds: false
+  destination:
+    create: true
+    name: rspamd-litellm
+    overwrite: true
+  mount: litellm
+  path: creds/rspamd
+  revoke: true
+  vaultAuthRef: rspamd
+---
+# DKIM private key for unkin.net, selector `mail` (field `private_key`).
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: dkim-unkin-net
+  namespace: mail
+spec:
+  destination:
+    create: true
+    name: dkim-unkin-net
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/mail/default/dkim-unkin-net
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+# Controller (web UI) password; config reads the `rspamadm pw` hash (field `password_hash`).
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: rspamd-controller
+  namespace: mail
+spec:
+  destination:
+    create: true
+    name: rspamd-controller
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/mail/default/rspamd-controller
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
@@ -0,0 +1,3 @@
+reject = 15;
+add_header = 6;
+greylist = null;
@@ -0,0 +1,8 @@
+clamav {
+  type = "clamav";
+  servers = "clamav.mail.svc.cluster.local:3310";
+  symbol = "CLAM_VIRUS";
+  action = "reject";
+  scan_mime_parts = true;
+  log_clean = false;
+}
@@ -0,0 +1,7 @@
+sign_authenticated = false;
+sign_local = true;
+sign_inbound = false;
+use_domain = "header";
+allow_hdrfrom_mismatch = false;
+selector = "mail";
+path = "/etc/rspamd/dkim/$domain.$selector.key";
@@ -0,0 +1,8 @@
+# Edge gateway public IPs (smarthost gateways.cidr): score the first hop before them.
+enabled = true;
+rules {
+  EDGE_RELAY {
+    strategy = "ip_map";
+    ip_map = ["192.0.2.1/32", "192.0.2.2/32"];
+  }
+}
@@ -0,0 +1,7 @@
+rules {
+  CLAMAV_UNAVAILABLE {
+    action = "soft reject";
+    expression = "CLAM_VIRUS_FAIL";
+    message = "Virus scanner unavailable, try again later";
+  }
+}
@@ -0,0 +1,22 @@
+enabled = true;
+type = "openai";
+url = "http://litellm.litellm.svc.cluster.local:4000/v1/chat/completions";
+api_key = "{= env.LITELLM_API_KEY =}";
+model = "claude-haiku-4-5";
+model_parameters = {
+  "claude-haiku-4-5" = {
+    max_tokens = 400,
+    temperature = 0.0,
+  }
+};
+timeout = 10s;
+# Inbound only (settings.conf disables it outbound); only uncertain scores (2 <= score < 12).
+condition = <<EOD
+return function(task)
+  local res = task:get_metric_result()
+  if not res then return false, 'no result' end
+  if res.passthrough then return false, 'passthrough' end
+  if res.score < 2 or res.score >= 12 then return false, 'score not uncertain' end
+  return true
+end
+EOD
@@ -0,0 +1 @@
+enabled = false;
@@ -0,0 +1 @@
+nrows = 5000;
@@ -0,0 +1,30 @@
+# Outbound sources (smarthost mynetworks): DKIM-signed, scored with reject, no GPT.
+127.0.0.0/8
+::1
+10.42.0.0/16
+10.10.12.200/32
+198.18.1.10/32
+198.18.2.160/27
+198.18.2.192/27
+198.18.13.0/24
+198.18.14.0/24
+198.18.15.0/24
+198.18.16.0/24
+198.18.17.0/24
+198.18.18.0/24
+198.18.19.0/24
+198.18.20.0/24
+198.18.21.0/24
+198.18.22.0/24
+198.18.23.0/24
+198.18.25.0/24
+198.18.26.0/24
+198.18.27.0/24
+198.18.28.0/24
+198.18.29.0/24
+198.18.64.0/24
+198.18.65.0/24
+198.18.66.0/24
+198.18.67.0/24
+198.18.68.0/24
+198.18.69.0/24
@@ -0,0 +1,2 @@
+use = ["x-spamd-result", "x-spam-status", "x-spam-level", "authentication-results"];
+skip_local = true;
@@ -0,0 +1,2 @@
+local_addrs = "/etc/rspamd/local.d/local_networks.map";
+task_timeout = 20s;
@@ -0,0 +1,4 @@
+sentinels = "rspamd-valkey-sentinel.mail.svc.cluster.local:26379";
+sentinel_masters_pattern = "^rspamd$";
+sentinel_watch_time = 10s;
+servers = "rspamd-valkey-0.rspamd-valkey.mail.svc.cluster.local.:6379";
@@ -0,0 +1,19 @@
+outbound {
+  priority = high;
+  ip_map = "/etc/rspamd/local.d/local_networks.map";
+  apply {
+    symbols_disabled = ["GPT_CHECK"];
+  }
+}
+inbound {
+  priority = high;
+  ip_map = "/etc/rspamd/local.d/local_networks.map";
+  inverse = true;
+  apply {
+    symbols_disabled = ["DKIM_SIGNED"];
+    # Tag only; reject stays defined (unreachable by score) so ClamAV's virus reject still applies.
+    actions {
+      reject = 1000;
+    }
+  }
+}
@@ -0,0 +1 @@
+ruleset = "/etc/rspamd/sa-rules/*.cf";
@@ -0,0 +1,3 @@
+password = "{= env.CONTROLLER_PASSWORD_HASH =}";
+enable_password = "{= env.CONTROLLER_PASSWORD_HASH =}";
+allow_file_and_shm_inputs = false;
@@ -0,0 +1,2 @@
+count = 2;
+allow_file_and_shm_inputs = false;
@@ -0,0 +1 @@
+allow_file_and_shm_inputs = false;