mail: use real edge MTA public IPs

Recursing public names in-cluster SERVFAILs a few percent of cold lookups and loses the cache on every restart, which breaks CI. DNSBLs refuse or poison queries relayed via public resolvers, so rspamd's RBL lookups must keep resolving directly.

- add global forwarders 1.1.1.1 and 8.8.8.8 (forward first) to bind-resolvers
- add empty-forwarder zones to the openforwarder view for the RBL domains rspamd queries, so they resolve iteratively

Reviewed-on: #555
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

apps/base/artifactapi/api-deployment.yaml

@@ -36,7 +36,7 @@ spec:
               mountPath: /combined-certs
       containers:
         - name: api
-          image: git.unkin.net/unkin/artifactapi:v3.12.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.14.0
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000

apps/base/artifactapi/configmap.yaml

@@ -5,6 +5,8 @@ metadata:
   name: api-env
   namespace: artifactapi
 data:
+  ARTIFACTAPI_OIDC_AUDIENCE: artifactapi
+  ARTIFACTAPI_OIDC_ISSUER: https://vault.service.consul:8200/v1/identity/oidc
   DBHOST: postgres-pooler
   DBNAME: artifacts
   DBPORT: "5432"

apps/base/artifactapi/image-keeper.yaml

@@ -0,0 +1,105 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: image-keeper
+  namespace: artifactapi
+spec:
+  selector:
+    matchLabels:
+      app: image-keeper
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: image-keeper
+    spec:
+      automountServiceAccountToken: false
+      priorityClassName: low
+      tolerations:
+        - operator: Exists
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        # artifactapi is distroless with no exit-0 flag, so run a static busybox as `true`
+        - name: copy-true
+          image: busybox:1.37.0-musl
+          imagePullPolicy: IfNotPresent
+          command: ["cp", "/bin/busybox", "/keeper/true"]
+          volumeMounts:
+            - name: keeper
+              mountPath: /keeper
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+        - name: api
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.14.0
+          imagePullPolicy: IfNotPresent
+          command: ["/keeper/true"]
+          volumeMounts:
+            - name: keeper
+              mountPath: /keeper
+              readOnly: true
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+        - name: ui
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.14.0
+          imagePullPolicy: IfNotPresent
+          command: ["true"]
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+      containers:
+        - name: pause
+          image: rancher/mirrored-pause:3.6
+          imagePullPolicy: IfNotPresent
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+      volumes:
+        - name: keeper
+          emptyDir: {}

apps/base/artifactapi/kustomization.yaml

@@ -11,6 +11,7 @@ resources:
   - cnpg_pooler.yaml
   - gateway.yaml
   - httproute.yaml
+  - image-keeper.yaml
   - namespace.yaml
   - oauth2-proxy-configmap.yaml
   - oauth2-proxy-deployment.yaml

apps/base/artifactapi/ui-deployment.yaml

@@ -22,7 +22,7 @@ spec:
       automountServiceAccountToken: true
       containers:
         - name: ui
-          image: git.unkin.net/unkin/artifactapi-ui:v3.12.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.14.0
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 80

apps/base/bind-internal/authoritative/unkin-net/a/kustomization.yaml

@@ -16,5 +16,6 @@ resources:
   - identity.yaml
   - lb1.yaml
   - logviewer.yaml
+  - mail.yaml
   - vlogs.yaml
   - watchstate.yaml

apps/base/bind-internal/authoritative/unkin-net/a/mail.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: mail-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: mail
+  type: A
+  ttl: 600
+  values:
+    # stalwart-lb PureLB VIP (IMAPS, submission, HTTPS).
+    - 198.18.200.18

apps/base/bind-internal/resolvers/acls.yaml

@@ -38,6 +38,9 @@ spec:
   clusterRef: bind-resolvers
   entries:
     - 198.18.24.0/24
+    - 172.18.30.0/24  # BinaryLane DMZ VPC syd (mail edge, via VPN)
+    - 172.18.31.0/24  # BinaryLane DMZ VPC mel (mail edge, via VPN)
+    - 172.18.32.0/24  # BinaryLane DMZ VPC bne (mail edge, via VPN)
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: BindACL

apps/base/bind-internal/resolvers/cluster.yaml

@@ -18,9 +18,10 @@ spec:
       purelb.io/service-group: common
       purelb.io/addresses: 198.18.200.7
       external-dns.alpha.kubernetes.io/hostname: bind-resolvers.k8s.syd1.au.unkin.net
+  # Public names go via global forwarders; RBL zones bypass them (view.yaml).
   forwarders:
-    - 8.8.8.8
     - 1.1.1.1
+    - 8.8.8.8
   # The internal split-horizon zones are served UNSIGNED by the in-cluster
   # authoritative, but their public parents publish DS records (e.g. unkin.net
   # is DNSSEC-signed on the Internet). With dnssec-validation on, the validator

apps/base/bind-internal/resolvers/view.yaml

@@ -21,3 +21,16 @@ spec:
   extraOptions:
     - "allow-recursion { any; }"
     - "allow-query-cache { any; }"
+    # pods have no IPv6 route; v6 NS attempts burn the recursion-query budget
+    - "server ::/0 { bogus yes; }"
+    # RBL zones resolve directly: DNSBLs refuse queries relayed via public resolvers.
+    - 'zone "spamhaus.org" { type forward; forwarders { }; }'
+    - 'zone "mailspike.net" { type forward; forwarders { }; }'
+    - 'zone "spameatingmonkey.net" { type forward; forwarders { }; }'
+    - 'zone "dnswl.org" { type forward; forwarders { }; }'
+    - 'zone "virusfree.cz" { type forward; forwarders { }; }'
+    - 'zone "blocklist.de" { type forward; forwarders { }; }'
+    - 'zone "msbl.org" { type forward; forwarders { }; }'
+    - 'zone "surbl.org" { type forward; forwarders { }; }'
+    - 'zone "uribl.com" { type forward; forwarders { }; }'
+    - 'zone "rspamd.com" { type forward; forwarders { }; }'

apps/base/cert-manager/certificate_wildcard-main-unkin-net.yaml

@@ -1,6 +1,6 @@
 ---
 # Let's Encrypt *.main.unkin.net wildcard for the haproxy edge (pve, arr stack,
-# jellyfin, stalwart webadmin/autoconfig). DNS-01 needs the delegated
+# jellyfin, stalwart webadmin/autoconfig) and the mail smarthost. DNS-01 needs the delegated
 # _acme-challenge.main.unkin.net CNAME in the public unkin.net zone.
 # _acme-challenge.main.unkin.net. CNAME _acme-challenge.main.acme.unkin.net.
 apiVersion: cert-manager.io/v1
@@ -13,9 +13,9 @@ spec:
   secretTemplate:
     annotations:
       reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
-      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy,mail"
       reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
-      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy,mail"
   privateKey:
     size: 4096
   dnsNames:

apps/base/cert-manager/certificate_wildcard-unkin-net.yaml

@@ -14,9 +14,9 @@ spec:
   secretTemplate:
     annotations:
       reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
-      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
       reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
-      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
   privateKey:
     size: 4096
   dnsNames:

apps/base/haproxy/backendtlspolicy.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: BackendTLSPolicy
+metadata:
+  name: haproxy
+  namespace: haproxy
+  labels:
+    app: haproxy
+spec:
+  targetRefs:
+    - group: ""
+      kind: Service
+      name: haproxy
+  validation:
+    wellKnownCACertificates: System
+    hostname: git.unkin.net

apps/base/haproxy/gateway.yaml

@@ -1,10 +1,8 @@
 ---
 # External (DMZ) front for the haproxy edge on the traefik-external LB VIP
-# 198.18.199.0. The :443 listener is TLS Passthrough: haproxy owns the three
-# wildcard certs and terminates behind Traefik, so there are no certificateRefs
-# here. Listener hostnames are deliberately unset and the routes carry the
-# explicit hostname list instead; allowedRoutes Same keeps other namespaces off
-# these listeners.
+# 198.18.199.0. Traefik terminates TLS so HTTP/2 connections coalesced from
+# other *.unkin.net hosts still route by Host; the hop to haproxy is re-encrypted via
+# BackendTLSPolicy. Routes carry the explicit hostname list.
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
@@ -21,11 +19,21 @@ spec:
       allowedRoutes:
         namespaces:
           from: Same
-    - name: https-passthrough
+    - name: https
       port: 443
-      protocol: TLS
+      protocol: HTTPS
       tls:
-        mode: Passthrough
+        mode: Terminate
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: wildcard-unkin-net-tls
+          - group: ""
+            kind: Secret
+            name: wildcard-main-unkin-net-tls
+          - group: ""
+            kind: Secret
+            name: wildcard-ceph-unkin-net-tls
       allowedRoutes:
         namespaces:
           from: Same

apps/base/haproxy/httproute.yaml

@@ -34,3 +34,36 @@ spec:
         - path:
             type: PathPrefix
             value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: haproxy
+  namespace: haproxy
+  labels:
+    app: haproxy
+spec:
+  hostnames:
+    - sonarr.main.unkin.net
+    - radarr.main.unkin.net
+    - lidarr.main.unkin.net
+    - readarr.main.unkin.net
+    - prowlarr.main.unkin.net
+    - nzbget.main.unkin.net
+    - jellyfin.main.unkin.net
+    - fafflix.unkin.net
+    - git.unkin.net
+    - dashboard.ceph.unkin.net
+    - auth.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: haproxy
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: haproxy
+          port: 443
+          weight: 1

apps/base/haproxy/kustomization.yaml

@@ -8,7 +8,7 @@ resources:
   - deployment.yaml
   - service.yaml
   - gateway.yaml
-  - tlsroute.yaml
+  - backendtlspolicy.yaml
   - httproute.yaml
   - pdb.yaml
   - vpa.yaml

apps/base/haproxy/service.yaml

@@ -6,7 +6,7 @@ metadata:
   namespace: haproxy
 spec:
   type: ClusterIP
-  # Reached only by the external Traefik's TLS-passthrough TLSRoute, so the
+  # Reached only by the external Traefik (re-encrypted HTTPRoute), so the
   # peer address here is a Traefik pod, not the client. sessionAffinity is
   # deliberately absent: keyed on ClientIP it would pin whole Traefik pods,
   # not clients. Backend persistence rests on the per-backend SRVNAME cookie.

apps/base/haproxy/tlsroute.yaml

@@ -1,33 +0,0 @@
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: TLSRoute
-metadata:
-  name: haproxy
-  namespace: haproxy
-  labels:
-    app: haproxy
-spec:
-  hostnames:
-    - sonarr.main.unkin.net
-    - radarr.main.unkin.net
-    - lidarr.main.unkin.net
-    - readarr.main.unkin.net
-    - prowlarr.main.unkin.net
-    - nzbget.main.unkin.net
-    - jellyfin.main.unkin.net
-    - fafflix.unkin.net
-    - git.unkin.net
-    - dashboard.ceph.unkin.net
-    - auth.unkin.net
-  parentRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: haproxy
-      sectionName: https-passthrough
-  rules:
-    - backendRefs:
-        - group: ""
-          kind: Service
-          name: haproxy
-          port: 443
-          weight: 1

apps/base/mail/clamav.yaml

@@ -0,0 +1,87 @@
+---
+# clamd + freshclam (the image's init runs both); signatures come from database.clamav.net.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: clamav
+  namespace: mail
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: clamav
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: clamav
+    spec:
+      automountServiceAccountToken: false
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: clamav
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: clamav
+          image: docker.io/clamav/clamav:1.5.4
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 3310
+              name: clamd
+              protocol: TCP
+          startupProbe:
+            tcpSocket:
+              port: clamd
+            periodSeconds: 10
+            failureThreshold: 60
+          readinessProbe:
+            tcpSocket:
+              port: clamd
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket:
+              port: clamd
+            periodSeconds: 30
+          resources:
+            requests:
+              cpu: 100m
+              memory: 1536Mi
+            limits:
+              cpu: "2"
+              memory: 3Gi
+          volumeMounts:
+            - name: db
+              mountPath: /var/lib/clamav
+      volumes:
+        - name: db
+          emptyDir: {}
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: clamav
+  namespace: mail
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: clamav
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: clamav
+  namespace: mail
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: clamav
+  ports:
+    - name: clamd
+      port: 3310
+      targetPort: clamd

apps/base/mail/cnpg_backup.yaml

@@ -0,0 +1,40 @@
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: cnpg-mail-backup
+  namespace: mail
+spec:
+  displayName: "CNPG backup owner (mail)"
+  uid: cnpg-mail-backup
+  maxBuckets: 5
+  secretName: cnpg-mail-backup-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: cnpg-mail
+  namespace: mail
+spec:
+  placementTarget: ec
+  bucketName: cnpg-mail
+  ownerRef: cnpg-mail-backup
+  versioning: false
+  tags:
+    app: mail
+    purpose: cnpg-backup
+  retainOnDelete: true
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-mail-nightly
+  namespace: mail
+spec:
+  schedule: "0 20 4 * * *"
+  immediate: false
+  backupOwnerReference: self
+  method: barmanObjectStore
+  cluster:
+    name: stalwart-db

apps/base/mail/cnpg_cluster.yaml

@@ -0,0 +1,118 @@
+---
+# Stalwart metadata store. No bootstrap secret: CNPG mints stalwart-db-app
+# (username/password/dbname/uri) for Stalwart to consume.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: stalwart-db
+  namespace: mail
+spec:
+  affinity:
+    podAntiAffinityType: preferred
+  backup:
+    retentionPolicy: 30d
+    barmanObjectStore:
+      destinationPath: s3://cnpg-mail
+      endpointURL: https://s3.ceph.unkin.net
+      endpointCA:
+        name: vault-ca-cert
+        key: ca.crt
+      s3Credentials:
+        accessKeyId:
+          name: cnpg-mail-backup-s3
+          key: AWS_ACCESS_KEY_ID
+        secretAccessKey:
+          name: cnpg-mail-backup-s3
+          key: AWS_SECRET_ACCESS_KEY
+      serverName: stalwart-db
+      data:
+        compression: bzip2
+        jobs: 2
+      wal:
+        compression: zstd
+        maxParallel: 2
+  bootstrap:
+    initdb:
+      database: stalwart
+      encoding: UTF8
+      localeCType: C
+      localeCollate: C
+      owner: stalwart
+  enablePDB: true
+  enableSuperuserAccess: false
+  failoverDelay: 0
+  imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie
+  instances: 3
+  logLevel: info
+  maxSyncReplicas: 0
+  minSyncReplicas: 0
+  monitoring:
+    customQueriesConfigMap:
+      - key: queries
+        name: cnpg-default-monitoring
+    disableDefaultQueries: false
+    enablePodMonitor: false
+  postgresql:
+    parameters:
+      archive_mode: "on"
+      archive_timeout: 5min
+      dynamic_shared_memory_type: posix
+      effective_cache_size: 256MB
+      full_page_writes: "on"
+      hot_standby_feedback: "on"
+      log_destination: csvlog
+      log_directory: /controller/log
+      log_filename: postgres
+      log_rotation_age: "0"
+      log_rotation_size: "0"
+      log_truncate_on_rotation: "false"
+      logging_collector: "on"
+      max_connections: "200"
+      max_parallel_workers: "16"
+      max_replication_slots: "16"
+      max_worker_processes: "16"
+      pg_stat_statements.max: "10000"
+      pg_stat_statements.track: top
+      shared_buffers: 128MB
+      shared_memory_type: mmap
+      ssl_max_protocol_version: TLSv1.3
+      ssl_min_protocol_version: TLSv1.3
+      wal_keep_size: 256MB
+      wal_level: logical
+      wal_log_hints: "on"
+      wal_receiver_timeout: 5s
+      wal_sender_timeout: 5s
+    shared_preload_libraries:
+      - pg_stat_statements
+    syncReplicaElectionConstraint:
+      enabled: false
+  primaryUpdateMethod: restart
+  primaryUpdateStrategy: unsupervised
+  probes:
+    liveness:
+      isolationCheck:
+        connectionTimeout: 1000
+        enabled: true
+        requestTimeout: 1000
+  replicationSlots:
+    highAvailability:
+      enabled: true
+      slotPrefix: _cnpg_
+    synchronizeReplicas:
+      enabled: true
+    updateInterval: 30
+  resources:
+    limits:
+      cpu: "1"
+      memory: 1Gi
+    requests:
+      cpu: 50m
+      memory: 512Mi
+  smartShutdownTimeout: 180
+  startDelay: 3600
+  stopDelay: 1800
+  storage:
+    resizeInUseVolumes: true
+    size: 10Gi
+    storageClass: cephrbd-fast-delete
+  switchoverDelay: 3600

apps/base/mail/gateway.yaml

@@ -0,0 +1,33 @@
+---
+# External (DMZ) front for mail.unkin.net web/JMAP via the external Traefik
+# (LB VIP 198.18.199.0), terminating with the reflected *.unkin.net wildcard.
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: external
+  name: stalwart-external
+  namespace: mail
+spec:
+  gatewayClassName: traefik-external
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      hostname: mail.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https
+      port: 443
+      protocol: HTTPS
+      hostname: mail.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: wildcard-unkin-net-tls

apps/base/mail/httproute.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: stalwart-external-http-redirect
+  namespace: mail
+spec:
+  hostnames:
+    - mail.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: stalwart-external
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: stalwart-external
+  namespace: mail
+spec:
+  hostnames:
+    - mail.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: stalwart-external
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: stalwart
+          port: 8080
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/mail/kustomization.yaml

@@ -0,0 +1,72 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - clamav.yaml
+  - cnpg_backup.yaml
+  - cnpg_cluster.yaml
+  - gateway.yaml
+  - httproute.yaml
+  - namespace.yaml
+  - rspamd-deployment.yaml
+  - rspamd-sa-rules.yaml
+  - rspamd-valkey.yaml
+  - rspamd-vault.yaml
+  - smarthost-networkpolicy.yaml
+  - smarthost-pdb.yaml
+  - smarthost-service.yaml
+  - smarthost-statefulset.yaml
+  - stalwart-apply-job.yaml
+  - stalwart-apply-rbac.yaml
+  - stalwart-blobs-bucket.yaml
+  - stalwart-configmap.yaml
+  - stalwart-pdb.yaml
+  - stalwart-service.yaml
+  - stalwart-statefulset.yaml
+  - valkey.yaml
+  - vaultauth.yaml
+  - vaultstaticsecret.yaml
+  - vmpodscrape.yaml
+
+configMapGenerator:
+  - name: rspamd-config
+    namespace: mail
+    options:
+      disableNameSuffixHash: true
+    files:
+      - rspamd/actions.conf
+      - rspamd/antivirus.conf
+      - rspamd/dkim_signing.conf
+      - rspamd/external_relay.conf
+      - rspamd/force_actions.conf
+      - rspamd/gpt.conf
+      - rspamd/greylist.conf
+      - rspamd/history_redis.conf
+      - rspamd/local_networks.map
+      - rspamd/milter_headers.conf
+      - rspamd/options.inc
+      - rspamd/redis.conf
+      - rspamd/settings.conf
+      - rspamd/spamassassin.conf
+      - rspamd/worker-controller.inc
+      - rspamd/worker-normal.inc
+      - rspamd/worker-proxy.inc
+  - name: rspamd-valkey-scripts
+    namespace: mail
+    options:
+      disableNameSuffixHash: true
+    files:
+      - rspamd-valkey/start.sh
+  - name: smarthost-config
+    namespace: mail
+    options:
+      disableNameSuffixHash: true
+    files:
+      - smarthost/main.cf
+      - smarthost/master.cf
+      - smarthost/relay_domains
+      - smarthost/transport
+      - smarthost/gateways.cidr
+      - smarthost/mynetworks.cidr
+      - smarthost/tls_policy

apps/base/mail/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: mail

apps/base/mail/rspamd-deployment.yaml

@@ -0,0 +1,140 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rspamd
+  namespace: mail
+  annotations:
+    reloader.stakater.com/auto: "true"
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: rspamd
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: rspamd
+    spec:
+      serviceAccountName: rspamd
+      automountServiceAccountToken: false
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: rspamd
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 11333
+        runAsGroup: 11333
+        fsGroup: 11333
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: rspamd
+          image: docker.io/rspamd/rspamd:4.2.2
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: RSPAMD_LOCAL_ADDR
+              value: "*"
+            - name: RSPAMD_LOG_TYPE
+              value: console
+            - name: RSPAMD_LITELLM_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: rspamd-litellm
+                  key: key
+            - name: RSPAMD_CONTROLLER_PASSWORD_HASH
+              valueFrom:
+                secretKeyRef:
+                  name: rspamd-controller
+                  key: password_hash
+          ports:
+            - containerPort: 11332
+              name: milter
+              protocol: TCP
+            - containerPort: 11334
+              name: controller
+              protocol: TCP
+          readinessProbe:
+            tcpSocket:
+              port: milter
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: controller
+            initialDelaySeconds: 60
+            periodSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 250m
+              memory: 512Mi
+            limits:
+              cpu: "2"
+              memory: 1536Mi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/rspamd/local.d
+              readOnly: true
+            - name: sa-rules
+              mountPath: /etc/rspamd/sa-rules
+              readOnly: true
+            - name: dkim
+              mountPath: /etc/rspamd/dkim
+              readOnly: true
+            - name: data
+              mountPath: /var/lib/rspamd
+      volumes:
+        - name: config
+          configMap:
+            name: rspamd-config
+        - name: sa-rules
+          configMap:
+            name: rspamd-sa-rules
+        - name: dkim
+          secret:
+            secretName: dkim-unkin-net
+            defaultMode: 0440
+            items:
+              - key: private_key
+                path: unkin.net.mail.key
+        - name: data
+          emptyDir: {}
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: rspamd
+  namespace: mail
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: rspamd
+---
+# Milter for the smarthost only (rspamd.mail.svc:11332); controller/web UI is in-cluster only.
+apiVersion: v1
+kind: Service
+metadata:
+  name: rspamd
+  namespace: mail
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: rspamd
+  ports:
+    - name: milter
+      port: 11332
+      targetPort: milter
+    - name: controller
+      port: 11334
+      targetPort: controller

apps/base/mail/rspamd-sa-rules.yaml

@@ -0,0 +1,8 @@
+---
+# SpamAssassin rules loaded by the rspamd spamassassin module (*.cf).
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: rspamd-sa-rules
+  namespace: mail
+data: {}

apps/base/mail/rspamd-valkey.yaml

@@ -0,0 +1,178 @@
+---
+# Sentinel-managed Valkey for rspamd (bayes, history, ratelimit, gpt cache).
+# rspamd has no Redis Cluster support, so the valkey-operator (cluster mode)
+# cannot be used. start.sh follows the master the sentinels report; on a full
+# restart the highest replication offset wins. Hostnames carry a trailing dot so
+# lookups of a deleted pod fail fast instead of walking the search list (sentinel TILT).
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: rspamd-valkey
+  namespace: mail
+  annotations:
+    configmap.reloader.stakater.com/reload: "rspamd-valkey-scripts"
+spec:
+  serviceName: rspamd-valkey
+  podManagementPolicy: Parallel
+  replicas: 3
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: rspamd-valkey
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: rspamd-valkey
+    spec:
+      automountServiceAccountToken: false
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: rspamd-valkey
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 999
+        runAsGroup: 999
+        fsGroup: 999
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: valkey
+          image: docker.io/valkey/valkey:9.0.0
+          imagePullPolicy: IfNotPresent
+          command: ["sh", "/scripts/start.sh", "valkey"]
+          env:
+            - name: DOMAIN_SUFFIX
+              value: .rspamd-valkey.mail.svc.cluster.local.
+          ports:
+            - containerPort: 6379
+              name: valkey
+              protocol: TCP
+          readinessProbe:
+            exec:
+              command: ["sh", "-c", "valkey-cli info replication | grep -q '^master_host:127.0.0.1' && exit 1; valkey-cli ping | grep -q PONG"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            exec:
+              command: ["valkey-cli", "ping"]
+            initialDelaySeconds: 30
+            periodSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+          volumeMounts:
+            - name: data
+              mountPath: /data
+            - name: scripts
+              mountPath: /scripts
+              readOnly: true
+        - name: sentinel
+          image: docker.io/valkey/valkey:9.0.0
+          imagePullPolicy: IfNotPresent
+          command: ["sh", "/scripts/start.sh", "sentinel"]
+          env:
+            - name: DOMAIN_SUFFIX
+              value: .rspamd-valkey.mail.svc.cluster.local.
+          ports:
+            - containerPort: 26379
+              name: sentinel
+              protocol: TCP
+          readinessProbe:
+            exec:
+              command: ["valkey-cli", "-p", "26379", "ping"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            exec:
+              command: ["valkey-cli", "-p", "26379", "ping"]
+            initialDelaySeconds: 30
+            periodSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 64Mi
+          volumeMounts:
+            - name: sentinel
+              mountPath: /sentinel
+            - name: scripts
+              mountPath: /scripts
+              readOnly: true
+      volumes:
+        - name: scripts
+          configMap:
+            name: rspamd-valkey-scripts
+        - name: sentinel
+          emptyDir: {}
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: [ReadWriteOnce]
+        storageClassName: cephrbd-fast-delete
+        resources:
+          requests:
+            storage: 5Gi
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: rspamd-valkey
+  namespace: mail
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: rspamd-valkey
+---
+# Stable per-pod DNS, published before readiness so replicas can find the seed.
+apiVersion: v1
+kind: Service
+metadata:
+  name: rspamd-valkey
+  namespace: mail
+spec:
+  clusterIP: None
+  publishNotReadyAddresses: true
+  selector:
+    app.kubernetes.io/name: rspamd-valkey
+  ports:
+    - name: valkey
+      port: 6379
+      targetPort: valkey
+    - name: sentinel
+      port: 26379
+      targetPort: sentinel
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: rspamd-valkey-sentinel
+  namespace: mail
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: rspamd-valkey
+  ports:
+    - name: sentinel
+      port: 26379
+      targetPort: sentinel

apps/base/mail/rspamd-valkey/start.sh

@@ -0,0 +1,82 @@
+#!/bin/sh
+# Usage: start.sh valkey|sentinel. valkey boots read-only as a replica of
+# 127.0.0.1:1 ("undecided") so peers can compare offsets; decide() then follows
+# the sentinels' master, a live peer master, or the highest-offset node.
+set -eu
+name="$(hostname)"
+sts="${name%-*}"
+self="${name}${DOMAIN_SUFFIX}"
+
+info() {
+  timeout 3 valkey-cli -h "$1" info replication 2>/dev/null | tr -d '\r' || true
+}
+
+field() {
+  printf '%s\n' "$1" | sed -n "s/^$2://p"
+}
+
+undecided() {
+  [ "$(field "$1" master_host)" = 127.0.0.1 ]
+}
+
+choose_master() {
+  deadline=$(($(date +%s) + 60))
+  while :; do
+    best=""
+    best_off=-1
+    answered=0
+    for i in 0 1 2; do
+      peer="${sts}-${i}${DOMAIN_SUFFIX}"
+      if [ "$peer" = "$self" ]; then
+        r="$(info 127.0.0.1)"
+      else
+        m="$(timeout 3 valkey-cli -h "$peer" -p 26379 --raw sentinel get-master-addr-by-name rspamd 2>/dev/null | head -n1 || true)"
+        [ -n "$m" ] && { echo "$m"; return; }
+        r="$(info "$peer")"
+        [ -n "$r" ] || continue
+        answered=$((answered + 1))
+        [ "$(field "$r" role)" = master ] && { echo "$peer"; return; }
+      fi
+      undecided "$r" || continue
+      off="$(field "$r" master_repl_offset)"
+      [ "${off:-0}" -gt "$best_off" ] && best="$peer" best_off="${off:-0}"
+    done
+    if [ "$best" = "$self" ] && { [ "$answered" -eq 2 ] || [ "$(date +%s)" -ge "$deadline" ]; }; then
+      echo "$self"
+      return
+    fi
+    sleep 3
+  done
+}
+
+decide() {
+  until valkey-cli ping >/dev/null 2>&1; do sleep 1; done
+  master="$(choose_master)"
+  target="$master 6379"
+  [ "$master" = "$self" ] && target="no one"
+  until valkey-cli replicaof $target; do sleep 1; done
+}
+
+if [ "$1" = sentinel ]; then
+  until r="$(info 127.0.0.1)" && [ -n "$(field "$r" role)" ] && ! undecided "$r"; do sleep 2; done
+  master="$self"
+  [ "$(field "$r" role)" = master ] || master="$(field "$r" master_host)"
+  cat > /sentinel/sentinel.conf <<CONF
+port 26379
+protected-mode no
+sentinel resolve-hostnames yes
+sentinel announce-hostnames yes
+sentinel announce-ip $self
+sentinel monitor rspamd $master 6379 2
+sentinel down-after-milliseconds rspamd 5000
+sentinel failover-timeout rspamd 60000
+sentinel parallel-syncs rspamd 1
+CONF
+  exec valkey-sentinel /sentinel/sentinel.conf
+fi
+
+decide &
+# RDB only: an AOF reload resets the replication offset that choose_master compares.
+exec valkey-server --dir /data --appendonly no --save "60 1" \
+  --maxmemory 384mb --maxmemory-policy volatile-ttl \
+  --protected-mode no --replica-announce-ip "$self" --replicaof 127.0.0.1 1

apps/base/mail/rspamd-vault.yaml

@@ -0,0 +1,79 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: rspamd
+  namespace: mail
+---
+# Separate auth for the litellm engine: the `rspamd` k8s role is the only one
+# whose policy grants litellm/creds/rspamd.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: rspamd
+  namespace: mail
+spec:
+  allowedNamespaces:
+    - mail
+  kubernetes:
+    audiences:
+      - vault
+    role: rspamd
+    serviceAccount: rspamd
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default
+---
+# Haiku-scoped LiteLLM virtual key for the gpt module (field `key`).
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultDynamicSecret
+metadata:
+  name: rspamd-litellm
+  namespace: mail
+spec:
+  allowStaticCreds: false
+  destination:
+    create: true
+    name: rspamd-litellm
+    overwrite: true
+  mount: litellm
+  path: creds/rspamd
+  revoke: true
+  vaultAuthRef: rspamd
+---
+# DKIM private key for unkin.net, selector `mail` (field `private_key`).
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: dkim-unkin-net
+  namespace: mail
+spec:
+  destination:
+    create: true
+    name: dkim-unkin-net
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/mail/default/dkim-unkin-net
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+# Controller (web UI) password; config reads the `rspamadm pw` hash (field `password_hash`).
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: rspamd-controller
+  namespace: mail
+spec:
+  destination:
+    create: true
+    name: rspamd-controller
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/mail/default/rspamd-controller
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/base/mail/rspamd/actions.conf

@@ -0,0 +1,3 @@
+reject = 15;
+add_header = 6;
+greylist = null;

apps/base/mail/rspamd/antivirus.conf

@@ -0,0 +1,8 @@
+clamav {
+  type = "clamav";
+  servers = "clamav.mail.svc.cluster.local:3310";
+  symbol = "CLAM_VIRUS";
+  action = "reject";
+  scan_mime_parts = true;
+  log_clean = false;
+}

apps/base/mail/rspamd/dkim_signing.conf

@@ -0,0 +1,7 @@
+sign_authenticated = false;
+sign_local = true;
+sign_inbound = false;
+use_domain = "header";
+allow_hdrfrom_mismatch = false;
+selector = "mail";
+path = "/etc/rspamd/dkim/$domain.$selector.key";

apps/base/mail/rspamd/external_relay.conf

@@ -0,0 +1,8 @@
+# Edge gateway public IPs (smarthost gateways.cidr): score the first hop before them.
+enabled = true;
+rules {
+  EDGE_RELAY {
+    strategy = "ip_map";
+    ip_map = ["66.226.148.207/32", "66.226.149.50/32"];
+  }
+}

apps/base/mail/rspamd/force_actions.conf

@@ -0,0 +1,7 @@
+rules {
+  CLAMAV_UNAVAILABLE {
+    action = "soft reject";
+    expression = "CLAM_VIRUS_FAIL";
+    message = "Virus scanner unavailable, try again later";
+  }
+}

apps/base/mail/rspamd/gpt.conf

@@ -0,0 +1,22 @@
+enabled = true;
+type = "openai";
+url = "http://litellm.litellm.svc.cluster.local:4000/v1/chat/completions";
+api_key = "{= env.LITELLM_API_KEY =}";
+model = "claude-haiku-4-5";
+model_parameters = {
+  "claude-haiku-4-5" = {
+    max_tokens = 400,
+    temperature = 0.0,
+  }
+};
+timeout = 10s;
+# Inbound only (settings.conf disables it outbound); only uncertain scores (2 <= score < 12).
+condition = <<EOD
+return function(task)
+  local res = task:get_metric_result()
+  if not res then return false, 'no result' end
+  if res.passthrough then return false, 'passthrough' end
+  if res.score < 2 or res.score >= 12 then return false, 'score not uncertain' end
+  return true
+end
+EOD

apps/base/mail/rspamd/greylist.conf

@@ -0,0 +1 @@
+enabled = false;

apps/base/mail/rspamd/history_redis.conf

@@ -0,0 +1 @@
+nrows = 5000;

apps/base/mail/rspamd/local_networks.map

@@ -0,0 +1,30 @@
+# Outbound sources (smarthost mynetworks): DKIM-signed, scored with reject, no GPT.
+127.0.0.0/8
+::1
+10.42.0.0/16
+10.10.12.200/32
+198.18.1.10/32
+198.18.2.160/27
+198.18.2.192/27
+198.18.13.0/24
+198.18.14.0/24
+198.18.15.0/24
+198.18.16.0/24
+198.18.17.0/24
+198.18.18.0/24
+198.18.19.0/24
+198.18.20.0/24
+198.18.21.0/24
+198.18.22.0/24
+198.18.23.0/24
+198.18.25.0/24
+198.18.26.0/24
+198.18.27.0/24
+198.18.28.0/24
+198.18.29.0/24
+198.18.64.0/24
+198.18.65.0/24
+198.18.66.0/24
+198.18.67.0/24
+198.18.68.0/24
+198.18.69.0/24

apps/base/mail/rspamd/milter_headers.conf

@@ -0,0 +1,2 @@
+use = ["x-spamd-result", "x-spam-status", "x-spam-level", "authentication-results"];
+skip_local = true;

apps/base/mail/rspamd/options.inc

@@ -0,0 +1,2 @@
+local_addrs = "/etc/rspamd/local.d/local_networks.map";
+task_timeout = 20s;

apps/base/mail/rspamd/redis.conf

@@ -0,0 +1,4 @@
+sentinels = "rspamd-valkey-sentinel.mail.svc.cluster.local:26379";
+sentinel_masters_pattern = "^rspamd$";
+sentinel_watch_time = 10s;
+servers = "rspamd-valkey-0.rspamd-valkey.mail.svc.cluster.local.:6379";

apps/base/mail/rspamd/settings.conf

@@ -0,0 +1,19 @@
+outbound {
+  priority = high;
+  ip_map = "/etc/rspamd/local.d/local_networks.map";
+  apply {
+    symbols_disabled = ["GPT_CHECK"];
+  }
+}
+inbound {
+  priority = high;
+  ip_map = "/etc/rspamd/local.d/local_networks.map";
+  inverse = true;
+  apply {
+    symbols_disabled = ["DKIM_SIGNED"];
+    # Tag only; reject stays defined (unreachable by score) so ClamAV's virus reject still applies.
+    actions {
+      reject = 1000;
+    }
+  }
+}

apps/base/mail/rspamd/spamassassin.conf

@@ -0,0 +1 @@
+ruleset = "/etc/rspamd/sa-rules/*.cf";

apps/base/mail/rspamd/worker-controller.inc

@@ -0,0 +1,3 @@
+password = "{= env.CONTROLLER_PASSWORD_HASH =}";
+enable_password = "{= env.CONTROLLER_PASSWORD_HASH =}";
+allow_file_and_shm_inputs = false;

apps/base/mail/rspamd/worker-normal.inc

@@ -0,0 +1,2 @@
+count = 2;
+allow_file_and_shm_inputs = false;

apps/base/mail/rspamd/worker-proxy.inc

@@ -0,0 +1 @@
+allow_file_and_shm_inputs = false;

apps/base/mail/smarthost-networkpolicy.yaml

@@ -0,0 +1,78 @@
+---
+# Gateways (public IPs, see smarthost/gateways.cidr), lab networks (incl. node IPs for kubelet probes) and opted-in pods in any namespace.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: smarthost
+  namespace: mail
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: smarthost
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - ipBlock:
+            cidr: 66.226.148.207/32
+        - ipBlock:
+            cidr: 66.226.149.50/32
+        - ipBlock:
+            cidr: 10.10.12.200/32
+        - ipBlock:
+            cidr: 198.18.1.10/32
+        - ipBlock:
+            cidr: 198.18.2.160/27
+        - ipBlock:
+            cidr: 198.18.2.192/27
+        - ipBlock:
+            cidr: 198.18.13.0/24
+        - ipBlock:
+            cidr: 198.18.14.0/24
+        - ipBlock:
+            cidr: 198.18.15.0/24
+        - ipBlock:
+            cidr: 198.18.16.0/24
+        - ipBlock:
+            cidr: 198.18.17.0/24
+        - ipBlock:
+            cidr: 198.18.18.0/24
+        - ipBlock:
+            cidr: 198.18.19.0/24
+        - ipBlock:
+            cidr: 198.18.20.0/24
+        - ipBlock:
+            cidr: 198.18.21.0/24
+        - ipBlock:
+            cidr: 198.18.22.0/24
+        - ipBlock:
+            cidr: 198.18.23.0/24
+        - ipBlock:
+            cidr: 198.18.25.0/24
+        - ipBlock:
+            cidr: 198.18.26.0/24
+        - ipBlock:
+            cidr: 198.18.27.0/24
+        - ipBlock:
+            cidr: 198.18.28.0/24
+        - ipBlock:
+            cidr: 198.18.29.0/24
+        - ipBlock:
+            cidr: 198.18.64.0/24
+        - ipBlock:
+            cidr: 198.18.65.0/24
+        - ipBlock:
+            cidr: 198.18.66.0/24
+        - ipBlock:
+            cidr: 198.18.67.0/24
+        - ipBlock:
+            cidr: 198.18.68.0/24
+        - ipBlock:
+            cidr: 198.18.69.0/24
+        - namespaceSelector: {}
+          podSelector:
+            matchLabels:
+              mail.unkin.net/smarthost-client: "true"
+      ports:
+        - port: smtp
+          protocol: TCP

apps/base/mail/smarthost-pdb.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: smarthost
+  namespace: mail
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: smarthost

apps/base/mail/smarthost-service.yaml

@@ -0,0 +1,33 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: smarthost
+  namespace: mail
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: smarthost
+  ports:
+    - name: smtp
+      port: 25
+      targetPort: smtp
+---
+# Gateways and lab networks; Local keeps client IPs for the postfix CIDR checks.
+apiVersion: v1
+kind: Service
+metadata:
+  name: smarthost-lb
+  namespace: mail
+  annotations:
+    purelb.io/addresses: 198.18.200.16
+    purelb.io/service-group: common
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: smarthost
+  ports:
+    - name: smtp
+      port: 25
+      targetPort: smtp

apps/base/mail/smarthost-statefulset.yaml

@@ -0,0 +1,127 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: smarthost
+  namespace: mail
+  annotations:
+    configmap.reloader.stakater.com/reload: "smarthost-config"
+    secret.reloader.stakater.com/reload: "vault-ca-cert"
+spec:
+  serviceName: smarthost
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: smarthost
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: smarthost
+    spec:
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: smarthost
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: postfix
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/postfix:0.1.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 25
+              name: smtp
+              protocol: TCP
+          readinessProbe:
+            tcpSocket:
+              port: smtp
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            exec:
+              command: ["postfix", "status"]
+            initialDelaySeconds: 30
+            periodSeconds: 30
+          # postfix master runs as root and drops to the postfix user per service.
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+              add:
+                - CHOWN
+                - DAC_OVERRIDE
+                - FOWNER
+                - SETUID
+                - SETGID
+                - NET_BIND_SERVICE
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+          volumeMounts:
+            - name: spool
+              mountPath: /var/spool/postfix
+            - name: tls
+              mountPath: /etc/postfix/tls
+              readOnly: true
+            - name: vault-ca-cert
+              mountPath: /etc/postfix/edge-ca
+              readOnly: true
+            - name: config
+              mountPath: /etc/postfix/main.cf
+              subPath: main.cf
+              readOnly: true
+            - name: config
+              mountPath: /etc/postfix/master.cf
+              subPath: master.cf
+              readOnly: true
+            - name: config
+              mountPath: /etc/postfix/relay_domains
+              subPath: relay_domains
+              readOnly: true
+            - name: config
+              mountPath: /etc/postfix/transport
+              subPath: transport
+              readOnly: true
+            - name: config
+              mountPath: /etc/postfix/gateways.cidr
+              subPath: gateways.cidr
+              readOnly: true
+            - name: config
+              mountPath: /etc/postfix/mynetworks.cidr
+              subPath: mynetworks.cidr
+              readOnly: true
+            - name: config
+              mountPath: /etc/postfix/tls_policy
+              subPath: tls_policy
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: smarthost-config
+        - name: tls
+          secret:
+            secretName: wildcard-main-unkin-net-tls
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
+  volumeClaimTemplates:
+    - metadata:
+        name: spool
+      spec:
+        accessModes: [ReadWriteOnce]
+        storageClassName: cephrbd-fast-delete
+        resources:
+          requests:
+            storage: 50Gi

apps/base/mail/smarthost/gateways.cidr

@@ -0,0 +1,3 @@
+# Gateway (mta1/mta2) public IPs: ausyd2nxvm2130, aubne1nxvm2132.
+66.226.148.207/32 reject_plaintext_session, permit
+66.226.149.50/32 reject_plaintext_session, permit

apps/base/mail/smarthost/main.cf

@@ -0,0 +1,42 @@
+compatibility_level = 3.5
+myhostname = mail-in.main.unkin.net
+myorigin = unkin.net
+mydestination =
+alias_maps =
+local_recipient_maps =
+inet_interfaces = all
+inet_protocols = ipv4
+maillog_file = /dev/stdout
+message_size_limit = 52428800
+
+# Relay-anywhere clients; gateways must stay out.
+mynetworks = cidr:/etc/postfix/mynetworks.cidr
+
+relay_domains = texthash:/etc/postfix/relay_domains
+# Default minus relay_domains: subdomains are not delivered and would loop back out via the gateways.
+parent_domain_matches_subdomains = debug_peer_list,fast_flush_domains,mynetworks,permit_mx_backup_networks,qmqpd_authorized_clients,smtpd_access_maps
+transport_maps = texthash:/etc/postfix/transport
+relayhost = [mta1.edge.unkin.net]:25
+smtp_fallback_relay = [mta2.edge.unkin.net]:25
+smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy
+smtp_tls_CAfile = /etc/pki/tls/certs/ca-bundle.crt
+smtpd_client_restrictions = permit_mynetworks, check_client_access cidr:/etc/postfix/gateways.cidr, reject
+smtpd_relay_restrictions = permit_mynetworks, reject_unauth_destination
+smtpd_recipient_restrictions = permit_mynetworks, reject_unverified_recipient
+unverified_recipient_reject_code = 550
+unverified_recipient_tempfail_action = defer_if_permit
+address_verify_negative_refresh_time = 5m
+address_verify_negative_expire_time = 1h
+address_verify_positive_refresh_time = 1h
+address_verify_positive_expire_time = 1d
+
+milter_default_action = tempfail
+smtpd_milters = inet:rspamd.mail.svc:11332
+smtpd_authorized_xforward_hosts = cidr:/etc/postfix/gateways.cidr
+
+smtpd_tls_security_level = may
+smtpd_tls_cert_file = /etc/postfix/tls/tls.crt
+smtpd_tls_key_file = /etc/postfix/tls/tls.key
+smtpd_tls_received_header = yes
+smtp_tls_security_level = may
+lmtp_tls_security_level = none

apps/base/mail/smarthost/master.cf

@@ -0,0 +1,27 @@
+# Gateways (inbound to relay_domains) and mynetworks (relay anywhere); all via the rspamd milter.
+25        inet  n       -       n       -       -       smtpd
+pickup    unix  n       -       n       60      1       pickup
+cleanup   unix  n       -       n       -       0       cleanup
+qmgr      unix  n       -       n       300     1       qmgr
+tlsmgr    unix  -       -       n       1000?   1       tlsmgr
+rewrite   unix  -       -       n       -       -       trivial-rewrite
+bounce    unix  -       -       n       -       0       bounce
+defer     unix  -       -       n       -       0       bounce
+trace     unix  -       -       n       -       0       bounce
+verify    unix  -       -       n       -       1       verify
+flush     unix  n       -       n       1000?   0       flush
+proxymap  unix  -       -       n       -       -       proxymap
+proxywrite unix -       -       n       -       1       proxymap
+smtp      unix  -       -       n       -       -       smtp
+relay     unix  -       -       n       -       -       smtp
+        -o syslog_name=postfix/$service_name
+showq     unix  n       -       n       -       -       showq
+error     unix  -       -       n       -       -       error
+retry     unix  -       -       n       -       -       error
+discard   unix  -       -       n       -       -       discard
+local     unix  -       n       n       -       -       local
+virtual   unix  -       n       n       -       -       virtual
+lmtp      unix  -       -       n       -       -       lmtp
+anvil     unix  -       -       n       -       1       anvil
+scache    unix  -       -       n       -       1       scache
+postlog   unix-dgram n  -       n       -       1       postlogd

apps/base/mail/smarthost/mynetworks.cidr

@@ -0,0 +1,31 @@
+# Pod CIDR (narrowed by smarthost-networkpolicy.yaml) plus the lab networks from bind-internal acl-main/acl-common/acl-nomad-jobs.
+# Keep in sync with the NetworkPolicy.
+10.42.0.0/16 OK
+10.10.12.200/32 OK
+198.18.1.10/32 OK
+198.18.2.160/27 OK
+198.18.2.192/27 OK
+198.18.21.160/27 OK
+198.18.21.192/27 OK
+198.18.13.0/24 OK
+198.18.14.0/24 OK
+198.18.15.0/24 OK
+198.18.16.0/24 OK
+198.18.17.0/24 OK
+198.18.18.0/24 OK
+198.18.19.0/24 OK
+198.18.20.0/24 OK
+198.18.21.0/24 OK
+198.18.22.0/24 OK
+198.18.23.0/24 OK
+198.18.25.0/24 OK
+198.18.26.0/24 OK
+198.18.27.0/24 OK
+198.18.28.0/24 OK
+198.18.29.0/24 OK
+198.18.64.0/24 OK
+198.18.65.0/24 OK
+198.18.66.0/24 OK
+198.18.67.0/24 OK
+198.18.68.0/24 OK
+198.18.69.0/24 OK

apps/base/mail/smarthost/relay_domains

@@ -0,0 +1,2 @@
+unkin.net OK
+main.unkin.net OK

apps/base/mail/smarthost/tls_policy

@@ -0,0 +1,2 @@
+[mta1.edge.unkin.net]:25 secure match=hostname tafile=/etc/postfix/edge-ca/ca.crt
+[mta2.edge.unkin.net]:25 secure match=hostname tafile=/etc/postfix/edge-ca/ca.crt

apps/base/mail/smarthost/transport

@@ -0,0 +1,2 @@
+unkin.net lmtp:[stalwart.mail.svc.cluster.local]:24
+main.unkin.net lmtp:[stalwart.mail.svc.cluster.local]:24

apps/base/mail/stalwart-apply-job.yaml

@@ -0,0 +1,108 @@
+---
+# Converges Stalwart's database-held settings to plan.ndjson after every sync,
+# then rolls the StatefulSet: listener, store and coordinator changes only bind
+# on startup. It rolls on every sync because neither image has a shell to diff the
+# apply output.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: stalwart-apply
+  namespace: mail
+  annotations:
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  activeDeadlineSeconds: 300
+  backoffLimit: 2
+  ttlSecondsAfterFinished: 600
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: stalwart-apply
+    spec:
+      restartPolicy: Never
+      serviceAccountName: stalwart-apply
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        - name: apply
+          image: docker.io/stalwartlabs/cli:1.0.13
+          args:
+            - apply
+            - --file
+            - /plan/plan.ndjson
+          env:
+            # the cli caches the server schema under $HOME/.cache
+            - name: HOME
+              value: /tmp
+            - name: STALWART_URL
+              value: http://stalwart.mail.svc:8080
+            - name: STALWART_USER
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: username
+            - name: STALWART_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: password
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 200m
+              memory: 128Mi
+            requests:
+              cpu: 50m
+              memory: 32Mi
+          volumeMounts:
+            - name: plan
+              mountPath: /plan
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+      containers:
+        - name: restart
+          image: registry.k8s.io/kubectl:v1.33.13
+          args:
+            - rollout
+            - restart
+            - statefulset/stalwart
+            - --namespace=mail
+          env:
+            - name: HOME
+              value: /tmp
+          securityContext:
+            runAsUser: 65532
+            runAsGroup: 65532
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 200m
+              memory: 128Mi
+            requests:
+              cpu: 50m
+              memory: 32Mi
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: plan
+          configMap:
+            name: stalwart-config
+            items:
+              - key: plan.ndjson
+                path: plan.ndjson
+        - name: tmp
+          emptyDir: {}

apps/base/mail/stalwart-apply-rbac.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: stalwart-apply
+  namespace: mail
+---
+# Lets the stalwart-apply Job roll the Stalwart pods after applying the plan.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: stalwart-apply
+  namespace: mail
+rules:
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    resourceNames:
+      - stalwart
+    verbs:
+      - get
+      - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: stalwart-apply
+  namespace: mail
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: stalwart-apply
+subjects:
+  - kind: ServiceAccount
+    name: stalwart-apply
+    namespace: mail

apps/base/mail/stalwart-blobs-bucket.yaml

@@ -0,0 +1,30 @@
+---
+# Stalwart S3 blob store. The owner user has read-write on its own bucket; the
+# operator mints AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / S3_ENDPOINT into
+# stalwart-blobs-s3.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: stalwart-blobs
+  namespace: mail
+spec:
+  displayName: "Stalwart blob store owner"
+  uid: mail-stalwart-blobs
+  maxBuckets: 5
+  secretName: stalwart-blobs-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: stalwart-blobs
+  namespace: mail
+spec:
+  placementTarget: ec
+  bucketName: stalwart-blobs
+  ownerRef: stalwart-blobs
+  versioning: false
+  tags:
+    app: stalwart
+    purpose: blob-store
+  retainOnDelete: true

apps/base/mail/stalwart-configmap.yaml

@@ -0,0 +1,43 @@
+---
+# config.json only names the datastore; everything else lives in Postgres and
+# is converged from plan.ndjson by the stalwart-apply PostSync Job.
+# The spam filter only scores rspamd's X-Spam header, feeding the native Junk path.
+# The Coordinator uses the plain Redis client: its cluster client never
+# resubscribes once the Valkey node it subscribed on is replaced.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: stalwart-config
+  namespace: mail
+data:
+  config.json: |
+    {"@type":"PostgreSql","host":"stalwart-db-rw.mail.svc","port":5432,"database":"stalwart","authUsername":"stalwart","authSecret":{"@type":"EnvironmentVariable","variableName":"DB_PASSWORD"},"useTls":true}
+  plan.ndjson: |
+    {"@type":"upsert","object":"Domain","matchOn":["name"],"value":{"unkin":{"name":"unkin.net","aliases":{"main.unkin.net":true},"certificateManagement":{"@type":"Manual"},"dkimManagement":{"@type":"Manual"},"dnsManagement":{"@type":"Manual"},"subAddressing":{"@type":"Enabled"}}}}
+    {"@type":"upsert","object":"Certificate","matchOn":["certificate"],"value":{"cert":{"certificate":{"@type":"File","filePath":"/etc/stalwart/tls/tls.crt"},"privateKey":{"@type":"File","filePath":"/etc/stalwart/tls/tls.key"}}}}
+    {"@type":"upsert","object":"Directory","matchOn":["description"],"value":{"authentik":{"@type":"Oidc","description":"Authentik","issuerUrl":"https://identity.unkin.net/application/o/stalwart/","requireAudience":"stalwart","requireScopes":{},"claimUsername":"preferred_username","usernameDomain":"unkin.net","claimName":"name"}}}
+    {"@type":"upsert","object":"Application","matchOn":["description"],"value":{"webui":{"description":"Stalwart Web Interface","resourceUrl":"https://github.com/stalwartlabs/webui/releases/download/v1.0.11/webui.zip","urlPrefix":{"/admin":true,"/account":true},"unpackDirectory":"/var/lib/stalwart/webui","oauthClientId":"stalwart"}}}
+    {"@type":"upsert","object":"NetworkListener","matchOn":["name"],"value":{"lmtp":{"name":"lmtp","bind":{"[::]:24":true},"protocol":"lmtp","useTls":false},"submission":{"name":"submission","bind":{"[::]:587":true},"protocol":"smtp","useTls":true,"tlsImplicit":false},"metrics":{"name":"metrics","bind":{"[::]:9090":true},"protocol":"http","useTls":false}}}
+    {"@type":"upsert","object":"MtaRoute","matchOn":["name"],"value":{"smarthost":{"@type":"Relay","name":"smarthost","address":"smarthost.mail.svc","port":25,"protocol":"smtp","implicitTls":false,"allowInvalidCerts":true,"authSecret":{"@type":"None"}}}}
+    {"@type":"upsert","object":"SieveUserScript","matchOn":["name"],"value":{"spamToJunk":{"name":"spam-to-junk","description":"Files mail rspamd tagged X-Spam: Yes into Junk","isActive":true,"contents":"require [\"fileinto\", \"special-use\"];\nif header :is \"X-Spam\" \"Yes\" {\n  fileinto :specialuse \"\\\\Junk\" \"Junk Mail\";\n  stop;\n}\n"}}}
+    {"@type":"upsert","object":"MtaInboundThrottle","matchOn":["description"],"value":{"senderIp":{"description":"Sender IP throttle","enable":true,"key":{"remoteIp":true},"match":{"match":{},"else":"local_port != 24"},"rate":{"period":1000,"count":5}},"senderRcpt":{"description":"Sender address to recipient throttle","enable":true,"key":{"senderDomain":true,"rcpt":true},"match":{"match":{},"else":"local_port != 24"},"rate":{"period":3600000,"count":25}}}}
+    {"@type":"reconcile","object":"Tracer","matchOn":"*","value":{"stdout":{"@type":"Stdout","enable":true,"level":"info","ansi":false,"multiline":false,"buffered":false,"lossy":false,"eventsPolicy":"exclude","events":{}}}}
+    {"@type":"update","object":"BlobStore","value":{"@type":"S3","bucket":"stalwart-blobs","region":{"@type":"Custom","customEndpoint":"https://s3.ceph.unkin.net","customRegion":"us-east-1"},"accessKey":{"@type":"EnvironmentVariable","variableName":"AWS_ACCESS_KEY_ID"},"secretKey":{"@type":"EnvironmentVariable","variableName":"AWS_SECRET_ACCESS_KEY"},"securityToken":{"@type":"None"},"sessionToken":{"@type":"None"}}}
+    {"@type":"update","object":"SearchStore","value":{"@type":"Default"}}
+    {"@type":"update","object":"InMemoryStore","value":{"@type":"RedisCluster","urls":{"redis://valkey-mail-valkey.mail.svc:6379":true},"authUsername":null,"authSecret":{"@type":"None"},"readFromReplicas":false,"protocolVersion":"resp3"}}
+    {"@type":"update","object":"Coordinator","value":{"@type":"Redis","url":"redis://valkey-mail-valkey.mail.svc:6379"}}
+    {"@type":"update","object":"Authentication","value":{"directoryId":"#authentik"}}
+    {"@type":"update","object":"MtaOutboundStrategy","value":{"route":{"match":{"0":{"if":"is_local_domain(rcpt_domain)","then":"'local'"}},"else":"'smarthost'"}}}
+    {"@type":"update","object":"MtaStageAuth","value":{"require":{"match":{},"else":"local_port != 25 && local_port != 24"}}}
+    {"@type":"update","object":"SenderAuth","value":{"dkimSignDomain":{"match":{},"else":"false"}}}
+    {"@type":"reconcile","object":"SpamRule","matchOn":["name"],"value":{"rspamd":{"@type":"Header","name":"RSPAMD_SPAM","enable":true,"priority":0,"condition":{"match":{"0":{"if":"name_lower == 'x-spam' && value_lower == 'yes'","then":"'RSPAMD_SPAM'"}},"else":"false"}}}}
+    {"@type":"reconcile","object":"SpamTag","matchOn":["tag"],"value":{"rspamd":{"@type":"Score","tag":"RSPAMD_SPAM","score":10.0}}}
+    {"@type":"destroy","object":"SpamDnsblServer"}
+    {"@type":"destroy","object":"HttpLookup"}
+    {"@type":"destroy","object":"MemoryLookupKey"}
+    {"@type":"update","object":"SpamClassifier","value":{"model":{"@type":"Disabled"}}}
+    {"@type":"update","object":"SpamPyzor","value":{"enable":false}}
+    {"@type":"update","object":"SpamSettings","value":{"enable":true,"spamFilterRulesUrl":null}}
+    {"@type":"update","object":"Metrics","value":{"prometheus":{"@type":"Enabled","authSecret":{"@type":"None"}}}}
+    {"@type":"update","object":"Http","value":{"allowedEndpoints":{"match":{"0":{"if":"starts_with(path, '/metrics') && local_port != 9090","then":"404"}},"else":"200"}}}
+    {"@type":"update","object":"SystemSettings","value":{"defaultHostname":"mail.unkin.net","defaultDomainId":"#unkin","defaultCertificateId":"#cert"}}

apps/base/mail/stalwart-pdb.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: stalwart
+  namespace: mail
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: stalwart

apps/base/mail/stalwart-service.yaml

@@ -0,0 +1,61 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: stalwart
+  namespace: mail
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: stalwart
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+    - name: lmtp
+      port: 24
+      targetPort: lmtp
+---
+# Governing Service for the StatefulSet's per-pod DNS.
+apiVersion: v1
+kind: Service
+metadata:
+  name: stalwart-headless
+  namespace: mail
+spec:
+  clusterIP: None
+  selector:
+    app.kubernetes.io/name: stalwart
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+---
+# Client-facing mail ports on the lab network. 443 is here too so the single
+# internal mail.unkin.net record serves web as well as IMAP/submission.
+apiVersion: v1
+kind: Service
+metadata:
+  name: stalwart-lb
+  namespace: mail
+  annotations:
+    purelb.io/addresses: 198.18.200.18
+    purelb.io/service-group: common
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: stalwart
+  ports:
+    - name: imaps
+      port: 993
+      targetPort: imaps
+    - name: submissions
+      port: 465
+      targetPort: submissions
+    - name: submission
+      port: 587
+      targetPort: submission
+    - name: https
+      port: 443
+      targetPort: https

apps/base/mail/stalwart-statefulset.yaml

@@ -0,0 +1,200 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: stalwart
+  namespace: mail
+  annotations:
+    secret.reloader.stakater.com/reload: "stalwart-admin,stalwart-db-app,stalwart-blobs-s3,wildcard-unkin-net-tls,vault-ca-cert,stalwart-db-ca"
+spec:
+  # Nodes coordinate through mail-valkey; each leases its cluster node id by
+  # hostname, so stable pod names keep a stable id across restarts.
+  replicas: 3
+  serviceName: stalwart-headless
+  podManagementPolicy: Parallel
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: stalwart
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: stalwart
+        mail.unkin.net/smarthost-client: "true"
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 2000
+        runAsGroup: 2000
+        fsGroup: 2000
+        seccompProfile:
+          type: RuntimeDefault
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: stalwart
+      # rustls loads SSL_CERT_FILE, so public roots, the internal CA (RGW) and the CNPG CA go in one bundle.
+      initContainers:
+        - name: combine-certs
+          image: docker.io/stalwartlabs/stalwart:v0.16.25
+          command:
+            - sh
+            - -c
+            - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt /db-ca/ca.crt > /combined-certs/ca-certificates.crt
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 100m
+              memory: 64Mi
+            requests:
+              cpu: 25m
+              memory: 32Mi
+          volumeMounts:
+            - name: vault-ca-cert
+              mountPath: /custom-ca
+              readOnly: true
+            - name: db-ca
+              mountPath: /db-ca
+              readOnly: true
+            - name: combined-certs
+              mountPath: /combined-certs
+      containers:
+        - name: stalwart
+          image: docker.io/stalwartlabs/stalwart:v0.16.25
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: STALWART_HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: SSL_CERT_FILE
+              value: /etc/ssl/combined/ca-certificates.crt
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-db-app
+                  key: password
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-blobs-s3
+                  key: AWS_ACCESS_KEY_ID
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-blobs-s3
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: ADMIN_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: username
+            - name: ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: password
+            - name: STALWART_RECOVERY_ADMIN
+              value: "$(ADMIN_USERNAME):$(ADMIN_PASSWORD)"
+          ports:
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+            - containerPort: 443
+              name: https
+              protocol: TCP
+            - containerPort: 24
+              name: lmtp
+              protocol: TCP
+            - containerPort: 465
+              name: submissions
+              protocol: TCP
+            - containerPort: 587
+              name: submission
+              protocol: TCP
+            - containerPort: 993
+              name: imaps
+              protocol: TCP
+            - containerPort: 9090
+              name: metrics
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz/live
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /healthz/ready
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+              # the binary carries a cap_net_bind_service file capability; exec fails without it
+              add:
+                - NET_BIND_SERVICE
+          resources:
+            limits:
+              cpu: "2"
+              memory: 2Gi
+            requests:
+              cpu: 100m
+              memory: 512Mi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/stalwart/config.json
+              subPath: config.json
+              readOnly: true
+            - name: tls
+              mountPath: /etc/stalwart/tls
+              readOnly: true
+            - name: combined-certs
+              mountPath: /etc/ssl/combined
+              readOnly: true
+            - name: data
+              mountPath: /var/lib/stalwart
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: stalwart-config
+        - name: tls
+          secret:
+            secretName: wildcard-unkin-net-tls
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: db-ca
+          secret:
+            secretName: stalwart-db-ca
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: combined-certs
+          emptyDir: {}
+        # Webui unpack dir only; mail data is in Postgres and S3.
+        - name: data
+          emptyDir: {}
+        - name: tmp
+          emptyDir: {}

apps/base/mail/valkey.yaml

@@ -0,0 +1,29 @@
+---
+# Stalwart's in-memory store (locks, rate limits, queue leases) and cluster
+# coordinator pub/sub. Ephemeral state, so no persistence; the operator runs it
+# cluster-mode, passwordless and without TLS behind valkey-mail-valkey.
+apiVersion: valkey.io/v1alpha1
+kind: ValkeyCluster
+metadata:
+  name: mail-valkey
+  namespace: mail
+spec:
+  shards: 1
+  replicas: 2
+  image: docker.io/valkey/valkey:9.0.0
+  exporter:
+    enabled: false
+  scheduling:
+    node:
+      spread:
+        shard:
+          mode: Required
+  podDisruptionBudget:
+    mode: Cluster
+  resources:
+    requests:
+      cpu: 50m
+      memory: 128Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi

apps/base/mail/vaultauth.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: default
+  namespace: mail
+spec:
+  allowedNamespaces:
+    - mail
+  kubernetes:
+    audiences:
+      - vault
+    role: default
+    serviceAccount: default
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default

apps/base/mail/vaultstaticsecret.yaml

@@ -0,0 +1,19 @@
+---
+# Fallback admin (username/password) for STALWART_RECOVERY_ADMIN and the apply
+# Job, from kv/kubernetes/namespace/mail/default/admin.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: stalwart-admin
+  namespace: mail
+spec:
+  destination:
+    create: true
+    name: stalwart-admin
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/mail/default/admin
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/base/mail/vmpodscrape.yaml

@@ -0,0 +1,14 @@
+---
+# Stalwart's Prometheus exporter, served only on the dedicated metrics listener.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: stalwart
+  namespace: mail
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: stalwart
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics/prometheus

apps/base/pdbmux/deployment.yaml

@@ -25,7 +25,7 @@ spec:
         - name: pdbmux
           # Image is published by the pdbmux repo's .woodpecker/docker.yaml on
           # a v* tag. It only exists after that tag is cut (see PR merge gates).
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/pdbmux:v0.4.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/pdbmux:v0.5.0
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8080

apps/base/woodpecker/kustomization.yaml

@@ -20,6 +20,7 @@ resources:
   - serviceaccount_repospawner_ci.yaml
   - serviceaccount_terraform_artifactapi.yaml
   - serviceaccount_terraform_authentik.yaml
+  - serviceaccount_terraform_binarylane.yaml
   - serviceaccount_terraform_enc.yaml
   - serviceaccount_terraform_git.yaml
   - serviceaccount_terraform_netbox.yaml

apps/base/woodpecker/serviceaccount_terraform_binarylane.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: terraform-binarylane
+  namespace: woodpecker

apps/overlays/au-syd1/mail/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/mail

argocd/applicationsets/platform.yaml

@@ -34,6 +34,7 @@ spec:
       - path: apps/overlays/*/jfrog
       - path: apps/overlays/*/k8up-system
       - path: apps/overlays/*/kanidm
+      - path: apps/overlays/*/mail
       - path: apps/overlays/*/netbox
       - path: apps/overlays/*/node-feature-discovery
       - path: apps/overlays/*/pdbmux

argocd/projects/platform.yaml

@@ -49,6 +49,8 @@ spec:
       server: https://kubernetes.default.svc
     - namespace: 'kanidm'
       server: https://kubernetes.default.svc
+    - namespace: 'mail'
+      server: https://kubernetes.default.svc
     - namespace: 'netbox'
       server: https://kubernetes.default.svc
     - namespace: 'node-feature-discovery'