Merge remote-tracking branch 'origin/benvin/mail-stalwart' into benvin/mail-sieve-reconcile

# Conflicts:
#	apps/base/mail/kustomization.yaml

.pre-commit-config.yaml

@@ -42,3 +42,9 @@ repos:
         entry: ci/validate-no-secrets.sh
         language: system
         pass_filenames: false
+      - id: sieve_reconcile_selftest
+        name: sieve-reconcile selftest
+        entry: python3 apps/base/mail/resources/sieve-reconcile.py --selftest
+        language: system
+        files: sieve-reconcile\.py$
+        pass_filenames: false

apps/base/artifactapi/api-deployment.yaml

@@ -36,7 +36,7 @@ spec:
               mountPath: /combined-certs
       containers:
         - name: api
-          image: git.unkin.net/unkin/artifactapi:v3.12.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.13.1
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000

apps/base/artifactapi/image-keeper.yaml

@@ -0,0 +1,105 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: image-keeper
+  namespace: artifactapi
+spec:
+  selector:
+    matchLabels:
+      app: image-keeper
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: image-keeper
+    spec:
+      automountServiceAccountToken: false
+      priorityClassName: low
+      tolerations:
+        - operator: Exists
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        # artifactapi is distroless with no exit-0 flag, so run a static busybox as `true`
+        - name: copy-true
+          image: busybox:1.37.0-musl
+          imagePullPolicy: IfNotPresent
+          command: ["cp", "/bin/busybox", "/keeper/true"]
+          volumeMounts:
+            - name: keeper
+              mountPath: /keeper
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+        - name: api
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.13.1
+          imagePullPolicy: IfNotPresent
+          command: ["/keeper/true"]
+          volumeMounts:
+            - name: keeper
+              mountPath: /keeper
+              readOnly: true
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+        - name: ui
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.13.1
+          imagePullPolicy: IfNotPresent
+          command: ["true"]
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+      containers:
+        - name: pause
+          image: rancher/mirrored-pause:3.6
+          imagePullPolicy: IfNotPresent
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+      volumes:
+        - name: keeper
+          emptyDir: {}

apps/base/artifactapi/kustomization.yaml

@@ -11,6 +11,7 @@ resources:
   - cnpg_pooler.yaml
   - gateway.yaml
   - httproute.yaml
+  - image-keeper.yaml
   - namespace.yaml
   - oauth2-proxy-configmap.yaml
   - oauth2-proxy-deployment.yaml

apps/base/artifactapi/ui-deployment.yaml

@@ -22,7 +22,7 @@ spec:
       automountServiceAccountToken: true
       containers:
         - name: ui
-          image: git.unkin.net/unkin/artifactapi-ui:v3.12.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.13.1
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 80

apps/base/bind-internal/authoritative/unkin-net/a/kustomization.yaml

@@ -16,5 +16,6 @@ resources:
   - identity.yaml
   - lb1.yaml
   - logviewer.yaml
+  - mail.yaml
   - vlogs.yaml
   - watchstate.yaml

apps/base/bind-internal/authoritative/unkin-net/a/mail.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: mail-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: mail
+  type: A
+  ttl: 600
+  values:
+    # stalwart-lb PureLB VIP (IMAPS, submission, HTTPS).
+    - 198.18.200.18

apps/base/cert-manager/certificate_wildcard-main-unkin-net.yaml

@@ -1,6 +1,6 @@
 ---
 # Let's Encrypt *.main.unkin.net wildcard for the haproxy edge (pve, arr stack,
-# jellyfin, stalwart webadmin/autoconfig). DNS-01 needs the delegated
+# jellyfin, stalwart webadmin/autoconfig) and the mail smarthost. DNS-01 needs the delegated
 # _acme-challenge.main.unkin.net CNAME in the public unkin.net zone.
 # _acme-challenge.main.unkin.net. CNAME _acme-challenge.main.acme.unkin.net.
 apiVersion: cert-manager.io/v1
@@ -13,9 +13,9 @@ spec:
   secretTemplate:
     annotations:
       reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
-      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy,mail"
       reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
-      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy,mail"
   privateKey:
     size: 4096
   dnsNames:

apps/base/cert-manager/certificate_wildcard-unkin-net.yaml

@@ -14,9 +14,9 @@ spec:
   secretTemplate:
     annotations:
       reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
-      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
       reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
-      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
   privateKey:
     size: 4096
   dnsNames:

apps/base/mail/clamav.yaml

@@ -0,0 +1,87 @@
+---
+# clamd + freshclam (the image's init runs both); signatures come from database.clamav.net.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: clamav
+  namespace: mail
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: clamav
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: clamav
+    spec:
+      automountServiceAccountToken: false
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: clamav
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: clamav
+          image: docker.io/clamav/clamav:1.5.4
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 3310
+              name: clamd
+              protocol: TCP
+          startupProbe:
+            tcpSocket:
+              port: clamd
+            periodSeconds: 10
+            failureThreshold: 60
+          readinessProbe:
+            tcpSocket:
+              port: clamd
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket:
+              port: clamd
+            periodSeconds: 30
+          resources:
+            requests:
+              cpu: 100m
+              memory: 1536Mi
+            limits:
+              cpu: "2"
+              memory: 3Gi
+          volumeMounts:
+            - name: db
+              mountPath: /var/lib/clamav
+      volumes:
+        - name: db
+          emptyDir: {}
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: clamav
+  namespace: mail
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: clamav
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: clamav
+  namespace: mail
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: clamav
+  ports:
+    - name: clamd
+      port: 3310
+      targetPort: clamd

apps/base/mail/cnpg_backup.yaml

@@ -0,0 +1,40 @@
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: cnpg-mail-backup
+  namespace: mail
+spec:
+  displayName: "CNPG backup owner (mail)"
+  uid: cnpg-mail-backup
+  maxBuckets: 5
+  secretName: cnpg-mail-backup-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: cnpg-mail
+  namespace: mail
+spec:
+  placementTarget: ec
+  bucketName: cnpg-mail
+  ownerRef: cnpg-mail-backup
+  versioning: false
+  tags:
+    app: mail
+    purpose: cnpg-backup
+  retainOnDelete: true
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-mail-nightly
+  namespace: mail
+spec:
+  schedule: "0 20 4 * * *"
+  immediate: false
+  backupOwnerReference: self
+  method: barmanObjectStore
+  cluster:
+    name: stalwart-db

apps/base/mail/cnpg_cluster.yaml

@@ -0,0 +1,118 @@
+---
+# Stalwart metadata store. No bootstrap secret: CNPG mints stalwart-db-app
+# (username/password/dbname/uri) for Stalwart to consume.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: stalwart-db
+  namespace: mail
+spec:
+  affinity:
+    podAntiAffinityType: preferred
+  backup:
+    retentionPolicy: 30d
+    barmanObjectStore:
+      destinationPath: s3://cnpg-mail
+      endpointURL: https://s3.ceph.unkin.net
+      endpointCA:
+        name: vault-ca-cert
+        key: ca.crt
+      s3Credentials:
+        accessKeyId:
+          name: cnpg-mail-backup-s3
+          key: AWS_ACCESS_KEY_ID
+        secretAccessKey:
+          name: cnpg-mail-backup-s3
+          key: AWS_SECRET_ACCESS_KEY
+      serverName: stalwart-db
+      data:
+        compression: bzip2
+        jobs: 2
+      wal:
+        compression: zstd
+        maxParallel: 2
+  bootstrap:
+    initdb:
+      database: stalwart
+      encoding: UTF8
+      localeCType: C
+      localeCollate: C
+      owner: stalwart
+  enablePDB: true
+  enableSuperuserAccess: false
+  failoverDelay: 0
+  imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie
+  instances: 3
+  logLevel: info
+  maxSyncReplicas: 0
+  minSyncReplicas: 0
+  monitoring:
+    customQueriesConfigMap:
+      - key: queries
+        name: cnpg-default-monitoring
+    disableDefaultQueries: false
+    enablePodMonitor: false
+  postgresql:
+    parameters:
+      archive_mode: "on"
+      archive_timeout: 5min
+      dynamic_shared_memory_type: posix
+      effective_cache_size: 256MB
+      full_page_writes: "on"
+      hot_standby_feedback: "on"
+      log_destination: csvlog
+      log_directory: /controller/log
+      log_filename: postgres
+      log_rotation_age: "0"
+      log_rotation_size: "0"
+      log_truncate_on_rotation: "false"
+      logging_collector: "on"
+      max_connections: "200"
+      max_parallel_workers: "16"
+      max_replication_slots: "16"
+      max_worker_processes: "16"
+      pg_stat_statements.max: "10000"
+      pg_stat_statements.track: top
+      shared_buffers: 128MB
+      shared_memory_type: mmap
+      ssl_max_protocol_version: TLSv1.3
+      ssl_min_protocol_version: TLSv1.3
+      wal_keep_size: 256MB
+      wal_level: logical
+      wal_log_hints: "on"
+      wal_receiver_timeout: 5s
+      wal_sender_timeout: 5s
+    shared_preload_libraries:
+      - pg_stat_statements
+    syncReplicaElectionConstraint:
+      enabled: false
+  primaryUpdateMethod: restart
+  primaryUpdateStrategy: unsupervised
+  probes:
+    liveness:
+      isolationCheck:
+        connectionTimeout: 1000
+        enabled: true
+        requestTimeout: 1000
+  replicationSlots:
+    highAvailability:
+      enabled: true
+      slotPrefix: _cnpg_
+    synchronizeReplicas:
+      enabled: true
+    updateInterval: 30
+  resources:
+    limits:
+      cpu: "1"
+      memory: 1Gi
+    requests:
+      cpu: 50m
+      memory: 512Mi
+  smartShutdownTimeout: 180
+  startDelay: 3600
+  stopDelay: 1800
+  storage:
+    resizeInUseVolumes: true
+    size: 10Gi
+    storageClass: cephrbd-fast-delete
+  switchoverDelay: 3600

apps/base/mail/gateway.yaml

@@ -0,0 +1,33 @@
+---
+# External (DMZ) front for mail.unkin.net web/JMAP via the external Traefik
+# (LB VIP 198.18.199.0), terminating with the reflected *.unkin.net wildcard.
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: external
+  name: stalwart-external
+  namespace: mail
+spec:
+  gatewayClassName: traefik-external
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      hostname: mail.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https
+      port: 443
+      protocol: HTTPS
+      hostname: mail.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: wildcard-unkin-net-tls

apps/base/mail/httproute.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: stalwart-external-http-redirect
+  namespace: mail
+spec:
+  hostnames:
+    - mail.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: stalwart-external
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: stalwart-external
+  namespace: mail
+spec:
+  hostnames:
+    - mail.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: stalwart-external
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: stalwart
+          port: 8080
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/mail/kustomization.yaml

@@ -0,0 +1,79 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - clamav.yaml
+  - cnpg_backup.yaml
+  - cnpg_cluster.yaml
+  - gateway.yaml
+  - httproute.yaml
+  - namespace.yaml
+  - rspamd-deployment.yaml
+  - rspamd-sa-rules.yaml
+  - rspamd-valkey.yaml
+  - rspamd-vault.yaml
+  - smarthost-networkpolicy.yaml
+  - smarthost-pdb.yaml
+  - smarthost-service.yaml
+  - smarthost-statefulset.yaml
+  - stalwart-apply-job.yaml
+  - stalwart-apply-rbac.yaml
+  - stalwart-blobs-bucket.yaml
+  - stalwart-configmap.yaml
+  - stalwart-pdb.yaml
+  - stalwart-service.yaml
+  - stalwart-sieve-reconcile-cronjob.yaml
+  - stalwart-statefulset.yaml
+  - valkey.yaml
+  - vaultauth.yaml
+  - vaultstaticsecret.yaml
+  - vmpodscrape.yaml
+
+configMapGenerator:
+  - name: rspamd-config
+    namespace: mail
+    options:
+      disableNameSuffixHash: true
+    files:
+      - rspamd/actions.conf
+      - rspamd/antivirus.conf
+      - rspamd/dkim_signing.conf
+      - rspamd/external_relay.conf
+      - rspamd/force_actions.conf
+      - rspamd/gpt.conf
+      - rspamd/greylist.conf
+      - rspamd/history_redis.conf
+      - rspamd/local_networks.map
+      - rspamd/milter_headers.conf
+      - rspamd/options.inc
+      - rspamd/redis.conf
+      - rspamd/settings.conf
+      - rspamd/spamassassin.conf
+      - rspamd/worker-controller.inc
+      - rspamd/worker-normal.inc
+      - rspamd/worker-proxy.inc
+  - name: rspamd-valkey-scripts
+    namespace: mail
+    options:
+      disableNameSuffixHash: true
+    files:
+      - rspamd-valkey/start.sh
+  - name: smarthost-config
+    namespace: mail
+    options:
+      disableNameSuffixHash: true
+    files:
+      - smarthost/main.cf
+      - smarthost/master.cf
+      - smarthost/relay_domains
+      - smarthost/transport
+      - smarthost/gateways.cidr
+      - smarthost/mynetworks.cidr
+      - smarthost/tls_policy
+  - name: stalwart-sieve-reconcile
+    namespace: mail
+    options:
+      disableNameSuffixHash: true
+    files:
+      - sieve-reconcile.py=resources/sieve-reconcile.py

apps/base/mail/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: mail

apps/base/mail/resources/sieve-reconcile.py

@@ -0,0 +1,158 @@
+import base64
+import json
+import os
+import re
+import sys
+import urllib.parse
+import urllib.request
+
+USING = ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:sieve", "urn:stalwart:jmap"]
+HEADER = 'require ["include"];\ninclude :global "spam-to-junk";\n'
+HAS_INCLUDE = re.compile(r'\binclude\b[^;]*"spam-to-junk"')
+TOKENS = re.compile(r'"(?:\\.|[^"\\])*"|#[^\n]*|/\*.*?\*/', re.S)
+
+
+def auth():
+    user = f"{os.environ['STALWART_USER']}:{os.environ['STALWART_PASSWORD']}"
+    return "Basic " + base64.b64encode(user.encode()).decode()
+
+
+def http(path, data=None, ctype="application/json"):
+    url = os.environ["STALWART_URL"].rstrip("/") + path
+    req = urllib.request.Request(url, data=data, headers={"Authorization": auth()})
+    if data is not None:
+        req.add_header("Content-Type", ctype)
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        return resp.read()
+
+
+def jmap(*calls):
+    body = json.dumps({"using": USING, "methodCalls": [[m, a, str(i)] for i, (m, a) in enumerate(calls)]})
+    responses = json.loads(http("/jmap", body.encode()))["methodResponses"]
+    for name, args, _ in responses:
+        if name == "error":
+            raise RuntimeError(args)
+    return [args for _, args, _ in responses]
+
+
+def blank(script):
+    # Same-length copy with comments as spaces and string contents as "x", so offsets match.
+    def mask(m):
+        t = m.group(0)
+        if t.startswith('"'):
+            return '"' + re.sub(r"[^\n]", "x", t[1:-1]) + '"'
+        return re.sub(r"[^\n]", " ", t)
+    return TOKENS.sub(mask, script)
+
+
+def commands(script):
+    s, start, depth = blank(script), None, 0
+    for p, c in enumerate(s):
+        if start is None and not c.isspace():
+            start = p
+        depth += {"{": 1, "}": -1}.get(c, 0)
+        if start is not None and depth == 0 and c in ";}":
+            yield re.match(r"\w*", s[start:]).group(0).lower(), start, p + 1
+            start = None
+
+
+def with_include(script):
+    at = 0
+    for name, _, end in commands(script):
+        if name != "require":
+            break
+        at = end
+    if at:
+        eol = script.find("\n", at)
+        eol = len(script) if eol < 0 else eol + 1
+        if not blank(script)[at:eol].strip():
+            at = eol
+        if not script[:at].endswith("\n"):
+            return script[:at] + "\n" + HEADER + script[at:]
+    return script[:at] + HEADER + script[at:]
+
+
+def has_include(script):
+    uncommented = TOKENS.sub(lambda m: m.group(0) if m.group(0).startswith('"') else " ", script)
+    return bool(HAS_INCLUDE.search(uncommented))
+
+
+def users():
+    ids, position = [], 0
+    while True:
+        (page,) = jmap(("x:Account/query", {"position": position, "limit": 500, "calculateTotal": True}))
+        ids += page["ids"]
+        position += len(page["ids"])
+        if not page["ids"] or position >= page.get("total", position):
+            break
+    for start in range(0, len(ids), 500):
+        (got,) = jmap(("x:Account/get", {"ids": ids[start:start + 500], "properties": ["@type", "name"]}))
+        yield from (a for a in got["list"] if a["@type"] == "User")
+
+
+def reconcile(account):
+    acct = account["id"]
+    (got,) = jmap(("SieveScript/get", {"accountId": acct}))
+    scripts = got["list"]
+    target = next((s for s in scripts if s["isActive"]), None) or next(
+        (s for s in scripts if s["name"] == "default"), None
+    )
+    content = ""
+    if target:
+        content = http(f"/jmap/download/{acct}/{target['blobId']}/script").decode()
+        if target["isActive"] and has_include(content):
+            return
+    if not has_include(content):
+        content = with_include(content)
+    blob = json.loads(http(f"/jmap/upload/{acct}/", content.encode(), "application/sieve"))["blobId"]
+    if target:
+        args = {"accountId": acct, "update": {target["id"]: {"blobId": blob}},
+                "onSuccessActivateScript": target["id"]}
+        action = f"added spam-to-junk include to {target['name']!r}"
+    else:
+        args = {"accountId": acct, "create": {"d": {"name": "default", "blobId": blob}},
+                "onSuccessActivateScript": "#d"}
+        action = "created and activated 'default'"
+    (res,) = jmap(("SieveScript/set", args))
+    failed = res.get("notCreated") or res.get("notUpdated")
+    if failed:
+        raise RuntimeError(failed)
+    print(f"{account['name']}: {action}", flush=True)
+
+
+SELFTEST_CASES = {
+    "empty": "",
+    "no require": "keep;\n",
+    "no trailing newline": 'require "fileinto";',
+    "plain": 'require ["fileinto"];\nif true { fileinto "A"; }\n',
+    "multiline after comment": '# mine\nrequire ["fileinto",\n  "mailbox"];\nkeep;\n',
+    "trailing comment": 'require "fileinto"; # needed; really\nfileinto "A";',
+    "bracketed comment": '/* require "x"; */ require /* ; */ ["fileinto"];\nrequire "mailbox";\nstop;\n',
+}
+
+
+def selftest():
+    for label, script in SELFTEST_CASES.items():
+        out = with_include(script)
+        names = [n for n, _, _ in commands(out)]
+        first = next((i for i, n in enumerate(names) if n != "require"), len(names))
+        assert "require" not in names[first:], (label, out)
+        assert names[first:first + 1] == ["include"], (label, out)
+        assert has_include(out) and not has_include(script), label
+        assert out.replace(HEADER, "", 1).rstrip("\n") == script.rstrip("\n"), label
+    assert not has_include('# include :global "spam-to-junk";\n')
+    print("selftest ok")
+
+
+if __name__ == "__main__":
+    if sys.argv[1:] == ["--selftest"]:
+        selftest()
+        sys.exit(0)
+    errors = 0
+    for account in users():
+        try:
+            reconcile(account)
+        except Exception as err:
+            errors += 1
+            print(f"{account['name']}: {err}", file=sys.stderr, flush=True)
+    sys.exit(1 if errors else 0)

apps/base/mail/rspamd-deployment.yaml

@@ -0,0 +1,140 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rspamd
+  namespace: mail
+  annotations:
+    reloader.stakater.com/auto: "true"
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: rspamd
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: rspamd
+    spec:
+      serviceAccountName: rspamd
+      automountServiceAccountToken: false
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: rspamd
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 11333
+        runAsGroup: 11333
+        fsGroup: 11333
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: rspamd
+          image: docker.io/rspamd/rspamd:4.2.2
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: RSPAMD_LOCAL_ADDR
+              value: "*"
+            - name: RSPAMD_LOG_TYPE
+              value: console
+            - name: RSPAMD_LITELLM_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: rspamd-litellm
+                  key: key
+            - name: RSPAMD_CONTROLLER_PASSWORD_HASH
+              valueFrom:
+                secretKeyRef:
+                  name: rspamd-controller
+                  key: password_hash
+          ports:
+            - containerPort: 11332
+              name: milter
+              protocol: TCP
+            - containerPort: 11334
+              name: controller
+              protocol: TCP
+          readinessProbe:
+            tcpSocket:
+              port: milter
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: controller
+            initialDelaySeconds: 60
+            periodSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 250m
+              memory: 512Mi
+            limits:
+              cpu: "2"
+              memory: 1536Mi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/rspamd/local.d
+              readOnly: true
+            - name: sa-rules
+              mountPath: /etc/rspamd/sa-rules
+              readOnly: true
+            - name: dkim
+              mountPath: /etc/rspamd/dkim
+              readOnly: true
+            - name: data
+              mountPath: /var/lib/rspamd
+      volumes:
+        - name: config
+          configMap:
+            name: rspamd-config
+        - name: sa-rules
+          configMap:
+            name: rspamd-sa-rules
+        - name: dkim
+          secret:
+            secretName: dkim-unkin-net
+            defaultMode: 0440
+            items:
+              - key: private_key
+                path: unkin.net.mail.key
+        - name: data
+          emptyDir: {}
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: rspamd
+  namespace: mail
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: rspamd
+---
+# Milter for the smarthost only (rspamd.mail.svc:11332); controller/web UI is in-cluster only.
+apiVersion: v1
+kind: Service
+metadata:
+  name: rspamd
+  namespace: mail
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: rspamd
+  ports:
+    - name: milter
+      port: 11332
+      targetPort: milter
+    - name: controller
+      port: 11334
+      targetPort: controller

apps/base/mail/rspamd-sa-rules.yaml

@@ -0,0 +1,8 @@
+---
+# SpamAssassin rules loaded by the rspamd spamassassin module (*.cf).
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: rspamd-sa-rules
+  namespace: mail
+data: {}

apps/base/mail/rspamd-valkey.yaml

@@ -0,0 +1,178 @@
+---
+# Sentinel-managed Valkey for rspamd (bayes, history, ratelimit, gpt cache).
+# rspamd has no Redis Cluster support, so the valkey-operator (cluster mode)
+# cannot be used. start.sh follows the master the sentinels report; on a full
+# restart the highest replication offset wins. Hostnames carry a trailing dot so
+# lookups of a deleted pod fail fast instead of walking the search list (sentinel TILT).
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: rspamd-valkey
+  namespace: mail
+  annotations:
+    configmap.reloader.stakater.com/reload: "rspamd-valkey-scripts"
+spec:
+  serviceName: rspamd-valkey
+  podManagementPolicy: Parallel
+  replicas: 3
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: rspamd-valkey
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: rspamd-valkey
+    spec:
+      automountServiceAccountToken: false
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: rspamd-valkey
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 999
+        runAsGroup: 999
+        fsGroup: 999
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: valkey
+          image: docker.io/valkey/valkey:9.0.0
+          imagePullPolicy: IfNotPresent
+          command: ["sh", "/scripts/start.sh", "valkey"]
+          env:
+            - name: DOMAIN_SUFFIX
+              value: .rspamd-valkey.mail.svc.cluster.local.
+          ports:
+            - containerPort: 6379
+              name: valkey
+              protocol: TCP
+          readinessProbe:
+            exec:
+              command: ["sh", "-c", "valkey-cli info replication | grep -q '^master_host:127.0.0.1' && exit 1; valkey-cli ping | grep -q PONG"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            exec:
+              command: ["valkey-cli", "ping"]
+            initialDelaySeconds: 30
+            periodSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+          volumeMounts:
+            - name: data
+              mountPath: /data
+            - name: scripts
+              mountPath: /scripts
+              readOnly: true
+        - name: sentinel
+          image: docker.io/valkey/valkey:9.0.0
+          imagePullPolicy: IfNotPresent
+          command: ["sh", "/scripts/start.sh", "sentinel"]
+          env:
+            - name: DOMAIN_SUFFIX
+              value: .rspamd-valkey.mail.svc.cluster.local.
+          ports:
+            - containerPort: 26379
+              name: sentinel
+              protocol: TCP
+          readinessProbe:
+            exec:
+              command: ["valkey-cli", "-p", "26379", "ping"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            exec:
+              command: ["valkey-cli", "-p", "26379", "ping"]
+            initialDelaySeconds: 30
+            periodSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 64Mi
+          volumeMounts:
+            - name: sentinel
+              mountPath: /sentinel
+            - name: scripts
+              mountPath: /scripts
+              readOnly: true
+      volumes:
+        - name: scripts
+          configMap:
+            name: rspamd-valkey-scripts
+        - name: sentinel
+          emptyDir: {}
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: [ReadWriteOnce]
+        storageClassName: cephrbd-fast-delete
+        resources:
+          requests:
+            storage: 5Gi
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: rspamd-valkey
+  namespace: mail
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: rspamd-valkey
+---
+# Stable per-pod DNS, published before readiness so replicas can find the seed.
+apiVersion: v1
+kind: Service
+metadata:
+  name: rspamd-valkey
+  namespace: mail
+spec:
+  clusterIP: None
+  publishNotReadyAddresses: true
+  selector:
+    app.kubernetes.io/name: rspamd-valkey
+  ports:
+    - name: valkey
+      port: 6379
+      targetPort: valkey
+    - name: sentinel
+      port: 26379
+      targetPort: sentinel
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: rspamd-valkey-sentinel
+  namespace: mail
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: rspamd-valkey
+  ports:
+    - name: sentinel
+      port: 26379
+      targetPort: sentinel

apps/base/mail/rspamd-valkey/start.sh

@@ -0,0 +1,82 @@
+#!/bin/sh
+# Usage: start.sh valkey|sentinel. valkey boots read-only as a replica of
+# 127.0.0.1:1 ("undecided") so peers can compare offsets; decide() then follows
+# the sentinels' master, a live peer master, or the highest-offset node.
+set -eu
+name="$(hostname)"
+sts="${name%-*}"
+self="${name}${DOMAIN_SUFFIX}"
+
+info() {
+  timeout 3 valkey-cli -h "$1" info replication 2>/dev/null | tr -d '\r' || true
+}
+
+field() {
+  printf '%s\n' "$1" | sed -n "s/^$2://p"
+}
+
+undecided() {
+  [ "$(field "$1" master_host)" = 127.0.0.1 ]
+}
+
+choose_master() {
+  deadline=$(($(date +%s) + 60))
+  while :; do
+    best=""
+    best_off=-1
+    answered=0
+    for i in 0 1 2; do
+      peer="${sts}-${i}${DOMAIN_SUFFIX}"
+      if [ "$peer" = "$self" ]; then
+        r="$(info 127.0.0.1)"
+      else
+        m="$(timeout 3 valkey-cli -h "$peer" -p 26379 --raw sentinel get-master-addr-by-name rspamd 2>/dev/null | head -n1 || true)"
+        [ -n "$m" ] && { echo "$m"; return; }
+        r="$(info "$peer")"
+        [ -n "$r" ] || continue
+        answered=$((answered + 1))
+        [ "$(field "$r" role)" = master ] && { echo "$peer"; return; }
+      fi
+      undecided "$r" || continue
+      off="$(field "$r" master_repl_offset)"
+      [ "${off:-0}" -gt "$best_off" ] && best="$peer" best_off="${off:-0}"
+    done
+    if [ "$best" = "$self" ] && { [ "$answered" -eq 2 ] || [ "$(date +%s)" -ge "$deadline" ]; }; then
+      echo "$self"
+      return
+    fi
+    sleep 3
+  done
+}
+
+decide() {
+  until valkey-cli ping >/dev/null 2>&1; do sleep 1; done
+  master="$(choose_master)"
+  target="$master 6379"
+  [ "$master" = "$self" ] && target="no one"
+  until valkey-cli replicaof $target; do sleep 1; done
+}
+
+if [ "$1" = sentinel ]; then
+  until r="$(info 127.0.0.1)" && [ -n "$(field "$r" role)" ] && ! undecided "$r"; do sleep 2; done
+  master="$self"
+  [ "$(field "$r" role)" = master ] || master="$(field "$r" master_host)"
+  cat > /sentinel/sentinel.conf <<CONF
+port 26379
+protected-mode no
+sentinel resolve-hostnames yes
+sentinel announce-hostnames yes
+sentinel announce-ip $self
+sentinel monitor rspamd $master 6379 2
+sentinel down-after-milliseconds rspamd 5000
+sentinel failover-timeout rspamd 60000
+sentinel parallel-syncs rspamd 1
+CONF
+  exec valkey-sentinel /sentinel/sentinel.conf
+fi
+
+decide &
+# RDB only: an AOF reload resets the replication offset that choose_master compares.
+exec valkey-server --dir /data --appendonly no --save "60 1" \
+  --maxmemory 384mb --maxmemory-policy volatile-ttl \
+  --protected-mode no --replica-announce-ip "$self" --replicaof 127.0.0.1 1

apps/base/mail/rspamd-vault.yaml

@@ -0,0 +1,79 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: rspamd
+  namespace: mail
+---
+# Separate auth for the litellm engine: the `rspamd` k8s role is the only one
+# whose policy grants litellm/creds/rspamd.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: rspamd
+  namespace: mail
+spec:
+  allowedNamespaces:
+    - mail
+  kubernetes:
+    audiences:
+      - vault
+    role: rspamd
+    serviceAccount: rspamd
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default
+---
+# Haiku-scoped LiteLLM virtual key for the gpt module (field `key`).
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultDynamicSecret
+metadata:
+  name: rspamd-litellm
+  namespace: mail
+spec:
+  allowStaticCreds: false
+  destination:
+    create: true
+    name: rspamd-litellm
+    overwrite: true
+  mount: litellm
+  path: creds/rspamd
+  revoke: true
+  vaultAuthRef: rspamd
+---
+# DKIM private key for unkin.net, selector `mail` (field `private_key`).
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: dkim-unkin-net
+  namespace: mail
+spec:
+  destination:
+    create: true
+    name: dkim-unkin-net
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/mail/default/dkim-unkin-net
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+# Controller (web UI) password; config reads the `rspamadm pw` hash (field `password_hash`).
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: rspamd-controller
+  namespace: mail
+spec:
+  destination:
+    create: true
+    name: rspamd-controller
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/mail/default/rspamd-controller
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/base/mail/rspamd/actions.conf

@@ -0,0 +1,3 @@
+reject = 15;
+add_header = 6;
+greylist = null;

apps/base/mail/rspamd/antivirus.conf

@@ -0,0 +1,8 @@
+clamav {
+  type = "clamav";
+  servers = "clamav.mail.svc.cluster.local:3310";
+  symbol = "CLAM_VIRUS";
+  action = "reject";
+  scan_mime_parts = true;
+  log_clean = false;
+}

apps/base/mail/rspamd/dkim_signing.conf

@@ -0,0 +1,7 @@
+sign_authenticated = false;
+sign_local = true;
+sign_inbound = false;
+use_domain = "header";
+allow_hdrfrom_mismatch = false;
+selector = "mail";
+path = "/etc/rspamd/dkim/$domain.$selector.key";

apps/base/mail/rspamd/external_relay.conf

@@ -0,0 +1,8 @@
+# Edge gateway public IPs (smarthost gateways.cidr): score the first hop before them.
+enabled = true;
+rules {
+  EDGE_RELAY {
+    strategy = "ip_map";
+    ip_map = ["192.0.2.1/32", "192.0.2.2/32"];
+  }
+}

apps/base/mail/rspamd/force_actions.conf

@@ -0,0 +1,7 @@
+rules {
+  CLAMAV_UNAVAILABLE {
+    action = "soft reject";
+    expression = "CLAM_VIRUS_FAIL";
+    message = "Virus scanner unavailable, try again later";
+  }
+}

apps/base/mail/rspamd/gpt.conf

@@ -0,0 +1,22 @@
+enabled = true;
+type = "openai";
+url = "http://litellm.litellm.svc.cluster.local:4000/v1/chat/completions";
+api_key = "{= env.LITELLM_API_KEY =}";
+model = "claude-haiku-4-5";
+model_parameters = {
+  "claude-haiku-4-5" = {
+    max_tokens = 400,
+    temperature = 0.0,
+  }
+};
+timeout = 10s;
+# Inbound only (settings.conf disables it outbound); only uncertain scores (2 <= score < 12).
+condition = <<EOD
+return function(task)
+  local res = task:get_metric_result()
+  if not res then return false, 'no result' end
+  if res.passthrough then return false, 'passthrough' end
+  if res.score < 2 or res.score >= 12 then return false, 'score not uncertain' end
+  return true
+end
+EOD

apps/base/mail/rspamd/greylist.conf

@@ -0,0 +1 @@
+enabled = false;

apps/base/mail/rspamd/history_redis.conf

@@ -0,0 +1 @@
+nrows = 5000;

apps/base/mail/rspamd/local_networks.map

@@ -0,0 +1,30 @@
+# Outbound sources (smarthost mynetworks): DKIM-signed, scored with reject, no GPT.
+127.0.0.0/8
+::1
+10.42.0.0/16
+10.10.12.200/32
+198.18.1.10/32
+198.18.2.160/27
+198.18.2.192/27
+198.18.13.0/24
+198.18.14.0/24
+198.18.15.0/24
+198.18.16.0/24
+198.18.17.0/24
+198.18.18.0/24
+198.18.19.0/24
+198.18.20.0/24
+198.18.21.0/24
+198.18.22.0/24
+198.18.23.0/24
+198.18.25.0/24
+198.18.26.0/24
+198.18.27.0/24
+198.18.28.0/24
+198.18.29.0/24
+198.18.64.0/24
+198.18.65.0/24
+198.18.66.0/24
+198.18.67.0/24
+198.18.68.0/24
+198.18.69.0/24

apps/base/mail/rspamd/milter_headers.conf

@@ -0,0 +1,2 @@
+use = ["x-spamd-result", "x-spam-status", "x-spam-level", "authentication-results"];
+skip_local = true;

apps/base/mail/rspamd/options.inc

@@ -0,0 +1,2 @@
+local_addrs = "/etc/rspamd/local.d/local_networks.map";
+task_timeout = 20s;

apps/base/mail/rspamd/redis.conf

@@ -0,0 +1,4 @@
+sentinels = "rspamd-valkey-sentinel.mail.svc.cluster.local:26379";
+sentinel_masters_pattern = "^rspamd$";
+sentinel_watch_time = 10s;
+servers = "rspamd-valkey-0.rspamd-valkey.mail.svc.cluster.local.:6379";

apps/base/mail/rspamd/settings.conf

@@ -0,0 +1,19 @@
+outbound {
+  priority = high;
+  ip_map = "/etc/rspamd/local.d/local_networks.map";
+  apply {
+    symbols_disabled = ["GPT_CHECK"];
+  }
+}
+inbound {
+  priority = high;
+  ip_map = "/etc/rspamd/local.d/local_networks.map";
+  inverse = true;
+  apply {
+    symbols_disabled = ["DKIM_SIGNED"];
+    # Tag only; reject stays defined (unreachable by score) so ClamAV's virus reject still applies.
+    actions {
+      reject = 1000;
+    }
+  }
+}

apps/base/mail/rspamd/spamassassin.conf

@@ -0,0 +1 @@
+ruleset = "/etc/rspamd/sa-rules/*.cf";

apps/base/mail/rspamd/worker-controller.inc

@@ -0,0 +1,3 @@
+password = "{= env.CONTROLLER_PASSWORD_HASH =}";
+enable_password = "{= env.CONTROLLER_PASSWORD_HASH =}";
+allow_file_and_shm_inputs = false;

apps/base/mail/rspamd/worker-normal.inc

@@ -0,0 +1,2 @@
+count = 2;
+allow_file_and_shm_inputs = false;

apps/base/mail/rspamd/worker-proxy.inc

@@ -0,0 +1 @@
+allow_file_and_shm_inputs = false;

apps/base/mail/smarthost-networkpolicy.yaml

@@ -0,0 +1,78 @@
+---
+# Gateways (public IPs, see smarthost/gateways.cidr), lab networks (incl. node IPs for kubelet probes) and opted-in pods in any namespace.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: smarthost
+  namespace: mail
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: smarthost
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - ipBlock:
+            cidr: 192.0.2.1/32
+        - ipBlock:
+            cidr: 192.0.2.2/32
+        - ipBlock:
+            cidr: 10.10.12.200/32
+        - ipBlock:
+            cidr: 198.18.1.10/32
+        - ipBlock:
+            cidr: 198.18.2.160/27
+        - ipBlock:
+            cidr: 198.18.2.192/27
+        - ipBlock:
+            cidr: 198.18.13.0/24
+        - ipBlock:
+            cidr: 198.18.14.0/24
+        - ipBlock:
+            cidr: 198.18.15.0/24
+        - ipBlock:
+            cidr: 198.18.16.0/24
+        - ipBlock:
+            cidr: 198.18.17.0/24
+        - ipBlock:
+            cidr: 198.18.18.0/24
+        - ipBlock:
+            cidr: 198.18.19.0/24
+        - ipBlock:
+            cidr: 198.18.20.0/24
+        - ipBlock:
+            cidr: 198.18.21.0/24
+        - ipBlock:
+            cidr: 198.18.22.0/24
+        - ipBlock:
+            cidr: 198.18.23.0/24
+        - ipBlock:
+            cidr: 198.18.25.0/24
+        - ipBlock:
+            cidr: 198.18.26.0/24
+        - ipBlock:
+            cidr: 198.18.27.0/24
+        - ipBlock:
+            cidr: 198.18.28.0/24
+        - ipBlock:
+            cidr: 198.18.29.0/24
+        - ipBlock:
+            cidr: 198.18.64.0/24
+        - ipBlock:
+            cidr: 198.18.65.0/24
+        - ipBlock:
+            cidr: 198.18.66.0/24
+        - ipBlock:
+            cidr: 198.18.67.0/24
+        - ipBlock:
+            cidr: 198.18.68.0/24
+        - ipBlock:
+            cidr: 198.18.69.0/24
+        - namespaceSelector: {}
+          podSelector:
+            matchLabels:
+              mail.unkin.net/smarthost-client: "true"
+      ports:
+        - port: smtp
+          protocol: TCP

apps/base/mail/smarthost-pdb.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: smarthost
+  namespace: mail
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: smarthost

apps/base/mail/smarthost-service.yaml

@@ -0,0 +1,33 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: smarthost
+  namespace: mail
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: smarthost
+  ports:
+    - name: smtp
+      port: 25
+      targetPort: smtp
+---
+# Gateways and lab networks; Local keeps client IPs for the postfix CIDR checks.
+apiVersion: v1
+kind: Service
+metadata:
+  name: smarthost-lb
+  namespace: mail
+  annotations:
+    purelb.io/addresses: 198.18.200.16
+    purelb.io/service-group: common
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: smarthost
+  ports:
+    - name: smtp
+      port: 25
+      targetPort: smtp

apps/base/mail/smarthost-statefulset.yaml

@@ -0,0 +1,117 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: smarthost
+  namespace: mail
+  annotations:
+    configmap.reloader.stakater.com/reload: "smarthost-config"
+spec:
+  serviceName: smarthost
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: smarthost
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: smarthost
+    spec:
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: smarthost
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: postfix
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/postfix:0.1.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 25
+              name: smtp
+              protocol: TCP
+          readinessProbe:
+            tcpSocket:
+              port: smtp
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            exec:
+              command: ["postfix", "status"]
+            initialDelaySeconds: 30
+            periodSeconds: 30
+          # postfix master runs as root and drops to the postfix user per service.
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+              add:
+                - CHOWN
+                - DAC_OVERRIDE
+                - FOWNER
+                - SETUID
+                - SETGID
+                - NET_BIND_SERVICE
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+          volumeMounts:
+            - name: spool
+              mountPath: /var/spool/postfix
+            - name: tls
+              mountPath: /etc/postfix/tls
+              readOnly: true
+            - name: config
+              mountPath: /etc/postfix/main.cf
+              subPath: main.cf
+              readOnly: true
+            - name: config
+              mountPath: /etc/postfix/master.cf
+              subPath: master.cf
+              readOnly: true
+            - name: config
+              mountPath: /etc/postfix/relay_domains
+              subPath: relay_domains
+              readOnly: true
+            - name: config
+              mountPath: /etc/postfix/transport
+              subPath: transport
+              readOnly: true
+            - name: config
+              mountPath: /etc/postfix/gateways.cidr
+              subPath: gateways.cidr
+              readOnly: true
+            - name: config
+              mountPath: /etc/postfix/mynetworks.cidr
+              subPath: mynetworks.cidr
+              readOnly: true
+            - name: config
+              mountPath: /etc/postfix/tls_policy
+              subPath: tls_policy
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: smarthost-config
+        - name: tls
+          secret:
+            secretName: wildcard-main-unkin-net-tls
+  volumeClaimTemplates:
+    - metadata:
+        name: spool
+      spec:
+        accessModes: [ReadWriteOnce]
+        storageClassName: cephrbd-fast-delete
+        resources:
+          requests:
+            storage: 50Gi

apps/base/mail/smarthost/gateways.cidr

@@ -0,0 +1,3 @@
+# Gateway (mta1/mta2) public IPs; TEST-NET placeholders until the BinaryLane servers exist.
+192.0.2.1/32 reject_plaintext_session, permit
+192.0.2.2/32 reject_plaintext_session, permit

apps/base/mail/smarthost/main.cf

@@ -0,0 +1,42 @@
+compatibility_level = 3.5
+myhostname = mail-in.main.unkin.net
+myorigin = unkin.net
+mydestination =
+alias_maps =
+local_recipient_maps =
+inet_interfaces = all
+inet_protocols = ipv4
+maillog_file = /dev/stdout
+message_size_limit = 52428800
+
+# Relay-anywhere clients; gateways must stay out.
+mynetworks = cidr:/etc/postfix/mynetworks.cidr
+
+relay_domains = texthash:/etc/postfix/relay_domains
+# Default minus relay_domains: subdomains are not delivered and would loop back out via the gateways.
+parent_domain_matches_subdomains = debug_peer_list,fast_flush_domains,mynetworks,permit_mx_backup_networks,qmqpd_authorized_clients,smtpd_access_maps
+transport_maps = texthash:/etc/postfix/transport
+relayhost = [mta1.edge.unkin.net]:25
+smtp_fallback_relay = [mta2.edge.unkin.net]:25
+smtp_tls_policy_maps = texthash:/etc/postfix/tls_policy
+smtp_tls_CAfile = /etc/pki/tls/certs/ca-bundle.crt
+smtpd_client_restrictions = permit_mynetworks, check_client_access cidr:/etc/postfix/gateways.cidr, reject
+smtpd_relay_restrictions = permit_mynetworks, reject_unauth_destination
+smtpd_recipient_restrictions = permit_mynetworks, reject_unverified_recipient
+unverified_recipient_reject_code = 550
+unverified_recipient_tempfail_action = defer_if_permit
+address_verify_negative_refresh_time = 5m
+address_verify_negative_expire_time = 1h
+address_verify_positive_refresh_time = 1h
+address_verify_positive_expire_time = 1d
+
+milter_default_action = tempfail
+smtpd_milters = inet:rspamd.mail.svc:11332
+smtpd_authorized_xforward_hosts = cidr:/etc/postfix/gateways.cidr
+
+smtpd_tls_security_level = may
+smtpd_tls_cert_file = /etc/postfix/tls/tls.crt
+smtpd_tls_key_file = /etc/postfix/tls/tls.key
+smtpd_tls_received_header = yes
+smtp_tls_security_level = may
+lmtp_tls_security_level = none

apps/base/mail/smarthost/master.cf

@@ -0,0 +1,27 @@
+# Gateways (inbound to relay_domains) and mynetworks (relay anywhere); all via the rspamd milter.
+25        inet  n       -       n       -       -       smtpd
+pickup    unix  n       -       n       60      1       pickup
+cleanup   unix  n       -       n       -       0       cleanup
+qmgr      unix  n       -       n       300     1       qmgr
+tlsmgr    unix  -       -       n       1000?   1       tlsmgr
+rewrite   unix  -       -       n       -       -       trivial-rewrite
+bounce    unix  -       -       n       -       0       bounce
+defer     unix  -       -       n       -       0       bounce
+trace     unix  -       -       n       -       0       bounce
+verify    unix  -       -       n       -       1       verify
+flush     unix  n       -       n       1000?   0       flush
+proxymap  unix  -       -       n       -       -       proxymap
+proxywrite unix -       -       n       -       1       proxymap
+smtp      unix  -       -       n       -       -       smtp
+relay     unix  -       -       n       -       -       smtp
+        -o syslog_name=postfix/$service_name
+showq     unix  n       -       n       -       -       showq
+error     unix  -       -       n       -       -       error
+retry     unix  -       -       n       -       -       error
+discard   unix  -       -       n       -       -       discard
+local     unix  -       n       n       -       -       local
+virtual   unix  -       n       n       -       -       virtual
+lmtp      unix  -       -       n       -       -       lmtp
+anvil     unix  -       -       n       -       1       anvil
+scache    unix  -       -       n       -       1       scache
+postlog   unix-dgram n  -       n       -       1       postlogd

apps/base/mail/smarthost/mynetworks.cidr

@@ -0,0 +1,31 @@
+# Pod CIDR (narrowed by smarthost-networkpolicy.yaml) plus the lab networks from bind-internal acl-main/acl-common/acl-nomad-jobs.
+# Keep in sync with the NetworkPolicy.
+10.42.0.0/16 OK
+10.10.12.200/32 OK
+198.18.1.10/32 OK
+198.18.2.160/27 OK
+198.18.2.192/27 OK
+198.18.21.160/27 OK
+198.18.21.192/27 OK
+198.18.13.0/24 OK
+198.18.14.0/24 OK
+198.18.15.0/24 OK
+198.18.16.0/24 OK
+198.18.17.0/24 OK
+198.18.18.0/24 OK
+198.18.19.0/24 OK
+198.18.20.0/24 OK
+198.18.21.0/24 OK
+198.18.22.0/24 OK
+198.18.23.0/24 OK
+198.18.25.0/24 OK
+198.18.26.0/24 OK
+198.18.27.0/24 OK
+198.18.28.0/24 OK
+198.18.29.0/24 OK
+198.18.64.0/24 OK
+198.18.65.0/24 OK
+198.18.66.0/24 OK
+198.18.67.0/24 OK
+198.18.68.0/24 OK
+198.18.69.0/24 OK

apps/base/mail/smarthost/relay_domains

@@ -0,0 +1 @@
+unkin.net OK

apps/base/mail/smarthost/tls_policy

@@ -0,0 +1,2 @@
+[mta1.edge.unkin.net]:25 secure match=hostname
+[mta2.edge.unkin.net]:25 secure match=hostname

apps/base/mail/smarthost/transport

@@ -0,0 +1 @@
+unkin.net lmtp:[stalwart.mail.svc.cluster.local]:24

apps/base/mail/stalwart-apply-job.yaml

@@ -0,0 +1,95 @@
+---
+# Converges Stalwart's database-held settings to plan.ndjson after every sync,
+# then rolls the StatefulSet: listener, store and coordinator changes only bind
+# on startup. It rolls on every sync because neither image has a shell to diff the
+# apply output.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: stalwart-apply
+  namespace: mail
+  annotations:
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  activeDeadlineSeconds: 300
+  backoffLimit: 2
+  ttlSecondsAfterFinished: 600
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: stalwart-apply
+    spec:
+      restartPolicy: Never
+      serviceAccountName: stalwart-apply
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        - name: apply
+          image: docker.io/stalwartlabs/cli:1.0.13
+          args:
+            - apply
+            - --file
+            - /plan/plan.ndjson
+          env:
+            - name: STALWART_URL
+              value: http://stalwart.mail.svc:8080
+            - name: STALWART_USER
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: username
+            - name: STALWART_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: password
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 200m
+              memory: 128Mi
+            requests:
+              cpu: 50m
+              memory: 32Mi
+          volumeMounts:
+            - name: plan
+              mountPath: /plan
+              readOnly: true
+      containers:
+        - name: restart
+          image: registry.k8s.io/kubectl:v1.33.13
+          args:
+            - rollout
+            - restart
+            - statefulset/stalwart
+            - --namespace=mail
+          securityContext:
+            runAsUser: 65532
+            runAsGroup: 65532
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 200m
+              memory: 128Mi
+            requests:
+              cpu: 50m
+              memory: 32Mi
+      volumes:
+        - name: plan
+          configMap:
+            name: stalwart-config
+            items:
+              - key: plan.ndjson
+                path: plan.ndjson

apps/base/mail/stalwart-apply-rbac.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: stalwart-apply
+  namespace: mail
+---
+# Lets the stalwart-apply Job roll the Stalwart pods after applying the plan.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: stalwart-apply
+  namespace: mail
+rules:
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    resourceNames:
+      - stalwart
+    verbs:
+      - get
+      - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: stalwart-apply
+  namespace: mail
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: stalwart-apply
+subjects:
+  - kind: ServiceAccount
+    name: stalwart-apply
+    namespace: mail

apps/base/mail/stalwart-blobs-bucket.yaml

@@ -0,0 +1,30 @@
+---
+# Stalwart S3 blob store. The owner user has read-write on its own bucket; the
+# operator mints AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / S3_ENDPOINT into
+# stalwart-blobs-s3.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: stalwart-blobs
+  namespace: mail
+spec:
+  displayName: "Stalwart blob store owner"
+  uid: mail-stalwart-blobs
+  maxBuckets: 5
+  secretName: stalwart-blobs-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: stalwart-blobs
+  namespace: mail
+spec:
+  placementTarget: ec
+  bucketName: stalwart-blobs
+  ownerRef: stalwart-blobs
+  versioning: false
+  tags:
+    app: stalwart
+    purpose: blob-store
+  retainOnDelete: true

apps/base/mail/stalwart-configmap.yaml

@@ -0,0 +1,35 @@
+---
+# config.json only names the datastore; everything else lives in Postgres and
+# is converged from plan.ndjson by the stalwart-apply PostSync Job.
+# The Coordinator uses the plain Redis client: its cluster client never
+# resubscribes once the Valkey node it subscribed on is replaced.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: stalwart-config
+  namespace: mail
+data:
+  config.json: |
+    {"@type":"PostgreSql","host":"stalwart-db-rw.mail.svc","port":5432,"database":"stalwart","authUsername":"stalwart","authSecret":{"@type":"EnvironmentVariable","variableName":"DB_PASSWORD"},"useTls":true}
+  plan.ndjson: |
+    {"@type":"upsert","object":"Domain","matchOn":["name"],"value":{"unkin":{"name":"unkin.net","certificateManagement":{"@type":"Manual"},"dkimManagement":{"@type":"Manual"},"dnsManagement":{"@type":"Manual"},"subAddressing":{"@type":"Enabled"}}}}
+    {"@type":"upsert","object":"Certificate","matchOn":["certificate"],"value":{"cert":{"certificate":{"@type":"File","filePath":"/etc/stalwart/tls/tls.crt"},"privateKey":{"@type":"File","filePath":"/etc/stalwart/tls/tls.key"}}}}
+    {"@type":"upsert","object":"Directory","matchOn":["description"],"value":{"authentik":{"@type":"Oidc","description":"Authentik","issuerUrl":"https://identity.unkin.net/application/o/stalwart/","requireAudience":"stalwart","requireScopes":{},"claimUsername":"preferred_username","usernameDomain":"unkin.net","claimName":"name"}}}
+    {"@type":"upsert","object":"Application","matchOn":["description"],"value":{"webui":{"description":"Stalwart Web Interface","resourceUrl":"https://github.com/stalwartlabs/webui/releases/download/v1.0.11/webui.zip","urlPrefix":{"/admin":true,"/account":true},"unpackDirectory":"/var/lib/stalwart/webui","oauthClientId":"stalwart"}}}
+    {"@type":"upsert","object":"NetworkListener","matchOn":["name"],"value":{"lmtp":{"name":"lmtp","bind":{"[::]:24":true},"protocol":"lmtp","useTls":false},"submission":{"name":"submission","bind":{"[::]:587":true},"protocol":"smtp","useTls":true,"tlsImplicit":false},"metrics":{"name":"metrics","bind":{"[::]:9090":true},"protocol":"http","useTls":false}}}
+    {"@type":"upsert","object":"MtaRoute","matchOn":["name"],"value":{"smarthost":{"@type":"Relay","name":"smarthost","address":"smarthost.mail.svc","port":25,"protocol":"smtp","implicitTls":false,"allowInvalidCerts":true,"authSecret":{"@type":"None"}}}}
+    {"@type":"upsert","object":"SieveUserScript","matchOn":["name"],"value":{"spamToJunk":{"name":"spam-to-junk","description":"Files mail rspamd tagged X-Spam: Yes into Junk","isActive":true,"contents":"require [\"fileinto\", \"special-use\"];\nif header :is \"X-Spam\" \"Yes\" {\n  fileinto :specialuse \"\\\\Junk\" \"Junk Mail\";\n  stop;\n}\n"}}}
+    {"@type":"upsert","object":"MtaInboundThrottle","matchOn":["description"],"value":{"senderIp":{"description":"Sender IP throttle","enable":true,"key":{"remoteIp":true},"match":{"match":{},"else":"local_port != 24"},"rate":{"period":1000,"count":5}},"senderRcpt":{"description":"Sender address to recipient throttle","enable":true,"key":{"senderDomain":true,"rcpt":true},"match":{"match":{},"else":"local_port != 24"},"rate":{"period":3600000,"count":25}}}}
+    {"@type":"reconcile","object":"Tracer","matchOn":"*","value":{"stdout":{"@type":"Stdout","enable":true,"level":"info","ansi":false,"multiline":false,"buffered":false,"lossy":false,"eventsPolicy":"exclude","events":{}}}}
+    {"@type":"update","object":"BlobStore","value":{"@type":"S3","bucket":"stalwart-blobs","region":{"@type":"Custom","customEndpoint":"https://s3.ceph.unkin.net","customRegion":"us-east-1"},"accessKey":{"@type":"EnvironmentVariable","variableName":"AWS_ACCESS_KEY_ID"},"secretKey":{"@type":"EnvironmentVariable","variableName":"AWS_SECRET_ACCESS_KEY"},"securityToken":{"@type":"None"},"sessionToken":{"@type":"None"}}}
+    {"@type":"update","object":"SearchStore","value":{"@type":"Default"}}
+    {"@type":"update","object":"InMemoryStore","value":{"@type":"RedisCluster","urls":{"redis://valkey-mail-valkey.mail.svc:6379":true},"authUsername":null,"authSecret":{"@type":"None"},"readFromReplicas":false,"protocolVersion":"resp3"}}
+    {"@type":"update","object":"Coordinator","value":{"@type":"Redis","url":"redis://valkey-mail-valkey.mail.svc:6379"}}
+    {"@type":"update","object":"Authentication","value":{"directoryId":"#authentik"}}
+    {"@type":"update","object":"MtaOutboundStrategy","value":{"route":{"match":{"0":{"if":"is_local_domain(rcpt_domain)","then":"'local'"}},"else":"'smarthost'"}}}
+    {"@type":"update","object":"MtaStageAuth","value":{"require":{"match":{},"else":"local_port != 25 && local_port != 24"}}}
+    {"@type":"update","object":"SenderAuth","value":{"dkimSignDomain":{"match":{},"else":"false"}}}
+    {"@type":"update","object":"SpamSettings","value":{"enable":false,"spamFilterRulesUrl":null}}
+    {"@type":"update","object":"Metrics","value":{"prometheus":{"@type":"Enabled","authSecret":{"@type":"None"}}}}
+    {"@type":"update","object":"Http","value":{"allowedEndpoints":{"match":{"0":{"if":"starts_with(path, '/metrics') && local_port != 9090","then":"404"}},"else":"200"}}}
+    {"@type":"update","object":"SystemSettings","value":{"defaultHostname":"mail.unkin.net","defaultDomainId":"#unkin","defaultCertificateId":"#cert"}}

apps/base/mail/stalwart-pdb.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: stalwart
+  namespace: mail
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: stalwart

apps/base/mail/stalwart-service.yaml

@@ -0,0 +1,61 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: stalwart
+  namespace: mail
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: stalwart
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+    - name: lmtp
+      port: 24
+      targetPort: lmtp
+---
+# Governing Service for the StatefulSet's per-pod DNS.
+apiVersion: v1
+kind: Service
+metadata:
+  name: stalwart-headless
+  namespace: mail
+spec:
+  clusterIP: None
+  selector:
+    app.kubernetes.io/name: stalwart
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+---
+# Client-facing mail ports on the lab network. 443 is here too so the single
+# internal mail.unkin.net record serves web as well as IMAP/submission.
+apiVersion: v1
+kind: Service
+metadata:
+  name: stalwart-lb
+  namespace: mail
+  annotations:
+    purelb.io/addresses: 198.18.200.18
+    purelb.io/service-group: common
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: stalwart
+  ports:
+    - name: imaps
+      port: 993
+      targetPort: imaps
+    - name: submissions
+      port: 465
+      targetPort: submissions
+    - name: submission
+      port: 587
+      targetPort: submission
+    - name: https
+      port: 443
+      targetPort: https

apps/base/mail/stalwart-sieve-reconcile-cronjob.yaml

@@ -0,0 +1,70 @@
+---
+# Keeps an active personal Sieve script including the global spam-to-junk
+# script on every individual account, so rspamd-tagged mail lands in Junk.
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: stalwart-sieve-reconcile
+  namespace: mail
+spec:
+  schedule: "*/15 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: 600
+      backoffLimit: 1
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: stalwart-sieve-reconcile
+        spec:
+          restartPolicy: Never
+          automountServiceAccountToken: false
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65532
+            runAsGroup: 65532
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: reconcile
+              image: docker.io/library/python:3.13-alpine
+              command:
+                - python
+                - /scripts/sieve-reconcile.py
+              env:
+                - name: STALWART_URL
+                  value: http://stalwart.mail.svc:8080
+                - name: STALWART_USER
+                  valueFrom:
+                    secretKeyRef:
+                      name: stalwart-admin
+                      key: username
+                - name: STALWART_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: stalwart-admin
+                      key: password
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                capabilities:
+                  drop:
+                    - ALL
+              resources:
+                limits:
+                  cpu: 200m
+                  memory: 128Mi
+                requests:
+                  cpu: 50m
+                  memory: 64Mi
+              volumeMounts:
+                - name: scripts
+                  mountPath: /scripts
+                  readOnly: true
+          volumes:
+            - name: scripts
+              configMap:
+                name: stalwart-sieve-reconcile

apps/base/mail/stalwart-statefulset.yaml

@@ -0,0 +1,200 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: stalwart
+  namespace: mail
+  annotations:
+    secret.reloader.stakater.com/reload: "stalwart-admin,stalwart-db-app,stalwart-blobs-s3,wildcard-unkin-net-tls,vault-ca-cert,stalwart-db-ca"
+spec:
+  # Nodes coordinate through mail-valkey; each leases its cluster node id by
+  # hostname, so stable pod names keep a stable id across restarts.
+  replicas: 3
+  serviceName: stalwart-headless
+  podManagementPolicy: Parallel
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: stalwart
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: stalwart
+        mail.unkin.net/smarthost-client: "true"
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 2000
+        runAsGroup: 2000
+        fsGroup: 2000
+        seccompProfile:
+          type: RuntimeDefault
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: stalwart
+      # rustls loads SSL_CERT_FILE, so public roots, the internal CA (RGW) and the CNPG CA go in one bundle.
+      initContainers:
+        - name: combine-certs
+          image: docker.io/stalwartlabs/stalwart:v0.16.25
+          command:
+            - sh
+            - -c
+            - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt /db-ca/ca.crt > /combined-certs/ca-certificates.crt
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 100m
+              memory: 64Mi
+            requests:
+              cpu: 25m
+              memory: 32Mi
+          volumeMounts:
+            - name: vault-ca-cert
+              mountPath: /custom-ca
+              readOnly: true
+            - name: db-ca
+              mountPath: /db-ca
+              readOnly: true
+            - name: combined-certs
+              mountPath: /combined-certs
+      containers:
+        - name: stalwart
+          image: docker.io/stalwartlabs/stalwart:v0.16.25
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: STALWART_HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: SSL_CERT_FILE
+              value: /etc/ssl/combined/ca-certificates.crt
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-db-app
+                  key: password
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-blobs-s3
+                  key: AWS_ACCESS_KEY_ID
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-blobs-s3
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: ADMIN_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: username
+            - name: ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: password
+            - name: STALWART_RECOVERY_ADMIN
+              value: "$(ADMIN_USERNAME):$(ADMIN_PASSWORD)"
+          ports:
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+            - containerPort: 443
+              name: https
+              protocol: TCP
+            - containerPort: 24
+              name: lmtp
+              protocol: TCP
+            - containerPort: 465
+              name: submissions
+              protocol: TCP
+            - containerPort: 587
+              name: submission
+              protocol: TCP
+            - containerPort: 993
+              name: imaps
+              protocol: TCP
+            - containerPort: 9090
+              name: metrics
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz/live
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /healthz/ready
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+              # the binary carries a cap_net_bind_service file capability; exec fails without it
+              add:
+                - NET_BIND_SERVICE
+          resources:
+            limits:
+              cpu: "2"
+              memory: 2Gi
+            requests:
+              cpu: 100m
+              memory: 512Mi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/stalwart/config.json
+              subPath: config.json
+              readOnly: true
+            - name: tls
+              mountPath: /etc/stalwart/tls
+              readOnly: true
+            - name: combined-certs
+              mountPath: /etc/ssl/combined
+              readOnly: true
+            - name: data
+              mountPath: /var/lib/stalwart
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: stalwart-config
+        - name: tls
+          secret:
+            secretName: wildcard-unkin-net-tls
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: db-ca
+          secret:
+            secretName: stalwart-db-ca
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: combined-certs
+          emptyDir: {}
+        # Webui unpack dir only; mail data is in Postgres and S3.
+        - name: data
+          emptyDir: {}
+        - name: tmp
+          emptyDir: {}

apps/base/mail/valkey.yaml

@@ -0,0 +1,29 @@
+---
+# Stalwart's in-memory store (locks, rate limits, queue leases) and cluster
+# coordinator pub/sub. Ephemeral state, so no persistence; the operator runs it
+# cluster-mode, passwordless and without TLS behind valkey-mail-valkey.
+apiVersion: valkey.io/v1alpha1
+kind: ValkeyCluster
+metadata:
+  name: mail-valkey
+  namespace: mail
+spec:
+  shards: 1
+  replicas: 2
+  image: docker.io/valkey/valkey:9.0.0
+  exporter:
+    enabled: false
+  scheduling:
+    node:
+      spread:
+        shard:
+          mode: Required
+  podDisruptionBudget:
+    mode: Cluster
+  resources:
+    requests:
+      cpu: 50m
+      memory: 128Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi

apps/base/mail/vaultauth.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: default
+  namespace: mail
+spec:
+  allowedNamespaces:
+    - mail
+  kubernetes:
+    audiences:
+      - vault
+    role: default
+    serviceAccount: default
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default

apps/base/mail/vaultstaticsecret.yaml

@@ -0,0 +1,19 @@
+---
+# Fallback admin (username/password) for STALWART_RECOVERY_ADMIN and the apply
+# Job, from kv/kubernetes/namespace/mail/default/admin.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: stalwart-admin
+  namespace: mail
+spec:
+  destination:
+    create: true
+    name: stalwart-admin
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/mail/default/admin
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/base/mail/vmpodscrape.yaml

@@ -0,0 +1,14 @@
+---
+# Stalwart's Prometheus exporter, served only on the dedicated metrics listener.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: stalwart
+  namespace: mail
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: stalwart
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics/prometheus

apps/base/pdbmux/deployment.yaml

@@ -25,7 +25,7 @@ spec:
         - name: pdbmux
           # Image is published by the pdbmux repo's .woodpecker/docker.yaml on
           # a v* tag. It only exists after that tag is cut (see PR merge gates).
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/pdbmux:v0.4.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/pdbmux:v0.5.0
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8080

apps/base/woodpecker/kustomization.yaml

@@ -20,6 +20,7 @@ resources:
   - serviceaccount_repospawner_ci.yaml
   - serviceaccount_terraform_artifactapi.yaml
   - serviceaccount_terraform_authentik.yaml
+  - serviceaccount_terraform_binarylane.yaml
   - serviceaccount_terraform_enc.yaml
   - serviceaccount_terraform_git.yaml
   - serviceaccount_terraform_netbox.yaml

apps/base/woodpecker/serviceaccount_terraform_binarylane.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: terraform-binarylane
+  namespace: woodpecker

apps/overlays/au-syd1/mail/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/mail

argocd/applicationsets/platform.yaml

@@ -34,6 +34,7 @@ spec:
       - path: apps/overlays/*/jfrog
       - path: apps/overlays/*/k8up-system
       - path: apps/overlays/*/kanidm
+      - path: apps/overlays/*/mail
       - path: apps/overlays/*/netbox
       - path: apps/overlays/*/node-feature-discovery
       - path: apps/overlays/*/pdbmux

argocd/projects/platform.yaml

@@ -49,6 +49,8 @@ spec:
       server: https://kubernetes.default.svc
     - namespace: 'kanidm'
       server: https://kubernetes.default.svc
+    - namespace: 'mail'
+      server: https://kubernetes.default.svc
     - namespace: 'netbox'
       server: https://kubernetes.default.svc
     - namespace: 'node-feature-discovery'