Verify the CNPG CA for Stalwart's Postgres, pin webui, bound the apply Job

Stalwart needs a Postgres metadata store and an S3 blob store before it can be deployed into the new `mail` namespace.

- add CNPG Cluster `stalwart-db` (db/owner `stalwart`, CNPG-minted `stalwart-db-app` creds) with barman backups to a dedicated `cnpg-mail` bucket and a nightly ScheduledBackup
- add cephrgw Bucket `stalwart-blobs` with an owner user writing RW keys to `stalwart-blobs-s3`
- add the au-syd1 overlay, platform ApplicationSet path and project destination

Reviewed-on: #538
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

apps/base/artifactapi/api-deployment.yaml

@@ -36,7 +36,7 @@ spec:
               mountPath: /combined-certs
       containers:
         - name: api
-          image: git.unkin.net/unkin/artifactapi:v3.12.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.13.1
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000

apps/base/artifactapi/image-keeper.yaml

@@ -0,0 +1,105 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: image-keeper
+  namespace: artifactapi
+spec:
+  selector:
+    matchLabels:
+      app: image-keeper
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: image-keeper
+    spec:
+      automountServiceAccountToken: false
+      priorityClassName: low
+      tolerations:
+        - operator: Exists
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        # artifactapi is distroless with no exit-0 flag, so run a static busybox as `true`
+        - name: copy-true
+          image: busybox:1.37.0-musl
+          imagePullPolicy: IfNotPresent
+          command: ["cp", "/bin/busybox", "/keeper/true"]
+          volumeMounts:
+            - name: keeper
+              mountPath: /keeper
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+        - name: api
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.13.1
+          imagePullPolicy: IfNotPresent
+          command: ["/keeper/true"]
+          volumeMounts:
+            - name: keeper
+              mountPath: /keeper
+              readOnly: true
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+        - name: ui
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.13.1
+          imagePullPolicy: IfNotPresent
+          command: ["true"]
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+      containers:
+        - name: pause
+          image: rancher/mirrored-pause:3.6
+          imagePullPolicy: IfNotPresent
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+      volumes:
+        - name: keeper
+          emptyDir: {}

apps/base/artifactapi/kustomization.yaml

@@ -11,6 +11,7 @@ resources:
   - cnpg_pooler.yaml
   - gateway.yaml
   - httproute.yaml
+  - image-keeper.yaml
   - namespace.yaml
   - oauth2-proxy-configmap.yaml
   - oauth2-proxy-deployment.yaml

apps/base/artifactapi/ui-deployment.yaml

@@ -22,7 +22,7 @@ spec:
       automountServiceAccountToken: true
       containers:
         - name: ui
-          image: git.unkin.net/unkin/artifactapi-ui:v3.12.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.13.1
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 80

apps/base/cert-manager/certificate_wildcard-unkin-net.yaml

@@ -14,9 +14,9 @@ spec:
   secretTemplate:
     annotations:
       reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
-      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
       reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
-      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
   privateKey:
     size: 4096
   dnsNames:

apps/base/mail/cnpg_backup.yaml

@@ -0,0 +1,40 @@
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: cnpg-mail-backup
+  namespace: mail
+spec:
+  displayName: "CNPG backup owner (mail)"
+  uid: cnpg-mail-backup
+  maxBuckets: 5
+  secretName: cnpg-mail-backup-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: cnpg-mail
+  namespace: mail
+spec:
+  placementTarget: ec
+  bucketName: cnpg-mail
+  ownerRef: cnpg-mail-backup
+  versioning: false
+  tags:
+    app: mail
+    purpose: cnpg-backup
+  retainOnDelete: true
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-mail-nightly
+  namespace: mail
+spec:
+  schedule: "0 20 4 * * *"
+  immediate: false
+  backupOwnerReference: self
+  method: barmanObjectStore
+  cluster:
+    name: stalwart-db

apps/base/mail/cnpg_cluster.yaml

@@ -0,0 +1,118 @@
+---
+# Stalwart metadata store. No bootstrap secret: CNPG mints stalwart-db-app
+# (username/password/dbname/uri) for Stalwart to consume.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: stalwart-db
+  namespace: mail
+spec:
+  affinity:
+    podAntiAffinityType: preferred
+  backup:
+    retentionPolicy: 30d
+    barmanObjectStore:
+      destinationPath: s3://cnpg-mail
+      endpointURL: https://s3.ceph.unkin.net
+      endpointCA:
+        name: vault-ca-cert
+        key: ca.crt
+      s3Credentials:
+        accessKeyId:
+          name: cnpg-mail-backup-s3
+          key: AWS_ACCESS_KEY_ID
+        secretAccessKey:
+          name: cnpg-mail-backup-s3
+          key: AWS_SECRET_ACCESS_KEY
+      serverName: stalwart-db
+      data:
+        compression: bzip2
+        jobs: 2
+      wal:
+        compression: zstd
+        maxParallel: 2
+  bootstrap:
+    initdb:
+      database: stalwart
+      encoding: UTF8
+      localeCType: C
+      localeCollate: C
+      owner: stalwart
+  enablePDB: true
+  enableSuperuserAccess: false
+  failoverDelay: 0
+  imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie
+  instances: 3
+  logLevel: info
+  maxSyncReplicas: 0
+  minSyncReplicas: 0
+  monitoring:
+    customQueriesConfigMap:
+      - key: queries
+        name: cnpg-default-monitoring
+    disableDefaultQueries: false
+    enablePodMonitor: false
+  postgresql:
+    parameters:
+      archive_mode: "on"
+      archive_timeout: 5min
+      dynamic_shared_memory_type: posix
+      effective_cache_size: 256MB
+      full_page_writes: "on"
+      hot_standby_feedback: "on"
+      log_destination: csvlog
+      log_directory: /controller/log
+      log_filename: postgres
+      log_rotation_age: "0"
+      log_rotation_size: "0"
+      log_truncate_on_rotation: "false"
+      logging_collector: "on"
+      max_connections: "200"
+      max_parallel_workers: "16"
+      max_replication_slots: "16"
+      max_worker_processes: "16"
+      pg_stat_statements.max: "10000"
+      pg_stat_statements.track: top
+      shared_buffers: 128MB
+      shared_memory_type: mmap
+      ssl_max_protocol_version: TLSv1.3
+      ssl_min_protocol_version: TLSv1.3
+      wal_keep_size: 256MB
+      wal_level: logical
+      wal_log_hints: "on"
+      wal_receiver_timeout: 5s
+      wal_sender_timeout: 5s
+    shared_preload_libraries:
+      - pg_stat_statements
+    syncReplicaElectionConstraint:
+      enabled: false
+  primaryUpdateMethod: restart
+  primaryUpdateStrategy: unsupervised
+  probes:
+    liveness:
+      isolationCheck:
+        connectionTimeout: 1000
+        enabled: true
+        requestTimeout: 1000
+  replicationSlots:
+    highAvailability:
+      enabled: true
+      slotPrefix: _cnpg_
+    synchronizeReplicas:
+      enabled: true
+    updateInterval: 30
+  resources:
+    limits:
+      cpu: "1"
+      memory: 1Gi
+    requests:
+      cpu: 50m
+      memory: 512Mi
+  smartShutdownTimeout: 180
+  startDelay: 3600
+  stopDelay: 1800
+  storage:
+    resizeInUseVolumes: true
+    size: 10Gi
+    storageClass: cephrbd-fast-delete
+  switchoverDelay: 3600

apps/base/mail/gateway.yaml

@@ -0,0 +1,33 @@
+---
+# External (DMZ) front for mail.unkin.net web/JMAP via the external Traefik
+# (LB VIP 198.18.199.0), terminating with the reflected *.unkin.net wildcard.
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: external
+  name: stalwart-external
+  namespace: mail
+spec:
+  gatewayClassName: traefik-external
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      hostname: mail.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https
+      port: 443
+      protocol: HTTPS
+      hostname: mail.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: wildcard-unkin-net-tls

apps/base/mail/httproute.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: stalwart-external-http-redirect
+  namespace: mail
+spec:
+  hostnames:
+    - mail.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: stalwart-external
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: stalwart-external
+  namespace: mail
+spec:
+  hostnames:
+    - mail.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: stalwart-external
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: stalwart
+          port: 8080
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/mail/kustomization.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - cnpg_backup.yaml
+  - cnpg_cluster.yaml
+  - gateway.yaml
+  - httproute.yaml
+  - namespace.yaml
+  - stalwart-apply-job.yaml
+  - stalwart-apply-rbac.yaml
+  - stalwart-blobs-bucket.yaml
+  - stalwart-configmap.yaml
+  - stalwart-deployment.yaml
+  - stalwart-service.yaml
+  - vaultauth.yaml
+  - vaultstaticsecret.yaml
+  - vmpodscrape.yaml

apps/base/mail/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: mail

apps/base/mail/stalwart-apply-job.yaml

@@ -0,0 +1,94 @@
+---
+# Converges Stalwart's database-held settings to plan.ndjson after every sync,
+# then restarts the pod: listener and store changes only bind on startup.
+# ponytail: restarts on every sync since neither image has a shell to diff apply output.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: stalwart-apply
+  namespace: mail
+  annotations:
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  activeDeadlineSeconds: 300
+  backoffLimit: 2
+  ttlSecondsAfterFinished: 600
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: stalwart-apply
+    spec:
+      restartPolicy: Never
+      serviceAccountName: stalwart-apply
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        - name: apply
+          image: docker.io/stalwartlabs/cli:1.0.13
+          args:
+            - apply
+            - --file
+            - /plan/plan.ndjson
+          env:
+            - name: STALWART_URL
+              value: http://stalwart.mail.svc:8080
+            - name: STALWART_USER
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: username
+            - name: STALWART_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: password
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 200m
+              memory: 128Mi
+            requests:
+              cpu: 50m
+              memory: 32Mi
+          volumeMounts:
+            - name: plan
+              mountPath: /plan
+              readOnly: true
+      containers:
+        - name: restart
+          image: registry.k8s.io/kubectl:v1.33.13
+          args:
+            - rollout
+            - restart
+            - deployment/stalwart
+            - --namespace=mail
+          securityContext:
+            runAsUser: 65532
+            runAsGroup: 65532
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 200m
+              memory: 128Mi
+            requests:
+              cpu: 50m
+              memory: 32Mi
+      volumes:
+        - name: plan
+          configMap:
+            name: stalwart-config
+            items:
+              - key: plan.ndjson
+                path: plan.ndjson

apps/base/mail/stalwart-apply-rbac.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: stalwart-apply
+  namespace: mail
+---
+# Lets the stalwart-apply Job roll the Stalwart pod after applying the plan.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: stalwart-apply
+  namespace: mail
+rules:
+  - apiGroups:
+      - apps
+    resources:
+      - deployments
+    resourceNames:
+      - stalwart
+    verbs:
+      - get
+      - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: stalwart-apply
+  namespace: mail
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: stalwart-apply
+subjects:
+  - kind: ServiceAccount
+    name: stalwart-apply
+    namespace: mail

apps/base/mail/stalwart-blobs-bucket.yaml

@@ -0,0 +1,30 @@
+---
+# Stalwart S3 blob store. The owner user has read-write on its own bucket; the
+# operator mints AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / S3_ENDPOINT into
+# stalwart-blobs-s3.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: stalwart-blobs
+  namespace: mail
+spec:
+  displayName: "Stalwart blob store owner"
+  uid: mail-stalwart-blobs
+  maxBuckets: 5
+  secretName: stalwart-blobs-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: stalwart-blobs
+  namespace: mail
+spec:
+  placementTarget: ec
+  bucketName: stalwart-blobs
+  ownerRef: stalwart-blobs
+  versioning: false
+  tags:
+    app: stalwart
+    purpose: blob-store
+  retainOnDelete: true

apps/base/mail/stalwart-configmap.yaml

@@ -0,0 +1,30 @@
+---
+# config.json only names the datastore; everything else lives in Postgres and
+# is converged from plan.ndjson by the stalwart-apply PostSync Job.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: stalwart-config
+  namespace: mail
+data:
+  config.json: |
+    {"@type":"PostgreSql","host":"stalwart-db-rw.mail.svc","port":5432,"database":"stalwart","authUsername":"stalwart","authSecret":{"@type":"EnvironmentVariable","variableName":"DB_PASSWORD"},"useTls":true}
+  plan.ndjson: |
+    {"@type":"upsert","object":"Domain","matchOn":["name"],"value":{"unkin":{"name":"unkin.net","certificateManagement":{"@type":"Manual"},"dkimManagement":{"@type":"Manual"},"dnsManagement":{"@type":"Manual"},"subAddressing":{"@type":"Enabled"}}}}
+    {"@type":"upsert","object":"Certificate","matchOn":["certificate"],"value":{"cert":{"certificate":{"@type":"File","filePath":"/etc/stalwart/tls/tls.crt"},"privateKey":{"@type":"File","filePath":"/etc/stalwart/tls/tls.key"}}}}
+    {"@type":"upsert","object":"Directory","matchOn":["description"],"value":{"authentik":{"@type":"Oidc","description":"Authentik","issuerUrl":"https://identity.unkin.net/application/o/stalwart/","requireAudience":"stalwart","requireScopes":{},"claimUsername":"preferred_username","usernameDomain":"unkin.net","claimName":"name"}}}
+    {"@type":"upsert","object":"Application","matchOn":["description"],"value":{"webui":{"description":"Stalwart Web Interface","resourceUrl":"https://github.com/stalwartlabs/webui/releases/download/v1.0.11/webui.zip","urlPrefix":{"/admin":true,"/account":true},"unpackDirectory":"/var/lib/stalwart/webui","oauthClientId":"stalwart"}}}
+    {"@type":"upsert","object":"NetworkListener","matchOn":["name"],"value":{"lmtp":{"name":"lmtp","bind":{"[::]:24":true},"protocol":"lmtp","useTls":false},"submission":{"name":"submission","bind":{"[::]:587":true},"protocol":"smtp","useTls":true,"tlsImplicit":false},"metrics":{"name":"metrics","bind":{"[::]:9090":true},"protocol":"http","useTls":false}}}
+    {"@type":"upsert","object":"MtaRoute","matchOn":["name"],"value":{"smarthost":{"@type":"Relay","name":"smarthost","address":"smarthost.mail.svc","port":25,"protocol":"smtp","implicitTls":false,"allowInvalidCerts":true,"authSecret":{"@type":"None"}}}}
+    {"@type":"reconcile","object":"Tracer","matchOn":"*","value":{"stdout":{"@type":"Stdout","enable":true,"level":"info","ansi":false,"multiline":false,"buffered":false,"lossy":false,"eventsPolicy":"exclude","events":{}}}}
+    {"@type":"update","object":"BlobStore","value":{"@type":"S3","bucket":"stalwart-blobs","region":{"@type":"Custom","customEndpoint":"https://s3.ceph.unkin.net","customRegion":"us-east-1"},"accessKey":{"@type":"EnvironmentVariable","variableName":"AWS_ACCESS_KEY_ID"},"secretKey":{"@type":"EnvironmentVariable","variableName":"AWS_SECRET_ACCESS_KEY"},"securityToken":{"@type":"None"},"sessionToken":{"@type":"None"}}}
+    {"@type":"update","object":"SearchStore","value":{"@type":"Default"}}
+    {"@type":"update","object":"InMemoryStore","value":{"@type":"Default"}}
+    {"@type":"update","object":"Authentication","value":{"directoryId":"#authentik"}}
+    {"@type":"update","object":"MtaOutboundStrategy","value":{"route":{"match":{"0":{"if":"is_local_domain(rcpt_domain)","then":"'local'"}},"else":"'smarthost'"}}}
+    {"@type":"update","object":"MtaStageAuth","value":{"require":{"match":{},"else":"local_port != 25 && local_port != 24"}}}
+    {"@type":"update","object":"SenderAuth","value":{"dkimSignDomain":{"match":{},"else":"false"}}}
+    {"@type":"update","object":"SpamSettings","value":{"enable":false,"spamFilterRulesUrl":null}}
+    {"@type":"update","object":"Metrics","value":{"prometheus":{"@type":"Enabled","authSecret":{"@type":"None"}}}}
+    {"@type":"update","object":"Http","value":{"allowedEndpoints":{"match":{"0":{"if":"starts_with(path, '/metrics') && local_port != 9090","then":"404"}},"else":"200"}}}
+    {"@type":"update","object":"SystemSettings","value":{"defaultHostname":"mail.unkin.net","defaultDomainId":"#unkin","defaultCertificateId":"#cert"}}

apps/base/mail/stalwart-deployment.yaml

@@ -0,0 +1,185 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: stalwart
+  namespace: mail
+  annotations:
+    secret.reloader.stakater.com/reload: "stalwart-admin,stalwart-db-app,stalwart-blobs-s3,wildcard-unkin-net-tls,vault-ca-cert,stalwart-db-ca"
+spec:
+  # ponytail: single replica, no cluster coordinator; add one before scaling out.
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: stalwart
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: stalwart
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 2000
+        runAsGroup: 2000
+        fsGroup: 2000
+        seccompProfile:
+          type: RuntimeDefault
+      # rustls loads SSL_CERT_FILE, so public roots, the internal CA (RGW) and the CNPG CA go in one bundle.
+      initContainers:
+        - name: combine-certs
+          image: docker.io/stalwartlabs/stalwart:v0.16.25
+          command:
+            - sh
+            - -c
+            - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt /db-ca/ca.crt > /combined-certs/ca-certificates.crt
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 100m
+              memory: 64Mi
+            requests:
+              cpu: 25m
+              memory: 32Mi
+          volumeMounts:
+            - name: vault-ca-cert
+              mountPath: /custom-ca
+              readOnly: true
+            - name: db-ca
+              mountPath: /db-ca
+              readOnly: true
+            - name: combined-certs
+              mountPath: /combined-certs
+      containers:
+        - name: stalwart
+          image: docker.io/stalwartlabs/stalwart:v0.16.25
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: SSL_CERT_FILE
+              value: /etc/ssl/combined/ca-certificates.crt
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-db-app
+                  key: password
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-blobs-s3
+                  key: AWS_ACCESS_KEY_ID
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-blobs-s3
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: ADMIN_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: username
+            - name: ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: password
+            - name: STALWART_RECOVERY_ADMIN
+              value: "$(ADMIN_USERNAME):$(ADMIN_PASSWORD)"
+          ports:
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+            - containerPort: 443
+              name: https
+              protocol: TCP
+            - containerPort: 24
+              name: lmtp
+              protocol: TCP
+            - containerPort: 465
+              name: submissions
+              protocol: TCP
+            - containerPort: 587
+              name: submission
+              protocol: TCP
+            - containerPort: 993
+              name: imaps
+              protocol: TCP
+            - containerPort: 9090
+              name: metrics
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz/live
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /healthz/ready
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+              # the binary carries a cap_net_bind_service file capability; exec fails without it
+              add:
+                - NET_BIND_SERVICE
+          resources:
+            limits:
+              cpu: "2"
+              memory: 2Gi
+            requests:
+              cpu: 100m
+              memory: 512Mi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/stalwart/config.json
+              subPath: config.json
+              readOnly: true
+            - name: tls
+              mountPath: /etc/stalwart/tls
+              readOnly: true
+            - name: combined-certs
+              mountPath: /etc/ssl/combined
+              readOnly: true
+            - name: data
+              mountPath: /var/lib/stalwart
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: stalwart-config
+        - name: tls
+          secret:
+            secretName: wildcard-unkin-net-tls
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: db-ca
+          secret:
+            secretName: stalwart-db-ca
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: combined-certs
+          emptyDir: {}
+        # Webui unpack dir only; mail data is in Postgres and S3.
+        - name: data
+          emptyDir: {}
+        - name: tmp
+          emptyDir: {}

apps/base/mail/stalwart-service.yaml

@@ -0,0 +1,46 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: stalwart
+  namespace: mail
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: stalwart
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+    - name: lmtp
+      port: 24
+      targetPort: lmtp
+---
+# Client-facing mail ports on the lab network. 443 is here too so the single
+# internal mail.unkin.net record serves web as well as IMAP/submission.
+apiVersion: v1
+kind: Service
+metadata:
+  name: stalwart-lb
+  namespace: mail
+  annotations:
+    purelb.io/addresses: 198.18.200.11
+    purelb.io/service-group: common
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: stalwart
+  ports:
+    - name: imaps
+      port: 993
+      targetPort: imaps
+    - name: submissions
+      port: 465
+      targetPort: submissions
+    - name: submission
+      port: 587
+      targetPort: submission
+    - name: https
+      port: 443
+      targetPort: https

apps/base/mail/vaultauth.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: default
+  namespace: mail
+spec:
+  allowedNamespaces:
+    - mail
+  kubernetes:
+    audiences:
+      - vault
+    role: default
+    serviceAccount: default
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default

apps/base/mail/vaultstaticsecret.yaml

@@ -0,0 +1,19 @@
+---
+# Fallback admin (username/password) for STALWART_RECOVERY_ADMIN and the apply
+# Job, from kv/kubernetes/namespace/mail/default/admin.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: stalwart-admin
+  namespace: mail
+spec:
+  destination:
+    create: true
+    name: stalwart-admin
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/mail/default/admin
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/base/mail/vmpodscrape.yaml

@@ -0,0 +1,14 @@
+---
+# Stalwart's Prometheus exporter, served only on the dedicated metrics listener.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: stalwart
+  namespace: mail
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: stalwart
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics/prometheus

apps/base/pdbmux/deployment.yaml

@@ -25,7 +25,7 @@ spec:
         - name: pdbmux
           # Image is published by the pdbmux repo's .woodpecker/docker.yaml on
           # a v* tag. It only exists after that tag is cut (see PR merge gates).
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/pdbmux:v0.4.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/pdbmux:v0.5.0
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8080

apps/base/woodpecker/kustomization.yaml

@@ -20,6 +20,7 @@ resources:
   - serviceaccount_repospawner_ci.yaml
   - serviceaccount_terraform_artifactapi.yaml
   - serviceaccount_terraform_authentik.yaml
+  - serviceaccount_terraform_binarylane.yaml
   - serviceaccount_terraform_enc.yaml
   - serviceaccount_terraform_git.yaml
   - serviceaccount_terraform_netbox.yaml

apps/base/woodpecker/serviceaccount_terraform_binarylane.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: terraform-binarylane
+  namespace: woodpecker

apps/overlays/au-syd1/mail/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/mail

argocd/applicationsets/platform.yaml

@@ -34,6 +34,7 @@ spec:
       - path: apps/overlays/*/jfrog
       - path: apps/overlays/*/k8up-system
       - path: apps/overlays/*/kanidm
+      - path: apps/overlays/*/mail
       - path: apps/overlays/*/netbox
       - path: apps/overlays/*/node-feature-discovery
       - path: apps/overlays/*/pdbmux

argocd/projects/platform.yaml

@@ -49,6 +49,8 @@ spec:
       server: https://kubernetes.default.svc
     - namespace: 'kanidm'
       server: https://kubernetes.default.svc
+    - namespace: 'mail'
+      server: https://kubernetes.default.svc
     - namespace: 'netbox'
       server: https://kubernetes.default.svc
     - namespace: 'node-feature-discovery'