Verify the CNPG CA for Stalwart's Postgres, pin webui, bound the apply Job

apps/base/cert-manager/certificate_wildcard-unkin-net.yaml

@@ -14,9 +14,9 @@ spec:
   secretTemplate:
     annotations:
       reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
-      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
       reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
-      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
   privateKey:
     size: 4096
   dnsNames:

apps/base/mail/gateway.yaml

@@ -0,0 +1,33 @@
+---
+# External (DMZ) front for mail.unkin.net web/JMAP via the external Traefik
+# (LB VIP 198.18.199.0), terminating with the reflected *.unkin.net wildcard.
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: external
+  name: stalwart-external
+  namespace: mail
+spec:
+  gatewayClassName: traefik-external
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      hostname: mail.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https
+      port: 443
+      protocol: HTTPS
+      hostname: mail.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: wildcard-unkin-net-tls

apps/base/mail/httproute.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: stalwart-external-http-redirect
+  namespace: mail
+spec:
+  hostnames:
+    - mail.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: stalwart-external
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: stalwart-external
+  namespace: mail
+spec:
+  hostnames:
+    - mail.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: stalwart-external
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: stalwart
+          port: 8080
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/mail/kustomization.yaml

@@ -5,5 +5,15 @@ kind: Kustomization
 resources:
   - cnpg_backup.yaml
   - cnpg_cluster.yaml
+  - gateway.yaml
+  - httproute.yaml
   - namespace.yaml
+  - stalwart-apply-job.yaml
+  - stalwart-apply-rbac.yaml
   - stalwart-blobs-bucket.yaml
+  - stalwart-configmap.yaml
+  - stalwart-deployment.yaml
+  - stalwart-service.yaml
+  - vaultauth.yaml
+  - vaultstaticsecret.yaml
+  - vmpodscrape.yaml

apps/base/mail/stalwart-apply-job.yaml

@@ -0,0 +1,94 @@
+---
+# Converges Stalwart's database-held settings to plan.ndjson after every sync,
+# then restarts the pod: listener and store changes only bind on startup.
+# ponytail: restarts on every sync since neither image has a shell to diff apply output.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: stalwart-apply
+  namespace: mail
+  annotations:
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  activeDeadlineSeconds: 300
+  backoffLimit: 2
+  ttlSecondsAfterFinished: 600
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: stalwart-apply
+    spec:
+      restartPolicy: Never
+      serviceAccountName: stalwart-apply
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        - name: apply
+          image: docker.io/stalwartlabs/cli:1.0.13
+          args:
+            - apply
+            - --file
+            - /plan/plan.ndjson
+          env:
+            - name: STALWART_URL
+              value: http://stalwart.mail.svc:8080
+            - name: STALWART_USER
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: username
+            - name: STALWART_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: password
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 200m
+              memory: 128Mi
+            requests:
+              cpu: 50m
+              memory: 32Mi
+          volumeMounts:
+            - name: plan
+              mountPath: /plan
+              readOnly: true
+      containers:
+        - name: restart
+          image: registry.k8s.io/kubectl:v1.33.13
+          args:
+            - rollout
+            - restart
+            - deployment/stalwart
+            - --namespace=mail
+          securityContext:
+            runAsUser: 65532
+            runAsGroup: 65532
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 200m
+              memory: 128Mi
+            requests:
+              cpu: 50m
+              memory: 32Mi
+      volumes:
+        - name: plan
+          configMap:
+            name: stalwart-config
+            items:
+              - key: plan.ndjson
+                path: plan.ndjson

apps/base/mail/stalwart-apply-rbac.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: stalwart-apply
+  namespace: mail
+---
+# Lets the stalwart-apply Job roll the Stalwart pod after applying the plan.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: stalwart-apply
+  namespace: mail
+rules:
+  - apiGroups:
+      - apps
+    resources:
+      - deployments
+    resourceNames:
+      - stalwart
+    verbs:
+      - get
+      - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: stalwart-apply
+  namespace: mail
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: stalwart-apply
+subjects:
+  - kind: ServiceAccount
+    name: stalwart-apply
+    namespace: mail

apps/base/mail/stalwart-configmap.yaml

@@ -0,0 +1,30 @@
+---
+# config.json only names the datastore; everything else lives in Postgres and
+# is converged from plan.ndjson by the stalwart-apply PostSync Job.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: stalwart-config
+  namespace: mail
+data:
+  config.json: |
+    {"@type":"PostgreSql","host":"stalwart-db-rw.mail.svc","port":5432,"database":"stalwart","authUsername":"stalwart","authSecret":{"@type":"EnvironmentVariable","variableName":"DB_PASSWORD"},"useTls":true}
+  plan.ndjson: |
+    {"@type":"upsert","object":"Domain","matchOn":["name"],"value":{"unkin":{"name":"unkin.net","certificateManagement":{"@type":"Manual"},"dkimManagement":{"@type":"Manual"},"dnsManagement":{"@type":"Manual"},"subAddressing":{"@type":"Enabled"}}}}
+    {"@type":"upsert","object":"Certificate","matchOn":["certificate"],"value":{"cert":{"certificate":{"@type":"File","filePath":"/etc/stalwart/tls/tls.crt"},"privateKey":{"@type":"File","filePath":"/etc/stalwart/tls/tls.key"}}}}
+    {"@type":"upsert","object":"Directory","matchOn":["description"],"value":{"authentik":{"@type":"Oidc","description":"Authentik","issuerUrl":"https://identity.unkin.net/application/o/stalwart/","requireAudience":"stalwart","requireScopes":{},"claimUsername":"preferred_username","usernameDomain":"unkin.net","claimName":"name"}}}
+    {"@type":"upsert","object":"Application","matchOn":["description"],"value":{"webui":{"description":"Stalwart Web Interface","resourceUrl":"https://github.com/stalwartlabs/webui/releases/download/v1.0.11/webui.zip","urlPrefix":{"/admin":true,"/account":true},"unpackDirectory":"/var/lib/stalwart/webui","oauthClientId":"stalwart"}}}
+    {"@type":"upsert","object":"NetworkListener","matchOn":["name"],"value":{"lmtp":{"name":"lmtp","bind":{"[::]:24":true},"protocol":"lmtp","useTls":false},"submission":{"name":"submission","bind":{"[::]:587":true},"protocol":"smtp","useTls":true,"tlsImplicit":false},"metrics":{"name":"metrics","bind":{"[::]:9090":true},"protocol":"http","useTls":false}}}
+    {"@type":"upsert","object":"MtaRoute","matchOn":["name"],"value":{"smarthost":{"@type":"Relay","name":"smarthost","address":"smarthost.mail.svc","port":25,"protocol":"smtp","implicitTls":false,"allowInvalidCerts":true,"authSecret":{"@type":"None"}}}}
+    {"@type":"reconcile","object":"Tracer","matchOn":"*","value":{"stdout":{"@type":"Stdout","enable":true,"level":"info","ansi":false,"multiline":false,"buffered":false,"lossy":false,"eventsPolicy":"exclude","events":{}}}}
+    {"@type":"update","object":"BlobStore","value":{"@type":"S3","bucket":"stalwart-blobs","region":{"@type":"Custom","customEndpoint":"https://s3.ceph.unkin.net","customRegion":"us-east-1"},"accessKey":{"@type":"EnvironmentVariable","variableName":"AWS_ACCESS_KEY_ID"},"secretKey":{"@type":"EnvironmentVariable","variableName":"AWS_SECRET_ACCESS_KEY"},"securityToken":{"@type":"None"},"sessionToken":{"@type":"None"}}}
+    {"@type":"update","object":"SearchStore","value":{"@type":"Default"}}
+    {"@type":"update","object":"InMemoryStore","value":{"@type":"Default"}}
+    {"@type":"update","object":"Authentication","value":{"directoryId":"#authentik"}}
+    {"@type":"update","object":"MtaOutboundStrategy","value":{"route":{"match":{"0":{"if":"is_local_domain(rcpt_domain)","then":"'local'"}},"else":"'smarthost'"}}}
+    {"@type":"update","object":"MtaStageAuth","value":{"require":{"match":{},"else":"local_port != 25 && local_port != 24"}}}
+    {"@type":"update","object":"SenderAuth","value":{"dkimSignDomain":{"match":{},"else":"false"}}}
+    {"@type":"update","object":"SpamSettings","value":{"enable":false,"spamFilterRulesUrl":null}}
+    {"@type":"update","object":"Metrics","value":{"prometheus":{"@type":"Enabled","authSecret":{"@type":"None"}}}}
+    {"@type":"update","object":"Http","value":{"allowedEndpoints":{"match":{"0":{"if":"starts_with(path, '/metrics') && local_port != 9090","then":"404"}},"else":"200"}}}
+    {"@type":"update","object":"SystemSettings","value":{"defaultHostname":"mail.unkin.net","defaultDomainId":"#unkin","defaultCertificateId":"#cert"}}

apps/base/mail/stalwart-deployment.yaml

@@ -0,0 +1,185 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: stalwart
+  namespace: mail
+  annotations:
+    secret.reloader.stakater.com/reload: "stalwart-admin,stalwart-db-app,stalwart-blobs-s3,wildcard-unkin-net-tls,vault-ca-cert,stalwart-db-ca"
+spec:
+  # ponytail: single replica, no cluster coordinator; add one before scaling out.
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: stalwart
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: stalwart
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 2000
+        runAsGroup: 2000
+        fsGroup: 2000
+        seccompProfile:
+          type: RuntimeDefault
+      # rustls loads SSL_CERT_FILE, so public roots, the internal CA (RGW) and the CNPG CA go in one bundle.
+      initContainers:
+        - name: combine-certs
+          image: docker.io/stalwartlabs/stalwart:v0.16.25
+          command:
+            - sh
+            - -c
+            - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt /db-ca/ca.crt > /combined-certs/ca-certificates.crt
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 100m
+              memory: 64Mi
+            requests:
+              cpu: 25m
+              memory: 32Mi
+          volumeMounts:
+            - name: vault-ca-cert
+              mountPath: /custom-ca
+              readOnly: true
+            - name: db-ca
+              mountPath: /db-ca
+              readOnly: true
+            - name: combined-certs
+              mountPath: /combined-certs
+      containers:
+        - name: stalwart
+          image: docker.io/stalwartlabs/stalwart:v0.16.25
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: SSL_CERT_FILE
+              value: /etc/ssl/combined/ca-certificates.crt
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-db-app
+                  key: password
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-blobs-s3
+                  key: AWS_ACCESS_KEY_ID
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-blobs-s3
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: ADMIN_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: username
+            - name: ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: stalwart-admin
+                  key: password
+            - name: STALWART_RECOVERY_ADMIN
+              value: "$(ADMIN_USERNAME):$(ADMIN_PASSWORD)"
+          ports:
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+            - containerPort: 443
+              name: https
+              protocol: TCP
+            - containerPort: 24
+              name: lmtp
+              protocol: TCP
+            - containerPort: 465
+              name: submissions
+              protocol: TCP
+            - containerPort: 587
+              name: submission
+              protocol: TCP
+            - containerPort: 993
+              name: imaps
+              protocol: TCP
+            - containerPort: 9090
+              name: metrics
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz/live
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /healthz/ready
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+              # the binary carries a cap_net_bind_service file capability; exec fails without it
+              add:
+                - NET_BIND_SERVICE
+          resources:
+            limits:
+              cpu: "2"
+              memory: 2Gi
+            requests:
+              cpu: 100m
+              memory: 512Mi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/stalwart/config.json
+              subPath: config.json
+              readOnly: true
+            - name: tls
+              mountPath: /etc/stalwart/tls
+              readOnly: true
+            - name: combined-certs
+              mountPath: /etc/ssl/combined
+              readOnly: true
+            - name: data
+              mountPath: /var/lib/stalwart
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: stalwart-config
+        - name: tls
+          secret:
+            secretName: wildcard-unkin-net-tls
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: db-ca
+          secret:
+            secretName: stalwart-db-ca
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: combined-certs
+          emptyDir: {}
+        # Webui unpack dir only; mail data is in Postgres and S3.
+        - name: data
+          emptyDir: {}
+        - name: tmp
+          emptyDir: {}

apps/base/mail/stalwart-service.yaml

@@ -0,0 +1,46 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: stalwart
+  namespace: mail
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: stalwart
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http
+    - name: lmtp
+      port: 24
+      targetPort: lmtp
+---
+# Client-facing mail ports on the lab network. 443 is here too so the single
+# internal mail.unkin.net record serves web as well as IMAP/submission.
+apiVersion: v1
+kind: Service
+metadata:
+  name: stalwart-lb
+  namespace: mail
+  annotations:
+    purelb.io/addresses: 198.18.200.11
+    purelb.io/service-group: common
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: stalwart
+  ports:
+    - name: imaps
+      port: 993
+      targetPort: imaps
+    - name: submissions
+      port: 465
+      targetPort: submissions
+    - name: submission
+      port: 587
+      targetPort: submission
+    - name: https
+      port: 443
+      targetPort: https

apps/base/mail/vaultauth.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: default
+  namespace: mail
+spec:
+  allowedNamespaces:
+    - mail
+  kubernetes:
+    audiences:
+      - vault
+    role: default
+    serviceAccount: default
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default

apps/base/mail/vaultstaticsecret.yaml

@@ -0,0 +1,19 @@
+---
+# Fallback admin (username/password) for STALWART_RECOVERY_ADMIN and the apply
+# Job, from kv/kubernetes/namespace/mail/default/admin.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: stalwart-admin
+  namespace: mail
+spec:
+  destination:
+    create: true
+    name: stalwart-admin
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/mail/default/admin
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/base/mail/vmpodscrape.yaml

@@ -0,0 +1,14 @@
+---
+# Stalwart's Prometheus exporter, served only on the dedicated metrics listener.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: stalwart
+  namespace: mail
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: stalwart
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics/prometheus