diff --git a/apps/base/cert-manager/certificate_wildcard-unkin-net.yaml b/apps/base/cert-manager/certificate_wildcard-unkin-net.yaml index a28c2cf..4e1e432 100644 --- a/apps/base/cert-manager/certificate_wildcard-unkin-net.yaml +++ b/apps/base/cert-manager/certificate_wildcard-unkin-net.yaml @@ -14,9 +14,9 @@ spec: secretTemplate: annotations: reflector.v1.k8s.emberstack.com/reflection-allowed: "true" - reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana" + reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail" reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true" - reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana" + reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail" privateKey: size: 4096 dnsNames: diff --git a/apps/base/mail/gateway.yaml b/apps/base/mail/gateway.yaml new file mode 100644 index 0000000..48f6f4d --- /dev/null +++ b/apps/base/mail/gateway.yaml @@ -0,0 +1,33 @@ +--- +# External (DMZ) front for mail.unkin.net web/JMAP via the external Traefik +# (LB VIP 198.18.199.0), terminating with the reflected *.unkin.net wildcard. +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + labels: + traefik.io/instance: external + name: stalwart-external + namespace: mail +spec: + gatewayClassName: traefik-external + listeners: + - name: http + port: 80 + protocol: HTTP + hostname: mail.unkin.net + allowedRoutes: + namespaces: + from: Same + - name: https + port: 443 + protocol: HTTPS + hostname: mail.unkin.net + allowedRoutes: + namespaces: + from: Same + tls: + mode: Terminate + certificateRefs: + - group: "" + kind: Secret + name: wildcard-unkin-net-tls diff --git a/apps/base/mail/httproute.yaml b/apps/base/mail/httproute.yaml new file mode 100644 index 0000000..9cfd8d4 --- /dev/null +++ b/apps/base/mail/httproute.yaml @@ -0,0 +1,49 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: stalwart-external-http-redirect + namespace: mail +spec: + hostnames: + - mail.unkin.net + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: stalwart-external + sectionName: http + rules: + - filters: + - type: RequestRedirect + requestRedirect: + scheme: https + statusCode: 301 + matches: + - path: + type: PathPrefix + value: / +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: stalwart-external + namespace: mail +spec: + hostnames: + - mail.unkin.net + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: stalwart-external + sectionName: https + rules: + - backendRefs: + - group: "" + kind: Service + name: stalwart + port: 8080 + weight: 1 + matches: + - path: + type: PathPrefix + value: / diff --git a/apps/base/mail/kustomization.yaml b/apps/base/mail/kustomization.yaml index 289d94f..4e2346f 100644 --- a/apps/base/mail/kustomization.yaml +++ b/apps/base/mail/kustomization.yaml @@ -6,6 +6,8 @@ resources: - clamav.yaml - cnpg_backup.yaml - cnpg_cluster.yaml + - gateway.yaml + - httproute.yaml - namespace.yaml - rspamd-deployment.yaml - rspamd-sa-rules.yaml @@ -15,8 +17,17 @@ resources: - smarthost-pdb.yaml - smarthost-service.yaml - smarthost-statefulset.yaml + - stalwart-apply-job.yaml + - stalwart-apply-rbac.yaml - stalwart-blobs-bucket.yaml + - stalwart-configmap.yaml + - stalwart-pdb.yaml + - stalwart-service.yaml + - stalwart-statefulset.yaml + - valkey.yaml - vaultauth.yaml + - vaultstaticsecret.yaml + - vmpodscrape.yaml configMapGenerator: - name: rspamd-config diff --git a/apps/base/mail/stalwart-apply-job.yaml b/apps/base/mail/stalwart-apply-job.yaml new file mode 100644 index 0000000..70059a3 --- /dev/null +++ b/apps/base/mail/stalwart-apply-job.yaml @@ -0,0 +1,95 @@ +--- +# Converges Stalwart's database-held settings to plan.ndjson after every sync, +# then rolls the StatefulSet: listener, store and coordinator changes only bind +# on startup. It rolls on every sync because neither image has a shell to diff the +# apply output. +apiVersion: batch/v1 +kind: Job +metadata: + name: stalwart-apply + namespace: mail + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + activeDeadlineSeconds: 300 + backoffLimit: 2 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: + app.kubernetes.io/name: stalwart-apply + spec: + restartPolicy: Never + serviceAccountName: stalwart-apply + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + initContainers: + - name: apply + image: docker.io/stalwartlabs/cli:1.0.13 + args: + - apply + - --file + - /plan/plan.ndjson + env: + - name: STALWART_URL + value: http://stalwart.mail.svc:8080 + - name: STALWART_USER + valueFrom: + secretKeyRef: + name: stalwart-admin + key: username + - name: STALWART_PASSWORD + valueFrom: + secretKeyRef: + name: stalwart-admin + key: password + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + limits: + cpu: 200m + memory: 128Mi + requests: + cpu: 50m + memory: 32Mi + volumeMounts: + - name: plan + mountPath: /plan + readOnly: true + containers: + - name: restart + image: registry.k8s.io/kubectl:v1.33.13 + args: + - rollout + - restart + - statefulset/stalwart + - --namespace=mail + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + limits: + cpu: 200m + memory: 128Mi + requests: + cpu: 50m + memory: 32Mi + volumes: + - name: plan + configMap: + name: stalwart-config + items: + - key: plan.ndjson + path: plan.ndjson diff --git a/apps/base/mail/stalwart-apply-rbac.yaml b/apps/base/mail/stalwart-apply-rbac.yaml new file mode 100644 index 0000000..bb47d35 --- /dev/null +++ b/apps/base/mail/stalwart-apply-rbac.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: stalwart-apply + namespace: mail +--- +# Lets the stalwart-apply Job roll the Stalwart pods after applying the plan. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: stalwart-apply + namespace: mail +rules: + - apiGroups: + - apps + resources: + - statefulsets + resourceNames: + - stalwart + verbs: + - get + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: stalwart-apply + namespace: mail +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: stalwart-apply +subjects: + - kind: ServiceAccount + name: stalwart-apply + namespace: mail diff --git a/apps/base/mail/stalwart-configmap.yaml b/apps/base/mail/stalwart-configmap.yaml new file mode 100644 index 0000000..cf485e4 --- /dev/null +++ b/apps/base/mail/stalwart-configmap.yaml @@ -0,0 +1,35 @@ +--- +# config.json only names the datastore; everything else lives in Postgres and +# is converged from plan.ndjson by the stalwart-apply PostSync Job. +# The Coordinator uses the plain Redis client: its cluster client never +# resubscribes once the Valkey node it subscribed on is replaced. +apiVersion: v1 +kind: ConfigMap +metadata: + name: stalwart-config + namespace: mail +data: + config.json: | + {"@type":"PostgreSql","host":"stalwart-db-rw.mail.svc","port":5432,"database":"stalwart","authUsername":"stalwart","authSecret":{"@type":"EnvironmentVariable","variableName":"DB_PASSWORD"},"useTls":true} + plan.ndjson: | + {"@type":"upsert","object":"Domain","matchOn":["name"],"value":{"unkin":{"name":"unkin.net","certificateManagement":{"@type":"Manual"},"dkimManagement":{"@type":"Manual"},"dnsManagement":{"@type":"Manual"},"subAddressing":{"@type":"Enabled"}}}} + {"@type":"upsert","object":"Certificate","matchOn":["certificate"],"value":{"cert":{"certificate":{"@type":"File","filePath":"/etc/stalwart/tls/tls.crt"},"privateKey":{"@type":"File","filePath":"/etc/stalwart/tls/tls.key"}}}} + {"@type":"upsert","object":"Directory","matchOn":["description"],"value":{"authentik":{"@type":"Oidc","description":"Authentik","issuerUrl":"https://identity.unkin.net/application/o/stalwart/","requireAudience":"stalwart","requireScopes":{},"claimUsername":"preferred_username","usernameDomain":"unkin.net","claimName":"name"}}} + {"@type":"upsert","object":"Application","matchOn":["description"],"value":{"webui":{"description":"Stalwart Web Interface","resourceUrl":"https://github.com/stalwartlabs/webui/releases/download/v1.0.11/webui.zip","urlPrefix":{"/admin":true,"/account":true},"unpackDirectory":"/var/lib/stalwart/webui","oauthClientId":"stalwart"}}} + {"@type":"upsert","object":"NetworkListener","matchOn":["name"],"value":{"lmtp":{"name":"lmtp","bind":{"[::]:24":true},"protocol":"lmtp","useTls":false},"submission":{"name":"submission","bind":{"[::]:587":true},"protocol":"smtp","useTls":true,"tlsImplicit":false},"metrics":{"name":"metrics","bind":{"[::]:9090":true},"protocol":"http","useTls":false}}} + {"@type":"upsert","object":"MtaRoute","matchOn":["name"],"value":{"smarthost":{"@type":"Relay","name":"smarthost","address":"smarthost.mail.svc","port":25,"protocol":"smtp","implicitTls":false,"allowInvalidCerts":true,"authSecret":{"@type":"None"}}}} + {"@type":"upsert","object":"SieveUserScript","matchOn":["name"],"value":{"spamToJunk":{"name":"spam-to-junk","description":"Files mail rspamd tagged X-Spam: Yes into Junk","isActive":true,"contents":"require [\"fileinto\", \"special-use\"];\nif header :is \"X-Spam\" \"Yes\" {\n fileinto :specialuse \"\\\\Junk\" \"Junk Mail\";\n stop;\n}\n"}}} + {"@type":"upsert","object":"MtaInboundThrottle","matchOn":["description"],"value":{"senderIp":{"description":"Sender IP throttle","enable":true,"key":{"remoteIp":true},"match":{"match":{},"else":"local_port != 24"},"rate":{"period":1000,"count":5}},"senderRcpt":{"description":"Sender address to recipient throttle","enable":true,"key":{"senderDomain":true,"rcpt":true},"match":{"match":{},"else":"local_port != 24"},"rate":{"period":3600000,"count":25}}}} + {"@type":"reconcile","object":"Tracer","matchOn":"*","value":{"stdout":{"@type":"Stdout","enable":true,"level":"info","ansi":false,"multiline":false,"buffered":false,"lossy":false,"eventsPolicy":"exclude","events":{}}}} + {"@type":"update","object":"BlobStore","value":{"@type":"S3","bucket":"stalwart-blobs","region":{"@type":"Custom","customEndpoint":"https://s3.ceph.unkin.net","customRegion":"us-east-1"},"accessKey":{"@type":"EnvironmentVariable","variableName":"AWS_ACCESS_KEY_ID"},"secretKey":{"@type":"EnvironmentVariable","variableName":"AWS_SECRET_ACCESS_KEY"},"securityToken":{"@type":"None"},"sessionToken":{"@type":"None"}}} + {"@type":"update","object":"SearchStore","value":{"@type":"Default"}} + {"@type":"update","object":"InMemoryStore","value":{"@type":"RedisCluster","urls":{"redis://valkey-mail-valkey.mail.svc:6379":true},"authUsername":null,"authSecret":{"@type":"None"},"readFromReplicas":false,"protocolVersion":"resp3"}} + {"@type":"update","object":"Coordinator","value":{"@type":"Redis","url":"redis://valkey-mail-valkey.mail.svc:6379"}} + {"@type":"update","object":"Authentication","value":{"directoryId":"#authentik"}} + {"@type":"update","object":"MtaOutboundStrategy","value":{"route":{"match":{"0":{"if":"is_local_domain(rcpt_domain)","then":"'local'"}},"else":"'smarthost'"}}} + {"@type":"update","object":"MtaStageAuth","value":{"require":{"match":{},"else":"local_port != 25 && local_port != 24"}}} + {"@type":"update","object":"SenderAuth","value":{"dkimSignDomain":{"match":{},"else":"false"}}} + {"@type":"update","object":"SpamSettings","value":{"enable":false,"spamFilterRulesUrl":null}} + {"@type":"update","object":"Metrics","value":{"prometheus":{"@type":"Enabled","authSecret":{"@type":"None"}}}} + {"@type":"update","object":"Http","value":{"allowedEndpoints":{"match":{"0":{"if":"starts_with(path, '/metrics') && local_port != 9090","then":"404"}},"else":"200"}}} + {"@type":"update","object":"SystemSettings","value":{"defaultHostname":"mail.unkin.net","defaultDomainId":"#unkin","defaultCertificateId":"#cert"}} diff --git a/apps/base/mail/stalwart-pdb.yaml b/apps/base/mail/stalwart-pdb.yaml new file mode 100644 index 0000000..6d43f7a --- /dev/null +++ b/apps/base/mail/stalwart-pdb.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: stalwart + namespace: mail +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: stalwart diff --git a/apps/base/mail/stalwart-service.yaml b/apps/base/mail/stalwart-service.yaml new file mode 100644 index 0000000..b8837c7 --- /dev/null +++ b/apps/base/mail/stalwart-service.yaml @@ -0,0 +1,61 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: stalwart + namespace: mail +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: stalwart + ports: + - name: http + port: 8080 + targetPort: http + - name: lmtp + port: 24 + targetPort: lmtp +--- +# Governing Service for the StatefulSet's per-pod DNS. +apiVersion: v1 +kind: Service +metadata: + name: stalwart-headless + namespace: mail +spec: + clusterIP: None + selector: + app.kubernetes.io/name: stalwart + ports: + - name: http + port: 8080 + targetPort: http +--- +# Client-facing mail ports on the lab network. 443 is here too so the single +# internal mail.unkin.net record serves web as well as IMAP/submission. +apiVersion: v1 +kind: Service +metadata: + name: stalwart-lb + namespace: mail + annotations: + purelb.io/addresses: 198.18.200.18 + purelb.io/service-group: common +spec: + type: LoadBalancer + externalTrafficPolicy: Local + selector: + app.kubernetes.io/name: stalwart + ports: + - name: imaps + port: 993 + targetPort: imaps + - name: submissions + port: 465 + targetPort: submissions + - name: submission + port: 587 + targetPort: submission + - name: https + port: 443 + targetPort: https diff --git a/apps/base/mail/stalwart-statefulset.yaml b/apps/base/mail/stalwart-statefulset.yaml new file mode 100644 index 0000000..964b105 --- /dev/null +++ b/apps/base/mail/stalwart-statefulset.yaml @@ -0,0 +1,200 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: stalwart + namespace: mail + annotations: + secret.reloader.stakater.com/reload: "stalwart-admin,stalwart-db-app,stalwart-blobs-s3,wildcard-unkin-net-tls,vault-ca-cert,stalwart-db-ca" +spec: + # Nodes coordinate through mail-valkey; each leases its cluster node id by + # hostname, so stable pod names keep a stable id across restarts. + replicas: 3 + serviceName: stalwart-headless + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: stalwart + template: + metadata: + labels: + app.kubernetes.io/name: stalwart + mail.unkin.net/smarthost-client: "true" + spec: + securityContext: + runAsNonRoot: true + runAsUser: 2000 + runAsGroup: 2000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: stalwart + # rustls loads SSL_CERT_FILE, so public roots, the internal CA (RGW) and the CNPG CA go in one bundle. + initContainers: + - name: combine-certs + image: docker.io/stalwartlabs/stalwart:v0.16.25 + command: + - sh + - -c + - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt /db-ca/ca.crt > /combined-certs/ca-certificates.crt + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 25m + memory: 32Mi + volumeMounts: + - name: vault-ca-cert + mountPath: /custom-ca + readOnly: true + - name: db-ca + mountPath: /db-ca + readOnly: true + - name: combined-certs + mountPath: /combined-certs + containers: + - name: stalwart + image: docker.io/stalwartlabs/stalwart:v0.16.25 + imagePullPolicy: IfNotPresent + env: + - name: STALWART_HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: SSL_CERT_FILE + value: /etc/ssl/combined/ca-certificates.crt + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: stalwart-db-app + key: password + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: stalwart-blobs-s3 + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: stalwart-blobs-s3 + key: AWS_SECRET_ACCESS_KEY + - name: ADMIN_USERNAME + valueFrom: + secretKeyRef: + name: stalwart-admin + key: username + - name: ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: stalwart-admin + key: password + - name: STALWART_RECOVERY_ADMIN + value: "$(ADMIN_USERNAME):$(ADMIN_PASSWORD)" + ports: + - containerPort: 8080 + name: http + protocol: TCP + - containerPort: 443 + name: https + protocol: TCP + - containerPort: 24 + name: lmtp + protocol: TCP + - containerPort: 465 + name: submissions + protocol: TCP + - containerPort: 587 + name: submission + protocol: TCP + - containerPort: 993 + name: imaps + protocol: TCP + - containerPort: 9090 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + path: /healthz/live + port: http + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /healthz/ready + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + # the binary carries a cap_net_bind_service file capability; exec fails without it + add: + - NET_BIND_SERVICE + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 100m + memory: 512Mi + volumeMounts: + - name: config + mountPath: /etc/stalwart/config.json + subPath: config.json + readOnly: true + - name: tls + mountPath: /etc/stalwart/tls + readOnly: true + - name: combined-certs + mountPath: /etc/ssl/combined + readOnly: true + - name: data + mountPath: /var/lib/stalwart + - name: tmp + mountPath: /tmp + volumes: + - name: config + configMap: + name: stalwart-config + - name: tls + secret: + secretName: wildcard-unkin-net-tls + - name: vault-ca-cert + secret: + secretName: vault-ca-cert + items: + - key: ca.crt + path: ca.crt + - name: db-ca + secret: + secretName: stalwart-db-ca + items: + - key: ca.crt + path: ca.crt + - name: combined-certs + emptyDir: {} + # Webui unpack dir only; mail data is in Postgres and S3. + - name: data + emptyDir: {} + - name: tmp + emptyDir: {} diff --git a/apps/base/mail/valkey.yaml b/apps/base/mail/valkey.yaml new file mode 100644 index 0000000..a495c71 --- /dev/null +++ b/apps/base/mail/valkey.yaml @@ -0,0 +1,29 @@ +--- +# Stalwart's in-memory store (locks, rate limits, queue leases) and cluster +# coordinator pub/sub. Ephemeral state, so no persistence; the operator runs it +# cluster-mode, passwordless and without TLS behind valkey-mail-valkey. +apiVersion: valkey.io/v1alpha1 +kind: ValkeyCluster +metadata: + name: mail-valkey + namespace: mail +spec: + shards: 1 + replicas: 2 + image: docker.io/valkey/valkey:9.0.0 + exporter: + enabled: false + scheduling: + node: + spread: + shard: + mode: Required + podDisruptionBudget: + mode: Cluster + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi diff --git a/apps/base/mail/vaultstaticsecret.yaml b/apps/base/mail/vaultstaticsecret.yaml new file mode 100644 index 0000000..2c89782 --- /dev/null +++ b/apps/base/mail/vaultstaticsecret.yaml @@ -0,0 +1,19 @@ +--- +# Fallback admin (username/password) for STALWART_RECOVERY_ADMIN and the apply +# Job, from kv/kubernetes/namespace/mail/default/admin. +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: stalwart-admin + namespace: mail +spec: + destination: + create: true + name: stalwart-admin + overwrite: true + hmacSecretData: true + mount: kv + path: kubernetes/namespace/mail/default/admin + refreshAfter: 5m + type: kv-v2 + vaultAuthRef: default diff --git a/apps/base/mail/vmpodscrape.yaml b/apps/base/mail/vmpodscrape.yaml new file mode 100644 index 0000000..b5afb59 --- /dev/null +++ b/apps/base/mail/vmpodscrape.yaml @@ -0,0 +1,14 @@ +--- +# Stalwart's Prometheus exporter, served only on the dedicated metrics listener. +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: stalwart + namespace: mail +spec: + selector: + matchLabels: + app.kubernetes.io/name: stalwart + podMetricsEndpoints: + - port: metrics + path: /metrics/prometheus