From 0d89a69c1894380841ac2b183ee1ce4b6fdf2383 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sun, 24 May 2026 00:48:49 +1000 Subject: [PATCH] feat(stalwart): deploy Stalwart mail server with CNPG and S3 - stalwart namespace with Deployment + HPA (2-6 replicas) - CNPG PostgreSQL cluster (3 instances, 20Gi cephrbd-fast-delete) with PgBouncer pooler - S3/Ceph-RGW for blob storage (stalwart-maildata bucket, lz4 compressed) - Secrets from Vault: postgres-credentials, s3-credentials, stalwart-admin - TLS cert via cert-manager (vault-issuer) for mail.main.unkin.net - SMTP relay on port 25 (internal ClusterIP, trusted pod CIDRs) - Submission on port 587, IMAP 143/993, HTTPS 443 via LoadBalancer - HTTP port 8080 for Traefik reverse proxy (web admin at mail.k8s.syd1.au.unkin.net) - Outbound mail routed through postfix.mailgateway.svc.cluster.local:25 - Spam filtering offloaded to postfix/rspamd (disabled internally) --- apps/base/stalwart/certificate.yaml | 36 +++ apps/base/stalwart/cnpg_cluster.yaml | 91 +++++++ apps/base/stalwart/cnpg_pooler.yaml | 33 +++ apps/base/stalwart/gateway.yaml | 38 +++ apps/base/stalwart/httproute.yaml | 16 ++ apps/base/stalwart/kustomization.yaml | 23 ++ apps/base/stalwart/namespace.yaml | 5 + apps/base/stalwart/resources/config.toml | 223 ++++++++++++++++++ apps/base/stalwart/services.yaml | 54 +++++ apps/base/stalwart/stalwart-deployment.yaml | 107 +++++++++ apps/base/stalwart/stalwart-hpa.yaml | 38 +++ apps/base/stalwart/vaultauth.yaml | 18 ++ apps/base/stalwart/vaultstaticsecret.yaml | 51 ++++ .../au-syd1/stalwart/kustomization.yaml | 6 + argocd/applicationsets/platform.yaml | 1 + 15 files changed, 740 insertions(+) create mode 100644 apps/base/stalwart/certificate.yaml create mode 100644 apps/base/stalwart/cnpg_cluster.yaml create mode 100644 apps/base/stalwart/cnpg_pooler.yaml create mode 100644 apps/base/stalwart/gateway.yaml create mode 100644 apps/base/stalwart/httproute.yaml create mode 100644 apps/base/stalwart/kustomization.yaml create mode 100644 apps/base/stalwart/namespace.yaml create mode 100644 apps/base/stalwart/resources/config.toml create mode 100644 apps/base/stalwart/services.yaml create mode 100644 apps/base/stalwart/stalwart-deployment.yaml create mode 100644 apps/base/stalwart/stalwart-hpa.yaml create mode 100644 apps/base/stalwart/vaultauth.yaml create mode 100644 apps/base/stalwart/vaultstaticsecret.yaml create mode 100644 apps/overlays/au-syd1/stalwart/kustomization.yaml diff --git a/apps/base/stalwart/certificate.yaml b/apps/base/stalwart/certificate.yaml new file mode 100644 index 0000000..dd15ae5 --- /dev/null +++ b/apps/base/stalwart/certificate.yaml @@ -0,0 +1,36 @@ +--- +# TLS cert for stalwart's own HTTPS/IMAPS/STARTTLS listeners (mail.main.unkin.net) +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: stalwart-tls + namespace: stalwart +spec: + secretName: stalwart-tls + issuerRef: + name: vault-issuer + kind: ClusterIssuer + commonName: mail.main.unkin.net + dnsNames: + - mail.main.unkin.net + privateKey: + size: 4096 + algorithm: RSA +--- +# TLS cert for Traefik Gateway (internal k8s admin URL) +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: stalwart-gateway-tls + namespace: stalwart +spec: + secretName: stalwart-gateway-tls + issuerRef: + name: vault-issuer + kind: ClusterIssuer + commonName: mail.k8s.syd1.au.unkin.net + dnsNames: + - mail.k8s.syd1.au.unkin.net + privateKey: + size: 4096 + algorithm: RSA diff --git a/apps/base/stalwart/cnpg_cluster.yaml b/apps/base/stalwart/cnpg_cluster.yaml new file mode 100644 index 0000000..ae566af --- /dev/null +++ b/apps/base/stalwart/cnpg_cluster.yaml @@ -0,0 +1,91 @@ +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: stalwart-postgres + namespace: stalwart +spec: + affinity: + podAntiAffinityType: preferred + bootstrap: + initdb: + database: stalwart + encoding: UTF8 + localeCType: C + localeCollate: C + owner: stalwart + secret: + name: postgres-credentials + enablePDB: true + enableSuperuserAccess: false + failoverDelay: 0 + imageName: ghcr.io/cloudnative-pg/postgresql:17-minimal-trixie + instances: 3 + logLevel: info + maxSyncReplicas: 0 + minSyncReplicas: 0 + monitoring: + customQueriesConfigMap: + - key: queries + name: cnpg-default-monitoring + disableDefaultQueries: false + enablePodMonitor: false + postgresql: + parameters: + archive_mode: "on" + archive_timeout: 5min + dynamic_shared_memory_type: posix + effective_cache_size: 256MB + full_page_writes: "on" + log_destination: csvlog + log_directory: /controller/log + log_filename: postgres + log_rotation_age: "0" + log_rotation_size: "0" + log_truncate_on_rotation: "false" + logging_collector: "on" + max_connections: "200" + max_parallel_workers: "16" + max_replication_slots: "16" + max_worker_processes: "16" + shared_buffers: 128MB + shared_memory_type: mmap + ssl_max_protocol_version: TLSv1.3 + ssl_min_protocol_version: TLSv1.3 + wal_keep_size: 256MB + wal_level: logical + wal_log_hints: "on" + wal_receiver_timeout: 5s + wal_sender_timeout: 5s + syncReplicaElectionConstraint: + enabled: false + primaryUpdateMethod: restart + primaryUpdateStrategy: unsupervised + probes: + liveness: + isolationCheck: + connectionTimeout: 1000 + enabled: true + requestTimeout: 1000 + replicationSlots: + highAvailability: + enabled: true + slotPrefix: _cnpg_ + synchronizeReplicas: + enabled: true + updateInterval: 30 + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + smartShutdownTimeout: 180 + startDelay: 3600 + stopDelay: 1800 + storage: + resizeInUseVolumes: true + size: 20Gi + storageClass: cephrbd-fast-delete + switchoverDelay: 3600 diff --git a/apps/base/stalwart/cnpg_pooler.yaml b/apps/base/stalwart/cnpg_pooler.yaml new file mode 100644 index 0000000..8eb4e3c --- /dev/null +++ b/apps/base/stalwart/cnpg_pooler.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Pooler +metadata: + name: stalwart-postgres-pooler + namespace: stalwart +spec: + cluster: + name: stalwart-postgres + instances: 2 + pgbouncer: + parameters: + default_pool_size: "50" + max_client_conn: "200" + paused: false + poolMode: session + template: + metadata: + labels: + app: pooler + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - pooler + topologyKey: kubernetes.io/hostname + containers: [] + type: rw diff --git a/apps/base/stalwart/gateway.yaml b/apps/base/stalwart/gateway.yaml new file mode 100644 index 0000000..2932c9f --- /dev/null +++ b/apps/base/stalwart/gateway.yaml @@ -0,0 +1,38 @@ +--- +# Traefik gateway for stalwart web admin UI (separate from the LoadBalancer HTTPS port) +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + labels: + traefik.io/instance: internal + annotations: + cert-manager.io/cluster-issuer: vault-issuer + cert-manager.io/common-name: mail.k8s.syd1.au.unkin.net + cert-manager.io/private-key-size: "4096" + external-dns.alpha.kubernetes.io/hostname: mail.k8s.syd1.au.unkin.net + external-dns.alpha.kubernetes.io/target: 198.18.200.4 + name: stalwart + namespace: stalwart +spec: + gatewayClassName: traefik-internal + listeners: + - allowedRoutes: + namespaces: + from: Same + hostname: mail.k8s.syd1.au.unkin.net + name: http + port: 80 + protocol: HTTP + - allowedRoutes: + namespaces: + from: Same + hostname: mail.k8s.syd1.au.unkin.net + name: https + port: 443 + protocol: HTTPS + tls: + certificateRefs: + - group: "" + kind: Secret + name: stalwart-gateway-tls + mode: Terminate diff --git a/apps/base/stalwart/httproute.yaml b/apps/base/stalwart/httproute.yaml new file mode 100644 index 0000000..81bf46e --- /dev/null +++ b/apps/base/stalwart/httproute.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: stalwart + namespace: stalwart +spec: + parentRefs: + - name: stalwart + namespace: stalwart + hostnames: + - mail.k8s.syd1.au.unkin.net + rules: + - backendRefs: + - name: stalwart + port: 8080 diff --git a/apps/base/stalwart/kustomization.yaml b/apps/base/stalwart/kustomization.yaml new file mode 100644 index 0000000..4191dec --- /dev/null +++ b/apps/base/stalwart/kustomization.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - certificate.yaml + - cnpg_cluster.yaml + - cnpg_pooler.yaml + - gateway.yaml + - httproute.yaml + - namespace.yaml + - services.yaml + - stalwart-deployment.yaml + - stalwart-hpa.yaml + - vaultauth.yaml + - vaultstaticsecret.yaml + +configMapGenerator: + - name: stalwart-config + files: + - config.toml=resources/config.toml + options: + disableNameSuffixHash: true diff --git a/apps/base/stalwart/namespace.yaml b/apps/base/stalwart/namespace.yaml new file mode 100644 index 0000000..5a22388 --- /dev/null +++ b/apps/base/stalwart/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: stalwart diff --git a/apps/base/stalwart/resources/config.toml b/apps/base/stalwart/resources/config.toml new file mode 100644 index 0000000..efff98e --- /dev/null +++ b/apps/base/stalwart/resources/config.toml @@ -0,0 +1,223 @@ +[server] +hostname = "mail.main.unkin.net" +greeting = "Stalwart ESMTP" + +# SMTP relay listener (receives from postfix for local delivery) +[server.listener."smtp-relay"] +bind = ["0.0.0.0:25"] +protocol = "smtp" +greeting = "Stalwart SMTP Relay" + +[server.listener."smtp-relay".proxy] +trusted-networks = ["127.0.0.0/8", "::1", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"] + +# SMTP submission listener (user-facing, TLS required) +[server.listener."submission"] +bind = ["0.0.0.0:587"] +protocol = "smtp" +greeting = "Stalwart SMTP Submission" +tls.require = true + +[server.listener."submission".proxy] +trusted-networks = ["127.0.0.0/8", "::1"] + +# IMAP listener +[server.listener."imap"] +bind = ["0.0.0.0:143"] +protocol = "imap" + +[server.listener."imap".proxy] +trusted-networks = ["127.0.0.0/8", "::1"] + +# IMAPS listener (implicit TLS) +[server.listener."imaps"] +bind = ["0.0.0.0:993"] +protocol = "imap" +tls.implicit = true + +[server.listener."imaps".proxy] +trusted-networks = ["127.0.0.0/8", "::1"] + +# HTTPS (web admin + JMAP) for external LoadBalancer +[server.listener."https"] +bind = ["0.0.0.0:443"] +protocol = "http" +tls.implicit = true + +[server.listener."https".proxy] +trusted-networks = ["127.0.0.0/8", "::1", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"] + +# Plain HTTP for Traefik reverse proxy (TLS terminated at Traefik) +[server.listener."http-internal"] +bind = ["0.0.0.0:8080"] +protocol = "http" + +[server.listener."http-internal".proxy] +trusted-networks = ["127.0.0.0/8", "::1", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"] + +[server.tls] +enable = true +implicit = false +certificate = "default" + +[server.http] +use-x-forwarded = true +permissive-cors = false + +[webadmin] +path = "/var/lib/stalwart/webadmin" +auto-update = true +resource = "https://github.com/stalwartlabs/webadmin/releases/latest/download/webadmin.zip" + +# PostgreSQL store (via CNPG pooler) +[store."postgresql"] +type = "postgresql" +host = "stalwart-postgres-pooler.stalwart.svc.cluster.local" +port = 5432 +database = "stalwart" +user = "stalwart" +password = "%{env:POSTGRES_PASSWORD}%" +timeout = "15s" + +[store."postgresql".tls] +enable = true +allow-invalid-certs = false + +[store."postgresql".pool] +max-connections = 10 + +[store."postgresql".purge] +frequency = "0 3 *" + +# S3/Ceph-RGW store for blobs +[store."s3"] +type = "s3" +bucket = "stalwart-maildata" +region = "syd1" +access-key = "%{env:S3_ACCESS_KEY}%" +secret-key = "%{env:S3_SECRET_KEY}%" +endpoint = "https://radosgw.service.consul" +timeout = "30s" +key-prefix = "stalwart/" +compression = "lz4" + +[store."s3".purge] +frequency = "30 5 *" + +# Storage assignment +[storage] +data = "postgresql" +fts = "postgresql" +blob = "s3" +lookup = "postgresql" +directory = "internal" +in-memory = "postgresql" + +# Directory configuration +[directory.internal] +type = "internal" +store = "postgresql" + +# Fallback admin (password hash from Vault) +[authentication.fallback-admin] +user = "admin" +secret = "%{env:ADMIN_PASSWORD_HASH}%" + +[authentication] +[authentication.directory] +directories = ["internal"] + +[authorization] +directory = "internal" + +# JMAP configuration +[jmap] +directory = "internal" + +[jmap.protocol] +request-max-size = 10485760 +get.max-objects = 500 +query.max-results = 5000 +changes.max-results = 5000 +upload.max-size = 50000000 +upload.ttl = "1h" + +# IMAP configuration +[imap] +directory = "internal" + +[imap.protocol] +max-requests = 64 + +# Inbound rate limiting +[[queue.limiter.inbound]] +key = ["remote_ip"] +rate = "500/1s" +enable = true + +# Outbound routing through postfix relay +[queue] +path = "/var/lib/stalwart/queue" + +[queue.schedule] +retry = ["2s", "5s", "1m", "5m", "15m", "30m", "1h", "2h"] +notify = ["1d", "3d"] +expire = "5d" + +[session.extensions] +future-release = "7d" + +# Route local domain mail locally, everything else through postfix relay +[queue.strategy] +route = [ + { if = "is_local_domain('', rcpt_domain)", then = "'local'" }, + { else = "'relay'" } +] + +[queue.route."local"] +type = "local" + +[queue.route."relay"] +type = "relay" +address = "postfix.mailgateway.svc.cluster.local" +port = 25 +protocol = "smtp" + +[queue.route."relay".tls] +implicit = false +allow-invalid-certs = false + +# Session configuration +[session.ehlo] +reject-non-fqdn = false + +[session.rcpt] +type = "internal" +store = "postgresql" +max-recipients = 25 + +[session.data] +max-messages = 10 +max-message-size = 52428800 + +# Spam filtering offloaded to postfix/rspamd — disable internal spam filter +[spam-filter] +enable = false + +# TLS certificate (cert-manager issues to /etc/stalwart/tls/) +[certificate."default"] +cert = "%{file:/etc/stalwart/tls/tls.crt}%" +private-key = "%{file:/etc/stalwart/tls/tls.key}%" +default = true + +# Logging +[tracer] +type = "log" +level = "info" +ansi = false +multiline = true + +# Metrics +[metrics] +prometheus.enable = true +prometheus.port = 9090 diff --git a/apps/base/stalwart/services.yaml b/apps/base/stalwart/services.yaml new file mode 100644 index 0000000..a1e90f7 --- /dev/null +++ b/apps/base/stalwart/services.yaml @@ -0,0 +1,54 @@ +--- +# Internal service - postfix connects here to deliver inbound mail +apiVersion: v1 +kind: Service +metadata: + name: stalwart + namespace: stalwart +spec: + selector: + app: stalwart + ports: + - name: smtp-relay + port: 25 + targetPort: 25 + protocol: TCP + - name: http-internal + port: 8080 + targetPort: 8080 + protocol: TCP + - name: metrics + port: 9090 + targetPort: 9090 + protocol: TCP +--- +# External LoadBalancer for user-facing mail services +apiVersion: v1 +kind: Service +metadata: + name: stalwart-external + namespace: stalwart + annotations: + external-dns.alpha.kubernetes.io/hostname: mail.main.unkin.net +spec: + type: LoadBalancer + externalTrafficPolicy: Local + selector: + app: stalwart + ports: + - name: submission + port: 587 + targetPort: 587 + protocol: TCP + - name: imap + port: 143 + targetPort: 143 + protocol: TCP + - name: imaps + port: 993 + targetPort: 993 + protocol: TCP + - name: https + port: 443 + targetPort: 443 + protocol: TCP diff --git a/apps/base/stalwart/stalwart-deployment.yaml b/apps/base/stalwart/stalwart-deployment.yaml new file mode 100644 index 0000000..2b1e1e1 --- /dev/null +++ b/apps/base/stalwart/stalwart-deployment.yaml @@ -0,0 +1,107 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: stalwart + namespace: stalwart +spec: + selector: + matchLabels: + app: stalwart + template: + metadata: + annotations: + reloader.stakater.com/auto: "true" + labels: + app: stalwart + spec: + securityContext: + runAsUser: 2000 + runAsGroup: 2000 + fsGroup: 2000 + containers: + - name: stalwart + image: ghcr.io/stalwartlabs/stalwart:v0.16.6 + ports: + - containerPort: 25 + name: smtp-relay + protocol: TCP + - containerPort: 587 + name: submission + protocol: TCP + - containerPort: 143 + name: imap + protocol: TCP + - containerPort: 993 + name: imaps + protocol: TCP + - containerPort: 443 + name: https + protocol: TCP + - containerPort: 8080 + name: http-internal + protocol: TCP + - containerPort: 9090 + name: metrics + protocol: TCP + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: password + - name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: s3-credentials + key: access_key + - name: S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: s3-credentials + key: secret_key + - name: ADMIN_PASSWORD_HASH + valueFrom: + secretKeyRef: + name: stalwart-admin + key: password_hash + livenessProbe: + tcpSocket: + port: 25 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 25 + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + volumeMounts: + - name: config + mountPath: /etc/stalwart/config.toml + subPath: config.toml + readOnly: true + - name: tls + mountPath: /etc/stalwart/tls + readOnly: true + - name: data + mountPath: /var/lib/stalwart + volumes: + - name: config + configMap: + name: stalwart-config + - name: tls + secret: + secretName: stalwart-tls + - name: data + emptyDir: {} diff --git a/apps/base/stalwart/stalwart-hpa.yaml b/apps/base/stalwart/stalwart-hpa.yaml new file mode 100644 index 0000000..1f068be --- /dev/null +++ b/apps/base/stalwart/stalwart-hpa.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: stalwart-hpa + namespace: stalwart +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: stalwart + minReplicas: 2 + maxReplicas: 6 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + selectPolicy: Max + policies: + - type: Percent + value: 100 + periodSeconds: 60 + - type: Pods + value: 2 + periodSeconds: 30 + scaleDown: + stabilizationWindowSeconds: 300 + selectPolicy: Min + policies: + - type: Percent + value: 30 + periodSeconds: 60 diff --git a/apps/base/stalwart/vaultauth.yaml b/apps/base/stalwart/vaultauth.yaml new file mode 100644 index 0000000..613e7eb --- /dev/null +++ b/apps/base/stalwart/vaultauth.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: default + namespace: stalwart +spec: + allowedNamespaces: + - stalwart + kubernetes: + audiences: + - vault + role: default + serviceAccount: default + tokenExpirationSeconds: 600 + method: kubernetes + mount: k8s/au/syd1 + vaultConnectionRef: vso-system/default diff --git a/apps/base/stalwart/vaultstaticsecret.yaml b/apps/base/stalwart/vaultstaticsecret.yaml new file mode 100644 index 0000000..88b3f6e --- /dev/null +++ b/apps/base/stalwart/vaultstaticsecret.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: postgres-credentials + namespace: stalwart +spec: + destination: + create: true + name: postgres-credentials + overwrite: true + hmacSecretData: true + mount: kv + path: kubernetes/namespace/stalwart/default/postgres-credentials + refreshAfter: 5m + type: kv-v2 + vaultAuthRef: default +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: s3-credentials + namespace: stalwart +spec: + destination: + create: true + name: s3-credentials + overwrite: true + hmacSecretData: true + mount: kv + path: kubernetes/namespace/stalwart/default/s3-credentials + refreshAfter: 5m + type: kv-v2 + vaultAuthRef: default +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: stalwart-admin + namespace: stalwart +spec: + destination: + create: true + name: stalwart-admin + overwrite: true + hmacSecretData: true + mount: kv + path: kubernetes/namespace/stalwart/default/stalwart-admin + refreshAfter: 5m + type: kv-v2 + vaultAuthRef: default diff --git a/apps/overlays/au-syd1/stalwart/kustomization.yaml b/apps/overlays/au-syd1/stalwart/kustomization.yaml new file mode 100644 index 0000000..964d173 --- /dev/null +++ b/apps/overlays/au-syd1/stalwart/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../base/stalwart diff --git a/argocd/applicationsets/platform.yaml b/argocd/applicationsets/platform.yaml index ac5afac..e52954a 100644 --- a/argocd/applicationsets/platform.yaml +++ b/argocd/applicationsets/platform.yaml @@ -26,6 +26,7 @@ spec: - path: apps/overlays/*/reflector-system - path: apps/overlays/*/reloader-system - path: apps/overlays/*/reposync + - path: apps/overlays/*/stalwart - path: apps/overlays/*/traefik-system - path: apps/overlays/*/vm-system - path: apps/overlays/*/vault