From 584524c80ddaf25a37cc8b4ba7f84d89192c3acf Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 26 Sep 2026 20:59:01 +1000 Subject: [PATCH 1/2] drop stalwart mail proxying --- apps/base/haproxy/configmap.yaml | 122 ------------------------------ apps/base/haproxy/deployment.yaml | 14 +--- apps/base/haproxy/service.yaml | 18 +---- 3 files changed, 2 insertions(+), 152 deletions(-) diff --git a/apps/base/haproxy/configmap.yaml b/apps/base/haproxy/configmap.yaml index e1ad1eb..9f3e1c4 100644 --- a/apps/base/haproxy/configmap.yaml +++ b/apps/base/haproxy/configmap.yaml @@ -23,9 +23,6 @@ data: git.unkin.net be_gitea grafana.unkin.net be_grafana dashboard.ceph.unkin.net be_ceph_dashboard - mail-webadmin.main.unkin.net be_stalwart_webadmin - autoconfig.main.unkin.net be_stalwart_webadmin - autodiscovery.main.unkin.net be_stalwart_webadmin auth.unkin.net be_k8s_kanidm fe_https.map: | @@ -40,9 +37,6 @@ data: git.unkin.net be_gitea grafana.unkin.net be_grafana dashboard.ceph.unkin.net be_ceph_dashboard - mail-webadmin.main.unkin.net be_stalwart_webadmin - autoconfig.main.unkin.net be_stalwart_webadmin - autodiscovery.main.unkin.net be_stalwart_webadmin auth.unkin.net be_k8s_kanidm haproxy.cfg: | @@ -101,26 +95,6 @@ data: use_backend be_letsencrypt if acl-letsencrypt use_backend %[req.hdr(host),lower,map(/usr/local/etc/haproxy/fe_https.map,be_default)] - frontend fe_imap - bind 0.0.0.0:143 - mode tcp - description Frontend for Stalwart IMAP (STARTTLS) - default_backend be_stalwart_imap - log global - option tcplog - tcp-request inspect-delay 5s - tcp-request content accept if { req_len 0 } - - frontend fe_imaps - bind 0.0.0.0:993 - mode tcp - description Frontend for Stalwart IMAPS (implicit TLS) - default_backend be_stalwart_imaps - log global - option tcplog - tcp-request inspect-delay 5s - tcp-request content accept if { req_len 0 } - frontend fe_metrics bind 0.0.0.0:8405 mode http @@ -129,26 +103,6 @@ data: http-request set-header X-Real-IP %[src] http-request use-service prometheus-exporter if { path /metrics } - frontend fe_smtp - bind 0.0.0.0:25 - mode tcp - description Frontend for Stalwart SMTP - default_backend be_stalwart_smtp - log global - option tcplog - tcp-request inspect-delay 5s - tcp-request content accept if { req_len 0 } - - frontend fe_submission - bind 0.0.0.0:587 - mode tcp - description Frontend for Stalwart SMTP Submission - default_backend be_stalwart_submission - log global - option tcplog - tcp-request inspect-delay 5s - tcp-request content accept if { req_len 0 } - backend be_ceph_dashboard description Backend for Ceph Dashboard from Mgr instances balance roundrobin @@ -332,82 +286,6 @@ data: redirect scheme https if !{ ssl_fc } server ausyd1nxvm2046 198.18.26.161:443 check cookie ausyd1nxvm2046 fall 2 inter 2s rise 3 ssl verify none - backend be_stalwart_imap - description Backend for Stalwart IMAP (STARTTLS) - balance roundrobin - mode tcp - option tcp-check - option prefer-last-server - stick on src - stick-table type ip size 200k expire 30m - tcp-check connect port 143 send-proxy - tcp-check expect string "* OK" - tcp-check send "A001 STARTTLS\r\n" - tcp-check expect rstring "A001 (OK|2.0.0)" - server ausyd1nxvm2124 198.18.28.76:143 check fall 3 inter 3s rise 2 send-proxy-v2 - server ausyd1nxvm2125 198.18.29.44:143 check fall 3 inter 3s rise 2 send-proxy-v2 - server ausyd1nxvm2126 198.18.25.160:143 check fall 3 inter 3s rise 2 send-proxy-v2 - - backend be_stalwart_imaps - description Backend for Stalwart IMAPS (implicit TLS) - balance roundrobin - mode tcp - option tcp-check - option prefer-last-server - stick on src - stick-table type ip size 200k expire 30m - tcp-check connect ssl send-proxy - tcp-check expect string "* OK" - server ausyd1nxvm2124 198.18.28.76:993 check fall 3 inter 3s rise 2 send-proxy-v2 ssl verify none - server ausyd1nxvm2125 198.18.29.44:993 check fall 3 inter 3s rise 2 send-proxy-v2 ssl verify none - server ausyd1nxvm2126 198.18.25.160:993 check fall 3 inter 3s rise 2 send-proxy-v2 ssl verify none - - backend be_stalwart_smtp - description Backend for Stalwart SMTP - balance roundrobin - mode tcp - option tcp-check - option prefer-last-server - stick on src - stick-table type ip size 200k expire 30m - tcp-check connect port 25 send-proxy - tcp-check expect string "220 " - server ausyd1nxvm2124 198.18.28.76:25 check fall 3 inter 3s rise 2 send-proxy-v2 - server ausyd1nxvm2125 198.18.29.44:25 check fall 3 inter 3s rise 2 send-proxy-v2 - server ausyd1nxvm2126 198.18.25.160:25 check fall 3 inter 3s rise 2 send-proxy-v2 - - backend be_stalwart_submission - description Backend for Stalwart SMTP Submission - balance roundrobin - mode tcp - option tcp-check - option prefer-last-server - stick on src - stick-table type ip size 200k expire 30m - tcp-check connect port 587 send-proxy - tcp-check expect string "220 " - server ausyd1nxvm2124 198.18.28.76:587 check fall 3 inter 3s rise 2 send-proxy-v2 - server ausyd1nxvm2125 198.18.29.44:587 check fall 3 inter 3s rise 2 send-proxy-v2 - server ausyd1nxvm2126 198.18.25.160:587 check fall 3 inter 3s rise 2 send-proxy-v2 - - backend be_stalwart_webadmin - description Backend for Stalwart Webadmin - balance roundrobin - cookie SRVNAME insert indirect nocache - http-check expect status 200 - http-request set-header X-Forwarded-Port %[dst_port] - http-request add-header X-Forwarded-Proto https if { dst_port 9443 } - http-reuse always - option httpchk GET / - option forwardfor - option http-keep-alive - option prefer-last-server - redirect scheme https if !{ ssl_fc } - stick-table type ip size 200k expire 30m - server ausyd1nxvm2124 198.18.28.76:443 check cookie ausyd1nxvm2124 fall 2 inter 2s rise 3 send-proxy-v2 ssl verify none - server ausyd1nxvm2125 198.18.29.44:443 check cookie ausyd1nxvm2125 fall 2 inter 2s rise 3 send-proxy-v2 ssl verify none - server ausyd1nxvm2126 198.18.25.160:443 check cookie ausyd1nxvm2126 fall 2 inter 2s rise 3 send-proxy-v2 ssl verify none - # The `peers au-syd1-prod` section is dropped: peer names must be static and a # Deployment cannot provide them. Service sessionAffinity: ClientIP pins a # client to one replica so the per-replica stick-tables behave as before. diff --git a/apps/base/haproxy/deployment.yaml b/apps/base/haproxy/deployment.yaml index 4a1f11e..a44724a 100644 --- a/apps/base/haproxy/deployment.yaml +++ b/apps/base/haproxy/deployment.yaml @@ -50,7 +50,7 @@ spec: readOnlyRootFilesystem: true capabilities: drop: [ALL] - # Frontends bind 25/80/143/443/587; the dst_port ACLs need the real ports. + # Frontends bind 80 and 443; the dst_port ACLs need the real ports. add: [NET_BIND_SERVICE] ports: - name: http @@ -59,18 +59,6 @@ spec: - name: https containerPort: 443 protocol: TCP - - name: smtp - containerPort: 25 - protocol: TCP - - name: imap - containerPort: 143 - protocol: TCP - - name: submission - containerPort: 587 - protocol: TCP - - name: imaps - containerPort: 993 - protocol: TCP - name: health containerPort: 8404 protocol: TCP diff --git a/apps/base/haproxy/service.yaml b/apps/base/haproxy/service.yaml index 388401c..a1f7d54 100644 --- a/apps/base/haproxy/service.yaml +++ b/apps/base/haproxy/service.yaml @@ -10,7 +10,7 @@ metadata: spec: type: LoadBalancer loadBalancerIP: "198.18.199.1" - # Source IP must survive for acl_internalsubnets, X-Real-IP and SMTP. + # Source IP must survive for X-Real-IP. externalTrafficPolicy: Local # Pins a client to one replica, standing in for the dropped stick-table peers. sessionAffinity: ClientIP @@ -25,19 +25,3 @@ spec: port: 443 protocol: TCP targetPort: https - - name: smtp - port: 25 - protocol: TCP - targetPort: smtp - - name: imap - port: 143 - protocol: TCP - targetPort: imap - - name: submission - port: 587 - protocol: TCP - targetPort: submission - - name: imaps - port: 993 - protocol: TCP - targetPort: imaps -- 2.47.3 From 475477f9b5ae0a7774d8bd4edb32fd174dd34484 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 26 Sep 2026 21:12:42 +1000 Subject: [PATCH 2/2] Move the haproxy edge behind the external Traefik --- apps/base/haproxy/configmap.yaml | 38 ++++------------------------ apps/base/haproxy/deployment.yaml | 6 ++--- apps/base/haproxy/gateway.yaml | 31 +++++++++++++++++++++++ apps/base/haproxy/httproute.yaml | 37 +++++++++++++++++++++++++++ apps/base/haproxy/kustomization.yaml | 3 +++ apps/base/haproxy/service.yaml | 18 ++++--------- apps/base/haproxy/tlsroute.yaml | 34 +++++++++++++++++++++++++ 7 files changed, 117 insertions(+), 50 deletions(-) create mode 100644 apps/base/haproxy/gateway.yaml create mode 100644 apps/base/haproxy/httproute.yaml create mode 100644 apps/base/haproxy/tlsroute.yaml diff --git a/apps/base/haproxy/configmap.yaml b/apps/base/haproxy/configmap.yaml index 9f3e1c4..9a79587 100644 --- a/apps/base/haproxy/configmap.yaml +++ b/apps/base/haproxy/configmap.yaml @@ -11,20 +11,6 @@ data: /etc/haproxy/certs/main-unkin-net/tls.crt /etc/haproxy/certs/ceph-unkin-net/tls.crt - fe_http.map: | - sonarr.main.unkin.net be_sonarr - radarr.main.unkin.net be_radarr - lidarr.main.unkin.net be_lidarr - readarr.main.unkin.net be_readarr - prowlarr.main.unkin.net be_prowlarr - nzbget.main.unkin.net be_nzbget - jellyfin.main.unkin.net be_jellyfin - fafflix.unkin.net be_jellyfin - git.unkin.net be_gitea - grafana.unkin.net be_grafana - dashboard.ceph.unkin.net be_ceph_dashboard - auth.unkin.net be_k8s_kanidm - fe_https.map: | sonarr.main.unkin.net be_sonarr radarr.main.unkin.net be_radarr @@ -73,26 +59,14 @@ data: timeout http-keep-alive 10s timeout check 10s - frontend fe_http - bind 0.0.0.0:80 - mode http - description Global HTTP Frontend - acl acl-letsencrypt path_beg /.well-known/acme-challenge/ - http-request set-header X-Forwarded-Proto https - http-request set-header X-Real-IP %[src] - use_backend be_letsencrypt if acl-letsencrypt - use_backend %[req.hdr(host),lower,map(/usr/local/etc/haproxy/fe_http.map,be_default)] - frontend fe_https bind 0.0.0.0:443 ssl crt-list /usr/local/etc/haproxy/certificate.list ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH force-tlsv12 mode http description Global HTTPS Frontend - acl acl-letsencrypt path_beg /.well-known/acme-challenge/ http-request set-header X-Forwarded-Proto https http-request set-header X-Real-IP %[src] http-response set-header X-Content-Type-Options nosniff http-response set-header X-XSS-Protection 1;mode=block - use_backend be_letsencrypt if acl-letsencrypt use_backend %[req.hdr(host),lower,map(/usr/local/etc/haproxy/fe_https.map,be_default)] frontend fe_metrics @@ -197,11 +171,6 @@ data: http-check expect status 200 server k8s-traefik-internal 198.18.200.4:443 ssl verify none check inter 2s rise 3 fall 2 sni str(auth.unkin.net) - backend be_letsencrypt - description Backend for LetsEncrypt Verifications - balance roundrobin - server ausyd1nxvm2057 198.18.25.3:8888 - backend be_lidarr description Backend for au-syd1 lidarr balance roundrobin @@ -287,8 +256,11 @@ data: server ausyd1nxvm2046 198.18.26.161:443 check cookie ausyd1nxvm2046 fall 2 inter 2s rise 3 ssl verify none # The `peers au-syd1-prod` section is dropped: peer names must be static and a - # Deployment cannot provide them. Service sessionAffinity: ClientIP pins a - # client to one replica so the per-replica stick-tables behave as before. + # Deployment cannot provide them. Behind the external Traefik's TLS + # passthrough `src` is a Traefik pod, so X-Real-IP, forwardfor and the + # `stick on src` tables all key on that; the SRVNAME cookie carries real + # session persistence. Traefik cannot emit PROXY protocol to a TLSRoute + # backend, so there is nothing to bind `accept-proxy` to. listen health bind 0.0.0.0:8404 diff --git a/apps/base/haproxy/deployment.yaml b/apps/base/haproxy/deployment.yaml index a44724a..f8a7c6c 100644 --- a/apps/base/haproxy/deployment.yaml +++ b/apps/base/haproxy/deployment.yaml @@ -50,12 +50,10 @@ spec: readOnlyRootFilesystem: true capabilities: drop: [ALL] - # Frontends bind 80 and 443; the dst_port ACLs need the real ports. + # fe_https binds the privileged port 443 as uid 99, and the + # dst_port ACLs need the real port. add: [NET_BIND_SERVICE] ports: - - name: http - containerPort: 80 - protocol: TCP - name: https containerPort: 443 protocol: TCP diff --git a/apps/base/haproxy/gateway.yaml b/apps/base/haproxy/gateway.yaml new file mode 100644 index 0000000..19591d4 --- /dev/null +++ b/apps/base/haproxy/gateway.yaml @@ -0,0 +1,31 @@ +--- +# External (DMZ) front for the haproxy edge on the traefik-external LB VIP +# 198.18.199.0. The :443 listener is TLS Passthrough: haproxy owns the three +# wildcard certs and terminates behind Traefik, so there are no certificateRefs +# here. Listener hostnames are deliberately unset and the routes carry the +# explicit hostname list instead; allowedRoutes Same keeps other namespaces off +# these listeners. +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: haproxy + namespace: haproxy + labels: + traefik.io/instance: external +spec: + gatewayClassName: traefik-external + listeners: + - name: http + port: 80 + protocol: HTTP + allowedRoutes: + namespaces: + from: Same + - name: https-passthrough + port: 443 + protocol: TLS + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: Same diff --git a/apps/base/haproxy/httproute.yaml b/apps/base/haproxy/httproute.yaml new file mode 100644 index 0000000..6778452 --- /dev/null +++ b/apps/base/haproxy/httproute.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: haproxy-http-redirect + namespace: haproxy + labels: + app: haproxy +spec: + hostnames: + - sonarr.main.unkin.net + - radarr.main.unkin.net + - lidarr.main.unkin.net + - readarr.main.unkin.net + - prowlarr.main.unkin.net + - nzbget.main.unkin.net + - jellyfin.main.unkin.net + - fafflix.unkin.net + - git.unkin.net + - grafana.unkin.net + - dashboard.ceph.unkin.net + - auth.unkin.net + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: haproxy + sectionName: http + rules: + - filters: + - type: RequestRedirect + requestRedirect: + scheme: https + statusCode: 301 + matches: + - path: + type: PathPrefix + value: / diff --git a/apps/base/haproxy/kustomization.yaml b/apps/base/haproxy/kustomization.yaml index 680c918..16f50f5 100644 --- a/apps/base/haproxy/kustomization.yaml +++ b/apps/base/haproxy/kustomization.yaml @@ -7,6 +7,9 @@ resources: - configmap.yaml - deployment.yaml - service.yaml + - gateway.yaml + - tlsroute.yaml + - httproute.yaml - pdb.yaml - vpa.yaml - vmpodscrape.yaml diff --git a/apps/base/haproxy/service.yaml b/apps/base/haproxy/service.yaml index a1f7d54..d98b2d7 100644 --- a/apps/base/haproxy/service.yaml +++ b/apps/base/haproxy/service.yaml @@ -4,23 +4,15 @@ kind: Service metadata: name: haproxy namespace: haproxy - annotations: - purelb.io/service-group: dmz - purelb.io/addresses: 198.18.199.1 spec: - type: LoadBalancer - loadBalancerIP: "198.18.199.1" - # Source IP must survive for X-Real-IP. - externalTrafficPolicy: Local - # Pins a client to one replica, standing in for the dropped stick-table peers. - sessionAffinity: ClientIP + type: ClusterIP + # Reached only by the external Traefik's TLS-passthrough TLSRoute, so the + # peer address here is a Traefik pod, not the client. sessionAffinity is + # deliberately absent: keyed on ClientIP it would pin whole Traefik pods, + # not clients. Backend persistence rests on the per-backend SRVNAME cookie. selector: app: haproxy ports: - - name: http - port: 80 - protocol: TCP - targetPort: http - name: https port: 443 protocol: TCP diff --git a/apps/base/haproxy/tlsroute.yaml b/apps/base/haproxy/tlsroute.yaml new file mode 100644 index 0000000..530c9d3 --- /dev/null +++ b/apps/base/haproxy/tlsroute.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: TLSRoute +metadata: + name: haproxy + namespace: haproxy + labels: + app: haproxy +spec: + hostnames: + - sonarr.main.unkin.net + - radarr.main.unkin.net + - lidarr.main.unkin.net + - readarr.main.unkin.net + - prowlarr.main.unkin.net + - nzbget.main.unkin.net + - jellyfin.main.unkin.net + - fafflix.unkin.net + - git.unkin.net + - grafana.unkin.net + - dashboard.ceph.unkin.net + - auth.unkin.net + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: haproxy + sectionName: https-passthrough + rules: + - backendRefs: + - group: "" + kind: Service + name: haproxy + port: 443 + weight: 1 -- 2.47.3