Move the haproxy edge behind the external Traefik (#492)

The haproxy edge holds its own DMZ VIP, a second public entry point alongside
traefik-external that must be firewalled and DNS'd separately. Traefik can
front it with TLS passthrough, leaving haproxy's certs and backends untouched.

- Add a `traefik-external` Gateway: HTTP :80 plus Passthrough TLS :443.
- TLSRoute the 12 `fe_https.map` hostnames to haproxy:443; HTTPRoute 301s :80.
- Make the Service ClusterIP on 443 only, releasing 198.18.199.1.
- Drop `fe_http`, `be_letsencrypt` and `fe_http.map`; certs are DNS-01 only.

Client IP now reads as a Traefik pod — the Gateway provider cannot emit PROXY protocol to a TLSRoute backend. `sessionAffinity` goes too (it would pin Traefik pods, not clients); SRVNAME cookies keep persistence.

Reviewed-on: #492
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

apps/base/haproxy/configmap.yaml

@@ -11,20 +11,6 @@ data:
     /etc/haproxy/certs/main-unkin-net/tls.crt
     /etc/haproxy/certs/ceph-unkin-net/tls.crt
 
-  fe_http.map: |
-    sonarr.main.unkin.net be_sonarr
-    radarr.main.unkin.net be_radarr
-    lidarr.main.unkin.net be_lidarr
-    readarr.main.unkin.net be_readarr
-    prowlarr.main.unkin.net be_prowlarr
-    nzbget.main.unkin.net be_nzbget
-    jellyfin.main.unkin.net be_jellyfin
-    fafflix.unkin.net be_jellyfin
-    git.unkin.net be_gitea
-    grafana.unkin.net be_grafana
-    dashboard.ceph.unkin.net be_ceph_dashboard
-    auth.unkin.net be_k8s_kanidm
-
   fe_https.map: |
     sonarr.main.unkin.net be_sonarr
     radarr.main.unkin.net be_radarr
@@ -73,26 +59,14 @@ data:
       timeout  http-keep-alive 10s
       timeout  check 10s
 
-    frontend fe_http
-      bind 0.0.0.0:80
-      mode http
-      description Global HTTP Frontend
-      acl acl-letsencrypt path_beg /.well-known/acme-challenge/
-      http-request set-header X-Forwarded-Proto https
-      http-request set-header X-Real-IP %[src]
-      use_backend be_letsencrypt if acl-letsencrypt
-      use_backend %[req.hdr(host),lower,map(/usr/local/etc/haproxy/fe_http.map,be_default)]
-
     frontend fe_https
       bind 0.0.0.0:443 ssl crt-list /usr/local/etc/haproxy/certificate.list ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH force-tlsv12
       mode http
       description Global HTTPS Frontend
-      acl acl-letsencrypt path_beg /.well-known/acme-challenge/
       http-request set-header X-Forwarded-Proto https
       http-request set-header X-Real-IP %[src]
       http-response set-header X-Content-Type-Options nosniff
       http-response set-header X-XSS-Protection 1;mode=block
-      use_backend be_letsencrypt if acl-letsencrypt
       use_backend %[req.hdr(host),lower,map(/usr/local/etc/haproxy/fe_https.map,be_default)]
 
     frontend fe_metrics
@@ -197,11 +171,6 @@ data:
       http-check expect status 200
       server k8s-traefik-internal 198.18.200.4:443 ssl verify none check inter 2s rise 3 fall 2 sni str(auth.unkin.net)
 
-    backend be_letsencrypt
-      description Backend for LetsEncrypt Verifications
-      balance roundrobin
-      server ausyd1nxvm2057 198.18.25.3:8888
-
     backend be_lidarr
       description Backend for au-syd1 lidarr
       balance roundrobin
@@ -287,8 +256,11 @@ data:
       server ausyd1nxvm2046 198.18.26.161:443 check cookie ausyd1nxvm2046 fall 2 inter 2s rise 3 ssl verify none
 
     # The `peers au-syd1-prod` section is dropped: peer names must be static and a
-    # Deployment cannot provide them. Service sessionAffinity: ClientIP pins a
+    # Deployment cannot provide them. Behind the external Traefik's TLS
-    # client to one replica so the per-replica stick-tables behave as before.
+    # passthrough `src` is a Traefik pod, so X-Real-IP, forwardfor and the
+    # `stick on src` tables all key on that; the SRVNAME cookie carries real
+    # session persistence. Traefik cannot emit PROXY protocol to a TLSRoute
+    # backend, so there is nothing to bind `accept-proxy` to.
 
     listen health
       bind 0.0.0.0:8404

apps/base/haproxy/deployment.yaml

@@ -50,12 +50,10 @@ spec:
             readOnlyRootFilesystem: true
             capabilities:
               drop: [ALL]
-              # Frontends bind 80 and 443; the dst_port ACLs need the real ports.
+              # fe_https binds the privileged port 443 as uid 99, and the
+              # dst_port ACLs need the real port.
               add: [NET_BIND_SERVICE]
           ports:
-            - name: http
-              containerPort: 80
-              protocol: TCP
             - name: https
               containerPort: 443
               protocol: TCP

apps/base/haproxy/gateway.yaml

@@ -0,0 +1,31 @@
+---
+# External (DMZ) front for the haproxy edge on the traefik-external LB VIP
+# 198.18.199.0. The :443 listener is TLS Passthrough: haproxy owns the three
+# wildcard certs and terminates behind Traefik, so there are no certificateRefs
+# here. Listener hostnames are deliberately unset and the routes carry the
+# explicit hostname list instead; allowedRoutes Same keeps other namespaces off
+# these listeners.
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: haproxy
+  namespace: haproxy
+  labels:
+    traefik.io/instance: external
+spec:
+  gatewayClassName: traefik-external
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https-passthrough
+      port: 443
+      protocol: TLS
+      tls:
+        mode: Passthrough
+      allowedRoutes:
+        namespaces:
+          from: Same

apps/base/haproxy/httproute.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: haproxy-http-redirect
+  namespace: haproxy
+  labels:
+    app: haproxy
+spec:
+  hostnames:
+    - sonarr.main.unkin.net
+    - radarr.main.unkin.net
+    - lidarr.main.unkin.net
+    - readarr.main.unkin.net
+    - prowlarr.main.unkin.net
+    - nzbget.main.unkin.net
+    - jellyfin.main.unkin.net
+    - fafflix.unkin.net
+    - git.unkin.net
+    - grafana.unkin.net
+    - dashboard.ceph.unkin.net
+    - auth.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: haproxy
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/haproxy/kustomization.yaml

@@ -7,6 +7,9 @@ resources:
   - configmap.yaml
   - deployment.yaml
   - service.yaml
+  - gateway.yaml
+  - tlsroute.yaml
+  - httproute.yaml
   - pdb.yaml
   - vpa.yaml
   - vmpodscrape.yaml

apps/base/haproxy/service.yaml

@@ -4,23 +4,15 @@ kind: Service
 metadata:
   name: haproxy
   namespace: haproxy
-  annotations:
-    purelb.io/service-group: dmz
-    purelb.io/addresses: 198.18.199.1
 spec:
-  type: LoadBalancer
+  type: ClusterIP
-  loadBalancerIP: "198.18.199.1"
+  # Reached only by the external Traefik's TLS-passthrough TLSRoute, so the
-  # Source IP must survive for X-Real-IP.
+  # peer address here is a Traefik pod, not the client. sessionAffinity is
-  externalTrafficPolicy: Local
+  # deliberately absent: keyed on ClientIP it would pin whole Traefik pods,
-  # Pins a client to one replica, standing in for the dropped stick-table peers.
+  # not clients. Backend persistence rests on the per-backend SRVNAME cookie.
-  sessionAffinity: ClientIP
   selector:
     app: haproxy
   ports:
-    - name: http
-      port: 80
-      protocol: TCP
-      targetPort: http
     - name: https
       port: 443
       protocol: TCP

apps/base/haproxy/tlsroute.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: TLSRoute
+metadata:
+  name: haproxy
+  namespace: haproxy
+  labels:
+    app: haproxy
+spec:
+  hostnames:
+    - sonarr.main.unkin.net
+    - radarr.main.unkin.net
+    - lidarr.main.unkin.net
+    - readarr.main.unkin.net
+    - prowlarr.main.unkin.net
+    - nzbget.main.unkin.net
+    - jellyfin.main.unkin.net
+    - fafflix.unkin.net
+    - git.unkin.net
+    - grafana.unkin.net
+    - dashboard.ceph.unkin.net
+    - auth.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: haproxy
+      sectionName: https-passthrough
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: haproxy
+          port: 443
+          weight: 1