Merge main into the haproxy traefik branch

Stalwart was only ever a test deployment. The daemon is dead on all three
backend VMs and nothing public depends on it — `unkin.net` MX points at Google —
so the edge is proxying mail to nowhere and the tcp frontends make `defaults`
emit 20 spurious HTTP-mode warnings.

- Drop the `fe_smtp`, `fe_submission`, `fe_imap` and `fe_imaps` frontends.
- Drop the five `be_stalwart_*` backends and their map entries in `fe_http.map`/`fe_https.map`.
- Drop the now-unused 25/143/587/993 Service and container ports.

`haproxy -c` on the rendered config: exit 0, 0 warnings (was 20), 0 alerts.

Reviewed-on: #491
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

apps/base/haproxy/configmap.yaml

@@ -11,23 +11,6 @@ data:
     /etc/haproxy/certs/main-unkin-net/tls.crt
     /etc/haproxy/certs/ceph-unkin-net/tls.crt
 
-  fe_http.map: |
-    sonarr.main.unkin.net be_sonarr
-    radarr.main.unkin.net be_radarr
-    lidarr.main.unkin.net be_lidarr
-    readarr.main.unkin.net be_readarr
-    prowlarr.main.unkin.net be_prowlarr
-    nzbget.main.unkin.net be_nzbget
-    jellyfin.main.unkin.net be_jellyfin
-    fafflix.unkin.net be_jellyfin
-    git.unkin.net be_gitea
-    grafana.unkin.net be_grafana
-    dashboard.ceph.unkin.net be_ceph_dashboard
-    mail-webadmin.main.unkin.net be_stalwart_webadmin
-    autoconfig.main.unkin.net be_stalwart_webadmin
-    autodiscovery.main.unkin.net be_stalwart_webadmin
-    auth.unkin.net be_k8s_kanidm
-
   fe_https.map: |
     sonarr.main.unkin.net be_sonarr
     radarr.main.unkin.net be_radarr
@@ -40,9 +23,6 @@ data:
     git.unkin.net be_gitea
     grafana.unkin.net be_grafana
     dashboard.ceph.unkin.net be_ceph_dashboard
-    mail-webadmin.main.unkin.net be_stalwart_webadmin
-    autoconfig.main.unkin.net be_stalwart_webadmin
-    autodiscovery.main.unkin.net be_stalwart_webadmin
     auth.unkin.net be_k8s_kanidm
 
   haproxy.cfg: |
@@ -79,76 +59,16 @@ data:
       timeout  http-keep-alive 10s
       timeout  check 10s
 
-    frontend fe_http
-      bind 0.0.0.0:80
-      mode http
-      description Global HTTP Frontend
-      acl acl-letsencrypt path_beg /.well-known/acme-challenge/
-      http-request set-header X-Forwarded-Proto https
-      http-request set-header X-Real-IP %[src]
-      use_backend be_letsencrypt if acl-letsencrypt
-      use_backend %[req.hdr(host),lower,map(/usr/local/etc/haproxy/fe_http.map,be_default)]
-
     frontend fe_https
       bind 0.0.0.0:443 ssl crt-list /usr/local/etc/haproxy/certificate.list ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH force-tlsv12
       mode http
       description Global HTTPS Frontend
-      acl acl-letsencrypt path_beg /.well-known/acme-challenge/
-      acl acl_sonarr req.hdr(host) -i sonarr.main.unkin.net
-      acl acl_radarr req.hdr(host) -i radarr.main.unkin.net
-      acl acl_lidarr req.hdr(host) -i lidarr.main.unkin.net
-      acl acl_readarr req.hdr(host) -i readarr.main.unkin.net
-      acl acl_prowlarr req.hdr(host) -i prowlarr.main.unkin.net
-      acl acl_nzbget req.hdr(host) -i nzbget.main.unkin.net
-      acl acl_jellyfin req.hdr(host) -i jellyfin.main.unkin.net
-      acl acl_fafflix req.hdr(host) -i fafflix.unkin.net
-      acl acl_gitea req.hdr(host) -i git.unkin.net
-      acl acl_grafana req.hdr(host) -i grafana.unkin.net
-      acl acl_ceph_dashboard req.hdr(host) -i dashboard.ceph.unkin.net
-      acl acl_stalwart_webadmin req.hdr(host) -i mail-webadmin.main.unkin.net
-      acl acl_stalwart_webadmin req.hdr(host) -i autoconfig.main.unkin.net
-      acl acl_stalwart_webadmin req.hdr(host) -i autodiscovery.main.unkin.net
-      acl acl_kanidm req.hdr(host) -i auth.unkin.net
       http-request set-header X-Forwarded-Proto https
       http-request set-header X-Real-IP %[src]
-      http-response set-header X-Frame-Options DENY if acl_sonarr
-      http-response set-header X-Frame-Options DENY if acl_radarr
-      http-response set-header X-Frame-Options DENY if acl_lidarr
-      http-response set-header X-Frame-Options DENY if acl_readarr
-      http-response set-header X-Frame-Options DENY if acl_prowlarr
-      http-response set-header X-Frame-Options DENY if acl_nzbget
-      http-response set-header X-Frame-Options DENY if acl_jellyfin
-      http-response set-header X-Frame-Options DENY if acl_fafflix
-      http-response set-header X-Frame-Options DENY if acl_gitea
-      http-response set-header X-Frame-Options DENY if acl_grafana
-      http-response set-header X-Frame-Options DENY if acl_ceph_dashboard
-      http-response set-header X-Frame-Options DENY if acl_stalwart_webadmin
-      http-response set-header X-Frame-Options DENY if acl_kanidm
       http-response set-header X-Content-Type-Options nosniff
       http-response set-header X-XSS-Protection 1;mode=block
-      use_backend be_letsencrypt if acl-letsencrypt
       use_backend %[req.hdr(host),lower,map(/usr/local/etc/haproxy/fe_https.map,be_default)]
 
-    frontend fe_imap
-      bind 0.0.0.0:143
-      mode tcp
-      description Frontend for Stalwart IMAP (STARTTLS)
-      default_backend be_stalwart_imap
-      log global
-      option tcplog
-      tcp-request inspect-delay 5s
-      tcp-request content accept if { req_len 0 }
-
-    frontend fe_imaps
-      bind 0.0.0.0:993
-      mode tcp
-      description Frontend for Stalwart IMAPS (implicit TLS)
-      default_backend be_stalwart_imaps
-      log global
-      option tcplog
-      tcp-request inspect-delay 5s
-      tcp-request content accept if { req_len 0 }
-
     frontend fe_metrics
       bind 0.0.0.0:8405
       mode http
@@ -157,26 +77,6 @@ data:
       http-request set-header X-Real-IP %[src]
       http-request use-service prometheus-exporter if { path /metrics }
 
-    frontend fe_smtp
-      bind 0.0.0.0:25
-      mode tcp
-      description Frontend for Stalwart SMTP
-      default_backend be_stalwart_smtp
-      log global
-      option tcplog
-      tcp-request inspect-delay 5s
-      tcp-request content accept if { req_len 0 }
-
-    frontend fe_submission
-      bind 0.0.0.0:587
-      mode tcp
-      description Frontend for Stalwart SMTP Submission
-      default_backend be_stalwart_submission
-      log global
-      option tcplog
-      tcp-request inspect-delay 5s
-      tcp-request content accept if { req_len 0 }
-
     backend be_ceph_dashboard
       description Backend for Ceph Dashboard from Mgr instances
       balance roundrobin
@@ -271,11 +171,6 @@ data:
       http-check expect status 200
       server k8s-traefik-internal 198.18.200.4:443 ssl verify none check inter 2s rise 3 fall 2 sni str(auth.unkin.net)
 
-    backend be_letsencrypt
-      description Backend for LetsEncrypt Verifications
-      balance roundrobin
-      server ausyd1nxvm2057 198.18.25.3:8888
-
     backend be_lidarr
       description Backend for au-syd1 lidarr
       balance roundrobin
@@ -360,85 +255,12 @@ data:
       redirect scheme https if !{ ssl_fc }
       server ausyd1nxvm2046 198.18.26.161:443 check cookie ausyd1nxvm2046 fall 2 inter 2s rise 3 ssl verify none
 
-    backend be_stalwart_imap
-      description Backend for Stalwart IMAP (STARTTLS)
-      balance roundrobin
-      mode tcp
-      option tcp-check
-      option prefer-last-server
-      stick on src
-      stick-table type ip size 200k expire 30m
-      tcp-check connect port 143 send-proxy
-      tcp-check expect string "* OK"
-      tcp-check send "A001 STARTTLS\r\n"
-      tcp-check expect rstring "A001 (OK|2.0.0)"
-      server ausyd1nxvm2124 198.18.28.76:143 check fall 3 inter 3s rise 2 send-proxy-v2
-      server ausyd1nxvm2125 198.18.29.44:143 check fall 3 inter 3s rise 2 send-proxy-v2
-      server ausyd1nxvm2126 198.18.25.160:143 check fall 3 inter 3s rise 2 send-proxy-v2
-
-    backend be_stalwart_imaps
-      description Backend for Stalwart IMAPS (implicit TLS)
-      balance roundrobin
-      mode tcp
-      option tcp-check
-      option prefer-last-server
-      stick on src
-      stick-table type ip size 200k expire 30m
-      tcp-check connect ssl send-proxy
-      tcp-check expect string "* OK"
-      server ausyd1nxvm2124 198.18.28.76:993 check fall 3 inter 3s rise 2 send-proxy-v2 ssl verify none
-      server ausyd1nxvm2125 198.18.29.44:993 check fall 3 inter 3s rise 2 send-proxy-v2 ssl verify none
-      server ausyd1nxvm2126 198.18.25.160:993 check fall 3 inter 3s rise 2 send-proxy-v2 ssl verify none
-
-    backend be_stalwart_smtp
-      description Backend for Stalwart SMTP
-      balance roundrobin
-      mode tcp
-      option tcp-check
-      option prefer-last-server
-      stick on src
-      stick-table type ip size 200k expire 30m
-      tcp-check connect port 25 send-proxy
-      tcp-check expect string "220 "
-      server ausyd1nxvm2124 198.18.28.76:25 check fall 3 inter 3s rise 2 send-proxy-v2
-      server ausyd1nxvm2125 198.18.29.44:25 check fall 3 inter 3s rise 2 send-proxy-v2
-      server ausyd1nxvm2126 198.18.25.160:25 check fall 3 inter 3s rise 2 send-proxy-v2
-
-    backend be_stalwart_submission
-      description Backend for Stalwart SMTP Submission
-      balance roundrobin
-      mode tcp
-      option tcp-check
-      option prefer-last-server
-      stick on src
-      stick-table type ip size 200k expire 30m
-      tcp-check connect port 587 send-proxy
-      tcp-check expect string "220 "
-      server ausyd1nxvm2124 198.18.28.76:587 check fall 3 inter 3s rise 2 send-proxy-v2
-      server ausyd1nxvm2125 198.18.29.44:587 check fall 3 inter 3s rise 2 send-proxy-v2
-      server ausyd1nxvm2126 198.18.25.160:587 check fall 3 inter 3s rise 2 send-proxy-v2
-
-    backend be_stalwart_webadmin
-      description Backend for Stalwart Webadmin
-      balance roundrobin
-      cookie SRVNAME insert indirect nocache
-      http-check expect status 200
-      http-request set-header X-Forwarded-Port %[dst_port]
-      http-request add-header X-Forwarded-Proto https if { dst_port 9443 }
-      http-reuse always
-      option httpchk GET /
-      option forwardfor
-      option http-keep-alive
-      option prefer-last-server
-      redirect scheme https if !{ ssl_fc }
-      stick-table type ip size 200k expire 30m
-      server ausyd1nxvm2124 198.18.28.76:443 check cookie ausyd1nxvm2124 fall 2 inter 2s rise 3 send-proxy-v2 ssl verify none
-      server ausyd1nxvm2125 198.18.29.44:443 check cookie ausyd1nxvm2125 fall 2 inter 2s rise 3 send-proxy-v2 ssl verify none
-      server ausyd1nxvm2126 198.18.25.160:443 check cookie ausyd1nxvm2126 fall 2 inter 2s rise 3 send-proxy-v2 ssl verify none
-
     # The `peers au-syd1-prod` section is dropped: peer names must be static and a
-    # Deployment cannot provide them. Service sessionAffinity: ClientIP pins a
-    # client to one replica so the per-replica stick-tables behave as before.
+    # Deployment cannot provide them. Behind the external Traefik's TLS
+    # passthrough `src` is a Traefik pod, so X-Real-IP, forwardfor and the
+    # `stick on src` tables all key on that; the SRVNAME cookie carries real
+    # session persistence. Traefik cannot emit PROXY protocol to a TLSRoute
+    # backend, so there is nothing to bind `accept-proxy` to.
 
     listen health
       bind 0.0.0.0:8404

apps/base/haproxy/deployment.yaml

@@ -50,27 +50,13 @@ spec:
             readOnlyRootFilesystem: true
             capabilities:
               drop: [ALL]
-              # Frontends bind 25/80/143/443/587; the dst_port ACLs need the real ports.
+              # fe_https binds the privileged port 443 as uid 99, and the
+              # dst_port ACLs need the real port.
               add: [NET_BIND_SERVICE]
           ports:
-            - name: http
-              containerPort: 80
-              protocol: TCP
             - name: https
               containerPort: 443
               protocol: TCP
-            - name: smtp
-              containerPort: 25
-              protocol: TCP
-            - name: imap
-              containerPort: 143
-              protocol: TCP
-            - name: submission
-              containerPort: 587
-              protocol: TCP
-            - name: imaps
-              containerPort: 993
-              protocol: TCP
             - name: health
               containerPort: 8404
               protocol: TCP

apps/base/haproxy/gateway.yaml

@@ -0,0 +1,31 @@
+---
+# External (DMZ) front for the haproxy edge on the traefik-external LB VIP
+# 198.18.199.0. The :443 listener is TLS Passthrough: haproxy owns the three
+# wildcard certs and terminates behind Traefik, so there are no certificateRefs
+# here. Listener hostnames are deliberately unset and the routes carry the
+# explicit hostname list instead; allowedRoutes Same keeps other namespaces off
+# these listeners.
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: haproxy
+  namespace: haproxy
+  labels:
+    traefik.io/instance: external
+spec:
+  gatewayClassName: traefik-external
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https-passthrough
+      port: 443
+      protocol: TLS
+      tls:
+        mode: Passthrough
+      allowedRoutes:
+        namespaces:
+          from: Same

apps/base/haproxy/httproute.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: haproxy-http-redirect
+  namespace: haproxy
+  labels:
+    app: haproxy
+spec:
+  hostnames:
+    - sonarr.main.unkin.net
+    - radarr.main.unkin.net
+    - lidarr.main.unkin.net
+    - readarr.main.unkin.net
+    - prowlarr.main.unkin.net
+    - nzbget.main.unkin.net
+    - jellyfin.main.unkin.net
+    - fafflix.unkin.net
+    - git.unkin.net
+    - grafana.unkin.net
+    - dashboard.ceph.unkin.net
+    - auth.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: haproxy
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/haproxy/kustomization.yaml

@@ -7,6 +7,9 @@ resources:
   - configmap.yaml
   - deployment.yaml
   - service.yaml
+  - gateway.yaml
+  - tlsroute.yaml
+  - httproute.yaml
   - pdb.yaml
   - vpa.yaml
   - vmpodscrape.yaml

apps/base/haproxy/service.yaml

@@ -4,40 +4,16 @@ kind: Service
 metadata:
   name: haproxy
   namespace: haproxy
-  annotations:
-    purelb.io/service-group: dmz
-    purelb.io/addresses: 198.18.199.1
 spec:
-  type: LoadBalancer
-  loadBalancerIP: "198.18.199.1"
-  # Source IP must survive for acl_internalsubnets, X-Real-IP and SMTP.
-  externalTrafficPolicy: Local
-  # Pins a client to one replica, standing in for the dropped stick-table peers.
-  sessionAffinity: ClientIP
+  type: ClusterIP
+  # Reached only by the external Traefik's TLS-passthrough TLSRoute, so the
+  # peer address here is a Traefik pod, not the client. sessionAffinity is
+  # deliberately absent: keyed on ClientIP it would pin whole Traefik pods,
+  # not clients. Backend persistence rests on the per-backend SRVNAME cookie.
   selector:
     app: haproxy
   ports:
-    - name: http
-      port: 80
-      protocol: TCP
-      targetPort: http
     - name: https
       port: 443
       protocol: TCP
       targetPort: https
-    - name: smtp
-      port: 25
-      protocol: TCP
-      targetPort: smtp
-    - name: imap
-      port: 143
-      protocol: TCP
-      targetPort: imap
-    - name: submission
-      port: 587
-      protocol: TCP
-      targetPort: submission
-    - name: imaps
-      port: 993
-      protocol: TCP
-      targetPort: imaps

apps/base/haproxy/tlsroute.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: TLSRoute
+metadata:
+  name: haproxy
+  namespace: haproxy
+  labels:
+    app: haproxy
+spec:
+  hostnames:
+    - sonarr.main.unkin.net
+    - radarr.main.unkin.net
+    - lidarr.main.unkin.net
+    - readarr.main.unkin.net
+    - prowlarr.main.unkin.net
+    - nzbget.main.unkin.net
+    - jellyfin.main.unkin.net
+    - fafflix.unkin.net
+    - git.unkin.net
+    - grafana.unkin.net
+    - dashboard.ceph.unkin.net
+    - auth.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: haproxy
+      sectionName: https-passthrough
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: haproxy
+          port: 443
+          weight: 1

apps/base/puppet/cronjob_g10k-code.yaml

@@ -11,11 +11,13 @@ metadata:
   namespace: puppet
 spec:
   schedule: "*/1 * * * *"
+  startingDeadlineSeconds: 200
   concurrencyPolicy: Forbid
   successfulJobsHistoryLimit: 3
   failedJobsHistoryLimit: 3
   jobTemplate:
     spec:
+      activeDeadlineSeconds: 300
       template:
         metadata:
           labels:

apps/base/woodpecker/gocache_bucket.yaml

@@ -0,0 +1,33 @@
+---
+# Shared Go build cache (GOCACHEPROG) for CI and developer laptops. Lives in the
+# woodpecker namespace because CI is the primary consumer and reads the Secret here.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: gocache
+  namespace: woodpecker
+spec:
+  displayName: "Go build cache owner"
+  uid: gocache
+  maxBuckets: 1
+  secretName: gocache-s3
+  retainOnDelete: false
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: gocache
+  namespace: woodpecker
+spec:
+  bucketName: gocache
+  ownerRef: gocache
+  versioning: false
+  # No placementTarget: default (replicated) placement, not the ec target the
+  # backup buckets use — a build cache is millions of small objects.
+  tags:
+    app: gocache
+    purpose: go-build-cache
+  retainOnDelete: false
+  # A cache bucket is never empty, and the operator refuses to delete a
+  # non-empty bucket without this, wedging the finalizer.
+  purgeOnDelete: true

apps/base/woodpecker/kustomization.yaml

@@ -7,6 +7,7 @@ resources:
   - cnpg_cluster.yaml
   - cnpg_backup.yaml
   - cnpg_pooler.yaml
+  - gocache_bucket.yaml
   - serviceaccount_arrproxy_ci.yaml
   - serviceaccount_autobackup_operator_ci.yaml
   - serviceaccount_ghp.yaml