From ae3edd9db5a1ca9175440303ee1f65d5c58b7e0f Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 26 Sep 2026 14:09:56 +1000 Subject: [PATCH 1/4] add *.main.unkin.net and *.ceph.unkin.net wildcard certificates --- .../certificate_wildcard-ceph-unkin-net.yaml | 25 +++++++++++++++++++ .../certificate_wildcard-main-unkin-net.yaml | 25 +++++++++++++++++++ .../certificate_wildcard-unkin-net.yaml | 4 +-- apps/base/cert-manager/kustomization.yaml | 2 ++ 4 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 apps/base/cert-manager/certificate_wildcard-ceph-unkin-net.yaml create mode 100644 apps/base/cert-manager/certificate_wildcard-main-unkin-net.yaml diff --git a/apps/base/cert-manager/certificate_wildcard-ceph-unkin-net.yaml b/apps/base/cert-manager/certificate_wildcard-ceph-unkin-net.yaml new file mode 100644 index 0000000..dbc780f --- /dev/null +++ b/apps/base/cert-manager/certificate_wildcard-ceph-unkin-net.yaml @@ -0,0 +1,25 @@ +--- +# Let's Encrypt *.ceph.unkin.net wildcard for the haproxy edge (ceph dashboard). +# DNS-01 needs the delegated _acme-challenge.ceph.unkin.net CNAME in the public +# unkin.net zone. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: wildcard-ceph-unkin-net + namespace: cert-manager +spec: + secretName: wildcard-ceph-unkin-net-tls + secretTemplate: + annotations: + reflector.v1.k8s.emberstack.com/reflection-allowed: "true" + reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy" + reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true" + reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy" + privateKey: + size: 4096 + dnsNames: + - "*.ceph.unkin.net" + issuerRef: + name: letsencrypt + kind: ClusterIssuer + group: cert-manager.io diff --git a/apps/base/cert-manager/certificate_wildcard-main-unkin-net.yaml b/apps/base/cert-manager/certificate_wildcard-main-unkin-net.yaml new file mode 100644 index 0000000..ba8f76c --- /dev/null +++ b/apps/base/cert-manager/certificate_wildcard-main-unkin-net.yaml @@ -0,0 +1,25 @@ +--- +# Let's Encrypt *.main.unkin.net wildcard for the haproxy edge (pve, arr stack, +# jellyfin, stalwart webadmin/autoconfig). DNS-01 needs the delegated +# _acme-challenge.main.unkin.net CNAME in the public unkin.net zone. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: wildcard-main-unkin-net + namespace: cert-manager +spec: + secretName: wildcard-main-unkin-net-tls + secretTemplate: + annotations: + reflector.v1.k8s.emberstack.com/reflection-allowed: "true" + reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy" + reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true" + reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy" + privateKey: + size: 4096 + dnsNames: + - "*.main.unkin.net" + issuerRef: + name: letsencrypt + kind: ClusterIssuer + group: cert-manager.io diff --git a/apps/base/cert-manager/certificate_wildcard-unkin-net.yaml b/apps/base/cert-manager/certificate_wildcard-unkin-net.yaml index d3b971c..7873433 100644 --- a/apps/base/cert-manager/certificate_wildcard-unkin-net.yaml +++ b/apps/base/cert-manager/certificate_wildcard-unkin-net.yaml @@ -14,9 +14,9 @@ spec: secretTemplate: annotations: reflector.v1.k8s.emberstack.com/reflection-allowed: "true" - reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner" + reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy" reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true" - reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner" + reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy" privateKey: size: 4096 dnsNames: diff --git a/apps/base/cert-manager/kustomization.yaml b/apps/base/cert-manager/kustomization.yaml index a102d8c..592a976 100644 --- a/apps/base/cert-manager/kustomization.yaml +++ b/apps/base/cert-manager/kustomization.yaml @@ -12,3 +12,5 @@ resources: - clusterissuer_letsencrypt.yaml - clusterissuer_letsencrypt-staging.yaml - certificate_wildcard-unkin-net.yaml + - certificate_wildcard-main-unkin-net.yaml + - certificate_wildcard-ceph-unkin-net.yaml -- 2.47.3 From 4fcee639252651a5977bd4d8d4fa4f67ac0305c7 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 26 Sep 2026 14:15:05 +1000 Subject: [PATCH 2/4] move the au-syd1 haproxy edge into kubernetes --- apps/base/haproxy/configmap.yaml | 484 ++++++++++++++++++ apps/base/haproxy/deployment.yaml | 152 ++++++ apps/base/haproxy/kustomization.yaml | 12 + apps/base/haproxy/namespace.yaml | 5 + apps/base/haproxy/pdb.yaml | 11 + apps/base/haproxy/service.yaml | 43 ++ apps/base/haproxy/vmpodscrape.yaml | 13 + apps/base/haproxy/vpa.yaml | 13 + .../au-syd1/haproxy/kustomization.yaml | 6 + argocd/applicationsets/platform.yaml | 1 + argocd/projects/platform.yaml | 2 + 11 files changed, 742 insertions(+) create mode 100644 apps/base/haproxy/configmap.yaml create mode 100644 apps/base/haproxy/deployment.yaml create mode 100644 apps/base/haproxy/kustomization.yaml create mode 100644 apps/base/haproxy/namespace.yaml create mode 100644 apps/base/haproxy/pdb.yaml create mode 100644 apps/base/haproxy/service.yaml create mode 100644 apps/base/haproxy/vmpodscrape.yaml create mode 100644 apps/base/haproxy/vpa.yaml create mode 100644 apps/overlays/au-syd1/haproxy/kustomization.yaml diff --git a/apps/base/haproxy/configmap.yaml b/apps/base/haproxy/configmap.yaml new file mode 100644 index 0000000..441908f --- /dev/null +++ b/apps/base/haproxy/configmap.yaml @@ -0,0 +1,484 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: haproxy-config + namespace: haproxy +data: + certificate.list: | + # First entry is the default cert for non-matching SNI. + /etc/haproxy/certs/unkin-net/tls.crt + /etc/haproxy/certs/main-unkin-net/tls.crt + /etc/haproxy/certs/ceph-unkin-net/tls.crt + + fe_http.map: | + au-syd1-pve.main.unkin.net be_ausyd1pve_web + au-syd1-pve-api.main.unkin.net be_ausyd1pve_api + sonarr.main.unkin.net be_sonarr + radarr.main.unkin.net be_radarr + lidarr.main.unkin.net be_lidarr + readarr.main.unkin.net be_readarr + prowlarr.main.unkin.net be_prowlarr + nzbget.main.unkin.net be_nzbget + jellyfin.main.unkin.net be_jellyfin + fafflix.unkin.net be_jellyfin + git.unkin.net be_gitea + grafana.unkin.net be_grafana + dashboard.ceph.unkin.net be_ceph_dashboard + mail-webadmin.main.unkin.net be_stalwart_webadmin + autoconfig.main.unkin.net be_stalwart_webadmin + autodiscovery.main.unkin.net be_stalwart_webadmin + auth.unkin.net be_k8s_kanidm + + fe_https.map: | + au-syd1-pve.main.unkin.net be_ausyd1pve_web + au-syd1-pve-api.main.unkin.net be_ausyd1pve_api + sonarr.main.unkin.net be_sonarr + radarr.main.unkin.net be_radarr + lidarr.main.unkin.net be_lidarr + readarr.main.unkin.net be_readarr + prowlarr.main.unkin.net be_prowlarr + nzbget.main.unkin.net be_nzbget + jellyfin.main.unkin.net be_jellyfin + fafflix.unkin.net be_jellyfin + git.unkin.net be_gitea + grafana.unkin.net be_grafana + dashboard.ceph.unkin.net be_ceph_dashboard + mail-webadmin.main.unkin.net be_stalwart_webadmin + autoconfig.main.unkin.net be_stalwart_webadmin + autodiscovery.main.unkin.net be_stalwart_webadmin + auth.unkin.net be_k8s_kanidm + + haproxy.cfg: | + global + log stdout format raw local0 + log stdout format raw local1 notice + maxconn 4000 + ssl-default-bind-ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH + ssl-default-bind-options ssl-min-ver TLSv1.2 ssl-max-ver TLSv1.3 + ssl-default-server-ciphers kEECDH+aRSA+AES:kRSA+AES:+AES256:RC4-SHA:!kEDH:!LOW:!EXP:!MD5:!aNULL:!eNULL + ssl-default-server-options no-sslv3 + stats timeout 30s + stats socket /var/lib/haproxy/stats + stats socket /var/lib/haproxy/admin.sock mode 660 level admin + tune.ssl.default-dh-param 2048 + + defaults + log global + maxconn 5000 + mode http + option httplog + option dontlognull + option http-server-close + option forwardfor except 127.0.0.0/8 + option redispatch + retries 3 + stats enable + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 5m + timeout server 5m + timeout http-keep-alive 10s + timeout check 10s + + frontend fe_http + bind 0.0.0.0:80 + mode http + description Global HTTP Frontend + acl acl-letsencrypt path_beg /.well-known/acme-challenge/ + http-request set-header X-Forwarded-Proto https + http-request set-header X-Real-IP %[src] + use_backend be_letsencrypt if acl-letsencrypt + use_backend %[req.hdr(host),lower,map(/usr/local/etc/haproxy/fe_http.map,be_default)] + + frontend fe_https + bind 0.0.0.0:443 ssl crt-list /usr/local/etc/haproxy/certificate.list ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH force-tlsv12 + mode http + description Global HTTPS Frontend + acl acl-letsencrypt path_beg /.well-known/acme-challenge/ + acl acl_ausyd1pve req.hdr(host) -i au-syd1-pve.main.unkin.net + acl acl_sonarr req.hdr(host) -i sonarr.main.unkin.net + acl acl_radarr req.hdr(host) -i radarr.main.unkin.net + acl acl_lidarr req.hdr(host) -i lidarr.main.unkin.net + acl acl_readarr req.hdr(host) -i readarr.main.unkin.net + acl acl_prowlarr req.hdr(host) -i prowlarr.main.unkin.net + acl acl_nzbget req.hdr(host) -i nzbget.main.unkin.net + acl acl_jellyfin req.hdr(host) -i jellyfin.main.unkin.net + acl acl_fafflix req.hdr(host) -i fafflix.unkin.net + acl acl_gitea req.hdr(host) -i git.unkin.net + acl acl_grafana req.hdr(host) -i grafana.unkin.net + acl acl_ceph_dashboard req.hdr(host) -i dashboard.ceph.unkin.net + acl acl_stalwart_webadmin req.hdr(host) -i mail-webadmin.main.unkin.net + acl acl_stalwart_webadmin req.hdr(host) -i autoconfig.main.unkin.net + acl acl_stalwart_webadmin req.hdr(host) -i autodiscovery.main.unkin.net + acl acl_kanidm req.hdr(host) -i auth.unkin.net + acl acl_internalsubnets src 198.18.0.0/16 10.10.12.0/24 + http-request set-header X-Forwarded-Proto https + http-request set-header X-Real-IP %[src] + http-request deny if { hdr_dom(host) -i au-syd1-pve.main.unkin.net } !acl_internalsubnets + http-response set-header X-Frame-Options DENY if acl_ausyd1pve + http-response set-header X-Frame-Options DENY if acl_sonarr + http-response set-header X-Frame-Options DENY if acl_radarr + http-response set-header X-Frame-Options DENY if acl_lidarr + http-response set-header X-Frame-Options DENY if acl_readarr + http-response set-header X-Frame-Options DENY if acl_prowlarr + http-response set-header X-Frame-Options DENY if acl_nzbget + http-response set-header X-Frame-Options DENY if acl_jellyfin + http-response set-header X-Frame-Options DENY if acl_fafflix + http-response set-header X-Frame-Options DENY if acl_gitea + http-response set-header X-Frame-Options DENY if acl_grafana + http-response set-header X-Frame-Options DENY if acl_ceph_dashboard + http-response set-header X-Frame-Options DENY if acl_stalwart_webadmin + http-response set-header X-Frame-Options DENY if acl_kanidm + http-response set-header X-Content-Type-Options nosniff + http-response set-header X-XSS-Protection 1;mode=block + use_backend be_letsencrypt if acl-letsencrypt + use_backend %[req.hdr(host),lower,map(/usr/local/etc/haproxy/fe_https.map,be_default)] + + frontend fe_imap + bind 0.0.0.0:143 + mode tcp + description Frontend for Stalwart IMAP (STARTTLS) + default_backend be_stalwart_imap + log global + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_len 0 } + + frontend fe_imaps + bind 0.0.0.0:993 + mode tcp + description Frontend for Stalwart IMAPS (implicit TLS) + default_backend be_stalwart_imaps + log global + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_len 0 } + + frontend fe_metrics + bind 0.0.0.0:8405 + mode http + description Metrics Frontend + http-request set-header X-Forwarded-Proto https + http-request set-header X-Real-IP %[src] + http-request use-service prometheus-exporter if { path /metrics } + + frontend fe_smtp + bind 0.0.0.0:25 + mode tcp + description Frontend for Stalwart SMTP + default_backend be_stalwart_smtp + log global + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_len 0 } + + frontend fe_submission + bind 0.0.0.0:587 + mode tcp + description Frontend for Stalwart SMTP Submission + default_backend be_stalwart_submission + log global + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_len 0 } + + backend be_ausyd1pve_api + description Backend for au-syd1 pve cluster (API only) + balance roundrobin + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { dst_port 443 } + http-reuse always + option httpchk GET / + option forwardfor + option http-keep-alive + option prefer-last-server + redirect scheme https if !{ ssl_fc } + + backend be_ausyd1pve_web + description Backend for au-syd1 pve cluster (Web) + balance roundrobin + cookie SRVNAME insert indirect nocache + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { dst_port 443 } + http-reuse always + option httpchk GET / + option forwardfor + option http-keep-alive + option prefer-last-server + redirect scheme https if !{ ssl_fc } + + backend be_ceph_dashboard + description Backend for Ceph Dashboard from Mgr instances + balance roundrobin + cookie SRVNAME insert indirect nocache + http-check expect status 200 + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { dst_port 9443 } + http-reuse always + option httpchk GET / + option forwardfor + option http-keep-alive + option prefer-last-server + redirect scheme https if !{ ssl_fc } + stick-table type ip size 200k expire 30m + server prodnxsr0009 198.18.23.9:9443 check cookie prodnxsr0009 fall 2 inter 2s rise 3 ssl verify none + server prodnxsr0010 198.18.23.10:9443 check cookie prodnxsr0010 fall 2 inter 2s rise 3 ssl verify none + server prodnxsr0011 198.18.23.11:9443 check cookie prodnxsr0011 fall 2 inter 2s rise 3 ssl verify none + server prodnxsr0012 198.18.23.12:9443 check cookie prodnxsr0012 fall 2 inter 2s rise 3 ssl verify none + server prodnxsr0013 198.18.23.13:9443 check cookie prodnxsr0013 fall 2 inter 2s rise 3 ssl verify none + + backend be_default + description Backend for unmatched HTTP traffic + balance roundrobin + cookie SRVNAME insert + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { dst_port 443 } + option httpchk GET / + option forwardfor + + backend be_gitea + description Backend for gitea cluster + balance roundrobin + cookie SRVNAME insert indirect nocache + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { dst_port 443 } + http-reuse always + option httpchk GET / + option forwardfor + option http-keep-alive + option prefer-last-server + redirect scheme https if !{ ssl_fc } + stick on src + stick-table type ip size 200k expire 30m + server ausyd1nxvm2080 198.18.26.18:443 check cookie ausyd1nxvm2080 fall 2 inter 2s rise 3 ssl verify none + server ausyd1nxvm2081 198.18.27.117:443 check cookie ausyd1nxvm2081 fall 2 inter 2s rise 3 ssl verify none + server ausyd1nxvm2082 198.18.28.71:443 check cookie ausyd1nxvm2082 fall 2 inter 2s rise 3 ssl verify none + + backend be_grafana + description Backend for grafana nodes + balance roundrobin + cookie SRVNAME insert indirect nocache + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { dst_port 443 } + http-reuse always + option httpchk GET / + option forwardfor + option http-keep-alive + option prefer-last-server + redirect scheme https if !{ ssl_fc } + stick on src + stick-table type ip size 200k expire 30m + server ausyd1nxvm2015 198.18.27.2:443 check cookie ausyd1nxvm2015 fall 2 inter 2s rise 3 ssl verify none + server ausyd1nxvm2016 198.18.28.189:443 check cookie ausyd1nxvm2016 fall 2 inter 2s rise 3 ssl verify none + + backend be_jellyfin + description Backend for au-syd1 jellyfin + balance roundrobin + cookie SRVNAME insert indirect nocache + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { dst_port 443 } + http-reuse always + option httpchk GET / + option forwardfor + option http-keep-alive + option prefer-last-server + redirect scheme https if !{ ssl_fc } + server ausyd1nxvm2051 198.18.25.164:443 check cookie ausyd1nxvm2051 fall 2 inter 2s rise 3 ssl verify none + + backend be_k8s_kanidm + description Backend for Kanidm (auth.unkin.net via Kubernetes internal Traefik) + balance roundrobin + http-reuse always + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { dst_port 443 } + redirect scheme https if !{ ssl_fc } + option httpchk + option forwardfor + option http-keep-alive + option prefer-last-server + http-check connect ssl sni auth.unkin.net + http-check send meth GET uri /status ver HTTP/1.1 hdr Host auth.unkin.net + http-check expect status 200 + server k8s-traefik-internal 198.18.200.4:443 ssl verify none check inter 2s rise 3 fall 2 sni str(auth.unkin.net) + + backend be_letsencrypt + description Backend for LetsEncrypt Verifications + balance roundrobin + server ausyd1nxvm2057 198.18.25.3:8888 + + backend be_lidarr + description Backend for au-syd1 lidarr + balance roundrobin + cookie SRVNAME insert indirect nocache + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { dst_port 443 } + http-reuse always + option httpchk GET /consul/health + option forwardfor + option http-keep-alive + option prefer-last-server + redirect scheme https if !{ ssl_fc } + server ausyd1nxvm2048 198.18.28.165:443 check cookie ausyd1nxvm2048 fall 2 inter 2s rise 3 ssl verify none + + backend be_nzbget + description Backend for au-syd1 nzbget + balance roundrobin + cookie SRVNAME insert indirect nocache + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { dst_port 443 } + http-reuse always + option httpchk GET /consul/health + option forwardfor + option http-keep-alive + option prefer-last-server + redirect scheme https if !{ ssl_fc } + server ausyd1nxvm2045 198.18.25.44:443 check cookie ausyd1nxvm2045 fall 2 inter 2s rise 3 ssl verify none + + backend be_prowlarr + description Backend for au-syd1 prowlarr + balance roundrobin + cookie SRVNAME insert indirect nocache + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { dst_port 443 } + http-reuse always + option httpchk GET /consul/health + option forwardfor + option http-keep-alive + option prefer-last-server + redirect scheme https if !{ ssl_fc } + server ausyd1nxvm2050 198.18.25.66:443 check cookie ausyd1nxvm2050 fall 2 inter 2s rise 3 ssl verify none + + backend be_radarr + description Backend for au-syd1 radarr + balance roundrobin + cookie SRVNAME insert indirect nocache + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { dst_port 443 } + http-reuse always + option httpchk GET /consul/health + option forwardfor + option http-keep-alive + option prefer-last-server + redirect scheme https if !{ ssl_fc } + server ausyd1nxvm2047 198.18.27.131:443 check cookie ausyd1nxvm2047 fall 2 inter 2s rise 3 ssl verify none + + backend be_readarr + description Backend for au-syd1 readarr + balance roundrobin + cookie SRVNAME insert indirect nocache + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { dst_port 443 } + http-reuse always + option httpchk GET /consul/health + option forwardfor + option http-keep-alive + option prefer-last-server + redirect scheme https if !{ ssl_fc } + server ausyd1nxvm2049 198.18.29.32:443 check cookie ausyd1nxvm2049 fall 2 inter 2s rise 3 ssl verify none + + backend be_sonarr + description Backend for au-syd1 sonarr + balance roundrobin + cookie SRVNAME insert indirect nocache + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { dst_port 443 } + http-reuse always + option httpchk GET /consul/health + option forwardfor + option http-keep-alive + option prefer-last-server + redirect scheme https if !{ ssl_fc } + server ausyd1nxvm2046 198.18.26.161:443 check cookie ausyd1nxvm2046 fall 2 inter 2s rise 3 ssl verify none + + backend be_stalwart_imap + description Backend for Stalwart IMAP (STARTTLS) + balance roundrobin + mode tcp + option tcp-check + option prefer-last-server + stick on src + stick-table type ip size 200k expire 30m + tcp-check connect port 143 send-proxy + tcp-check expect string "* OK" + tcp-check send "A001 STARTTLS\r\n" + tcp-check expect rstring "A001 (OK|2.0.0)" + server ausyd1nxvm2124 198.18.28.76:143 check fall 3 inter 3s rise 2 send-proxy-v2 + server ausyd1nxvm2125 198.18.29.44:143 check fall 3 inter 3s rise 2 send-proxy-v2 + server ausyd1nxvm2126 198.18.25.160:143 check fall 3 inter 3s rise 2 send-proxy-v2 + + backend be_stalwart_imaps + description Backend for Stalwart IMAPS (implicit TLS) + balance roundrobin + mode tcp + option tcp-check + option prefer-last-server + stick on src + stick-table type ip size 200k expire 30m + tcp-check connect ssl send-proxy + tcp-check expect string "* OK" + server ausyd1nxvm2124 198.18.28.76:993 check fall 3 inter 3s rise 2 send-proxy-v2 ssl verify none + server ausyd1nxvm2125 198.18.29.44:993 check fall 3 inter 3s rise 2 send-proxy-v2 ssl verify none + server ausyd1nxvm2126 198.18.25.160:993 check fall 3 inter 3s rise 2 send-proxy-v2 ssl verify none + + backend be_stalwart_smtp + description Backend for Stalwart SMTP + balance roundrobin + mode tcp + option tcp-check + option prefer-last-server + stick on src + stick-table type ip size 200k expire 30m + tcp-check connect port 25 send-proxy + tcp-check expect string "220 " + server ausyd1nxvm2124 198.18.28.76:25 check fall 3 inter 3s rise 2 send-proxy-v2 + server ausyd1nxvm2125 198.18.29.44:25 check fall 3 inter 3s rise 2 send-proxy-v2 + server ausyd1nxvm2126 198.18.25.160:25 check fall 3 inter 3s rise 2 send-proxy-v2 + + backend be_stalwart_submission + description Backend for Stalwart SMTP Submission + balance roundrobin + mode tcp + option tcp-check + option prefer-last-server + stick on src + stick-table type ip size 200k expire 30m + tcp-check connect port 587 send-proxy + tcp-check expect string "220 " + server ausyd1nxvm2124 198.18.28.76:587 check fall 3 inter 3s rise 2 send-proxy-v2 + server ausyd1nxvm2125 198.18.29.44:587 check fall 3 inter 3s rise 2 send-proxy-v2 + server ausyd1nxvm2126 198.18.25.160:587 check fall 3 inter 3s rise 2 send-proxy-v2 + + backend be_stalwart_webadmin + description Backend for Stalwart Webadmin + balance roundrobin + cookie SRVNAME insert indirect nocache + http-check expect status 200 + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { dst_port 9443 } + http-reuse always + option httpchk GET / + option forwardfor + option http-keep-alive + option prefer-last-server + redirect scheme https if !{ ssl_fc } + stick-table type ip size 200k expire 30m + server ausyd1nxvm2124 198.18.28.76:443 check cookie ausyd1nxvm2124 fall 2 inter 2s rise 3 send-proxy-v2 ssl verify none + server ausyd1nxvm2125 198.18.29.44:443 check cookie ausyd1nxvm2125 fall 2 inter 2s rise 3 send-proxy-v2 ssl verify none + server ausyd1nxvm2126 198.18.25.160:443 check cookie ausyd1nxvm2126 fall 2 inter 2s rise 3 send-proxy-v2 ssl verify none + + # The `peers au-syd1-prod` section is dropped: peer names must be static and a + # Deployment cannot provide them. Service sessionAffinity: ClientIP pins a + # client to one replica so the per-replica stick-tables behave as before. + + listen health + bind 0.0.0.0:8404 + mode http + monitor-uri /healthz + + listen stats + bind 0.0.0.0:9090 + mode http + stats uri / + stats auth admin:admin diff --git a/apps/base/haproxy/deployment.yaml b/apps/base/haproxy/deployment.yaml new file mode 100644 index 0000000..ca849ef --- /dev/null +++ b/apps/base/haproxy/deployment.yaml @@ -0,0 +1,152 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: haproxy + namespace: haproxy + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: 3 + selector: + matchLabels: + app: haproxy + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + app: haproxy + spec: + automountServiceAccountToken: false + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: haproxy + topologyKey: kubernetes.io/hostname + securityContext: + runAsNonRoot: true + runAsUser: 99 + runAsGroup: 99 + seccompProfile: + type: RuntimeDefault + containers: + - name: haproxy + image: haproxy:3.2.24-alpine + imagePullPolicy: IfNotPresent + command: + - haproxy + - -W + - -db + - -f + - /usr/local/etc/haproxy/haproxy.cfg + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + # Frontends bind 25/80/143/443/587; the dst_port ACLs need the real ports. + add: [NET_BIND_SERVICE] + ports: + - name: http + containerPort: 80 + protocol: TCP + - name: https + containerPort: 443 + protocol: TCP + - name: smtp + containerPort: 25 + protocol: TCP + - name: imap + containerPort: 143 + protocol: TCP + - name: submission + containerPort: 587 + protocol: TCP + - name: imaps + containerPort: 993 + protocol: TCP + - name: health + containerPort: 8404 + protocol: TCP + - name: metrics + containerPort: 8405 + protocol: TCP + - name: stats + containerPort: 9090 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: health + initialDelaySeconds: 15 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthz + port: health + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: 2 + memory: 1Gi + volumeMounts: + - name: config + mountPath: /usr/local/etc/haproxy + readOnly: true + - name: cert-unkin-net + mountPath: /etc/haproxy/certs/unkin-net + readOnly: true + - name: cert-main-unkin-net + mountPath: /etc/haproxy/certs/main-unkin-net + readOnly: true + - name: cert-ceph-unkin-net + mountPath: /etc/haproxy/certs/ceph-unkin-net + readOnly: true + - name: run + mountPath: /var/lib/haproxy + volumes: + - name: config + configMap: + name: haproxy-config + # ssl-load-extra-files loads .key by default, so the key is + # projected next to the cert as tls.crt.key. + - name: cert-unkin-net + secret: + secretName: wildcard-unkin-net-tls + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.crt.key + - name: cert-main-unkin-net + secret: + secretName: wildcard-main-unkin-net-tls + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.crt.key + - name: cert-ceph-unkin-net + secret: + secretName: wildcard-ceph-unkin-net-tls + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.crt.key + - name: run + emptyDir: {} + restartPolicy: Always diff --git a/apps/base/haproxy/kustomization.yaml b/apps/base/haproxy/kustomization.yaml new file mode 100644 index 0000000..680c918 --- /dev/null +++ b/apps/base/haproxy/kustomization.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - configmap.yaml + - deployment.yaml + - service.yaml + - pdb.yaml + - vpa.yaml + - vmpodscrape.yaml diff --git a/apps/base/haproxy/namespace.yaml b/apps/base/haproxy/namespace.yaml new file mode 100644 index 0000000..bba7910 --- /dev/null +++ b/apps/base/haproxy/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: haproxy diff --git a/apps/base/haproxy/pdb.yaml b/apps/base/haproxy/pdb.yaml new file mode 100644 index 0000000..6552183 --- /dev/null +++ b/apps/base/haproxy/pdb.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: haproxy + namespace: haproxy +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: haproxy diff --git a/apps/base/haproxy/service.yaml b/apps/base/haproxy/service.yaml new file mode 100644 index 0000000..388401c --- /dev/null +++ b/apps/base/haproxy/service.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: haproxy + namespace: haproxy + annotations: + purelb.io/service-group: dmz + purelb.io/addresses: 198.18.199.1 +spec: + type: LoadBalancer + loadBalancerIP: "198.18.199.1" + # Source IP must survive for acl_internalsubnets, X-Real-IP and SMTP. + externalTrafficPolicy: Local + # Pins a client to one replica, standing in for the dropped stick-table peers. + sessionAffinity: ClientIP + selector: + app: haproxy + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + - name: https + port: 443 + protocol: TCP + targetPort: https + - name: smtp + port: 25 + protocol: TCP + targetPort: smtp + - name: imap + port: 143 + protocol: TCP + targetPort: imap + - name: submission + port: 587 + protocol: TCP + targetPort: submission + - name: imaps + port: 993 + protocol: TCP + targetPort: imaps diff --git a/apps/base/haproxy/vmpodscrape.yaml b/apps/base/haproxy/vmpodscrape.yaml new file mode 100644 index 0000000..4bb80c5 --- /dev/null +++ b/apps/base/haproxy/vmpodscrape.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: haproxy + namespace: haproxy +spec: + selector: + matchLabels: + app: haproxy + podMetricsEndpoints: + - port: metrics + path: /metrics diff --git a/apps/base/haproxy/vpa.yaml b/apps/base/haproxy/vpa.yaml new file mode 100644 index 0000000..37ccc6a --- /dev/null +++ b/apps/base/haproxy/vpa.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: haproxy-vpa + namespace: haproxy +spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: haproxy + updatePolicy: + updateMode: "Off" diff --git a/apps/overlays/au-syd1/haproxy/kustomization.yaml b/apps/overlays/au-syd1/haproxy/kustomization.yaml new file mode 100644 index 0000000..e73b6a9 --- /dev/null +++ b/apps/overlays/au-syd1/haproxy/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../base/haproxy diff --git a/argocd/applicationsets/platform.yaml b/argocd/applicationsets/platform.yaml index fae41e9..2a6cfe5 100644 --- a/argocd/applicationsets/platform.yaml +++ b/argocd/applicationsets/platform.yaml @@ -29,6 +29,7 @@ spec: - path: apps/overlays/*/ghp - path: apps/overlays/*/gitea - path: apps/overlays/*/grafana-system + - path: apps/overlays/*/haproxy - path: apps/overlays/*/inteldeviceplugins-system - path: apps/overlays/*/jfrog - path: apps/overlays/*/k8up-system diff --git a/argocd/projects/platform.yaml b/argocd/projects/platform.yaml index ef702c8..d5be156 100644 --- a/argocd/projects/platform.yaml +++ b/argocd/projects/platform.yaml @@ -43,6 +43,8 @@ spec: server: https://kubernetes.default.svc - namespace: 'gitea' server: https://kubernetes.default.svc + - namespace: 'haproxy' + server: https://kubernetes.default.svc - namespace: 'jfrog' server: https://kubernetes.default.svc - namespace: 'kanidm' -- 2.47.3 From 2cbac810157129c782f73b01a3bf7557c1f7742c Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 26 Sep 2026 14:38:25 +1000 Subject: [PATCH 3/4] drain haproxy on shutdown and keep the stats listener on loopback --- apps/base/haproxy/configmap.yaml | 3 ++- apps/base/haproxy/deployment.yaml | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/base/haproxy/configmap.yaml b/apps/base/haproxy/configmap.yaml index 441908f..14f9ee7 100644 --- a/apps/base/haproxy/configmap.yaml +++ b/apps/base/haproxy/configmap.yaml @@ -54,6 +54,7 @@ data: log stdout format raw local0 log stdout format raw local1 notice maxconn 4000 + hard-stop-after 2m ssl-default-bind-ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH ssl-default-bind-options ssl-min-ver TLSv1.2 ssl-max-ver TLSv1.3 ssl-default-server-ciphers kEECDH+aRSA+AES:kRSA+AES:+AES256:RC4-SHA:!kEDH:!LOW:!EXP:!MD5:!aNULL:!eNULL @@ -478,7 +479,7 @@ data: monitor-uri /healthz listen stats - bind 0.0.0.0:9090 + bind 127.0.0.1:9090 mode http stats uri / stats auth admin:admin diff --git a/apps/base/haproxy/deployment.yaml b/apps/base/haproxy/deployment.yaml index ca849ef..4a1f11e 100644 --- a/apps/base/haproxy/deployment.yaml +++ b/apps/base/haproxy/deployment.yaml @@ -21,6 +21,7 @@ spec: app: haproxy spec: automountServiceAccountToken: false + terminationGracePeriodSeconds: 150 affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -79,6 +80,15 @@ spec: - name: stats containerPort: 9090 protocol: TCP + lifecycle: + preStop: + exec: + # SIGUSR1 to the master soft-stops the workers; hard-stop-after + # caps the drain. Wait so kubelet holds SIGTERM until it is done. + command: + - /bin/sh + - -c + - kill -s USR1 1; while kill -0 1 2>/dev/null; do sleep 1; done livenessProbe: httpGet: path: /healthz -- 2.47.3 From be5270e7a9e26442aa046366fdf51995f7481dc4 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 26 Sep 2026 17:45:40 +1000 Subject: [PATCH 4/4] drop proxmox routing --- apps/base/haproxy/configmap.yaml | 33 -------------------------------- 1 file changed, 33 deletions(-) diff --git a/apps/base/haproxy/configmap.yaml b/apps/base/haproxy/configmap.yaml index 14f9ee7..d81ac50 100644 --- a/apps/base/haproxy/configmap.yaml +++ b/apps/base/haproxy/configmap.yaml @@ -12,8 +12,6 @@ data: /etc/haproxy/certs/ceph-unkin-net/tls.crt fe_http.map: | - au-syd1-pve.main.unkin.net be_ausyd1pve_web - au-syd1-pve-api.main.unkin.net be_ausyd1pve_api sonarr.main.unkin.net be_sonarr radarr.main.unkin.net be_radarr lidarr.main.unkin.net be_lidarr @@ -31,8 +29,6 @@ data: auth.unkin.net be_k8s_kanidm fe_https.map: | - au-syd1-pve.main.unkin.net be_ausyd1pve_web - au-syd1-pve-api.main.unkin.net be_ausyd1pve_api sonarr.main.unkin.net be_sonarr radarr.main.unkin.net be_radarr lidarr.main.unkin.net be_lidarr @@ -98,7 +94,6 @@ data: mode http description Global HTTPS Frontend acl acl-letsencrypt path_beg /.well-known/acme-challenge/ - acl acl_ausyd1pve req.hdr(host) -i au-syd1-pve.main.unkin.net acl acl_sonarr req.hdr(host) -i sonarr.main.unkin.net acl acl_radarr req.hdr(host) -i radarr.main.unkin.net acl acl_lidarr req.hdr(host) -i lidarr.main.unkin.net @@ -114,11 +109,8 @@ data: acl acl_stalwart_webadmin req.hdr(host) -i autoconfig.main.unkin.net acl acl_stalwart_webadmin req.hdr(host) -i autodiscovery.main.unkin.net acl acl_kanidm req.hdr(host) -i auth.unkin.net - acl acl_internalsubnets src 198.18.0.0/16 10.10.12.0/24 http-request set-header X-Forwarded-Proto https http-request set-header X-Real-IP %[src] - http-request deny if { hdr_dom(host) -i au-syd1-pve.main.unkin.net } !acl_internalsubnets - http-response set-header X-Frame-Options DENY if acl_ausyd1pve http-response set-header X-Frame-Options DENY if acl_sonarr http-response set-header X-Frame-Options DENY if acl_radarr http-response set-header X-Frame-Options DENY if acl_lidarr @@ -185,31 +177,6 @@ data: tcp-request inspect-delay 5s tcp-request content accept if { req_len 0 } - backend be_ausyd1pve_api - description Backend for au-syd1 pve cluster (API only) - balance roundrobin - http-request set-header X-Forwarded-Port %[dst_port] - http-request add-header X-Forwarded-Proto https if { dst_port 443 } - http-reuse always - option httpchk GET / - option forwardfor - option http-keep-alive - option prefer-last-server - redirect scheme https if !{ ssl_fc } - - backend be_ausyd1pve_web - description Backend for au-syd1 pve cluster (Web) - balance roundrobin - cookie SRVNAME insert indirect nocache - http-request set-header X-Forwarded-Port %[dst_port] - http-request add-header X-Forwarded-Proto https if { dst_port 443 } - http-reuse always - option httpchk GET / - option forwardfor - option http-keep-alive - option prefer-last-server - redirect scheme https if !{ ssl_fc } - backend be_ceph_dashboard description Backend for Ceph Dashboard from Mgr instances balance roundrobin -- 2.47.3