Deliver the cert helpers to the puppet compiler pods

certmanager and sshsignhost are invoked server-side by generate() during catalog compilation but only exist on the legacy VM masters, so compiles on the k8s compilers fail with "No such file or directory".

- Stage a self-contained EL9 python runtime on the shared bins volume
- Render both helpers from their puppet-prod ERB templates on the code volume rather than copying the scripts here
- Mount a name-dispatching launcher at /usr/local/bin/{certmanager,sshsignhost}
- Mount kubernetes-auth configs at /opt/{certmanager,sshsignhost}/config.yaml

Needs terraform-vault #152 applied and the puppet-prod kubernetes-auth PR merged.

apps/base/authentik/cnpg_cluster.yaml

@@ -64,12 +64,8 @@ spec:
       archive_mode: "on"
       archive_timeout: 5min
       dynamic_shared_memory_type: posix
-      effective_cache_size: 1536MB
+      effective_cache_size: 256MB
       full_page_writes: "on"
-      # Replicas report their oldest xmin to the primary, so multi-second reads on
-      # a hot standby stop exhausting max_standby_streaming_delay and being
-      # cancelled. Retained-dead-tuple cost is negligible on a ~155MB database.
-      hot_standby_feedback: "on"
       log_destination: csvlog
       log_directory: /controller/log
       log_filename: postgres
@@ -81,12 +77,7 @@ spec:
       max_parallel_workers: "16"
       max_replication_slots: "16"
       max_worker_processes: "16"
-      # A pg_stat_statements.* parameter is what makes CNPG treat the extension as
+      shared_buffers: 128MB
-      # managed and run CREATE EXTENSION in every database; preloading alone does
-      # not create it.
-      pg_stat_statements.max: "10000"
-      pg_stat_statements.track: top
-      shared_buffers: 512MB
       shared_memory_type: mmap
       ssl_max_protocol_version: TLSv1.3
       ssl_min_protocol_version: TLSv1.3
@@ -95,9 +86,6 @@ spec:
       wal_log_hints: "on"
       wal_receiver_timeout: 5s
       wal_sender_timeout: 5s
-    # CNPG merges this with the libraries it manages itself.
-    shared_preload_libraries:
-      - pg_stat_statements
     syncReplicaElectionConstraint:
       enabled: false
   primaryUpdateMethod: restart
@@ -117,16 +105,13 @@ spec:
     updateInterval: 30
   resources:
     limits:
-      # 500m is a 50ms CFS quota per 100ms period, exhausted by bursts even at
+      cpu: 500m
-      # ~0.01 cores average, so every query pays throttle latency.
-      cpu: "2"
       # 512Mi OOMKilled replicas under load (shared_buffers 128MB +
       # max_connections 200 leave no headroom) — see incident 2026-07-28.
-      # shared_buffers 512MB needs the same headroom multiple, hence 2Gi.
-      memory: 2Gi
-    requests:
-      cpu: 500m
       memory: 1Gi
+    requests:
+      cpu: 50m
+      memory: 512Mi
   smartShutdownTimeout: 180
   startDelay: 3600
   stopDelay: 1800

apps/base/authentik/httproute.yaml

@@ -37,22 +37,6 @@ spec:
       name: authentik
       sectionName: https
   rules:
-    - backendRefs:
-        - group: ""
-          kind: Service
-          name: authentik-server
-          port: 80
-          weight: 1
-      filters:
-        - type: URLRewrite
-          urlRewrite:
-            path:
-              type: ReplaceFullPath
-              replaceFullPath: /application/o/token/
-      matches:
-        - path:
-            type: Exact
-            value: /application/o/token
     - backendRefs:
         - group: ""
           kind: Service
@@ -102,22 +86,6 @@ spec:
       name: authentik-internal
       sectionName: https
   rules:
-    - backendRefs:
-        - group: ""
-          kind: Service
-          name: authentik-server
-          port: 80
-          weight: 1
-      filters:
-        - type: URLRewrite
-          urlRewrite:
-            path:
-              type: ReplaceFullPath
-              replaceFullPath: /application/o/token/
-      matches:
-        - path:
-            type: Exact
-            value: /application/o/token
     - backendRefs:
         - group: ""
           kind: Service

apps/base/authentik/kustomization.yaml

@@ -19,7 +19,6 @@ resources:
   - redis-deployment.yaml
   - redis-pvc.yaml
   - redis-service.yaml
-  - server-vmpodscrape.yaml
   - vaultauth.yaml
   - vaultstaticsecret.yaml
   - vmpodscrape.yaml

apps/base/authentik/server-vmpodscrape.yaml

@@ -1,16 +0,0 @@
----
-# Scrape the authentik server's django_prometheus endpoint (:9300). Picked up
-# by the observability VMAgent (selectAllByDefault).
-apiVersion: operator.victoriametrics.com/v1beta1
-kind: VMPodScrape
-metadata:
-  name: authentik-server
-  namespace: authentik
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: authentik
-      app.kubernetes.io/component: server
-  podMetricsEndpoints:
-    - port: metrics
-      path: /metrics

apps/base/bind-external/kustomization.yaml

@@ -7,5 +7,4 @@ resources:
   - cluster.yaml
   - tsigkey.yaml
   - zones.yaml
-  - records.yaml
   - agent-dns-rolebinding.yaml

apps/base/bind-external/records.yaml

@@ -1,36 +0,0 @@
-# Authoritative delegation records for acme.unkin.net. Without these the zone
-# only holds the operator's seed apex (NS ns1.acme.unkin.net glued to the
-# primary pod IP), which is unroutable off-cluster and goes stale on
-# reschedule. DNSRecords must live in the same namespace as their BindZone.
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: acme-apex-ns
-  namespace: bind-external
-spec:
-  zoneRef: acme-unkin-net
-  # "@" is the zone apex.
-  name: "@"
-  type: NS
-  ttl: 3600
-  values:
-    # Matches the parent delegation in Google Cloud DNS. Out of zone, so the
-    # child needs no glue of its own.
-    - acme-ns1.unkin.net.
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: acme-ns1-a
-  namespace: bind-external
-spec:
-  zoneRef: acme-unkin-net
-  name: ns1
-  type: A
-  ttl: 3600
-  values:
-    # Public address of this cluster's external BIND, same target as
-    # acme-ns1.unkin.net. Resolvers that cached the seeded ns1.acme.unkin.net
-    # NS name must still reach the zone.
-    - 103.216.191.185

apps/base/bind-external/zones.yaml

@@ -17,14 +17,3 @@ spec:
   updateKeyRef: certmanager
   allowTransfer:
     - key certmanager
-  # Published apex NS. acme-ns1 is what the parent delegates to and glues; ns1 is
-  # in-zone, so its address is declared below or a reseed would glue it to the
-  # primary pod IP.
-  nameservers:
-    - acme-ns1.unkin.net.
-    - ns1.acme.unkin.net.
-  records:
-    - name: ns1
-      type: A
-      ttl: 3600
-      values: ["103.216.191.185"]

apps/base/bind-system/deployment.yaml

@@ -21,7 +21,7 @@ spec:
         runAsNonRoot: true
       containers:
         - name: operator
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.3.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.2.6
           args:
             - --metrics-bind-address=:8080
             - --health-probe-bind-address=:8081

apps/base/bind-system/kustomization.yaml

@@ -6,7 +6,7 @@ resources:
   - namespace.yaml
   # CRDs are pulled from the bind-operator repo at the matching tag rather than
   # vendored here, so they never drift from the operator.
-  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.3.0/config/crd/install.yaml
+  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.2.6/config/crd/install.yaml
   - rbac.yaml
   - agent-dns-rbac.yaml
   - deployment.yaml

apps/base/cert-manager/certificate_wildcard-ceph-unkin-net.yaml

@@ -1,26 +0,0 @@
----
-# Let's Encrypt *.ceph.unkin.net wildcard for the haproxy edge (ceph dashboard).
-# DNS-01 needs the delegated _acme-challenge.ceph.unkin.net CNAME in the public
-# unkin.net zone.
-# _acme-challenge.ceph.unkin.net. CNAME _acme-challenge.ceph.acme.unkin.net.
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: wildcard-ceph-unkin-net
-  namespace: cert-manager
-spec:
-  secretName: wildcard-ceph-unkin-net-tls
-  secretTemplate:
-    annotations:
-      reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
-      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy"
-      reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
-      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy"
-  privateKey:
-    size: 4096
-  dnsNames:
-    - "*.ceph.unkin.net"
-  issuerRef:
-    name: letsencrypt
-    kind: ClusterIssuer
-    group: cert-manager.io

apps/base/cert-manager/certificate_wildcard-main-unkin-net.yaml

@@ -1,26 +0,0 @@
----
-# Let's Encrypt *.main.unkin.net wildcard for the haproxy edge (pve, arr stack,
-# jellyfin, stalwart webadmin/autoconfig). DNS-01 needs the delegated
-# _acme-challenge.main.unkin.net CNAME in the public unkin.net zone.
-# _acme-challenge.main.unkin.net. CNAME _acme-challenge.main.acme.unkin.net.
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: wildcard-main-unkin-net
-  namespace: cert-manager
-spec:
-  secretName: wildcard-main-unkin-net-tls
-  secretTemplate:
-    annotations:
-      reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
-      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy"
-      reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
-      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy"
-  privateKey:
-    size: 4096
-  dnsNames:
-    - "*.main.unkin.net"
-  issuerRef:
-    name: letsencrypt
-    kind: ClusterIssuer
-    group: cert-manager.io

apps/base/cert-manager/certificate_wildcard-unkin-net.yaml

@@ -14,9 +14,9 @@ spec:
   secretTemplate:
     annotations:
       reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
-      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner"
       reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
-      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner"
   privateKey:
     size: 4096
   dnsNames:

apps/base/cert-manager/kustomization.yaml

@@ -12,5 +12,3 @@ resources:
   - clusterissuer_letsencrypt.yaml
   - clusterissuer_letsencrypt-staging.yaml
   - certificate_wildcard-unkin-net.yaml
-  - certificate_wildcard-main-unkin-net.yaml
-  - certificate_wildcard-ceph-unkin-net.yaml

apps/base/cheeztv/plugin-configmap.yaml

@@ -26,7 +26,7 @@ data:
           </key>
           <value>
             <PluginConfiguration>
-              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
               <OidClientId>jellyfin</OidClientId>
               <OidSecret>@@CLIENT_SECRET@@</OidSecret>
               <Enabled>true</Enabled>

apps/base/cheeztv/service.yaml

@@ -13,4 +13,6 @@ spec:
       targetPort: http
   selector:
     app: cheeztv
+  # Pin each client to one replica to reduce transcode-session churn/takeover.
+  sessionAffinity: ClientIP
   type: ClusterIP

apps/base/cheeztv/statefulset.yaml

@@ -4,8 +4,6 @@ kind: StatefulSet
 metadata:
   name: cheeztv
   namespace: cheeztv
-  annotations:
-    configmap.reloader.stakater.com/auto: "true"
 spec:
   # HA: two replicas coordinate transcode session ownership through Valkey and
   # resume each other's HLS segments off the shared RWX transcode PVC. Stable
@@ -164,7 +162,7 @@ spec:
               readOnly: true
       containers:
         - name: cheeztv
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.4.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

apps/base/fafflix/plugin-configmap.yaml

@@ -26,7 +26,7 @@ data:
           </key>
           <value>
             <PluginConfiguration>
-              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
               <OidClientId>jellyfin</OidClientId>
               <OidSecret>@@CLIENT_SECRET@@</OidSecret>
               <Enabled>true</Enabled>

apps/base/fafflix/service.yaml

@@ -13,4 +13,6 @@ spec:
       targetPort: http
   selector:
     app: fafflix
+  # Pin each client to one replica to reduce transcode-session churn/takeover.
+  sessionAffinity: ClientIP
   type: ClusterIP

apps/base/fafflix/statefulset.yaml

@@ -4,8 +4,6 @@ kind: StatefulSet
 metadata:
   name: fafflix
   namespace: fafflix
-  annotations:
-    configmap.reloader.stakater.com/auto: "true"
 spec:
   # HA: two replicas coordinate transcode session ownership through Valkey and
   # resume each other's HLS segments off the shared RWX transcode PVC. Stable
@@ -164,7 +162,7 @@ spec:
               readOnly: true
       containers:
         - name: fafflix
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.4.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

apps/base/grafana/grafanadatasource.yaml

@@ -20,22 +20,3 @@ spec:
     jsonData:
       timeInterval: "15s"
       httpMethod: "POST"
----
-apiVersion: grafana.integreatly.org/v1beta1
-kind: GrafanaDatasource
-metadata:
-  name: victorialogs
-  namespace: grafana
-spec:
-  instanceSelector:
-    matchLabels:
-      dashboards: "grafana"
-  plugins:
-    - name: victoriametrics-logs-datasource
-      version: 0.32.0
-  datasource:
-    name: "VictoriaLogs"
-    type: "victoriametrics-logs-datasource"
-    uid: "victorialogs"
-    access: "proxy"
-    url: "http://vlselect-logs.logging.svc.cluster.local:9471"

apps/base/haproxy/configmap.yaml

@@ -1,274 +0,0 @@
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: haproxy-config
-  namespace: haproxy
-data:
-  certificate.list: |
-    # First entry is the default cert for non-matching SNI.
-    /etc/haproxy/certs/unkin-net/tls.crt
-    /etc/haproxy/certs/main-unkin-net/tls.crt
-    /etc/haproxy/certs/ceph-unkin-net/tls.crt
-
-  fe_https.map: |
-    sonarr.main.unkin.net be_sonarr
-    radarr.main.unkin.net be_radarr
-    lidarr.main.unkin.net be_lidarr
-    readarr.main.unkin.net be_readarr
-    prowlarr.main.unkin.net be_prowlarr
-    nzbget.main.unkin.net be_nzbget
-    jellyfin.main.unkin.net be_jellyfin
-    fafflix.unkin.net be_jellyfin
-    git.unkin.net be_gitea
-    grafana.unkin.net be_grafana
-    dashboard.ceph.unkin.net be_ceph_dashboard
-    auth.unkin.net be_k8s_kanidm
-
-  haproxy.cfg: |
-    global
-      log stdout format raw local0
-      log stdout format raw local1 notice
-      maxconn  4000
-      hard-stop-after  2m
-      ssl-default-bind-ciphers  EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH
-      ssl-default-bind-options  ssl-min-ver TLSv1.2 ssl-max-ver TLSv1.3
-      ssl-default-server-ciphers  kEECDH+aRSA+AES:kRSA+AES:+AES256:RC4-SHA:!kEDH:!LOW:!EXP:!MD5:!aNULL:!eNULL
-      ssl-default-server-options  no-sslv3
-      stats  timeout 30s
-      stats  socket /var/lib/haproxy/stats
-      stats  socket /var/lib/haproxy/admin.sock mode 660 level admin
-      tune.ssl.default-dh-param  2048
-
-    defaults
-      log  global
-      maxconn  5000
-      mode  http
-      option  httplog
-      option  dontlognull
-      option  http-server-close
-      option  forwardfor except 127.0.0.0/8
-      option  redispatch
-      retries  3
-      stats  enable
-      timeout  http-request 10s
-      timeout  queue 1m
-      timeout  connect 10s
-      timeout  client 5m
-      timeout  server 5m
-      timeout  http-keep-alive 10s
-      timeout  check 10s
-
-    frontend fe_https
-      bind 0.0.0.0:443 ssl crt-list /usr/local/etc/haproxy/certificate.list ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH force-tlsv12
-      mode http
-      description Global HTTPS Frontend
-      http-request set-header X-Forwarded-Proto https
-      http-request set-header X-Real-IP %[src]
-      http-response set-header X-Content-Type-Options nosniff
-      http-response set-header X-XSS-Protection 1;mode=block
-      use_backend %[req.hdr(host),lower,map(/usr/local/etc/haproxy/fe_https.map,be_default)]
-
-    frontend fe_metrics
-      bind 0.0.0.0:8405
-      mode http
-      description Metrics Frontend
-      http-request set-header X-Forwarded-Proto https
-      http-request set-header X-Real-IP %[src]
-      http-request use-service prometheus-exporter if { path /metrics }
-
-    backend be_ceph_dashboard
-      description Backend for Ceph Dashboard from Mgr instances
-      balance roundrobin
-      cookie SRVNAME insert indirect nocache
-      http-check expect status 200
-      http-request set-header X-Forwarded-Port %[dst_port]
-      http-request add-header X-Forwarded-Proto https if { dst_port 9443 }
-      http-reuse always
-      option httpchk GET /
-      option forwardfor
-      option http-keep-alive
-      option prefer-last-server
-      redirect scheme https if !{ ssl_fc }
-      stick-table type ip size 200k expire 30m
-      server prodnxsr0009 198.18.23.9:9443 check cookie prodnxsr0009 fall 2 inter 2s rise 3 ssl verify none
-      server prodnxsr0010 198.18.23.10:9443 check cookie prodnxsr0010 fall 2 inter 2s rise 3 ssl verify none
-      server prodnxsr0011 198.18.23.11:9443 check cookie prodnxsr0011 fall 2 inter 2s rise 3 ssl verify none
-      server prodnxsr0012 198.18.23.12:9443 check cookie prodnxsr0012 fall 2 inter 2s rise 3 ssl verify none
-      server prodnxsr0013 198.18.23.13:9443 check cookie prodnxsr0013 fall 2 inter 2s rise 3 ssl verify none
-
-    backend be_default
-      description Backend for unmatched HTTP traffic
-      balance roundrobin
-      cookie SRVNAME insert
-      http-request set-header X-Forwarded-Port %[dst_port]
-      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
-      option httpchk GET /
-      option forwardfor
-
-    backend be_gitea
-      description Backend for gitea cluster
-      balance roundrobin
-      cookie SRVNAME insert indirect nocache
-      http-request set-header X-Forwarded-Port %[dst_port]
-      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
-      http-reuse always
-      option httpchk GET /
-      option forwardfor
-      option http-keep-alive
-      option prefer-last-server
-      redirect scheme https if !{ ssl_fc }
-      stick on src
-      stick-table type ip size 200k expire 30m
-      server ausyd1nxvm2080 198.18.26.18:443 check cookie ausyd1nxvm2080 fall 2 inter 2s rise 3 ssl verify none
-      server ausyd1nxvm2081 198.18.27.117:443 check cookie ausyd1nxvm2081 fall 2 inter 2s rise 3 ssl verify none
-      server ausyd1nxvm2082 198.18.28.71:443 check cookie ausyd1nxvm2082 fall 2 inter 2s rise 3 ssl verify none
-
-    backend be_grafana
-      description Backend for grafana nodes
-      balance roundrobin
-      cookie SRVNAME insert indirect nocache
-      http-request set-header X-Forwarded-Port %[dst_port]
-      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
-      http-reuse always
-      option httpchk GET /
-      option forwardfor
-      option http-keep-alive
-      option prefer-last-server
-      redirect scheme https if !{ ssl_fc }
-      stick on src
-      stick-table type ip size 200k expire 30m
-      server ausyd1nxvm2015 198.18.27.2:443 check cookie ausyd1nxvm2015 fall 2 inter 2s rise 3 ssl verify none
-      server ausyd1nxvm2016 198.18.28.189:443 check cookie ausyd1nxvm2016 fall 2 inter 2s rise 3 ssl verify none
-
-    backend be_jellyfin
-      description Backend for au-syd1 jellyfin
-      balance roundrobin
-      cookie SRVNAME insert indirect nocache
-      http-request set-header X-Forwarded-Port %[dst_port]
-      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
-      http-reuse always
-      option httpchk GET /
-      option forwardfor
-      option http-keep-alive
-      option prefer-last-server
-      redirect scheme https if !{ ssl_fc }
-      server ausyd1nxvm2051 198.18.25.164:443 check cookie ausyd1nxvm2051 fall 2 inter 2s rise 3 ssl verify none
-
-    backend be_k8s_kanidm
-      description Backend for Kanidm (auth.unkin.net via Kubernetes internal Traefik)
-      balance roundrobin
-      http-reuse always
-      http-request set-header X-Forwarded-Port %[dst_port]
-      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
-      redirect scheme https if !{ ssl_fc }
-      option httpchk
-      option forwardfor
-      option http-keep-alive
-      option prefer-last-server
-      http-check connect ssl sni auth.unkin.net
-      http-check send meth GET uri /status ver HTTP/1.1 hdr Host auth.unkin.net
-      http-check expect status 200
-      server k8s-traefik-internal 198.18.200.4:443 ssl verify none check inter 2s rise 3 fall 2 sni str(auth.unkin.net)
-
-    backend be_lidarr
-      description Backend for au-syd1 lidarr
-      balance roundrobin
-      cookie SRVNAME insert indirect nocache
-      http-request set-header X-Forwarded-Port %[dst_port]
-      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
-      http-reuse always
-      option httpchk GET /consul/health
-      option forwardfor
-      option http-keep-alive
-      option prefer-last-server
-      redirect scheme https if !{ ssl_fc }
-      server ausyd1nxvm2048 198.18.28.165:443 check cookie ausyd1nxvm2048 fall 2 inter 2s rise 3 ssl verify none
-
-    backend be_nzbget
-      description Backend for au-syd1 nzbget
-      balance roundrobin
-      cookie SRVNAME insert indirect nocache
-      http-request set-header X-Forwarded-Port %[dst_port]
-      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
-      http-reuse always
-      option httpchk GET /consul/health
-      option forwardfor
-      option http-keep-alive
-      option prefer-last-server
-      redirect scheme https if !{ ssl_fc }
-      server ausyd1nxvm2045 198.18.25.44:443 check cookie ausyd1nxvm2045 fall 2 inter 2s rise 3 ssl verify none
-
-    backend be_prowlarr
-      description Backend for au-syd1 prowlarr
-      balance roundrobin
-      cookie SRVNAME insert indirect nocache
-      http-request set-header X-Forwarded-Port %[dst_port]
-      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
-      http-reuse always
-      option httpchk GET /consul/health
-      option forwardfor
-      option http-keep-alive
-      option prefer-last-server
-      redirect scheme https if !{ ssl_fc }
-      server ausyd1nxvm2050 198.18.25.66:443 check cookie ausyd1nxvm2050 fall 2 inter 2s rise 3 ssl verify none
-
-    backend be_radarr
-      description Backend for au-syd1 radarr
-      balance roundrobin
-      cookie SRVNAME insert indirect nocache
-      http-request set-header X-Forwarded-Port %[dst_port]
-      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
-      http-reuse always
-      option httpchk GET /consul/health
-      option forwardfor
-      option http-keep-alive
-      option prefer-last-server
-      redirect scheme https if !{ ssl_fc }
-      server ausyd1nxvm2047 198.18.27.131:443 check cookie ausyd1nxvm2047 fall 2 inter 2s rise 3 ssl verify none
-
-    backend be_readarr
-      description Backend for au-syd1 readarr
-      balance roundrobin
-      cookie SRVNAME insert indirect nocache
-      http-request set-header X-Forwarded-Port %[dst_port]
-      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
-      http-reuse always
-      option httpchk GET /consul/health
-      option forwardfor
-      option http-keep-alive
-      option prefer-last-server
-      redirect scheme https if !{ ssl_fc }
-      server ausyd1nxvm2049 198.18.29.32:443 check cookie ausyd1nxvm2049 fall 2 inter 2s rise 3 ssl verify none
-
-    backend be_sonarr
-      description Backend for au-syd1 sonarr
-      balance roundrobin
-      cookie SRVNAME insert indirect nocache
-      http-request set-header X-Forwarded-Port %[dst_port]
-      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
-      http-reuse always
-      option httpchk GET /consul/health
-      option forwardfor
-      option http-keep-alive
-      option prefer-last-server
-      redirect scheme https if !{ ssl_fc }
-      server ausyd1nxvm2046 198.18.26.161:443 check cookie ausyd1nxvm2046 fall 2 inter 2s rise 3 ssl verify none
-
-    # The `peers au-syd1-prod` section is dropped: peer names must be static and a
-    # Deployment cannot provide them. Behind the external Traefik's TLS
-    # passthrough `src` is a Traefik pod, so X-Real-IP, forwardfor and the
-    # `stick on src` tables all key on that; the SRVNAME cookie carries real
-    # session persistence. Traefik cannot emit PROXY protocol to a TLSRoute
-    # backend, so there is nothing to bind `accept-proxy` to.
-
-    listen health
-      bind 0.0.0.0:8404
-      mode http
-      monitor-uri /healthz
-
-    listen stats
-      bind 127.0.0.1:9090
-      mode http
-      stats uri /
-      stats auth admin:admin

apps/base/haproxy/deployment.yaml

@@ -1,148 +0,0 @@
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: haproxy
-  namespace: haproxy
-  annotations:
-    reloader.stakater.com/auto: "true"
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: haproxy
-  strategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-  template:
-    metadata:
-      labels:
-        app: haproxy
-    spec:
-      automountServiceAccountToken: false
-      terminationGracePeriodSeconds: 150
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchLabels:
-                  app: haproxy
-              topologyKey: kubernetes.io/hostname
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 99
-        runAsGroup: 99
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: haproxy
-          image: haproxy:3.2.24-alpine
-          imagePullPolicy: IfNotPresent
-          command:
-            - haproxy
-            - -W
-            - -db
-            - -f
-            - /usr/local/etc/haproxy/haproxy.cfg
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: [ALL]
-              # fe_https binds the privileged port 443 as uid 99, and the
-              # dst_port ACLs need the real port.
-              add: [NET_BIND_SERVICE]
-          ports:
-            - name: https
-              containerPort: 443
-              protocol: TCP
-            - name: health
-              containerPort: 8404
-              protocol: TCP
-            - name: metrics
-              containerPort: 8405
-              protocol: TCP
-            - name: stats
-              containerPort: 9090
-              protocol: TCP
-          lifecycle:
-            preStop:
-              exec:
-                # SIGUSR1 to the master soft-stops the workers; hard-stop-after
-                # caps the drain. Wait so kubelet holds SIGTERM until it is done.
-                command:
-                  - /bin/sh
-                  - -c
-                  - kill -s USR1 1; while kill -0 1 2>/dev/null; do sleep 1; done
-          livenessProbe:
-            httpGet:
-              path: /healthz
-              port: health
-            initialDelaySeconds: 15
-            periodSeconds: 30
-            timeoutSeconds: 5
-            failureThreshold: 3
-          readinessProbe:
-            httpGet:
-              path: /healthz
-              port: health
-            initialDelaySeconds: 5
-            periodSeconds: 5
-            timeoutSeconds: 5
-            failureThreshold: 3
-          resources:
-            requests:
-              cpu: 200m
-              memory: 256Mi
-            limits:
-              cpu: 2
-              memory: 1Gi
-          volumeMounts:
-            - name: config
-              mountPath: /usr/local/etc/haproxy
-              readOnly: true
-            - name: cert-unkin-net
-              mountPath: /etc/haproxy/certs/unkin-net
-              readOnly: true
-            - name: cert-main-unkin-net
-              mountPath: /etc/haproxy/certs/main-unkin-net
-              readOnly: true
-            - name: cert-ceph-unkin-net
-              mountPath: /etc/haproxy/certs/ceph-unkin-net
-              readOnly: true
-            - name: run
-              mountPath: /var/lib/haproxy
-      volumes:
-        - name: config
-          configMap:
-            name: haproxy-config
-        # ssl-load-extra-files loads <crtfile>.key by default, so the key is
-        # projected next to the cert as tls.crt.key.
-        - name: cert-unkin-net
-          secret:
-            secretName: wildcard-unkin-net-tls
-            items:
-              - key: tls.crt
-                path: tls.crt
-              - key: tls.key
-                path: tls.crt.key
-        - name: cert-main-unkin-net
-          secret:
-            secretName: wildcard-main-unkin-net-tls
-            items:
-              - key: tls.crt
-                path: tls.crt
-              - key: tls.key
-                path: tls.crt.key
-        - name: cert-ceph-unkin-net
-          secret:
-            secretName: wildcard-ceph-unkin-net-tls
-            items:
-              - key: tls.crt
-                path: tls.crt
-              - key: tls.key
-                path: tls.crt.key
-        - name: run
-          emptyDir: {}
-      restartPolicy: Always

apps/base/haproxy/gateway.yaml

@@ -1,31 +0,0 @@
----
-# External (DMZ) front for the haproxy edge on the traefik-external LB VIP
-# 198.18.199.0. The :443 listener is TLS Passthrough: haproxy owns the three
-# wildcard certs and terminates behind Traefik, so there are no certificateRefs
-# here. Listener hostnames are deliberately unset and the routes carry the
-# explicit hostname list instead; allowedRoutes Same keeps other namespaces off
-# these listeners.
-apiVersion: gateway.networking.k8s.io/v1
-kind: Gateway
-metadata:
-  name: haproxy
-  namespace: haproxy
-  labels:
-    traefik.io/instance: external
-spec:
-  gatewayClassName: traefik-external
-  listeners:
-    - name: http
-      port: 80
-      protocol: HTTP
-      allowedRoutes:
-        namespaces:
-          from: Same
-    - name: https-passthrough
-      port: 443
-      protocol: TLS
-      tls:
-        mode: Passthrough
-      allowedRoutes:
-        namespaces:
-          from: Same

apps/base/haproxy/httproute.yaml

@@ -1,37 +0,0 @@
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: haproxy-http-redirect
-  namespace: haproxy
-  labels:
-    app: haproxy
-spec:
-  hostnames:
-    - sonarr.main.unkin.net
-    - radarr.main.unkin.net
-    - lidarr.main.unkin.net
-    - readarr.main.unkin.net
-    - prowlarr.main.unkin.net
-    - nzbget.main.unkin.net
-    - jellyfin.main.unkin.net
-    - fafflix.unkin.net
-    - git.unkin.net
-    - grafana.unkin.net
-    - dashboard.ceph.unkin.net
-    - auth.unkin.net
-  parentRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: haproxy
-      sectionName: http
-  rules:
-    - filters:
-        - type: RequestRedirect
-          requestRedirect:
-            scheme: https
-            statusCode: 301
-      matches:
-        - path:
-            type: PathPrefix
-            value: /

apps/base/haproxy/kustomization.yaml

@@ -1,15 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - namespace.yaml
-  - configmap.yaml
-  - deployment.yaml
-  - service.yaml
-  - gateway.yaml
-  - tlsroute.yaml
-  - httproute.yaml
-  - pdb.yaml
-  - vpa.yaml
-  - vmpodscrape.yaml

apps/base/haproxy/namespace.yaml

@@ -1,5 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: haproxy

apps/base/haproxy/pdb.yaml

@@ -1,11 +0,0 @@
----
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
-  name: haproxy
-  namespace: haproxy
-spec:
-  maxUnavailable: 1
-  selector:
-    matchLabels:
-      app: haproxy

apps/base/haproxy/service.yaml

@@ -1,19 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: haproxy
-  namespace: haproxy
-spec:
-  type: ClusterIP
-  # Reached only by the external Traefik's TLS-passthrough TLSRoute, so the
-  # peer address here is a Traefik pod, not the client. sessionAffinity is
-  # deliberately absent: keyed on ClientIP it would pin whole Traefik pods,
-  # not clients. Backend persistence rests on the per-backend SRVNAME cookie.
-  selector:
-    app: haproxy
-  ports:
-    - name: https
-      port: 443
-      protocol: TCP
-      targetPort: https

apps/base/haproxy/tlsroute.yaml

@@ -1,34 +0,0 @@
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: TLSRoute
-metadata:
-  name: haproxy
-  namespace: haproxy
-  labels:
-    app: haproxy
-spec:
-  hostnames:
-    - sonarr.main.unkin.net
-    - radarr.main.unkin.net
-    - lidarr.main.unkin.net
-    - readarr.main.unkin.net
-    - prowlarr.main.unkin.net
-    - nzbget.main.unkin.net
-    - jellyfin.main.unkin.net
-    - fafflix.unkin.net
-    - git.unkin.net
-    - grafana.unkin.net
-    - dashboard.ceph.unkin.net
-    - auth.unkin.net
-  parentRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: haproxy
-      sectionName: https-passthrough
-  rules:
-    - backendRefs:
-        - group: ""
-          kind: Service
-          name: haproxy
-          port: 443
-          weight: 1

apps/base/haproxy/vmpodscrape.yaml

@@ -1,13 +0,0 @@
----
-apiVersion: operator.victoriametrics.com/v1beta1
-kind: VMPodScrape
-metadata:
-  name: haproxy
-  namespace: haproxy
-spec:
-  selector:
-    matchLabels:
-      app: haproxy
-  podMetricsEndpoints:
-    - port: metrics
-      path: /metrics

apps/base/haproxy/vpa.yaml

@@ -1,13 +0,0 @@
----
-apiVersion: autoscaling.k8s.io/v1
-kind: VerticalPodAutoscaler
-metadata:
-  name: haproxy-vpa
-  namespace: haproxy
-spec:
-  targetRef:
-    apiVersion: apps/v1
-    kind: Deployment
-    name: haproxy
-  updatePolicy:
-    updateMode: "Off"

apps/base/logging/gateway.yaml

@@ -1,13 +1,16 @@
 ---
-# Log ingestion endpoint for puppet-managed VMs (and any non-k8s client):
+# Log ingestion endpoint for puppet-managed VMs (and any non-k8s client).
-# fronts the VLCluster vlinsert service over TLS at a name VMs can resolve.
+# Reuses the internal Traefik gateway + cert-manager + external-dns pattern so
+# VMs reach the Vector aggregator's HTTP source over TLS at a DNS name they can
+# resolve. The puppet-side Vector rollout ships NDJSON to
+# https://logs-ingest.k8s.syd1.au.unkin.net/ (a later task).
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
   name: logs-ingest
   namespace: logging
   labels:
-    app.kubernetes.io/name: victorialogs
+    app.kubernetes.io/name: vector-aggregator
     app.kubernetes.io/component: ingest
     traefik.io/instance: internal
   annotations:

apps/base/logging/httproute.yaml

@@ -5,7 +5,7 @@ metadata:
   name: logs-ingest-http-redirect
   namespace: logging
   labels:
-    app.kubernetes.io/name: victorialogs
+    app.kubernetes.io/name: vector-aggregator
     app.kubernetes.io/component: ingest
 spec:
   hostnames:
@@ -32,7 +32,7 @@ metadata:
   name: logs-ingest
   namespace: logging
   labels:
-    app.kubernetes.io/name: victorialogs
+    app.kubernetes.io/name: vector-aggregator
     app.kubernetes.io/component: ingest
 spec:
   hostnames:
@@ -46,8 +46,8 @@ spec:
     - backendRefs:
         - group: ""
           kind: Service
-          name: vlinsert-logs
+          name: vector-vm-ingest
-          port: 9481
+          port: 8080
           weight: 1
       matches:
         - path:

apps/base/logging/kustomization.yaml

@@ -10,7 +10,6 @@ resources:
   - job_clickhouse-schema.yaml
   - nats-bootstrap-job.yaml
   - cephrgw.yaml
-  - vlcluster.yaml
   - gateway.yaml
   - httproute.yaml
   - serviceaccount_logarchiver.yaml

apps/base/logging/vlcluster.yaml

@@ -1,47 +0,0 @@
----
-apiVersion: operator.victoriametrics.com/v1
-kind: VLCluster
-metadata:
-  name: logs
-  namespace: logging
-spec:
-  clusterVersion: v1.52.0
-  vlinsert:
-    replicaCount: 2
-    resources:
-      requests:
-        cpu: 500m
-        memory: 1Gi
-      limits:
-        cpu: "2"
-        memory: 4Gi
-  vlselect:
-    replicaCount: 2
-    resources:
-      requests:
-        cpu: 500m
-        memory: 1Gi
-      limits:
-        cpu: "2"
-        memory: 4Gi
-  vlstorage:
-    replicaCount: 3
-    retentionPeriod: 180d
-    # ~3 GiB/day measured; 220GiB/node cap keeps 180d time-based, not disk-bound
-    retentionMaxDiskSpaceUsageBytes: 220GiB
-    storage:
-      volumeClaimTemplate:
-        spec:
-          accessModes:
-            - ReadWriteOnce
-          storageClassName: cephrbd-fast-delete
-          resources:
-            requests:
-              storage: 250Gi
-    resources:
-      requests:
-        cpu: "1"
-        memory: 2Gi
-      limits:
-        cpu: "4"
-        memory: 8Gi

apps/base/puppet/cronjob_g10k-code.yaml

@@ -11,13 +11,11 @@ metadata:
   namespace: puppet
 spec:
   schedule: "*/1 * * * *"
-  startingDeadlineSeconds: 200
   concurrencyPolicy: Forbid
   successfulJobsHistoryLimit: 3
   failedJobsHistoryLimit: 3
   jobTemplate:
     spec:
-      activeDeadlineSeconds: 300
       template:
         metadata:
           labels:

apps/base/puppet/deployment_puppetserver-compiler.yaml

@@ -99,23 +99,24 @@ spec:
         - mountPath: /docker-custom-entrypoint.d/post-startup/additional-ruby-gems.sh
           name: additional-ruby-gems
           subPath: additional-ruby-gems.sh
+        - mountPath: /usr/local/bin/certmanager
+          name: cert-helpers
+          subPath: vault-helper
+        - mountPath: /usr/local/bin/sshsignhost
+          name: cert-helpers
+          subPath: vault-helper
+        - mountPath: /opt/certmanager/config.yaml
+          name: cert-helpers
+          subPath: certmanager-config.yaml
+        - mountPath: /opt/sshsignhost/config.yaml
+          name: cert-helpers
+          subPath: sshsignhost-config.yaml
         - mountPath: /configmaps/auth.conf
           name: compiler-auth-conf
           subPath: auth.conf
         - mountPath: /docker-custom-entrypoint.d/pre-default/10-auth-conf.sh
           name: compiler-auth-conf-seed
           subPath: 10-auth-conf.sh
-        - mountPath: /docker-custom-entrypoint.d/pre-default/20-vault-helpers.sh
-          name: compiler-vault-helpers-seed
-          subPath: 20-vault-helpers.sh
-        - mountPath: /opt/certmanager/config.yaml
-          name: certmanager-config
-          subPath: certmanager.yaml
-          readOnly: true
-        - mountPath: /opt/sshsignhost/config.yaml
-          name: sshsignhost-config
-          subPath: sshsignhost.yaml
-          readOnly: true
       initContainers:
       - name: copy-configmaps
         image: busybox:1.35
@@ -213,41 +214,67 @@ spec:
           echo "$EXPECTED  encapic" | sha256sum -c -
           install -m 0755 encapic /opt/bin/encapic
 
-          # Puppet shells out to these two from generate() during catalog
-          # compilation: profiles::pki::vault runs certmanager and
-          # profiles::ssh::sign runs sshsignhost.
-          install_release() {
-            name=$1
-            version=$2
-            asset="$name-linux-amd64"
-            base="https://git.unkin.net/unkin/$name/releases/download/$version"
-            curl -fsSL -o "$name" "$base/$asset"
-            curl -fsSL -o "$name.checksums" "$base/checksums.txt"
-            # checksums.txt covers every release asset; pick the line for the
-            # one we downloaded and verify it under our local filename.
-            expected=$(awk -v a="$asset" '$NF == a || $NF == "*"a {print $1}' "$name.checksums")
-            if [ -z "$expected" ]; then
-              echo "no checksum for $asset in $version checksums.txt" >&2
-              exit 1
-            fi
-            echo "$expected  $name" | sha256sum -c -
-            install -m 0755 "$name" "/opt/bin/$name"
-          }
-
-          install_release certmanager v0.2.0
-          install_release sshsignhost v0.1.0
-
           echo "Shared binaries setup completed"
-        resources:
-          limits:
-            cpu: 300m
-            memory: 256Mi
-          requests:
-            cpu: 100m
-            memory: 64Mi
         volumeMounts:
         - mountPath: /opt/bin/
           name: puppet-shared-bins
+
+      - name: setup-cert-helpers
+        image: git.unkin.net/unkin/almalinux9-base:20260606
+        command:
+        - sh
+        - -c
+        args:
+        - |
+          set -e
+          CH=/opt/bin/certhelpers
+          PYROOT=$CH/py-el9-1
+          TPL=/etc/puppetlabs/code/environments/develop/site/profiles/templates/helpers
+          mkdir -p "$CH"
+
+          # The helpers are python3 (requests, pyyaml) and openvoxserver ships
+          # no python, so stage a self-contained EL9 tree once per volume.
+          if [ ! -f "$PYROOT/.ready" ]; then
+            echo "Staging python runtime for the cert helpers..."
+            TMP=$CH/.py-el9-1.$$
+            rm -rf "$TMP"
+            dnf -y --installroot="$TMP" --releasever=9 --nodocs \
+              --setopt=install_weak_deps=0 --disablerepo=unkin install \
+              python3 python3-requests python3-pyyaml python3-six
+            rm -rf "$TMP/var/cache" "$TMP/var/lib/dnf" "$TMP/var/lib/rpm" \
+              "$TMP/usr/share/locale"
+            # Both hardcode EL absolute paths that only exist inside the tree.
+            SP=$TMP/usr/lib/python3.9/site-packages
+            ln -sfn ../../six.py "$SP/urllib3/packages/six.py"
+            sed -i "s|'/etc/pki/tls/certs/ca-bundle.crt'|'$PYROOT/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem'|" \
+              "$SP/requests/certs.py"
+            touch "$TMP/.ready"
+            mv -T "$TMP" "$PYROOT" || rm -rf "$TMP"
+          fi
+
+          # Render from the puppet-prod ERB templates on the code volume so this
+          # repo never carries a second copy of the scripts.
+          for n in certmanager sshsignhost; do
+            sed -e "s|<%= @venv_path %>|$PYROOT/usr|g" \
+                -e "s|<%= @config_path %>|/opt/$n/config.yaml|g" \
+                "$TPL/$n.erb" > "$CH/.$n.$$"
+            chmod 0755 "$CH/.$n.$$"
+            mv "$CH/.$n.$$" "$CH/$n"
+          done
+          echo "Cert helpers setup completed"
+        resources:
+          limits:
+            cpu: 1
+            memory: 1Gi
+          requests:
+            cpu: 200m
+            memory: 256Mi
+        volumeMounts:
+        - mountPath: /opt/bin/
+          name: puppet-shared-bins
+        - mountPath: /etc/puppetlabs/code/
+          name: puppet-code-volume
+          readOnly: true
       securityContext:
         fsGroup: 999
         seccompProfile:
@@ -282,6 +309,19 @@ spec:
         configMap:
           name: additional-ruby-gems
           defaultMode: 0755
+      - name: cert-helpers
+        configMap:
+          name: cert-helpers
+          items:
+          - key: vault-helper
+            path: vault-helper
+            mode: 0755
+          - key: certmanager-config.yaml
+            path: certmanager-config.yaml
+            mode: 0444
+          - key: sshsignhost-config.yaml
+            path: sshsignhost-config.yaml
+            mode: 0444
       - name: compiler-auth-conf
         configMap:
           name: compiler-auth.conf
@@ -289,15 +329,5 @@ spec:
         configMap:
           name: compiler-auth-conf-seed
           defaultMode: 0755
-      - name: compiler-vault-helpers-seed
-        configMap:
-          name: compiler-vault-helpers-seed
-          defaultMode: 0755
-      - name: certmanager-config
-        configMap:
-          name: certmanager-config
-      - name: sshsignhost-config
-        configMap:
-          name: sshsignhost-config
   strategy:
     type: RollingUpdate

apps/base/puppet/kustomization.yaml

@@ -64,23 +64,15 @@ configMapGenerator:
       - resources/compiler/10-auth-conf.sh
     options:
       disableNameSuffixHash: true
-  - name: compiler-vault-helpers-seed
-    files:
-      - resources/compiler/20-vault-helpers.sh
-    options:
-      disableNameSuffixHash: true
-  - name: certmanager-config
-    files:
-      - resources/compiler/certmanager.yaml
-    options:
-      disableNameSuffixHash: true
-  - name: sshsignhost-config
-    files:
-      - resources/compiler/sshsignhost.yaml
-    options:
-      disableNameSuffixHash: true
   - name: additional-ruby-gems
     files:
       - resources/additional-ruby-gems.sh
     options:
       disableNameSuffixHash: true
+  - name: cert-helpers
+    files:
+      - resources/cert-helpers/vault-helper
+      - resources/cert-helpers/certmanager-config.yaml
+      - resources/cert-helpers/sshsignhost-config.yaml
+    options:
+      disableNameSuffixHash: true

apps/base/puppet/resources/additional-ruby-gems.sh

@@ -6,6 +6,4 @@ echo "Installing additional Ruby gems..."
 /opt/puppetlabs/puppet/bin/gem install ipaddr
 /opt/puppetlabs/puppet/bin/gem install hiera-eyaml
 /opt/puppetlabs/puppet/bin/gem install toml
-# Under set -e a failed install kills the entrypoint post-startup hooks, taking down an already-serving compiler.
-/opt/puppetlabs/bin/puppetserver gem install toml
 echo "Additional Ruby gems installed successfully"

apps/base/puppet/resources/cert-helpers/certmanager-config.yaml

@@ -0,0 +1,12 @@
+---
+# profiles::helpers::certmanager::vault_config, with kubernetes auth: the
+# certmanager approle is CIDR-bound to the legacy VM masters.
+vault:
+  addr: 'https://vault.service.consul:8200'
+  auth_method: 'kubernetes'
+  k8s_mount: 'k8s/au/syd1'
+  k8s_role: 'puppet_certmanager'
+  jwt_path: '/var/run/secrets/kubernetes.io/serviceaccount/token'
+  mount_point: 'pki_int'
+  role_name: 'servers_default'
+output_path: '/tmp/certmanager'

apps/base/puppet/resources/cert-helpers/sshsignhost-config.yaml

@@ -0,0 +1,11 @@
+---
+# profiles::helpers::sshsignhost::vault_config, with kubernetes auth.
+vault:
+  addr: 'https://vault.service.consul:8200'
+  auth_method: 'kubernetes'
+  k8s_mount: 'k8s/au/syd1'
+  k8s_role: 'puppet_sshsigner'
+  jwt_path: '/var/run/secrets/kubernetes.io/serviceaccount/token'
+  mount_point: 'ssh-host-signer'
+  role_name: 'hostrole'
+output_path: '/tmp/sshsignhost'

apps/base/puppet/resources/cert-helpers/vault-helper

@@ -0,0 +1,11 @@
+#!/bin/sh
+# Runs the certmanager/sshsignhost helper matching the name it is invoked as.
+# The openvoxserver image has no python, so the EL9 tree staged on the shared
+# bins volume is started through its own dynamic loader.
+set -eu
+
+PYROOT=/opt/bin/certhelpers/py-el9-1
+
+exec "${PYROOT}/lib64/ld-linux-x86-64.so.2" \
+  --library-path "${PYROOT}/lib64:${PYROOT}/usr/lib64" \
+  "${PYROOT}/usr/bin/python3.9" "/opt/bin/certhelpers/${0##*/}" "$@"

apps/base/puppet/resources/compiler/20-vault-helpers.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-BIN_DIR=/opt/bin
-CA=/opt/vault-ca-cert.crt
-
-if [ ! -s "$CA" ]; then
-  echo "FATAL: $CA missing or empty; certmanager and sshsignhost cannot verify Vault" >&2
-  exit 1
-fi
-
-# profiles::pki::vault and profiles::ssh::sign shell out to fixed /usr/local/bin
-# paths from generate(); the binaries ship on the shared PVC, and /usr/local/bin
-# lives in the image. Wrappers rather than symlinks because neither binary reads
-# a CA path from its config: SSL_CERT_FILE scopes the internal CA to these two
-# processes instead of the puppetserver JVM's own trust store.
-for bin in certmanager sshsignhost; do
-  if [ ! -x "$BIN_DIR/$bin" ]; then
-    echo "FATAL: $BIN_DIR/$bin missing; generate() would abort every catalog compile" >&2
-    exit 1
-  fi
-  cat > "/usr/local/bin/$bin" <<WRAPPER
-#!/bin/sh
-SSL_CERT_FILE=$CA
-export SSL_CERT_FILE
-exec $BIN_DIR/$bin "\$@"
-WRAPPER
-  chmod 0755 "/usr/local/bin/$bin"
-done

apps/base/puppet/resources/compiler/certmanager.yaml

@@ -1,12 +0,0 @@
----
-vault:
-  addr: https://vault.service.consul:8200
-  auth_method: kubernetes
-  k8s_mount: k8s/au/syd1
-  k8s_role: puppet_certmanager
-  jwt_path: /var/run/secrets/kubernetes.io/serviceaccount/token
-  mount_point: pki_int
-  role_name: servers_default
-  output_path: /tmp/certmanager
-  tls_skip_verify: false
-  timeout: 30s

apps/base/puppet/resources/compiler/sshsignhost.yaml

@@ -1,11 +0,0 @@
----
-vault:
-  addr: https://vault.service.consul:8200
-  auth_method: kubernetes
-  k8s_mount: k8s/au/syd1
-  k8s_role: puppet_sshsigner
-  jwt_path: /var/run/secrets/kubernetes.io/serviceaccount/token
-  mount_point: sshca
-  role_name: signhost
-  tls_skip_verify: false
-  timeout: 30s

apps/base/woodpecker/configmap_gocache-nginx.yaml

@@ -1,39 +0,0 @@
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: gocache-nginx
-  namespace: woodpecker
-  labels:
-    app.kubernetes.io/name: gocache
-    app.kubernetes.io/component: proxy
-data:
-  nginx.conf: |
-    worker_processes auto;
-    error_log /dev/stderr warn;
-    pid /tmp/nginx.pid;
-
-    events {
-      worker_connections 512;
-    }
-
-    # GOCACHEPROG is a raw byte stream, not HTTP, so this must be stream{} not http{}.
-    stream {
-      server {
-        listen 9090;
-
-        # The protocol has no authentication: anyone who can reach this port can
-        # write cache entries, which become code in every build that reads them.
-        # Loopback is the kubectl port-forward fallback; in a pod netns it is
-        # only these two containers.
-        allow 127.0.0.1/32;
-        allow 10.10.12.200/32;
-        allow 10.42.0.0/16;
-        deny all;
-
-        # A connect session lasts the whole build; the 10m default cuts long builds off.
-        proxy_timeout 2h;
-        proxy_connect_timeout 5s;
-        proxy_pass 127.0.0.1:9080;
-      }
-    }

apps/base/woodpecker/deployment_gocache.yaml

@@ -1,131 +0,0 @@
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: gocache
-  namespace: woodpecker
-  annotations:
-    configmap.reloader.stakater.com/reload: "gocache-nginx"
-    secret.reloader.stakater.com/reload: "gocache-s3,vault-ca-cert"
-  labels:
-    app.kubernetes.io/name: gocache
-    app.kubernetes.io/component: cache
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: gocache
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: gocache
-        app.kubernetes.io/component: cache
-    spec:
-      serviceAccountName: default
-      automountServiceAccountToken: false
-      securityContext:
-        runAsNonRoot: true
-        fsGroup: 65532
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: go-cache-plugin
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/go-cache-plugin:v0.1.0
-          imagePullPolicy: IfNotPresent
-          # Root flags must precede the subcommand; only --plugin belongs to serve.
-          args:
-            - --cache-dir=/var/cache/gocache
-            - --bucket=gocache
-            # Explicit region skips the GetBucketLocation probe, which RGW handles poorly.
-            - --region=us-east-1
-            - --s3-endpoint-url=https://s3.ceph.unkin.net
-            - --s3-path-style
-            - serve
-            - --plugin=9080
-          env:
-            - name: AWS_ACCESS_KEY_ID
-              valueFrom:
-                secretKeyRef:
-                  name: gocache-s3
-                  key: AWS_ACCESS_KEY_ID
-            - name: AWS_SECRET_ACCESS_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: gocache-s3
-                  key: AWS_SECRET_ACCESS_KEY
-            # s3.ceph.unkin.net is served by the estate CA, not a public root.
-            - name: AWS_CA_BUNDLE
-              value: /etc/ssl/vault-ca/ca.crt
-          volumeMounts:
-            - name: cache
-              mountPath: /var/cache/gocache
-            - name: vault-ca
-              mountPath: /etc/ssl/vault-ca
-              readOnly: true
-          securityContext:
-            runAsUser: 65532
-            runAsGroup: 65532
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop:
-                - ALL
-          resources:
-            requests:
-              cpu: 200m
-              memory: 256Mi
-            limits:
-              cpu: "2"
-              memory: 2Gi
-        - name: nginx
-          image: docker.io/nginx:1.29.8-alpine
-          imagePullPolicy: IfNotPresent
-          # Bypass the image entrypoint: its config scripts write to a read-only rootfs.
-          command:
-            - nginx
-            - -g
-            - daemon off;
-          ports:
-            - containerPort: 9090
-              name: gocache
-              protocol: TCP
-          volumeMounts:
-            - name: nginx-config
-              mountPath: /etc/nginx/nginx.conf
-              subPath: nginx.conf
-              readOnly: true
-            - name: tmp
-              mountPath: /tmp
-          securityContext:
-            runAsUser: 101
-            runAsGroup: 101
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop:
-                - ALL
-          resources:
-            requests:
-              cpu: 25m
-              memory: 32Mi
-            limits:
-              cpu: 500m
-              memory: 128Mi
-      volumes:
-        # Staging cache in front of S3: losing it costs a repopulate, not data.
-        - name: cache
-          emptyDir:
-            sizeLimit: 20Gi
-        - name: nginx-config
-          configMap:
-            name: gocache-nginx
-        - name: tmp
-          emptyDir: {}
-        - name: vault-ca
-          secret:
-            secretName: vault-ca-cert
-            items:
-              - key: ca.crt
-                path: ca.crt

apps/base/woodpecker/gocache_bucket.yaml

@@ -1,33 +0,0 @@
----
-# Shared Go build cache (GOCACHEPROG) for CI and developer laptops. Lives in the
-# woodpecker namespace because CI is the primary consumer and reads the Secret here.
-apiVersion: ceph.unkin.net/v1alpha1
-kind: ObjectStoreUser
-metadata:
-  name: gocache
-  namespace: woodpecker
-spec:
-  displayName: "Go build cache owner"
-  uid: gocache
-  maxBuckets: 1
-  secretName: gocache-s3
-  retainOnDelete: false
----
-apiVersion: ceph.unkin.net/v1alpha1
-kind: Bucket
-metadata:
-  name: gocache
-  namespace: woodpecker
-spec:
-  bucketName: gocache
-  ownerRef: gocache
-  versioning: false
-  # No placementTarget: default (replicated) placement, not the ec target the
-  # backup buckets use — a build cache is millions of small objects.
-  tags:
-    app: gocache
-    purpose: go-build-cache
-  retainOnDelete: false
-  # A cache bucket is never empty, and the operator refuses to delete a
-  # non-empty bucket without this, wedging the finalizer.
-  purgeOnDelete: true

apps/base/woodpecker/kustomization.yaml

@@ -7,10 +7,6 @@ resources:
   - cnpg_cluster.yaml
   - cnpg_backup.yaml
   - cnpg_pooler.yaml
-  - gocache_bucket.yaml
-  - configmap_gocache-nginx.yaml
-  - deployment_gocache.yaml
-  - service_gocache.yaml
   - serviceaccount_arrproxy_ci.yaml
   - serviceaccount_autobackup_operator_ci.yaml
   - serviceaccount_ghp.yaml
@@ -19,7 +15,6 @@ resources:
   - serviceaccount_mediamark_ci.yaml
   - serviceaccount_plugin_docker_buildx.yaml
   - serviceaccount_jellyfin_ha_src.yaml
-  - serviceaccount_jellyfin_plugin_sso.yaml
   - serviceaccount_repospawner_ci.yaml
   - serviceaccount_terraform_artifactapi.yaml
   - serviceaccount_terraform_authentik.yaml

apps/base/woodpecker/service_gocache.yaml

@@ -1,23 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: gocache
-  namespace: woodpecker
-  annotations:
-    purelb.io/addresses: 198.18.200.11
-    purelb.io/service-group: common
-  labels:
-    app.kubernetes.io/name: gocache
-spec:
-  type: LoadBalancer
-  # Cluster SNATs off-node traffic to a node address, which would defeat the
-  # nginx allow rules; Local preserves the wireguard client IP.
-  externalTrafficPolicy: Local
-  selector:
-    app.kubernetes.io/name: gocache
-  ports:
-    - name: gocache
-      port: 9090
-      targetPort: gocache
-      protocol: TCP

apps/base/woodpecker/serviceaccount_jellyfin_plugin_sso.yaml

@@ -1,6 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: jellyfin-plugin-sso
-  namespace: woodpecker

apps/overlays/au-syd1/haproxy/kustomization.yaml

@@ -1,6 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - ../../../base/haproxy

apps/overlays/au-syd1/vm-system/kustomization.yaml

@@ -10,7 +10,7 @@ resources:
 helmCharts:
   - name: victoria-metrics-operator
     repo: https://victoriametrics.github.io/helm-charts/
-    version: "0.67.3"
+    version: "0.57.1"
     releaseName: victoria-metrics-operator
     namespace: vm-system
     valuesFile: values.yaml

argocd/applicationsets/platform.yaml

@@ -29,7 +29,6 @@ spec:
       - path: apps/overlays/*/ghp
       - path: apps/overlays/*/gitea
       - path: apps/overlays/*/grafana-system
-      - path: apps/overlays/*/haproxy
       - path: apps/overlays/*/inteldeviceplugins-system
       - path: apps/overlays/*/jfrog
       - path: apps/overlays/*/k8up-system

argocd/projects/platform.yaml

@@ -43,8 +43,6 @@ spec:
       server: https://kubernetes.default.svc
     - namespace: 'gitea'
       server: https://kubernetes.default.svc
-    - namespace: 'haproxy'
-      server: https://kubernetes.default.svc
     - namespace: 'jfrog'
       server: https://kubernetes.default.svc
     - namespace: 'kanidm'

clusters/au-syd1/bootstrap/argocd-cm-patch.yaml

@@ -26,10 +26,6 @@ data:
     issuer: https://identity.unkin.net/application/o/argocd/
     clientID: argocd
     clientSecret: $argocd-oidc:client_secret
-    # The Authentik client is public (the iOS app can't hold a secret), so
-    # Authentik no longer enforces clientSecret; PKCE replaces it as the
-    # protection against authorization-code interception.
-    enablePKCEAuthentication: true
     # identity.unkin.net now serves the LetsEncrypt *.unkin.net wildcard, so the
     # stock image trust store validates it; no rootCA pin.
     requestedScopes: