Deliver the cert helpers to the puppet compiler pods

certmanager and sshsignhost are invoked server-side by generate() during catalog compilation but only exist on the legacy VM masters, so compiles on the k8s compilers fail with "No such file or directory".

- Stage a self-contained EL9 python runtime on the shared bins volume
- Render both helpers from their puppet-prod ERB templates on the code volume rather than copying the scripts here
- Mount a name-dispatching launcher at /usr/local/bin/{certmanager,sshsignhost}
- Mount kubernetes-auth configs at /opt/{certmanager,sshsignhost}/config.yaml

Needs terraform-vault #152 applied and the puppet-prod kubernetes-auth PR merged.

apps/base/authentik/cnpg_cluster.yaml

@@ -64,12 +64,8 @@ spec:
       archive_mode: "on"
       archive_timeout: 5min
       dynamic_shared_memory_type: posix
-      effective_cache_size: 1536MB
+      effective_cache_size: 256MB
       full_page_writes: "on"
-      # Replicas report their oldest xmin to the primary, so multi-second reads on
-      # a hot standby stop exhausting max_standby_streaming_delay and being
-      # cancelled. Retained-dead-tuple cost is negligible on a ~155MB database.
-      hot_standby_feedback: "on"
       log_destination: csvlog
       log_directory: /controller/log
       log_filename: postgres
@@ -81,12 +77,7 @@ spec:
       max_parallel_workers: "16"
       max_replication_slots: "16"
       max_worker_processes: "16"
-      # A pg_stat_statements.* parameter is what makes CNPG treat the extension as
-      # managed and run CREATE EXTENSION in every database; preloading alone does
-      # not create it.
-      pg_stat_statements.max: "10000"
-      pg_stat_statements.track: top
-      shared_buffers: 512MB
+      shared_buffers: 128MB
       shared_memory_type: mmap
       ssl_max_protocol_version: TLSv1.3
       ssl_min_protocol_version: TLSv1.3
@@ -95,9 +86,6 @@ spec:
       wal_log_hints: "on"
       wal_receiver_timeout: 5s
       wal_sender_timeout: 5s
-    # CNPG merges this with the libraries it manages itself.
-    shared_preload_libraries:
-      - pg_stat_statements
     syncReplicaElectionConstraint:
       enabled: false
   primaryUpdateMethod: restart
@@ -117,16 +105,13 @@ spec:
     updateInterval: 30
   resources:
     limits:
-      # 500m is a 50ms CFS quota per 100ms period, exhausted by bursts even at
-      # ~0.01 cores average, so every query pays throttle latency.
-      cpu: "2"
+      cpu: 500m
       # 512Mi OOMKilled replicas under load (shared_buffers 128MB +
       # max_connections 200 leave no headroom) — see incident 2026-07-28.
-      # shared_buffers 512MB needs the same headroom multiple, hence 2Gi.
-      memory: 2Gi
-    requests:
-      cpu: 500m
       memory: 1Gi
+    requests:
+      cpu: 50m
+      memory: 512Mi
   smartShutdownTimeout: 180
   startDelay: 3600
   stopDelay: 1800

apps/base/authentik/httproute.yaml

@@ -37,22 +37,6 @@ spec:
       name: authentik
       sectionName: https
   rules:
-    - backendRefs:
-        - group: ""
-          kind: Service
-          name: authentik-server
-          port: 80
-          weight: 1
-      filters:
-        - type: URLRewrite
-          urlRewrite:
-            path:
-              type: ReplaceFullPath
-              replaceFullPath: /application/o/token/
-      matches:
-        - path:
-            type: Exact
-            value: /application/o/token
     - backendRefs:
         - group: ""
           kind: Service
@@ -102,22 +86,6 @@ spec:
       name: authentik-internal
       sectionName: https
   rules:
-    - backendRefs:
-        - group: ""
-          kind: Service
-          name: authentik-server
-          port: 80
-          weight: 1
-      filters:
-        - type: URLRewrite
-          urlRewrite:
-            path:
-              type: ReplaceFullPath
-              replaceFullPath: /application/o/token/
-      matches:
-        - path:
-            type: Exact
-            value: /application/o/token
     - backendRefs:
         - group: ""
           kind: Service

apps/base/authentik/kustomization.yaml

@@ -19,7 +19,6 @@ resources:
   - redis-deployment.yaml
   - redis-pvc.yaml
   - redis-service.yaml
-  - server-vmpodscrape.yaml
   - vaultauth.yaml
   - vaultstaticsecret.yaml
   - vmpodscrape.yaml

apps/base/authentik/server-vmpodscrape.yaml

@@ -1,16 +0,0 @@
----
-# Scrape the authentik server's django_prometheus endpoint (:9300). Picked up
-# by the observability VMAgent (selectAllByDefault).
-apiVersion: operator.victoriametrics.com/v1beta1
-kind: VMPodScrape
-metadata:
-  name: authentik-server
-  namespace: authentik
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: authentik
-      app.kubernetes.io/component: server
-  podMetricsEndpoints:
-    - port: metrics
-      path: /metrics

apps/base/bind-system/deployment.yaml

@@ -21,7 +21,7 @@ spec:
         runAsNonRoot: true
       containers:
         - name: operator
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.2.7
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.2.6
           args:
             - --metrics-bind-address=:8080
             - --health-probe-bind-address=:8081

apps/base/bind-system/kustomization.yaml

@@ -6,7 +6,7 @@ resources:
   - namespace.yaml
   # CRDs are pulled from the bind-operator repo at the matching tag rather than
   # vendored here, so they never drift from the operator.
-  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.2.7/config/crd/install.yaml
+  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.2.6/config/crd/install.yaml
   - rbac.yaml
   - agent-dns-rbac.yaml
   - deployment.yaml

apps/base/cheeztv/plugin-configmap.yaml

@@ -26,7 +26,7 @@ data:
           </key>
           <value>
             <PluginConfiguration>
-              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
               <OidClientId>jellyfin</OidClientId>
               <OidSecret>@@CLIENT_SECRET@@</OidSecret>
               <Enabled>true</Enabled>

apps/base/cheeztv/service.yaml

@@ -13,4 +13,6 @@ spec:
       targetPort: http
   selector:
     app: cheeztv
+  # Pin each client to one replica to reduce transcode-session churn/takeover.
+  sessionAffinity: ClientIP
   type: ClusterIP

apps/base/cheeztv/statefulset.yaml

@@ -4,8 +4,6 @@ kind: StatefulSet
 metadata:
   name: cheeztv
   namespace: cheeztv
-  annotations:
-    configmap.reloader.stakater.com/auto: "true"
 spec:
   # HA: two replicas coordinate transcode session ownership through Valkey and
   # resume each other's HLS segments off the shared RWX transcode PVC. Stable
@@ -164,7 +162,7 @@ spec:
               readOnly: true
       containers:
         - name: cheeztv
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.3.3
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

apps/base/fafflix/plugin-configmap.yaml

@@ -26,7 +26,7 @@ data:
           </key>
           <value>
             <PluginConfiguration>
-              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
               <OidClientId>jellyfin</OidClientId>
               <OidSecret>@@CLIENT_SECRET@@</OidSecret>
               <Enabled>true</Enabled>

apps/base/fafflix/service.yaml

@@ -13,4 +13,6 @@ spec:
       targetPort: http
   selector:
     app: fafflix
+  # Pin each client to one replica to reduce transcode-session churn/takeover.
+  sessionAffinity: ClientIP
   type: ClusterIP

apps/base/fafflix/statefulset.yaml

@@ -4,8 +4,6 @@ kind: StatefulSet
 metadata:
   name: fafflix
   namespace: fafflix
-  annotations:
-    configmap.reloader.stakater.com/auto: "true"
 spec:
   # HA: two replicas coordinate transcode session ownership through Valkey and
   # resume each other's HLS segments off the shared RWX transcode PVC. Stable
@@ -164,7 +162,7 @@ spec:
               readOnly: true
       containers:
         - name: fafflix
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.3.3
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

apps/base/puppet/deployment_puppetserver-compiler.yaml

@@ -99,23 +99,24 @@ spec:
         - mountPath: /docker-custom-entrypoint.d/post-startup/additional-ruby-gems.sh
           name: additional-ruby-gems
           subPath: additional-ruby-gems.sh
+        - mountPath: /usr/local/bin/certmanager
+          name: cert-helpers
+          subPath: vault-helper
+        - mountPath: /usr/local/bin/sshsignhost
+          name: cert-helpers
+          subPath: vault-helper
+        - mountPath: /opt/certmanager/config.yaml
+          name: cert-helpers
+          subPath: certmanager-config.yaml
+        - mountPath: /opt/sshsignhost/config.yaml
+          name: cert-helpers
+          subPath: sshsignhost-config.yaml
         - mountPath: /configmaps/auth.conf
           name: compiler-auth-conf
           subPath: auth.conf
         - mountPath: /docker-custom-entrypoint.d/pre-default/10-auth-conf.sh
           name: compiler-auth-conf-seed
           subPath: 10-auth-conf.sh
-        - mountPath: /docker-custom-entrypoint.d/pre-default/20-vault-helpers.sh
-          name: compiler-vault-helpers-seed
-          subPath: 20-vault-helpers.sh
-        - mountPath: /opt/certmanager/config.yaml
-          name: certmanager-config
-          subPath: certmanager.yaml
-          readOnly: true
-        - mountPath: /opt/sshsignhost/config.yaml
-          name: sshsignhost-config
-          subPath: sshsignhost.yaml
-          readOnly: true
       initContainers:
       - name: copy-configmaps
         image: busybox:1.35
@@ -213,41 +214,67 @@ spec:
           echo "$EXPECTED  encapic" | sha256sum -c -
           install -m 0755 encapic /opt/bin/encapic
 
-          # Puppet shells out to these two from generate() during catalog
-          # compilation: profiles::pki::vault runs certmanager and
-          # profiles::ssh::sign runs sshsignhost.
-          install_release() {
-            name=$1
-            version=$2
-            asset="$name-linux-amd64"
-            base="https://git.unkin.net/unkin/$name/releases/download/$version"
-            curl -fsSL -o "$name" "$base/$asset"
-            curl -fsSL -o "$name.checksums" "$base/checksums.txt"
-            # checksums.txt covers every release asset; pick the line for the
-            # one we downloaded and verify it under our local filename.
-            expected=$(awk -v a="$asset" '$NF == a || $NF == "*"a {print $1}' "$name.checksums")
-            if [ -z "$expected" ]; then
-              echo "no checksum for $asset in $version checksums.txt" >&2
-              exit 1
-            fi
-            echo "$expected  $name" | sha256sum -c -
-            install -m 0755 "$name" "/opt/bin/$name"
-          }
-
-          install_release certmanager v0.2.0
-          install_release sshsignhost v0.1.0
-
           echo "Shared binaries setup completed"
-        resources:
-          limits:
-            cpu: 300m
-            memory: 256Mi
-          requests:
-            cpu: 100m
-            memory: 64Mi
         volumeMounts:
         - mountPath: /opt/bin/
           name: puppet-shared-bins
+
+      - name: setup-cert-helpers
+        image: git.unkin.net/unkin/almalinux9-base:20260606
+        command:
+        - sh
+        - -c
+        args:
+        - |
+          set -e
+          CH=/opt/bin/certhelpers
+          PYROOT=$CH/py-el9-1
+          TPL=/etc/puppetlabs/code/environments/develop/site/profiles/templates/helpers
+          mkdir -p "$CH"
+
+          # The helpers are python3 (requests, pyyaml) and openvoxserver ships
+          # no python, so stage a self-contained EL9 tree once per volume.
+          if [ ! -f "$PYROOT/.ready" ]; then
+            echo "Staging python runtime for the cert helpers..."
+            TMP=$CH/.py-el9-1.$$
+            rm -rf "$TMP"
+            dnf -y --installroot="$TMP" --releasever=9 --nodocs \
+              --setopt=install_weak_deps=0 --disablerepo=unkin install \
+              python3 python3-requests python3-pyyaml python3-six
+            rm -rf "$TMP/var/cache" "$TMP/var/lib/dnf" "$TMP/var/lib/rpm" \
+              "$TMP/usr/share/locale"
+            # Both hardcode EL absolute paths that only exist inside the tree.
+            SP=$TMP/usr/lib/python3.9/site-packages
+            ln -sfn ../../six.py "$SP/urllib3/packages/six.py"
+            sed -i "s|'/etc/pki/tls/certs/ca-bundle.crt'|'$PYROOT/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem'|" \
+              "$SP/requests/certs.py"
+            touch "$TMP/.ready"
+            mv -T "$TMP" "$PYROOT" || rm -rf "$TMP"
+          fi
+
+          # Render from the puppet-prod ERB templates on the code volume so this
+          # repo never carries a second copy of the scripts.
+          for n in certmanager sshsignhost; do
+            sed -e "s|<%= @venv_path %>|$PYROOT/usr|g" \
+                -e "s|<%= @config_path %>|/opt/$n/config.yaml|g" \
+                "$TPL/$n.erb" > "$CH/.$n.$$"
+            chmod 0755 "$CH/.$n.$$"
+            mv "$CH/.$n.$$" "$CH/$n"
+          done
+          echo "Cert helpers setup completed"
+        resources:
+          limits:
+            cpu: 1
+            memory: 1Gi
+          requests:
+            cpu: 200m
+            memory: 256Mi
+        volumeMounts:
+        - mountPath: /opt/bin/
+          name: puppet-shared-bins
+        - mountPath: /etc/puppetlabs/code/
+          name: puppet-code-volume
+          readOnly: true
       securityContext:
         fsGroup: 999
         seccompProfile:
@@ -282,6 +309,19 @@ spec:
         configMap:
           name: additional-ruby-gems
           defaultMode: 0755
+      - name: cert-helpers
+        configMap:
+          name: cert-helpers
+          items:
+          - key: vault-helper
+            path: vault-helper
+            mode: 0755
+          - key: certmanager-config.yaml
+            path: certmanager-config.yaml
+            mode: 0444
+          - key: sshsignhost-config.yaml
+            path: sshsignhost-config.yaml
+            mode: 0444
       - name: compiler-auth-conf
         configMap:
           name: compiler-auth.conf
@@ -289,15 +329,5 @@ spec:
         configMap:
           name: compiler-auth-conf-seed
           defaultMode: 0755
-      - name: compiler-vault-helpers-seed
-        configMap:
-          name: compiler-vault-helpers-seed
-          defaultMode: 0755
-      - name: certmanager-config
-        configMap:
-          name: certmanager-config
-      - name: sshsignhost-config
-        configMap:
-          name: sshsignhost-config
   strategy:
     type: RollingUpdate

apps/base/puppet/kustomization.yaml

@@ -64,23 +64,15 @@ configMapGenerator:
       - resources/compiler/10-auth-conf.sh
     options:
       disableNameSuffixHash: true
-  - name: compiler-vault-helpers-seed
-    files:
-      - resources/compiler/20-vault-helpers.sh
-    options:
-      disableNameSuffixHash: true
-  - name: certmanager-config
-    files:
-      - resources/compiler/certmanager.yaml
-    options:
-      disableNameSuffixHash: true
-  - name: sshsignhost-config
-    files:
-      - resources/compiler/sshsignhost.yaml
-    options:
-      disableNameSuffixHash: true
   - name: additional-ruby-gems
     files:
       - resources/additional-ruby-gems.sh
     options:
       disableNameSuffixHash: true
+  - name: cert-helpers
+    files:
+      - resources/cert-helpers/vault-helper
+      - resources/cert-helpers/certmanager-config.yaml
+      - resources/cert-helpers/sshsignhost-config.yaml
+    options:
+      disableNameSuffixHash: true

apps/base/puppet/resources/additional-ruby-gems.sh

@@ -6,6 +6,4 @@ echo "Installing additional Ruby gems..."
 /opt/puppetlabs/puppet/bin/gem install ipaddr
 /opt/puppetlabs/puppet/bin/gem install hiera-eyaml
 /opt/puppetlabs/puppet/bin/gem install toml
-# Under set -e a failed install kills the entrypoint post-startup hooks, taking down an already-serving compiler.
-/opt/puppetlabs/bin/puppetserver gem install toml
 echo "Additional Ruby gems installed successfully"

apps/base/puppet/resources/cert-helpers/certmanager-config.yaml

@@ -0,0 +1,12 @@
+---
+# profiles::helpers::certmanager::vault_config, with kubernetes auth: the
+# certmanager approle is CIDR-bound to the legacy VM masters.
+vault:
+  addr: 'https://vault.service.consul:8200'
+  auth_method: 'kubernetes'
+  k8s_mount: 'k8s/au/syd1'
+  k8s_role: 'puppet_certmanager'
+  jwt_path: '/var/run/secrets/kubernetes.io/serviceaccount/token'
+  mount_point: 'pki_int'
+  role_name: 'servers_default'
+output_path: '/tmp/certmanager'

apps/base/puppet/resources/cert-helpers/sshsignhost-config.yaml

@@ -0,0 +1,11 @@
+---
+# profiles::helpers::sshsignhost::vault_config, with kubernetes auth.
+vault:
+  addr: 'https://vault.service.consul:8200'
+  auth_method: 'kubernetes'
+  k8s_mount: 'k8s/au/syd1'
+  k8s_role: 'puppet_sshsigner'
+  jwt_path: '/var/run/secrets/kubernetes.io/serviceaccount/token'
+  mount_point: 'ssh-host-signer'
+  role_name: 'hostrole'
+output_path: '/tmp/sshsignhost'

apps/base/puppet/resources/cert-helpers/vault-helper

@@ -0,0 +1,11 @@
+#!/bin/sh
+# Runs the certmanager/sshsignhost helper matching the name it is invoked as.
+# The openvoxserver image has no python, so the EL9 tree staged on the shared
+# bins volume is started through its own dynamic loader.
+set -eu
+
+PYROOT=/opt/bin/certhelpers/py-el9-1
+
+exec "${PYROOT}/lib64/ld-linux-x86-64.so.2" \
+  --library-path "${PYROOT}/lib64:${PYROOT}/usr/lib64" \
+  "${PYROOT}/usr/bin/python3.9" "/opt/bin/certhelpers/${0##*/}" "$@"

apps/base/puppet/resources/compiler/20-vault-helpers.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-BIN_DIR=/opt/bin
-CA=/opt/vault-ca-cert.crt
-
-if [ ! -s "$CA" ]; then
-  echo "FATAL: $CA missing or empty; certmanager and sshsignhost cannot verify Vault" >&2
-  exit 1
-fi
-
-# profiles::pki::vault and profiles::ssh::sign shell out to fixed /usr/local/bin
-# paths from generate(); the binaries ship on the shared PVC, and /usr/local/bin
-# lives in the image. Wrappers rather than symlinks because neither binary reads
-# a CA path from its config: SSL_CERT_FILE scopes the internal CA to these two
-# processes instead of the puppetserver JVM's own trust store.
-for bin in certmanager sshsignhost; do
-  if [ ! -x "$BIN_DIR/$bin" ]; then
-    echo "FATAL: $BIN_DIR/$bin missing; generate() would abort every catalog compile" >&2
-    exit 1
-  fi
-  cat > "/usr/local/bin/$bin" <<WRAPPER
-#!/bin/sh
-SSL_CERT_FILE=$CA
-export SSL_CERT_FILE
-exec $BIN_DIR/$bin "\$@"
-WRAPPER
-  chmod 0755 "/usr/local/bin/$bin"
-done

apps/base/puppet/resources/compiler/certmanager.yaml

@@ -1,12 +0,0 @@
----
-vault:
-  addr: https://vault.service.consul:8200
-  auth_method: kubernetes
-  k8s_mount: k8s/au/syd1
-  k8s_role: puppet_certmanager
-  jwt_path: /var/run/secrets/kubernetes.io/serviceaccount/token
-  mount_point: pki_int
-  role_name: servers_default
-  output_path: /tmp/certmanager
-  tls_skip_verify: false
-  timeout: 30s

apps/base/puppet/resources/compiler/sshsignhost.yaml

@@ -1,11 +0,0 @@
----
-vault:
-  addr: https://vault.service.consul:8200
-  auth_method: kubernetes
-  k8s_mount: k8s/au/syd1
-  k8s_role: puppet_sshsigner
-  jwt_path: /var/run/secrets/kubernetes.io/serviceaccount/token
-  mount_point: sshca
-  role_name: signhost
-  tls_skip_verify: false
-  timeout: 30s

apps/base/woodpecker/kustomization.yaml

@@ -15,7 +15,6 @@ resources:
   - serviceaccount_mediamark_ci.yaml
   - serviceaccount_plugin_docker_buildx.yaml
   - serviceaccount_jellyfin_ha_src.yaml
-  - serviceaccount_jellyfin_plugin_sso.yaml
   - serviceaccount_repospawner_ci.yaml
   - serviceaccount_terraform_artifactapi.yaml
   - serviceaccount_terraform_authentik.yaml

apps/base/woodpecker/serviceaccount_jellyfin_plugin_sso.yaml

@@ -1,6 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: jellyfin-plugin-sso
-  namespace: woodpecker

clusters/au-syd1/bootstrap/argocd-cm-patch.yaml

@@ -26,10 +26,6 @@ data:
     issuer: https://identity.unkin.net/application/o/argocd/
     clientID: argocd
     clientSecret: $argocd-oidc:client_secret
-    # The Authentik client is public (the iOS app can't hold a secret), so
-    # Authentik no longer enforces clientSecret; PKCE replaces it as the
-    # protection against authorization-code interception.
-    enablePKCEAuthentication: true
     # identity.unkin.net now serves the LetsEncrypt *.unkin.net wildcard, so the
     # stock image trust store validates it; no rootCA pin.
     requestedScopes: