Give the puppetserver compilers the Vault cert helpers (#482)

profiles::pki::vault and profiles::ssh::sign shell out to
/usr/local/bin/certmanager and /usr/local/bin/sshsignhost from generate()
during catalog compilation. Neither binary exists in the compiler image, so
every node using them fails to compile.

- install certmanager v0.2.0 and sshsignhost v0.1.0 onto the shared bin volume with sha256 verification
- wrap both at /usr/local/bin from a pre-default entrypoint hook, failing startup loudly if either is missing
- mount read-only Vault configs for both: kubernetes auth on k8s/au/syd1, internal CA verified rather than skipped

Reviewed-on: #482
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

apps/base/authentik/cnpg_cluster.yaml

@@ -64,8 +64,12 @@ spec:
       archive_mode: "on"
       archive_timeout: 5min
       dynamic_shared_memory_type: posix
-      effective_cache_size: 256MB
+      effective_cache_size: 1536MB
       full_page_writes: "on"
+      # Replicas report their oldest xmin to the primary, so multi-second reads on
+      # a hot standby stop exhausting max_standby_streaming_delay and being
+      # cancelled. Retained-dead-tuple cost is negligible on a ~155MB database.
+      hot_standby_feedback: "on"
       log_destination: csvlog
       log_directory: /controller/log
       log_filename: postgres
@@ -77,7 +81,12 @@ spec:
       max_parallel_workers: "16"
       max_replication_slots: "16"
       max_worker_processes: "16"
-      shared_buffers: 128MB
+      # A pg_stat_statements.* parameter is what makes CNPG treat the extension as
+      # managed and run CREATE EXTENSION in every database; preloading alone does
+      # not create it.
+      pg_stat_statements.max: "10000"
+      pg_stat_statements.track: top
+      shared_buffers: 512MB
       shared_memory_type: mmap
       ssl_max_protocol_version: TLSv1.3
       ssl_min_protocol_version: TLSv1.3
@@ -86,6 +95,9 @@ spec:
       wal_log_hints: "on"
       wal_receiver_timeout: 5s
       wal_sender_timeout: 5s
+    # CNPG merges this with the libraries it manages itself.
+    shared_preload_libraries:
+      - pg_stat_statements
     syncReplicaElectionConstraint:
       enabled: false
   primaryUpdateMethod: restart
@@ -105,13 +117,16 @@ spec:
     updateInterval: 30
   resources:
     limits:
-      cpu: 500m
+      # 500m is a 50ms CFS quota per 100ms period, exhausted by bursts even at
+      # ~0.01 cores average, so every query pays throttle latency.
+      cpu: "2"
       # 512Mi OOMKilled replicas under load (shared_buffers 128MB +
       # max_connections 200 leave no headroom) — see incident 2026-07-28.
-      memory: 1Gi
+      # shared_buffers 512MB needs the same headroom multiple, hence 2Gi.
+      memory: 2Gi
     requests:
-      cpu: 50m
+      cpu: 500m
-      memory: 512Mi
+      memory: 1Gi
   smartShutdownTimeout: 180
   startDelay: 3600
   stopDelay: 1800

apps/base/authentik/httproute.yaml

@@ -37,6 +37,22 @@ spec:
       name: authentik
       sectionName: https
   rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: authentik-server
+          port: 80
+          weight: 1
+      filters:
+        - type: URLRewrite
+          urlRewrite:
+            path:
+              type: ReplaceFullPath
+              replaceFullPath: /application/o/token/
+      matches:
+        - path:
+            type: Exact
+            value: /application/o/token
     - backendRefs:
         - group: ""
           kind: Service
@@ -86,6 +102,22 @@ spec:
       name: authentik-internal
       sectionName: https
   rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: authentik-server
+          port: 80
+          weight: 1
+      filters:
+        - type: URLRewrite
+          urlRewrite:
+            path:
+              type: ReplaceFullPath
+              replaceFullPath: /application/o/token/
+      matches:
+        - path:
+            type: Exact
+            value: /application/o/token
     - backendRefs:
         - group: ""
           kind: Service

apps/base/authentik/kustomization.yaml

@@ -19,6 +19,7 @@ resources:
   - redis-deployment.yaml
   - redis-pvc.yaml
   - redis-service.yaml
+  - server-vmpodscrape.yaml
   - vaultauth.yaml
   - vaultstaticsecret.yaml
   - vmpodscrape.yaml

apps/base/authentik/server-vmpodscrape.yaml

@@ -0,0 +1,16 @@
+---
+# Scrape the authentik server's django_prometheus endpoint (:9300). Picked up
+# by the observability VMAgent (selectAllByDefault).
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: authentik-server
+  namespace: authentik
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: authentik
+      app.kubernetes.io/component: server
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics

apps/base/bind-system/deployment.yaml

@@ -21,7 +21,7 @@ spec:
         runAsNonRoot: true
       containers:
         - name: operator
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.2.6
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.2.7
           args:
             - --metrics-bind-address=:8080
             - --health-probe-bind-address=:8081

apps/base/bind-system/kustomization.yaml

@@ -6,7 +6,7 @@ resources:
   - namespace.yaml
   # CRDs are pulled from the bind-operator repo at the matching tag rather than
   # vendored here, so they never drift from the operator.
-  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.2.6/config/crd/install.yaml
+  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.2.7/config/crd/install.yaml
   - rbac.yaml
   - agent-dns-rbac.yaml
   - deployment.yaml

apps/base/cheeztv/plugin-configmap.yaml

@@ -26,7 +26,7 @@ data:
           </key>
           <value>
             <PluginConfiguration>
-              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
               <OidClientId>jellyfin</OidClientId>
               <OidSecret>@@CLIENT_SECRET@@</OidSecret>
               <Enabled>true</Enabled>

apps/base/cheeztv/service.yaml

@@ -13,6 +13,4 @@ spec:
       targetPort: http
   selector:
     app: cheeztv
-  # Pin each client to one replica to reduce transcode-session churn/takeover.
-  sessionAffinity: ClientIP
   type: ClusterIP

apps/base/cheeztv/statefulset.yaml

@@ -4,6 +4,8 @@ kind: StatefulSet
 metadata:
   name: cheeztv
   namespace: cheeztv
+  annotations:
+    configmap.reloader.stakater.com/auto: "true"
 spec:
   # HA: two replicas coordinate transcode session ownership through Valkey and
   # resume each other's HLS segments off the shared RWX transcode PVC. Stable
@@ -162,7 +164,7 @@ spec:
               readOnly: true
       containers:
         - name: cheeztv
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.4.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

apps/base/fafflix/plugin-configmap.yaml

@@ -26,7 +26,7 @@ data:
           </key>
           <value>
             <PluginConfiguration>
-              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
               <OidClientId>jellyfin</OidClientId>
               <OidSecret>@@CLIENT_SECRET@@</OidSecret>
               <Enabled>true</Enabled>

apps/base/fafflix/service.yaml

@@ -13,6 +13,4 @@ spec:
       targetPort: http
   selector:
     app: fafflix
-  # Pin each client to one replica to reduce transcode-session churn/takeover.
-  sessionAffinity: ClientIP
   type: ClusterIP

apps/base/fafflix/statefulset.yaml

@@ -4,6 +4,8 @@ kind: StatefulSet
 metadata:
   name: fafflix
   namespace: fafflix
+  annotations:
+    configmap.reloader.stakater.com/auto: "true"
 spec:
   # HA: two replicas coordinate transcode session ownership through Valkey and
   # resume each other's HLS segments off the shared RWX transcode PVC. Stable
@@ -162,7 +164,7 @@ spec:
               readOnly: true
       containers:
         - name: fafflix
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.4.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

apps/base/puppet/deployment_puppetserver-compiler.yaml

@@ -105,6 +105,17 @@ spec:
         - mountPath: /docker-custom-entrypoint.d/pre-default/10-auth-conf.sh
           name: compiler-auth-conf-seed
           subPath: 10-auth-conf.sh
+        - mountPath: /docker-custom-entrypoint.d/pre-default/20-vault-helpers.sh
+          name: compiler-vault-helpers-seed
+          subPath: 20-vault-helpers.sh
+        - mountPath: /opt/certmanager/config.yaml
+          name: certmanager-config
+          subPath: certmanager.yaml
+          readOnly: true
+        - mountPath: /opt/sshsignhost/config.yaml
+          name: sshsignhost-config
+          subPath: sshsignhost.yaml
+          readOnly: true
       initContainers:
       - name: copy-configmaps
         image: busybox:1.35
@@ -202,7 +213,38 @@ spec:
           echo "$EXPECTED  encapic" | sha256sum -c -
           install -m 0755 encapic /opt/bin/encapic
 
+          # Puppet shells out to these two from generate() during catalog
+          # compilation: profiles::pki::vault runs certmanager and
+          # profiles::ssh::sign runs sshsignhost.
+          install_release() {
+            name=$1
+            version=$2
+            asset="$name-linux-amd64"
+            base="https://git.unkin.net/unkin/$name/releases/download/$version"
+            curl -fsSL -o "$name" "$base/$asset"
+            curl -fsSL -o "$name.checksums" "$base/checksums.txt"
+            # checksums.txt covers every release asset; pick the line for the
+            # one we downloaded and verify it under our local filename.
+            expected=$(awk -v a="$asset" '$NF == a || $NF == "*"a {print $1}' "$name.checksums")
+            if [ -z "$expected" ]; then
+              echo "no checksum for $asset in $version checksums.txt" >&2
+              exit 1
+            fi
+            echo "$expected  $name" | sha256sum -c -
+            install -m 0755 "$name" "/opt/bin/$name"
+          }
+
+          install_release certmanager v0.2.0
+          install_release sshsignhost v0.1.0
+
           echo "Shared binaries setup completed"
+        resources:
+          limits:
+            cpu: 300m
+            memory: 256Mi
+          requests:
+            cpu: 100m
+            memory: 64Mi
         volumeMounts:
         - mountPath: /opt/bin/
           name: puppet-shared-bins
@@ -247,5 +289,15 @@ spec:
         configMap:
           name: compiler-auth-conf-seed
           defaultMode: 0755
+      - name: compiler-vault-helpers-seed
+        configMap:
+          name: compiler-vault-helpers-seed
+          defaultMode: 0755
+      - name: certmanager-config
+        configMap:
+          name: certmanager-config
+      - name: sshsignhost-config
+        configMap:
+          name: sshsignhost-config
   strategy:
     type: RollingUpdate

apps/base/puppet/kustomization.yaml

@@ -64,6 +64,21 @@ configMapGenerator:
       - resources/compiler/10-auth-conf.sh
     options:
       disableNameSuffixHash: true
+  - name: compiler-vault-helpers-seed
+    files:
+      - resources/compiler/20-vault-helpers.sh
+    options:
+      disableNameSuffixHash: true
+  - name: certmanager-config
+    files:
+      - resources/compiler/certmanager.yaml
+    options:
+      disableNameSuffixHash: true
+  - name: sshsignhost-config
+    files:
+      - resources/compiler/sshsignhost.yaml
+    options:
+      disableNameSuffixHash: true
   - name: additional-ruby-gems
     files:
       - resources/additional-ruby-gems.sh

apps/base/puppet/resources/additional-ruby-gems.sh

@@ -6,4 +6,6 @@ echo "Installing additional Ruby gems..."
 /opt/puppetlabs/puppet/bin/gem install ipaddr
 /opt/puppetlabs/puppet/bin/gem install hiera-eyaml
 /opt/puppetlabs/puppet/bin/gem install toml
+# Under set -e a failed install kills the entrypoint post-startup hooks, taking down an already-serving compiler.
+/opt/puppetlabs/bin/puppetserver gem install toml
 echo "Additional Ruby gems installed successfully"

apps/base/puppet/resources/compiler/20-vault-helpers.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -euo pipefail
+
+BIN_DIR=/opt/bin
+CA=/opt/vault-ca-cert.crt
+
+if [ ! -s "$CA" ]; then
+  echo "FATAL: $CA missing or empty; certmanager and sshsignhost cannot verify Vault" >&2
+  exit 1
+fi
+
+# profiles::pki::vault and profiles::ssh::sign shell out to fixed /usr/local/bin
+# paths from generate(); the binaries ship on the shared PVC, and /usr/local/bin
+# lives in the image. Wrappers rather than symlinks because neither binary reads
+# a CA path from its config: SSL_CERT_FILE scopes the internal CA to these two
+# processes instead of the puppetserver JVM's own trust store.
+for bin in certmanager sshsignhost; do
+  if [ ! -x "$BIN_DIR/$bin" ]; then
+    echo "FATAL: $BIN_DIR/$bin missing; generate() would abort every catalog compile" >&2
+    exit 1
+  fi
+  cat > "/usr/local/bin/$bin" <<WRAPPER
+#!/bin/sh
+SSL_CERT_FILE=$CA
+export SSL_CERT_FILE
+exec $BIN_DIR/$bin "\$@"
+WRAPPER
+  chmod 0755 "/usr/local/bin/$bin"
+done

apps/base/puppet/resources/compiler/certmanager.yaml

@@ -0,0 +1,12 @@
+---
+vault:
+  addr: https://vault.service.consul:8200
+  auth_method: kubernetes
+  k8s_mount: k8s/au/syd1
+  k8s_role: puppet_certmanager
+  jwt_path: /var/run/secrets/kubernetes.io/serviceaccount/token
+  mount_point: pki_int
+  role_name: servers_default
+  output_path: /tmp/certmanager
+  tls_skip_verify: false
+  timeout: 30s

apps/base/puppet/resources/compiler/sshsignhost.yaml

@@ -0,0 +1,11 @@
+---
+vault:
+  addr: https://vault.service.consul:8200
+  auth_method: kubernetes
+  k8s_mount: k8s/au/syd1
+  k8s_role: puppet_sshsigner
+  jwt_path: /var/run/secrets/kubernetes.io/serviceaccount/token
+  mount_point: sshca
+  role_name: signhost
+  tls_skip_verify: false
+  timeout: 30s

apps/base/woodpecker/kustomization.yaml

@@ -15,6 +15,7 @@ resources:
   - serviceaccount_mediamark_ci.yaml
   - serviceaccount_plugin_docker_buildx.yaml
   - serviceaccount_jellyfin_ha_src.yaml
+  - serviceaccount_jellyfin_plugin_sso.yaml
   - serviceaccount_repospawner_ci.yaml
   - serviceaccount_terraform_artifactapi.yaml
   - serviceaccount_terraform_authentik.yaml

apps/base/woodpecker/serviceaccount_jellyfin_plugin_sso.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: jellyfin-plugin-sso
+  namespace: woodpecker

clusters/au-syd1/bootstrap/argocd-cm-patch.yaml

@@ -26,6 +26,10 @@ data:
     issuer: https://identity.unkin.net/application/o/argocd/
     clientID: argocd
     clientSecret: $argocd-oidc:client_secret
+    # The Authentik client is public (the iOS app can't hold a secret), so
+    # Authentik no longer enforces clientSecret; PKCE replaces it as the
+    # protection against authorization-code interception.
+    enablePKCEAuthentication: true
     # identity.unkin.net now serves the LetsEncrypt *.unkin.net wildcard, so the
     # stock image trust store validates it; no rootCA pin.
     requestedScopes: