Check kube-version inside the build recipe

Render charts with --helm-kube-version from ci/kube-version so kustomize
no longer inherits the local helm binary's default Kubernetes version.

AGENTS.md

@@ -22,10 +22,10 @@ make kubeconform
 make clean
 
 # Quick build + inspect without persisting output
-kustomize build --enable-helm apps/overlays/au-syd1/<app-name>
+kustomize build --enable-helm --helm-kube-version "$(cat ci/kube-version)" apps/overlays/au-syd1/<app-name>
 
 # Check all resource kinds produced by an overlay
-kustomize build --enable-helm apps/overlays/au-syd1/<app-name> | grep "^kind:" | sort | uniq -c
+kustomize build --enable-helm --helm-kube-version "$(cat ci/kube-version)" apps/overlays/au-syd1/<app-name> | grep "^kind:" | sort | uniq -c
 
 # Run pre-commit checks against all files
 uvx pre-commit run --all-files
@@ -118,7 +118,7 @@ In `argocd/projects/platform.yaml` (or `storage.yaml`):
 
 ### 5. Validate
 ```bash
-kustomize build --enable-helm apps/overlays/au-syd1/<app-name>
+kustomize build --enable-helm --helm-kube-version "$(cat ci/kube-version)" apps/overlays/au-syd1/<app-name>
 make kubeconform
 ```
 
@@ -215,7 +215,7 @@ Examples:
 
 Some overlays vendor Helm charts locally under `apps/overlays/au-syd1/<app-name>/charts/<chart-name>/`. When a chart is vendored, the overlay's `kustomization.yaml` references the local path. When not vendored, it references the OCI or HTTP repo directly.
 
-Current Kubernetes target version: **1.33.7** (used by kubeconform in CI).
+The Kubernetes target version lives in `ci/kube-version`; the Makefile and CI scripts read it for both the Helm capabilities version and kubeconform.
 
 ---
 

Makefile

@@ -3,8 +3,11 @@
 # Build a kustomization path to manifests directory
 # Usage: make build clusters/au-syd1/bootstrap
 build:
-	@mkdir -p manifests/$(filter-out $@,$(MAKECMDGOALS))
-	@kustomize build --enable-helm $(filter-out $@,$(MAKECMDGOALS)) --output manifests/$(filter-out $@,$(MAKECMDGOALS))
+	@kube_version="$$(cat ci/kube-version)"; \
+	if [ -z "$$kube_version" ]; then echo "cannot read ci/kube-version" >&2; exit 1; fi; \
+	target="$(filter-out $@,$(MAKECMDGOALS))"; \
+	mkdir -p "manifests/$$target"; \
+	kustomize build --enable-helm --helm-kube-version "$$kube_version" "$$target" --output "manifests/$$target"
 
 # Generate JSON schemas from CRDs and Kubernetes swagger spec (run manually, results committed)
 schemas:

apps/base/authentik/cnpg_cluster.yaml

@@ -64,8 +64,12 @@ spec:
       archive_mode: "on"
       archive_timeout: 5min
       dynamic_shared_memory_type: posix
-      effective_cache_size: 256MB
+      effective_cache_size: 1536MB
       full_page_writes: "on"
+      # Replicas report their oldest xmin to the primary, so multi-second reads on
+      # a hot standby stop exhausting max_standby_streaming_delay and being
+      # cancelled. Retained-dead-tuple cost is negligible on a ~155MB database.
+      hot_standby_feedback: "on"
       log_destination: csvlog
       log_directory: /controller/log
       log_filename: postgres
@@ -77,7 +81,12 @@ spec:
       max_parallel_workers: "16"
       max_replication_slots: "16"
       max_worker_processes: "16"
-      shared_buffers: 128MB
+      # A pg_stat_statements.* parameter is what makes CNPG treat the extension as
+      # managed and run CREATE EXTENSION in every database; preloading alone does
+      # not create it.
+      pg_stat_statements.max: "10000"
+      pg_stat_statements.track: top
+      shared_buffers: 512MB
       shared_memory_type: mmap
       ssl_max_protocol_version: TLSv1.3
       ssl_min_protocol_version: TLSv1.3
@@ -86,6 +95,9 @@ spec:
       wal_log_hints: "on"
       wal_receiver_timeout: 5s
       wal_sender_timeout: 5s
+    # CNPG merges this with the libraries it manages itself.
+    shared_preload_libraries:
+      - pg_stat_statements
     syncReplicaElectionConstraint:
       enabled: false
   primaryUpdateMethod: restart
@@ -105,13 +117,16 @@ spec:
     updateInterval: 30
   resources:
     limits:
-      cpu: 500m
+      # 500m is a 50ms CFS quota per 100ms period, exhausted by bursts even at
+      # ~0.01 cores average, so every query pays throttle latency.
+      cpu: "2"
       # 512Mi OOMKilled replicas under load (shared_buffers 128MB +
       # max_connections 200 leave no headroom) — see incident 2026-07-28.
-      memory: 1Gi
+      # shared_buffers 512MB needs the same headroom multiple, hence 2Gi.
+      memory: 2Gi
     requests:
-      cpu: 50m
-      memory: 512Mi
+      cpu: 500m
+      memory: 1Gi
   smartShutdownTimeout: 180
   startDelay: 3600
   stopDelay: 1800

apps/base/authentik/httproute.yaml

@@ -37,6 +37,22 @@ spec:
       name: authentik
       sectionName: https
   rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: authentik-server
+          port: 80
+          weight: 1
+      filters:
+        - type: URLRewrite
+          urlRewrite:
+            path:
+              type: ReplaceFullPath
+              replaceFullPath: /application/o/token/
+      matches:
+        - path:
+            type: Exact
+            value: /application/o/token
     - backendRefs:
         - group: ""
           kind: Service
@@ -86,6 +102,22 @@ spec:
       name: authentik-internal
       sectionName: https
   rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: authentik-server
+          port: 80
+          weight: 1
+      filters:
+        - type: URLRewrite
+          urlRewrite:
+            path:
+              type: ReplaceFullPath
+              replaceFullPath: /application/o/token/
+      matches:
+        - path:
+            type: Exact
+            value: /application/o/token
     - backendRefs:
         - group: ""
           kind: Service

apps/base/authentik/kustomization.yaml

@@ -19,6 +19,7 @@ resources:
   - redis-deployment.yaml
   - redis-pvc.yaml
   - redis-service.yaml
+  - server-vmpodscrape.yaml
   - vaultauth.yaml
   - vaultstaticsecret.yaml
   - vmpodscrape.yaml

apps/base/authentik/server-vmpodscrape.yaml

@@ -0,0 +1,16 @@
+---
+# Scrape the authentik server's django_prometheus endpoint (:9300). Picked up
+# by the observability VMAgent (selectAllByDefault).
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: authentik-server
+  namespace: authentik
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: authentik
+      app.kubernetes.io/component: server
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics

apps/base/bind-system/deployment.yaml

@@ -21,7 +21,7 @@ spec:
         runAsNonRoot: true
       containers:
         - name: operator
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.2.6
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.2.7
           args:
             - --metrics-bind-address=:8080
             - --health-probe-bind-address=:8081

apps/base/bind-system/kustomization.yaml

@@ -6,7 +6,7 @@ resources:
   - namespace.yaml
   # CRDs are pulled from the bind-operator repo at the matching tag rather than
   # vendored here, so they never drift from the operator.
-  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.2.6/config/crd/install.yaml
+  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.2.7/config/crd/install.yaml
   - rbac.yaml
   - agent-dns-rbac.yaml
   - deployment.yaml

apps/base/cheeztv/plugin-configmap.yaml

@@ -26,7 +26,7 @@ data:
           </key>
           <value>
             <PluginConfiguration>
-              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
               <OidClientId>jellyfin</OidClientId>
               <OidSecret>@@CLIENT_SECRET@@</OidSecret>
               <Enabled>true</Enabled>

apps/base/cheeztv/service.yaml

@@ -13,6 +13,4 @@ spec:
       targetPort: http
   selector:
     app: cheeztv
-  # Pin each client to one replica to reduce transcode-session churn/takeover.
-  sessionAffinity: ClientIP
   type: ClusterIP

apps/base/cheeztv/statefulset.yaml

@@ -4,6 +4,8 @@ kind: StatefulSet
 metadata:
   name: cheeztv
   namespace: cheeztv
+  annotations:
+    configmap.reloader.stakater.com/auto: "true"
 spec:
   # HA: two replicas coordinate transcode session ownership through Valkey and
   # resume each other's HLS segments off the shared RWX transcode PVC. Stable
@@ -162,7 +164,7 @@ spec:
               readOnly: true
       containers:
         - name: cheeztv
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.4.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

apps/base/fafflix/plugin-configmap.yaml

@@ -26,7 +26,7 @@ data:
           </key>
           <value>
             <PluginConfiguration>
-              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
               <OidClientId>jellyfin</OidClientId>
               <OidSecret>@@CLIENT_SECRET@@</OidSecret>
               <Enabled>true</Enabled>

apps/base/fafflix/service.yaml

@@ -13,6 +13,4 @@ spec:
       targetPort: http
   selector:
     app: fafflix
-  # Pin each client to one replica to reduce transcode-session churn/takeover.
-  sessionAffinity: ClientIP
   type: ClusterIP

apps/base/fafflix/statefulset.yaml

@@ -4,6 +4,8 @@ kind: StatefulSet
 metadata:
   name: fafflix
   namespace: fafflix
+  annotations:
+    configmap.reloader.stakater.com/auto: "true"
 spec:
   # HA: two replicas coordinate transcode session ownership through Valkey and
   # resume each other's HLS segments off the shared RWX transcode PVC. Stable
@@ -162,7 +164,7 @@ spec:
               readOnly: true
       containers:
         - name: fafflix
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.4.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

apps/base/puppet/deployment_puppetserver-compiler.yaml

@@ -105,6 +105,17 @@ spec:
         - mountPath: /docker-custom-entrypoint.d/pre-default/10-auth-conf.sh
           name: compiler-auth-conf-seed
           subPath: 10-auth-conf.sh
+        - mountPath: /docker-custom-entrypoint.d/pre-default/20-vault-helpers.sh
+          name: compiler-vault-helpers-seed
+          subPath: 20-vault-helpers.sh
+        - mountPath: /opt/certmanager/config.yaml
+          name: certmanager-config
+          subPath: certmanager.yaml
+          readOnly: true
+        - mountPath: /opt/sshsignhost/config.yaml
+          name: sshsignhost-config
+          subPath: sshsignhost.yaml
+          readOnly: true
       initContainers:
       - name: copy-configmaps
         image: busybox:1.35
@@ -202,7 +213,38 @@ spec:
           echo "$EXPECTED  encapic" | sha256sum -c -
           install -m 0755 encapic /opt/bin/encapic
 
+          # Puppet shells out to these two from generate() during catalog
+          # compilation: profiles::pki::vault runs certmanager and
+          # profiles::ssh::sign runs sshsignhost.
+          install_release() {
+            name=$1
+            version=$2
+            asset="$name-linux-amd64"
+            base="https://git.unkin.net/unkin/$name/releases/download/$version"
+            curl -fsSL -o "$name" "$base/$asset"
+            curl -fsSL -o "$name.checksums" "$base/checksums.txt"
+            # checksums.txt covers every release asset; pick the line for the
+            # one we downloaded and verify it under our local filename.
+            expected=$(awk -v a="$asset" '$NF == a || $NF == "*"a {print $1}' "$name.checksums")
+            if [ -z "$expected" ]; then
+              echo "no checksum for $asset in $version checksums.txt" >&2
+              exit 1
+            fi
+            echo "$expected  $name" | sha256sum -c -
+            install -m 0755 "$name" "/opt/bin/$name"
+          }
+
+          install_release certmanager v0.2.0
+          install_release sshsignhost v0.1.0
+
           echo "Shared binaries setup completed"
+        resources:
+          limits:
+            cpu: 300m
+            memory: 256Mi
+          requests:
+            cpu: 100m
+            memory: 64Mi
         volumeMounts:
         - mountPath: /opt/bin/
           name: puppet-shared-bins
@@ -247,5 +289,15 @@ spec:
         configMap:
           name: compiler-auth-conf-seed
           defaultMode: 0755
+      - name: compiler-vault-helpers-seed
+        configMap:
+          name: compiler-vault-helpers-seed
+          defaultMode: 0755
+      - name: certmanager-config
+        configMap:
+          name: certmanager-config
+      - name: sshsignhost-config
+        configMap:
+          name: sshsignhost-config
   strategy:
     type: RollingUpdate

apps/base/puppet/kustomization.yaml

@@ -64,6 +64,21 @@ configMapGenerator:
       - resources/compiler/10-auth-conf.sh
     options:
       disableNameSuffixHash: true
+  - name: compiler-vault-helpers-seed
+    files:
+      - resources/compiler/20-vault-helpers.sh
+    options:
+      disableNameSuffixHash: true
+  - name: certmanager-config
+    files:
+      - resources/compiler/certmanager.yaml
+    options:
+      disableNameSuffixHash: true
+  - name: sshsignhost-config
+    files:
+      - resources/compiler/sshsignhost.yaml
+    options:
+      disableNameSuffixHash: true
   - name: additional-ruby-gems
     files:
       - resources/additional-ruby-gems.sh

apps/base/puppet/resources/additional-ruby-gems.sh

@@ -6,4 +6,6 @@ echo "Installing additional Ruby gems..."
 /opt/puppetlabs/puppet/bin/gem install ipaddr
 /opt/puppetlabs/puppet/bin/gem install hiera-eyaml
 /opt/puppetlabs/puppet/bin/gem install toml
+# Under set -e a failed install kills the entrypoint post-startup hooks, taking down an already-serving compiler.
+/opt/puppetlabs/bin/puppetserver gem install toml
 echo "Additional Ruby gems installed successfully"

apps/base/puppet/resources/compiler/20-vault-helpers.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -euo pipefail
+
+BIN_DIR=/opt/bin
+CA=/opt/vault-ca-cert.crt
+
+if [ ! -s "$CA" ]; then
+  echo "FATAL: $CA missing or empty; certmanager and sshsignhost cannot verify Vault" >&2
+  exit 1
+fi
+
+# profiles::pki::vault and profiles::ssh::sign shell out to fixed /usr/local/bin
+# paths from generate(); the binaries ship on the shared PVC, and /usr/local/bin
+# lives in the image. Wrappers rather than symlinks because neither binary reads
+# a CA path from its config: SSL_CERT_FILE scopes the internal CA to these two
+# processes instead of the puppetserver JVM's own trust store.
+for bin in certmanager sshsignhost; do
+  if [ ! -x "$BIN_DIR/$bin" ]; then
+    echo "FATAL: $BIN_DIR/$bin missing; generate() would abort every catalog compile" >&2
+    exit 1
+  fi
+  cat > "/usr/local/bin/$bin" <<WRAPPER
+#!/bin/sh
+SSL_CERT_FILE=$CA
+export SSL_CERT_FILE
+exec $BIN_DIR/$bin "\$@"
+WRAPPER
+  chmod 0755 "/usr/local/bin/$bin"
+done

apps/base/puppet/resources/compiler/certmanager.yaml

@@ -0,0 +1,12 @@
+---
+vault:
+  addr: https://vault.service.consul:8200
+  auth_method: kubernetes
+  k8s_mount: k8s/au/syd1
+  k8s_role: puppet_certmanager
+  jwt_path: /var/run/secrets/kubernetes.io/serviceaccount/token
+  mount_point: pki_int
+  role_name: servers_default
+  output_path: /tmp/certmanager
+  tls_skip_verify: false
+  timeout: 30s

apps/base/puppet/resources/compiler/sshsignhost.yaml

@@ -0,0 +1,11 @@
+---
+vault:
+  addr: https://vault.service.consul:8200
+  auth_method: kubernetes
+  k8s_mount: k8s/au/syd1
+  k8s_role: puppet_sshsigner
+  jwt_path: /var/run/secrets/kubernetes.io/serviceaccount/token
+  mount_point: sshca
+  role_name: signhost
+  tls_skip_verify: false
+  timeout: 30s

apps/base/woodpecker/kustomization.yaml

@@ -15,6 +15,7 @@ resources:
   - serviceaccount_mediamark_ci.yaml
   - serviceaccount_plugin_docker_buildx.yaml
   - serviceaccount_jellyfin_ha_src.yaml
+  - serviceaccount_jellyfin_plugin_sso.yaml
   - serviceaccount_repospawner_ci.yaml
   - serviceaccount_terraform_artifactapi.yaml
   - serviceaccount_terraform_authentik.yaml

apps/base/woodpecker/serviceaccount_jellyfin_plugin_sso.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: jellyfin-plugin-sso
+  namespace: woodpecker

ci/kube-version

@@ -0,0 +1 @@
+1.33.13

ci/validate-apps.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-KUBE_VERSION="1.33.7"
+KUBE_VERSION="$(cat ci/kube-version)"
 
 SCHEMA_DIR="${SCHEMA_DIR:-schemas}"
 
@@ -15,7 +15,7 @@ while IFS= read -r -d "" k; do
   dir="$(dirname "$k")"
   echo "==> kubeconform: $dir" >&2
 
-  kustomize build --enable-helm "$dir" \
+  kustomize build --enable-helm --helm-kube-version "$KUBE_VERSION" "$dir" \
     | kubeconform \
         -kubernetes-version "$KUBE_VERSION" \
         -summary \

ci/validate-clusters.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-KUBE_VERSION="1.33.7"
+KUBE_VERSION="$(cat ci/kube-version)"
 
 SCHEMA_DIR="${SCHEMA_DIR:-schemas}"
 
@@ -15,7 +15,7 @@ while IFS= read -r -d "" k; do
   dir="$(dirname "$k")"
   echo "==> kubeconform: $dir" >&2
 
-  kustomize build --enable-helm "$dir" \
+  kustomize build --enable-helm --helm-kube-version "$KUBE_VERSION" "$dir" \
     | kubeconform \
         -kubernetes-version "$KUBE_VERSION" \
         -summary \

clusters/au-syd1/bootstrap/argocd-cm-patch.yaml

@@ -26,6 +26,10 @@ data:
     issuer: https://identity.unkin.net/application/o/argocd/
     clientID: argocd
     clientSecret: $argocd-oidc:client_secret
+    # The Authentik client is public (the iOS app can't hold a secret), so
+    # Authentik no longer enforces clientSecret; PKCE replaces it as the
+    # protection against authorization-code interception.
+    enablePKCEAuthentication: true
     # identity.unkin.net now serves the LetsEncrypt *.unkin.net wildcard, so the
     # stock image trust store validates it; no rootCA pin.
     requestedScopes:

migration.md

@@ -49,8 +49,8 @@
 - Use existing patterns like `apps/overlays/*/csi-*` or `apps/overlays/*/<app-name>`
 
 ### 9. Validation
-- Run `kustomize build --enable-helm apps/overlays/au-syd1/<app-name>` to generate all resources
-- Check resource types: `kustomize build --enable-helm apps/overlays/au-syd1/<app-name> | grep "^kind:" | sort | uniq -c`
+- Run `kustomize build --enable-helm --helm-kube-version "$(cat ci/kube-version)" apps/overlays/au-syd1/<app-name>` to generate all resources
+- Check resource types: `kustomize build --enable-helm --helm-kube-version "$(cat ci/kube-version)" apps/overlays/au-syd1/<app-name> | grep "^kind:" | sort | uniq -c`
 - Verify all resource types are permitted in the target project's `clusterResourceWhitelist` and `namespaceResourceWhitelist`
 - Run `make kubeconform` to validate all resources
 - Fix any validation errors