Deliver the cert helpers to the puppet compiler pods

certmanager and sshsignhost are invoked server-side by generate() during catalog compilation but only exist on the legacy VM masters, so compiles on the k8s compilers fail with "No such file or directory".

- Stage a self-contained EL9 python runtime on the shared bins volume
- Render both helpers from their puppet-prod ERB templates on the code volume rather than copying the scripts here
- Mount a name-dispatching launcher at /usr/local/bin/{certmanager,sshsignhost}
- Mount kubernetes-auth configs at /opt/{certmanager,sshsignhost}/config.yaml

Needs terraform-vault #152 applied and the puppet-prod kubernetes-auth PR merged.

AGENTS.md

@@ -22,10 +22,10 @@ make kubeconform
 make clean
 
 # Quick build + inspect without persisting output
-kustomize build --enable-helm --helm-kube-version "$(cat ci/kube-version)" apps/overlays/au-syd1/<app-name>
+kustomize build --enable-helm apps/overlays/au-syd1/<app-name>
 
 # Check all resource kinds produced by an overlay
-kustomize build --enable-helm --helm-kube-version "$(cat ci/kube-version)" apps/overlays/au-syd1/<app-name> | grep "^kind:" | sort | uniq -c
+kustomize build --enable-helm apps/overlays/au-syd1/<app-name> | grep "^kind:" | sort | uniq -c
 
 # Run pre-commit checks against all files
 uvx pre-commit run --all-files
@@ -118,7 +118,7 @@ In `argocd/projects/platform.yaml` (or `storage.yaml`):
 
 ### 5. Validate
 ```bash
-kustomize build --enable-helm --helm-kube-version "$(cat ci/kube-version)" apps/overlays/au-syd1/<app-name>
+kustomize build --enable-helm apps/overlays/au-syd1/<app-name>
 make kubeconform
 ```
 
@@ -215,7 +215,7 @@ Examples:
 
 Some overlays vendor Helm charts locally under `apps/overlays/au-syd1/<app-name>/charts/<chart-name>/`. When a chart is vendored, the overlay's `kustomization.yaml` references the local path. When not vendored, it references the OCI or HTTP repo directly.
 
-The Kubernetes target version lives in `ci/kube-version`; the Makefile and CI scripts read it for both the Helm capabilities version and kubeconform.
+Current Kubernetes target version: **1.33.7** (used by kubeconform in CI).
 
 ---
 

Makefile

@@ -3,11 +3,8 @@
 # Build a kustomization path to manifests directory
 # Usage: make build clusters/au-syd1/bootstrap
 build:
-	@kube_version="$$(cat ci/kube-version)"; \
-	if [ -z "$$kube_version" ]; then echo "cannot read ci/kube-version" >&2; exit 1; fi; \
-	target="$(filter-out $@,$(MAKECMDGOALS))"; \
-	mkdir -p "manifests/$$target"; \
-	kustomize build --enable-helm --helm-kube-version "$$kube_version" "$$target" --output "manifests/$$target"
+	@mkdir -p manifests/$(filter-out $@,$(MAKECMDGOALS))
+	@kustomize build --enable-helm $(filter-out $@,$(MAKECMDGOALS)) --output manifests/$(filter-out $@,$(MAKECMDGOALS))
 
 # Generate JSON schemas from CRDs and Kubernetes swagger spec (run manually, results committed)
 schemas:

apps/base/cheeztv/plugin-configmap.yaml

@@ -26,7 +26,7 @@ data:
           </key>
           <value>
             <PluginConfiguration>
-              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
               <OidClientId>jellyfin</OidClientId>
               <OidSecret>@@CLIENT_SECRET@@</OidSecret>
               <Enabled>true</Enabled>

apps/base/cheeztv/statefulset.yaml

@@ -162,7 +162,7 @@ spec:
               readOnly: true
       containers:
         - name: cheeztv
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.3.2
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

apps/base/fafflix/plugin-configmap.yaml

@@ -26,7 +26,7 @@ data:
           </key>
           <value>
             <PluginConfiguration>
-              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
               <OidClientId>jellyfin</OidClientId>
               <OidSecret>@@CLIENT_SECRET@@</OidSecret>
               <Enabled>true</Enabled>

apps/base/fafflix/statefulset.yaml

@@ -162,7 +162,7 @@ spec:
               readOnly: true
       containers:
         - name: fafflix
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.3.2
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

apps/base/puppet/deployment_puppetserver-compiler.yaml

@@ -99,6 +99,18 @@ spec:
         - mountPath: /docker-custom-entrypoint.d/post-startup/additional-ruby-gems.sh
           name: additional-ruby-gems
           subPath: additional-ruby-gems.sh
+        - mountPath: /usr/local/bin/certmanager
+          name: cert-helpers
+          subPath: vault-helper
+        - mountPath: /usr/local/bin/sshsignhost
+          name: cert-helpers
+          subPath: vault-helper
+        - mountPath: /opt/certmanager/config.yaml
+          name: cert-helpers
+          subPath: certmanager-config.yaml
+        - mountPath: /opt/sshsignhost/config.yaml
+          name: cert-helpers
+          subPath: sshsignhost-config.yaml
         - mountPath: /configmaps/auth.conf
           name: compiler-auth-conf
           subPath: auth.conf
@@ -206,6 +218,63 @@ spec:
         volumeMounts:
         - mountPath: /opt/bin/
           name: puppet-shared-bins
+
+      - name: setup-cert-helpers
+        image: git.unkin.net/unkin/almalinux9-base:20260606
+        command:
+        - sh
+        - -c
+        args:
+        - |
+          set -e
+          CH=/opt/bin/certhelpers
+          PYROOT=$CH/py-el9-1
+          TPL=/etc/puppetlabs/code/environments/develop/site/profiles/templates/helpers
+          mkdir -p "$CH"
+
+          # The helpers are python3 (requests, pyyaml) and openvoxserver ships
+          # no python, so stage a self-contained EL9 tree once per volume.
+          if [ ! -f "$PYROOT/.ready" ]; then
+            echo "Staging python runtime for the cert helpers..."
+            TMP=$CH/.py-el9-1.$$
+            rm -rf "$TMP"
+            dnf -y --installroot="$TMP" --releasever=9 --nodocs \
+              --setopt=install_weak_deps=0 --disablerepo=unkin install \
+              python3 python3-requests python3-pyyaml python3-six
+            rm -rf "$TMP/var/cache" "$TMP/var/lib/dnf" "$TMP/var/lib/rpm" \
+              "$TMP/usr/share/locale"
+            # Both hardcode EL absolute paths that only exist inside the tree.
+            SP=$TMP/usr/lib/python3.9/site-packages
+            ln -sfn ../../six.py "$SP/urllib3/packages/six.py"
+            sed -i "s|'/etc/pki/tls/certs/ca-bundle.crt'|'$PYROOT/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem'|" \
+              "$SP/requests/certs.py"
+            touch "$TMP/.ready"
+            mv -T "$TMP" "$PYROOT" || rm -rf "$TMP"
+          fi
+
+          # Render from the puppet-prod ERB templates on the code volume so this
+          # repo never carries a second copy of the scripts.
+          for n in certmanager sshsignhost; do
+            sed -e "s|<%= @venv_path %>|$PYROOT/usr|g" \
+                -e "s|<%= @config_path %>|/opt/$n/config.yaml|g" \
+                "$TPL/$n.erb" > "$CH/.$n.$$"
+            chmod 0755 "$CH/.$n.$$"
+            mv "$CH/.$n.$$" "$CH/$n"
+          done
+          echo "Cert helpers setup completed"
+        resources:
+          limits:
+            cpu: 1
+            memory: 1Gi
+          requests:
+            cpu: 200m
+            memory: 256Mi
+        volumeMounts:
+        - mountPath: /opt/bin/
+          name: puppet-shared-bins
+        - mountPath: /etc/puppetlabs/code/
+          name: puppet-code-volume
+          readOnly: true
       securityContext:
         fsGroup: 999
         seccompProfile:
@@ -240,6 +309,19 @@ spec:
         configMap:
           name: additional-ruby-gems
           defaultMode: 0755
+      - name: cert-helpers
+        configMap:
+          name: cert-helpers
+          items:
+          - key: vault-helper
+            path: vault-helper
+            mode: 0755
+          - key: certmanager-config.yaml
+            path: certmanager-config.yaml
+            mode: 0444
+          - key: sshsignhost-config.yaml
+            path: sshsignhost-config.yaml
+            mode: 0444
       - name: compiler-auth-conf
         configMap:
           name: compiler-auth.conf

apps/base/puppet/kustomization.yaml

@@ -69,3 +69,10 @@ configMapGenerator:
       - resources/additional-ruby-gems.sh
     options:
       disableNameSuffixHash: true
+  - name: cert-helpers
+    files:
+      - resources/cert-helpers/vault-helper
+      - resources/cert-helpers/certmanager-config.yaml
+      - resources/cert-helpers/sshsignhost-config.yaml
+    options:
+      disableNameSuffixHash: true

apps/base/puppet/resources/additional-ruby-gems.sh

@@ -6,6 +6,4 @@ echo "Installing additional Ruby gems..."
 /opt/puppetlabs/puppet/bin/gem install ipaddr
 /opt/puppetlabs/puppet/bin/gem install hiera-eyaml
 /opt/puppetlabs/puppet/bin/gem install toml
-# Under set -e a failed install kills the entrypoint post-startup hooks, taking down an already-serving compiler.
-/opt/puppetlabs/bin/puppetserver gem install toml
 echo "Additional Ruby gems installed successfully"

apps/base/puppet/resources/cert-helpers/certmanager-config.yaml

@@ -0,0 +1,12 @@
+---
+# profiles::helpers::certmanager::vault_config, with kubernetes auth: the
+# certmanager approle is CIDR-bound to the legacy VM masters.
+vault:
+  addr: 'https://vault.service.consul:8200'
+  auth_method: 'kubernetes'
+  k8s_mount: 'k8s/au/syd1'
+  k8s_role: 'puppet_certmanager'
+  jwt_path: '/var/run/secrets/kubernetes.io/serviceaccount/token'
+  mount_point: 'pki_int'
+  role_name: 'servers_default'
+output_path: '/tmp/certmanager'

apps/base/puppet/resources/cert-helpers/sshsignhost-config.yaml

@@ -0,0 +1,11 @@
+---
+# profiles::helpers::sshsignhost::vault_config, with kubernetes auth.
+vault:
+  addr: 'https://vault.service.consul:8200'
+  auth_method: 'kubernetes'
+  k8s_mount: 'k8s/au/syd1'
+  k8s_role: 'puppet_sshsigner'
+  jwt_path: '/var/run/secrets/kubernetes.io/serviceaccount/token'
+  mount_point: 'ssh-host-signer'
+  role_name: 'hostrole'
+output_path: '/tmp/sshsignhost'

apps/base/puppet/resources/cert-helpers/vault-helper

@@ -0,0 +1,11 @@
+#!/bin/sh
+# Runs the certmanager/sshsignhost helper matching the name it is invoked as.
+# The openvoxserver image has no python, so the EL9 tree staged on the shared
+# bins volume is started through its own dynamic loader.
+set -eu
+
+PYROOT=/opt/bin/certhelpers/py-el9-1
+
+exec "${PYROOT}/lib64/ld-linux-x86-64.so.2" \
+  --library-path "${PYROOT}/lib64:${PYROOT}/usr/lib64" \
+  "${PYROOT}/usr/bin/python3.9" "/opt/bin/certhelpers/${0##*/}" "$@"

ci/kube-version

@@ -1 +0,0 @@
-1.33.13

ci/validate-apps.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-KUBE_VERSION="$(cat ci/kube-version)"
+KUBE_VERSION="1.33.7"
 
 SCHEMA_DIR="${SCHEMA_DIR:-schemas}"
 
@@ -15,7 +15,7 @@ while IFS= read -r -d "" k; do
   dir="$(dirname "$k")"
   echo "==> kubeconform: $dir" >&2
 
-  kustomize build --enable-helm --helm-kube-version "$KUBE_VERSION" "$dir" \
+  kustomize build --enable-helm "$dir" \
     | kubeconform \
         -kubernetes-version "$KUBE_VERSION" \
         -summary \

ci/validate-clusters.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-KUBE_VERSION="$(cat ci/kube-version)"
+KUBE_VERSION="1.33.7"
 
 SCHEMA_DIR="${SCHEMA_DIR:-schemas}"
 
@@ -15,7 +15,7 @@ while IFS= read -r -d "" k; do
   dir="$(dirname "$k")"
   echo "==> kubeconform: $dir" >&2
 
-  kustomize build --enable-helm --helm-kube-version "$KUBE_VERSION" "$dir" \
+  kustomize build --enable-helm "$dir" \
     | kubeconform \
         -kubernetes-version "$KUBE_VERSION" \
         -summary \

migration.md

@@ -49,8 +49,8 @@
 - Use existing patterns like `apps/overlays/*/csi-*` or `apps/overlays/*/<app-name>`
 
 ### 9. Validation
-- Run `kustomize build --enable-helm --helm-kube-version "$(cat ci/kube-version)" apps/overlays/au-syd1/<app-name>` to generate all resources
-- Check resource types: `kustomize build --enable-helm --helm-kube-version "$(cat ci/kube-version)" apps/overlays/au-syd1/<app-name> | grep "^kind:" | sort | uniq -c`
+- Run `kustomize build --enable-helm apps/overlays/au-syd1/<app-name>` to generate all resources
+- Check resource types: `kustomize build --enable-helm apps/overlays/au-syd1/<app-name> | grep "^kind:" | sort | uniq -c`
 - Verify all resource types are permitted in the target project's `clusterResourceWhitelist` and `namespaceResourceWhitelist`
 - Run `make kubeconform` to validate all resources
 - Fix any validation errors