Deliver the cert helpers to the puppet compiler pods

certmanager and sshsignhost are invoked server-side by generate() during catalog compilation but only exist on the legacy VM masters, so compiles on the k8s compilers fail with "No such file or directory".

- Stage a self-contained EL9 python runtime on the shared bins volume
- Render both helpers from their puppet-prod ERB templates on the code volume rather than copying the scripts here
- Mount a name-dispatching launcher at /usr/local/bin/{certmanager,sshsignhost}
- Mount kubernetes-auth configs at /opt/{certmanager,sshsignhost}/config.yaml

Needs terraform-vault #152 applied and the puppet-prod kubernetes-auth PR merged.

apps/base/puppet/deployment_puppetserver-compiler.yaml

@@ -99,6 +99,18 @@ spec:
         - mountPath: /docker-custom-entrypoint.d/post-startup/additional-ruby-gems.sh
           name: additional-ruby-gems
           subPath: additional-ruby-gems.sh
+        - mountPath: /usr/local/bin/certmanager
+          name: cert-helpers
+          subPath: vault-helper
+        - mountPath: /usr/local/bin/sshsignhost
+          name: cert-helpers
+          subPath: vault-helper
+        - mountPath: /opt/certmanager/config.yaml
+          name: cert-helpers
+          subPath: certmanager-config.yaml
+        - mountPath: /opt/sshsignhost/config.yaml
+          name: cert-helpers
+          subPath: sshsignhost-config.yaml
         - mountPath: /configmaps/auth.conf
           name: compiler-auth-conf
           subPath: auth.conf
@@ -206,6 +218,63 @@ spec:
         volumeMounts:
         - mountPath: /opt/bin/
           name: puppet-shared-bins
+
+      - name: setup-cert-helpers
+        image: git.unkin.net/unkin/almalinux9-base:20260606
+        command:
+        - sh
+        - -c
+        args:
+        - |
+          set -e
+          CH=/opt/bin/certhelpers
+          PYROOT=$CH/py-el9-1
+          TPL=/etc/puppetlabs/code/environments/develop/site/profiles/templates/helpers
+          mkdir -p "$CH"
+
+          # The helpers are python3 (requests, pyyaml) and openvoxserver ships
+          # no python, so stage a self-contained EL9 tree once per volume.
+          if [ ! -f "$PYROOT/.ready" ]; then
+            echo "Staging python runtime for the cert helpers..."
+            TMP=$CH/.py-el9-1.$$
+            rm -rf "$TMP"
+            dnf -y --installroot="$TMP" --releasever=9 --nodocs \
+              --setopt=install_weak_deps=0 --disablerepo=unkin install \
+              python3 python3-requests python3-pyyaml python3-six
+            rm -rf "$TMP/var/cache" "$TMP/var/lib/dnf" "$TMP/var/lib/rpm" \
+              "$TMP/usr/share/locale"
+            # Both hardcode EL absolute paths that only exist inside the tree.
+            SP=$TMP/usr/lib/python3.9/site-packages
+            ln -sfn ../../six.py "$SP/urllib3/packages/six.py"
+            sed -i "s|'/etc/pki/tls/certs/ca-bundle.crt'|'$PYROOT/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem'|" \
+              "$SP/requests/certs.py"
+            touch "$TMP/.ready"
+            mv -T "$TMP" "$PYROOT" || rm -rf "$TMP"
+          fi
+
+          # Render from the puppet-prod ERB templates on the code volume so this
+          # repo never carries a second copy of the scripts.
+          for n in certmanager sshsignhost; do
+            sed -e "s|<%= @venv_path %>|$PYROOT/usr|g" \
+                -e "s|<%= @config_path %>|/opt/$n/config.yaml|g" \
+                "$TPL/$n.erb" > "$CH/.$n.$$"
+            chmod 0755 "$CH/.$n.$$"
+            mv "$CH/.$n.$$" "$CH/$n"
+          done
+          echo "Cert helpers setup completed"
+        resources:
+          limits:
+            cpu: 1
+            memory: 1Gi
+          requests:
+            cpu: 200m
+            memory: 256Mi
+        volumeMounts:
+        - mountPath: /opt/bin/
+          name: puppet-shared-bins
+        - mountPath: /etc/puppetlabs/code/
+          name: puppet-code-volume
+          readOnly: true
       securityContext:
         fsGroup: 999
         seccompProfile:
@@ -240,6 +309,19 @@ spec:
         configMap:
           name: additional-ruby-gems
           defaultMode: 0755
+      - name: cert-helpers
+        configMap:
+          name: cert-helpers
+          items:
+          - key: vault-helper
+            path: vault-helper
+            mode: 0755
+          - key: certmanager-config.yaml
+            path: certmanager-config.yaml
+            mode: 0444
+          - key: sshsignhost-config.yaml
+            path: sshsignhost-config.yaml
+            mode: 0444
       - name: compiler-auth-conf
         configMap:
           name: compiler-auth.conf

apps/base/puppet/kustomization.yaml

@@ -69,3 +69,10 @@ configMapGenerator:
       - resources/additional-ruby-gems.sh
     options:
       disableNameSuffixHash: true
+  - name: cert-helpers
+    files:
+      - resources/cert-helpers/vault-helper
+      - resources/cert-helpers/certmanager-config.yaml
+      - resources/cert-helpers/sshsignhost-config.yaml
+    options:
+      disableNameSuffixHash: true

apps/base/puppet/resources/cert-helpers/certmanager-config.yaml

@@ -0,0 +1,12 @@
+---
+# profiles::helpers::certmanager::vault_config, with kubernetes auth: the
+# certmanager approle is CIDR-bound to the legacy VM masters.
+vault:
+  addr: 'https://vault.service.consul:8200'
+  auth_method: 'kubernetes'
+  k8s_mount: 'k8s/au/syd1'
+  k8s_role: 'puppet_certmanager'
+  jwt_path: '/var/run/secrets/kubernetes.io/serviceaccount/token'
+  mount_point: 'pki_int'
+  role_name: 'servers_default'
+output_path: '/tmp/certmanager'

apps/base/puppet/resources/cert-helpers/sshsignhost-config.yaml

@@ -0,0 +1,11 @@
+---
+# profiles::helpers::sshsignhost::vault_config, with kubernetes auth.
+vault:
+  addr: 'https://vault.service.consul:8200'
+  auth_method: 'kubernetes'
+  k8s_mount: 'k8s/au/syd1'
+  k8s_role: 'puppet_sshsigner'
+  jwt_path: '/var/run/secrets/kubernetes.io/serviceaccount/token'
+  mount_point: 'ssh-host-signer'
+  role_name: 'hostrole'
+output_path: '/tmp/sshsignhost'

apps/base/puppet/resources/cert-helpers/vault-helper

@@ -0,0 +1,11 @@
+#!/bin/sh
+# Runs the certmanager/sshsignhost helper matching the name it is invoked as.
+# The openvoxserver image has no python, so the EL9 tree staged on the shared
+# bins volume is started through its own dynamic loader.
+set -eu
+
+PYROOT=/opt/bin/certhelpers/py-el9-1
+
+exec "${PYROOT}/lib64/ld-linux-x86-64.so.2" \
+  --library-path "${PYROOT}/lib64:${PYROOT}/usr/lib64" \
+  "${PYROOT}/usr/bin/python3.9" "/opt/bin/certhelpers/${0##*/}" "$@"