Give the puppetserver compilers the Vault cert helpers (#482)

profiles::pki::vault and profiles::ssh::sign shell out to
/usr/local/bin/certmanager and /usr/local/bin/sshsignhost from generate()
during catalog compilation. Neither binary exists in the compiler image, so
every node using them fails to compile.

- install certmanager v0.2.0 and sshsignhost v0.1.0 onto the shared bin volume with sha256 verification
- wrap both at /usr/local/bin from a pre-default entrypoint hook, failing startup loudly if either is missing
- mount read-only Vault configs for both: kubernetes auth on k8s/au/syd1, internal CA verified rather than skipped

Reviewed-on: #482
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

apps/base/artifactapi/oauth2-proxy-configmap.yaml

@@ -39,3 +39,8 @@ data:
   OAUTH2_PROXY_REVERSE_PROXY: "true"
   OAUTH2_PROXY_CODE_CHALLENGE_METHOD: "S256"
   OAUTH2_PROXY_SKIP_PROVIDER_BUTTON: "true"
+  # Back-channel discovery/token calls resolve the issuer inside the cluster,
+  # where it is served under the internal unkin.net CA rather than the publicly
+  # trusted cert the browser sees. Trust the bundle the combine-certs init
+  # container assembles, as every other oauth2-proxy in the estate does.
+  OAUTH2_PROXY_PROVIDER_CA_FILES: "/etc/ssl/combined/ca-certificates.crt"

apps/base/artifactapi/oauth2-proxy-deployment.yaml

@@ -6,7 +6,7 @@ metadata:
   namespace: artifactapi
   annotations:
     configmap.reloader.stakater.com/auto: "true"
-    secret.reloader.stakater.com/reload: "oauth-credentials"
+    secret.reloader.stakater.com/reload: "oauth-credentials,vault-ca-cert"
 spec:
   replicas: 2
   selector:
@@ -30,6 +30,36 @@ spec:
         fsGroup: 65532
         seccompProfile:
           type: RuntimeDefault
+      initContainers:
+        # The Authentik issuer is served behind the internal unkin.net CA;
+        # combine the system roots with it so oauth2-proxy's OIDC HTTP client
+        # trusts the discovery endpoint.
+        - name: combine-certs
+          image: docker.io/library/alpine:3
+          imagePullPolicy: IfNotPresent
+          command:
+            - sh
+            - -c
+            - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt
+          volumeMounts:
+            - name: vault-ca-cert
+              mountPath: /custom-ca
+              readOnly: true
+            - name: combined-certs
+              mountPath: /combined-certs
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 50m
+              memory: 32Mi
+            limits:
+              cpu: 200m
+              memory: 64Mi
       containers:
         - name: oauth2-proxy
           image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.3
@@ -83,6 +113,10 @@ spec:
             capabilities:
               drop:
                 - ALL
+          volumeMounts:
+            - name: combined-certs
+              mountPath: /etc/ssl/combined
+              readOnly: true
           resources:
             requests:
               cpu: 50m
@@ -90,4 +124,13 @@ spec:
             limits:
               cpu: 500m
               memory: 256Mi
+      volumes:
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: combined-certs
+          emptyDir: {}
       restartPolicy: Always

apps/base/authentik/cnpg_cluster.yaml

@@ -64,8 +64,12 @@ spec:
       archive_mode: "on"
       archive_timeout: 5min
       dynamic_shared_memory_type: posix
-      effective_cache_size: 256MB
+      effective_cache_size: 1536MB
       full_page_writes: "on"
+      # Replicas report their oldest xmin to the primary, so multi-second reads on
+      # a hot standby stop exhausting max_standby_streaming_delay and being
+      # cancelled. Retained-dead-tuple cost is negligible on a ~155MB database.
+      hot_standby_feedback: "on"
       log_destination: csvlog
       log_directory: /controller/log
       log_filename: postgres
@@ -77,7 +81,12 @@ spec:
       max_parallel_workers: "16"
       max_replication_slots: "16"
       max_worker_processes: "16"
-      shared_buffers: 128MB
+      # A pg_stat_statements.* parameter is what makes CNPG treat the extension as
+      # managed and run CREATE EXTENSION in every database; preloading alone does
+      # not create it.
+      pg_stat_statements.max: "10000"
+      pg_stat_statements.track: top
+      shared_buffers: 512MB
       shared_memory_type: mmap
       ssl_max_protocol_version: TLSv1.3
       ssl_min_protocol_version: TLSv1.3
@@ -86,6 +95,9 @@ spec:
       wal_log_hints: "on"
       wal_receiver_timeout: 5s
       wal_sender_timeout: 5s
+    # CNPG merges this with the libraries it manages itself.
+    shared_preload_libraries:
+      - pg_stat_statements
     syncReplicaElectionConstraint:
       enabled: false
   primaryUpdateMethod: restart
@@ -105,13 +117,16 @@ spec:
     updateInterval: 30
   resources:
     limits:
-      cpu: 500m
+      # 500m is a 50ms CFS quota per 100ms period, exhausted by bursts even at
+      # ~0.01 cores average, so every query pays throttle latency.
+      cpu: "2"
       # 512Mi OOMKilled replicas under load (shared_buffers 128MB +
       # max_connections 200 leave no headroom) — see incident 2026-07-28.
-      memory: 1Gi
+      # shared_buffers 512MB needs the same headroom multiple, hence 2Gi.
+      memory: 2Gi
     requests:
-      cpu: 50m
+      cpu: 500m
-      memory: 512Mi
+      memory: 1Gi
   smartShutdownTimeout: 180
   startDelay: 3600
   stopDelay: 1800

apps/base/authentik/httproute.yaml

@@ -37,6 +37,22 @@ spec:
       name: authentik
       sectionName: https
   rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: authentik-server
+          port: 80
+          weight: 1
+      filters:
+        - type: URLRewrite
+          urlRewrite:
+            path:
+              type: ReplaceFullPath
+              replaceFullPath: /application/o/token/
+      matches:
+        - path:
+            type: Exact
+            value: /application/o/token
     - backendRefs:
         - group: ""
           kind: Service
@@ -86,6 +102,22 @@ spec:
       name: authentik-internal
       sectionName: https
   rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: authentik-server
+          port: 80
+          weight: 1
+      filters:
+        - type: URLRewrite
+          urlRewrite:
+            path:
+              type: ReplaceFullPath
+              replaceFullPath: /application/o/token/
+      matches:
+        - path:
+            type: Exact
+            value: /application/o/token
     - backendRefs:
         - group: ""
           kind: Service

apps/base/authentik/kustomization.yaml

@@ -19,6 +19,7 @@ resources:
   - redis-deployment.yaml
   - redis-pvc.yaml
   - redis-service.yaml
+  - server-vmpodscrape.yaml
   - vaultauth.yaml
   - vaultstaticsecret.yaml
   - vmpodscrape.yaml

apps/base/authentik/server-vmpodscrape.yaml

@@ -0,0 +1,16 @@
+---
+# Scrape the authentik server's django_prometheus endpoint (:9300). Picked up
+# by the observability VMAgent (selectAllByDefault).
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: authentik-server
+  namespace: authentik
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: authentik
+      app.kubernetes.io/component: server
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics

apps/base/bind-system/deployment.yaml

@@ -21,7 +21,7 @@ spec:
         runAsNonRoot: true
       containers:
         - name: operator
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.2.6
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.2.7
           args:
             - --metrics-bind-address=:8080
             - --health-probe-bind-address=:8081

apps/base/bind-system/kustomization.yaml

@@ -6,7 +6,7 @@ resources:
   - namespace.yaml
   # CRDs are pulled from the bind-operator repo at the matching tag rather than
   # vendored here, so they never drift from the operator.
-  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.2.6/config/crd/install.yaml
+  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.2.7/config/crd/install.yaml
   - rbac.yaml
   - agent-dns-rbac.yaml
   - deployment.yaml

apps/base/cheeztv/plugin-configmap.yaml

@@ -26,7 +26,7 @@ data:
           </key>
           <value>
             <PluginConfiguration>
-              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
               <OidClientId>jellyfin</OidClientId>
               <OidSecret>@@CLIENT_SECRET@@</OidSecret>
               <Enabled>true</Enabled>

apps/base/cheeztv/service.yaml

@@ -13,6 +13,4 @@ spec:
       targetPort: http
   selector:
     app: cheeztv
-  # Pin each client to one replica to reduce transcode-session churn/takeover.
-  sessionAffinity: ClientIP
   type: ClusterIP

apps/base/cheeztv/statefulset.yaml

@@ -4,6 +4,8 @@ kind: StatefulSet
 metadata:
   name: cheeztv
   namespace: cheeztv
+  annotations:
+    configmap.reloader.stakater.com/auto: "true"
 spec:
   # HA: two replicas coordinate transcode session ownership through Valkey and
   # resume each other's HLS segments off the shared RWX transcode PVC. Stable
@@ -162,7 +164,7 @@ spec:
               readOnly: true
       containers:
         - name: cheeztv
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.4.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

apps/base/fafflix/plugin-configmap.yaml

@@ -26,7 +26,7 @@ data:
           </key>
           <value>
             <PluginConfiguration>
-              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
               <OidClientId>jellyfin</OidClientId>
               <OidSecret>@@CLIENT_SECRET@@</OidSecret>
               <Enabled>true</Enabled>

apps/base/fafflix/service.yaml

@@ -13,6 +13,4 @@ spec:
       targetPort: http
   selector:
     app: fafflix
-  # Pin each client to one replica to reduce transcode-session churn/takeover.
-  sessionAffinity: ClientIP
   type: ClusterIP

apps/base/fafflix/statefulset.yaml

@@ -4,6 +4,8 @@ kind: StatefulSet
 metadata:
   name: fafflix
   namespace: fafflix
+  annotations:
+    configmap.reloader.stakater.com/auto: "true"
 spec:
   # HA: two replicas coordinate transcode session ownership through Valkey and
   # resume each other's HLS segments off the shared RWX transcode PVC. Stable
@@ -162,7 +164,7 @@ spec:
               readOnly: true
       containers:
         - name: fafflix
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.4.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

apps/base/pdbmux/deployment.yaml

@@ -25,7 +25,7 @@ spec:
         - name: pdbmux
           # Image is published by the pdbmux repo's .woodpecker/docker.yaml on
           # a v* tag. It only exists after that tag is cut (see PR merge gates).
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/pdbmux:v0.2.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/pdbmux:v0.4.0
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8080

apps/base/puppet/deployment_puppetserver-compiler.yaml

@@ -99,6 +99,23 @@ spec:
         - mountPath: /docker-custom-entrypoint.d/post-startup/additional-ruby-gems.sh
           name: additional-ruby-gems
           subPath: additional-ruby-gems.sh
+        - mountPath: /configmaps/auth.conf
+          name: compiler-auth-conf
+          subPath: auth.conf
+        - mountPath: /docker-custom-entrypoint.d/pre-default/10-auth-conf.sh
+          name: compiler-auth-conf-seed
+          subPath: 10-auth-conf.sh
+        - mountPath: /docker-custom-entrypoint.d/pre-default/20-vault-helpers.sh
+          name: compiler-vault-helpers-seed
+          subPath: 20-vault-helpers.sh
+        - mountPath: /opt/certmanager/config.yaml
+          name: certmanager-config
+          subPath: certmanager.yaml
+          readOnly: true
+        - mountPath: /opt/sshsignhost/config.yaml
+          name: sshsignhost-config
+          subPath: sshsignhost.yaml
+          readOnly: true
       initContainers:
       - name: copy-configmaps
         image: busybox:1.35
@@ -196,7 +213,38 @@ spec:
           echo "$EXPECTED  encapic" | sha256sum -c -
           install -m 0755 encapic /opt/bin/encapic
 
+          # Puppet shells out to these two from generate() during catalog
+          # compilation: profiles::pki::vault runs certmanager and
+          # profiles::ssh::sign runs sshsignhost.
+          install_release() {
+            name=$1
+            version=$2
+            asset="$name-linux-amd64"
+            base="https://git.unkin.net/unkin/$name/releases/download/$version"
+            curl -fsSL -o "$name" "$base/$asset"
+            curl -fsSL -o "$name.checksums" "$base/checksums.txt"
+            # checksums.txt covers every release asset; pick the line for the
+            # one we downloaded and verify it under our local filename.
+            expected=$(awk -v a="$asset" '$NF == a || $NF == "*"a {print $1}' "$name.checksums")
+            if [ -z "$expected" ]; then
+              echo "no checksum for $asset in $version checksums.txt" >&2
+              exit 1
+            fi
+            echo "$expected  $name" | sha256sum -c -
+            install -m 0755 "$name" "/opt/bin/$name"
+          }
+
+          install_release certmanager v0.2.0
+          install_release sshsignhost v0.1.0
+
           echo "Shared binaries setup completed"
+        resources:
+          limits:
+            cpu: 300m
+            memory: 256Mi
+          requests:
+            cpu: 100m
+            memory: 64Mi
         volumeMounts:
         - mountPath: /opt/bin/
           name: puppet-shared-bins
@@ -234,5 +282,22 @@ spec:
         configMap:
           name: additional-ruby-gems
           defaultMode: 0755
+      - name: compiler-auth-conf
+        configMap:
+          name: compiler-auth.conf
+      - name: compiler-auth-conf-seed
+        configMap:
+          name: compiler-auth-conf-seed
+          defaultMode: 0755
+      - name: compiler-vault-helpers-seed
+        configMap:
+          name: compiler-vault-helpers-seed
+          defaultMode: 0755
+      - name: certmanager-config
+        configMap:
+          name: certmanager-config
+      - name: sshsignhost-config
+        configMap:
+          name: sshsignhost-config
   strategy:
     type: RollingUpdate

apps/base/puppet/kustomization.yaml

@@ -54,6 +54,31 @@ configMapGenerator:
       - resources/compiler/puppetdb.conf
     options:
       disableNameSuffixHash: true
+  - name: compiler-auth.conf
+    files:
+      - resources/compiler/auth.conf
+    options:
+      disableNameSuffixHash: true
+  - name: compiler-auth-conf-seed
+    files:
+      - resources/compiler/10-auth-conf.sh
+    options:
+      disableNameSuffixHash: true
+  - name: compiler-vault-helpers-seed
+    files:
+      - resources/compiler/20-vault-helpers.sh
+    options:
+      disableNameSuffixHash: true
+  - name: certmanager-config
+    files:
+      - resources/compiler/certmanager.yaml
+    options:
+      disableNameSuffixHash: true
+  - name: sshsignhost-config
+    files:
+      - resources/compiler/sshsignhost.yaml
+    options:
+      disableNameSuffixHash: true
   - name: additional-ruby-gems
     files:
       - resources/additional-ruby-gems.sh

apps/base/puppet/resources/additional-ruby-gems.sh

@@ -6,4 +6,6 @@ echo "Installing additional Ruby gems..."
 /opt/puppetlabs/puppet/bin/gem install ipaddr
 /opt/puppetlabs/puppet/bin/gem install hiera-eyaml
 /opt/puppetlabs/puppet/bin/gem install toml
+# Under set -e a failed install kills the entrypoint post-startup hooks, taking down an already-serving compiler.
+/opt/puppetlabs/bin/puppetserver gem install toml
 echo "Additional Ruby gems installed successfully"

apps/base/puppet/resources/compiler/10-auth-conf.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -euo pipefail
+
+SRC=/configmaps/auth.conf
+DST=/etc/puppetlabs/puppetserver/conf.d/auth.conf
+
+# Copied rather than mounted: the entrypoint chowns conf.d and rewrites auth.conf,
+# both of which fail on a read-only configmap mount and abort container startup.
+if [ ! -s "$SRC" ]; then
+  echo "FATAL: $SRC missing or empty; refusing to start on the image default auth.conf" >&2
+  exit 1
+fi
+
+cp "$SRC" "$DST"

apps/base/puppet/resources/compiler/20-vault-helpers.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -euo pipefail
+
+BIN_DIR=/opt/bin
+CA=/opt/vault-ca-cert.crt
+
+if [ ! -s "$CA" ]; then
+  echo "FATAL: $CA missing or empty; certmanager and sshsignhost cannot verify Vault" >&2
+  exit 1
+fi
+
+# profiles::pki::vault and profiles::ssh::sign shell out to fixed /usr/local/bin
+# paths from generate(); the binaries ship on the shared PVC, and /usr/local/bin
+# lives in the image. Wrappers rather than symlinks because neither binary reads
+# a CA path from its config: SSL_CERT_FILE scopes the internal CA to these two
+# processes instead of the puppetserver JVM's own trust store.
+for bin in certmanager sshsignhost; do
+  if [ ! -x "$BIN_DIR/$bin" ]; then
+    echo "FATAL: $BIN_DIR/$bin missing; generate() would abort every catalog compile" >&2
+    exit 1
+  fi
+  cat > "/usr/local/bin/$bin" <<WRAPPER
+#!/bin/sh
+SSL_CERT_FILE=$CA
+export SSL_CERT_FILE
+exec $BIN_DIR/$bin "\$@"
+WRAPPER
+  chmod 0755 "/usr/local/bin/$bin"
+done

apps/base/puppet/resources/compiler/auth.conf

@@ -0,0 +1,320 @@
+# Copied into conf.d at startup by 10-auth-conf.sh; the entrypoint then appends the
+# admin API cache rule and re-renders the result, so the running file is not byte-identical.
+authorization: {
+    version: 1
+    rules: [
+        {
+            # Allow nodes to retrieve their own catalog
+            match-request: {
+                path: "^/puppet/v3/catalog/([^/]+)$"
+                type: regex
+                method: [get, post]
+            }
+            allow: "$1"
+            sort-order: 500
+            name: "puppetlabs v3 catalog from agents"
+        },
+        {
+            # Allow catalog-diff to retrieve catalogs on behalf of others.
+            # sort-order 400 must stay lower than the puppetlabs deny that follows: rules
+            # sort by [sort-order, name] and the first match wins.
+            match-request: {
+                path: "^/puppet/v4/catalog/?$"
+                type: regex
+                method: post
+            }
+            allow: "catalog-diff.main.unkin.net"
+            sort-order: 400
+            name: "unkin v4 catalog for catalog-diff"
+        },
+        {
+            # Allow services to retrieve catalogs on behalf of others
+            match-request: {
+                path: "^/puppet/v4/catalog/?$"
+                type: regex
+                method: post
+            }
+            deny: "*"
+            sort-order: 500
+            name: "puppetlabs v4 catalog for services"
+        },
+        {
+            # Allow nodes to retrieve the certificate they requested earlier
+            match-request: {
+                path: "/puppet-ca/v1/certificate/"
+                type: path
+                method: get
+            }
+            allow-unauthenticated: true
+            sort-order: 500
+            name: "puppetlabs certificate"
+        },
+        {
+            # Allow all nodes to access the certificate revocation list
+            match-request: {
+                path: "/puppet-ca/v1/certificate_revocation_list/ca"
+                type: path
+                method: get
+            }
+            allow-unauthenticated: true
+            sort-order: 500
+            name: "puppetlabs crl"
+        },
+        {
+            # Allow nodes to request a new certificate
+            match-request: {
+                path: "/puppet-ca/v1/certificate_request"
+                type: path
+                method: [get, put]
+            }
+            allow-unauthenticated: true
+            sort-order: 500
+            name: "puppetlabs csr"
+        },
+        {
+            # Allow nodes to renew their certificate
+            match-request: {
+                path: "/puppet-ca/v1/certificate_renewal"
+                type: path
+                method: post
+            }
+            # this endpoint should never be unauthenticated, as it requires the cert to be provided.
+            allow: "*"
+            sort-order: 500
+            name: "puppetlabs certificate renewal"
+        },
+        {
+            # Allow the CA CLI to access the certificate_status endpoint
+            match-request: {
+                path: "/puppet-ca/v1/certificate_status"
+                type: path
+                method: [get, put, delete]
+            }
+            allow: {
+               extensions: {
+                   pp_cli_auth: "true"
+               }
+            }
+            sort-order: 500
+            name: "puppetlabs cert status"
+        },
+        {
+            match-request: {
+                path: "^/puppet-ca/v1/certificate_revocation_list$"
+                type: regex
+                method: put
+            }
+            allow: {
+               extensions: {
+                   pp_cli_auth: "true"
+               }
+            }
+            sort-order: 500
+            name: "puppetlabs CRL update"
+        },
+        {
+            # Allow the CA CLI to access the certificate_statuses endpoint
+            match-request: {
+                path: "/puppet-ca/v1/certificate_statuses"
+                type: path
+                method: get
+            }
+            allow: {
+               extensions: {
+                   pp_cli_auth: "true"
+               }
+            }
+            sort-order: 500
+            name: "puppetlabs cert statuses"
+        },
+        {
+            # Allow authenticated access to the CA expirations endpoint
+            match-request: {
+                path: "/puppet-ca/v1/expirations"
+                type: path
+                method: get
+            }
+            allow: "*"
+            sort-order: 500
+            name: "puppetlabs CA cert and CRL expirations"
+        },
+        {
+            # Allow the CA CLI to access the certificate clean endpoint
+            match-request: {
+                path: "/puppet-ca/v1/clean"
+                type: path
+                method: put
+            }
+            allow: {
+               extensions: {
+                   pp_cli_auth: "true"
+               }
+            }
+            sort-order: 500
+            name: "puppetlabs cert clean"
+        },
+        {
+            # Allow the CA CLI to access the certificate sign endpoint
+            match-request: {
+                path: "/puppet-ca/v1/sign"
+                type: path
+                method: post
+            }
+            allow: {
+               extensions: {
+                   pp_cli_auth: "true"
+               }
+            }
+            sort-order: 500
+            name: "puppetlabs cert sign"
+        },
+        {
+            # Allow the CA CLI to access the certificate sign all endpoint
+            match-request: {
+                path: "/puppet-ca/v1/sign/all"
+                type: path
+                method: post
+            }
+            allow: {
+               extensions: {
+                   pp_cli_auth: "true"
+               }
+            }
+            sort-order: 500
+            name: "puppetlabs cert sign all"
+        },
+        {
+            # Allow unauthenticated access to the status service endpoint
+            match-request: {
+                path: "/status/v1/services"
+                type: path
+                method: get
+            }
+            allow-unauthenticated: true
+            sort-order: 500
+            name: "puppetlabs status service - full"
+        },
+        {
+            match-request: {
+                path: "/status/v1/simple"
+                type: path
+                method: get
+            }
+            allow-unauthenticated: true
+            sort-order: 500
+            name: "puppetlabs status service - simple"
+        },
+        {
+            match-request: {
+                path: "/puppet/v3/environments"
+                type: path
+                method: get
+            }
+            allow: "*"
+            sort-order: 500
+            name: "puppetlabs environments"
+        },
+        {
+            # Allow nodes to access all file_bucket_files.  Note that access for
+            # the 'delete' method is forbidden by Puppet regardless of the
+            # configuration of this rule.
+            match-request: {
+                path: "/puppet/v3/file_bucket_file"
+                type: path
+                method: [get, head, post, put]
+            }
+            allow: "*"
+            sort-order: 500
+            name: "puppetlabs file bucket file"
+        },
+        {
+            # Allow nodes to access all file_content.  Note that access for the
+            # 'delete' method is forbidden by Puppet regardless of the
+            # configuration of this rule.
+            match-request: {
+                path: "/puppet/v3/file_content"
+                type: path
+                method: [get, post]
+            }
+            allow: "*"
+            sort-order: 500
+            name: "puppetlabs file content"
+        },
+        {
+            # Allow nodes to access all file_metadata.  Note that access for the
+            # 'delete' method is forbidden by Puppet regardless of the
+            # configuration of this rule.
+            match-request: {
+                path: "/puppet/v3/file_metadata"
+                type: path
+                method: [get, post]
+            }
+            allow: "*"
+            sort-order: 500
+            name: "puppetlabs file metadata"
+        },
+        {
+            # Allow nodes to retrieve only their own node definition
+            match-request: {
+                path: "^/puppet/v3/node/([^/]+)$"
+                type: regex
+                method: get
+            }
+            allow: "$1"
+            sort-order: 500
+            name: "puppetlabs node"
+        },
+        {
+            # Allow nodes to store only their own reports
+            match-request: {
+                path: "^/puppet/v3/report/([^/]+)$"
+                type: regex
+                method: put
+            }
+            allow: "$1"
+            sort-order: 500
+            name: "puppetlabs report"
+        },
+        {
+            # Allow nodes to update their own facts
+            match-request: {
+                path: "^/puppet/v3/facts/([^/]+)$"
+                type: regex
+                method: put
+            }
+            allow: "$1"
+            sort-order: 500
+            name: "puppetlabs facts"
+        },
+        {
+            match-request: {
+                path: "/puppet/v3/static_file_content"
+                type: path
+                method: get
+            }
+            allow: "*"
+            sort-order: 500
+            name: "puppetlabs static file content"
+        },
+        {
+            match-request: {
+                path: "/puppet/v3/tasks"
+                type: path
+            }
+            allow: "*"
+            sort-order: 500
+            name: "puppet tasks information"
+        },
+        {
+            # Deny everything else. This ACL is not strictly
+            # necessary, but illustrates the default policy
+            match-request: {
+                path: "/"
+                type: path
+            }
+            deny: "*"
+            sort-order: 999
+            name: "puppetlabs deny all"
+        }
+    ]
+}

apps/base/puppet/resources/compiler/certmanager.yaml

@@ -0,0 +1,12 @@
+---
+vault:
+  addr: https://vault.service.consul:8200
+  auth_method: kubernetes
+  k8s_mount: k8s/au/syd1
+  k8s_role: puppet_certmanager
+  jwt_path: /var/run/secrets/kubernetes.io/serviceaccount/token
+  mount_point: pki_int
+  role_name: servers_default
+  output_path: /tmp/certmanager
+  tls_skip_verify: false
+  timeout: 30s

apps/base/puppet/resources/compiler/sshsignhost.yaml

@@ -0,0 +1,11 @@
+---
+vault:
+  addr: https://vault.service.consul:8200
+  auth_method: kubernetes
+  k8s_mount: k8s/au/syd1
+  k8s_role: puppet_sshsigner
+  jwt_path: /var/run/secrets/kubernetes.io/serviceaccount/token
+  mount_point: sshca
+  role_name: signhost
+  tls_skip_verify: false
+  timeout: 30s

apps/base/woodpecker/kustomization.yaml

@@ -15,6 +15,7 @@ resources:
   - serviceaccount_mediamark_ci.yaml
   - serviceaccount_plugin_docker_buildx.yaml
   - serviceaccount_jellyfin_ha_src.yaml
+  - serviceaccount_jellyfin_plugin_sso.yaml
   - serviceaccount_repospawner_ci.yaml
   - serviceaccount_terraform_artifactapi.yaml
   - serviceaccount_terraform_authentik.yaml

apps/base/woodpecker/serviceaccount_jellyfin_plugin_sso.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: jellyfin-plugin-sso
+  namespace: woodpecker

apps/overlays/au-syd1/woodpecker/values.yaml

@@ -4,7 +4,7 @@ agent:
     WOODPECKER_MAX_WORKFLOWS: "8"
     WOODPECKER_BACKEND_K8S_PRIORITY_CLASS: power
     WOODPECKER_BACKEND_K8S_STORAGE_CLASS: cephrbd-fast-delete
-    WOODPECKER_BACKEND_K8S_VOLUME_SIZE: 10G
+    WOODPECKER_BACKEND_K8S_VOLUME_SIZE: 20Gi
     WOODPECKER_BACKEND_K8S_STORAGE_RWX: false
     # Required from woodpecker 3.16.0 (GHSA-qf34-295c-26v8): step-level
     # serviceAccountName is gated behind this agent flag (default false).

clusters/au-syd1/bootstrap/argocd-cm-patch.yaml

@@ -26,6 +26,10 @@ data:
     issuer: https://identity.unkin.net/application/o/argocd/
     clientID: argocd
     clientSecret: $argocd-oidc:client_secret
+    # The Authentik client is public (the iOS app can't hold a secret), so
+    # Authentik no longer enforces clientSecret; PKCE replaces it as the
+    # protection against authorization-code interception.
+    enablePKCEAuthentication: true
     # identity.unkin.net now serves the LetsEncrypt *.unkin.net wildcard, so the
     # stock image trust store validates it; no rootCA pin.
     requestedScopes: