Trust internal CA for Authentik SSO; make identity.unkin.net canonical for netbox

identity.unkin.net is now the canonical Authentik host. Grafana, LiteLLM and
NetBox reach it over TLS served by the internal unkin.net CA, which their images
don't trust, so OIDC/discovery failed with x509 unknown-authority. NetBox also
still pointed at the secondary admin host.

- grafana: mount the reflected vault-ca-cert and set generic_oauth tls_client_ca.
- litellm: combine-certs init builds a public+internal CA bundle; SSL_CERT_FILE
  and REQUESTS_CA_BUNDLE point at it.
- netbox: flip the OIDC issuer to identity.unkin.net; same combine-certs bundle
  for python-social-auth (requests).
- docs: record the Rancher manual runtime step (issuer + CA in the auth config).

Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
Signed-off-by: Ben Vincent <ben@unkin.net>

apps/base/grafana/grafana.yaml

@@ -26,6 +26,13 @@ spec:
                     secretKeyRef:
                       name: oauth-credentials
                       key: client_secret
+              # identity.unkin.net is served by the internal unkin.net CA, which
+              # the stock Grafana image doesn't trust. Mount the reflected
+              # vault-ca-cert and point generic_oauth's tls_client_ca at it.
+              volumeMounts:
+                - name: vault-ca-cert
+                  mountPath: /etc/grafana/vault-ca
+                  readOnly: true
               resources:
                 requests:
                   cpu: 100m
@@ -33,6 +40,13 @@ spec:
                 limits:
                   cpu: "1"
                   memory: 1Gi
+          volumes:
+            - name: vault-ca-cert
+              secret:
+                secretName: vault-ca-cert
+                items:
+                  - key: ca.crt
+                    path: ca.crt
   config:
     server:
       root_url: "https://grafana.k8s.syd1.au.unkin.net"
@@ -57,6 +71,9 @@ spec:
       auth_url: "https://identity.unkin.net/application/o/authorize/"
       token_url: "https://identity.unkin.net/application/o/token/"
       api_url: "https://identity.unkin.net/application/o/userinfo/"
+      # Trust the internal unkin.net CA that signs identity.unkin.net's cert
+      # (mounted from the reflected vault-ca-cert Secret).
+      tls_client_ca: "/etc/grafana/vault-ca/ca.crt"
       # Authentik permission groups -> Grafana roles. akP-grafana-admin is granted
       # to akR-global-admin members (and direct members) via terraform-authentik.
       role_attribute_path: "contains(ak_groups[*], 'akP-grafana-admin') && 'Admin' || 'Viewer'"

apps/base/litellm/deployment.yaml

@@ -15,6 +15,23 @@ spec:
       labels:
         app: litellm
     spec:
+      # LiteLLM's SSO client reaches identity.unkin.net, whose cert is signed by
+      # the internal unkin.net CA. Combine the image's public roots with the
+      # reflected vault-ca-cert into one bundle (SSL_CERT_FILE/REQUESTS_CA_BUNDLE
+      # in litellm-env point at it) so internal OIDC and public HTTPS both work.
+      initContainers:
+        - name: combine-certs
+          image: alpine:3
+          command:
+            - sh
+            - -c
+            - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt
+          volumeMounts:
+            - name: vault-ca-cert
+              mountPath: /custom-ca
+              readOnly: true
+            - name: combined-certs
+              mountPath: /combined-certs
       containers:
         - name: litellm
           image: docker.litellm.ai/berriai/litellm-database:main-stable
@@ -72,8 +89,19 @@ spec:
             - mountPath: /app/config.yaml
               name: config
               subPath: config.yaml
+            - name: combined-certs
+              mountPath: /etc/ssl/combined
+              readOnly: true
       restartPolicy: Always
       volumes:
         - name: config
           configMap:
             name: litellm-config
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: combined-certs
+          emptyDir: {}

apps/base/litellm/kustomization.yaml

@@ -39,5 +39,9 @@ configMapGenerator:
       - GENERIC_SCOPE=openid email profile litellm_role
       - GENERIC_USER_ROLE_ATTRIBUTE=litellm_role
       - PROXY_BASE_URL=https://litellm.k8s.syd1.au.unkin.net
+      # Trust the internal unkin.net CA (identity.unkin.net) via the combined
+      # bundle assembled by the combine-certs init container.
+      - SSL_CERT_FILE=/etc/ssl/combined/ca-certificates.crt
+      - REQUESTS_CA_BUNDLE=/etc/ssl/combined/ca-certificates.crt
     options:
       disableNameSuffixHash: true

apps/overlays/au-syd1/netbox/values.yaml

@@ -89,9 +89,11 @@ remoteAuth:
     - social_core.backends.open_id_connect.OpenIdConnectAuth
   autoCreateUser: true
 extraConfig:
-  # index 0 -> /run/config/extra/0/extra-0.yaml (non-secret OIDC config)
+  # index 0 -> /run/config/extra/0/extra-0.yaml (non-secret OIDC config).
+  # Canonical Authentik host identity.unkin.net (served by the internal unkin.net
+  # CA; trusted via the combined bundle mounted below).
   - values:
-      SOCIAL_AUTH_OIDC_OIDC_ENDPOINT: https://identity.k8s.syd1.au.unkin.net/application/o/netbox/
+      SOCIAL_AUTH_OIDC_OIDC_ENDPOINT: https://identity.unkin.net/application/o/netbox/
       SOCIAL_AUTH_OIDC_KEY: netbox
   # index 1 -> /run/config/extra/1/oidc.yaml (client secret, from Vault via VSO)
   - secret:
@@ -99,3 +101,39 @@ extraConfig:
       items:
         - key: oidc.yaml
           path: oidc.yaml
+
+# python-social-auth uses `requests` to reach identity.unkin.net, whose cert is
+# signed by the internal unkin.net CA. Combine the image's public roots with the
+# reflected vault-ca-cert into one bundle and point requests/OpenSSL at it, so
+# both internal (OIDC) and public HTTPS keep working.
+initContainers:
+  - name: combine-certs
+    image: alpine:3
+    command:
+      - sh
+      - -c
+      - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt
+    volumeMounts:
+      - name: vault-ca-cert
+        mountPath: /custom-ca
+        readOnly: true
+      - name: combined-certs
+        mountPath: /combined-certs
+extraVolumes:
+  - name: vault-ca-cert
+    secret:
+      secretName: vault-ca-cert
+      items:
+        - key: ca.crt
+          path: ca.crt
+  - name: combined-certs
+    emptyDir: {}
+extraVolumeMounts:
+  - name: combined-certs
+    mountPath: /etc/ssl/combined
+    readOnly: true
+extraEnvs:
+  - name: REQUESTS_CA_BUNDLE
+    value: /etc/ssl/combined/ca-certificates.crt
+  - name: SSL_CERT_FILE
+    value: /etc/ssl/combined/ca-certificates.crt