Trust internal CA for Authentik SSO; canonical identity.unkin.net for NetBox (#314)

Authentik is canonical at https://identity.unkin.net, served by the internal
unkin.net CA. Grafana, LiteLLM and NetBox failed OIDC discovery because their
images don't trust that CA (x509: unknown authority); NetBox also still pointed
at the secondary admin host.

- grafana: mount the reflected vault-ca-cert; set generic_oauth `tls_client_ca`.
- litellm: `combine-certs` init builds public+internal CA bundle; `SSL_CERT_FILE`
  + `REQUESTS_CA_BUNDLE` point at it.
- netbox: flip OIDC issuer to identity.unkin.net; same combine bundle for
  python-social-auth (`requests`).
- docs: record the Rancher manual runtime step (issuer + CA in the auth config).

Validated: kustomize build + kubeconform + pre-commit.

https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
Reviewed-on: #314
Co-authored-by: Ben Vincent <ben@unkin.net>
Co-committed-by: Ben Vincent <ben@unkin.net>

apps/base/grafana/grafana.yaml

@@ -26,6 +26,13 @@ spec:
                     secretKeyRef:
                       name: oauth-credentials
                       key: client_secret
+              # identity.unkin.net is served by the internal unkin.net CA, which
+              # the stock Grafana image doesn't trust. Mount the reflected
+              # vault-ca-cert and point generic_oauth's tls_client_ca at it.
+              volumeMounts:
+                - name: vault-ca-cert
+                  mountPath: /etc/grafana/vault-ca
+                  readOnly: true
               resources:
                 requests:
                   cpu: 100m
@@ -33,6 +40,13 @@ spec:
                 limits:
                   cpu: "1"
                   memory: 1Gi
+          volumes:
+            - name: vault-ca-cert
+              secret:
+                secretName: vault-ca-cert
+                items:
+                  - key: ca.crt
+                    path: ca.crt
   config:
     server:
       root_url: "https://grafana.k8s.syd1.au.unkin.net"
@@ -57,6 +71,9 @@ spec:
       auth_url: "https://identity.unkin.net/application/o/authorize/"
       token_url: "https://identity.unkin.net/application/o/token/"
       api_url: "https://identity.unkin.net/application/o/userinfo/"
+      # Trust the internal unkin.net CA that signs identity.unkin.net's cert
+      # (mounted from the reflected vault-ca-cert Secret).
+      tls_client_ca: "/etc/grafana/vault-ca/ca.crt"
       # Authentik permission groups -> Grafana roles. akP-grafana-admin is granted
       # to akR-global-admin members (and direct members) via terraform-authentik.
       role_attribute_path: "contains(ak_groups[*], 'akP-grafana-admin') && 'Admin' || 'Viewer'"

apps/base/litellm/deployment.yaml

@@ -15,6 +15,23 @@ spec:
       labels:
         app: litellm
     spec:
+      # LiteLLM's SSO client reaches identity.unkin.net, whose cert is signed by
+      # the internal unkin.net CA. Combine the image's public roots with the
+      # reflected vault-ca-cert into one bundle (SSL_CERT_FILE/REQUESTS_CA_BUNDLE
+      # in litellm-env point at it) so internal OIDC and public HTTPS both work.
+      initContainers:
+        - name: combine-certs
+          image: alpine:3
+          command:
+            - sh
+            - -c
+            - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt
+          volumeMounts:
+            - name: vault-ca-cert
+              mountPath: /custom-ca
+              readOnly: true
+            - name: combined-certs
+              mountPath: /combined-certs
       containers:
         - name: litellm
           image: docker.litellm.ai/berriai/litellm-database:main-stable
@@ -72,8 +89,19 @@ spec:
             - mountPath: /app/config.yaml
               name: config
               subPath: config.yaml
+            - name: combined-certs
+              mountPath: /etc/ssl/combined
+              readOnly: true
       restartPolicy: Always
       volumes:
         - name: config
           configMap:
             name: litellm-config
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: combined-certs
+          emptyDir: {}

apps/base/litellm/kustomization.yaml

@@ -39,5 +39,9 @@ configMapGenerator:
       - GENERIC_SCOPE=openid email profile litellm_role
       - GENERIC_USER_ROLE_ATTRIBUTE=litellm_role
       - PROXY_BASE_URL=https://litellm.k8s.syd1.au.unkin.net
+      # Trust the internal unkin.net CA (identity.unkin.net) via the combined
+      # bundle assembled by the combine-certs init container.
+      - SSL_CERT_FILE=/etc/ssl/combined/ca-certificates.crt
+      - REQUESTS_CA_BUNDLE=/etc/ssl/combined/ca-certificates.crt
     options:
       disableNameSuffixHash: true

apps/overlays/au-syd1/netbox/values.yaml

@@ -89,9 +89,11 @@ remoteAuth:
     - social_core.backends.open_id_connect.OpenIdConnectAuth
   autoCreateUser: true
 extraConfig:
-  # index 0 -> /run/config/extra/0/extra-0.yaml (non-secret OIDC config)
+  # index 0 -> /run/config/extra/0/extra-0.yaml (non-secret OIDC config).
+  # Canonical Authentik host identity.unkin.net (served by the internal unkin.net
+  # CA; trusted via the combined bundle mounted below).
   - values:
-      SOCIAL_AUTH_OIDC_OIDC_ENDPOINT: https://identity.k8s.syd1.au.unkin.net/application/o/netbox/
+      SOCIAL_AUTH_OIDC_OIDC_ENDPOINT: https://identity.unkin.net/application/o/netbox/
       SOCIAL_AUTH_OIDC_KEY: netbox
   # index 1 -> /run/config/extra/1/oidc.yaml (client secret, from Vault via VSO)
   - secret:
@@ -99,3 +101,39 @@ extraConfig:
       items:
         - key: oidc.yaml
           path: oidc.yaml
+
+# python-social-auth uses `requests` to reach identity.unkin.net, whose cert is
+# signed by the internal unkin.net CA. Combine the image's public roots with the
+# reflected vault-ca-cert into one bundle and point requests/OpenSSL at it, so
+# both internal (OIDC) and public HTTPS keep working.
+initContainers:
+  - name: combine-certs
+    image: alpine:3
+    command:
+      - sh
+      - -c
+      - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt
+    volumeMounts:
+      - name: vault-ca-cert
+        mountPath: /custom-ca
+        readOnly: true
+      - name: combined-certs
+        mountPath: /combined-certs
+extraVolumes:
+  - name: vault-ca-cert
+    secret:
+      secretName: vault-ca-cert
+      items:
+        - key: ca.crt
+          path: ca.crt
+  - name: combined-certs
+    emptyDir: {}
+extraVolumeMounts:
+  - name: combined-certs
+    mountPath: /etc/ssl/combined
+    readOnly: true
+extraEnvs:
+  - name: REQUESTS_CA_BUNDLE
+    value: /etc/ssl/combined/ca-certificates.crt
+  - name: SSL_CERT_FILE
+    value: /etc/ssl/combined/ca-certificates.crt