diff --git a/apps/base/grafana/grafana.yaml b/apps/base/grafana/grafana.yaml index 1665d71..82b6a1f 100644 --- a/apps/base/grafana/grafana.yaml +++ b/apps/base/grafana/grafana.yaml @@ -26,6 +26,13 @@ spec: secretKeyRef: name: oauth-credentials key: client_secret + # identity.unkin.net is served by the internal unkin.net CA, which + # the stock Grafana image doesn't trust. Mount the reflected + # vault-ca-cert and point generic_oauth's tls_client_ca at it. + volumeMounts: + - name: vault-ca-cert + mountPath: /etc/grafana/vault-ca + readOnly: true resources: requests: cpu: 100m @@ -33,6 +40,13 @@ spec: limits: cpu: "1" memory: 1Gi + volumes: + - name: vault-ca-cert + secret: + secretName: vault-ca-cert + items: + - key: ca.crt + path: ca.crt config: server: root_url: "https://grafana.k8s.syd1.au.unkin.net" @@ -57,6 +71,9 @@ spec: auth_url: "https://identity.unkin.net/application/o/authorize/" token_url: "https://identity.unkin.net/application/o/token/" api_url: "https://identity.unkin.net/application/o/userinfo/" + # Trust the internal unkin.net CA that signs identity.unkin.net's cert + # (mounted from the reflected vault-ca-cert Secret). + tls_client_ca: "/etc/grafana/vault-ca/ca.crt" # Authentik permission groups -> Grafana roles. akP-grafana-admin is granted # to akR-global-admin members (and direct members) via terraform-authentik. role_attribute_path: "contains(ak_groups[*], 'akP-grafana-admin') && 'Admin' || 'Viewer'" diff --git a/apps/base/litellm/deployment.yaml b/apps/base/litellm/deployment.yaml index 68230d6..0a70f65 100644 --- a/apps/base/litellm/deployment.yaml +++ b/apps/base/litellm/deployment.yaml @@ -15,6 +15,23 @@ spec: labels: app: litellm spec: + # LiteLLM's SSO client reaches identity.unkin.net, whose cert is signed by + # the internal unkin.net CA. Combine the image's public roots with the + # reflected vault-ca-cert into one bundle (SSL_CERT_FILE/REQUESTS_CA_BUNDLE + # in litellm-env point at it) so internal OIDC and public HTTPS both work. + initContainers: + - name: combine-certs + image: alpine:3 + command: + - sh + - -c + - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt + volumeMounts: + - name: vault-ca-cert + mountPath: /custom-ca + readOnly: true + - name: combined-certs + mountPath: /combined-certs containers: - name: litellm image: docker.litellm.ai/berriai/litellm-database:main-stable @@ -72,8 +89,19 @@ spec: - mountPath: /app/config.yaml name: config subPath: config.yaml + - name: combined-certs + mountPath: /etc/ssl/combined + readOnly: true restartPolicy: Always volumes: - name: config configMap: name: litellm-config + - name: vault-ca-cert + secret: + secretName: vault-ca-cert + items: + - key: ca.crt + path: ca.crt + - name: combined-certs + emptyDir: {} diff --git a/apps/base/litellm/kustomization.yaml b/apps/base/litellm/kustomization.yaml index 32f4171..5a582cd 100644 --- a/apps/base/litellm/kustomization.yaml +++ b/apps/base/litellm/kustomization.yaml @@ -39,5 +39,9 @@ configMapGenerator: - GENERIC_SCOPE=openid email profile litellm_role - GENERIC_USER_ROLE_ATTRIBUTE=litellm_role - PROXY_BASE_URL=https://litellm.k8s.syd1.au.unkin.net + # Trust the internal unkin.net CA (identity.unkin.net) via the combined + # bundle assembled by the combine-certs init container. + - SSL_CERT_FILE=/etc/ssl/combined/ca-certificates.crt + - REQUESTS_CA_BUNDLE=/etc/ssl/combined/ca-certificates.crt options: disableNameSuffixHash: true diff --git a/apps/overlays/au-syd1/netbox/values.yaml b/apps/overlays/au-syd1/netbox/values.yaml index 6d5665a..967cde9 100644 --- a/apps/overlays/au-syd1/netbox/values.yaml +++ b/apps/overlays/au-syd1/netbox/values.yaml @@ -89,9 +89,11 @@ remoteAuth: - social_core.backends.open_id_connect.OpenIdConnectAuth autoCreateUser: true extraConfig: - # index 0 -> /run/config/extra/0/extra-0.yaml (non-secret OIDC config) + # index 0 -> /run/config/extra/0/extra-0.yaml (non-secret OIDC config). + # Canonical Authentik host identity.unkin.net (served by the internal unkin.net + # CA; trusted via the combined bundle mounted below). - values: - SOCIAL_AUTH_OIDC_OIDC_ENDPOINT: https://identity.k8s.syd1.au.unkin.net/application/o/netbox/ + SOCIAL_AUTH_OIDC_OIDC_ENDPOINT: https://identity.unkin.net/application/o/netbox/ SOCIAL_AUTH_OIDC_KEY: netbox # index 1 -> /run/config/extra/1/oidc.yaml (client secret, from Vault via VSO) - secret: @@ -99,3 +101,39 @@ extraConfig: items: - key: oidc.yaml path: oidc.yaml + +# python-social-auth uses `requests` to reach identity.unkin.net, whose cert is +# signed by the internal unkin.net CA. Combine the image's public roots with the +# reflected vault-ca-cert into one bundle and point requests/OpenSSL at it, so +# both internal (OIDC) and public HTTPS keep working. +initContainers: + - name: combine-certs + image: alpine:3 + command: + - sh + - -c + - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt + volumeMounts: + - name: vault-ca-cert + mountPath: /custom-ca + readOnly: true + - name: combined-certs + mountPath: /combined-certs +extraVolumes: + - name: vault-ca-cert + secret: + secretName: vault-ca-cert + items: + - key: ca.crt + path: ca.crt + - name: combined-certs + emptyDir: {} +extraVolumeMounts: + - name: combined-certs + mountPath: /etc/ssl/combined + readOnly: true +extraEnvs: + - name: REQUESTS_CA_BUNDLE + value: /etc/ssl/combined/ca-certificates.crt + - name: SSL_CERT_FILE + value: /etc/ssl/combined/ca-certificates.crt diff --git a/docs/README.md b/docs/README.md index 51f96d1..8ab0681 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,3 +6,4 @@ Operational notes for the manifests in this repo. | --- | --- | | [cnpg-backups.md](cnpg-backups.md) | How CNPG Postgres backups (WAL archiving + nightly base backups) to Ceph RGW are configured. | | [cnpg-restore.md](cnpg-restore.md) | Restoring a CNPG cluster: full recovery, point-in-time recovery, cutover, and gotchas. | +| [authentik-rancher-sso.md](authentik-rancher-sso.md) | Manual runtime step to point Rancher's OIDC auth at the canonical `identity.unkin.net` issuer and trust the internal CA. | diff --git a/docs/authentik-rancher-sso.md b/docs/authentik-rancher-sso.md new file mode 100644 index 0000000..afdf0cb --- /dev/null +++ b/docs/authentik-rancher-sso.md @@ -0,0 +1,59 @@ +# Rancher Authentik SSO — manual runtime step + +Rancher's Authentik/OIDC login is a cluster-scoped **runtime** object +(`authconfigs.management.cattle.io`, name `keycloakoidc`). It is enabled through +Rancher's verify-auth flow (see `terraform-rancher`), not GitOps, and it is not +declaratively reconcilable without risking admin lockout — so the two fields +below must be set by hand in the Rancher UI (or API). This doc is the record of +that step; nothing in this repo applies it. + +## Why this is needed + +- **Canonical issuer.** Authentik is canonical at `https://identity.unkin.net`. + Rancher's OIDC issuer must be `https://identity.unkin.net/application/o/rancher/`. +- **Internal CA trust.** `identity.unkin.net` presents a cert signed by the + internal `unkin.net` CA. Rancher's Go OIDC client does not trust it out of the + box, so discovery fails with: + + ``` + Get "https://identity.unkin.net/application/o/rancher/.well-known/openid-configuration": + x509: certificate signed by unknown authority + ``` + + Rancher's Keycloak-OIDC auth provider has a **Certificate** field that seeds an + extra trust anchor for exactly this. Paste the `unkin.net` CA chain there. + +## The step + +1. Grab the CA chain (root + intermediate PEM — same bundle as the reflected + `vault-ca-cert` Secret / argocd-apps #305): + + ```sh + vault read -field=ca_chain pki_int/cert/ca_chain + ``` + +2. In Rancher: **☰ → Users & Authentication → Auth Provider → Keycloak (OIDC)** + (or `PUT /v3/keycloakOIDCConfigs/keycloakoidc` via the API) and set: + + | Field | Value | + | --- | --- | + | Issuer / `issuer` | `https://identity.unkin.net/application/o/rancher/` | + | Rancher URL / `rancherUrl` | `https://rancher.k8s.syd1.au.unkin.net/verify-auth` | + | Client ID | `rancher` | + | Certificate / `certificate` | *(paste the full PEM chain from step 1)* | + + Leave Client Secret and the `unrestricted` access mode as configured by + `terraform-rancher`. + +3. Save. Rancher re-runs discovery against `identity.unkin.net`; with the CA in + the Certificate field the `x509` error clears and a test login succeeds. + +## Notes + +- `terraform-rancher` (rancher2 provider, `rancher2_auth_config_keycloak_oidc`) + *can* set `issuer`/`certificate` declaratively. It does not manage the + certificate today; adding `certificate = file(...)` there and re-applying is the + recommended long-term home for this so it survives a re-provision. Until then, + this manual step is authoritative. +- The Certificate field trusts an extra CA; it does not replace Rancher's system + trust, so public TLS is unaffected.