diff --git a/apps/base/kanidm/README.md b/apps/base/kanidm/README.md index 8861d79..de13a85 100644 --- a/apps/base/kanidm/README.md +++ b/apps/base/kanidm/README.md @@ -1,32 +1,51 @@ # kanidm -Single-replica kanidm identity server deployment. +Three-replica kanidm identity server with Vault-managed replication certificates. + +## Architecture + +- Per-pod `server-N.toml` in `resources/` — each has its own replication origin hardcoded +- `config-init` busybox init container copies the right config and injects peer certs from the + vault-synced `kanidm-repl-certs` Secret at pod startup +- `reloader.stakater.com/auto: "true"` triggers a rolling restart when the ConfigMap or Secret changes +- Vault path: `kv/kubernetes/namespace/kanidm/default/repl-certs` + - Keys: `kanidm-0`, `kanidm-1`, `kanidm-2` — each holds that pod's replication certificate ## Initial setup -After the pod starts for the first time, generate the admin and idm_admin credentials: +After the first pod starts, generate the admin credentials: ```bash -kubectl exec -n kanidm kanidm-0 -- /sbin/kanidmd recover-account admin -kubectl exec -n kanidm kanidm-0 -- /sbin/kanidmd recover-account idm_admin +kubectl exec -n kanidm kanidm-0 -- /sbin/kanidmd recover-account -c /config/server.toml admin +kubectl exec -n kanidm kanidm-0 -- /sbin/kanidmd recover-account -c /config/server.toml idm_admin ``` -## Adding replication +## Replication certificate rotation -If replication is needed in the future: - -1. Scale the StatefulSet to 3 replicas and add `podAntiAffinity` to spread across nodes. -2. Add a `[replication]` section to `configmap.yaml` per pod (origin is pod-specific: - `repl://kanidm-N.kanidm-headless.kanidm.svc.cluster.local:8444`). -3. Add the replication port (8444) back to the StatefulSet container ports and headless service. -4. Restore `rbac.yaml` for the cert-publisher sidecar, or exchange certificates manually: +When certs need to be renewed, update vault and reloader will roll the StatefulSet: ```bash -# On each pod, get its replication certificate -kubectl exec -n kanidm kanidm-0 -- /sbin/kanidmd renew-replication-certificate +# Get new cert from a pod +kubectl exec -it -n kanidm kanidm-N -- /sbin/kanidmd renew-replication-certificate -c /config/server.toml -# Add each peer's certificate to the other pods' configs under: -# [replication."repl://:8444"] -# type = "mutual-pull" -# partner_cert = "" +# Write updated cert to vault (reloader triggers restart automatically) +vault kv patch kv/kubernetes/namespace/kanidm/default/repl-certs "kanidm-N=" +``` + +## Resolving domain UUID mismatch + +If pods initialized independently (each with a different domain UUID), replication will fail with +`Consumer Domain UUID does not match`. Fix by resetting kanidm-1 and kanidm-2 to sync from +kanidm-0 (the authoritative node): + +```bash +# Scale down to avoid split-brain during reset +kubectl scale statefulset -n kanidm kanidm --replicas=1 + +# Delete the stale PVCs for the replica pods +kubectl delete pvc -n kanidm data-kanidm-1 data-kanidm-2 + +# Scale back up — replicas start with empty DBs and automatic_refresh=true +# will trigger a full sync from kanidm-0 once TLS peer certs are verified +kubectl scale statefulset -n kanidm kanidm --replicas=3 ``` diff --git a/apps/base/kanidm/kustomization.yaml b/apps/base/kanidm/kustomization.yaml index 05456f2..6608e74 100644 --- a/apps/base/kanidm/kustomization.yaml +++ b/apps/base/kanidm/kustomization.yaml @@ -5,6 +5,8 @@ kind: Kustomization resources: - namespace.yaml - serviceaccount.yaml + - vaultauth.yaml + - vaultstaticsecret.yaml - certificate.yaml - service.yaml - statefulset.yaml diff --git a/apps/base/kanidm/resources/server-0.toml b/apps/base/kanidm/resources/server-0.toml index 4ad3253..1fb9fd5 100644 --- a/apps/base/kanidm/resources/server-0.toml +++ b/apps/base/kanidm/resources/server-0.toml @@ -17,11 +17,4 @@ versions = 7 [replication] origin = "repl://kanidm-0.kanidm-headless.kanidm.svc.cluster.local:8444" bindaddress = "[::]:8444" - -[replication."repl://kanidm-1.kanidm-headless.kanidm.svc.cluster.local:8444"] -type = "mutual-pull" -partner_cert = "MIIB-TCCAZ-gAwIBAgIRASqOpORz60wiv7wF_7oBOxQwCgYIKoZIzj0EAwIwTDEtMCsGA1UEAwwkMmE4ZWE0ZTQtNzNlYi00YzIyLWJmYmMtMDVmZmJhMDEzYjE0MRswGQYDVQQKDBJLYW5pZG0gUmVwbGljYXRpb24wHhcNMjYwNTI1MTMyODM5WhcNMzAwNTI1MTMyODM5WjBMMS0wKwYDVQQDDCQyYThlYTRlNC03M2ViLTRjMjItYmZiYy0wNWZmYmEwMTNiMTQxGzAZBgNVBAoMEkthbmlkbSBSZXBsaWNhdGlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFQP3zpFRt7TCOhzrUpOJBojn-sC2LmqZUub8P2ymVdIQbmoAyh4Q8Me0hNWJFyuFDnnqO06dt5I2iv0910-X6KjYjBgMCAGA1UdJQEB_wQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATA8BgNVHREENTAzgjFrYW5pZG0tMS5rYW5pZG0taGVhZGxlc3Mua2FuaWRtLnN2Yy5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0gAMEUCIGjl58U6apcDjMEPIca8Wwg_JMfuMvV-uVcJI49Gl_9GAiEA2tFdb9rnFeBI7mwysScf5UsmY3ZziMD3UVm1vWN2IKs" - -[replication."repl://kanidm-2.kanidm-headless.kanidm.svc.cluster.local:8444"] -type = "mutual-pull" -partner_cert = "MIIB-TCCAZ-gAwIBAgIRAeFGUAJbCkJ2vzf_Vv4qjeUwCgYIKoZIzj0EAwIwTDEtMCsGA1UEAwwkZTE0NjUwMDItNWIwYS00Mjc2LWJmMzctZmY1NmZlMmE4ZGU1MRswGQYDVQQKDBJLYW5pZG0gUmVwbGljYXRpb24wHhcNMjYwNTI1MTMyOTEwWhcNMzAwNTI1MTMyOTEwWjBMMS0wKwYDVQQDDCRlMTQ2NTAwMi01YjBhLTQyNzYtYmYzNy1mZjU2ZmUyYThkZTUxGzAZBgNVBAoMEkthbmlkbSBSZXBsaWNhdGlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCrncHSbDNSV3_aOSZ14plbVfrvSXQQL9MOqvrDKlf_Q6WbcA8OrTUjs3Jt0Q2beWjC3Z5-5c9fGu8M_k2iVWf-jYjBgMCAGA1UdJQEB_wQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATA8BgNVHREENTAzgjFrYW5pZG0tMi5rYW5pZG0taGVhZGxlc3Mua2FuaWRtLnN2Yy5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0gAMEUCIQDHY5Yl-bhDTuJaYnHSMSiSAEWPrDcRVzvfmOJukuJ1QQIgSwgyeSG3K0MY87DI1RDYAdZlpP1YOK3Yatj7-YSXPC0" +automatic_refresh = true diff --git a/apps/base/kanidm/resources/server-1.toml b/apps/base/kanidm/resources/server-1.toml index add72fb..f365c9b 100644 --- a/apps/base/kanidm/resources/server-1.toml +++ b/apps/base/kanidm/resources/server-1.toml @@ -17,11 +17,4 @@ versions = 7 [replication] origin = "repl://kanidm-1.kanidm-headless.kanidm.svc.cluster.local:8444" bindaddress = "[::]:8444" - -[replication."repl://kanidm-0.kanidm-headless.kanidm.svc.cluster.local:8444"] -type = "mutual-pull" -partner_cert = "MIIB-jCCAZ-gAwIBAgIRAVKuoPDpF0IBnvFjCwdK41EwCgYIKoZIzj0EAwIwTDEtMCsGA1UEAwwkNTJhZWEwZjAtZTkxNy00MjAxLTllZjEtNjMwYjA3NGFlMzUxMRswGQYDVQQKDBJLYW5pZG0gUmVwbGljYXRpb24wHhcNMjYwNTI1MTMzNzQ5WhcNMzAwNTI1MTMzNzQ5WjBMMS0wKwYDVQQDDCQ1MmFlYTBmMC1lOTE3LTQyMDEtOWVmMS02MzBiMDc0YWUzNTExGzAZBgNVBAoMEkthbmlkbSBSZXBsaWNhdGlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABGejqjk0Eet-RILHI236wHYqISdnPlebqnkuUTh4W2mCzkmqKibyjxGIUOs8LBrUeTR2DxVR1VV6H2rYQk2wdROjYjBgMCAGA1UdJQEB_wQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATA8BgNVHREENTAzgjFrYW5pZG0tMC5rYW5pZG0taGVhZGxlc3Mua2FuaWRtLnN2Yy5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0kAMEYCIQCSkFj2A-KVWv2tKJLFzb18J5eWWKtsvWewZTn-FVnRnQIhAKJbt84IoZ9oXxgfp0VOLyVZiAgUgwMFS6JOfno3D-Nw" - -[replication."repl://kanidm-2.kanidm-headless.kanidm.svc.cluster.local:8444"] -type = "mutual-pull" -partner_cert = "MIIB-TCCAZ-gAwIBAgIRAeFGUAJbCkJ2vzf_Vv4qjeUwCgYIKoZIzj0EAwIwTDEtMCsGA1UEAwwkZTE0NjUwMDItNWIwYS00Mjc2LWJmMzctZmY1NmZlMmE4ZGU1MRswGQYDVQQKDBJLYW5pZG0gUmVwbGljYXRpb24wHhcNMjYwNTI1MTMyOTEwWhcNMzAwNTI1MTMyOTEwWjBMMS0wKwYDVQQDDCRlMTQ2NTAwMi01YjBhLTQyNzYtYmYzNy1mZjU2ZmUyYThkZTUxGzAZBgNVBAoMEkthbmlkbSBSZXBsaWNhdGlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCrncHSbDNSV3_aOSZ14plbVfrvSXQQL9MOqvrDKlf_Q6WbcA8OrTUjs3Jt0Q2beWjC3Z5-5c9fGu8M_k2iVWf-jYjBgMCAGA1UdJQEB_wQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATA8BgNVHREENTAzgjFrYW5pZG0tMi5rYW5pZG0taGVhZGxlc3Mua2FuaWRtLnN2Yy5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0gAMEUCIQDHY5Yl-bhDTuJaYnHSMSiSAEWPrDcRVzvfmOJukuJ1QQIgSwgyeSG3K0MY87DI1RDYAdZlpP1YOK3Yatj7-YSXPC0" +automatic_refresh = true diff --git a/apps/base/kanidm/resources/server-2.toml b/apps/base/kanidm/resources/server-2.toml index 87ba6ae..7b019fc 100644 --- a/apps/base/kanidm/resources/server-2.toml +++ b/apps/base/kanidm/resources/server-2.toml @@ -17,11 +17,4 @@ versions = 7 [replication] origin = "repl://kanidm-2.kanidm-headless.kanidm.svc.cluster.local:8444" bindaddress = "[::]:8444" - -[replication."repl://kanidm-0.kanidm-headless.kanidm.svc.cluster.local:8444"] -type = "mutual-pull" -partner_cert = "MIIB-jCCAZ-gAwIBAgIRAVKuoPDpF0IBnvFjCwdK41EwCgYIKoZIzj0EAwIwTDEtMCsGA1UEAwwkNTJhZWEwZjAtZTkxNy00MjAxLTllZjEtNjMwYjA3NGFlMzUxMRswGQYDVQQKDBJLYW5pZG0gUmVwbGljYXRpb24wHhcNMjYwNTI1MTMzNzQ5WhcNMzAwNTI1MTMzNzQ5WjBMMS0wKwYDVQQDDCQ1MmFlYTBmMC1lOTE3LTQyMDEtOWVmMS02MzBiMDc0YWUzNTExGzAZBgNVBAoMEkthbmlkbSBSZXBsaWNhdGlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABGejqjk0Eet-RILHI236wHYqISdnPlebqnkuUTh4W2mCzkmqKibyjxGIUOs8LBrUeTR2DxVR1VV6H2rYQk2wdROjYjBgMCAGA1UdJQEB_wQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATA8BgNVHREENTAzgjFrYW5pZG0tMC5rYW5pZG0taGVhZGxlc3Mua2FuaWRtLnN2Yy5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0kAMEYCIQCSkFj2A-KVWv2tKJLFzb18J5eWWKtsvWewZTn-FVnRnQIhAKJbt84IoZ9oXxgfp0VOLyVZiAgUgwMFS6JOfno3D-Nw" - -[replication."repl://kanidm-1.kanidm-headless.kanidm.svc.cluster.local:8444"] -type = "mutual-pull" -partner_cert = "MIIB-TCCAZ-gAwIBAgIRASqOpORz60wiv7wF_7oBOxQwCgYIKoZIzj0EAwIwTDEtMCsGA1UEAwwkMmE4ZWE0ZTQtNzNlYi00YzIyLWJmYmMtMDVmZmJhMDEzYjE0MRswGQYDVQQKDBJLYW5pZG0gUmVwbGljYXRpb24wHhcNMjYwNTI1MTMyODM5WhcNMzAwNTI1MTMyODM5WjBMMS0wKwYDVQQDDCQyYThlYTRlNC03M2ViLTRjMjItYmZiYy0wNWZmYmEwMTNiMTQxGzAZBgNVBAoMEkthbmlkbSBSZXBsaWNhdGlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFQP3zpFRt7TCOhzrUpOJBojn-sC2LmqZUub8P2ymVdIQbmoAyh4Q8Me0hNWJFyuFDnnqO06dt5I2iv0910-X6KjYjBgMCAGA1UdJQEB_wQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATA8BgNVHREENTAzgjFrYW5pZG0tMS5rYW5pZG0taGVhZGxlc3Mua2FuaWRtLnN2Yy5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0gAMEUCIGjl58U6apcDjMEPIca8Wwg_JMfuMvV-uVcJI49Gl_9GAiEA2tFdb9rnFeBI7mwysScf5UsmY3ZziMD3UVm1vWN2IKs" +automatic_refresh = true diff --git a/apps/base/kanidm/statefulset.yaml b/apps/base/kanidm/statefulset.yaml index 397e1e4..2eaa778 100644 --- a/apps/base/kanidm/statefulset.yaml +++ b/apps/base/kanidm/statefulset.yaml @@ -4,6 +4,8 @@ kind: StatefulSet metadata: name: kanidm namespace: kanidm + annotations: + reloader.stakater.com/auto: "true" labels: app.kubernetes.io/name: kanidm app.kubernetes.io/instance: kanidm @@ -39,7 +41,17 @@ spec: image: busybox:1.36 command: ["/bin/sh", "-c"] args: - - cp "/config-template/server-${POD_NAME##*-}.toml" /config/server.toml + - | + set -e + cp "/config-template/server-${POD_NAME##*-}.toml" /config/server.toml + for peer in kanidm-0 kanidm-1 kanidm-2; do + [ "${peer}" = "${POD_NAME}" ] && continue + cert_file="/repl-certs/${peer}" + [ -s "${cert_file}" ] || continue + fqdn="${peer}.kanidm-headless.kanidm.svc.cluster.local" + printf '\n[replication."repl://%s:8444"]\ntype = "mutual-pull"\npartner_cert = "%s"\n' \ + "${fqdn}" "$(cat ${cert_file})" >> /config/server.toml + done env: - name: POD_NAME valueFrom: @@ -51,6 +63,9 @@ spec: readOnly: true - name: config mountPath: /config + - name: repl-certs + mountPath: /repl-certs + readOnly: true securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true @@ -101,6 +116,9 @@ spec: name: kanidm-config - name: config emptyDir: {} + - name: repl-certs + secret: + secretName: kanidm-repl-certs - name: tls secret: secretName: kanidm-tls diff --git a/apps/base/kanidm/vaultauth.yaml b/apps/base/kanidm/vaultauth.yaml new file mode 100644 index 0000000..de33b5c --- /dev/null +++ b/apps/base/kanidm/vaultauth.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: default + namespace: kanidm + labels: + app.kubernetes.io/name: kanidm + app.kubernetes.io/instance: kanidm +spec: + method: kubernetes + mount: k8s/au/syd1 + vaultConnectionRef: vso-system/default + allowedNamespaces: + - kanidm + kubernetes: + role: default + serviceAccount: kanidm + audiences: + - vault + tokenExpirationSeconds: 600 diff --git a/apps/base/kanidm/vaultstaticsecret.yaml b/apps/base/kanidm/vaultstaticsecret.yaml new file mode 100644 index 0000000..d6716ad --- /dev/null +++ b/apps/base/kanidm/vaultstaticsecret.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: repl-certs + namespace: kanidm + labels: + app.kubernetes.io/name: kanidm + app.kubernetes.io/instance: kanidm +spec: + vaultAuthRef: default + mount: kv + type: kv-v2 + path: kubernetes/namespace/kanidm/default/repl-certs + refreshAfter: 5m + destination: + name: kanidm-repl-certs + create: true + overwrite: true + hmacSecretData: true