Update apps/base/kanidm/vaultauth.yaml

Fix the VaultAuth object

.gitignore

@@ -1,3 +1,2 @@
 manifests/
 apps/**/charts/
-ci/crd-schemas/

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
       - id: check-executables-have-shebangs
       - id: check-json
       - id: check-added-large-files
-        args: ['--maxkb=2048']
+        args: ['--maxkb=500']
       - id: check-merge-conflict
       - id: check-shebang-scripts-are-executable
       - id: check-symlinks

.woodpecker/kubeconform.yaml

@@ -2,21 +2,6 @@ when:
   - event: pull_request
 
 steps:
-  - name: generate-schemas
-    image: git.unkin.net/unkin/almalinux9-kubetest:20260319
-    commands:
-      - make generate-schemas
-    backend_options:
-      kubernetes:
-        serviceAccountName: default
-        resources:
-          requests:
-            memory: 512Mi
-            cpu: 1
-          limits:
-            memory: 2Gi
-            cpu: 2
-
   - name: kubeconform
     image: git.unkin.net/unkin/almalinux9-kubetest:20260319
     commands:

Makefile

@@ -11,11 +11,6 @@ kubeconform:
 	@ci/validate-apps.sh && \
 	ci/validate-clusters.sh
 
-# Generate JSON schemas from CRD definitions for kubeconform
-# Run when CRD versions change, then commit ci/crd-schemas/
-generate-schemas:
-	@ci/generate-crd-schemas.sh
-
 # Clean all generated manifests
 clean:
 	@rm -rf manifests/

apps/base/kanidm/README.md

@@ -1,32 +1,51 @@
 # kanidm
 
-Single-replica kanidm identity server deployment.
+Three-replica kanidm identity server with Vault-managed replication certificates.
+
+## Architecture
+
+- Per-pod `server-N.toml` in `resources/` — each has its own replication origin hardcoded
+- `config-init` busybox init container copies the right config and injects peer certs from the
+  vault-synced `kanidm-repl-certs` Secret at pod startup
+- `reloader.stakater.com/auto: "true"` triggers a rolling restart when the ConfigMap or Secret changes
+- Vault path: `kv/kubernetes/namespace/kanidm/default/repl-certs`
+  - Keys: `kanidm-0`, `kanidm-1`, `kanidm-2` — each holds that pod's replication certificate
 
 ## Initial setup
 
-After the pod starts for the first time, generate the admin and idm_admin credentials:
+After the first pod starts, generate the admin credentials:
 
 ```bash
-kubectl exec -n kanidm kanidm-0 -- /sbin/kanidmd recover-account admin
+kubectl exec -n kanidm kanidm-0 -- /sbin/kanidmd recover-account -c /config/server.toml admin
-kubectl exec -n kanidm kanidm-0 -- /sbin/kanidmd recover-account idm_admin
+kubectl exec -n kanidm kanidm-0 -- /sbin/kanidmd recover-account -c /config/server.toml idm_admin
 ```
 
-## Adding replication
+## Replication certificate rotation
 
-If replication is needed in the future:
+When certs need to be renewed, update vault and reloader will roll the StatefulSet:
-
-1. Scale the StatefulSet to 3 replicas and add `podAntiAffinity` to spread across nodes.
-2. Add a `[replication]` section to `configmap.yaml` per pod (origin is pod-specific:
-   `repl://kanidm-N.kanidm-headless.kanidm.svc.cluster.local:8444`).
-3. Add the replication port (8444) back to the StatefulSet container ports and headless service.
-4. Restore `rbac.yaml` for the cert-publisher sidecar, or exchange certificates manually:
 
 ```bash
-# On each pod, get its replication certificate
+# Get new cert from a pod
-kubectl exec -n kanidm kanidm-0 -- /sbin/kanidmd renew-replication-certificate
+kubectl exec -it -n kanidm kanidm-N -- /sbin/kanidmd renew-replication-certificate -c /config/server.toml
 
-# Add each peer's certificate to the other pods' configs under:
+# Write updated cert to vault (reloader triggers restart automatically)
-# [replication."repl://<peer-fqdn>:8444"]
+vault kv patch kv/kubernetes/namespace/kanidm/default/repl-certs "kanidm-N=<cert>"
-# type = "mutual-pull"
+```
-# partner_cert = "<cert>"
+
+## Resolving domain UUID mismatch
+
+If pods initialized independently (each with a different domain UUID), replication will fail with
+`Consumer Domain UUID does not match`. Fix by resetting kanidm-1 and kanidm-2 to sync from
+kanidm-0 (the authoritative node):
+
+```bash
+# Scale down to avoid split-brain during reset
+kubectl scale statefulset -n kanidm kanidm --replicas=1
+
+# Delete the stale PVCs for the replica pods
+kubectl delete pvc -n kanidm data-kanidm-1 data-kanidm-2
+
+# Scale back up — replicas start with empty DBs and automatic_refresh=true
+# will trigger a full sync from kanidm-0 once TLS peer certs are verified
+kubectl scale statefulset -n kanidm kanidm --replicas=3
 ```

apps/base/kanidm/kustomization.yaml

@@ -5,6 +5,8 @@ kind: Kustomization
 resources:
   - namespace.yaml
   - serviceaccount.yaml
+  - vaultauth.yaml
+  - vaultstaticsecret.yaml
   - certificate.yaml
   - service.yaml
   - statefulset.yaml

apps/base/kanidm/resources/server-0.toml

@@ -17,11 +17,4 @@ versions = 7
 [replication]
 origin = "repl://kanidm-0.kanidm-headless.kanidm.svc.cluster.local:8444"
 bindaddress = "[::]:8444"
-
+automatic_refresh = true
-[replication."repl://kanidm-1.kanidm-headless.kanidm.svc.cluster.local:8444"]
-type = "mutual-pull"
-partner_cert = "MIIB-TCCAZ-gAwIBAgIRASqOpORz60wiv7wF_7oBOxQwCgYIKoZIzj0EAwIwTDEtMCsGA1UEAwwkMmE4ZWE0ZTQtNzNlYi00YzIyLWJmYmMtMDVmZmJhMDEzYjE0MRswGQYDVQQKDBJLYW5pZG0gUmVwbGljYXRpb24wHhcNMjYwNTI1MTMyODM5WhcNMzAwNTI1MTMyODM5WjBMMS0wKwYDVQQDDCQyYThlYTRlNC03M2ViLTRjMjItYmZiYy0wNWZmYmEwMTNiMTQxGzAZBgNVBAoMEkthbmlkbSBSZXBsaWNhdGlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFQP3zpFRt7TCOhzrUpOJBojn-sC2LmqZUub8P2ymVdIQbmoAyh4Q8Me0hNWJFyuFDnnqO06dt5I2iv0910-X6KjYjBgMCAGA1UdJQEB_wQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATA8BgNVHREENTAzgjFrYW5pZG0tMS5rYW5pZG0taGVhZGxlc3Mua2FuaWRtLnN2Yy5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0gAMEUCIGjl58U6apcDjMEPIca8Wwg_JMfuMvV-uVcJI49Gl_9GAiEA2tFdb9rnFeBI7mwysScf5UsmY3ZziMD3UVm1vWN2IKs"
-
-[replication."repl://kanidm-2.kanidm-headless.kanidm.svc.cluster.local:8444"]
-type = "mutual-pull"
-partner_cert = "MIIB-TCCAZ-gAwIBAgIRAeFGUAJbCkJ2vzf_Vv4qjeUwCgYIKoZIzj0EAwIwTDEtMCsGA1UEAwwkZTE0NjUwMDItNWIwYS00Mjc2LWJmMzctZmY1NmZlMmE4ZGU1MRswGQYDVQQKDBJLYW5pZG0gUmVwbGljYXRpb24wHhcNMjYwNTI1MTMyOTEwWhcNMzAwNTI1MTMyOTEwWjBMMS0wKwYDVQQDDCRlMTQ2NTAwMi01YjBhLTQyNzYtYmYzNy1mZjU2ZmUyYThkZTUxGzAZBgNVBAoMEkthbmlkbSBSZXBsaWNhdGlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCrncHSbDNSV3_aOSZ14plbVfrvSXQQL9MOqvrDKlf_Q6WbcA8OrTUjs3Jt0Q2beWjC3Z5-5c9fGu8M_k2iVWf-jYjBgMCAGA1UdJQEB_wQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATA8BgNVHREENTAzgjFrYW5pZG0tMi5rYW5pZG0taGVhZGxlc3Mua2FuaWRtLnN2Yy5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0gAMEUCIQDHY5Yl-bhDTuJaYnHSMSiSAEWPrDcRVzvfmOJukuJ1QQIgSwgyeSG3K0MY87DI1RDYAdZlpP1YOK3Yatj7-YSXPC0"

apps/base/kanidm/resources/server-1.toml

@@ -17,11 +17,4 @@ versions = 7
 [replication]
 origin = "repl://kanidm-1.kanidm-headless.kanidm.svc.cluster.local:8444"
 bindaddress = "[::]:8444"
-
+automatic_refresh = true
-[replication."repl://kanidm-0.kanidm-headless.kanidm.svc.cluster.local:8444"]
-type = "mutual-pull"
-partner_cert = "MIIB-jCCAZ-gAwIBAgIRAVKuoPDpF0IBnvFjCwdK41EwCgYIKoZIzj0EAwIwTDEtMCsGA1UEAwwkNTJhZWEwZjAtZTkxNy00MjAxLTllZjEtNjMwYjA3NGFlMzUxMRswGQYDVQQKDBJLYW5pZG0gUmVwbGljYXRpb24wHhcNMjYwNTI1MTMzNzQ5WhcNMzAwNTI1MTMzNzQ5WjBMMS0wKwYDVQQDDCQ1MmFlYTBmMC1lOTE3LTQyMDEtOWVmMS02MzBiMDc0YWUzNTExGzAZBgNVBAoMEkthbmlkbSBSZXBsaWNhdGlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABGejqjk0Eet-RILHI236wHYqISdnPlebqnkuUTh4W2mCzkmqKibyjxGIUOs8LBrUeTR2DxVR1VV6H2rYQk2wdROjYjBgMCAGA1UdJQEB_wQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATA8BgNVHREENTAzgjFrYW5pZG0tMC5rYW5pZG0taGVhZGxlc3Mua2FuaWRtLnN2Yy5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0kAMEYCIQCSkFj2A-KVWv2tKJLFzb18J5eWWKtsvWewZTn-FVnRnQIhAKJbt84IoZ9oXxgfp0VOLyVZiAgUgwMFS6JOfno3D-Nw"
-
-[replication."repl://kanidm-2.kanidm-headless.kanidm.svc.cluster.local:8444"]
-type = "mutual-pull"
-partner_cert = "MIIB-TCCAZ-gAwIBAgIRAeFGUAJbCkJ2vzf_Vv4qjeUwCgYIKoZIzj0EAwIwTDEtMCsGA1UEAwwkZTE0NjUwMDItNWIwYS00Mjc2LWJmMzctZmY1NmZlMmE4ZGU1MRswGQYDVQQKDBJLYW5pZG0gUmVwbGljYXRpb24wHhcNMjYwNTI1MTMyOTEwWhcNMzAwNTI1MTMyOTEwWjBMMS0wKwYDVQQDDCRlMTQ2NTAwMi01YjBhLTQyNzYtYmYzNy1mZjU2ZmUyYThkZTUxGzAZBgNVBAoMEkthbmlkbSBSZXBsaWNhdGlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCrncHSbDNSV3_aOSZ14plbVfrvSXQQL9MOqvrDKlf_Q6WbcA8OrTUjs3Jt0Q2beWjC3Z5-5c9fGu8M_k2iVWf-jYjBgMCAGA1UdJQEB_wQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATA8BgNVHREENTAzgjFrYW5pZG0tMi5rYW5pZG0taGVhZGxlc3Mua2FuaWRtLnN2Yy5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0gAMEUCIQDHY5Yl-bhDTuJaYnHSMSiSAEWPrDcRVzvfmOJukuJ1QQIgSwgyeSG3K0MY87DI1RDYAdZlpP1YOK3Yatj7-YSXPC0"

apps/base/kanidm/resources/server-2.toml

@@ -17,11 +17,4 @@ versions = 7
 [replication]
 origin = "repl://kanidm-2.kanidm-headless.kanidm.svc.cluster.local:8444"
 bindaddress = "[::]:8444"
-
+automatic_refresh = true
-[replication."repl://kanidm-0.kanidm-headless.kanidm.svc.cluster.local:8444"]
-type = "mutual-pull"
-partner_cert = "MIIB-jCCAZ-gAwIBAgIRAVKuoPDpF0IBnvFjCwdK41EwCgYIKoZIzj0EAwIwTDEtMCsGA1UEAwwkNTJhZWEwZjAtZTkxNy00MjAxLTllZjEtNjMwYjA3NGFlMzUxMRswGQYDVQQKDBJLYW5pZG0gUmVwbGljYXRpb24wHhcNMjYwNTI1MTMzNzQ5WhcNMzAwNTI1MTMzNzQ5WjBMMS0wKwYDVQQDDCQ1MmFlYTBmMC1lOTE3LTQyMDEtOWVmMS02MzBiMDc0YWUzNTExGzAZBgNVBAoMEkthbmlkbSBSZXBsaWNhdGlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABGejqjk0Eet-RILHI236wHYqISdnPlebqnkuUTh4W2mCzkmqKibyjxGIUOs8LBrUeTR2DxVR1VV6H2rYQk2wdROjYjBgMCAGA1UdJQEB_wQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATA8BgNVHREENTAzgjFrYW5pZG0tMC5rYW5pZG0taGVhZGxlc3Mua2FuaWRtLnN2Yy5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0kAMEYCIQCSkFj2A-KVWv2tKJLFzb18J5eWWKtsvWewZTn-FVnRnQIhAKJbt84IoZ9oXxgfp0VOLyVZiAgUgwMFS6JOfno3D-Nw"
-
-[replication."repl://kanidm-1.kanidm-headless.kanidm.svc.cluster.local:8444"]
-type = "mutual-pull"
-partner_cert = "MIIB-TCCAZ-gAwIBAgIRASqOpORz60wiv7wF_7oBOxQwCgYIKoZIzj0EAwIwTDEtMCsGA1UEAwwkMmE4ZWE0ZTQtNzNlYi00YzIyLWJmYmMtMDVmZmJhMDEzYjE0MRswGQYDVQQKDBJLYW5pZG0gUmVwbGljYXRpb24wHhcNMjYwNTI1MTMyODM5WhcNMzAwNTI1MTMyODM5WjBMMS0wKwYDVQQDDCQyYThlYTRlNC03M2ViLTRjMjItYmZiYy0wNWZmYmEwMTNiMTQxGzAZBgNVBAoMEkthbmlkbSBSZXBsaWNhdGlvbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFQP3zpFRt7TCOhzrUpOJBojn-sC2LmqZUub8P2ymVdIQbmoAyh4Q8Me0hNWJFyuFDnnqO06dt5I2iv0910-X6KjYjBgMCAGA1UdJQEB_wQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATA8BgNVHREENTAzgjFrYW5pZG0tMS5rYW5pZG0taGVhZGxlc3Mua2FuaWRtLnN2Yy5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0gAMEUCIGjl58U6apcDjMEPIca8Wwg_JMfuMvV-uVcJI49Gl_9GAiEA2tFdb9rnFeBI7mwysScf5UsmY3ZziMD3UVm1vWN2IKs"

apps/base/kanidm/statefulset.yaml

@@ -4,6 +4,8 @@ kind: StatefulSet
 metadata:
   name: kanidm
   namespace: kanidm
+  annotations:
+    reloader.stakater.com/auto: "true"
   labels:
     app.kubernetes.io/name: kanidm
     app.kubernetes.io/instance: kanidm
@@ -39,7 +41,17 @@ spec:
           image: busybox:1.36
           command: ["/bin/sh", "-c"]
           args:
-            - cp "/config-template/server-${POD_NAME##*-}.toml" /config/server.toml
+            - |
+              set -e
+              cp "/config-template/server-${POD_NAME##*-}.toml" /config/server.toml
+              for peer in kanidm-0 kanidm-1 kanidm-2; do
+                [ "${peer}" = "${POD_NAME}" ] && continue
+                cert_file="/repl-certs/${peer}"
+                [ -s "${cert_file}" ] || continue
+                fqdn="${peer}.kanidm-headless.kanidm.svc.cluster.local"
+                printf '\n[replication."repl://%s:8444"]\ntype = "mutual-pull"\npartner_cert = "%s"\n' \
+                  "${fqdn}" "$(cat ${cert_file})" >> /config/server.toml
+              done
           env:
             - name: POD_NAME
               valueFrom:
@@ -51,6 +63,9 @@ spec:
               readOnly: true
             - name: config
               mountPath: /config
+            - name: repl-certs
+              mountPath: /repl-certs
+              readOnly: true
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
@@ -101,6 +116,9 @@ spec:
             name: kanidm-config
         - name: config
           emptyDir: {}
+        - name: repl-certs
+          secret:
+            secretName: kanidm-repl-certs
         - name: tls
           secret:
             secretName: kanidm-tls

apps/base/kanidm/vaultauth.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: default
+  namespace: kanidm
+spec:
+  allowedNamespaces:
+    - kanidm
+  kubernetes:
+    audiences:
+      - vault
+    role: default
+    serviceAccount: default
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default

apps/base/kanidm/vaultstaticsecret.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: repl-certs
+  namespace: kanidm
+  labels:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
+spec:
+  vaultAuthRef: default
+  mount: kv
+  type: kv-v2
+  path: kubernetes/namespace/kanidm/default/repl-certs
+  refreshAfter: 5m
+  destination:
+    name: kanidm-repl-certs
+    create: true
+    overwrite: true
+  hmacSecretData: true

ci/generate-crd-schemas.py

@@ -1,49 +0,0 @@
-#!/usr/bin/env -S uv run
-# /// script
-# requires-python = ">=3.11"
-# dependencies = ["pyyaml"]
-# ///
-"""
-Extract OpenAPI v3 schemas from CRD YAML on stdin and write JSON schema files
-to the output directory for use with kubeconform.
-
-Usage: kustomize build ... | ci/generate-crd-schemas.py <output-dir>
-"""
-import sys
-import json
-import os
-
-import yaml
-
-
-def main() -> int:
-    output_dir = sys.argv[1] if len(sys.argv) > 1 else "ci/crd-schemas"
-    count = 0
-
-    for doc in yaml.safe_load_all(sys.stdin):
-        if not doc or doc.get("kind") != "CustomResourceDefinition":
-            continue
-
-        group = doc["spec"]["group"]
-        kind = doc["spec"]["names"]["kind"]
-        group_dir = os.path.join(output_dir, group)
-        os.makedirs(group_dir, exist_ok=True)
-
-        for ver in doc["spec"].get("versions", []):
-            if not ver.get("served", True):
-                continue
-            schema = ver.get("schema", {}).get("openAPIV3Schema")
-            if not schema:
-                continue
-            fname = os.path.join(group_dir, f"{kind.lower()}_{ver['name']}.json")
-            with open(fname, "w") as f:
-                json.dump({"$schema": "http://json-schema.org/schema#", **schema}, f, indent=2)
-                f.write("\n")
-            print(f"  wrote {fname}", file=sys.stderr)
-            count += 1
-
-    return count
-
-
-if __name__ == "__main__":
-    print(main())

ci/generate-crd-schemas.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-# Extract OpenAPI v3 schemas from CRD definitions in all kustomize overlays
-# and write JSON schema files to ci/crd-schemas/ for kubeconform validation.
-#
-# Run this script whenever CRD versions change, then commit the output.
-# Usage: ci/generate-crd-schemas.sh [output-dir]
-
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-OUTPUT_DIR="${1:-${SCRIPT_DIR}/crd-schemas}"
-mkdir -p "$OUTPUT_DIR"
-
-total=0
-
-while IFS= read -r -d "" k; do
-  dir="$(dirname "$k")"
-  n=$(kustomize build --enable-helm "$dir" 2>/dev/null \
-    | "$SCRIPT_DIR/generate-crd-schemas.py" "$OUTPUT_DIR") || continue
-  total=$((total + n))
-done < <(find apps/overlays clusters -name kustomization.yaml -print0 | sort -z)
-
-echo "Generated ${total} schema(s) in ${OUTPUT_DIR}" >&2

ci/validate-apps.sh

@@ -6,7 +6,6 @@ KUBE_VERSION="1.33.7"
 schema_args=(
   -schema-location "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/{{.NormalizedKubernetesVersion}}-standalone{{.StrictSuffix}}/{{.ResourceKind}}{{.KindSuffix}}.json"
   -schema-location "https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json"
-  -schema-location "file://${PWD}/ci/crd-schemas/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json"
 )
 
 while IFS= read -r -d "" k; do
@@ -19,6 +18,6 @@ while IFS= read -r -d "" k; do
         -summary \
         -output pretty \
         -verbose \
-        -skip GpuDevicePlugin,LBNodeAgent,ServiceGroup \
+        -skip CustomResourceDefinition,GpuDevicePlugin,LBNodeAgent,ServiceGroup \
         "${schema_args[@]}"
 done < <(find apps/overlays -name kustomization.yaml -print0)

ci/validate-clusters.sh

@@ -6,7 +6,6 @@ KUBE_VERSION="1.33.7"
 schema_args=(
   -schema-location "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/{{.NormalizedKubernetesVersion}}-standalone{{.StrictSuffix}}/{{.ResourceKind}}{{.KindSuffix}}.json"
   -schema-location "https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json"
-  -schema-location "file://${PWD}/ci/crd-schemas/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json"
 )
 
 while IFS= read -r -d "" k; do