Merge branch 'main' into fix-kanidm-replication-config

Reviewed-on: #179

apps/base/artifactapi/resources/conf.d/remote-docker.yaml

@@ -6,6 +6,7 @@ remotes:
     immutable_patterns:
       - "^cloudnative-pg/cloudnative-pg"
       - "^emberstack/helm-charts"
+      - "^open-webui/open-webui"
       - "^openvoxproject/"
       - "^stakater/reloader"
       - "^stalwartlabs/stalwart"

apps/base/artifactapi/resources/conf.d/remote-generic.yaml

@@ -36,6 +36,7 @@ remotes:
       - "neovim/neovim/.*/nvim-linux-x86_64.tar.gz$"
       - "nzbgetcom/nzbget/.*/nzbget-.*.x86_64.rpm$"
       - "onedr0p/exportarr/.*/exportarr_.*_linux_amd64.tar.gz$"
+      - "open-policy-agent/conftest/.*/conftest_.*_Linux_x86_64.tar.gz$"
       - "openbao/openbao-plugins/.*/openbao-plugin-secrets-consul_linux_amd64_.*.tar.gz$"
       - "openbao/openbao-plugins/.*/openbao-plugin-secrets-nomad_linux_amd64_.*.tar.gz$"
       - "prometheus-community/bind_exporter/.*/bind_exporter-.*.linux-amd64.tar.gz$"

apps/base/kanidm/README.md

@@ -0,0 +1,51 @@
+# kanidm
+
+Three-replica kanidm identity server with Vault-managed replication certificates.
+
+## Architecture
+
+- Per-pod `server-N.toml` in `resources/` — each has its own replication origin hardcoded
+- `config-init` busybox init container copies the right config and injects peer certs from the
+  vault-synced `kanidm-repl-certs` Secret at pod startup
+- `reloader.stakater.com/auto: "true"` triggers a rolling restart when the ConfigMap or Secret changes
+- Vault path: `kv/kubernetes/namespace/kanidm/default/repl-certs`
+  - Keys: `kanidm-0`, `kanidm-1`, `kanidm-2` — each holds that pod's replication certificate
+
+## Initial setup
+
+After the first pod starts, generate the admin credentials:
+
+```bash
+kubectl exec -n kanidm kanidm-0 -- /sbin/kanidmd recover-account -c /config/server.toml admin
+kubectl exec -n kanidm kanidm-0 -- /sbin/kanidmd recover-account -c /config/server.toml idm_admin
+```
+
+## Replication certificate rotation
+
+When certs need to be renewed, update vault and reloader will roll the StatefulSet:
+
+```bash
+# Get new cert from a pod
+kubectl exec -it -n kanidm kanidm-N -- /sbin/kanidmd renew-replication-certificate -c /config/server.toml
+
+# Write updated cert to vault (reloader triggers restart automatically)
+vault kv patch kv/kubernetes/namespace/kanidm/default/repl-certs "kanidm-N=<cert>"
+```
+
+## Resolving domain UUID mismatch
+
+If pods initialized independently (each with a different domain UUID), replication will fail with
+`Consumer Domain UUID does not match`. Fix by resetting kanidm-1 and kanidm-2 to sync from
+kanidm-0 (the authoritative node):
+
+```bash
+# Scale down to avoid split-brain during reset
+kubectl scale statefulset -n kanidm kanidm --replicas=1
+
+# Delete the stale PVCs for the replica pods
+kubectl delete pvc -n kanidm data-kanidm-1 data-kanidm-2
+
+# Scale back up — replicas start with empty DBs and automatic_refresh=true
+# will trigger a full sync from kanidm-0 once TLS peer certs are verified
+kubectl scale statefulset -n kanidm kanidm --replicas=3
+```

apps/base/kanidm/configmap.yaml

@@ -1,40 +0,0 @@
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kanidm-config
-  namespace: kanidm
-  labels:
-    app.kubernetes.io/name: kanidm
-    app.kubernetes.io/instance: kanidm
-data:
-  server.toml: |
-    version = "2"
-
-    domain = "auth.unkin.net"
-    origin = "https://auth.unkin.net"
-    bindaddress = "[::]:8443"
-    db_path = "/data/kanidm.db"
-    db_arc_size = 2048
-    tls_chain = "/data/tls/tls.crt"
-    tls_key = "/data/tls/tls.key"
-    log_level = "info"
-
-    [online_backup]
-    path = "/data/backups/"
-    schedule = "0 22 * * *"
-    versions = 7
-
-    [replication]
-    origin = "__REPL_ORIGIN__"
-    bindaddress = "[::]:8444"
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kanidm-repl-certs
-  namespace: kanidm
-  labels:
-    app.kubernetes.io/name: kanidm
-    app.kubernetes.io/instance: kanidm
-data: {}

apps/base/kanidm/kustomization.yaml

@@ -5,12 +5,25 @@ kind: Kustomization
 resources:
   - namespace.yaml
   - serviceaccount.yaml
-  - rbac.yaml
+  - vaultauth.yaml
+  - vaultstaticsecret.yaml
   - certificate.yaml
-  - configmap.yaml
   - service.yaml
   - statefulset.yaml
   - poddisruptionbudget.yaml
   - gateway.yaml
   - httproute.yaml
   - tlsroute.yaml
+
+configMapGenerator:
+  - name: kanidm-config
+    namespace: kanidm
+    options:
+      disableNameSuffixHash: true
+      labels:
+        app.kubernetes.io/name: kanidm
+        app.kubernetes.io/instance: kanidm
+    files:
+      - server-0.toml=resources/server-0.toml
+      - server-1.toml=resources/server-1.toml
+      - server-2.toml=resources/server-2.toml

apps/base/kanidm/resources/server-0.toml

@@ -0,0 +1,19 @@
+version = "2"
+
+domain = "auth.unkin.net"
+origin = "https://auth.unkin.net"
+bindaddress = "[::]:8443"
+db_path = "/data/kanidm.db"
+db_arc_size = 2048
+tls_chain = "/data/tls/tls.crt"
+tls_key = "/data/tls/tls.key"
+log_level = "info"
+
+[online_backup]
+path = "/data/backups/"
+schedule = "0 22 * * *"
+versions = 7
+
+[replication]
+origin = "repl://kanidm-0.kanidm-headless.kanidm.svc.cluster.local:8444"
+bindaddress = "[::]:8444"

apps/base/kanidm/resources/server-1.toml

@@ -0,0 +1,19 @@
+version = "2"
+
+domain = "auth.unkin.net"
+origin = "https://auth.unkin.net"
+bindaddress = "[::]:8443"
+db_path = "/data/kanidm.db"
+db_arc_size = 2048
+tls_chain = "/data/tls/tls.crt"
+tls_key = "/data/tls/tls.key"
+log_level = "info"
+
+[online_backup]
+path = "/data/backups/"
+schedule = "0 22 * * *"
+versions = 7
+
+[replication]
+origin = "repl://kanidm-1.kanidm-headless.kanidm.svc.cluster.local:8444"
+bindaddress = "[::]:8444"

apps/base/kanidm/resources/server-2.toml

@@ -0,0 +1,19 @@
+version = "2"
+
+domain = "auth.unkin.net"
+origin = "https://auth.unkin.net"
+bindaddress = "[::]:8443"
+db_path = "/data/kanidm.db"
+db_arc_size = 2048
+tls_chain = "/data/tls/tls.crt"
+tls_key = "/data/tls/tls.key"
+log_level = "info"
+
+[online_backup]
+path = "/data/backups/"
+schedule = "0 22 * * *"
+versions = 7
+
+[replication]
+origin = "repl://kanidm-2.kanidm-headless.kanidm.svc.cluster.local:8444"
+bindaddress = "[::]:8444"

apps/base/kanidm/statefulset.yaml

@@ -4,6 +4,8 @@ kind: StatefulSet
 metadata:
   name: kanidm
   namespace: kanidm
+  annotations:
+    reloader.stakater.com/auto: "true"
   labels:
     app.kubernetes.io/name: kanidm
     app.kubernetes.io/instance: kanidm
@@ -36,23 +38,19 @@ spec:
         fsGroup: 1000
       initContainers:
         - name: config-init
-          image: kanidm/server:1.10.3
+          image: busybox:1.36
           command: ["/bin/sh", "-c"]
           args:
             - |
               set -e
-              REPL_ORIGIN="repl://${POD_NAME}.kanidm-headless.kanidm.svc.cluster.local:8444"
-              sed "s|__REPL_ORIGIN__|${REPL_ORIGIN}|g" /config-template/server.toml > /config/server.toml
+              cp "/config-template/server-${POD_NAME##*-}.toml" /config/server.toml
               for peer in kanidm-0 kanidm-1 kanidm-2; do
-                if [ "${peer}" = "${POD_NAME}" ]; then
-                  continue
-                fi
+                [ "${peer}" = "${POD_NAME}" ] && continue
                 cert_file="/repl-certs/${peer}"
-                if [ -s "${cert_file}" ]; then
-                  fqdn="${peer}.kanidm-headless.kanidm.svc.cluster.local"
-                  printf '\n[replication."repl://%s:8444"]\ntype = "mutual-pull"\npartner_cert = "%s"\n' \
-                    "${fqdn}" "$(cat ${cert_file})" >> /config/server.toml
-                fi
+                [ -s "${cert_file}" ] || continue
+                fqdn="${peer}.kanidm-headless.kanidm.svc.cluster.local"
+                printf '\n[replication."repl://%s:8444"]\ntype = "mutual-pull"\npartner_cert = "%s"\n' \
+                  "${fqdn}" "$(cat ${cert_file})" >> /config/server.toml
               done
           env:
             - name: POD_NAME
@@ -62,6 +60,7 @@ spec:
           volumeMounts:
             - name: config-template
               mountPath: /config-template
+              readOnly: true
             - name: config
               mountPath: /config
             - name: repl-certs
@@ -70,33 +69,6 @@ spec:
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
-        - name: repl-cert-publisher
-          image: bitnami/kubectl:1.33
-          restartPolicy: Always
-          command: ["/bin/sh", "-c"]
-          args:
-            - |
-              until kubectl exec "${POD_NAME}" -c kanidm -- /sbin/kanidmd renew-replication-certificate 2>/dev/null | grep -q '^# certificate:'; do
-                sleep 30
-              done
-              while true; do
-                cert=$(kubectl exec "${POD_NAME}" -c kanidm -- /sbin/kanidmd renew-replication-certificate 2>/dev/null \
-                  | grep '^# certificate:' | sed 's/^# certificate: "\(.*\)"$/\1/')
-                if [ -n "${cert}" ]; then
-                  kubectl patch configmap kanidm-repl-certs \
-                    --type=merge \
-                    -p "{\"data\":{\"${POD_NAME}\":\"${cert}\"}}"
-                fi
-                sleep 3600
-              done
-          env:
-            - name: POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: false
       containers:
         - name: kanidm
           image: kanidm/server:1.10.3
@@ -145,8 +117,8 @@ spec:
         - name: config
           emptyDir: {}
         - name: repl-certs
-          configMap:
-            name: kanidm-repl-certs
+          secret:
+            secretName: kanidm-repl-certs
         - name: tls
           secret:
             secretName: kanidm-tls

apps/base/kanidm/tlsroute.yaml

@@ -13,9 +13,14 @@ spec:
     - auth.unkin.net
     - au.auth.unkin.net
   parentRefs:
-    - name: kanidm
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: kanidm
       sectionName: https-passthrough
   rules:
     - backendRefs:
-        - name: kanidm
+        - group: ""
+          kind: Service
+          name: kanidm
           port: 8443
+          weight: 1

apps/base/kanidm/vaultauth.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: default
+  namespace: kanidm
+spec:
+  allowedNamespaces:
+    - kanidm
+  kubernetes:
+    audiences:
+      - vault
+    role: default
+    serviceAccount: default
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default

apps/base/kanidm/vaultstaticsecret.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: repl-certs
+  namespace: kanidm
+  labels:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
+spec:
+  vaultAuthRef: default
+  mount: kv
+  type: kv-v2
+  path: kubernetes/namespace/kanidm/default/repl-certs
+  refreshAfter: 5m
+  destination:
+    name: kanidm-repl-certs
+    create: true
+    overwrite: true
+  hmacSecretData: true

apps/base/priority-classes/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - priorityclasses.yaml

apps/base/priority-classes/priorityclasses.yaml

@@ -0,0 +1,36 @@
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: low
+value: 100
+preemptionPolicy: Never
+globalDefault: false
+description: "Low-importance workloads. Can be evicted under pressure but will not preempt other pods."
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: power
+value: 100
+preemptionPolicy: Never
+globalDefault: false
+description: "Compute-heavy workloads with low scheduling importance. Evictable under pressure."
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: medium
+value: 10000
+preemptionPolicy: PreemptLowerPriority
+globalDefault: false
+description: "Standard workloads. Will preempt low-priority pods if the cluster is under pressure."
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: high
+value: 100000
+preemptionPolicy: PreemptLowerPriority
+globalDefault: false
+description: "High-importance services. Will preempt medium- and low-priority pods if necessary."

apps/base/puppet/service_puppetdb.yaml

@@ -9,7 +9,6 @@ metadata:
   name: puppetdb
   namespace: puppet
 spec:
-  clusterIP: null
   ports:
   - name: pdb-http
     port: 8080

apps/base/vault/kustomization.yaml

@@ -6,3 +6,4 @@ resources:
   - namespace.yaml
   - gateway.yaml
   - httproute.yaml
+  - role_k8s-service-registration.yaml

apps/base/vault/role_k8s-service-registration.yaml

@@ -0,0 +1,24 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: vault-k8s-service-registration
+  namespace: vault
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: vault-k8s-service-registration
+  namespace: vault
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: vault-k8s-service-registration
+subjects:
+  - kind: ServiceAccount
+    name: vault
+    namespace: vault

apps/overlays/au-syd1/consul/values.yaml

@@ -37,7 +37,7 @@ server:
       cpu: 100m
     limits:
       memory: 2Gi
-      cpu: 1000m
+      cpu: "1"
 
 client:
   enabled: false

apps/overlays/au-syd1/priority-classes/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/priority-classes

apps/overlays/au-syd1/vault/values.yaml

@@ -40,9 +40,7 @@ server:
           }
         }
 
-        service_registration "consul" {
-          address = "consul-server.consul.svc.cluster.local:8500"
-        }
+        service_registration "kubernetes" {}
 
   dataStorage:
     enabled: true
@@ -50,6 +48,14 @@ server:
     storageClass: cephrbd-fast-delete
     accessMode: ReadWriteOnce
 
+  extraEnv:
+    - name: VAULT_K8S_NAMESPACE
+      value: vault
+    - name: VAULT_K8S_POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+
   statefulSet:
     securityContext:
       container:

apps/overlays/au-syd1/woodpecker/values.yaml

@@ -2,6 +2,7 @@ agent:
   replicaCount: 3
   env:
     WOODPECKER_MAX_WORKFLOWS: "8"
+    WOODPECKER_BACKEND_K8S_PRIORITY_CLASS: power
     WOODPECKER_BACKEND_K8S_STORAGE_CLASS: cephrbd-fast-delete
     WOODPECKER_BACKEND_K8S_VOLUME_SIZE: 10G
     WOODPECKER_BACKEND_K8S_STORAGE_RWX: false

argocd/applicationsets/platform.yaml

@@ -22,6 +22,7 @@ spec:
       - path: apps/overlays/*/jfrog
       - path: apps/overlays/*/kanidm
       - path: apps/overlays/*/node-feature-discovery
+      - path: apps/overlays/*/priority-classes
       - path: apps/overlays/*/puppet
       - path: apps/overlays/*/purelb
       - path: apps/overlays/*/reflector-system

argocd/projects/platform.yaml

@@ -31,6 +31,8 @@ spec:
       server: https://kubernetes.default.svc
     - namespace: 'node-feature-discovery'
       server: https://kubernetes.default.svc
+    - namespace: 'priority-classes'
+      server: https://kubernetes.default.svc
     - namespace: 'purelb'
       server: https://kubernetes.default.svc
     - namespace: 'puppet'