fix(kanidm): prevent ArgoCD from overwriting repl-cert ConfigMap data

Remove the data keys from kanidm-repl-certs in git so ArgoCD never takes SSA ownership of them. Add ignoreDifferences for /data on that ConfigMap in the ApplicationSet template so ArgoCD doesn't flag sidecar-patched cert values as out-of-sync.
feat(kanidm): automate replication cert exchange via native sidecar
2026-05-24 00:07:02 +10:00 · 2026-05-24 00:02:40 +10:00 · 2026-05-23 23:49:30 +10:00 · 2026-05-23 23:16:52 +10:00 · 2026-05-23 23:12:52 +10:00 · 2026-05-23 22:40:49 +10:00
58 changed files with 1329 additions and 176 deletions
@@ -0,0 +1,36 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: artifactapi.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    external-dns.alpha.kubernetes.io/hostname: artifactapi.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+  name: artifactapi
+  namespace: artifactapi
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: artifactapi.k8s.syd1.au.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: artifactapi.k8s.syd1.au.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - kind: Secret
+            name: artifactapi-tls
+        mode: Terminate
@@ -0,0 +1,42 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: artifactapi-http-redirect
+  namespace: artifactapi
+spec:
+  hostnames:
+    - artifactapi.k8s.syd1.au.unkin.net
+  parentRefs:
+    - name: artifactapi
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: artifactapi
+  namespace: artifactapi
+spec:
+  hostnames:
+    - artifactapi.k8s.syd1.au.unkin.net
+  parentRefs:
+    - name: artifactapi
+      sectionName: https
+  rules:
+    - backendRefs:
+        - name: artifactapi-api
+          port: 80
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
@@ -1,32 +0,0 @@
---
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  annotations:
-    cert-manager.io/cluster-issuer: vault-issuer
-    cert-manager.io/common-name: artifactapi.k8s.syd1.au.unkin.net
-    cert-manager.io/private-key-size: "4096"
-    external-dns.alpha.kubernetes.io/hostname: artifactapi.k8s.syd1.au.unkin.net
-    external-dns.alpha.kubernetes.io/target: 198.18.200.0
-    nginx.ingress.kubernetes.io/proxy-body-size: 10g
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
-    nginx.ingress.kubernetes.io/ssl-redirect: "false"
-  name: artifactapi-ingress
-  namespace: artifactapi
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: artifactapi.k8s.syd1.au.unkin.net
-      http:
-        paths:
-          - backend:
-              service:
-                name: artifactapi-api
-                port:
-                  number: 80
-            path: /
-            pathType: Prefix
-  tls:
-    - hosts:
-        - artifactapi.k8s.syd1.au.unkin.net
-      secretName: artifactapi-tls
@@ -6,7 +6,8 @@ resources:
  - artifactapi-deployment.yaml
  - artifactapi-hpa.yaml
  - configmap.yaml
-  - ingress.yaml
+  - gateway.yaml
+  - httproute.yaml
  - namespace.yaml
  - postgres-deployment.yaml
  - pvc.yaml
@@ -30,7 +30,8 @@ remotes:
      - "^bitnami/"
      - "^curlimages/curl"
      - "^emberstack/kubernetes-reflector"
-      - "^hashicorp/vault-secrets-operator"
+      - "^hashicorp/consul"
+      - "^hashicorp/vault"
      - "^jfrog/"
      - "^rancher/"
      - "^traefik/"
@@ -15,6 +15,13 @@ metadata:
 spec:
  gatewayClassName: traefik-internal
  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: rancher.k8s.syd1.au.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
    - allowedRoutes:
        namespaces:
          from: Same
@@ -1,6 +1,28 @@
 ---
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
+metadata:
+  name: rancher-http-redirect
+  namespace: cattle-system
+spec:
+  hostnames:
+    - rancher.k8s.syd1.au.unkin.net
+  parentRefs:
+    - name: rancher
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
 metadata:
  name: rancher
  namespace: cattle-system
@@ -0,0 +1,51 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: consul
+  namespace: consul
+  labels:
+    app.kubernetes.io/name: consul
+    app.kubernetes.io/instance: consul
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: consul.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    cert-manager.io/alt-names: consul.service.consul
+    external-dns.alpha.kubernetes.io/hostname: consul.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      hostname: consul.k8s.syd1.au.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https
+      port: 443
+      protocol: HTTPS
+      hostname: consul.k8s.syd1.au.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - kind: Secret
+            name: consul-tls
+    - name: consul-svc
+      port: 443
+      protocol: HTTPS
+      hostname: consul.service.consul
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - kind: Secret
+            name: consul-tls
@@ -0,0 +1,71 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: consul-http-redirect
+  namespace: consul
+  labels:
+    app.kubernetes.io/name: consul
+    app.kubernetes.io/instance: consul
+spec:
+  hostnames:
+    - consul.k8s.syd1.au.unkin.net
+  parentRefs:
+    - name: consul
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: consul
+  namespace: consul
+  labels:
+    app.kubernetes.io/name: consul
+    app.kubernetes.io/instance: consul
+spec:
+  hostnames:
+    - consul.k8s.syd1.au.unkin.net
+  parentRefs:
+    - name: consul
+      sectionName: https
+  rules:
+    - backendRefs:
+        - name: consul-ui
+          port: 80
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: consul-svc
+  namespace: consul
+  labels:
+    app.kubernetes.io/name: consul
+    app.kubernetes.io/instance: consul
+spec:
+  hostnames:
+    - consul.service.consul
+  parentRefs:
+    - name: consul
+      sectionName: consul-svc
+  rules:
+    - backendRefs:
+        - name: consul-ui
+          port: 80
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
@@ -0,0 +1,8 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - gateway.yaml
+  - httproute.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: consul
@@ -0,0 +1,26 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: kanidm-tls
+  namespace: kanidm
+  labels:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
+spec:
+  secretName: kanidm-tls
+  issuerRef:
+    kind: ClusterIssuer
+    name: vault-issuer
+  commonName: auth.unkin.net
+  dnsNames:
+    - auth.unkin.net
+    - au.auth.unkin.net
+    - kanidm.k8s.syd1.au.unkin.net
+    - kanidm.kanidm.svc.cluster.local
+    - kanidm-0.kanidm-headless.kanidm.svc.cluster.local
+    - kanidm-1.kanidm-headless.kanidm.svc.cluster.local
+    - kanidm-2.kanidm-headless.kanidm.svc.cluster.local
+  privateKey:
+    algorithm: RSA
+    size: 4096
@@ -0,0 +1,40 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kanidm-config
+  namespace: kanidm
+  labels:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
+data:
+  server.toml: |
+    version = "2"
+
+    domain = "auth.unkin.net"
+    origin = "https://auth.unkin.net"
+    bindaddress = "[::]:8443"
+    db_path = "/data/kanidm.db"
+    db_arc_size = 2048
+    tls_chain = "/data/tls/tls.crt"
+    tls_key = "/data/tls/tls.key"
+    log_level = "info"
+
+    [online_backup]
+    path = "/data/backups/"
+    schedule = "0 22 * * *"
+    versions = 7
+
+    [replication]
+    origin = "__REPL_ORIGIN__"
+    bindaddress = "[::]:8444"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kanidm-repl-certs
+  namespace: kanidm
+  labels:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
+data: {}
@@ -0,0 +1,30 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: kanidm
+  namespace: kanidm
+  labels:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
+    traefik.io/instance: internal
+  annotations:
+    external-dns.alpha.kubernetes.io/hostname: kanidm.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https-passthrough
+      port: 443
+      protocol: TLS
+      tls:
+        mode: Passthrough
+      allowedRoutes:
+        namespaces:
+          from: Same
@@ -0,0 +1,27 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: kanidm-http-redirect
+  namespace: kanidm
+  labels:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
+spec:
+  hostnames:
+    - kanidm.k8s.syd1.au.unkin.net
+    - auth.unkin.net
+    - au.auth.unkin.net
+  parentRefs:
+    - name: kanidm
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
@@ -0,0 +1,16 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - serviceaccount.yaml
+  - rbac.yaml
+  - certificate.yaml
+  - configmap.yaml
+  - service.yaml
+  - statefulset.yaml
+  - poddisruptionbudget.yaml
+  - gateway.yaml
+  - httproute.yaml
+  - tlsroute.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kanidm
@@ -0,0 +1,15 @@
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: kanidm
+  namespace: kanidm
+  labels:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kanidm
+      app.kubernetes.io/instance: kanidm
@@ -0,0 +1,37 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kanidm-repl
+  namespace: kanidm
+  labels:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    resourceNames: ["kanidm-repl-certs"]
+    verbs: ["get", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kanidm-repl
+  namespace: kanidm
+  labels:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
+subjects:
+  - kind: ServiceAccount
+    name: kanidm
+    namespace: kanidm
+roleRef:
+  kind: Role
+  name: kanidm-repl
+  apiGroup: rbac.authorization.k8s.io
@@ -0,0 +1,43 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kanidm
+  namespace: kanidm
+  labels:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
+spec:
+  type: ClusterIP
+  ports:
+    - name: https
+      port: 8443
+      targetPort: https
+      protocol: TCP
+  selector:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kanidm-headless
+  namespace: kanidm
+  labels:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
+spec:
+  type: ClusterIP
+  clusterIP: None
+  ports:
+    - name: https
+      port: 8443
+      targetPort: https
+      protocol: TCP
+    - name: replication
+      port: 8444
+      targetPort: replication
+      protocol: TCP
+  selector:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kanidm
+  namespace: kanidm
+  labels:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
@@ -0,0 +1,161 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: kanidm
+  namespace: kanidm
+  labels:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
+spec:
+  serviceName: kanidm-headless
+  replicas: 3
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kanidm
+      app.kubernetes.io/instance: kanidm
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kanidm
+        app.kubernetes.io/instance: kanidm
+    spec:
+      serviceAccountName: kanidm
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app.kubernetes.io/name: kanidm
+                  app.kubernetes.io/instance: kanidm
+              topologyKey: kubernetes.io/hostname
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        runAsNonRoot: true
+        fsGroup: 1000
+      initContainers:
+        - name: config-init
+          image: ghcr.io/kanidm/server:1.10.3
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -e
+              REPL_ORIGIN="repl://${POD_NAME}.kanidm-headless.kanidm.svc.cluster.local:8444"
+              sed "s|__REPL_ORIGIN__|${REPL_ORIGIN}|g" /config-template/server.toml > /config/server.toml
+              for peer in kanidm-0 kanidm-1 kanidm-2; do
+                if [ "${peer}" = "${POD_NAME}" ]; then
+                  continue
+                fi
+                cert_file="/repl-certs/${peer}"
+                if [ -s "${cert_file}" ]; then
+                  fqdn="${peer}.kanidm-headless.kanidm.svc.cluster.local"
+                  printf '\n[replication."repl://%s:8444"]\ntype = "mutual-pull"\npartner_cert = "%s"\n' \
+                    "${fqdn}" "$(cat ${cert_file})" >> /config/server.toml
+                fi
+              done
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          volumeMounts:
+            - name: config-template
+              mountPath: /config-template
+            - name: config
+              mountPath: /config
+            - name: repl-certs
+              mountPath: /repl-certs
+              readOnly: true
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+        - name: repl-cert-publisher
+          image: bitnami/kubectl:1.33
+          restartPolicy: Always
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              until kubectl exec "${POD_NAME}" -c kanidm -- /sbin/kanidmd renew-replication-certificate 2>/dev/null | grep -q '^# certificate:'; do
+                sleep 30
+              done
+              while true; do
+                cert=$(kubectl exec "${POD_NAME}" -c kanidm -- /sbin/kanidmd renew-replication-certificate 2>/dev/null \
+                  | grep '^# certificate:' | sed 's/^# certificate: "\(.*\)"$/\1/')
+                if [ -n "${cert}" ]; then
+                  kubectl patch configmap kanidm-repl-certs \
+                    --type=merge \
+                    -p "{\"data\":{\"${POD_NAME}\":\"${cert}\"}}"
+                fi
+                sleep 3600
+              done
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: false
+      containers:
+        - name: kanidm
+          image: ghcr.io/kanidm/server:1.10.3
+          command: ["/sbin/kanidmd"]
+          args: ["server", "-c", "/config/server.toml"]
+          ports:
+            - name: https
+              containerPort: 8443
+              protocol: TCP
+            - name: replication
+              containerPort: 8444
+              protocol: TCP
+          volumeMounts:
+            - name: data
+              mountPath: /data
+            - name: config
+              mountPath: /config
+              readOnly: true
+            - name: tls
+              mountPath: /data/tls
+              readOnly: true
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: false
+          resources:
+            requests:
+              memory: 256Mi
+              cpu: 100m
+            limits:
+              memory: 1Gi
+              cpu: 500m
+          readinessProbe:
+            tcpSocket:
+              port: 8443
+            initialDelaySeconds: 15
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket:
+              port: 8443
+            initialDelaySeconds: 30
+            periodSeconds: 30
+      volumes:
+        - name: config-template
+          configMap:
+            name: kanidm-config
+        - name: config
+          emptyDir: {}
+        - name: repl-certs
+          configMap:
+            name: kanidm-repl-certs
+        - name: tls
+          secret:
+            secretName: kanidm-tls
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: [ReadWriteOnce]
+        storageClassName: cephrbd-fast-delete
+        resources:
+          requests:
+            storage: 10Gi
@@ -0,0 +1,21 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: TLSRoute
+metadata:
+  name: kanidm
+  namespace: kanidm
+  labels:
+    app.kubernetes.io/name: kanidm
+    app.kubernetes.io/instance: kanidm
+spec:
+  hostnames:
+    - kanidm.k8s.syd1.au.unkin.net
+    - auth.unkin.net
+    - au.auth.unkin.net
+  parentRefs:
+    - name: kanidm
+      sectionName: https-passthrough
+  rules:
+    - backendRefs:
+        - name: kanidm
+          port: 8443
@@ -76,11 +76,11 @@ spec:
    updateInterval: 30
  resources:
    limits:
-      cpu: 500m
-      memory: 512Mi
+      cpu: 1
+      memory: 1024Mi
    requests:
      cpu: 250m
-      memory: 256Mi
+      memory: 512Mi
  smartShutdownTimeout: 180
  startDelay: 3600
  stopDelay: 1800
@@ -56,10 +56,10 @@ spec:
          resources:
            limits:
              cpu: "2"
-              memory: 6Gi
+              memory: 8Gi
            requests:
              cpu: 250m
-              memory: 2Gi
+              memory: 6Gi
          volumeMounts:
            - mountPath: /app/config.yaml
              name: config
@@ -0,0 +1,36 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: litellm.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    external-dns.alpha.kubernetes.io/hostname: litellm.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+  name: litellm
+  namespace: litellm
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: litellm.k8s.syd1.au.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: litellm.k8s.syd1.au.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - kind: Secret
+            name: litellm-tls
+        mode: Terminate
@@ -10,14 +10,14 @@ spec:
    kind: Deployment
    name: litellm
  minReplicas: 2
-  maxReplicas: 10
+  maxReplicas: 4
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
-          averageUtilization: 60
+          averageUtilization: 80
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 0
@@ -25,7 +25,7 @@ spec:
      policies:
        - type: Percent
          value: 100
-          periodSeconds: 30
+          periodSeconds: 60
        - type: Pods
          value: 4
          periodSeconds: 30
@@ -34,7 +34,7 @@ spec:
      selectPolicy: Min
      policies:
        - type: Percent
-          value: 10
+          value: 30
          periodSeconds: 60
        - type: Pods
          value: 2
@@ -0,0 +1,42 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: litellm-http-redirect
+  namespace: litellm
+spec:
+  hostnames:
+    - litellm.k8s.syd1.au.unkin.net
+  parentRefs:
+    - name: litellm
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: litellm
+  namespace: litellm
+spec:
+  hostnames:
+    - litellm.k8s.syd1.au.unkin.net
+  parentRefs:
+    - name: litellm
+      sectionName: https
+  rules:
+    - backendRefs:
+        - name: litellm
+          port: 4000
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
@@ -1,29 +0,0 @@
---
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  annotations:
-    kubernetes.io/ingress.class: nginx
-    external-dns.alpha.kubernetes.io/hostname: litellm.k8s.syd1.au.unkin.net
-    external-dns.alpha.kubernetes.io/target: 198.18.200.0
-    cert-manager.io/cluster-issuer: vault-issuer
-    cert-manager.io/common-name: litellm.k8s.syd1.au.unkin.net
-    cert-manager.io/private-key-size: "4096"
-  name: litellm
-  namespace: litellm
-spec:
-  rules:
-    - host: litellm.k8s.syd1.au.unkin.net
-      http:
-        paths:
-          - backend:
-              service:
-                name: litellm
-                port:
-                  number: 4000
-            path: /
-            pathType: Prefix
-  tls:
-    - hosts:
-        - litellm.k8s.syd1.au.unkin.net
-      secretName: litellm-tls
@@ -7,7 +7,8 @@ resources:
  - cnpg_pooler.yaml
  - deployment.yaml
  - hpa.yaml
-  - ingress.yaml
+  - gateway.yaml
+  - httproute.yaml
  - namespace.yaml
  - redis-deployment.yaml
  - redis-pvc.yaml
@@ -0,0 +1,36 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: paperclip.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    external-dns.alpha.kubernetes.io/hostname: paperclip.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+  name: paperclip
+  namespace: paperclip
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: paperclip.k8s.syd1.au.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: paperclip.k8s.syd1.au.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - kind: Secret
+            name: paperclip-tls
+        mode: Terminate
@@ -0,0 +1,42 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: paperclip-http-redirect
+  namespace: paperclip
+spec:
+  hostnames:
+    - paperclip.k8s.syd1.au.unkin.net
+  parentRefs:
+    - name: paperclip
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: paperclip
+  namespace: paperclip
+spec:
+  hostnames:
+    - paperclip.k8s.syd1.au.unkin.net
+  parentRefs:
+    - name: paperclip
+      sectionName: https
+  rules:
+    - backendRefs:
+        - name: paperclip
+          port: 3100
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
@@ -1,29 +0,0 @@
---
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  annotations:
-    kubernetes.io/ingress.class: nginx
-    external-dns.alpha.kubernetes.io/hostname: paperclip.k8s.syd1.au.unkin.net
-    external-dns.alpha.kubernetes.io/target: 198.18.200.0
-    cert-manager.io/cluster-issuer: vault-issuer
-    cert-manager.io/common-name: paperclip.k8s.syd1.au.unkin.net
-    cert-manager.io/private-key-size: "4096"
-  name: paperclip
-  namespace: paperclip
-spec:
-  rules:
-    - host: paperclip.k8s.syd1.au.unkin.net
-      http:
-        paths:
-          - backend:
-              service:
-                name: paperclip
-                port:
-                  number: 3100
-            path: /
-            pathType: Prefix
-  tls:
-    - hosts:
-        - paperclip.k8s.syd1.au.unkin.net
-      secretName: paperclip-tls
@@ -6,7 +6,8 @@ resources:
  - cnpg_cluster.yaml
  - cnpg_pooler.yaml
  - deployment.yaml
-  - ingress.yaml
+  - gateway.yaml
+  - httproute.yaml
  - namespace.yaml
  - services.yaml
  - vaultauth.yaml
@@ -0,0 +1,40 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: puppetboard.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    external-dns.alpha.kubernetes.io/hostname: puppetboard.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+  labels:
+    traefik.io/instance: internal
+    app.kubernetes.io/component: puppetboard
+    app.kubernetes.io/instance: puppetserver
+    app.kubernetes.io/name: puppetserver
+    app.kubernetes.io/version: 8.8.0
+  name: puppetboard
+  namespace: puppet
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: puppetboard.k8s.syd1.au.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: puppetboard.k8s.syd1.au.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - kind: Secret
+            name: puppetboard-tls
+        mode: Terminate
@@ -0,0 +1,33 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: puppetdb.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    external-dns.alpha.kubernetes.io/hostname: puppetdb.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+  labels:
+    traefik.io/instance: internal
+    app.kubernetes.io/component: puppetdb
+    app.kubernetes.io/instance: puppetserver
+    app.kubernetes.io/name: puppetserver
+    app.kubernetes.io/version: 8.8.0
+  name: puppetdb
+  namespace: puppet
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: puppetdb.k8s.syd1.au.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - kind: Secret
+            name: puppetdb-tls
+        mode: Terminate
@@ -0,0 +1,52 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  labels:
+    app.kubernetes.io/component: puppetboard
+    app.kubernetes.io/instance: puppetserver
+    app.kubernetes.io/name: puppetserver
+    app.kubernetes.io/version: 8.8.0
+  name: puppetboard-http-redirect
+  namespace: puppet
+spec:
+  hostnames:
+    - puppetboard.k8s.syd1.au.unkin.net
+  parentRefs:
+    - name: puppetboard
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  labels:
+    app.kubernetes.io/component: puppetboard
+    app.kubernetes.io/instance: puppetserver
+    app.kubernetes.io/name: puppetserver
+    app.kubernetes.io/version: 8.8.0
+  name: puppetboard
+  namespace: puppet
+spec:
+  hostnames:
+    - puppetboard.k8s.syd1.au.unkin.net
+  parentRefs:
+    - name: puppetboard
+      sectionName: https
+  rules:
+    - backendRefs:
+        - name: puppetboard
+          port: 80
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
@@ -0,0 +1,25 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  labels:
+    app.kubernetes.io/component: puppetdb
+    app.kubernetes.io/instance: puppetserver
+    app.kubernetes.io/name: puppetserver
+    app.kubernetes.io/version: 8.8.0
+  name: puppetdb
+  namespace: puppet
+spec:
+  hostnames:
+    - puppetdb.k8s.syd1.au.unkin.net
+  parentRefs:
+    - name: puppetdb
+      sectionName: https
+  rules:
+    - backendRefs:
+        - name: puppetdb
+          port: 8080
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
@@ -1,34 +0,0 @@
---
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  annotations:
-    kubernetes.io/ingress.class: nginx
-    external-dns.alpha.kubernetes.io/hostname: puppetboard.k8s.syd1.au.unkin.net
-    external-dns.alpha.kubernetes.io/target: 198.18.200.0
-    cert-manager.io/cluster-issuer: vault-issuer
-    cert-manager.io/common-name: puppetboard.k8s.syd1.au.unkin.net
-    cert-manager.io/private-key-size: "4096"
-  labels:
-    app.kubernetes.io/component: puppetboard
-    app.kubernetes.io/instance: puppetserver
-    app.kubernetes.io/name: puppetserver
-    app.kubernetes.io/version: 8.8.0
-  name: puppetboard
-  namespace: puppet
-spec:
-  rules:
-    - host: puppetboard.k8s.syd1.au.unkin.net
-      http:
-        paths:
-          - backend:
-              service:
-                name: puppetboard
-                port:
-                  number: 80
-            path: /
-            pathType: Prefix
-  tls:
-    - hosts:
-        - puppetboard.k8s.syd1.au.unkin.net
-      secretName: puppetboard-tls
@@ -1,34 +0,0 @@
---
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  annotations:
-    kubernetes.io/ingress.class: nginx
-    external-dns.alpha.kubernetes.io/hostname: puppetdb.k8s.syd1.au.unkin.net
-    external-dns.alpha.kubernetes.io/target: 198.18.200.0
-    cert-manager.io/cluster-issuer: vault-issuer
-    cert-manager.io/common-name: puppetdb.k8s.syd1.au.unkin.net
-    cert-manager.io/private-key-size: "4096"
-  labels:
-    app.kubernetes.io/component: puppetdb
-    app.kubernetes.io/instance: puppetserver
-    app.kubernetes.io/name: puppetserver
-    app.kubernetes.io/version: 8.8.0
-  name: puppetdb
-  namespace: puppet
-spec:
-  rules:
-    - host: puppetdb.k8s.syd1.au.unkin.net
-      http:
-        paths:
-          - backend:
-              service:
-                name: puppetdb
-                port:
-                  number: 8080
-            path: /
-            pathType: Prefix
-  tls:
-    - hosts:
-        - puppetdb.k8s.syd1.au.unkin.net
-      secretName: puppetdb-tls
@@ -25,8 +25,10 @@ resources:
  - horizontalpodautoscaler_puppetserver-masters-autoscaler.yaml
  - horizontalpodautoscaler_puppetserver-puppetboard-autoscaler.yaml
  - horizontalpodautoscaler_puppetserver-puppetdb-autoscaler.yaml
-  - ingress_puppetboard.yaml
-  - ingress_puppetdb.yaml
+  - gateway_puppetboard.yaml
+  - httproute_puppetboard.yaml
+  - gateway_puppetdb.yaml
+  - httproute_puppetdb.yaml
  - service_puppetserver-agents-to-puppet.yaml
  - service_puppet-headless.yaml
  - service_puppet.yaml
@@ -0,0 +1,50 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: vault
+  namespace: vault
+  labels:
+    app.kubernetes.io/name: vault
+    app.kubernetes.io/instance: vault
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: vault.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    cert-manager.io/alt-names: vault.service.consul,vault.query.consul
+    external-dns.alpha.kubernetes.io/hostname: vault.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      hostname: vault.k8s.syd1.au.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https
+      port: 443
+      protocol: HTTPS
+      hostname: vault.k8s.syd1.au.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - kind: Secret
+            name: vault-tls
+    - name: vault-direct
+      port: 8200
+      protocol: HTTPS
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - kind: Secret
+            name: vault-tls
@@ -0,0 +1,72 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: vault-http-redirect
+  namespace: vault
+  labels:
+    app.kubernetes.io/name: vault
+    app.kubernetes.io/instance: vault
+spec:
+  hostnames:
+    - vault.k8s.syd1.au.unkin.net
+  parentRefs:
+    - name: vault
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: vault
+  namespace: vault
+  labels:
+    app.kubernetes.io/name: vault
+    app.kubernetes.io/instance: vault
+spec:
+  hostnames:
+    - vault.k8s.syd1.au.unkin.net
+  parentRefs:
+    - name: vault
+      sectionName: https
+  rules:
+    - backendRefs:
+        - name: vault
+          port: 8200
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: vault-consul
+  namespace: vault
+  labels:
+    app.kubernetes.io/name: vault
+    app.kubernetes.io/instance: vault
+spec:
+  hostnames:
+    - vault.service.consul
+    - vault.query.consul
+  parentRefs:
+    - name: vault
+      sectionName: vault-direct
+  rules:
+    - backendRefs:
+        - name: vault
+          port: 8200
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
@@ -0,0 +1,8 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - gateway.yaml
+  - httproute.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: vault
@@ -8,7 +8,7 @@ resources:
 helmCharts:
  - name: cert-manager
    repo: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/virtual/helm
-    version: "v1.19.2"
+    version: "v1.20.2"
    releaseName: cert-manager
    namespace: cert-manager
    valuesFile: values.yaml
@@ -1,6 +1,11 @@
 crds:
  enabled: true

+config:
+  apiVersion: controller.config.cert-manager.io/v1alpha1
+  kind: ControllerConfiguration
+  enableGatewayAPI: true
+
 replicaCount: 2

 resources:
@@ -0,0 +1,16 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/consul
+
+helmCharts:
+  - name: consul
+    repo: https://helm.releases.hashicorp.com
+    version: "1.9.7"
+    releaseName: consul
+    namespace: consul
+    valuesFile: values.yaml
+    apiVersions:
+      - policy/v1/PodDisruptionBudget
@@ -0,0 +1,58 @@
+global:
+  name: consul
+  datacenter: au-syd1
+  domain: consul
+
+server:
+  image: hashicorp/consul:1.22.7
+  replicas: 5
+  bootstrapExpect: 5
+  storage: 10Gi
+  storageClass: cephrbd-fast-delete
+
+  connect: true
+
+  disruptionBudget:
+    maxUnavailable: 1
+
+  extraConfig: |
+    {
+      "disable_remote_exec": true,
+      "disable_update_check": true,
+      "performance": {
+        "raft_multiplier": 10
+      },
+      "ports": {
+        "dns": 8600,
+        "grpc": 8502,
+        "http": 8500,
+        "https": -1
+      },
+      "primary_datacenter": "au-syd1"
+    }
+
+  resources:
+    requests:
+      memory: 256Mi
+      cpu: 100m
+    limits:
+      memory: 2Gi
+      cpu: 1000m
+
+client:
+  enabled: false
+
+ui:
+  enabled: true
+  service:
+    type: ClusterIP
+
+connectInject:
+  enabled: false
+
+dns:
+  enabled: true
+  type: LoadBalancer
+  annotations: |
+    purelb.io/service-group: "common"
+    purelb.io/addresses: 198.18.200.5
@@ -8,7 +8,7 @@ resources:
 helmCharts:
  - name: external-dns
    repo: https://artifactapi.k8s.syd1.au.unkin.net/api/v1/virtual/helm
-    version: "1.19.0"
+    version: "1.21.1"
    releaseName: externaldns
    namespace: externaldns
    valuesFile: values.yaml
@@ -25,6 +25,7 @@ sources:
  - service
  - ingress
  - gateway-httproute
+  - gateway-grpcroute

 # Environment variables for TSIG secret and algorithm from Vault
 env:
@@ -50,3 +51,5 @@ extraArgs:
  - --rfc2136-tsig-axfr
  - --rfc2136-tsig-secret=$(EXTERNAL_DNS_RFC2136_TSIG_SECRET)
  - --ingress-class=nginx
+
+logLevel: debug
@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/kanidm
@@ -94,5 +94,7 @@ ports:
    port: 80
  websecure:
    port: 443
+  vault-direct:
+    port: 8200

 enabled: true
@@ -0,0 +1,14 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/vault
+
+helmCharts:
+  - name: vault
+    repo: https://helm.releases.hashicorp.com
+    version: "0.32.0"
+    releaseName: vault
+    namespace: vault
+    valuesFile: values.yaml
@@ -0,0 +1,73 @@
+server:
+  image:
+    repository: hashicorp/vault
+    tag: "2.0.1"
+
+  ha:
+    enabled: true
+    replicas: 5
+
+    raft:
+      enabled: true
+      setNodeId: true
+      config: |
+        ui            = true
+        disable_mlock = true
+
+        listener "tcp" {
+          address         = "[::]:8200"
+          cluster_address = "[::]:8201"
+          tls_disable     = "true"
+        }
+
+        storage "raft" {
+          path = "/vault/data"
+
+          retry_join {
+            leader_api_addr = "http://vault-0.vault-internal.vault.svc.cluster.local:8200"
+          }
+          retry_join {
+            leader_api_addr = "http://vault-1.vault-internal.vault.svc.cluster.local:8200"
+          }
+          retry_join {
+            leader_api_addr = "http://vault-2.vault-internal.vault.svc.cluster.local:8200"
+          }
+          retry_join {
+            leader_api_addr = "http://vault-3.vault-internal.vault.svc.cluster.local:8200"
+          }
+          retry_join {
+            leader_api_addr = "http://vault-4.vault-internal.vault.svc.cluster.local:8200"
+          }
+        }
+
+        service_registration "consul" {
+          address = "consul-server.consul.svc.cluster.local:8500"
+        }
+
+  dataStorage:
+    enabled: true
+    size: 10Gi
+    storageClass: cephrbd-fast-delete
+    accessMode: ReadWriteOnce
+
+  statefulSet:
+    securityContext:
+      container:
+        capabilities:
+          add:
+            - IPC_LOCK
+
+  resources:
+    requests:
+      memory: 256Mi
+      cpu: 100m
+    limits:
+      memory: 2Gi
+      cpu: 1000m
+
+injector:
+  enabled: false
+
+ui:
+  enabled: true
+  serviceType: ClusterIP
@@ -8,7 +8,7 @@ resources:
 helmCharts:
  - name: woodpecker
    repo: oci://ghcr.io/woodpecker-ci/helm
-    version: "3.5.1"
+    version: "3.6.3"
    releaseName: woodpecker
    namespace: woodpecker
    valuesFile: values.yaml
@@ -15,10 +15,12 @@ spec:
      - path: apps/overlays/*/cert-manager
      - path: apps/overlays/*/certificates
      - path: apps/overlays/*/cnpg-system
+      - path: apps/overlays/*/consul
      - path: apps/overlays/*/elastic-system
      - path: apps/overlays/*/externaldns
      - path: apps/overlays/*/inteldeviceplugins-system
      - path: apps/overlays/*/jfrog
+      - path: apps/overlays/*/kanidm
      - path: apps/overlays/*/node-feature-discovery
      - path: apps/overlays/*/puppet
      - path: apps/overlays/*/purelb
@@ -27,6 +29,7 @@ spec:
      - path: apps/overlays/*/reposync
      - path: apps/overlays/*/traefik-system
      - path: apps/overlays/*/vm-system
+      - path: apps/overlays/*/vault
      - path: apps/overlays/*/vso-system
      - path: apps/overlays/*/woodpecker
  template:
@@ -41,6 +44,12 @@ spec:
      destination:
        server: https://kubernetes.default.svc
        namespace: '{{path[3]}}'  # Use directory name as namespace
+      ignoreDifferences:
+      - group: ""
+        kind: ConfigMap
+        name: kanidm-repl-certs
+        jsonPointers:
+        - /data
      syncPolicy:
        automated:
          prune: true
@@ -21,10 +21,14 @@ spec:
      server: https://kubernetes.default.svc
    - namespace: 'certificates'
      server: https://kubernetes.default.svc
+    - namespace: 'consul'
+      server: https://kubernetes.default.svc
    - namespace: 'externaldns'
      server: https://kubernetes.default.svc
    - namespace: 'jfrog'
      server: https://kubernetes.default.svc
+    - namespace: 'kanidm'
+      server: https://kubernetes.default.svc
    - namespace: 'node-feature-discovery'
      server: https://kubernetes.default.svc
    - namespace: 'purelb'
@@ -33,6 +37,8 @@ spec:
      server: https://kubernetes.default.svc
    - namespace: 'reposync'
      server: https://kubernetes.default.svc
+    - namespace: 'vault'
+      server: https://kubernetes.default.svc
    - namespace: 'woodpecker'
      server: https://kubernetes.default.svc
  clusterResourceWhitelist:
Author	SHA1	Message	Date
unkinben	0fd269ed9a	fix(kanidm): prevent ArgoCD from overwriting repl-cert ConfigMap data Remove the data keys from kanidm-repl-certs in git so ArgoCD never takes SSA ownership of them. Add ignoreDifferences for /data on that ConfigMap in the ApplicationSet template so ArgoCD doesn't flag sidecar-patched cert values as out-of-sync.	2026-05-24 00:07:02 +10:00
unkinben	1112309a60	feat(kanidm): automate replication cert exchange via native sidecar Add a native sidecar (bitnami/kubectl, restartPolicy: Always) that runs kanidmd renew-replication-certificate on each pod and patches the result into the kanidm-repl-certs ConfigMap (certs are public keys, not secrets). The config-init init container reads peer certs from the ConfigMap at startup, building the replication stanza automatically — no manual cert exchange required after first deploy. Add RBAC (Role + RoleBinding) granting the kanidm service account pods/exec and configmap patch permissions scoped to the kanidm namespace.	2026-05-24 00:02:40 +10:00
unkinben	b87844b4cc	feat(kanidm): 3 replicas, PDB maxUnavailable=1, host anti-affinity - Increase replicas from 2 to 3 - Add kanidm-2 headless DNS SAN to TLS certificate - Add PodDisruptionBudget (maxUnavailable: 1) to maintain quorum during node drains - Add requiredDuringSchedulingIgnoredDuringExecution pod anti-affinity on kubernetes.io/hostname to spread replicas across distinct hosts - Update replication peers comment to include kanidm-2 cert exchange step	2026-05-23 23:49:30 +10:00
unkinben	2ddd2173e3	fix(argocd): add consul and vault to platform project destinations Same fix as benvin/fix-platform-destinations applied to this branch.	2026-05-23 23:16:52 +10:00
unkinben	1e88102a46	feat(kanidm): deploy Kanidm 1.10.3 with 2-replica replication ## Summary - Deploys Kanidm 1.10.3 (ghcr.io/kanidm/server:1.10.3) as a 2-replica StatefulSet with built-in replication enabled - Domain: auth.unkin.net (primary WebAuthn origin); au.auth.unkin.net is an additional hostname for this au-syd1 instance - TLS: cert-manager Certificate (vault-issuer) covering auth.unkin.net, au.auth.unkin.net, kanidm.k8s.syd1.au.unkin.net, and internal headless pod DNS names — Kanidm terminates TLS itself (passthrough) - Gateway: TLS passthrough on port 443 via TLSRoute; HTTP on port 80 redirects to HTTPS; external-dns creates kanidm.k8s.syd1.au.unkin.net (not used in server.toml; canonical origin is auth.unkin.net only) - Replication: init container generates per-pod server.toml with the correct repl:// origin (kanidm-{N}.kanidm-headless.kanidm.svc); populate kanidm-repl-peers ConfigMap post-deployment after running `kanidmd show-replication-certificate` on each pod - Storage: 10Gi cephrbd-fast-delete PVC per pod via volumeClaimTemplates - Security: runAsUser/runAsGroup 1000, runAsNonRoot, no privilege escalation, allowPrivilegeEscalation=false - ArgoCD: platform ApplicationSet and Project updated for kanidm namespace ## Requires - PR benvin/kanidm-artifactapi (add ^kanidm/ to ghcr immutable patterns) to be merged first so artifactapi can cache ghcr.io/kanidm/server ## Post-deployment steps 1. Wait for both pods to reach Running state 2. Exchange replication certificates between pods: kubectl exec -n kanidm kanidm-0 -- kanidmd show-replication-certificate kubectl exec -n kanidm kanidm-1 -- kanidmd show-replication-certificate 3. Edit kanidm-repl-peers ConfigMap with both nodes' certs (see template in configmap.yaml comments) 4. kubectl rollout restart statefulset/kanidm -n kanidm ## Test plan - [x] Sandbox tested in sandbox-kanidm: all 11 resources server dry-run OK - [ ] After merge: ArgoCD syncs kanidm namespace - [ ] Verify auth.unkin.net and au.auth.unkin.net reachable via Gateway - [ ] Verify kanidm.k8s.syd1.au.unkin.net DNS record created by external-dns - [ ] Complete replication cert exchange and verify replication active	2026-05-23 23:12:52 +10:00
unkinben	11ac2ae91e	feat(consul): deploy HashiCorp Consul 1.22.7 via Helm chart (5-replica cluster) (#149 ) ## Summary - Deploys HashiCorp Consul 1.22.7 using Helm chart 1.9.7 with 5 server replicas - Configuration modelled on production consul: \`datacenter=au-syd1\`, \`connect=true\`, \`raft_multiplier=10\`, HTTP on 8500, GRPC on 8502, HTTPS disabled - 5-replica server cluster with \`bootstrapExpect=5\` - 10Gi cephrbd-fast-delete PVC per server pod - Gateway API: HTTPS gateway + HTTPRoute (443→consul-consul-ui:80→8500) at \`consul.k8s.syd1.au.unkin.net\` - PodDisruptionBudget patched from \`policy/v1beta1\` to \`policy/v1\` (k8s 1.25+ compatibility) - ArgoCD platform ApplicationSet updated to include consul overlay path - Clients disabled (server-only deployment) - ConnectInject disabled (can be enabled later for service mesh) ## Requires - PR #147 (artifactapi: add hashicorp/consul to docker immutable patterns) to be merged first ## Test plan - [ ] Sandbox tested in \`sandbox-consul\`: all 5 server pods 1/1 Running, cluster formed - [ ] After merge: ArgoCD syncs consul namespace - [ ] Verify \`consul.k8s.syd1.au.unkin.net\` is accessible via Gateway Reviewed-on: #149	2026-05-23 22:40:49 +10:00
unkinben	d2be521878	feat(vault): deploy HashiCorp Vault 2.0.1 via Helm chart (5-replica HA raft) (#148 ) ## Summary - Deploys HashiCorp Vault 2.0.1 using Helm chart 0.32.0 in HA raft mode (5 replicas) - Configuration modelled on production vault: \`disable_mlock=true\`, headless-DNS retry_join for all 5 pods - IPC_LOCK capability added via \`server.statefulSet.securityContext.container\` - 10Gi cephrbd-fast-delete PVC per pod via \`dataStorage\` - Gateway API: HTTPS gateway + HTTPRoute (443→vault service port 8200) at \`vault.k8s.syd1.au.unkin.net\` - ArgoCD platform ApplicationSet updated to include vault overlay path - Injector disabled (no agent sidecar injection needed) ## Requires - PR #147 (artifactapi: add hashicorp/vault to docker immutable patterns) to be merged first ## Test plan - [ ] Sandbox tested in \`sandbox-vault\`: all 5 pods Running, raft cluster forming - [ ] After merge: ArgoCD syncs vault namespace - [ ] Operator runs \`vault operator init\` to initialize, then unseals all 5 nodes - [ ] Verify \`vault.k8s.syd1.au.unkin.net\` is accessible via Gateway Reviewed-on: #148	2026-05-23 22:39:41 +10:00
unkinben	bcd4c1a722	feat(cert-manager): upgrade to v1.20.2 and enable Gateway API support (#150 ) ## Summary - Upgrades cert-manager from v1.19.2 to v1.20.2 - Enables `enableGatewayAPI: true` via the `ControllerConfiguration` config block ## Why cert-manager's Gateway API integration was not enabled. Without it, `cert-manager.io/*` annotations on Gateway resources are ignored and no certificates are issued. This is required for the consul and vault PRs (#148, #149) to have their TLS certs automatically provisioned from their Gateway annotations. In v1.20.2, `ExperimentalGatewayAPISupport` is BETA and defaults to true — enabling `enableGatewayAPI` in the controller config activates the gateway-shim controller. ## Test plan - [ ] cert-manager rolls out cleanly (v1.20.2 pods become Ready) - [ ] After rollout, existing Gateway-annotated services (artifactapi, puppet, litellm) retain valid certs - [ ] New Gateway resources with `cert-manager.io/cluster-issuer` annotations trigger Certificate creation Reviewed-on: #150	2026-05-23 22:38:39 +10:00
unkinben	6d9530b1ee	feat(artifactapi): add hashicorp/consul and hashicorp/vault to docker immutable patterns (#147 ) ## Summary - Adds \`^hashicorp/consul\` and \`^hashicorp/vault\` to the dockerhub immutable_patterns in artifactapi's remote-docker.yaml - Replaces the more specific \`^hashicorp/vault-secrets-operator\` pattern since \`^hashicorp/vault\` subsumes it - Required for the benvin/vault and benvin/consul branches (vault:2.0.1 and consul:1.22.7) ## Test plan - [ ] Verify artifactapi accepts requests for hashicorp/vault and hashicorp/consul images after merge Reviewed-on: #147	2026-05-23 18:21:25 +10:00
unkinben	dcea768c15	feat(woodpecker): upgrade to v3.14.1 (chart 3.6.3) (#146 ) Reviewed-on: #146	2026-05-23 18:00:55 +10:00
unkinben	e05f9bfd83	feat: increase litellm resources (#144 ) finding litellm performance has dropped, crashed in multiple cases, and then it had scaled to the maximum level using the majority of memory in cluster. - reduce the rate at which litellm autoscales - increase the requests/limits to match usage Reviewed-on: #144	2026-05-23 17:59:43 +10:00
unkinben	445d8b6e7e	feat: add HTTP→HTTPS redirect to Gateway API services (#145 ) Add port 80 HTTP listener and redirect HTTPRoute to artifactapi, cattle-system (rancher), litellm, paperclip, and puppetboard — restoring the redirect behaviour that existed on the previous nginx/traefik Ingress resources. Reviewed-on: #145	2026-05-23 17:34:07 +10:00
unkinben	c2637da068	feat(artifactapi): migrate Ingress to Gateway API (#129 ) ## Summary - Replace `Ingress` (nginx) with `Gateway` + `HTTPRoute` using `traefik-internal` GatewayClass - TLS terminated at the Gateway listener; cert-manager provisions the certificate via `vault-issuer` - external-dns annotations moved to the Gateway ## Notes The original Ingress had nginx-specific annotations (`proxy-body-size: 10g`, `proxy-read-timeout: 600`) which are not portable to Gateway API. These can be re-introduced via a Traefik `Middleware` CRD if needed. ## Test plan - [ ] ArgoCD syncs the app cleanly - [ ] cert-manager issues the `artifactapi-tls` certificate - [ ] external-dns creates the DNS record - [ ] `https://artifactapi.k8s.syd1.au.unkin.net` is reachable Reviewed-on: #129	2026-05-23 16:06:33 +10:00
unkinben	90ddd932fe	feat(puppet): migrate puppetdb Ingress to Gateway API (#131 ) ## Summary - Replace `Ingress` (nginx) with `Gateway` + `HTTPRoute` using `traefik-internal` GatewayClass - TLS terminated at the Gateway listener; cert-manager provisions the certificate via `vault-issuer` - external-dns annotations moved to the Gateway - `ingress_puppetboard.yaml` is unchanged in this PR (separate PR) ## Test plan - [ ] ArgoCD syncs the puppet app cleanly - [ ] cert-manager issues the `puppetdb-tls` certificate - [ ] external-dns creates the DNS record - [ ] `https://puppetdb.k8s.syd1.au.unkin.net` is reachable Reviewed-on: #131	2026-05-23 16:05:26 +10:00
unkinben	2c6d88aa6b	feat(puppet): migrate puppetboard Ingress to Gateway API (#130 ) ## Summary - Replace `Ingress` (nginx) with `Gateway` + `HTTPRoute` using `traefik-internal` GatewayClass - TLS terminated at the Gateway listener; cert-manager provisions the certificate via `vault-issuer` - external-dns annotations moved to the Gateway - `ingress_puppetdb.yaml` is unchanged in this PR (separate PR) ## Test plan - [ ] ArgoCD syncs the puppet app cleanly - [ ] cert-manager issues the `puppetboard-tls` certificate - [ ] external-dns creates the DNS record - [ ] `https://puppetboard.k8s.syd1.au.unkin.net` is reachable Reviewed-on: #130	2026-05-23 01:31:28 +10:00
unkinben	58368948d9	feat(paperclip): migrate Ingress to Gateway API (#133 ) ## Summary - Replace `Ingress` (nginx) with `Gateway` + `HTTPRoute` using `traefik-internal` GatewayClass - TLS terminated at the Gateway listener; cert-manager provisions the certificate via `vault-issuer` - external-dns annotations moved to the Gateway ## Test plan - [ ] ArgoCD syncs the paperclip app cleanly - [ ] cert-manager issues the `paperclip-tls` certificate - [ ] external-dns creates the DNS record - [ ] `https://paperclip.k8s.syd1.au.unkin.net` is reachable Reviewed-on: #133	2026-05-23 01:31:03 +10:00
unkinben	4f5c3f7ea0	feat(litellm): migrate Ingress to Gateway API (#134 ) ## Summary - Replace `Ingress` (nginx) with `Gateway` + `HTTPRoute` using `traefik-internal` GatewayClass - TLS terminated at the Gateway listener; cert-manager provisions the certificate via `vault-issuer` - external-dns annotations moved to the Gateway ## Test plan - [ ] ArgoCD syncs the litellm app cleanly - [ ] cert-manager issues the `litellm-tls` certificate - [ ] external-dns creates the DNS record - [ ] `https://litellm.k8s.syd1.au.unkin.net` is reachable Reviewed-on: #134	2026-05-23 01:29:54 +10:00
unkinben	fd87cb96b5	feat(externaldns): upgrade to 1.21.1, fix sources for installed CRDs (#143 ) ## Changes - Upgrade external-dns chart from 1.19.0 → 1.21.1 (app v0.19.0 → v0.21.0) - Remove `gateway-tcproute` source — `TCPRoute` CRD is not installed, causing crash-loop - Add `gateway-tlsroute` source — `TLSRoute` CRD is installed (Gateway API 1.5.1) ## Why The pod was crash-looping every minute with `failed to list *v1alpha2.TCPRoute: the server could not find the requested resource` because the TCPRoute CRD doesn't exist in this cluster. TLSRoute was previously removed but its CRD does exist. Reviewed-on: #143	2026-05-23 01:28:20 +10:00
unkinben	d619f9195e	benvin/externaldns_compatability (#142 ) Reviewed-on: #142	2026-05-23 01:17:20 +10:00
unkinben	1944dbbfcd	temp: enable debug logging on externaldns to diagnose TLSRoute sync timeout (#140 ) Temporary: enable --log-level=debug to understand why the TLSRoute informer never reports HasSynced within the 1m interval. To be closed/reverted after root cause is found. Reviewed-on: #140	2026-05-23 01:07:45 +10:00