feat(open-webui): HA deployment with CNPG, PDB, and session persistence

- Switch from SQLite/PVC to CNPG PostgreSQL (3 instances, low-resource)
  with a transaction-mode PgBouncer pooler (2 instances)
- Raise open-webui replicas to 3 with priorityClassName: power
- Add PodDisruptionBudget (minAvailable: 1)
- Add Gateway API sessionPersistence (cookie) on the HTTPS HTTPRoute
  so WebSocket connections stick to the same backend pod
- Add postgres-credentials VaultStaticSecret; DATABASE_URL must be
  added to kv/kubernetes/namespace/open-webui/default/open-webui-credentials

.gitignore

@@ -1,3 +1,2 @@
 manifests/
 apps/**/charts/
-ci/crd-schemas/

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
       - id: check-executables-have-shebangs
       - id: check-json
       - id: check-added-large-files
-        args: ['--maxkb=2048']
+        args: ['--maxkb=500']
       - id: check-merge-conflict
       - id: check-shebang-scripts-are-executable
       - id: check-symlinks

.woodpecker/kubeconform.yaml

@@ -2,21 +2,6 @@ when:
   - event: pull_request
 
 steps:
-  - name: generate-schemas
-    image: git.unkin.net/unkin/almalinux9-kubetest:20260319
-    commands:
-      - make generate-schemas
-    backend_options:
-      kubernetes:
-        serviceAccountName: default
-        resources:
-          requests:
-            memory: 512Mi
-            cpu: 1
-          limits:
-            memory: 2Gi
-            cpu: 2
-
   - name: kubeconform
     image: git.unkin.net/unkin/almalinux9-kubetest:20260319
     commands:

Makefile

@@ -11,11 +11,6 @@ kubeconform:
 	@ci/validate-apps.sh && \
 	ci/validate-clusters.sh
 
-# Generate JSON schemas from CRD definitions for kubeconform
-# Run when CRD versions change, then commit ci/crd-schemas/
-generate-schemas:
-	@ci/generate-crd-schemas.sh
-
 # Clean all generated manifests
 clean:
 	@rm -rf manifests/

apps/base/artifactapi/resources/conf.d/remote-docker.yaml

@@ -6,7 +6,6 @@ remotes:
     immutable_patterns:
       - "^cloudnative-pg/cloudnative-pg"
       - "^emberstack/helm-charts"
-      - "^open-webui/open-webui"
       - "^openvoxproject/"
       - "^stakater/reloader"
       - "^stalwartlabs/stalwart"

apps/base/open-webui/cnpg_cluster.yaml

@@ -0,0 +1,91 @@
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: open-webui-postgres
+  namespace: open-webui
+spec:
+  affinity:
+    podAntiAffinityType: preferred
+  bootstrap:
+    initdb:
+      database: open-webui
+      encoding: UTF8
+      localeCType: C
+      localeCollate: C
+      owner: open-webui
+      secret:
+        name: postgres-credentials
+  enablePDB: true
+  enableSuperuserAccess: false
+  failoverDelay: 0
+  imageName: ghcr.io/cloudnative-pg/postgresql:17-minimal-trixie
+  instances: 3
+  logLevel: info
+  maxSyncReplicas: 0
+  minSyncReplicas: 0
+  monitoring:
+    customQueriesConfigMap:
+      - key: queries
+        name: cnpg-default-monitoring
+    disableDefaultQueries: false
+    enablePodMonitor: false
+  postgresql:
+    parameters:
+      archive_mode: "on"
+      archive_timeout: 5min
+      dynamic_shared_memory_type: posix
+      effective_cache_size: 128MB
+      full_page_writes: "on"
+      log_destination: csvlog
+      log_directory: /controller/log
+      log_filename: postgres
+      log_rotation_age: "0"
+      log_rotation_size: "0"
+      log_truncate_on_rotation: "false"
+      logging_collector: "on"
+      max_connections: "100"
+      max_parallel_workers: "4"
+      max_replication_slots: "16"
+      max_worker_processes: "4"
+      shared_buffers: 64MB
+      shared_memory_type: mmap
+      ssl_max_protocol_version: TLSv1.3
+      ssl_min_protocol_version: TLSv1.3
+      wal_keep_size: 128MB
+      wal_level: logical
+      wal_log_hints: "on"
+      wal_receiver_timeout: 5s
+      wal_sender_timeout: 5s
+    syncReplicaElectionConstraint:
+      enabled: false
+  primaryUpdateMethod: restart
+  primaryUpdateStrategy: unsupervised
+  probes:
+    liveness:
+      isolationCheck:
+        connectionTimeout: 1000
+        enabled: true
+        requestTimeout: 1000
+  replicationSlots:
+    highAvailability:
+      enabled: true
+      slotPrefix: _cnpg_
+    synchronizeReplicas:
+      enabled: true
+    updateInterval: 30
+  resources:
+    limits:
+      cpu: 500m
+      memory: 512Mi
+    requests:
+      cpu: 100m
+      memory: 256Mi
+  smartShutdownTimeout: 180
+  startDelay: 3600
+  stopDelay: 1800
+  storage:
+    resizeInUseVolumes: true
+    size: 5Gi
+    storageClass: cephrbd-fast-delete
+  switchoverDelay: 3600

apps/base/open-webui/cnpg_pooler.yaml

@@ -0,0 +1,33 @@
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Pooler
+metadata:
+  name: open-webui-postgres-pooler
+  namespace: open-webui
+spec:
+  cluster:
+    name: open-webui-postgres
+  instances: 2
+  pgbouncer:
+    parameters:
+      default_pool_size: "50"
+      max_client_conn: "200"
+    paused: false
+    poolMode: transaction
+  template:
+    metadata:
+      labels:
+        app: pooler
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - pooler
+              topologyKey: kubernetes.io/hostname
+      containers: []
+  type: rw

apps/base/open-webui/deployment.yaml

@@ -0,0 +1,61 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: open-webui
+  namespace: open-webui
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: open-webui
+  template:
+    metadata:
+      annotations:
+        reloader.stakater.com/auto: "true"
+      labels:
+        app: open-webui
+    spec:
+      priorityClassName: power
+      containers:
+        - name: open-webui
+          image: ghcr.io/open-webui/open-webui:main
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+          env:
+            - name: OPENAI_API_BASE_URL
+              value: https://litellm.k8s.syd1.au.unkin.net
+            - name: WEBUI_URL
+              value: https://chat.k8s.syd1.au.unkin.net
+          envFrom:
+            - secretRef:
+                name: open-webui-credentials
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            successThreshold: 1
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            failureThreshold: 3
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 5
+          resources:
+            limits:
+              cpu: "2"
+              memory: 4Gi
+            requests:
+              cpu: 250m
+              memory: 512Mi
+      restartPolicy: Always

apps/base/open-webui/gateway.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: chat.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    external-dns.alpha.kubernetes.io/hostname: chat.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+  name: open-webui
+  namespace: open-webui
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: chat.k8s.syd1.au.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: chat.k8s.syd1.au.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: open-webui-tls
+        mode: Terminate

apps/base/open-webui/httproute.yaml

@@ -0,0 +1,53 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: open-webui-http-redirect
+  namespace: open-webui
+spec:
+  hostnames:
+    - chat.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: open-webui
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: open-webui
+  namespace: open-webui
+spec:
+  hostnames:
+    - chat.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: open-webui
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: open-webui
+          port: 8080
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+      sessionPersistence:
+        type: Cookie
+        cookieName: open-webui-backend
+        absoluteTimeout: 24h0m0s

apps/base/open-webui/kustomization.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - cnpg_cluster.yaml
+  - cnpg_pooler.yaml
+  - deployment.yaml
+  - pdb.yaml
+  - service.yaml
+  - gateway.yaml
+  - httproute.yaml
+  - vaultauth.yaml
+  - vaultstaticsecret.yaml

apps/base/open-webui/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: open-webui

apps/base/open-webui/pdb.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: open-webui
+  namespace: open-webui
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: open-webui

apps/base/open-webui/service.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: open-webui
+  namespace: open-webui
+spec:
+  internalTrafficPolicy: Cluster
+  ports:
+    - name: http
+      port: 8080
+      protocol: TCP
+      targetPort: http
+  selector:
+    app: open-webui
+  sessionAffinity: None
+  type: ClusterIP

apps/base/open-webui/vaultauth.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: default
+  namespace: open-webui
+spec:
+  allowedNamespaces:
+    - open-webui
+  kubernetes:
+    audiences:
+      - vault
+    role: default
+    serviceAccount: default
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default

apps/base/open-webui/vaultstaticsecret.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: postgres-credentials
+  namespace: open-webui
+spec:
+  destination:
+    create: true
+    name: postgres-credentials
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/open-webui/default/postgres-credentials
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: open-webui-credentials
+  namespace: open-webui
+spec:
+  destination:
+    create: true
+    name: open-webui-credentials
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/open-webui/default/open-webui-credentials
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/base/priority-classes/priorityclasses.yaml

@@ -1,36 +0,0 @@
----
-apiVersion: scheduling.k8s.io/v1
-kind: PriorityClass
-metadata:
-  name: low
-value: 100
-preemptionPolicy: Never
-globalDefault: false
-description: "Low-importance workloads. Can be evicted under pressure but will not preempt other pods."
----
-apiVersion: scheduling.k8s.io/v1
-kind: PriorityClass
-metadata:
-  name: power
-value: 100
-preemptionPolicy: Never
-globalDefault: false
-description: "Compute-heavy workloads with low scheduling importance. Evictable under pressure."
----
-apiVersion: scheduling.k8s.io/v1
-kind: PriorityClass
-metadata:
-  name: medium
-value: 10000
-preemptionPolicy: PreemptLowerPriority
-globalDefault: false
-description: "Standard workloads. Will preempt low-priority pods if the cluster is under pressure."
----
-apiVersion: scheduling.k8s.io/v1
-kind: PriorityClass
-metadata:
-  name: high
-value: 100000
-preemptionPolicy: PreemptLowerPriority
-globalDefault: false
-description: "High-importance services. Will preempt medium- and low-priority pods if necessary."

apps/base/vault/kustomization.yaml

@@ -6,4 +6,3 @@ resources:
   - namespace.yaml
   - gateway.yaml
   - httproute.yaml
-  - role_k8s-service-registration.yaml

apps/base/vault/role_k8s-service-registration.yaml

@@ -1,24 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: vault-k8s-service-registration
-  namespace: vault
-rules:
-  - apiGroups: [""]
-    resources: ["pods"]
-    verbs: ["get", "update", "patch"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: vault-k8s-service-registration
-  namespace: vault
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: vault-k8s-service-registration
-subjects:
-  - kind: ServiceAccount
-    name: vault
-    namespace: vault

apps/overlays/au-syd1/open-webui/kustomization.yaml

@@ -3,4 +3,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-  - priorityclasses.yaml
+  - ../../../base/open-webui

apps/overlays/au-syd1/priority-classes/kustomization.yaml

@@ -1,6 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - ../../../base/priority-classes

apps/overlays/au-syd1/vault/values.yaml

@@ -40,7 +40,9 @@ server:
           }
         }
 
-        service_registration "kubernetes" {}
+        service_registration "consul" {
+          address = "consul-server.consul.svc.cluster.local:8500"
+        }
 
   dataStorage:
     enabled: true
@@ -48,14 +50,6 @@ server:
     storageClass: cephrbd-fast-delete
     accessMode: ReadWriteOnce
 
-  extraEnv:
-    - name: VAULT_K8S_NAMESPACE
-      value: vault
-    - name: VAULT_K8S_POD_NAME
-      valueFrom:
-        fieldRef:
-          fieldPath: metadata.name
-
   statefulSet:
     securityContext:
       container:

apps/overlays/au-syd1/woodpecker/values.yaml

@@ -2,7 +2,6 @@ agent:
   replicaCount: 3
   env:
     WOODPECKER_MAX_WORKFLOWS: "8"
-    WOODPECKER_BACKEND_K8S_PRIORITY_CLASS: power
     WOODPECKER_BACKEND_K8S_STORAGE_CLASS: cephrbd-fast-delete
     WOODPECKER_BACKEND_K8S_VOLUME_SIZE: 10G
     WOODPECKER_BACKEND_K8S_STORAGE_RWX: false

argocd/applicationsets/aitooling.yaml

@@ -11,6 +11,7 @@ spec:
       revision: HEAD
       directories:
       - path: apps/overlays/*/litellm
+      - path: apps/overlays/*/open-webui
       - path: apps/overlays/*/paperclip
   template:
     metadata:

argocd/applicationsets/platform.yaml

@@ -22,7 +22,6 @@ spec:
       - path: apps/overlays/*/jfrog
       - path: apps/overlays/*/kanidm
       - path: apps/overlays/*/node-feature-discovery
-      - path: apps/overlays/*/priority-classes
       - path: apps/overlays/*/puppet
       - path: apps/overlays/*/purelb
       - path: apps/overlays/*/reflector-system

argocd/projects/aitooling.yaml

@@ -11,6 +11,8 @@ spec:
   destinations:
     - namespace: 'litellm'
       server: https://kubernetes.default.svc
+    - namespace: 'open-webui'
+      server: https://kubernetes.default.svc
     - namespace: 'paperclip'
       server: https://kubernetes.default.svc
   clusterResourceWhitelist:

argocd/projects/platform.yaml

@@ -31,8 +31,6 @@ spec:
       server: https://kubernetes.default.svc
     - namespace: 'node-feature-discovery'
       server: https://kubernetes.default.svc
-    - namespace: 'priority-classes'
-      server: https://kubernetes.default.svc
     - namespace: 'purelb'
       server: https://kubernetes.default.svc
     - namespace: 'puppet'

ci/generate-crd-schemas.py

@@ -1,49 +0,0 @@
-#!/usr/bin/env -S uv run
-# /// script
-# requires-python = ">=3.11"
-# dependencies = ["pyyaml"]
-# ///
-"""
-Extract OpenAPI v3 schemas from CRD YAML on stdin and write JSON schema files
-to the output directory for use with kubeconform.
-
-Usage: kustomize build ... | ci/generate-crd-schemas.py <output-dir>
-"""
-import sys
-import json
-import os
-
-import yaml
-
-
-def main() -> int:
-    output_dir = sys.argv[1] if len(sys.argv) > 1 else "ci/crd-schemas"
-    count = 0
-
-    for doc in yaml.safe_load_all(sys.stdin):
-        if not doc or doc.get("kind") != "CustomResourceDefinition":
-            continue
-
-        group = doc["spec"]["group"]
-        kind = doc["spec"]["names"]["kind"]
-        group_dir = os.path.join(output_dir, group)
-        os.makedirs(group_dir, exist_ok=True)
-
-        for ver in doc["spec"].get("versions", []):
-            if not ver.get("served", True):
-                continue
-            schema = ver.get("schema", {}).get("openAPIV3Schema")
-            if not schema:
-                continue
-            fname = os.path.join(group_dir, f"{kind.lower()}_{ver['name']}.json")
-            with open(fname, "w") as f:
-                json.dump({"$schema": "http://json-schema.org/schema#", **schema}, f, indent=2)
-                f.write("\n")
-            print(f"  wrote {fname}", file=sys.stderr)
-            count += 1
-
-    return count
-
-
-if __name__ == "__main__":
-    print(main())

ci/generate-crd-schemas.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-# Extract OpenAPI v3 schemas from CRD definitions in all kustomize overlays
-# and write JSON schema files to ci/crd-schemas/ for kubeconform validation.
-#
-# Run this script whenever CRD versions change, then commit the output.
-# Usage: ci/generate-crd-schemas.sh [output-dir]
-
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-OUTPUT_DIR="${1:-${SCRIPT_DIR}/crd-schemas}"
-mkdir -p "$OUTPUT_DIR"
-
-total=0
-
-while IFS= read -r -d "" k; do
-  dir="$(dirname "$k")"
-  n=$(kustomize build --enable-helm "$dir" 2>/dev/null \
-    | "$SCRIPT_DIR/generate-crd-schemas.py" "$OUTPUT_DIR") || continue
-  total=$((total + n))
-done < <(find apps/overlays clusters -name kustomization.yaml -print0 | sort -z)
-
-echo "Generated ${total} schema(s) in ${OUTPUT_DIR}" >&2

ci/validate-apps.sh

@@ -6,7 +6,6 @@ KUBE_VERSION="1.33.7"
 schema_args=(
   -schema-location "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/{{.NormalizedKubernetesVersion}}-standalone{{.StrictSuffix}}/{{.ResourceKind}}{{.KindSuffix}}.json"
   -schema-location "https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json"
-  -schema-location "file://${PWD}/ci/crd-schemas/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json"
 )
 
 while IFS= read -r -d "" k; do
@@ -19,6 +18,6 @@ while IFS= read -r -d "" k; do
         -summary \
         -output pretty \
         -verbose \
-        -skip GpuDevicePlugin,LBNodeAgent,ServiceGroup \
+        -skip CustomResourceDefinition,GpuDevicePlugin,LBNodeAgent,ServiceGroup \
         "${schema_args[@]}"
 done < <(find apps/overlays -name kustomization.yaml -print0)

ci/validate-clusters.sh

@@ -6,7 +6,6 @@ KUBE_VERSION="1.33.7"
 schema_args=(
   -schema-location "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/{{.NormalizedKubernetesVersion}}-standalone{{.StrictSuffix}}/{{.ResourceKind}}{{.KindSuffix}}.json"
   -schema-location "https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json"
-  -schema-location "file://${PWD}/ci/crd-schemas/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json"
 )
 
 while IFS= read -r -d "" k; do