From 1bcb88d3dd6d8ccf90f604b35bfe5b6eb7e7c903 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Tue, 26 May 2026 00:11:25 +1000 Subject: [PATCH 1/4] feat(open-webui): deploy Open WebUI with litellm backend Deploys Open WebUI (chat.k8s.syd1.au.unkin.net) into the open-webui namespace via the aitooling ArgoCD project. Uses SQLite with a 10Gi cephrbd PVC for persistence, routes model requests to the existing litellm deployment, and exposes the UI through the traefik-external gateway. Credentials (OPENAI_API_KEY, WEBUI_SECRET_KEY) are injected via VaultStaticSecret from kv/kubernetes/namespace/open-webui/default. Closes #155 --- apps/base/open-webui/deployment.yaml | 69 +++++++++++++++++++ apps/base/open-webui/gateway.yaml | 37 ++++++++++ apps/base/open-webui/httproute.yaml | 49 +++++++++++++ apps/base/open-webui/kustomization.yaml | 13 ++++ apps/base/open-webui/namespace.yaml | 5 ++ apps/base/open-webui/pvc.yaml | 13 ++++ apps/base/open-webui/service.yaml | 17 +++++ apps/base/open-webui/vaultauth.yaml | 18 +++++ apps/base/open-webui/vaultstaticsecret.yaml | 17 +++++ .../au-syd1/open-webui/kustomization.yaml | 6 ++ argocd/applicationsets/aitooling.yaml | 1 + argocd/projects/aitooling.yaml | 2 + 12 files changed, 247 insertions(+) create mode 100644 apps/base/open-webui/deployment.yaml create mode 100644 apps/base/open-webui/gateway.yaml create mode 100644 apps/base/open-webui/httproute.yaml create mode 100644 apps/base/open-webui/kustomization.yaml create mode 100644 apps/base/open-webui/namespace.yaml create mode 100644 apps/base/open-webui/pvc.yaml create mode 100644 apps/base/open-webui/service.yaml create mode 100644 apps/base/open-webui/vaultauth.yaml create mode 100644 apps/base/open-webui/vaultstaticsecret.yaml create mode 100644 apps/overlays/au-syd1/open-webui/kustomization.yaml diff --git a/apps/base/open-webui/deployment.yaml b/apps/base/open-webui/deployment.yaml new file mode 100644 index 0000000..4c52373 --- /dev/null +++ b/apps/base/open-webui/deployment.yaml @@ -0,0 +1,69 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: open-webui + namespace: open-webui +spec: + replicas: 1 + selector: + matchLabels: + app: open-webui + strategy: + type: Recreate + template: + metadata: + annotations: + reloader.stakater.com/auto: "true" + labels: + app: open-webui + spec: + containers: + - name: open-webui + image: ghcr.io/open-webui/open-webui:main + imagePullPolicy: Always + ports: + - containerPort: 8080 + name: http + protocol: TCP + env: + - name: OPENAI_API_BASE_URL + value: http://litellm-pooler.litellm.svc.cluster.local:4000 + - name: WEBUI_URL + value: https://chat.k8s.syd1.au.unkin.net + envFrom: + - secretRef: + name: open-webui-credentials + livenessProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + resources: + limits: + cpu: "2" + memory: 4Gi + requests: + cpu: 250m + memory: 512Mi + volumeMounts: + - mountPath: /app/backend/data + name: data + restartPolicy: Always + volumes: + - name: data + persistentVolumeClaim: + claimName: open-webui-data diff --git a/apps/base/open-webui/gateway.yaml b/apps/base/open-webui/gateway.yaml new file mode 100644 index 0000000..99cb2c9 --- /dev/null +++ b/apps/base/open-webui/gateway.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + labels: + traefik.io/instance: external + annotations: + cert-manager.io/cluster-issuer: vault-issuer + cert-manager.io/common-name: chat.k8s.syd1.au.unkin.net + cert-manager.io/private-key-size: "4096" + external-dns.alpha.kubernetes.io/hostname: chat.k8s.syd1.au.unkin.net + external-dns.alpha.kubernetes.io/target: 198.18.200.0 + name: open-webui + namespace: open-webui +spec: + gatewayClassName: traefik-external + listeners: + - allowedRoutes: + namespaces: + from: Same + hostname: chat.k8s.syd1.au.unkin.net + name: http + port: 80 + protocol: HTTP + - allowedRoutes: + namespaces: + from: Same + hostname: chat.k8s.syd1.au.unkin.net + name: https + port: 443 + protocol: HTTPS + tls: + certificateRefs: + - group: "" + kind: Secret + name: open-webui-tls + mode: Terminate diff --git a/apps/base/open-webui/httproute.yaml b/apps/base/open-webui/httproute.yaml new file mode 100644 index 0000000..17f8019 --- /dev/null +++ b/apps/base/open-webui/httproute.yaml @@ -0,0 +1,49 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: open-webui-http-redirect + namespace: open-webui +spec: + hostnames: + - chat.k8s.syd1.au.unkin.net + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: open-webui + sectionName: http + rules: + - filters: + - type: RequestRedirect + requestRedirect: + scheme: https + statusCode: 301 + matches: + - path: + type: PathPrefix + value: / +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: open-webui + namespace: open-webui +spec: + hostnames: + - chat.k8s.syd1.au.unkin.net + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: open-webui + sectionName: https + rules: + - backendRefs: + - group: "" + kind: Service + name: open-webui + port: 8080 + weight: 1 + matches: + - path: + type: PathPrefix + value: / diff --git a/apps/base/open-webui/kustomization.yaml b/apps/base/open-webui/kustomization.yaml new file mode 100644 index 0000000..b52c2f7 --- /dev/null +++ b/apps/base/open-webui/kustomization.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - pvc.yaml + - deployment.yaml + - service.yaml + - gateway.yaml + - httproute.yaml + - vaultauth.yaml + - vaultstaticsecret.yaml diff --git a/apps/base/open-webui/namespace.yaml b/apps/base/open-webui/namespace.yaml new file mode 100644 index 0000000..b399aef --- /dev/null +++ b/apps/base/open-webui/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: open-webui diff --git a/apps/base/open-webui/pvc.yaml b/apps/base/open-webui/pvc.yaml new file mode 100644 index 0000000..9069130 --- /dev/null +++ b/apps/base/open-webui/pvc.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: open-webui-data + namespace: open-webui +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: cephrbd-fast-delete diff --git a/apps/base/open-webui/service.yaml b/apps/base/open-webui/service.yaml new file mode 100644 index 0000000..773c7f5 --- /dev/null +++ b/apps/base/open-webui/service.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: open-webui + namespace: open-webui +spec: + internalTrafficPolicy: Cluster + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: http + selector: + app: open-webui + sessionAffinity: None + type: ClusterIP diff --git a/apps/base/open-webui/vaultauth.yaml b/apps/base/open-webui/vaultauth.yaml new file mode 100644 index 0000000..4fa4282 --- /dev/null +++ b/apps/base/open-webui/vaultauth.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: default + namespace: open-webui +spec: + allowedNamespaces: + - open-webui + kubernetes: + audiences: + - vault + role: default + serviceAccount: default + tokenExpirationSeconds: 600 + method: kubernetes + mount: k8s/au/syd1 + vaultConnectionRef: vso-system/default diff --git a/apps/base/open-webui/vaultstaticsecret.yaml b/apps/base/open-webui/vaultstaticsecret.yaml new file mode 100644 index 0000000..ca10d84 --- /dev/null +++ b/apps/base/open-webui/vaultstaticsecret.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: open-webui-credentials + namespace: open-webui +spec: + destination: + create: true + name: open-webui-credentials + overwrite: true + hmacSecretData: true + mount: kv + path: kubernetes/namespace/open-webui/default/open-webui-credentials + refreshAfter: 5m + type: kv-v2 + vaultAuthRef: default diff --git a/apps/overlays/au-syd1/open-webui/kustomization.yaml b/apps/overlays/au-syd1/open-webui/kustomization.yaml new file mode 100644 index 0000000..6716c15 --- /dev/null +++ b/apps/overlays/au-syd1/open-webui/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../base/open-webui diff --git a/argocd/applicationsets/aitooling.yaml b/argocd/applicationsets/aitooling.yaml index 7d0e81b..8869416 100644 --- a/argocd/applicationsets/aitooling.yaml +++ b/argocd/applicationsets/aitooling.yaml @@ -11,6 +11,7 @@ spec: revision: HEAD directories: - path: apps/overlays/*/litellm + - path: apps/overlays/*/open-webui - path: apps/overlays/*/paperclip template: metadata: diff --git a/argocd/projects/aitooling.yaml b/argocd/projects/aitooling.yaml index a60e350..15ad092 100644 --- a/argocd/projects/aitooling.yaml +++ b/argocd/projects/aitooling.yaml @@ -11,6 +11,8 @@ spec: destinations: - namespace: 'litellm' server: https://kubernetes.default.svc + - namespace: 'open-webui' + server: https://kubernetes.default.svc - namespace: 'paperclip' server: https://kubernetes.default.svc clusterResourceWhitelist: -- 2.47.3 From 16dabbbf8d1c9fffb949796be2aa8128267b1d72 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Tue, 26 May 2026 23:25:26 +1000 Subject: [PATCH 2/4] fix(open-webui): use litellm external hostname as OPENAI_API_BASE_URL --- apps/base/open-webui/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/base/open-webui/deployment.yaml b/apps/base/open-webui/deployment.yaml index 4c52373..b83f64d 100644 --- a/apps/base/open-webui/deployment.yaml +++ b/apps/base/open-webui/deployment.yaml @@ -28,7 +28,7 @@ spec: protocol: TCP env: - name: OPENAI_API_BASE_URL - value: http://litellm-pooler.litellm.svc.cluster.local:4000 + value: https://litellm.k8s.syd1.au.unkin.net - name: WEBUI_URL value: https://chat.k8s.syd1.au.unkin.net envFrom: -- 2.47.3 From 85a8cfe47d37e18329286125da8b56cdf10bcf8d Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Tue, 26 May 2026 23:26:01 +1000 Subject: [PATCH 3/4] fix(open-webui): use traefik-internal gateway for chat hostname --- apps/base/open-webui/gateway.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/base/open-webui/gateway.yaml b/apps/base/open-webui/gateway.yaml index 99cb2c9..eeb7b00 100644 --- a/apps/base/open-webui/gateway.yaml +++ b/apps/base/open-webui/gateway.yaml @@ -3,17 +3,17 @@ apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: labels: - traefik.io/instance: external + traefik.io/instance: internal annotations: cert-manager.io/cluster-issuer: vault-issuer cert-manager.io/common-name: chat.k8s.syd1.au.unkin.net cert-manager.io/private-key-size: "4096" external-dns.alpha.kubernetes.io/hostname: chat.k8s.syd1.au.unkin.net - external-dns.alpha.kubernetes.io/target: 198.18.200.0 + external-dns.alpha.kubernetes.io/target: 198.18.200.4 name: open-webui namespace: open-webui spec: - gatewayClassName: traefik-external + gatewayClassName: traefik-internal listeners: - allowedRoutes: namespaces: -- 2.47.3 From 3d85105afd770b84ecba3b88f923976e9e958e50 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Tue, 26 May 2026 23:37:10 +1000 Subject: [PATCH 4/4] feat(open-webui): HA deployment with CNPG, PDB, and session persistence - Switch from SQLite/PVC to CNPG PostgreSQL (3 instances, low-resource) with a transaction-mode PgBouncer pooler (2 instances) - Raise open-webui replicas to 3 with priorityClassName: power - Add PodDisruptionBudget (minAvailable: 1) - Add Gateway API sessionPersistence (cookie) on the HTTPS HTTPRoute so WebSocket connections stick to the same backend pod - Add postgres-credentials VaultStaticSecret; DATABASE_URL must be added to kv/kubernetes/namespace/open-webui/default/open-webui-credentials --- apps/base/open-webui/cnpg_cluster.yaml | 91 +++++++++++++++++++++ apps/base/open-webui/cnpg_pooler.yaml | 33 ++++++++ apps/base/open-webui/deployment.yaml | 12 +-- apps/base/open-webui/httproute.yaml | 4 + apps/base/open-webui/kustomization.yaml | 4 +- apps/base/open-webui/pdb.yaml | 11 +++ apps/base/open-webui/pvc.yaml | 13 --- apps/base/open-webui/vaultstaticsecret.yaml | 17 ++++ 8 files changed, 161 insertions(+), 24 deletions(-) create mode 100644 apps/base/open-webui/cnpg_cluster.yaml create mode 100644 apps/base/open-webui/cnpg_pooler.yaml create mode 100644 apps/base/open-webui/pdb.yaml delete mode 100644 apps/base/open-webui/pvc.yaml diff --git a/apps/base/open-webui/cnpg_cluster.yaml b/apps/base/open-webui/cnpg_cluster.yaml new file mode 100644 index 0000000..9e9dad3 --- /dev/null +++ b/apps/base/open-webui/cnpg_cluster.yaml @@ -0,0 +1,91 @@ +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: open-webui-postgres + namespace: open-webui +spec: + affinity: + podAntiAffinityType: preferred + bootstrap: + initdb: + database: open-webui + encoding: UTF8 + localeCType: C + localeCollate: C + owner: open-webui + secret: + name: postgres-credentials + enablePDB: true + enableSuperuserAccess: false + failoverDelay: 0 + imageName: ghcr.io/cloudnative-pg/postgresql:17-minimal-trixie + instances: 3 + logLevel: info + maxSyncReplicas: 0 + minSyncReplicas: 0 + monitoring: + customQueriesConfigMap: + - key: queries + name: cnpg-default-monitoring + disableDefaultQueries: false + enablePodMonitor: false + postgresql: + parameters: + archive_mode: "on" + archive_timeout: 5min + dynamic_shared_memory_type: posix + effective_cache_size: 128MB + full_page_writes: "on" + log_destination: csvlog + log_directory: /controller/log + log_filename: postgres + log_rotation_age: "0" + log_rotation_size: "0" + log_truncate_on_rotation: "false" + logging_collector: "on" + max_connections: "100" + max_parallel_workers: "4" + max_replication_slots: "16" + max_worker_processes: "4" + shared_buffers: 64MB + shared_memory_type: mmap + ssl_max_protocol_version: TLSv1.3 + ssl_min_protocol_version: TLSv1.3 + wal_keep_size: 128MB + wal_level: logical + wal_log_hints: "on" + wal_receiver_timeout: 5s + wal_sender_timeout: 5s + syncReplicaElectionConstraint: + enabled: false + primaryUpdateMethod: restart + primaryUpdateStrategy: unsupervised + probes: + liveness: + isolationCheck: + connectionTimeout: 1000 + enabled: true + requestTimeout: 1000 + replicationSlots: + highAvailability: + enabled: true + slotPrefix: _cnpg_ + synchronizeReplicas: + enabled: true + updateInterval: 30 + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 256Mi + smartShutdownTimeout: 180 + startDelay: 3600 + stopDelay: 1800 + storage: + resizeInUseVolumes: true + size: 5Gi + storageClass: cephrbd-fast-delete + switchoverDelay: 3600 diff --git a/apps/base/open-webui/cnpg_pooler.yaml b/apps/base/open-webui/cnpg_pooler.yaml new file mode 100644 index 0000000..993c1c9 --- /dev/null +++ b/apps/base/open-webui/cnpg_pooler.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Pooler +metadata: + name: open-webui-postgres-pooler + namespace: open-webui +spec: + cluster: + name: open-webui-postgres + instances: 2 + pgbouncer: + parameters: + default_pool_size: "50" + max_client_conn: "200" + paused: false + poolMode: transaction + template: + metadata: + labels: + app: pooler + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - pooler + topologyKey: kubernetes.io/hostname + containers: [] + type: rw diff --git a/apps/base/open-webui/deployment.yaml b/apps/base/open-webui/deployment.yaml index b83f64d..cb00ec5 100644 --- a/apps/base/open-webui/deployment.yaml +++ b/apps/base/open-webui/deployment.yaml @@ -5,12 +5,10 @@ metadata: name: open-webui namespace: open-webui spec: - replicas: 1 + replicas: 3 selector: matchLabels: app: open-webui - strategy: - type: Recreate template: metadata: annotations: @@ -18,6 +16,7 @@ spec: labels: app: open-webui spec: + priorityClassName: power containers: - name: open-webui image: ghcr.io/open-webui/open-webui:main @@ -59,11 +58,4 @@ spec: requests: cpu: 250m memory: 512Mi - volumeMounts: - - mountPath: /app/backend/data - name: data restartPolicy: Always - volumes: - - name: data - persistentVolumeClaim: - claimName: open-webui-data diff --git a/apps/base/open-webui/httproute.yaml b/apps/base/open-webui/httproute.yaml index 17f8019..b82a66b 100644 --- a/apps/base/open-webui/httproute.yaml +++ b/apps/base/open-webui/httproute.yaml @@ -47,3 +47,7 @@ spec: - path: type: PathPrefix value: / + sessionPersistence: + type: Cookie + cookieName: open-webui-backend + absoluteTimeout: 24h0m0s diff --git a/apps/base/open-webui/kustomization.yaml b/apps/base/open-webui/kustomization.yaml index b52c2f7..3af4aab 100644 --- a/apps/base/open-webui/kustomization.yaml +++ b/apps/base/open-webui/kustomization.yaml @@ -4,8 +4,10 @@ kind: Kustomization resources: - namespace.yaml - - pvc.yaml + - cnpg_cluster.yaml + - cnpg_pooler.yaml - deployment.yaml + - pdb.yaml - service.yaml - gateway.yaml - httproute.yaml diff --git a/apps/base/open-webui/pdb.yaml b/apps/base/open-webui/pdb.yaml new file mode 100644 index 0000000..9abebff --- /dev/null +++ b/apps/base/open-webui/pdb.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: open-webui + namespace: open-webui +spec: + minAvailable: 1 + selector: + matchLabels: + app: open-webui diff --git a/apps/base/open-webui/pvc.yaml b/apps/base/open-webui/pvc.yaml deleted file mode 100644 index 9069130..0000000 --- a/apps/base/open-webui/pvc.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: open-webui-data - namespace: open-webui -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: cephrbd-fast-delete diff --git a/apps/base/open-webui/vaultstaticsecret.yaml b/apps/base/open-webui/vaultstaticsecret.yaml index ca10d84..eeebb44 100644 --- a/apps/base/open-webui/vaultstaticsecret.yaml +++ b/apps/base/open-webui/vaultstaticsecret.yaml @@ -1,6 +1,23 @@ --- apiVersion: secrets.hashicorp.com/v1beta1 kind: VaultStaticSecret +metadata: + name: postgres-credentials + namespace: open-webui +spec: + destination: + create: true + name: postgres-credentials + overwrite: true + hmacSecretData: true + mount: kv + path: kubernetes/namespace/open-webui/default/postgres-credentials + refreshAfter: 5m + type: kv-v2 + vaultAuthRef: default +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret metadata: name: open-webui-credentials namespace: open-webui -- 2.47.3