refactor(ci): convert CRD schema generator to uv inline script

Add scripts to extract OpenAPI v3 schemas from CRD definitions in all
kustomize overlays and write JSON schema files to ci/crd-schemas/ for
kubeconform validation. This allows kubeconform to validate CRD instances
(Elasticsearch, Kibana, CNPG Cluster, VictoriaMetrics, etc.) instead of
skipping or erroring on them.

- ci/generate-crd-schemas.py: extracts schemas from CRD YAML on stdin
- ci/generate-crd-schemas.sh: iterates overlays, pipes to Python script
- ci/validate-apps.sh, ci/validate-clusters.sh: add local schema-location fallback
- Makefile: add generate-schemas target
- add generate-schemas step to kubeconform woodpecker pipeline so schemas

.gitignore

@@ -1,2 +1,3 @@
 manifests/
 apps/**/charts/
+ci/crd-schemas/

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
       - id: check-executables-have-shebangs
       - id: check-json
       - id: check-added-large-files
-        args: ['--maxkb=500']
+        args: ['--maxkb=2048']
       - id: check-merge-conflict
       - id: check-shebang-scripts-are-executable
       - id: check-symlinks

.woodpecker/kubeconform.yaml

@@ -2,6 +2,21 @@ when:
   - event: pull_request
 
 steps:
+  - name: generate-schemas
+    image: git.unkin.net/unkin/almalinux9-kubetest:20260319
+    commands:
+      - make generate-schemas
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 512Mi
+            cpu: 1
+          limits:
+            memory: 2Gi
+            cpu: 2
+
   - name: kubeconform
     image: git.unkin.net/unkin/almalinux9-kubetest:20260319
     commands:

Makefile

@@ -11,6 +11,11 @@ kubeconform:
 	@ci/validate-apps.sh && \
 	ci/validate-clusters.sh
 
+# Generate JSON schemas from CRD definitions for kubeconform
+# Run when CRD versions change, then commit ci/crd-schemas/
+generate-schemas:
+	@ci/generate-crd-schemas.sh
+
 # Clean all generated manifests
 clean:
 	@rm -rf manifests/

apps/base/artifactapi/resources/conf.d/remote-docker.yaml

@@ -6,6 +6,7 @@ remotes:
     immutable_patterns:
       - "^cloudnative-pg/cloudnative-pg"
       - "^emberstack/helm-charts"
+      - "^open-webui/open-webui"
       - "^openvoxproject/"
       - "^stakater/reloader"
       - "^stalwartlabs/stalwart"

apps/base/open-webui/cnpg_cluster.yaml

@@ -1,91 +0,0 @@
----
-apiVersion: postgresql.cnpg.io/v1
-kind: Cluster
-metadata:
-  name: open-webui-postgres
-  namespace: open-webui
-spec:
-  affinity:
-    podAntiAffinityType: preferred
-  bootstrap:
-    initdb:
-      database: open-webui
-      encoding: UTF8
-      localeCType: C
-      localeCollate: C
-      owner: open-webui
-      secret:
-        name: postgres-credentials
-  enablePDB: true
-  enableSuperuserAccess: false
-  failoverDelay: 0
-  imageName: ghcr.io/cloudnative-pg/postgresql:17-minimal-trixie
-  instances: 3
-  logLevel: info
-  maxSyncReplicas: 0
-  minSyncReplicas: 0
-  monitoring:
-    customQueriesConfigMap:
-      - key: queries
-        name: cnpg-default-monitoring
-    disableDefaultQueries: false
-    enablePodMonitor: false
-  postgresql:
-    parameters:
-      archive_mode: "on"
-      archive_timeout: 5min
-      dynamic_shared_memory_type: posix
-      effective_cache_size: 128MB
-      full_page_writes: "on"
-      log_destination: csvlog
-      log_directory: /controller/log
-      log_filename: postgres
-      log_rotation_age: "0"
-      log_rotation_size: "0"
-      log_truncate_on_rotation: "false"
-      logging_collector: "on"
-      max_connections: "100"
-      max_parallel_workers: "4"
-      max_replication_slots: "16"
-      max_worker_processes: "4"
-      shared_buffers: 64MB
-      shared_memory_type: mmap
-      ssl_max_protocol_version: TLSv1.3
-      ssl_min_protocol_version: TLSv1.3
-      wal_keep_size: 128MB
-      wal_level: logical
-      wal_log_hints: "on"
-      wal_receiver_timeout: 5s
-      wal_sender_timeout: 5s
-    syncReplicaElectionConstraint:
-      enabled: false
-  primaryUpdateMethod: restart
-  primaryUpdateStrategy: unsupervised
-  probes:
-    liveness:
-      isolationCheck:
-        connectionTimeout: 1000
-        enabled: true
-        requestTimeout: 1000
-  replicationSlots:
-    highAvailability:
-      enabled: true
-      slotPrefix: _cnpg_
-    synchronizeReplicas:
-      enabled: true
-    updateInterval: 30
-  resources:
-    limits:
-      cpu: 500m
-      memory: 512Mi
-    requests:
-      cpu: 100m
-      memory: 256Mi
-  smartShutdownTimeout: 180
-  startDelay: 3600
-  stopDelay: 1800
-  storage:
-    resizeInUseVolumes: true
-    size: 5Gi
-    storageClass: cephrbd-fast-delete
-  switchoverDelay: 3600

apps/base/open-webui/cnpg_pooler.yaml

@@ -1,33 +0,0 @@
----
-apiVersion: postgresql.cnpg.io/v1
-kind: Pooler
-metadata:
-  name: open-webui-postgres-pooler
-  namespace: open-webui
-spec:
-  cluster:
-    name: open-webui-postgres
-  instances: 2
-  pgbouncer:
-    parameters:
-      default_pool_size: "50"
-      max_client_conn: "200"
-    paused: false
-    poolMode: transaction
-  template:
-    metadata:
-      labels:
-        app: pooler
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchExpressions:
-                  - key: app
-                    operator: In
-                    values:
-                      - pooler
-              topologyKey: kubernetes.io/hostname
-      containers: []
-  type: rw

apps/base/open-webui/deployment.yaml

@@ -1,61 +0,0 @@
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: open-webui
-  namespace: open-webui
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: open-webui
-  template:
-    metadata:
-      annotations:
-        reloader.stakater.com/auto: "true"
-      labels:
-        app: open-webui
-    spec:
-      priorityClassName: power
-      containers:
-        - name: open-webui
-          image: ghcr.io/open-webui/open-webui:main
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 8080
-              name: http
-              protocol: TCP
-          env:
-            - name: OPENAI_API_BASE_URL
-              value: https://litellm.k8s.syd1.au.unkin.net
-            - name: WEBUI_URL
-              value: https://chat.k8s.syd1.au.unkin.net
-          envFrom:
-            - secretRef:
-                name: open-webui-credentials
-          livenessProbe:
-            httpGet:
-              path: /health
-              port: 8080
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 30
-            successThreshold: 1
-            timeoutSeconds: 5
-          readinessProbe:
-            httpGet:
-              path: /health
-              port: 8080
-            failureThreshold: 3
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            successThreshold: 1
-            timeoutSeconds: 5
-          resources:
-            limits:
-              cpu: "2"
-              memory: 4Gi
-            requests:
-              cpu: 250m
-              memory: 512Mi
-      restartPolicy: Always

apps/base/open-webui/gateway.yaml

@@ -1,37 +0,0 @@
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: Gateway
-metadata:
-  labels:
-    traefik.io/instance: internal
-  annotations:
-    cert-manager.io/cluster-issuer: vault-issuer
-    cert-manager.io/common-name: chat.k8s.syd1.au.unkin.net
-    cert-manager.io/private-key-size: "4096"
-    external-dns.alpha.kubernetes.io/hostname: chat.k8s.syd1.au.unkin.net
-    external-dns.alpha.kubernetes.io/target: 198.18.200.4
-  name: open-webui
-  namespace: open-webui
-spec:
-  gatewayClassName: traefik-internal
-  listeners:
-    - allowedRoutes:
-        namespaces:
-          from: Same
-      hostname: chat.k8s.syd1.au.unkin.net
-      name: http
-      port: 80
-      protocol: HTTP
-    - allowedRoutes:
-        namespaces:
-          from: Same
-      hostname: chat.k8s.syd1.au.unkin.net
-      name: https
-      port: 443
-      protocol: HTTPS
-      tls:
-        certificateRefs:
-          - group: ""
-            kind: Secret
-            name: open-webui-tls
-        mode: Terminate

apps/base/open-webui/httproute.yaml

@@ -1,53 +0,0 @@
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: open-webui-http-redirect
-  namespace: open-webui
-spec:
-  hostnames:
-    - chat.k8s.syd1.au.unkin.net
-  parentRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: open-webui
-      sectionName: http
-  rules:
-    - filters:
-        - type: RequestRedirect
-          requestRedirect:
-            scheme: https
-            statusCode: 301
-      matches:
-        - path:
-            type: PathPrefix
-            value: /
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: open-webui
-  namespace: open-webui
-spec:
-  hostnames:
-    - chat.k8s.syd1.au.unkin.net
-  parentRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: open-webui
-      sectionName: https
-  rules:
-    - backendRefs:
-        - group: ""
-          kind: Service
-          name: open-webui
-          port: 8080
-          weight: 1
-      matches:
-        - path:
-            type: PathPrefix
-            value: /
-      sessionPersistence:
-        type: Cookie
-        cookieName: open-webui-backend
-        absoluteTimeout: 24h0m0s

apps/base/open-webui/kustomization.yaml

@@ -1,15 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - namespace.yaml
-  - cnpg_cluster.yaml
-  - cnpg_pooler.yaml
-  - deployment.yaml
-  - pdb.yaml
-  - service.yaml
-  - gateway.yaml
-  - httproute.yaml
-  - vaultauth.yaml
-  - vaultstaticsecret.yaml

apps/base/open-webui/namespace.yaml

@@ -1,5 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: open-webui

apps/base/open-webui/pdb.yaml

@@ -1,11 +0,0 @@
----
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
-  name: open-webui
-  namespace: open-webui
-spec:
-  minAvailable: 1
-  selector:
-    matchLabels:
-      app: open-webui

apps/base/open-webui/service.yaml

@@ -1,17 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: open-webui
-  namespace: open-webui
-spec:
-  internalTrafficPolicy: Cluster
-  ports:
-    - name: http
-      port: 8080
-      protocol: TCP
-      targetPort: http
-  selector:
-    app: open-webui
-  sessionAffinity: None
-  type: ClusterIP

apps/base/open-webui/vaultauth.yaml

@@ -1,18 +0,0 @@
----
-apiVersion: secrets.hashicorp.com/v1beta1
-kind: VaultAuth
-metadata:
-  name: default
-  namespace: open-webui
-spec:
-  allowedNamespaces:
-    - open-webui
-  kubernetes:
-    audiences:
-      - vault
-    role: default
-    serviceAccount: default
-    tokenExpirationSeconds: 600
-  method: kubernetes
-  mount: k8s/au/syd1
-  vaultConnectionRef: vso-system/default

apps/base/open-webui/vaultstaticsecret.yaml

@@ -1,34 +0,0 @@
----
-apiVersion: secrets.hashicorp.com/v1beta1
-kind: VaultStaticSecret
-metadata:
-  name: postgres-credentials
-  namespace: open-webui
-spec:
-  destination:
-    create: true
-    name: postgres-credentials
-    overwrite: true
-  hmacSecretData: true
-  mount: kv
-  path: kubernetes/namespace/open-webui/default/postgres-credentials
-  refreshAfter: 5m
-  type: kv-v2
-  vaultAuthRef: default
----
-apiVersion: secrets.hashicorp.com/v1beta1
-kind: VaultStaticSecret
-metadata:
-  name: open-webui-credentials
-  namespace: open-webui
-spec:
-  destination:
-    create: true
-    name: open-webui-credentials
-    overwrite: true
-  hmacSecretData: true
-  mount: kv
-  path: kubernetes/namespace/open-webui/default/open-webui-credentials
-  refreshAfter: 5m
-  type: kv-v2
-  vaultAuthRef: default

apps/base/priority-classes/kustomization.yaml

@@ -3,4 +3,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-  - ../../../base/open-webui
+  - priorityclasses.yaml

apps/base/priority-classes/priorityclasses.yaml

@@ -0,0 +1,36 @@
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: low
+value: 100
+preemptionPolicy: Never
+globalDefault: false
+description: "Low-importance workloads. Can be evicted under pressure but will not preempt other pods."
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: power
+value: 100
+preemptionPolicy: Never
+globalDefault: false
+description: "Compute-heavy workloads with low scheduling importance. Evictable under pressure."
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: medium
+value: 10000
+preemptionPolicy: PreemptLowerPriority
+globalDefault: false
+description: "Standard workloads. Will preempt low-priority pods if the cluster is under pressure."
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: high
+value: 100000
+preemptionPolicy: PreemptLowerPriority
+globalDefault: false
+description: "High-importance services. Will preempt medium- and low-priority pods if necessary."

apps/base/vault/kustomization.yaml

@@ -6,3 +6,4 @@ resources:
   - namespace.yaml
   - gateway.yaml
   - httproute.yaml
+  - role_k8s-service-registration.yaml

apps/base/vault/role_k8s-service-registration.yaml

@@ -0,0 +1,24 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: vault-k8s-service-registration
+  namespace: vault
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: vault-k8s-service-registration
+  namespace: vault
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: vault-k8s-service-registration
+subjects:
+  - kind: ServiceAccount
+    name: vault
+    namespace: vault

apps/overlays/au-syd1/priority-classes/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/priority-classes

apps/overlays/au-syd1/vault/values.yaml

@@ -40,9 +40,7 @@ server:
           }
         }
 
-        service_registration "consul" {
+        service_registration "kubernetes" {}
-          address = "consul-server.consul.svc.cluster.local:8500"
-        }
 
   dataStorage:
     enabled: true
@@ -50,6 +48,14 @@ server:
     storageClass: cephrbd-fast-delete
     accessMode: ReadWriteOnce
 
+  extraEnv:
+    - name: VAULT_K8S_NAMESPACE
+      value: vault
+    - name: VAULT_K8S_POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+
   statefulSet:
     securityContext:
       container:

apps/overlays/au-syd1/woodpecker/values.yaml

@@ -2,6 +2,7 @@ agent:
   replicaCount: 3
   env:
     WOODPECKER_MAX_WORKFLOWS: "8"
+    WOODPECKER_BACKEND_K8S_PRIORITY_CLASS: power
     WOODPECKER_BACKEND_K8S_STORAGE_CLASS: cephrbd-fast-delete
     WOODPECKER_BACKEND_K8S_VOLUME_SIZE: 10G
     WOODPECKER_BACKEND_K8S_STORAGE_RWX: false

argocd/applicationsets/aitooling.yaml

@@ -11,7 +11,6 @@ spec:
       revision: HEAD
       directories:
       - path: apps/overlays/*/litellm
-      - path: apps/overlays/*/open-webui
       - path: apps/overlays/*/paperclip
   template:
     metadata:

argocd/applicationsets/platform.yaml

@@ -22,6 +22,7 @@ spec:
       - path: apps/overlays/*/jfrog
       - path: apps/overlays/*/kanidm
       - path: apps/overlays/*/node-feature-discovery
+      - path: apps/overlays/*/priority-classes
       - path: apps/overlays/*/puppet
       - path: apps/overlays/*/purelb
       - path: apps/overlays/*/reflector-system

argocd/projects/aitooling.yaml

@@ -11,8 +11,6 @@ spec:
   destinations:
     - namespace: 'litellm'
       server: https://kubernetes.default.svc
-    - namespace: 'open-webui'
-      server: https://kubernetes.default.svc
     - namespace: 'paperclip'
       server: https://kubernetes.default.svc
   clusterResourceWhitelist:

argocd/projects/platform.yaml

@@ -31,6 +31,8 @@ spec:
       server: https://kubernetes.default.svc
     - namespace: 'node-feature-discovery'
       server: https://kubernetes.default.svc
+    - namespace: 'priority-classes'
+      server: https://kubernetes.default.svc
     - namespace: 'purelb'
       server: https://kubernetes.default.svc
     - namespace: 'puppet'

ci/generate-crd-schemas.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env -S uv run
+# /// script
+# requires-python = ">=3.11"
+# dependencies = ["pyyaml"]
+# ///
+"""
+Extract OpenAPI v3 schemas from CRD YAML on stdin and write JSON schema files
+to the output directory for use with kubeconform.
+
+Usage: kustomize build ... | ci/generate-crd-schemas.py <output-dir>
+"""
+import sys
+import json
+import os
+
+import yaml
+
+
+def main() -> int:
+    output_dir = sys.argv[1] if len(sys.argv) > 1 else "ci/crd-schemas"
+    count = 0
+
+    for doc in yaml.safe_load_all(sys.stdin):
+        if not doc or doc.get("kind") != "CustomResourceDefinition":
+            continue
+
+        group = doc["spec"]["group"]
+        kind = doc["spec"]["names"]["kind"]
+        group_dir = os.path.join(output_dir, group)
+        os.makedirs(group_dir, exist_ok=True)
+
+        for ver in doc["spec"].get("versions", []):
+            if not ver.get("served", True):
+                continue
+            schema = ver.get("schema", {}).get("openAPIV3Schema")
+            if not schema:
+                continue
+            fname = os.path.join(group_dir, f"{kind.lower()}_{ver['name']}.json")
+            with open(fname, "w") as f:
+                json.dump({"$schema": "http://json-schema.org/schema#", **schema}, f, indent=2)
+                f.write("\n")
+            print(f"  wrote {fname}", file=sys.stderr)
+            count += 1
+
+    return count
+
+
+if __name__ == "__main__":
+    print(main())

ci/generate-crd-schemas.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Extract OpenAPI v3 schemas from CRD definitions in all kustomize overlays
+# and write JSON schema files to ci/crd-schemas/ for kubeconform validation.
+#
+# Run this script whenever CRD versions change, then commit the output.
+# Usage: ci/generate-crd-schemas.sh [output-dir]
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT_DIR="${1:-${SCRIPT_DIR}/crd-schemas}"
+mkdir -p "$OUTPUT_DIR"
+
+total=0
+
+while IFS= read -r -d "" k; do
+  dir="$(dirname "$k")"
+  n=$(kustomize build --enable-helm "$dir" 2>/dev/null \
+    | "$SCRIPT_DIR/generate-crd-schemas.py" "$OUTPUT_DIR") || continue
+  total=$((total + n))
+done < <(find apps/overlays clusters -name kustomization.yaml -print0 | sort -z)
+
+echo "Generated ${total} schema(s) in ${OUTPUT_DIR}" >&2

ci/validate-apps.sh

@@ -6,6 +6,7 @@ KUBE_VERSION="1.33.7"
 schema_args=(
   -schema-location "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/{{.NormalizedKubernetesVersion}}-standalone{{.StrictSuffix}}/{{.ResourceKind}}{{.KindSuffix}}.json"
   -schema-location "https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json"
+  -schema-location "file://${PWD}/ci/crd-schemas/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json"
 )
 
 while IFS= read -r -d "" k; do
@@ -18,6 +19,6 @@ while IFS= read -r -d "" k; do
         -summary \
         -output pretty \
         -verbose \
-        -skip CustomResourceDefinition,GpuDevicePlugin,LBNodeAgent,ServiceGroup \
+        -skip GpuDevicePlugin,LBNodeAgent,ServiceGroup \
         "${schema_args[@]}"
 done < <(find apps/overlays -name kustomization.yaml -print0)

ci/validate-clusters.sh

@@ -6,6 +6,7 @@ KUBE_VERSION="1.33.7"
 schema_args=(
   -schema-location "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/{{.NormalizedKubernetesVersion}}-standalone{{.StrictSuffix}}/{{.ResourceKind}}{{.KindSuffix}}.json"
   -schema-location "https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json"
+  -schema-location "file://${PWD}/ci/crd-schemas/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json"
 )
 
 while IFS= read -r -d "" k; do