Reduce media PR to jellyfin-only in its own namespace

Why:
Jellyfin ships and gets validated first, ahead of the rest of the media stack.
Scoping this PR to jellyfin alone keeps the initial rollout small and lets the
HA fork prove out against the real library before the download and manager apps
follow.

How:
- Drop sonarr, radarr, prowlarr, bazarr, nzbget, and jellyseerr and their shared
  media-apps foundation from this PR; they land in later PRs.
- Move jellyfin into its own jellyfin namespace and fold the namespace and the
  static mediafs PV plus its RWX claim into the jellyfin base.
- Keep the static CephFS PV bound to the in-use mediafs library with
  reclaimPolicy Retain and staticVolume true so nothing can reclaim it, mounted
  into jellyfin by the movies and tvseries subPaths; keep redis, the fresh RWX
  transcode scratch, the intel iGPU nodeSelector and i915 request, gateway, and
  httproute.
- Scope the media AppProject and ApplicationSet to the single jellyfin
  namespace and app, extensible as the remaining apps are added.

.woodpecker/vector-test.yaml

@@ -14,9 +14,9 @@ steps:
       # Transform tier + VM ingest: unit-tested transforms.
       - vector test apps/base/logging/vector/aggregator.yaml apps/base/logging/vector/aggregator-tests.yaml
       - vector test apps/base/logging/vector/vm-ingest.yaml apps/base/logging/vector/vm-ingest-tests.yaml
-      # Agent + archiver have no transforms to unit-test; validate they build.
+      # Agent has no transforms to unit-test; validate it builds. (The archiver
+      # leg is now the logarchiver service, not a Vector pipeline.)
       - vector validate --no-environment apps/base/logging/vector/agent.yaml
-      - vector validate --no-environment apps/base/logging/vector/archiver.yaml
     backend_options:
       kubernetes:
         serviceAccountName: default

apps/base/age-api/deployment.yaml

@@ -12,13 +12,13 @@ spec:
   template:
     metadata:
       annotations:
-        reloader.stakater.com/auto: "true"
+        configmap.reloader.stakater.com/auto: "true"
       labels:
         app: age-api
     spec:
       containers:
         - name: age-api
-          image: git.unkin.net/unkin/age-api:v0.1.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/age-api:v0.1.0
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8080

apps/base/artifactapi/api-deployment.yaml

@@ -5,7 +5,8 @@ metadata:
   name: api
   namespace: artifactapi
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "vault-ca-cert"
 spec:
   selector:
     matchLabels:

apps/base/artifactapi/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-artifactapi
   namespace: artifactapi
 spec:
+  placementTarget: ec
   bucketName: cnpg-artifactapi
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.

apps/base/artifactapi/httproute.yaml

@@ -1,30 +1,6 @@
 ---
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
-metadata:
-  name: http-redirect
-  namespace: artifactapi
-spec:
-  hostnames:
-    - artifactapi.k8s.syd1.au.unkin.net
-  parentRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: artifactapi
-      sectionName: http
-  rules:
-    - filters:
-        - type: RequestRedirect
-          requestRedirect:
-            scheme: https
-            statusCode: 301
-      matches:
-        - path:
-            type: PathPrefix
-            value: /
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
 metadata:
   name: api-route
   namespace: artifactapi
@@ -32,6 +8,12 @@ spec:
   hostnames:
     - artifactapi.k8s.syd1.au.unkin.net
   parentRefs:
+    # Early-boot clients (anaconda/kickstart, yum in %post, PXE) need plain HTTP
+    # for the rpm repos; serve the app directly on port 80 instead of redirecting.
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: artifactapi
+      sectionName: http
     - group: gateway.networking.k8s.io
       kind: Gateway
       name: artifactapi

apps/base/artifactapi/ui-deployment.yaml

@@ -5,7 +5,7 @@ metadata:
   name: ui
   namespace: artifactapi
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
 spec:
   selector:
     matchLabels:

apps/base/authentik/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-authentik
   namespace: authentik
 spec:
+  placementTarget: ec
   bucketName: cnpg-authentik
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.

apps/base/bind-external/agent-dns-rolebinding.yaml

@@ -0,0 +1,16 @@
+---
+# Confines the agent-dns service account (in bind-system) to the agent-dns
+# ClusterRole within this namespace.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: agent-dns
+  namespace: bind-external
+subjects:
+  - kind: ServiceAccount
+    name: agent-dns
+    namespace: bind-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: agent-dns

apps/base/bind-external/cluster.yaml

@@ -0,0 +1,52 @@
+---
+# Externally-reachable authoritative BIND for zones we delegate to ourselves.
+# First tenant: acme.unkin.net, the DNS-01 challenge zone Let's Encrypt validates
+# via a one-time _acme-challenge.unkin.net CNAME. Authoritative-only, recursion
+# off, no forwarding, no open transfers -- the primaryService is the single
+# dmz-pinned LoadBalancer that public NAT targets and that cert-manager writes to.
+apiVersion: bind.unkin.net/v1alpha1
+kind: BindCluster
+metadata:
+  name: bind-external
+  namespace: bind-external
+spec:
+  mode: authoritative
+  recursion: false
+  replicas: 2
+  storageClassName: cephrbd-fast-delete
+  storageSize: 1Gi
+  # Public server: answer queries from anywhere (Let's Encrypt validates over the
+  # internet), deny recursion and open zone transfers. localhost + pod net are
+  # implied by "any" and cover in-pod nsupdate and secondary SOA refresh; per-zone
+  # allow-transfer (catalog + acme zone) still permits key-authenticated AXFR.
+  extraOptions:
+    - "allow-query { any; }"
+    - "allow-transfer { none; }"
+  service:
+    type: ClusterIP
+  primaryService:
+    type: LoadBalancer
+    externalTrafficPolicy: Local
+    annotations:
+      purelb.io/service-group: dmz
+      purelb.io/addresses: 198.18.199.53
+      external-dns.alpha.kubernetes.io/hostname: bind-external-primary.k8s.syd1.au.unkin.net
+  resources:
+    requests:
+      cpu: 20m
+      memory: 128Mi
+    limits:
+      cpu: "1"
+      memory: 512Mi
+---
+# Catalog zone so the acme zone replicates onto the secondary (AXFR/IXFR keyed
+# with the certmanager TSIG key, reused here as the transfer key).
+apiVersion: bind.unkin.net/v1alpha1
+kind: BindCatalogZone
+metadata:
+  name: bind-external-catalog
+  namespace: bind-external
+spec:
+  clusterRef: bind-external
+  zoneName: catalog.external
+  transferKeyRef: certmanager

apps/base/bind-external/kustomization.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - cluster.yaml
+  - tsigkey.yaml
+  - zones.yaml
+  - agent-dns-rolebinding.yaml

apps/base/bind-external/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: bind-external

apps/base/bind-external/tsigkey.yaml

@@ -0,0 +1,20 @@
+---
+# TSIG key cert-manager uses to send RFC2136 dynamic updates (the DNS-01 TXT
+# records) to the primary, and that the secondary reuses for AXFR. The operator
+# generates the material into Secret certmanager-tsig in this namespace;
+# secretTemplate stamps emberstack reflector hints so the Secret is mirrored into
+# the cert-manager namespace, where the rfc2136 solver reads its "secret" key.
+apiVersion: bind.unkin.net/v1alpha1
+kind: BindTSIGKey
+metadata:
+  name: certmanager
+  namespace: bind-external
+spec:
+  clusterRef: bind-external
+  algorithm: hmac-sha256
+  secretTemplate:
+    annotations:
+      reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cert-manager"
+      reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cert-manager"

apps/base/bind-external/zones.yaml

@@ -0,0 +1,19 @@
+---
+# Self-delegated ACME challenge zone. Google Cloud DNS holds a one-time
+# _acme-challenge.unkin.net CNAME -> _acme-challenge.acme.unkin.net and an
+# acme.unkin.net NS delegation pointing here; cert-manager writes the challenge
+# TXT records via RFC2136 authenticated with the certmanager key.
+apiVersion: bind.unkin.net/v1alpha1
+kind: BindZone
+metadata:
+  name: acme-unkin-net
+  namespace: bind-external
+spec:
+  clusterRef: bind-external
+  zoneName: acme.unkin.net
+  type: primary
+  defaultTTL: 60
+  dynamicUpdate: true
+  updateKeyRef: certmanager
+  allowTransfer:
+    - key certmanager

apps/base/bind-internal/agent-dns-rolebinding.yaml

@@ -0,0 +1,16 @@
+---
+# Confines the agent-dns service account (in bind-system) to the agent-dns
+# ClusterRole within this namespace.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: agent-dns
+  namespace: bind-internal
+subjects:
+  - kind: ServiceAccount
+    name: agent-dns
+    namespace: bind-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: agent-dns

apps/base/bind-internal/authoritative/cluster.yaml

@@ -18,6 +18,9 @@ spec:
   # without it every dynamic update is "denied due to allow-query".
   extraOptions:
     - "allow-query { localhost; auth-acl-main; 10.42.0.0/16; }"
+    # Enable query logging for the Tier-2 vector bind_query pipeline (see the
+    # resolvers cluster for the routing rationale).
+    - "querylog yes"
   service:
     type: LoadBalancer
     externalTrafficPolicy: Local

apps/base/bind-internal/authoritative/records.yaml

@@ -20,6 +20,32 @@ spec:
     # identity.unkin.net hostname there.
     - 198.18.200.4
 ---
+# PRODUCTION CUTOVER RECORD — intentionally commented out.
+# git.unkin.net currently resolves to the LIVE VM forge (HAProxy VRRP VIP
+# 198.18.19.17), which holds every repo the estate depends on. Uncommenting this
+# repoints the whole org's git.unkin.net at the new k8s Gitea gateway VIP, so it
+# is the FINAL step of the forge migration — gated on the data migration (gitea
+# dump/restore + SECRET_KEY copy) in argocd-apps docs/gitea-migration.md.
+# NOTE: the live git.unkin.net answer is served by the puppet DNS master today
+# (profiles::dns::master, records from PuppetDB); this k8s apex zone holds only
+# SOA+NS + a few DNSRecords so far. Confirm the k8s bind cluster is the live
+# authority for unkin.net (or update the puppet record instead) before relying
+# on this CR at cutover.
+# ---
+# apiVersion: bind.unkin.net/v1alpha1
+# kind: DNSRecord
+# metadata:
+#   name: git-dns-internal
+#   namespace: bind-internal
+# spec:
+#   zoneRef: unkin-net
+#   name: git
+#   type: A
+#   ttl: 600
+#   values:
+#     # traefik-internal gateway VIP; the gitea Gateway serves git.unkin.net there.
+#     - 198.18.200.4
+---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:

apps/base/bind-internal/kustomization.yaml

@@ -8,3 +8,4 @@ resources:
   - resolvers
   - externaldns
   - tsig-api
+  - agent-dns-rolebinding.yaml

apps/base/bind-internal/resolvers/cluster.yaml

@@ -30,6 +30,11 @@ spec:
   # (incl. k8s.syd1.au.unkin.net); 18.198.in-addr.arpa covers every reverse zone.
   extraOptions:
     - "validate-except { unkin.net; 18.198.in-addr.arpa; consul; }"
+    # Enable query logging so the Tier-2 vector bind_query pipeline can parse
+    # client/qname/qtype. Routes to the `queries` category which, with no explicit
+    # logging{} clause, follows the default category to the named foreground
+    # stderr channel -> pod stdout -> vector (subject logs.k8s.bind-internal.*).
+    - "querylog yes"
   resources:
     requests:
       cpu: 20m

apps/base/bind-internal/tsig-api/tsig-api.yaml

@@ -13,7 +13,7 @@ metadata:
   name: bind-tsig-api
   namespace: bind-internal
 spec:
-  image: git.unkin.net/unkin/bind-tsig-api:v0.2.3
+  image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-tsig-api:v0.2.3
   replicas: 1
   port: 8443
   # targetNamespace defaults to this resource's namespace (bind-internal), where

apps/base/bind-system/agent-dns-rbac.yaml

@@ -0,0 +1,38 @@
+---
+# Static service account that Vault's kubernetes secret engine mints scoped
+# tokens for (agent-dns role). RBAC is confined to the bind namespaces via the
+# per-namespace RoleBindings below, not a ClusterRoleBinding.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: agent-dns
+  namespace: bind-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: agent-dns
+rules:
+  - apiGroups: ["bind.unkin.net"]
+    resources: ["*"]
+    verbs: ["*"]
+  - apiGroups: [""]
+    resources: ["pods", "services", "configmaps", "events"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods/log"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: agent-dns
+  namespace: bind-system
+subjects:
+  - kind: ServiceAccount
+    name: agent-dns
+    namespace: bind-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: agent-dns

apps/base/bind-system/deployment.yaml

@@ -21,7 +21,7 @@ spec:
         runAsNonRoot: true
       containers:
         - name: operator
-          image: git.unkin.net/unkin/bind-operator:v0.2.6
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.2.6
           args:
             - --metrics-bind-address=:8080
             - --health-probe-bind-address=:8081

apps/base/bind-system/kustomization.yaml

@@ -8,5 +8,6 @@ resources:
   # vendored here, so they never drift from the operator.
   - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.2.6/config/crd/install.yaml
   - rbac.yaml
+  - agent-dns-rbac.yaml
   - deployment.yaml
   - vpa.yaml

apps/base/cephrgw-system/deployment.yaml

@@ -7,8 +7,10 @@ metadata:
   labels:
     app.kubernetes.io/name: cephrgw-operator
   annotations:
-    # Restart the operator when the credentials Secret rotates.
-    reloader.stakater.com/auto: "true"
+    # Restart on internal CA rotation only; cephrgw-credentials is Vault-rotated
+    # (VSO) and deliberately excluded so routine key rotation causes no restart.
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "vault-ca-cert"
 spec:
   replicas: 1
   selector:
@@ -24,7 +26,7 @@ spec:
         runAsNonRoot: true
       containers:
         - name: operator
-          image: git.unkin.net/unkin/cephrgw-operator:v0.3.1
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/cephrgw-operator:v0.4.0
           args:
             - --metrics-bind-address=:8080
             - --health-probe-bind-address=:8081

apps/base/cephrgw-system/kustomization.yaml

@@ -6,7 +6,7 @@ resources:
   - namespace.yaml
   # CRDs are pulled from the cephrgw-operator repo at the matching tag rather
   # than vendored here, so they never drift from the operator.
-  - https://git.unkin.net/unkin/cephrgw-operator/raw/tag/v0.3.1/config/crd/install.yaml
+  - https://git.unkin.net/unkin/cephrgw-operator/raw/tag/v0.4.0/config/crd/install.yaml
   - rbac.yaml
   - deployment.yaml
   - vaultauth.yaml

apps/base/cert-manager/clusterissuer_letsencrypt-staging.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-staging
+spec:
+  acme:
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    email: admin@unkin.net
+    privateKeySecretRef:
+      name: letsencrypt-staging-account-key
+    solvers:
+      - dns01:
+          cnameStrategy: Follow
+          rfc2136:
+            nameserver: "198.18.199.53:53"
+            tsigKeyName: certmanager
+            tsigAlgorithm: HMACSHA256
+            tsigSecretSecretRef:
+              name: certmanager-tsig
+              key: secret

apps/base/cert-manager/clusterissuer_letsencrypt.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: admin@unkin.net
+    privateKeySecretRef:
+      name: letsencrypt-account-key
+    solvers:
+      - dns01:
+          cnameStrategy: Follow
+          rfc2136:
+            nameserver: "198.18.199.53:53"
+            tsigKeyName: certmanager
+            tsigAlgorithm: HMACSHA256
+            tsigSecretSecretRef:
+              name: certmanager-tsig
+              key: secret

apps/base/cert-manager/clusterissuer_vault-issuer.yaml

@@ -0,0 +1,27 @@
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: vault-issuer
+  labels:
+    app.kubernetes.io/instance: cert-manager-config
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: cert-manager-config
+  annotations:
+    meta.helm.sh/release-name: cert-manager-clusterissuer
+    meta.helm.sh/release-namespace: cert-manager
+spec:
+  vault:
+    server: https://vault.service.consul:8200
+    path: pki_int/sign/servers_default
+    caBundleSecretRef:
+      key: ca.crt
+      name: vault-ca-cert
+    auth:
+      kubernetes:
+        mountPath: /v1/auth/k8s/au/syd1
+        role: cert_manager_issuer
+        serviceAccountRef:
+          name: cert-manager-vault-issuer
+          audiences:
+            - vault

apps/base/cert-manager/kustomization.yaml

@@ -7,4 +7,7 @@ resources:
   - serviceaccount.yaml
   - clusterrole.yaml
   - clusterrolebinding.yaml
+  - clusterissuer_vault-issuer.yaml
   - vmservicescrape.yaml
+  - clusterissuer_letsencrypt.yaml
+  - clusterissuer_letsencrypt-staging.yaml

apps/base/certificates/vault-ca-cert.yaml

@@ -1,4 +1,6 @@
 ---
+# pre-commit: allow-plain-secret -- public CA bundle; this secret bootstraps
+# trust in Vault itself and therefore cannot be Vault-sourced.
 apiVersion: v1
 kind: Secret
 metadata:
@@ -28,7 +30,7 @@ stringData:
     mitItX+RAgMBAAGjgewwgekwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB
     Af8wHQYDVR0OBBYEFEp/+grAdVqRSeb9xJjSeZYNW32MMB8GA1UdIwQYMBaAFBqc
     v6Y+hfHt4EjgKa/uoQGEHTknMEcGCCsGAQUFBwEBBDswOTA3BggrBgEFBQcwAoYr
-    aHR0cHM6Ly92YXVsdC5zZXJ2aWNlLmNvbnN1bC92MS9wa2lfcm9vdC9jYTA9BgNV
+    aHR0cHM6Ly92YXVsdC5zZXJ2dWNlLmNvbnN1bC92MS9wa2lfcm9vdC9jYTA9BgNV
     HR8ENjA0MDKgMKAuhixodHRwczovL3ZhdWx0LnNlcnZpY2UuY29uc3VsL3YxL3Br
     aV9yb290L2NybDANBgkqhkiG9w0BAQsFAAOCAQEAM0FS8tscZe7yly/gM7jO6lx5
     muMFusifjUIrcQGnZBkoECeuUVPNTs3e/Th+XaxjCnmSpqSNT3z9Irr6Hhxf7n03

apps/base/dhcp-system/cr/keaapi.yaml

@@ -0,0 +1,25 @@
+---
+# Terraform-friendly REST API for KeaSubnet/KeaClientClass CRUD. The bearer
+# token Secret is generated by the operator when absent (no plain Secret is
+# committed here); it can later be pre-seeded from Vault under the same name.
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaAPI
+metadata:
+  name: kea-api
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  replicas: 1
+  image: git.unkin.net/unkin/kea-api:v0.1.3
+  tokenSecretName: kea-api-token
+  service:
+    type: ClusterIP
+    port: 8080
+  resources:
+    requests:
+      cpu: 100m
+      memory: 64Mi
+    limits:
+      cpu: "1"
+      memory: 256Mi

apps/base/dhcp-system/cr/keaclientclasses.yaml

@@ -0,0 +1,27 @@
+# PXE boot classes matching client architecture (option 93), replacing the
+# legacy dhcpd "Legacy" and "UEFI-64" classes. Object names are lowercased to
+# satisfy RFC1123 (the operator renders the kea class name from metadata.name).
+---
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaClientClass
+metadata:
+  name: legacy
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  clusterRef: kea
+  archHex: ["0x0000"]
+  bootFileName: /undionly.kpxe
+---
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaClientClass
+metadata:
+  name: uefi-64
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  clusterRef: kea
+  archHex: ["0x0007", "0x0009"]
+  bootFileName: /ipxe.efi

apps/base/dhcp-system/cr/keacluster.yaml

@@ -0,0 +1,35 @@
+---
+# HA pair fronted by a PureLB anycast Service on a NEW, unused common-pool IP
+# (198.18.200.10). This is intentionally NOT the current isc-dhcpd anycast
+# address (198.18.19.18) -- the production cutover is a separate later task.
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaCluster
+metadata:
+  name: kea
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  replicas: 2
+  image: git.unkin.net/unkin/kea:v0.1.3
+  domainName: main.unkin.net
+  defaultLeaseTime: 1200
+  maxLeaseTime: 86400
+  # No ntpServers: DHCP option 42 (ntp-servers) carries IPv4 addresses only, so
+  # the rotating AU pool.ntp.org hostnames cannot be delivered this way (kea
+  # rejects them at config load). Add concrete NTP server IPs here if needed.
+  ha:
+    mode: hot-standby
+  service:
+    type: LoadBalancer
+    ipAddressPool: common
+    loadBalancerIP: 198.18.200.10
+    annotations:
+      purelb.io/addresses: 198.18.200.10
+  resources:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: "1"
+      memory: 512Mi

apps/base/dhcp-system/cr/keasubnets.yaml

@@ -0,0 +1,90 @@
+# Translation of the legacy ISC dhcpd pools (puppet
+# roles/infra/dhcp/server.yaml): 198.18.13-17.0/24, each a .200-.220 pool,
+# next-server 198.18.19.19. Gateways per the original config:
+# .13/.14/.15/.16 -> .254, .17 -> .1. DNS points at the in-cluster
+# bind-resolvers PureLB IP (198.18.200.7), not the legacy 198.18.19.15.
+---
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaSubnet
+metadata:
+  name: net-198-18-13
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  clusterRef: kea
+  subnet: 198.18.13.0/24
+  pools:
+    - 198.18.13.200 - 198.18.13.220
+  routers: [198.18.13.254]
+  dnsServers: [198.18.200.7]
+  domainName: main.unkin.net
+  nextServer: 198.18.19.19
+---
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaSubnet
+metadata:
+  name: net-198-18-14
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  clusterRef: kea
+  subnet: 198.18.14.0/24
+  pools:
+    - 198.18.14.200 - 198.18.14.220
+  routers: [198.18.14.254]
+  dnsServers: [198.18.200.7]
+  domainName: main.unkin.net
+  nextServer: 198.18.19.19
+---
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaSubnet
+metadata:
+  name: net-198-18-15
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  clusterRef: kea
+  subnet: 198.18.15.0/24
+  pools:
+    - 198.18.15.200 - 198.18.15.220
+  routers: [198.18.15.254]
+  dnsServers: [198.18.200.7]
+  domainName: main.unkin.net
+  nextServer: 198.18.19.19
+---
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaSubnet
+metadata:
+  name: net-198-18-16
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  clusterRef: kea
+  subnet: 198.18.16.0/24
+  pools:
+    - 198.18.16.200 - 198.18.16.220
+  routers: [198.18.16.254]
+  dnsServers: [198.18.200.7]
+  domainName: main.unkin.net
+  nextServer: 198.18.19.19
+---
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaSubnet
+metadata:
+  name: net-198-18-17
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  clusterRef: kea
+  subnet: 198.18.17.0/24
+  pools:
+    - 198.18.17.200 - 198.18.17.220
+  routers: [198.18.17.1]
+  dnsServers: [198.18.200.7]
+  domainName: main.unkin.net
+  nextServer: 198.18.19.19

apps/base/dhcp-system/deployment.yaml

@@ -0,0 +1,56 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kea-operator
+  namespace: dhcp-system
+  labels:
+    app.kubernetes.io/name: kea-operator
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kea-operator
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kea-operator
+    spec:
+      serviceAccountName: kea-operator
+      securityContext:
+        runAsNonRoot: true
+      containers:
+        - name: operator
+          image: git.unkin.net/unkin/kea-operator:v0.1.3
+          args:
+            - --metrics-bind-address=:8080
+            - --health-probe-bind-address=:8081
+          ports:
+            - containerPort: 8080
+              name: metrics
+            - containerPort: 8081
+              name: health
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: 8081
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8081
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 500m
+              memory: 256Mi

apps/base/dhcp-system/kustomization.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  # CRDs are pulled from the kea-operator repo at the matching tag rather than
+  # vendored here, so they never drift from the operator.
+  - https://git.unkin.net/unkin/kea-operator/raw/tag/v0.1.0/config/crd/install.yaml
+  - rbac.yaml
+  - deployment.yaml
+  - vpa.yaml
+  # CRs (sync-wave 1) reconcile after the operator + CRDs are established.
+  - cr/keacluster.yaml
+  - cr/keasubnets.yaml
+  - cr/keaclientclasses.yaml
+  - cr/keaapi.yaml

apps/base/dhcp-system/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: dhcp-system

apps/base/dhcp-system/rbac.yaml

@@ -0,0 +1,46 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kea-operator
+  namespace: dhcp-system
+---
+# Sourced from the kea-operator repo config/rbac/role.yaml (v0.1.0). Leader
+# election is disabled so no coordination.k8s.io/leases grant is needed.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kea-operator
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps", "secrets", "serviceaccounts", "services"]
+    verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments", "statefulsets"]
+    verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
+  - apiGroups: ["kea.unkin.net"]
+    resources: ["keaapis", "keaclientclasses", "keaclusters", "keasubnets"]
+    verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
+  - apiGroups: ["kea.unkin.net"]
+    resources:
+      ["keaapis/status", "keaclientclasses/status", "keaclusters/status", "keasubnets/status"]
+    verbs: ["get", "patch", "update"]
+  - apiGroups: ["rbac.authorization.k8s.io"]
+    resources: ["rolebindings", "roles"]
+    verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kea-operator
+subjects:
+  - kind: ServiceAccount
+    name: kea-operator
+    namespace: dhcp-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kea-operator

apps/base/dhcp-system/vpa.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: autoscaling.k8s.io/v1
+kind: VerticalPodAutoscaler
+metadata:
+  name: kea-operator-vpa
+  namespace: dhcp-system
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: kea-operator
+  updatePolicy:
+    updateMode: "Off"

apps/base/encapi/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-encapi
   namespace: encapi
 spec:
+  placementTarget: ec
   bucketName: cnpg-encapi
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.

apps/base/encapi/deployment.yaml

@@ -5,7 +5,7 @@ metadata:
   name: encapi
   namespace: encapi
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
 spec:
   replicas: 2
   selector:
@@ -23,7 +23,7 @@ spec:
       automountServiceAccountToken: true
       containers:
         - name: encapi
-          image: git.unkin.net/unkin/encapi:v0.1.1
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/encapi:v0.1.1
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000

apps/base/externaldns/agent-dns-rolebinding.yaml

@@ -0,0 +1,16 @@
+---
+# Confines the agent-dns service account (in bind-system) to the agent-dns
+# ClusterRole within this namespace.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: agent-dns
+  namespace: externaldns
+subjects:
+  - kind: ServiceAccount
+    name: agent-dns
+    namespace: bind-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: agent-dns

apps/base/externaldns/kustomization.yaml

@@ -6,3 +6,4 @@ resources:
   - namespace.yaml
   - vaultauth.yaml
   - vaultstaticsecret.yaml
+  - agent-dns-rolebinding.yaml

apps/base/gitea/cnpg_backup.yaml

@@ -0,0 +1,46 @@
+---
+# Ceph RGW (S3) backup target for the gitea CNPG cluster, provisioned by the
+# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: cnpg-gitea-backup
+  namespace: gitea
+spec:
+  displayName: "CNPG backup owner (gitea)"
+  # RGW users are global; keep the uid namespace-qualified so it never collides.
+  uid: cnpg-gitea-backup
+  maxBuckets: 5
+  # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
+  # S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it.
+  secretName: cnpg-gitea-backup-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: cnpg-gitea
+  namespace: gitea
+spec:
+  bucketName: cnpg-gitea
+  ownerRef: cnpg-gitea-backup
+  versioning: false
+  tags:
+    app: gitea
+    purpose: cnpg-backup
+  retainOnDelete: true
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-gitea-nightly
+  namespace: gitea
+spec:
+  # 6-field CNPG cron (seconds first). 04:00 — next free slot after netbox
+  # (03:40), keeping the estate's 20-minute stagger.
+  schedule: "0 0 4 * * *"
+  immediate: false
+  backupOwnerReference: self
+  method: barmanObjectStore
+  cluster:
+    name: gitea-postgres

apps/base/gitea/cnpg_cluster.yaml

@@ -0,0 +1,90 @@
+---
+# Postgres for the k8s Gitea (replaces the Patroni-shared DB the VM uses). Gitea
+# already runs on Postgres, so cutover is a plain pg dump/restore (no engine
+# conversion). App-user creds come from the postgres-credentials Vault secret.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: gitea-postgres
+  namespace: gitea
+spec:
+  affinity:
+    podAntiAffinityType: preferred
+  backup:
+    # 30-day retention. Enforced by CNPG against the object store on each
+    # successful base backup.
+    retentionPolicy: 30d
+    barmanObjectStore:
+      # Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it).
+      destinationPath: s3://cnpg-gitea
+      endpointURL: https://s3.ceph.unkin.net
+      # radosgw serves a Vault-PKI cert; trust the internal CA (reflected into
+      # every namespace as the vault-ca-cert Secret).
+      endpointCA:
+        name: vault-ca-cert
+        key: ca.crt
+      # Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded.
+      s3Credentials:
+        accessKeyId:
+          name: cnpg-gitea-backup-s3
+          key: AWS_ACCESS_KEY_ID
+        secretAccessKey:
+          name: cnpg-gitea-backup-s3
+          key: AWS_SECRET_ACCESS_KEY
+      # Path prefix within the bucket; keep stable across restores (see docs).
+      serverName: gitea
+      data:
+        compression: bzip2
+        jobs: 2
+      wal:
+        compression: zstd
+        maxParallel: 2
+  bootstrap:
+    initdb:
+      database: gitea
+      encoding: UTF8
+      localeCType: C
+      localeCollate: C
+      owner: gitea
+      secret:
+        name: postgres-credentials
+  enablePDB: true
+  enableSuperuserAccess: false
+  failoverDelay: 0
+  imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie
+  instances: 2
+  logLevel: info
+  monitoring:
+    customQueriesConfigMap:
+      - key: queries
+        name: cnpg-default-monitoring
+    disableDefaultQueries: false
+    enablePodMonitor: false
+  postgresql:
+    parameters:
+      max_connections: "200"
+      shared_buffers: 256MB
+  primaryUpdateMethod: restart
+  primaryUpdateStrategy: unsupervised
+  replicationSlots:
+    highAvailability:
+      enabled: true
+      slotPrefix: _cnpg_
+    synchronizeReplicas:
+      enabled: true
+    updateInterval: 30
+  resources:
+    limits:
+      cpu: "2"
+      memory: 2Gi
+    requests:
+      cpu: 250m
+      memory: 512Mi
+  smartShutdownTimeout: 180
+  startDelay: 3600
+  stopDelay: 1800
+  storage:
+    resizeInUseVolumes: true
+    size: 20Gi
+    storageClass: cephrbd-fast-delete
+  switchoverDelay: 3600

apps/base/gitea/cnpg_pooler.yaml

@@ -0,0 +1,36 @@
+---
+# pgbouncer in front of the primary. Gitea opens a connection per request and
+# benefits from pooling under multiple app replicas. Session mode keeps Gitea's
+# occasional session-scoped state (advisory locks, LISTEN/NOTIFY) working.
+apiVersion: postgresql.cnpg.io/v1
+kind: Pooler
+metadata:
+  name: gitea-postgres-pooler-rw
+  namespace: gitea
+spec:
+  cluster:
+    name: gitea-postgres
+  instances: 2
+  pgbouncer:
+    parameters:
+      default_pool_size: "50"
+      max_client_conn: "200"
+    paused: false
+    poolMode: session
+  template:
+    metadata:
+      labels:
+        app: pooler-rw
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - pooler-rw
+              topologyKey: kubernetes.io/hostname
+      containers: []
+  type: rw

apps/base/gitea/gateway.yaml

@@ -0,0 +1,69 @@
+---
+# HTTPS front for the k8s Gitea, served on two names:
+#   git.unkin.net              — canonical/production (apex, bind-operator zone;
+#                                DNS flip is the gated cutover step, see the doc)
+#   git.k8s.syd1.au.unkin.net  — admin/backup route (external-dns k8s.syd1 zone),
+#                                same dual-name pattern as identity.unkin.net.
+# The cert-manager Certificate (vault-issuer) takes CN git.unkin.net and gets a
+# DNS SAN for each TLS listener hostname automatically.
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: gitea
+  namespace: gitea
+  labels:
+    app.kubernetes.io/name: gitea
+    app.kubernetes.io/instance: gitea
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: git.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    # Only the k8s admin route is published by external-dns (it owns just the
+    # k8s.syd1.au.unkin.net zone). git.unkin.net lives in the apex zone and is
+    # flipped at cutover — NOT managed here.
+    external-dns.alpha.kubernetes.io/hostname: git.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - name: http-primary
+      port: 80
+      protocol: HTTP
+      hostname: git.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https-primary
+      port: 443
+      protocol: HTTPS
+      hostname: git.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: gitea-tls
+    - name: http-admin
+      port: 80
+      protocol: HTTP
+      hostname: git.k8s.syd1.au.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https-admin
+      port: 443
+      protocol: HTTPS
+      hostname: git.k8s.syd1.au.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: gitea-tls

apps/base/gitea/httproute.yaml

@@ -0,0 +1,65 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: gitea-http-redirect
+  namespace: gitea
+  labels:
+    app.kubernetes.io/name: gitea
+    app.kubernetes.io/instance: gitea
+spec:
+  hostnames:
+    - git.unkin.net
+    - git.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gitea
+      sectionName: http-primary
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gitea
+      sectionName: http-admin
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: gitea
+  namespace: gitea
+  labels:
+    app.kubernetes.io/name: gitea
+    app.kubernetes.io/instance: gitea
+spec:
+  hostnames:
+    - git.unkin.net
+    - git.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gitea
+      sectionName: https-primary
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gitea
+      sectionName: https-admin
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: gitea-http
+          port: 3000
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/gitea/kustomization.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - cnpg_cluster.yaml
+  - cnpg_backup.yaml
+  - cnpg_pooler.yaml
+  - valkey-deployment.yaml
+  - valkey-pvc.yaml
+  - valkey-service.yaml
+  - vaultauth.yaml
+  - vaultstaticsecret.yaml
+  - gateway.yaml
+  - httproute.yaml

apps/base/gitea/namespace.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    app.kubernetes.io/name: gitea
+  name: gitea

apps/base/gitea/valkey-deployment.yaml

@@ -0,0 +1,89 @@
+---
+# Standalone Valkey (Redis-compatible) for Gitea's session store, cache and
+# queue. The Gitea chart bundles a redis-cluster subchart, but we run our own
+# standalone Valkey here: it keeps image control in-estate (valkey/valkey,
+# already allowlisted through the artifactapi dockerhub mirror) and matches the
+# standalone-cache pattern used by litellm/netbox. One instance serves three
+# logical DBs: DB 0 = session, DB 1 = cache, DB 2 = queue. AOF persistence is
+# enabled so queued actions/webhook deliveries survive a restart.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gitea-valkey
+  namespace: gitea
+  labels:
+    app.kubernetes.io/name: gitea
+    app.kubernetes.io/component: valkey
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: gitea-valkey
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: gitea-valkey
+        app.kubernetes.io/name: gitea
+        app.kubernetes.io/component: valkey
+    spec:
+      securityContext:
+        fsGroup: 999
+      containers:
+        - name: valkey
+          image: valkey/valkey:8-alpine
+          imagePullPolicy: IfNotPresent
+          command:
+            - valkey-server
+            - --appendonly
+            - "yes"
+            - --save
+            - "60"
+            - "1"
+          ports:
+            - containerPort: 6379
+              name: valkey
+              protocol: TCP
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsNonRoot: true
+            runAsUser: 999
+            capabilities:
+              drop:
+                - ALL
+          livenessProbe:
+            exec:
+              command:
+                - valkey-cli
+                - ping
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            successThreshold: 1
+            timeoutSeconds: 5
+          readinessProbe:
+            exec:
+              command:
+                - valkey-cli
+                - ping
+            failureThreshold: 3
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 5
+          resources:
+            limits:
+              cpu: 500m
+              memory: 512Mi
+            requests:
+              cpu: 50m
+              memory: 128Mi
+          volumeMounts:
+            - mountPath: /data
+              name: data
+      restartPolicy: Always
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: gitea-valkey-data

apps/base/gitea/valkey-pvc.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: gitea-valkey-data
+  namespace: gitea
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+  storageClassName: cephrbd-fast-delete
+  volumeMode: Filesystem

apps/base/gitea/valkey-service.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: gitea-valkey
+  namespace: gitea
+  labels:
+    app.kubernetes.io/name: gitea
+    app.kubernetes.io/component: valkey
+spec:
+  internalTrafficPolicy: Cluster
+  ports:
+    - name: valkey
+      port: 6379
+      protocol: TCP
+      targetPort: valkey
+  selector:
+    app: gitea-valkey
+  sessionAffinity: None
+  type: ClusterIP

apps/base/gitea/vaultauth.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: default
+  namespace: gitea
+spec:
+  allowedNamespaces:
+    - gitea
+  kubernetes:
+    audiences:
+      - vault
+    role: default
+    serviceAccount: default
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default

apps/base/gitea/vaultstaticsecret.yaml

@@ -0,0 +1,83 @@
+---
+# CNPG app-user credentials (keys: username, password). Consumed by the Cluster
+# bootstrap (initdb.secret) AND by Gitea (gitea.config.database.PASSWD via the
+# chart's existingSecret wiring). One-time Vault seed — see the PR description.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: postgres-credentials
+  namespace: gitea
+spec:
+  destination:
+    create: true
+    name: postgres-credentials
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/gitea/default/postgres-credentials
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+# Initial Gitea admin (keys: username, password, email). Applied by the chart's
+# init job on first boot (gitea.admin.existingSecret). Local fallback account
+# that survives the Authentik OIDC cutover. One-time Vault seed.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: gitea-admin
+  namespace: gitea
+spec:
+  destination:
+    create: true
+    name: gitea-admin
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/gitea/default/gitea-admin
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+# Gitea internal secrets (keys: SECRET_KEY, INTERNAL_TOKEN). Pinned here rather
+# than chart-generated so all replicas share identical values AND so the data
+# cutover can replace them with the VM's app.ini values (SECRET_KEY encrypts
+# 2FA/mirror/oauth secrets in the DB — it MUST match the restored database).
+# One-time Vault seed.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: gitea-inner
+  namespace: gitea
+spec:
+  destination:
+    create: true
+    name: gitea-inner
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/gitea/default/gitea-inner
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+# Authentik OIDC client secret (key: client_secret). Read by the
+# terraform-authentik provider runner (policy already grants
+# kv/.../namespace/+/default/oauth-credentials) AND mounted into Gitea to
+# register the OIDC login source. One-time Vault seed.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: oauth-credentials
+  namespace: gitea
+spec:
+  destination:
+    create: true
+    name: oauth-credentials
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/gitea/default/oauth-credentials
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/base/grafana/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-grafana
   namespace: grafana
 spec:
+  placementTarget: ec
   bucketName: cnpg-grafana
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.

apps/base/grafana/grafana.yaml

@@ -26,6 +26,13 @@ spec:
                     secretKeyRef:
                       name: oauth-credentials
                       key: client_secret
+              # identity.unkin.net is served by the internal unkin.net CA, which
+              # the stock Grafana image doesn't trust. Mount the reflected
+              # vault-ca-cert and point generic_oauth's tls_client_ca at it.
+              volumeMounts:
+                - name: vault-ca-cert
+                  mountPath: /etc/grafana/vault-ca
+                  readOnly: true
               resources:
                 requests:
                   cpu: 100m
@@ -33,6 +40,13 @@ spec:
                 limits:
                   cpu: "1"
                   memory: 1Gi
+          volumes:
+            - name: vault-ca-cert
+              secret:
+                secretName: vault-ca-cert
+                items:
+                  - key: ca.crt
+                    path: ca.crt
   config:
     server:
       root_url: "https://grafana.k8s.syd1.au.unkin.net"
@@ -57,6 +71,9 @@ spec:
       auth_url: "https://identity.unkin.net/application/o/authorize/"
       token_url: "https://identity.unkin.net/application/o/token/"
       api_url: "https://identity.unkin.net/application/o/userinfo/"
+      # Trust the internal unkin.net CA that signs identity.unkin.net's cert
+      # (mounted from the reflected vault-ca-cert Secret).
+      tls_client_ca: "/etc/grafana/vault-ca/ca.crt"
       # Authentik permission groups -> Grafana roles. akP-grafana-admin is granted
       # to akR-global-admin members (and direct members) via terraform-authentik.
       role_attribute_path: "contains(ak_groups[*], 'akP-grafana-admin') && 'Admin' || 'Viewer'"

apps/base/jellyfin/deployment.yaml

@@ -0,0 +1,97 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: jellyfin
+  namespace: jellyfin
+spec:
+  # Single-replica for now. The jellyfin-ha fork adds the Redis transcode store
+  # and RWX transcode scratch that make scaling to true HA a follow-up.
+  replicas: 1
+  strategy:
+    # Config PVC is RWO; Recreate avoids two pods contending for it.
+    type: Recreate
+  selector:
+    matchLabels:
+      app: jellyfin
+  template:
+    metadata:
+      labels:
+        app: jellyfin
+    spec:
+      securityContext:
+        fsGroup: 1000
+      nodeSelector:
+        feature.node.kubernetes.io/pci-0300_8086.present: "true"
+      containers:
+        - name: jellyfin
+          image: git.unkin.net/unkin/jellyfin-ha:v0.1.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 8096
+              protocol: TCP
+          env:
+            - name: TZ
+              value: Australia/Sydney
+            - name: PUID
+              value: "1000"
+            - name: PGID
+              value: "1000"
+            - name: JELLYFIN_PublishedServerUrl
+              value: https://jellyfin.k8s.syd1.au.unkin.net
+            # Distributed transcode session store (jellyfin-ha additions).
+            - name: Jellyfin__TranscodeStore__RedisConnectionString
+              value: "jellyfin-redis:6379,abortConnect=false"
+            - name: Jellyfin__TranscodeStore__LeaseDurationSeconds
+              value: "30"
+          resources:
+            requests:
+              cpu: 100m
+              memory: 1Gi
+            limits:
+              cpu: "4"
+              memory: 8Gi
+              gpu.intel.com/i915: "1"
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          volumeMounts:
+            - name: config
+              mountPath: /config
+            - name: cache
+              mountPath: /cache
+            - name: transcode
+              mountPath: /transcode
+            - name: media-library
+              mountPath: /mnt/movies
+              subPath: movies
+            - name: media-library
+              mountPath: /mnt/tvseries
+              subPath: tvseries
+      volumes:
+        - name: config
+          persistentVolumeClaim:
+            claimName: jellyfin-config
+        - name: cache
+          persistentVolumeClaim:
+            claimName: jellyfin-cache
+        - name: transcode
+          persistentVolumeClaim:
+            claimName: jellyfin-transcode
+        - name: media-library
+          persistentVolumeClaim:
+            claimName: media-library

apps/base/jellyfin/gateway.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: jellyfin.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    external-dns.alpha.kubernetes.io/hostname: jellyfin.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+  name: jellyfin
+  namespace: jellyfin
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: jellyfin.k8s.syd1.au.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: jellyfin.k8s.syd1.au.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: jellyfin-tls
+        mode: Terminate

apps/base/jellyfin/httproute.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: http-redirect
+  namespace: jellyfin
+spec:
+  hostnames:
+    - jellyfin.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: jellyfin
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: jellyfin
+  namespace: jellyfin
+spec:
+  hostnames:
+    - jellyfin.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: jellyfin
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: jellyfin
+          port: 8096
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/jellyfin/kustomization.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - pv_media-library.yaml
+  - pvc_media-library.yaml
+  - pvc_config.yaml
+  - pvc_cache.yaml
+  - pvc_transcode.yaml
+  - deployment.yaml
+  - service.yaml
+  - redis-deployment.yaml
+  - redis-service.yaml
+  - redis-pvc.yaml
+  - gateway.yaml
+  - httproute.yaml

apps/base/jellyfin/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: jellyfin

apps/base/jellyfin/pv_media-library.yaml

@@ -0,0 +1,41 @@
+---
+# Static CephFS PersistentVolume bound to the pre-existing, ACTIVELY-USED
+# puppet media library (ceph filesystem "mediafs", mounted by the VM/incus
+# instances at /shared/media). ceph-csi only mounts this volume; staticVolume
+# tells it the storage pre-exists and it must never provision or delete it.
+#
+# reclaimPolicy MUST stay Retain: deleting this PV or its PVC must NEVER be able
+# to reclaim or destroy the underlying CephFS data that the VM instances use.
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: jellyfin-media-library
+spec:
+  accessModes:
+    - ReadWriteMany
+  capacity:
+    storage: 10Ti
+  # Load-bearing safety control. Do not change to Delete.
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: ""
+  volumeMode: Filesystem
+  # Pre-bind to the media-library claim so nothing else can grab this PV.
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: media-library
+    namespace: jellyfin
+  csi:
+    driver: cephfs.csi.ceph.com
+    volumeHandle: jellyfin-media-library-static
+    nodeStageSecretRef:
+      name: csi-cephfs-secret
+      namespace: csi-cephfs
+    volumeAttributes:
+      # clusterID maps (in the csi-cephfs ceph-csi-config) to the mon set that
+      # also serves mediafs; for a static volume only the mon lookup is used.
+      clusterID: cephfs_csi_ssd_ec_4_1
+      fsName: mediafs
+      staticVolume: "true"
+      # Filesystem-internal root of the library (mediafs root == /shared/media).
+      rootPath: /

apps/base/jellyfin/pvc_cache.yaml

@@ -0,0 +1,15 @@
+---
+# Local transcode/image cache. Scratch, delete reclaim policy.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: jellyfin-cache
+  namespace: jellyfin
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 200Gi
+  storageClassName: cephrbd-fast-delete
+  volumeMode: Filesystem

apps/base/jellyfin/pvc_config.yaml

@@ -0,0 +1,15 @@
+---
+# Jellyfin config + SQLite library database. Single-writer, block storage.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: jellyfin-config
+  namespace: jellyfin
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+  storageClassName: cephrbd-fast-retain
+  volumeMode: Filesystem

apps/base/jellyfin/pvc_media-library.yaml

@@ -0,0 +1,19 @@
+---
+# Claim bound to the static mediafs PV. RWX so every app in the stack shares the
+# one library. storageClassName "" + volumeName pin it to the static PV (no
+# dynamic provisioning). Deleting this claim cannot reclaim the data (PV is
+# Retain).
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: media-library
+  namespace: jellyfin
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: ""
+  volumeName: jellyfin-media-library
+  resources:
+    requests:
+      storage: 10Ti
+  volumeMode: Filesystem

apps/base/jellyfin/pvc_transcode.yaml

@@ -0,0 +1,17 @@
+---
+# Shared transcode scratch. ReadWriteMany is the hard requirement for the HA
+# fork: a taking-over pod must read the in-flight HLS segments written by the
+# pod it replaces. Scratch data, so delete reclaim policy.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: jellyfin-transcode
+  namespace: jellyfin
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 100Gi
+  storageClassName: cephfs-raid5-delete
+  volumeMode: Filesystem

apps/base/jellyfin/redis-deployment.yaml

@@ -0,0 +1,64 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: jellyfin-redis
+  namespace: jellyfin
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: jellyfin-redis
+  template:
+    metadata:
+      labels:
+        app: jellyfin-redis
+    spec:
+      restartPolicy: Always
+      containers:
+        - name: redis
+          image: redis:7-alpine
+          imagePullPolicy: IfNotPresent
+          command:
+            - redis-server
+            - --save
+            - "20"
+            - "1"
+          ports:
+            - name: redis
+              containerPort: 6379
+              protocol: TCP
+          livenessProbe:
+            exec:
+              command:
+                - redis-cli
+                - ping
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            exec:
+              command:
+                - redis-cli
+                - ping
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+          volumeMounts:
+            - name: data
+              mountPath: /data
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: jellyfin-redis-data

apps/base/jellyfin/redis-pvc.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: jellyfin-redis-data
+  namespace: jellyfin
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+  storageClassName: cephrbd-fast-delete
+  volumeMode: Filesystem

apps/base/jellyfin/redis-service.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: jellyfin-redis
+  namespace: jellyfin
+spec:
+  type: ClusterIP
+  internalTrafficPolicy: Cluster
+  sessionAffinity: None
+  selector:
+    app: jellyfin-redis
+  ports:
+    - name: redis
+      port: 6379
+      targetPort: redis
+      protocol: TCP

apps/base/jellyfin/service.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: jellyfin
+  namespace: jellyfin
+spec:
+  type: ClusterIP
+  internalTrafficPolicy: Cluster
+  sessionAffinity: None
+  selector:
+    app: jellyfin
+  ports:
+    - name: http
+      port: 8096
+      targetPort: http
+      protocol: TCP

apps/base/kanidm/statefulset.yaml

@@ -5,7 +5,8 @@ metadata:
   name: kanidm
   namespace: kanidm
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "kanidm-tls"
   labels:
     app.kubernetes.io/name: kanidm
     app.kubernetes.io/instance: kanidm

apps/base/litellm/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-litellm
   namespace: litellm
 spec:
+  placementTarget: ec
   bucketName: cnpg-litellm
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.

apps/base/litellm/cnpg_cluster.yaml

@@ -48,7 +48,7 @@ spec:
   enablePDB: true
   enableSuperuserAccess: false
   failoverDelay: 0
-  imageName: ghcr.io/cloudnative-pg/postgresql:17-minimal-trixie
+  imageName: ghcr.io/cloudnative-pg/postgresql:17-system-trixie
   instances: 3
   logLevel: info
   maxSyncReplicas: 0

apps/base/litellm/deployment.yaml

@@ -11,10 +11,28 @@ spec:
   template:
     metadata:
       annotations:
-        reloader.stakater.com/auto: "true"
+        configmap.reloader.stakater.com/auto: "true"
+        secret.reloader.stakater.com/reload: "vault-ca-cert"
       labels:
         app: litellm
     spec:
+      # LiteLLM's SSO client reaches identity.unkin.net, whose cert is signed by
+      # the internal unkin.net CA. Combine the image's public roots with the
+      # reflected vault-ca-cert into one bundle (SSL_CERT_FILE/REQUESTS_CA_BUNDLE
+      # in litellm-env point at it) so internal OIDC and public HTTPS both work.
+      initContainers:
+        - name: combine-certs
+          image: alpine:3
+          command:
+            - sh
+            - -c
+            - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt
+          volumeMounts:
+            - name: vault-ca-cert
+              mountPath: /custom-ca
+              readOnly: true
+            - name: combined-certs
+              mountPath: /combined-certs
       containers:
         - name: litellm
           image: docker.litellm.ai/berriai/litellm-database:main-stable
@@ -72,8 +90,19 @@ spec:
             - mountPath: /app/config.yaml
               name: config
               subPath: config.yaml
+            - name: combined-certs
+              mountPath: /etc/ssl/combined
+              readOnly: true
       restartPolicy: Always
       volumes:
         - name: config
           configMap:
             name: litellm-config
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: combined-certs
+          emptyDir: {}

apps/base/litellm/kustomization.yaml

@@ -27,6 +27,9 @@ configMapGenerator:
   - name: litellm-env
     literals:
       - STORE_MODEL_IN_DB=True
+      # Emit structured JSON logs so the Tier-2 vector litellm pipeline can parse
+      # model/tokens/latency/key/status (logs.k8s.litellm.*).
+      - JSON_LOGS=True
       # Authentik OIDC SSO (generic). Client secret is injected from the
       # oauth-credentials Secret in the Deployment; endpoints match the other
       # apps (identity.unkin.net). PROXY_BASE_URL is required for SSO.
@@ -39,5 +42,9 @@ configMapGenerator:
       - GENERIC_SCOPE=openid email profile litellm_role
       - GENERIC_USER_ROLE_ATTRIBUTE=litellm_role
       - PROXY_BASE_URL=https://litellm.k8s.syd1.au.unkin.net
+      # Trust the internal unkin.net CA (identity.unkin.net) via the combined
+      # bundle assembled by the combine-certs init container.
+      - SSL_CERT_FILE=/etc/ssl/combined/ca-certificates.crt
+      - REQUESTS_CA_BUNDLE=/etc/ssl/combined/ca-certificates.crt
     options:
       disableNameSuffixHash: true

apps/base/logging/configmap_logarchiver.yaml

@@ -0,0 +1,40 @@
+---
+# logarchiver non-secret config. Secrets (NATS/S3/ClickHouse creds) and the
+# subject filter come from env; everything else uses the binary's built-in
+# defaults, which already target this stack. ack_wait MUST exceed batch.max_age
+# so unacked messages in an open batch are not redelivered mid-batch.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logarchiver-config
+  namespace: logging
+data:
+  config.yaml: |
+    nats:
+      ack_wait: 5m
+      fetch_batch: 512
+    batch:
+      max_bytes: 67108864
+      max_events: 200000
+      max_age: 2m
+    # Pin the proven RGW endpoint/bucket; ignore the secret's S3_ENDPOINT/BUCKET_NAME
+    # (AWS creds still come from the secret env). endpoint_env/bucket_env off.
+    s3:
+      endpoint: "https://s3.ceph.unkin.net"
+      bucket: "logs-archive"
+      region: "us-east-1"
+      path_style: true
+      ca_file: /etc/vault-ca/ca.crt
+      endpoint_env: ""
+      bucket_env: ""
+    crypto:
+      key_name: logarchive
+      pubkey_source: vault
+      vault:
+        address: "https://vault.service.consul:8200"
+        mount: gpg
+        auth_method: kubernetes
+        k8s_mount: k8s/au/syd1
+        k8s_role: logging_logarchiver
+        k8s_jwt_path: /var/run/secrets/vault/token
+        ca_file: /etc/vault-ca/ca.crt

apps/base/logging/deployment_logarchiver.yaml

@@ -0,0 +1,134 @@
+---
+# logarchiver — replaces the vector-archiver leg. Independent JetStream durable
+# consumer (archiver) that seals raw logs to S3 as zstd + OpenPGP objects and
+# indexes each object in ClickHouse (logs.archive_index). Acks only after the
+# object is in S3 AND indexed. Reuses the same NATS/S3/CA wiring the Vector
+# archiver used; the OpenPGP public key is delivered as a mounted file.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: logarchiver
+  namespace: logging
+  annotations:
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "vault-ca-cert"
+  labels:
+    app.kubernetes.io/name: logarchiver
+    app.kubernetes.io/component: archiver
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: logarchiver
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: logarchiver
+        vector.dev/exclude: "true"
+    spec:
+      # Dedicated SA whose projected vault-audience token authenticates the
+      # k8s-auth login used to fetch the logarchive public key from the gpg engine.
+      serviceAccountName: logarchiver
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: logarchiver
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/logarchiver:v0.1.0
+          imagePullPolicy: IfNotPresent
+          args: ["run"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          ports:
+            - containerPort: 9090
+              name: metrics
+              protocol: TCP
+          env:
+            - name: LOGARCHIVER_CONFIG
+              value: /etc/logarchiver/config.yaml
+            # Server-side subject filter; must match the archiver consumer's filter.
+            - name: ARCHIVE_SUBJECTS
+              value: "logs.k8s.vault.>"
+            - name: NATS_CONSUMER_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: nats-auth
+                  key: consumer_password
+            - name: CLICKHOUSE_USER
+              valueFrom:
+                secretKeyRef:
+                  name: clickhouse-credentials
+                  key: username
+            - name: CLICKHOUSE_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: clickhouse-credentials
+                  key: password
+          # S3 creds + S3_ENDPOINT + BUCKET_NAME from the cephrgw BucketAccess Secret.
+          envFrom:
+            - secretRef:
+                name: logs-archive-s3
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 15
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: "1"
+              memory: 1Gi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/logarchiver/config.yaml
+              subPath: config.yaml
+              readOnly: true
+            - name: vault-token
+              mountPath: /var/run/secrets/vault
+              readOnly: true
+            - name: vault-ca-cert
+              mountPath: /etc/vault-ca/ca.crt
+              subPath: ca.crt
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: logarchiver-config
+        # Projected SA token with audience "vault" for the gpg-engine k8s login.
+        - name: vault-token
+          projected:
+            sources:
+              - serviceAccountToken:
+                  path: token
+                  audience: vault
+                  expirationSeconds: 600
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+        - name: tmp
+          emptyDir: {}

apps/base/logging/job_clickhouse-schema.yaml

@@ -88,6 +88,32 @@ spec:
               ORDER BY (source, namespace, host, timestamp)
               TTL toDateTime(timestamp) + INTERVAL 3 DAY
               SETTINGS index_granularity = 8192;
+
+              -- One row per archived S3 object (written by logarchiver). No TTL:
+              -- the index must outlive logs.raw so the long-term S3 archive stays
+              -- searchable. Keep in sync with logarchiver internal/index/ddl.go.
+              CREATE TABLE IF NOT EXISTS logs.archive_index
+              (
+                  object_key       String,
+                  bucket           LowCardinality(String),
+                  subject          LowCardinality(String),
+                  hosts            Array(LowCardinality(String)),
+                  min_ts           DateTime64(3),
+                  max_ts           DateTime64(3),
+                  event_count      UInt64,
+                  raw_bytes        UInt64,
+                  stored_bytes     UInt64,
+                  compression      LowCardinality(String),
+                  cipher           LowCardinality(String),
+                  container_format LowCardinality(String),
+                  key_name         LowCardinality(String),
+                  key_fingerprint  String,
+                  created_at       DateTime64(3) DEFAULT now64(3),
+                  INDEX idx_hosts hosts TYPE bloom_filter GRANULARITY 1
+              )
+              ENGINE = MergeTree
+              PARTITION BY toYYYYMM(min_ts)
+              ORDER BY (subject, min_ts, object_key);
               EOSQL
               echo "Schema applied."
           resources:

apps/base/logging/kustomization.yaml

@@ -12,6 +12,9 @@ resources:
   - cephrgw.yaml
   - gateway.yaml
   - httproute.yaml
+  - serviceaccount_logarchiver.yaml
+  - configmap_logarchiver.yaml
+  - deployment_logarchiver.yaml
 
 # Vector pipelines are the single source of truth (also validated by
 # `vector test` in CI). Mounted into each tier via `existingConfigMaps`.
@@ -44,8 +47,3 @@ configMapGenerator:
       - vm-ingest.yaml=vector/vm-ingest.yaml
     options:
       disableNameSuffixHash: true
-  - name: vector-archiver-config
-    files:
-      - archiver.yaml=vector/archiver.yaml
-    options:
-      disableNameSuffixHash: true

apps/base/logging/nats-bootstrap-job.yaml

@@ -59,6 +59,11 @@ spec:
       containers:
         - name: nats-bootstrap
           image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/natsio/nats-box:0.18.0
+          # nats CLI stats the working directory when loading its response
+          # schemas; under readOnlyRootFilesystem + runAsUser 1000 the image's
+          # default WORKDIR is not accessible ("stat .: permission denied"), so
+          # run from the writable /tmp emptyDir.
+          workingDir: /tmp
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true

apps/base/logging/serviceaccount_logarchiver.yaml

@@ -0,0 +1,9 @@
+---
+# Dedicated SA for logarchiver's Vault k8s-auth login (role logging_logarchiver,
+# terraform-vault). Only used to fetch the logarchive public key.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: logarchiver
+  namespace: logging
+automountServiceAccountToken: false

apps/base/logging/vector/aggregator-tests.yaml

@@ -61,3 +61,608 @@ tests:
               assert_eq!(.severity, "info")
               assert_eq!(.message, "sshd started")
               assert_eq!(.labels.role, "database")
+
+  # --- catch-all preservation: an unclaimed k8s event still flows app_route ->
+  # generic route -> k8s_shape (proves the two-stage chain keeps the fallback) ---
+  - name: unclaimed_k8s_falls_through_to_generic
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.shop.web"
+          message: "plain app log"
+    outputs:
+      - extract_from: route.k8s
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.message, "plain app log")
+
+  # --- Tier-1: Authentik SSO (k8s, LIVE NOW) ---
+  - name: authentik_routes_by_subject
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.authentik.server"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.authentik
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: authentik_parse_extracts_event
+    inputs:
+      - insert_at: authentik_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.authentik.server"
+          stream: "stdout"
+          kubernetes.pod_namespace: "authentik"
+          kubernetes.container_name: "server"
+          kubernetes.pod_node_name: "node-2"
+          message: '{"event":"login","action":"login","user":"alice","client_ip":"203.0.113.9","result":"success","level":"info","logger":"authentik.events","timestamp":"2026-07-27T00:00:00Z"}'
+    outputs:
+      - extract_from: authentik_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "authentik")
+              assert_eq!(.container, "server")
+              assert_eq!(.severity, "info")
+              assert_eq!(.message, "login")
+              assert_eq!(.labels.app, "authentik")
+              assert_eq!(.fields.event, "login")
+              assert_eq!(.fields.action, "login")
+              assert_eq!(.fields.user, "alice")
+              assert_eq!(.fields.client_ip, "203.0.113.9")
+              assert_eq!(.fields.result, "success")
+
+  # --- Tier-1: Traefik ingress (k8s, JSON access logs) ---
+  - name: traefik_routes_by_subject
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.traefik-system.traefik"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.traefik
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: traefik_parse_extracts_access_fields
+    inputs:
+      - insert_at: traefik_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.traefik-system.traefik"
+          kubernetes.pod_namespace: "traefik-system"
+          kubernetes.container_name: "traefik"
+          kubernetes.pod_node_name: "node-3"
+          message: '{"RouterName":"web@kubernetes","ServiceName":"shop-svc@kubernetes","RequestMethod":"GET","RequestPath":"/api","RequestHost":"shop.example.net","RequestProtocol":"HTTP/1.1","DownstreamStatus":200,"Duration":5000000,"ClientHost":"203.0.113.5","StartUTC":"2026-07-27T00:00:00Z"}'
+    outputs:
+      - extract_from: traefik_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "traefik-system")
+              assert_eq!(.labels.app, "traefik")
+              assert_eq!(.message, "GET /api 200")
+              assert_eq!(.fields.route, "web@kubernetes")
+              assert_eq!(.fields.service, "shop-svc@kubernetes")
+              assert_eq!(.fields.method, "GET")
+              assert_eq!(.fields.path, "/api")
+              assert_eq!(.fields.host, "shop.example.net")
+              assert_eq!(.fields.status, "200")
+              assert_eq!(.fields.duration_ms, "5")
+              assert_eq!(.fields.client_ip, "203.0.113.5")
+
+  # --- Tier-1: Vault/OpenBao file audit (VM, awaiting VM vector) ---
+  - name: vault_routes_by_file
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.vm.vault1_syd1"
+          file: "/var/log/vault_audit.log"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.vault
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: vault_parse_extracts_request
+    inputs:
+      - insert_at: vault_parse
+        type: log
+        log_fields:
+          subject: "logs.vm.vault1_syd1"
+          host: "vault1"
+          file: "/var/log/vault_audit.log"
+          message: '{"time":"2026-07-27T00:00:00Z","type":"response","auth":{"display_name":"token"},"request":{"operation":"read","path":"secret/data/app","remote_address":"10.0.0.9"},"error":""}'
+    outputs:
+      - extract_from: vault_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "vm")
+              assert_eq!(.host, "vault1")
+              assert_eq!(.labels.app, "vault")
+              assert_eq!(.message, "read secret/data/app")
+              assert_eq!(.fields.type, "response")
+              assert_eq!(.fields.display_name, "token")
+              assert_eq!(.fields.operation, "read")
+              assert_eq!(.fields.path, "secret/data/app")
+              assert_eq!(.fields.remote_address, "10.0.0.9")
+
+  # --- Tier-1: nginx access (VM, awaiting VM vector) ---
+  - name: nginx_access_routes_by_file
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.vm.web1_syd1"
+          file: "/var/log/nginx/shop_access.log"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.nginx_access
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: nginx_access_parse_extracts_combined
+    inputs:
+      - insert_at: nginx_access_parse
+        type: log
+        log_fields:
+          subject: "logs.vm.web1_syd1"
+          host: "web1"
+          file: "/var/log/nginx/shop_access.log"
+          message: '192.0.2.10 - - [27/Jul/2026:00:00:00 +0000] "GET /index.html HTTP/1.1" 200 1024 "https://ref.example/" "Mozilla/5.0" 0.012'
+    outputs:
+      - extract_from: nginx_access_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "vm")
+              assert_eq!(.stream, "access")
+              assert_eq!(.labels.log_type, "access")
+              assert_eq!(.fields.client_ip, "192.0.2.10")
+              assert_eq!(.fields.method, "GET")
+              assert_eq!(.fields.path, "/index.html")
+              assert_eq!(.fields.status, "200")
+              assert_eq!(.fields.bytes, "1024")
+              assert_eq!(.fields.referer, "https://ref.example/")
+              assert_eq!(.fields.user_agent, "Mozilla/5.0")
+              assert_eq!(.fields.request_time, "0.012")
+
+  # --- Tier-1: nginx error (VM, awaiting VM vector) ---
+  - name: nginx_error_routes_by_file
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.vm.web1_syd1"
+          file: "/var/log/nginx/shop_error.log"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.nginx_error
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: nginx_error_parse_extracts_fields
+    inputs:
+      - insert_at: nginx_error_parse
+        type: log
+        log_fields:
+          subject: "logs.vm.web1_syd1"
+          host: "web1"
+          file: "/var/log/nginx/shop_error.log"
+          message: '2026/07/27 00:00:00 [error] 1234#0: *5 open() "/var/www/x" failed (2: No such file or directory), client: 192.0.2.20, server: shop, request: "GET / HTTP/1.1", host: "shop"'
+    outputs:
+      - extract_from: nginx_error_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.stream, "error")
+              assert_eq!(.severity, "error")
+              assert_eq!(.labels.log_type, "error")
+              assert_eq!(.fields.level, "error")
+              assert_eq!(.fields.pid, "1234")
+              assert_eq!(.fields.cid, "5")
+              assert_eq!(.fields.client_ip, "192.0.2.20")
+
+  # --- Tier-1: HAProxy httplog (VM journald, awaiting VM vector) ---
+  - name: haproxy_routes_by_identifier
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.vm.halb1_syd1"
+          SYSLOG_IDENTIFIER: "haproxy"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.haproxy
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: haproxy_parse_extracts_timers
+    inputs:
+      - insert_at: haproxy_parse
+        type: log
+        log_fields:
+          subject: "logs.vm.halb1_syd1"
+          host: "halb1"
+          SYSLOG_IDENTIFIER: "haproxy"
+          message: '192.0.2.30:54321 [27/Jul/2026:00:00:00.123] fe_http be_app/app1 10/0/1/2/13 200 512 - - ---- 5/4/3/2/0 0/0 "GET /health HTTP/1.1"'
+    outputs:
+      - extract_from: haproxy_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "vm")
+              assert_eq!(.labels.app, "haproxy")
+              assert_eq!(.fields.client_ip, "192.0.2.30")
+              assert_eq!(.fields.frontend, "fe_http")
+              assert_eq!(.fields.backend, "be_app")
+              assert_eq!(.fields.server, "app1")
+              assert_eq!(.fields.tq, "10")
+              assert_eq!(.fields.tw, "0")
+              assert_eq!(.fields.tc, "1")
+              assert_eq!(.fields.tr, "2")
+              assert_eq!(.fields.tt, "13")
+              assert_eq!(.fields.termination_state, "----")
+              assert_eq!(.fields.retries, "0")
+              assert_eq!(.fields.status, "200")
+              assert_eq!(.fields.bytes, "512")
+
+  # --- Tier-1: glauth LDAP (VM, awaiting VM vector) ---
+  - name: glauth_routes_by_identifier
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.vm.ldap1_syd1"
+          SYSLOG_IDENTIFIER: "glauth"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.glauth
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: glauth_parse_extracts_bind
+    inputs:
+      - insert_at: glauth_parse
+        type: log
+        log_fields:
+          subject: "logs.vm.ldap1_syd1"
+          host: "ldap1"
+          SYSLOG_IDENTIFIER: "glauth"
+          message: '{"level":"info","msg":"Bind success as user","bindDN":"cn=admin,dc=example,dc=com","src":"192.0.2.40:1234","time":"2026-07-27T00:00:00Z"}'
+    outputs:
+      - extract_from: glauth_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "vm")
+              assert_eq!(.host, "ldap1")
+              assert_eq!(.severity, "info")
+              assert_eq!(.labels.app, "glauth")
+              assert_eq!(.fields.bindDN, "cn=admin,dc=example,dc=com")
+              assert_eq!(.fields.remote, "192.0.2.40:1234")
+              assert_eq!(.fields.success, "true")
+
+  # ================= Tier-2 (stacks on #318) =================
+
+  # --- BIND query logs (k8s bind-* + VM named) ---
+  - name: bind_routes_k8s_by_subject
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.bind-internal.named"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.bind_query
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: bind_routes_vm_by_identifier
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.vm.dns1_syd1"
+          SYSLOG_IDENTIFIER: "named"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.bind_query
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: bind_parse_extracts_query
+    inputs:
+      - insert_at: bind_query_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.bind-internal.named"
+          kubernetes.pod_namespace: "bind-internal"
+          kubernetes.container_name: "named"
+          kubernetes.pod_node_name: "node-4"
+          message: '02-Aug-2026 00:00:00.123 client @0x7f 192.0.2.1#40426 (www.example.com): view internal: query: www.example.com IN A +E(0)K (198.18.200.7)'
+    outputs:
+      - extract_from: bind_query_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "bind-internal")
+              assert_eq!(.labels.app, "bind")
+              assert_eq!(.message, "query www.example.com A")
+              assert_eq!(.fields.client_ip, "192.0.2.1")
+              assert_eq!(.fields.qname, "www.example.com")
+              assert_eq!(.fields.qclass, "IN")
+              assert_eq!(.fields.qtype, "A")
+              assert_eq!(.fields.view, "internal")
+
+  # --- Rancher audit (k8s, cattle-system sidecar) ---
+  - name: rancher_routes_by_subject
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.cattle-system.rancher-audit-log"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.rancher_audit
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: rancher_parse_extracts_audit
+    inputs:
+      - insert_at: rancher_audit_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.cattle-system.rancher-audit-log"
+          kubernetes.pod_namespace: "cattle-system"
+          kubernetes.container_name: "rancher-audit-log"
+          kubernetes.pod_node_name: "node-5"
+          message: '{"auditID":"abc-123","requestURI":"/v3/tokens","user":{"name":"u-alice","group":["admins"]},"method":"GET","remoteAddr":"10.42.0.9:1234","responseCode":200,"requestTimestamp":"2026-08-01T00:00:00Z"}'
+    outputs:
+      - extract_from: rancher_audit_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "cattle-system")
+              assert_eq!(.labels.app, "rancher")
+              assert_eq!(.labels.log_type, "audit")
+              assert_eq!(.message, "GET /v3/tokens 200")
+              assert_eq!(.fields.user, "u-alice")
+              assert_eq!(.fields.verb, "GET")
+              assert_eq!(.fields.uri, "/v3/tokens")
+              assert_eq!(.fields.status, "200")
+
+  # --- CNPG Postgres (ONE transform, all clusters) ---
+  - name: cnpg_routes_by_postgres_container
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.litellm.postgres"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.cnpg_pg
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  # mutual exclusivity: an app-namespace CNPG pod (authentik) is claimed by
+  # cnpg_pg, NOT the authentik app route (which now carves out .postgres).
+  - name: cnpg_authentik_postgres_routes_to_cnpg
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.authentik.postgres"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.cnpg_pg
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: cnpg_parse_extracts_record
+    inputs:
+      - insert_at: cnpg_pg_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.litellm.postgres"
+          kubernetes.pod_namespace: "litellm"
+          kubernetes.container_name: "postgres"
+          kubernetes.pod_node_name: "node-6"
+          kubernetes.pod_labels."cnpg.io/cluster": "litellm-postgres"
+          message: '{"level":"info","ts":"2026-08-01T00:00:00Z","logger":"postgres","msg":"record","record":{"user_name":"litellm","database_name":"litellm","error_severity":"LOG","message":"duration: 12.345 ms  statement: SELECT 1","query":""}}'
+    outputs:
+      - extract_from: cnpg_pg_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "litellm")
+              assert_eq!(.severity, "LOG")
+              assert_eq!(.labels.app, "cnpg")
+              assert_eq!(.labels.cluster, "litellm-postgres")
+              assert_eq!(.fields.error_severity, "LOG")
+              assert_eq!(.fields.duration_ms, "12.345")
+              assert_eq!(.fields.user, "litellm")
+              assert_eq!(.fields.database, "litellm")
+
+  # --- Gitea router/access (k8s + VM) ---
+  - name: gitea_routes_k8s_by_subject
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.gitea.gitea"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.gitea
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: gitea_parse_router_line
+    inputs:
+      - insert_at: gitea_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.gitea.gitea"
+          kubernetes.pod_namespace: "gitea"
+          kubernetes.container_name: "gitea"
+          kubernetes.pod_node_name: "node-7"
+          message: '2026/08/01 00:00:00 .../router.go:100:func() [I] router: completed GET /user/login for 10.0.0.1:0, 200 OK in 12.3ms @ web/base.go:1'
+    outputs:
+      - extract_from: gitea_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "gitea")
+              assert_eq!(.labels.app, "gitea")
+              assert_eq!(.message, "GET /user/login 200")
+              assert_eq!(.fields.method, "GET")
+              assert_eq!(.fields.path, "/user/login")
+              assert_eq!(.fields.status, "200")
+              assert_eq!(.fields.latency, "12.3ms")
+  - name: gitea_parse_access_line
+    inputs:
+      - insert_at: gitea_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.gitea.gitea"
+          kubernetes.pod_namespace: "gitea"
+          kubernetes.container_name: "gitea"
+          message: '10.0.0.5 - alice [01/Aug/2026:00:00:00 +0000] "POST /repo/foo HTTP/1.1" 201 512 "-" "git/2.0"'
+    outputs:
+      - extract_from: gitea_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.fields.method, "POST")
+              assert_eq!(.fields.path, "/repo/foo")
+              assert_eq!(.fields.status, "201")
+              assert_eq!(.fields.user, "alice")
+              assert_eq!(.fields.client_ip, "10.0.0.5")
+
+  # --- PuppetServer / PuppetDB (k8s stdout) ---
+  - name: puppet_routes_by_subject
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.puppet.puppetserver"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.puppet
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: puppet_parse_logback_line
+    inputs:
+      - insert_at: puppet_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.puppet.puppetserver"
+          kubernetes.pod_namespace: "puppet"
+          kubernetes.container_name: "puppetserver"
+          kubernetes.pod_node_name: "node-8"
+          message: '2026-08-01 00:00:00,123 INFO  [qtp123-45] [puppetserver] Compiled catalog for web01.unkin.net in environment production in 1.23 seconds'
+    outputs:
+      - extract_from: puppet_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "puppet")
+              assert_eq!(.severity, "INFO")
+              assert_eq!(.labels.app, "puppet")
+              assert_eq!(.fields.level, "INFO")
+              assert_eq!(.fields.logger, "puppetserver")
+              assert_eq!(.fields.node, "web01.unkin.net")
+
+  # --- LiteLLM request logs (k8s JSON) ---
+  - name: litellm_routes_by_subject
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.litellm.litellm"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.litellm
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: litellm_parse_extracts_request
+    inputs:
+      - insert_at: litellm_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.litellm.litellm"
+          kubernetes.pod_namespace: "litellm"
+          kubernetes.container_name: "litellm"
+          kubernetes.pod_node_name: "node-9"
+          message: '{"message":"Request completed","level":"info","model":"gpt-4o","total_tokens":1234,"response_time":0.532,"api_key":"sk-abc","status":"success","timestamp":"2026-08-01T00:00:00Z"}'
+    outputs:
+      - extract_from: litellm_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "litellm")
+              assert_eq!(.severity, "info")
+              assert_eq!(.message, "Request completed")
+              assert_eq!(.labels.app, "litellm")
+              assert_eq!(.fields.model, "gpt-4o")
+              assert_eq!(.fields.tokens, "1234")
+              assert_eq!(.fields.latency, "0.532")
+              assert_eq!(.fields.key, "sk-abc")
+              assert_eq!(.fields.status, "success")
+
+  # --- Postfix maillog (VM, per-line best-effort) ---
+  - name: postfix_routes_by_identifier
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.vm.mail1_syd1"
+          SYSLOG_IDENTIFIER: "postfix/qmgr"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.postfix
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: postfix_parse_extracts_line
+    inputs:
+      - insert_at: postfix_parse
+        type: log
+        log_fields:
+          subject: "logs.vm.mail1_syd1"
+          host: "mail1"
+          SYSLOG_IDENTIFIER: "postfix/smtp"
+          message: 'ABC123DEF: to=<rcpt@example.com>, relay=mx.example.com[1.2.3.4]:25, delay=1.2, delays=0.1/0/0.5/0.6, dsn=2.0.0, status=sent (250 OK)'
+    outputs:
+      - extract_from: postfix_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "vm")
+              assert_eq!(.host, "mail1")
+              assert_eq!(.labels.app, "postfix")
+              assert_eq!(.fields.qid, "ABC123DEF")
+              assert_eq!(.fields.to, "rcpt@example.com")
+              assert_eq!(.fields.relay, "mx.example.com[1.2.3.4]:25")
+              assert_eq!(.fields.delay, "1.2")
+              assert_eq!(.fields.status, "sent")
+              assert_eq!(.fields.program, "postfix/smtp")

apps/base/logging/vector/aggregator.yaml

@@ -3,9 +3,25 @@
 # by `vector test` in CI. Consumes the whole log stream from JetStream via the
 # durable `transform` consumer (at-least-once; durable offsets tracked by
 # JetStream), routes by subject, normalises into the logs.raw columns, and is
-# the ONLY ClickHouse writer. Per-app parsing is added here as follow-ups:
-# insert a transform and append its id to the clickhouse sink `inputs` — no edge
-# or VM rollout required.
+# the ONLY ClickHouse writer.
+#
+# Routing model (two stages):
+#   1. app_route — peels off Tier-1 per-app streams by subject / source tag and
+#      hands each to a dedicated parse transform that emits the full logs.raw
+#      shape plus structured .fields. Conditions are MUTUALLY EXCLUSIVE, so an
+#      event is claimed by at most one app (no double-insert).
+#   2. route (generic catch-all) — everything app_route did NOT claim
+#      (app_route._unmatched) is split k8s/vm and shaped generically. This is the
+#      fallback for all un-parsed traffic and MUST stay intact.
+# Add a new per-app pipeline by appending a mutually-exclusive route to
+# app_route, a parse transform, and its id to the clickhouse sink `inputs`.
+#
+# Structured fields go into the logs.raw `fields Map(String,String)` column — no
+# DDL change is needed (values are stringified; empties are compacted away).
+#
+# VM source-tag convention (the puppet-side vector rollout MUST follow it so
+# these transforms light up): file sources set `.file` (absolute log path);
+# journald sources set `.SYSLOG_IDENTIFIER` (falls back to `.program`/`.appname`).
 #
 # Durability model: JetStream (3d / 130 GiB, S2-compressed) is the SOLE
 # durability layer and the replay window. This
@@ -39,10 +55,68 @@ sources:
       codec: json
 
 transforms:
-  route:
+  # Stage 1: peel off Tier-1 per-app streams. Mutually exclusive conditions;
+  # anything unmatched falls through to the generic `route` below.
+  app_route:
     type: route
     inputs:
       - js_in
+    route:
+      # k8s: authentik SSO — structlog JSON on stdout. The `.postgres` container
+      # is the authentik-namespace CNPG cluster; carve it out so it is claimed by
+      # the single `cnpg_pg` route below (keeps app_route mutually exclusive).
+      authentik: 'starts_with(to_string(.subject) ?? "", "logs.k8s.authentik.") && !ends_with(to_string(.subject) ?? "", ".postgres")'
+      # k8s: Traefik ingress — JSON access logs (requires logs.access.format=json,
+      # flipped in the traefik-system overlay values in this same change).
+      traefik: 'starts_with(to_string(.subject) ?? "", "logs.k8s.traefik-system.")'
+      # VM: Vault/OpenBao file audit device (/var/log/vault_audit.log), JSON.
+      vault: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && contains(to_string(.file) ?? "", "vault_audit")'
+      # VM: nginx combined access log (/var/log/nginx/<vhost>_access.log).
+      nginx_access: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && contains(to_string(.file) ?? "", "nginx") && ends_with(to_string(.file) ?? "", "access.log")'
+      # VM: nginx error log (/var/log/nginx/<vhost>_error.log).
+      nginx_error: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && contains(to_string(.file) ?? "", "nginx") && ends_with(to_string(.file) ?? "", "error.log")'
+      # VM: HAProxy httplog via journald.
+      haproxy: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && ((to_string(.SYSLOG_IDENTIFIER) ?? "") == "haproxy" || (to_string(.program) ?? "") == "haproxy" || (to_string(.appname) ?? "") == "haproxy")'
+      # VM: glauth LDAP — structuredlog (logrus) JSON.
+      glauth: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && (contains(to_string(.file) ?? "", "glauth") || (to_string(.SYSLOG_IDENTIFIER) ?? "") == "glauth" || (to_string(.program) ?? "") == "glauth" || (to_string(.appname) ?? "") == "glauth")'
+
+      # --- Tier-2 (stacks on #318) ---
+      # BIND query logs, k8s + VM. k8s: any bind-* namespace (bind-internal DNS
+      # servers, bind-system operator) — query logging enabled via `querylog yes`
+      # in the BindCluster extraOptions in this change. VM: puppet-managed named
+      # (file /var/log/named/*.log or journald `named`) — puppet-side enable is a
+      # required follow-up (profiles/dns/server.pp).
+      bind_query: 'starts_with(to_string(.subject) ?? "", "logs.k8s.bind") || (starts_with(to_string(.subject) ?? "", "logs.vm.") && (contains(to_string(.file) ?? "", "named") || (to_string(.SYSLOG_IDENTIFIER) ?? "") == "named" || (to_string(.program) ?? "") == "named" || (to_string(.appname) ?? "") == "named"))'
+      # k8s: Rancher audit log — JSON, emitted by the `rancher-audit-log` sidecar
+      # (auditLog.enabled level 1, already on in the cattle-system overlay).
+      rancher_audit: 'starts_with(to_string(.subject) ?? "", "logs.k8s.cattle-system.rancher-audit-log")'
+      # k8s: CNPG Postgres — ONE route for ALL clusters. The CNPG main container is
+      # always named `postgres`, so logs.k8s.<ns>.postgres uniquely identifies every
+      # cluster across all namespaces (authentik/litellm/artifactapi/woodpecker/
+      # puppet/paperclip/grafana/netbox/gitea/encapi). Mutually exclusive because the
+      # app routes above/below carve out `.postgres`.
+      cnpg_pg: 'starts_with(to_string(.subject) ?? "", "logs.k8s.") && ends_with(to_string(.subject) ?? "", ".postgres")'
+      # Gitea router/access logs. k8s: the new k8s gitea (ns gitea) with router +
+      # access logging enabled in the overlay values in this change — carve out
+      # `.postgres` (gitea-namespace CNPG). VM: puppet-managed gitea (file or
+      # journald `gitea`) — puppet-side log-format enable is a follow-up.
+      gitea: '(starts_with(to_string(.subject) ?? "", "logs.k8s.gitea.") && !ends_with(to_string(.subject) ?? "", ".postgres")) || (starts_with(to_string(.subject) ?? "", "logs.vm.") && (contains(to_string(.file) ?? "", "gitea") || (to_string(.SYSLOG_IDENTIFIER) ?? "") == "gitea" || (to_string(.program) ?? "") == "gitea" || (to_string(.appname) ?? "") == "gitea"))'
+      # PuppetServer / PuppetDB. k8s: openvoxserver/openvoxdb stdout (ns puppet) —
+      # carve out `.postgres` (puppet-namespace CNPG). VM file logs (multiline
+      # logback + puppetserver-access.log) are a puppet-side vector concern (the
+      # multiline join must happen at the edge) — follow-up.
+      puppet: 'starts_with(to_string(.subject) ?? "", "logs.k8s.puppet.") && !ends_with(to_string(.subject) ?? "", ".postgres")'
+      # k8s: LiteLLM request logs — JSON once JSON_LOGS=True (flipped in the litellm
+      # env in this change). Carve out `.postgres` (litellm-namespace CNPG).
+      litellm: 'starts_with(to_string(.subject) ?? "", "logs.k8s.litellm.") && !ends_with(to_string(.subject) ?? "", ".postgres")'
+      # VM: Postfix maillog — journald (SYSLOG_IDENTIFIER postfix/*) or file maillog.
+      postfix: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && (starts_with(to_string(.SYSLOG_IDENTIFIER) ?? "", "postfix") || starts_with(to_string(.program) ?? "", "postfix") || starts_with(to_string(.appname) ?? "", "postfix") || contains(to_string(.file) ?? "", "maillog"))'
+
+  # Stage 2: generic catch-all for everything app_route did not claim.
+  route:
+    type: route
+    inputs:
+      - app_route._unmatched
     route:
       k8s: 'starts_with(to_string(.subject) ?? "", "logs.k8s.")'
       vm: 'starts_with(to_string(.subject) ?? "", "logs.vm.")'
@@ -102,12 +176,707 @@ transforms:
         "fields": {}
       }
 
+  # --- Tier-1 per-app parse transforms (each emits the full logs.raw shape) ---
+
+  # Authentik SSO (k8s, ns authentik) — structlog JSON on stdout.
+  # LIVE NOW: authentik pods already stream to logs.k8s.authentik.*.
+  authentik_parse:
+    type: remap
+    inputs:
+      - app_route.authentik
+    source: |
+      node = to_string(.kubernetes.pod_node_name || "") ?? ""
+      pod = to_string(.kubernetes.pod_name || "") ?? ""
+      container = to_string(.kubernetes.container_name || "") ?? ""
+      strm = to_string(.stream || "") ?? ""
+      raw = to_string(.message || "") ?? ""
+      ev = object(parse_json(raw) ?? {}) ?? {}
+      ts = ev.timestamp || .timestamp || now()
+      user = ""
+      if is_string(ev.user) {
+        user = to_string(ev.user) ?? ""
+      } else if is_object(ev.user) {
+        user = to_string(ev.user.username) ?? ""
+      }
+      fields = compact({
+        "event": to_string(ev.event) ?? "",
+        "action": to_string(ev.action) ?? "",
+        "user": user,
+        "client_ip": to_string(ev.client_ip) ?? "",
+        "result": to_string(ev.result) ?? "",
+        "logger": to_string(ev.logger) ?? ""
+      }, string: true)
+      sev = to_string(ev.level) ?? ""
+      msg = raw
+      if ev.event != null {
+        msg = to_string(ev.event) ?? raw
+      }
+      . = {
+        "timestamp": ts,
+        "host": node,
+        "source": "k8s",
+        "namespace": "authentik",
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": sev,
+        "message": msg,
+        "labels": {"app": "authentik"},
+        "fields": fields
+      }
+
+  # Traefik ingress (k8s, ns traefik-system) — JSON access logs. Non-access
+  # traefik lines (app logs) simply parse to no access fields and keep .message.
+  # geoip on client_ip is a PREREQUISITE (no enrichment table yet — see PR note).
+  traefik_parse:
+    type: remap
+    inputs:
+      - app_route.traefik
+    source: |
+      node = to_string(.kubernetes.pod_node_name || "") ?? ""
+      pod = to_string(.kubernetes.pod_name || "") ?? ""
+      container = to_string(.kubernetes.container_name || "") ?? ""
+      strm = to_string(.stream || "") ?? ""
+      raw = to_string(.message || "") ?? ""
+      ev = object(parse_json(raw) ?? {}) ?? {}
+      ts = ev.StartUTC || ev.time || .timestamp || now()
+      status = ""
+      if ev.DownstreamStatus != null {
+        status = to_string(ev.DownstreamStatus) ?? ""
+      }
+      dur_ns = to_int(ev.Duration) ?? 0
+      dur_ms = ""
+      if dur_ns > 0 {
+        dur_ms = to_string(dur_ns / 1000000)
+      }
+      method = to_string(ev.RequestMethod) ?? ""
+      path = to_string(ev.RequestPath) ?? ""
+      fields = compact({
+        "route": to_string(ev.RouterName) ?? "",
+        "service": to_string(ev.ServiceName) ?? "",
+        "method": method,
+        "path": path,
+        "host": to_string(ev.RequestHost) ?? "",
+        "status": status,
+        "duration_ms": dur_ms,
+        "client_ip": to_string(ev.ClientHost) ?? "",
+        "protocol": to_string(ev.RequestProtocol) ?? ""
+      }, string: true)
+      msg = raw
+      if method != "" {
+        msg = method + " " + path + " " + status
+      }
+      . = {
+        "timestamp": ts,
+        "host": node,
+        "source": "k8s",
+        "namespace": "traefik-system",
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": "",
+        "message": msg,
+        "labels": {"app": "traefik"},
+        "fields": fields
+      }
+
+  # Vault/OpenBao file audit device (VM, /var/log/vault_audit.log) — JSON, one
+  # object per request/response. AWAITING VM VECTOR (in-cluster vault is quiet;
+  # lights up when the puppet vector rollout ships logs.vm.* with .file set).
+  vault_parse:
+    type: remap
+    inputs:
+      - app_route.vault
+    source: |
+      host = to_string(.host || .hostname || "") ?? ""
+      raw = to_string(.message || .msg || "") ?? ""
+      ev = object(parse_json(raw) ?? {}) ?? {}
+      ts = ev.time || .timestamp || .ts || now()
+      auth = object(ev.auth) ?? {}
+      req = object(ev.request) ?? {}
+      fields = compact({
+        "type": to_string(ev.type) ?? "",
+        "display_name": to_string(auth.display_name) ?? "",
+        "operation": to_string(req.operation) ?? "",
+        "path": to_string(req.path) ?? "",
+        "remote_address": to_string(req.remote_address) ?? "",
+        "error": to_string(ev.error) ?? ""
+      }, string: true)
+      op = to_string(req.operation) ?? ""
+      pth = to_string(req.path) ?? ""
+      msg = raw
+      if op != "" || pth != "" {
+        msg = op + " " + pth
+      }
+      . = {
+        "timestamp": ts,
+        "host": host,
+        "source": "vm",
+        "namespace": "",
+        "pod": "",
+        "container": "",
+        "stream": "",
+        "severity": "",
+        "message": msg,
+        "labels": {"app": "vault"},
+        "fields": fields
+      }
+
+  # nginx access log (VM) — combined/CLF + optional trailing request_time.
+  # AWAITING VM VECTOR. geoip on client_ip is a PREREQUISITE (see PR note).
+  nginx_access_parse:
+    type: remap
+    inputs:
+      - app_route.nginx_access
+    source: |
+      host = to_string(.host || .hostname || "") ?? ""
+      raw = to_string(.message || .msg || "") ?? ""
+      ts = .timestamp || .ts || now()
+      m = parse_regex(raw, r'^(?P<client_ip>\S+) \S+ (?P<user>\S+) \[(?P<time_local>[^\]]+)\] "(?P<method>\S+) (?P<path>\S+) (?P<protocol>[^"]*)" (?P<status>\d{3}) (?P<bytes>\d+|-) "(?P<referer>[^"]*)" "(?P<user_agent>[^"]*)"(?: (?P<request_time>[\d.]+))?') ?? {}
+      fields = compact({
+        "client_ip": to_string(m.client_ip),
+        "method": to_string(m.method),
+        "path": to_string(m.path),
+        "status": to_string(m.status),
+        "bytes": to_string(m.bytes),
+        "referer": to_string(m.referer),
+        "user_agent": to_string(m.user_agent),
+        "request_time": to_string(m.request_time)
+      }, string: true)
+      . = {
+        "timestamp": ts,
+        "host": host,
+        "source": "vm",
+        "namespace": "",
+        "pod": "",
+        "container": "",
+        "stream": "access",
+        "severity": "",
+        "message": raw,
+        "labels": {"app": "nginx", "log_type": "access"},
+        "fields": fields
+      }
+
+  # nginx error log (VM). AWAITING VM VECTOR.
+  nginx_error_parse:
+    type: remap
+    inputs:
+      - app_route.nginx_error
+    source: |
+      host = to_string(.host || .hostname || "") ?? ""
+      raw = to_string(.message || .msg || "") ?? ""
+      ts = .timestamp || .ts || now()
+      m = parse_regex(raw, r'^(?P<time_local>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?P<level>\w+)\] (?P<pid>\d+)#(?P<tid>\d+): (?:\*(?P<cid>\d+) )?(?P<err>.*)$') ?? {}
+      c = parse_regex(raw, r'client: (?P<client_ip>[0-9a-fA-F:.]+)') ?? {}
+      lvl = to_string(m.level)
+      err = to_string(m.err)
+      fields = compact({
+        "level": lvl,
+        "pid": to_string(m.pid),
+        "cid": to_string(m.cid),
+        "client_ip": to_string(c.client_ip),
+        "error": err
+      }, string: true)
+      msg = raw
+      if err != "" {
+        msg = err
+      }
+      . = {
+        "timestamp": ts,
+        "host": host,
+        "source": "vm",
+        "namespace": "",
+        "pod": "",
+        "container": "",
+        "stream": "error",
+        "severity": lvl,
+        "message": msg,
+        "labels": {"app": "nginx", "log_type": "error"},
+        "fields": fields
+      }
+
+  # HAProxy httplog (VM, journald). AWAITING VM VECTOR.
+  # httplog: %ci:%cp [%tr] %ft %b/%s %Tq/%Tw/%Tc/%Tr/%Tt %ST %B %CC %CS %tsc
+  #          %ac/%fc/%bc/%sc/%rc %sq/%bq {hdrs} "%r"
+  haproxy_parse:
+    type: remap
+    inputs:
+      - app_route.haproxy
+    source: |
+      host = to_string(.host || .hostname || "") ?? ""
+      raw = to_string(.message || .msg || "") ?? ""
+      ts = .timestamp || .ts || now()
+      m = parse_regex(raw, r'(?P<client_ip>\d{1,3}(?:\.\d{1,3}){3}):(?P<client_port>\d+) \[(?P<accept_date>[^\]]+)\] (?P<frontend>\S+) (?P<backend>[^/ ]+)/(?P<server>\S+) (?P<tq>-?\d+)/(?P<tw>-?\d+)/(?P<tc>-?\d+)/(?P<tr>-?\d+)/(?P<tt>[+-]?\d+) (?P<status>\d{3}) (?P<bytes>\d+) \S+ \S+ (?P<termination_state>\S{4}) (?P<actconn>\d+)/(?P<feconn>\d+)/(?P<beconn>\d+)/(?P<srvconn>\d+)/(?P<retries>\d+) (?P<srv_queue>\d+)/(?P<backend_queue>\d+)') ?? {}
+      fields = compact({
+        "client_ip": to_string(m.client_ip),
+        "frontend": to_string(m.frontend),
+        "backend": to_string(m.backend),
+        "server": to_string(m.server),
+        "tq": to_string(m.tq),
+        "tw": to_string(m.tw),
+        "tc": to_string(m.tc),
+        "tr": to_string(m.tr),
+        "tt": to_string(m.tt),
+        "termination_state": to_string(m.termination_state),
+        "retries": to_string(m.retries),
+        "status": to_string(m.status),
+        "bytes": to_string(m.bytes)
+      }, string: true)
+      . = {
+        "timestamp": ts,
+        "host": host,
+        "source": "vm",
+        "namespace": "",
+        "pod": "",
+        "container": "",
+        "stream": "",
+        "severity": "",
+        "message": raw,
+        "labels": {"app": "haproxy"},
+        "fields": fields
+      }
+
+  # glauth LDAP (VM) — structuredlog (logrus) JSON. AWAITING VM VECTOR.
+  glauth_parse:
+    type: remap
+    inputs:
+      - app_route.glauth
+    source: |
+      host = to_string(.host || .hostname || "") ?? ""
+      raw = to_string(.message || .msg || "") ?? ""
+      ev = object(parse_json(raw) ?? {}) ?? {}
+      ts = ev.time || .timestamp || .ts || now()
+      binddn = to_string(ev.bindDN) ?? ""
+      if binddn == "" {
+        binddn = to_string(ev.binddn) ?? ""
+      }
+      remote = to_string(ev.src) ?? ""
+      if remote == "" {
+        remote = to_string(ev.remoteAddr) ?? ""
+      }
+      lvl = to_string(ev.level) ?? ""
+      gmsg = to_string(ev.msg) ?? ""
+      success = "false"
+      if contains(downcase(gmsg), "success") || (lvl == "info" && contains(downcase(gmsg), "bind")) {
+        success = "true"
+      }
+      fields = compact({
+        "bindDN": binddn,
+        "remote": remote,
+        "success": success,
+        "level": lvl,
+        "msg": gmsg
+      }, string: true)
+      msg = gmsg
+      if msg == "" {
+        msg = raw
+      }
+      . = {
+        "timestamp": ts,
+        "host": host,
+        "source": "vm",
+        "namespace": "",
+        "pod": "",
+        "container": "",
+        "stream": "",
+        "severity": lvl,
+        "message": msg,
+        "labels": {"app": "glauth"},
+        "fields": fields
+      }
+
+  # --- Tier-2 per-app parse transforms (stacks on #318) ---
+
+  # BIND query logs (k8s bind-* namespaces + VM named). LIVE on k8s once the
+  # `querylog yes` extraOptions (this change) roll out; VM AWAITS the puppet-side
+  # enable (profiles/dns/server.pp). rcode is NOT present in standard query-log
+  # lines (that needs response logging / dnstap) — extracted only if a
+  # response-style `status:` line is seen. Non-query lines keep .message.
+  bind_query_parse:
+    type: remap
+    inputs:
+      - app_route.bind_query
+    source: |
+      subj = to_string(.subject) ?? ""
+      is_k8s = starts_with(subj, "logs.k8s.")
+      raw = to_string(.message || .msg || "") ?? ""
+      ts = .timestamp || .ts || now()
+      node = ""
+      ns = ""
+      pod = ""
+      container = ""
+      strm = ""
+      hostv = ""
+      src = "vm"
+      if is_k8s {
+        src = "k8s"
+        node = to_string(.kubernetes.pod_node_name || "") ?? ""
+        ns = to_string(.kubernetes.pod_namespace || "") ?? ""
+        pod = to_string(.kubernetes.pod_name || "") ?? ""
+        container = to_string(.kubernetes.container_name || "") ?? ""
+        strm = to_string(.stream || "") ?? ""
+        hostv = node
+      } else {
+        hostv = to_string(.host || .hostname || "") ?? ""
+      }
+      m = parse_regex(raw, r'client\s+(?:@\S+\s+)?(?P<client_ip>[0-9a-fA-F:.]+)#(?P<port>\d+)(?:\s+\([^)]*\))?:\s+(?:view\s+(?P<view>\S+):\s+)?query:\s+(?P<qname>\S+)\s+(?P<qclass>\S+)\s+(?P<qtype>\S+)(?:\s+(?P<flags>\S+))?') ?? {}
+      rc = parse_regex(raw, r'status:\s+(?P<rcode>\w+)') ?? {}
+      fields = compact({
+        "client_ip": to_string(m.client_ip),
+        "qname": to_string(m.qname),
+        "qtype": to_string(m.qtype),
+        "qclass": to_string(m.qclass),
+        "view": to_string(m.view),
+        "flags": to_string(m.flags),
+        "rcode": to_string(rc.rcode)
+      }, string: true)
+      qn = to_string(m.qname)
+      msg = raw
+      if qn != "" {
+        msg = "query " + qn + " " + to_string(m.qtype)
+      }
+      . = {
+        "timestamp": ts,
+        "host": hostv,
+        "source": src,
+        "namespace": ns,
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": "",
+        "message": msg,
+        "labels": {"app": "bind"},
+        "fields": fields
+      }
+
+  # Rancher audit log (k8s, cattle-system rancher-audit-log sidecar) — JSON,
+  # auditLog level 1 (already enabled in the overlay). LIVE NOW.
+  rancher_audit_parse:
+    type: remap
+    inputs:
+      - app_route.rancher_audit
+    source: |
+      node = to_string(.kubernetes.pod_node_name || "") ?? ""
+      pod = to_string(.kubernetes.pod_name || "") ?? ""
+      container = to_string(.kubernetes.container_name || "") ?? ""
+      strm = to_string(.stream || "") ?? ""
+      raw = to_string(.message || "") ?? ""
+      ev = object(parse_json(raw) ?? {}) ?? {}
+      ts = ev.requestTimestamp || ev.time || .timestamp || now()
+      user = ""
+      if is_object(ev.user) {
+        user = to_string(ev.user.name) ?? ""
+      } else if is_string(ev.user) {
+        user = to_string(ev.user) ?? ""
+      }
+      verb = to_string(ev.method) ?? ""
+      if verb == "" { verb = to_string(ev.verb) ?? "" }
+      uri = to_string(ev.requestURI) ?? ""
+      if uri == "" { uri = to_string(ev.uri) ?? "" }
+      status = ""
+      if ev.responseCode != null { status = to_string(ev.responseCode) ?? "" }
+      if status == "" && is_object(ev.responseStatus) { status = to_string(ev.responseStatus.code) ?? "" }
+      fields = compact({
+        "user": user,
+        "verb": verb,
+        "uri": uri,
+        "status": status,
+        "auditID": to_string(ev.auditID) ?? "",
+        "remote_addr": to_string(ev.remoteAddr) ?? ""
+      }, string: true)
+      msg = raw
+      if verb != "" || uri != "" {
+        msg = verb + " " + uri + " " + status
+      }
+      . = {
+        "timestamp": ts,
+        "host": node,
+        "source": "k8s",
+        "namespace": "cattle-system",
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": "",
+        "message": msg,
+        "labels": {"app": "rancher", "log_type": "audit"},
+        "fields": fields
+      }
+
+  # CNPG Postgres — ONE transform for ALL clusters (10 namespaces). The instance
+  # manager wraps postgres logs as JSON on stdout; the postgres CSV columns nest
+  # under `.record` (logger == "postgres"). Non-postgres lines (instance-manager
+  # operator logs) keep .message and set no PG fields. LIVE NOW.
+  cnpg_pg_parse:
+    type: remap
+    inputs:
+      - app_route.cnpg_pg
+    source: |
+      node = to_string(.kubernetes.pod_node_name || "") ?? ""
+      ns = to_string(.kubernetes.pod_namespace || "") ?? ""
+      pod = to_string(.kubernetes.pod_name || "") ?? ""
+      container = to_string(.kubernetes.container_name || "") ?? ""
+      strm = to_string(.stream || "") ?? ""
+      raw = to_string(.message || "") ?? ""
+      cluster = to_string(.kubernetes.pod_labels."cnpg.io/cluster" || "") ?? ""
+      ev = object(parse_json(raw) ?? {}) ?? {}
+      ts = .timestamp || now()
+      rec = object(ev.record) ?? {}
+      logger = to_string(ev.logger) ?? ""
+      sev = ""
+      pgmsg = ""
+      fields = {}
+      if logger == "postgres" {
+        sev = to_string(rec.error_severity) ?? ""
+        pgmsg = to_string(rec.message) ?? ""
+        dm = parse_regex(pgmsg, r'duration:\s+(?P<ms>[0-9.]+)\s+ms') ?? {}
+        fields = compact({
+          "error_severity": sev,
+          "message": pgmsg,
+          "query": to_string(rec.query) ?? "",
+          "duration_ms": to_string(dm.ms),
+          "user": to_string(rec.user_name) ?? "",
+          "database": to_string(rec.database_name) ?? ""
+        }, string: true)
+      }
+      lbls = {"app": "cnpg"}
+      if cluster != "" {
+        lbls = {"app": "cnpg", "cluster": cluster}
+      }
+      msg = raw
+      if pgmsg != "" { msg = pgmsg }
+      . = {
+        "timestamp": ts,
+        "host": node,
+        "source": "k8s",
+        "namespace": ns,
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": sev,
+        "message": msg,
+        "labels": lbls,
+        "fields": fields
+      }
+
+  # Gitea router/access logs (k8s gitea + VM gitea). Router "completed" lines give
+  # method/path/status/latency; NCSA access lines give method/path/status/user.
+  # k8s LIVE once the overlay log config (this change) rolls out; VM AWAITS the
+  # puppet-side log-format enable.
+  gitea_parse:
+    type: remap
+    inputs:
+      - app_route.gitea
+    source: |
+      subj = to_string(.subject) ?? ""
+      is_k8s = starts_with(subj, "logs.k8s.")
+      raw = to_string(.message || .msg || "") ?? ""
+      ts = .timestamp || .ts || now()
+      node = ""
+      ns = ""
+      pod = ""
+      container = ""
+      strm = ""
+      hostv = ""
+      src = "vm"
+      if is_k8s {
+        src = "k8s"
+        node = to_string(.kubernetes.pod_node_name || "") ?? ""
+        ns = to_string(.kubernetes.pod_namespace || "") ?? ""
+        pod = to_string(.kubernetes.pod_name || "") ?? ""
+        container = to_string(.kubernetes.container_name || "") ?? ""
+        strm = to_string(.stream || "") ?? ""
+        hostv = node
+      } else {
+        hostv = to_string(.host || .hostname || "") ?? ""
+      }
+      r = parse_regex(raw, r'completed (?P<method>\S+) (?P<path>\S+) for (?P<client>\S+), (?P<status>\d{3}) [^ ]+ in (?P<latency>[0-9.]+\w+)') ?? {}
+      a = parse_regex(raw, r'^(?P<client_ip>\S+) \S+ (?P<user>\S+) \[[^\]]+\] "(?P<method>\S+) (?P<path>\S+) [^"]*" (?P<status>\d{3})') ?? {}
+      method = to_string(r.method)
+      if method == "" { method = to_string(a.method) }
+      path = to_string(r.path)
+      if path == "" { path = to_string(a.path) }
+      status = to_string(r.status)
+      if status == "" { status = to_string(a.status) }
+      user = to_string(a.user)
+      if user == "-" { user = "" }
+      fields = compact({
+        "method": method,
+        "path": path,
+        "status": status,
+        "latency": to_string(r.latency),
+        "user": user,
+        "client_ip": to_string(a.client_ip)
+      }, string: true)
+      msg = raw
+      if method != "" {
+        msg = method + " " + path + " " + status
+      }
+      . = {
+        "timestamp": ts,
+        "host": hostv,
+        "source": src,
+        "namespace": ns,
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": "",
+        "message": msg,
+        "labels": {"app": "gitea"},
+        "fields": fields
+      }
+
+  # PuppetServer / PuppetDB (k8s openvoxserver/openvoxdb stdout, ns puppet). Per
+  # line logback parse (level/logger/message + node) and an access-log line
+  # (method/status/node) where present. VM multiline stacktrace join +
+  # puppetserver-access.log are a puppet-side edge concern (follow-up). LIVE NOW.
+  puppet_parse:
+    type: remap
+    inputs:
+      - app_route.puppet
+    source: |
+      node = to_string(.kubernetes.pod_node_name || "") ?? ""
+      pod = to_string(.kubernetes.pod_name || "") ?? ""
+      container = to_string(.kubernetes.container_name || "") ?? ""
+      strm = to_string(.stream || "") ?? ""
+      raw = to_string(.message || "") ?? ""
+      ts = .timestamp || now()
+      lb = parse_regex(raw, r'^(?P<ts>\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}[,.]\d+)\s+(?P<level>[A-Z]+)\s+\[(?P<thread>[^\]]*)\]\s+\[(?P<logger>[^\]]*)\]\s+(?P<msg>.*)$') ?? {}
+      ac = parse_regex(raw, r'^(?P<client_ip>\S+) \S+ \S+ \[[^\]]+\] "(?P<method>\S+) (?P<path>\S+) [^"]*" (?P<status>\d{3})') ?? {}
+      nd = parse_regex(raw, r'(?:catalog for|for node)\s+(?P<node>[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+)') ?? {}
+      lvl = to_string(lb.level)
+      pmsg = to_string(lb.msg)
+      err = ""
+      if lvl == "ERROR" { err = pmsg }
+      fields = compact({
+        "level": lvl,
+        "logger": to_string(lb.logger),
+        "node": to_string(nd.node),
+        "method": to_string(ac.method),
+        "status": to_string(ac.status),
+        "path": to_string(ac.path),
+        "client_ip": to_string(ac.client_ip),
+        "error": err
+      }, string: true)
+      msg = raw
+      if pmsg != "" { msg = pmsg }
+      . = {
+        "timestamp": ts,
+        "host": node,
+        "source": "k8s",
+        "namespace": "puppet",
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": lvl,
+        "message": msg,
+        "labels": {"app": "puppet"},
+        "fields": fields
+      }
+
+  # LiteLLM request logs (k8s) — JSON once JSON_LOGS=True (flipped in the litellm
+  # env this change). parse_json -> model/tokens/latency/key/status (best-effort
+  # against litellm's JSON schema); non-JSON lines keep .message. Field keys light
+  # up once the env flip rolls out.
+  litellm_parse:
+    type: remap
+    inputs:
+      - app_route.litellm
+    source: |
+      node = to_string(.kubernetes.pod_node_name || "") ?? ""
+      pod = to_string(.kubernetes.pod_name || "") ?? ""
+      container = to_string(.kubernetes.container_name || "") ?? ""
+      strm = to_string(.stream || "") ?? ""
+      raw = to_string(.message || "") ?? ""
+      ev = object(parse_json(raw) ?? {}) ?? {}
+      ts = ev.timestamp || .timestamp || now()
+      sev = to_string(ev.level) ?? ""
+      lmsg = to_string(ev.message) ?? ""
+      fields = compact({
+        "model": to_string(ev.model) ?? "",
+        "tokens": to_string(ev.total_tokens) ?? "",
+        "latency": to_string(ev.response_time) ?? "",
+        "key": to_string(ev.api_key) ?? "",
+        "status": to_string(ev.status) ?? "",
+        "user": to_string(ev.user) ?? ""
+      }, string: true)
+      msg = raw
+      if lmsg != "" { msg = lmsg }
+      . = {
+        "timestamp": ts,
+        "host": node,
+        "source": "k8s",
+        "namespace": "litellm",
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": sev,
+        "message": msg,
+        "labels": {"app": "litellm"},
+        "fields": fields
+      }
+
+  # Postfix maillog (VM) — best-effort PER-LINE parse (qid + from/to/status/relay/
+  # delay). Full qid-lifecycle correlation is a query-time GROUP BY qid in
+  # ClickHouse, NOT a stateless-aggregator job (stitching the multi-line lifecycle
+  # needs a stateful reduce). AWAITS VM VECTOR.
+  postfix_parse:
+    type: remap
+    inputs:
+      - app_route.postfix
+    source: |
+      host = to_string(.host || .hostname || "") ?? ""
+      raw = to_string(.message || .msg || "") ?? ""
+      ts = .timestamp || .ts || now()
+      prog = to_string(.SYSLOG_IDENTIFIER || .program || .appname || "") ?? ""
+      q = parse_regex(raw, r'^(?P<qid>[0-9A-F]{6,}):') ?? {}
+      frm = parse_regex(raw, r'from=<(?P<from>[^>]*)>') ?? {}
+      rcpt = parse_regex(raw, r'to=<(?P<to>[^>]*)>') ?? {}
+      st = parse_regex(raw, r'status=(?P<status>\w+)') ?? {}
+      rel = parse_regex(raw, r'relay=(?P<relay>[^,]+)') ?? {}
+      dly = parse_regex(raw, r'delay=(?P<delay>[0-9.]+)') ?? {}
+      fields = compact({
+        "qid": to_string(q.qid),
+        "from": to_string(frm.from),
+        "to": to_string(rcpt.to),
+        "status": to_string(st.status),
+        "relay": to_string(rel.relay),
+        "delay": to_string(dly.delay),
+        "program": prog
+      }, string: true)
+      . = {
+        "timestamp": ts,
+        "host": host,
+        "source": "vm",
+        "namespace": "",
+        "pod": "",
+        "container": "",
+        "stream": "",
+        "severity": "",
+        "message": raw,
+        "labels": {"app": "postfix"},
+        "fields": fields
+      }
+
 sinks:
   clickhouse:
     type: clickhouse
     inputs:
       - k8s_shape
       - vm_shape
+      - authentik_parse
+      - traefik_parse
+      - vault_parse
+      - nginx_access_parse
+      - nginx_error_parse
+      - haproxy_parse
+      - glauth_parse
+      - bind_query_parse
+      - rancher_audit_parse
+      - cnpg_pg_parse
+      - gitea_parse
+      - puppet_parse
+      - litellm_parse
+      - postfix_parse
     endpoint: http://clickhouse-logs.logging.svc.cluster.local:8123
     database: logs
     table: raw

apps/base/logging/vector/archiver.yaml

@@ -1,62 +0,0 @@
----
-# Vector ARCHIVER tier — long-term raw-log backup to S3 (Ceph RGW). Independent
-# durable JetStream consumer (`archiver`) so its offsets/lag are fully isolated
-# from the ClickHouse transform path (archive lag can never stall ingest — true
-# fan-out). Writes RAW, pre-transform events (as they sit in JetStream) as
-# gzipped NDJSON, partitioned by subject + date. This is the long-horizon replay
-# source beyond JetStream's 3d retention window.
-data_dir: /vector-data-dir
-
-api:
-  enabled: true
-  address: 0.0.0.0:8686
-
-sources:
-  js_archive:
-    type: nats
-    url: nats://nats.logging.svc.cluster.local:4222
-    connection_name: vector-archiver
-    subject: "logs.>"
-    jetstream:
-      stream: LOGS
-      consumer: archiver
-    auth:
-      strategy: user_password
-      user_password:
-        user: log-consumer
-        password: ${NATS_CONSUMER_PASSWORD}
-    decoding:
-      codec: json
-
-sinks:
-  s3:
-    type: aws_s3
-    inputs:
-      - js_archive
-    bucket: logs-archive
-    endpoint: https://s3.ceph.unkin.net
-    region: us-east-1
-    force_path_style: true
-    tls:
-      ca_file: /etc/vault-ca/ca.crt
-    # AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY come from the logs-archive-s3
-    # Secret (cephrgw-operator) via envFrom on the deployment.
-    key_prefix: "raw/{{ subject }}/%Y/%m/%d/"
-    compression: gzip
-    encoding:
-      codec: json
-    framing:
-      method: newline_delimited
-    filename_time_format: "%Y%m%dT%H%M%SZ"
-    filename_append_uuid: true
-    batch:
-      max_bytes: 134217728
-      timeout_secs: 300
-    buffer:
-      type: memory
-      max_events: 5000
-      when_full: block
-    # Disabled so slow BucketAccess credential propagation doesn't crash-loop
-    # the pod; RGW reachability is proven by the operator's own health.
-    healthcheck:
-      enabled: false

apps/base/netbox/cnpg_backup.yaml

@@ -25,6 +25,7 @@ metadata:
   name: cnpg-netbox
   namespace: netbox
 spec:
+  placementTarget: ec
   bucketName: cnpg-netbox
   ownerRef: cnpg-netbox-backup
   versioning: false

apps/base/netbox/vaultstaticsecret.yaml

@@ -19,7 +19,13 @@ spec:
   type: kv-v2
   vaultAuthRef: default
 ---
-# Django SECRET_KEY (key: secret_key). One-time Vault seed.
+# Config secret. Keys:
+#   secret_key         : Django SECRET_KEY (50+ random chars). One-time Vault seed.
+#   api_token_peppers  : JSON object {"1": "<50+ char random>"} used to HMAC-hash
+#                        v2 API tokens. One-time Vault seed — rotating a pepper
+#                        invalidates existing v2 tokens, so set it once.
+# VSO syncs every key at the path into the destination Secret, and the NetBox
+# chart mounts both keys from it (existingSecret) — no explicit key mapping needed.
 apiVersion: secrets.hashicorp.com/v1beta1
 kind: VaultStaticSecret
 metadata:

apps/base/paperclip/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-paperclip
   namespace: paperclip
 spec:
+  placementTarget: ec
   bucketName: cnpg-paperclip
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.

apps/base/paperclip/cnpg_cluster.yaml

@@ -48,7 +48,7 @@ spec:
   enablePDB: true
   enableSuperuserAccess: false
   failoverDelay: 0
-  imageName: ghcr.io/cloudnative-pg/postgresql:17-minimal-trixie
+  imageName: ghcr.io/cloudnative-pg/postgresql:17-system-trixie
   instances: 3
   logLevel: info
   maxSyncReplicas: 0

apps/base/pdbmux/deployment.yaml

@@ -5,7 +5,7 @@ metadata:
   name: pdbmux
   namespace: pdbmux
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
 spec:
   replicas: 2
   selector:
@@ -25,7 +25,7 @@ spec:
         - name: pdbmux
           # Image is published by the pdbmux repo's .woodpecker/docker.yaml on
           # a v* tag. It only exists after that tag is cut (see PR merge gates).
-          image: git.unkin.net/unkin/pdbmux:v0.1.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/pdbmux:v0.1.0
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8080

apps/base/puppet/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-puppet
   namespace: puppet
 spec:
+  placementTarget: ec
   bucketName: cnpg-puppet
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.

apps/base/puppet/cnpg_cluster.yaml

@@ -69,7 +69,7 @@ spec:
   enablePDB: true
   enableSuperuserAccess: false
   failoverDelay: 0
-  imageName: ghcr.io/cloudnative-pg/postgresql:17-minimal-trixie
+  imageName: ghcr.io/cloudnative-pg/postgresql:17-system-trixie
   instances: 3
   logLevel: info
   maxSyncReplicas: 0

apps/base/puppet/cronjob_generate-types.yaml

@@ -52,20 +52,16 @@ spec:
               securityContext:
                 runAsUser: 0
                 runAsNonRoot: false
+                allowPrivilegeEscalation: false
+                # Root to `gem install` into the image's root-owned gem dir and
+                # to `runuser` for `puppet generate types` (SETUID/SETGID).
                 capabilities:
                   add:
-                    - CAP_CHOWN
-                    - CAP_SETUID
-                    - CAP_SETGID
-                    - CAP_DAC_OVERRIDE
-                    - CAP_AUDIT_WRITE
-                    - CAP_FOWNER
                     - CHOWN
-                    - SETUID
-                    - SETGID
                     - DAC_OVERRIDE
-                    - AUDIT_WRITE
                     - FOWNER
+                    - SETGID
+                    - SETUID
                   drop:
                     - all
               volumeMounts:
@@ -76,6 +72,8 @@ spec:
           restartPolicy: OnFailure
           securityContext:
             fsGroup: 999
+            seccompProfile:
+              type: RuntimeDefault
           volumes:
             - name: puppet-code-volume
               persistentVolumeClaim:

apps/base/puppet/deployment_puppetboard.yaml

@@ -21,7 +21,7 @@ spec:
   template:
     metadata:
       annotations:
-        reloader.stakater.com/auto: "true"
+        configmap.reloader.stakater.com/auto: "true"
       labels:
         app.kubernetes.io/component: puppetboard
         app.kubernetes.io/instance: puppetserver
@@ -29,6 +29,11 @@ spec:
         app.kubernetes.io/version: 8.8.0
     spec:
       enableServiceLinks: false
+      securityContext:
+        fsGroup: 1000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
       initContainers:
         - name: wait-puppetserver
           image: curlimages/curl:8.11.1
@@ -115,9 +120,6 @@ spec:
               chmod 600 ${CERT_DIR}/${HOSTNAME}.key
               chmod 644 ${CERT_DIR}/ca.pem
 
-              # Change ownership to puppetboard user (1000:1000)
-              chown -R 1000:1000 ${CERT_DIR}
-
               echo "Certificate generation completed for ${HOSTNAME}"
           volumeMounts:
             - name: puppetboard-certs
@@ -130,9 +132,13 @@ spec:
               cpu: 50m
               memory: 64Mi
           securityContext:
-            runAsUser: 0
-            runAsGroup: 0
-            allowPrivilegeEscalation: true
+            runAsUser: 1000
+            runAsGroup: 1000
+            runAsNonRoot: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - all
       containers:
         - name: puppetboard
           image: ghcr.io/voxpupuli/puppetboard:7.0.1

apps/base/puppet/deployment_puppetdb.yaml

@@ -20,7 +20,7 @@ spec:
   template:
     metadata:
       annotations:
-        reloader.stakater.com/auto: "true"
+        configmap.reloader.stakater.com/auto: "true"
       labels:
         app.kubernetes.io/component: puppetdb
         app.kubernetes.io/instance: puppetserver
@@ -49,6 +49,10 @@ spec:
             - configMapRef:
                 name: puppetdb-config
           env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
             - name: OPENVOXDB_POSTGRES_PASSWORD
               valueFrom:
                 secretKeyRef:
@@ -71,23 +75,24 @@ spec:
                   name: postgres-read-credentials
           securityContext:
             allowPrivilegeEscalation: false
+            # Root entrypoint drops to the puppetdb user via `runuser` (needs
+            # SETUID/SETGID) after chowning SSL/data dirs (CHOWN). Cannot run
+            # non-root: the image entrypoint requires a root start.
             capabilities:
               add:
-                - CAP_FOWNER
-                - CAP_CHOWN
-                - CAP_SETUID
-                - CAP_SETGID
-                - CAP_DAC_OVERRIDE
-                - FOWNER
                 - CHOWN
-                - SETUID
-                - SETGID
                 - DAC_OVERRIDE
+                - FOWNER
+                - SETGID
+                - SETUID
               drop:
                 - all
           volumeMounts:
             - mountPath: /opt/puppetlabs/server/data/puppetdb
               name: puppetdb-storage
+            - mountPath: /opt/puppetlabs/server/data/puppetdb/stockpile
+              name: puppetdb-storage
+              subPathExpr: stockpile/$(POD_NAME)
             - mountPath: /etc/puppetlabs/puppetdb/conf.d/read-database.conf
               name: puppetdb-read-database-conf
               subPath: read-database.conf
@@ -98,7 +103,12 @@ spec:
             - sh
             - -c
           args:
-            - mkdir -p /opt/puppetlabs/server/data/puppetdb/logs && chown 999:999 /opt/puppetlabs/server/data/puppetdb/logs
+            - mkdir -p /opt/puppetlabs/server/data/puppetdb/logs /opt/puppetlabs/server/data/puppetdb/stockpile
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
           resources:
             limits:
               cpu: 20m
@@ -107,10 +117,19 @@ spec:
               cpu: 20m
               memory: 32Mi
           securityContext:
-            runAsUser: 0
+            runAsUser: 999
+            runAsGroup: 999
+            runAsNonRoot: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - all
           volumeMounts:
             - mountPath: /opt/puppetlabs/server/data/puppetdb
               name: puppetdb-storage
+            - mountPath: /opt/puppetlabs/server/data/puppetdb/stockpile
+              name: puppetdb-storage
+              subPathExpr: stockpile/$(POD_NAME)
 
         - name: pgchecker
           image: docker.io/busybox:1.37
@@ -163,6 +182,11 @@ spec:
             runAsGroup: 1000
             runAsNonRoot: true
             allowPrivilegeEscalation: false
+      securityContext:
+        fsGroup: 999
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
       volumes:
         - name: puppetdb-storage
           persistentVolumeClaim:

apps/base/puppet/deployment_puppetserver-compiler.yaml

@@ -2,7 +2,8 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "vault-ca-cert"
   labels:
     app.kubernetes.io/component: puppetserver-compilers
     app.kubernetes.io/instance: puppetserver
@@ -65,20 +66,16 @@ spec:
           timeoutSeconds: 20
         securityContext:
           allowPrivilegeEscalation: false
+          # Root entrypoint chowns baked-in dirs (CHOWN) then drops the JVM to
+          # the puppet user via `runuser` (needs SETUID/SETGID). Cannot run
+          # non-root: the image entrypoint requires a root start.
           capabilities:
             add:
-            - CAP_CHOWN
-            - CAP_SETUID
-            - CAP_SETGID
-            - CAP_DAC_OVERRIDE
-            - CAP_AUDIT_WRITE
-            - CAP_FOWNER
             - CHOWN
-            - SETUID
-            - SETGID
             - DAC_OVERRIDE
-            - AUDIT_WRITE
             - FOWNER
+            - SETGID
+            - SETUID
             drop:
             - all
         startupProbe:
@@ -158,19 +155,13 @@ spec:
         securityContext:
           runAsUser: 0
           runAsNonRoot: false
+          allowPrivilegeEscalation: false
+          # Runs as root to chown the mounted PVC dirs to puppet:puppet before
+          # the main container starts (CHOWN); does not drop privileges itself.
           capabilities:
             add:
-            - CAP_CHOWN
-            - CAP_SETUID
-            - CAP_SETGID
-            - CAP_DAC_OVERRIDE
-            - CAP_AUDIT_WRITE
-            - CAP_FOWNER
             - CHOWN
-            - SETUID
-            - SETGID
             - DAC_OVERRIDE
-            - AUDIT_WRITE
             - FOWNER
             drop:
             - all
@@ -211,6 +202,8 @@ spec:
           name: puppet-shared-bins
       securityContext:
         fsGroup: 999
+        seccompProfile:
+          type: RuntimeDefault
       volumes:
       - name: puppet-code-volume
         persistentVolumeClaim:

apps/base/puppet/deployment_puppetserver-master.yaml

@@ -2,7 +2,8 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "vault-ca-cert"
   labels:
     app.kubernetes.io/component: puppetserver
     app.kubernetes.io/instance: puppetserver
@@ -11,16 +12,17 @@ metadata:
   name: puppetserver-master
   namespace: puppet
 spec:
+  replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/component: puppetserver
       app.kubernetes.io/name: puppetserver
   strategy:
-    type: RollingUpdate
+    type: Recreate
   template:
     metadata:
       annotations:
-        reloader.stakater.com/auto: "true"
+        configmap.reloader.stakater.com/auto: "true"
       labels:
         app.kubernetes.io/component: puppetserver
         app.kubernetes.io/instance: puppetserver
@@ -64,20 +66,16 @@ spec:
             timeoutSeconds: 20
           securityContext:
             allowPrivilegeEscalation: false
+            # Root entrypoint chowns baked-in dirs (CHOWN) then drops the JVM to
+            # the puppet user via `runuser` (needs SETUID/SETGID). Cannot run
+            # non-root: the image entrypoint requires a root start.
             capabilities:
               add:
-                - CAP_CHOWN
-                - CAP_SETUID
-                - CAP_SETGID
-                - CAP_DAC_OVERRIDE
-                - CAP_AUDIT_WRITE
-                - CAP_FOWNER
                 - CHOWN
-                - SETUID
-                - SETGID
                 - DAC_OVERRIDE
-                - AUDIT_WRITE
                 - FOWNER
+                - SETGID
+                - SETUID
               drop:
                 - all
           startupProbe:
@@ -131,19 +129,13 @@ spec:
           securityContext:
             runAsUser: 0
             runAsNonRoot: false
+            allowPrivilegeEscalation: false
+            # Runs as root to chown the mounted PVC dirs to puppet:puppet before
+            # the main container starts (CHOWN); does not drop privileges itself.
             capabilities:
               add:
-                - CAP_CHOWN
-                - CAP_SETUID
-                - CAP_SETGID
-                - CAP_DAC_OVERRIDE
-                - CAP_AUDIT_WRITE
-                - CAP_FOWNER
                 - CHOWN
-                - SETUID
-                - SETGID
                 - DAC_OVERRIDE
-                - AUDIT_WRITE
                 - FOWNER
               drop:
                 - all
@@ -155,6 +147,8 @@ spec:
               subPath: check_for_masters.sh
       securityContext:
         fsGroup: 999
+        seccompProfile:
+          type: RuntimeDefault
       volumes:
         - name: puppet-ca-storage
           persistentVolumeClaim:

apps/base/puppet/horizontalpodautoscaler_puppetserver-masters-autoscaler.yaml

@@ -1,37 +0,0 @@
-apiVersion: autoscaling/v2
-kind: HorizontalPodAutoscaler
-metadata:
-  labels:
-    app.kubernetes.io/component: puppetserver
-    app.kubernetes.io/instance: puppetserver
-    app.kubernetes.io/name: puppetserver
-    app.kubernetes.io/version: 8.8.0
-  name: puppetserver-masters-autoscaler
-  namespace: puppet
-spec:
-  scaleTargetRef:
-    apiVersion: apps/v1
-    kind: Deployment
-    name: puppetserver-master
-  minReplicas: 2
-  maxReplicas: 5
-  metrics:
-  - resource:
-      name: cpu
-      target:
-        averageUtilization: 75
-        type: Utilization
-    type: Resource
-  behavior:
-    scaleUp:
-      stabilizationWindowSeconds: 60
-      policies:
-      - type: Percent
-        value: 50
-        periodSeconds: 15
-    scaleDown:
-      stabilizationWindowSeconds: 300
-      policies:
-      - type: Percent
-        value: 25
-        periodSeconds: 60

apps/base/puppet/kustomization.yaml

@@ -23,7 +23,6 @@ resources:
   - deployment_puppetdb.yaml
   - deployment_puppetserver-master.yaml
   - horizontalpodautoscaler_puppetserver-compilers-autoscaler.yaml
-  - horizontalpodautoscaler_puppetserver-masters-autoscaler.yaml
   - horizontalpodautoscaler_puppetserver-puppetboard-autoscaler.yaml
   - horizontalpodautoscaler_puppetserver-puppetdb-autoscaler.yaml
   - gateway_puppetboard.yaml

apps/base/puppet/vpa.yaml

@@ -52,9 +52,9 @@ kind: VerticalPodAutoscaler
 metadata:
   name: puppetserver-master-vpa
   namespace: puppet
-  # NOTE: this workload also has an HPA. updateMode Off is recommendation-only
-  # and does not act, so there is no HPA/VPA conflict today. Do not flip to Auto/
-  # Initial without first moving the HPA off CPU/memory (VPA owns those under Auto).
+  # NOTE: the master is a pinned single replica (Recreate, no HPA) so the CA/master
+  # never coexists. updateMode Off keeps this recommendation-only; do not flip to
+  # Auto/Initial, which would evict and briefly recreate the singleton pod.
 spec:
   targetRef:
     apiVersion: apps/v1

apps/base/woodpecker/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-woodpecker
   namespace: woodpecker
 spec:
+  placementTarget: ec
   bucketName: cnpg-woodpecker
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.