traefik-internal: add ldaps :636 entrypoint for authentik LDAPS

Why:
- traefik-internal has no 636 entrypoint, so the authentik-ldap Gateway's
  636/TLS passthrough listeners are invalid and the whole Gateway is
  rejected (Accepted: False), taking LDAPS down.
- The traefik-internal LoadBalancer only publishes 80/443, so nothing
  reaches the Authentik LDAP outpost on 636.

How:
- Add an `ldaps` entrypoint on :636 (raw TCP, no TLS termination) to the
  traefik-internal helm values so 636/TLS passthrough streams pass through
  to the LDAP outpost.
- Expose 636 on the traefik-internal LoadBalancer via the port's
  `expose.default`, keeping the pinned 198.18.200.4 / common group.

.woodpecker/vector-test.yaml

@@ -14,9 +14,9 @@ steps:
       # Transform tier + VM ingest: unit-tested transforms.
       - vector test apps/base/logging/vector/aggregator.yaml apps/base/logging/vector/aggregator-tests.yaml
       - vector test apps/base/logging/vector/vm-ingest.yaml apps/base/logging/vector/vm-ingest-tests.yaml
-      # Agent + archiver have no transforms to unit-test; validate they build.
+      # Agent has no transforms to unit-test; validate it builds. (The archiver
+      # leg is now the logarchiver service, not a Vector pipeline.)
       - vector validate --no-environment apps/base/logging/vector/agent.yaml
-      - vector validate --no-environment apps/base/logging/vector/archiver.yaml
     backend_options:
       kubernetes:
         serviceAccountName: default

apps/base/age-api/deployment.yaml

@@ -12,13 +12,13 @@ spec:
   template:
     metadata:
       annotations:
-        reloader.stakater.com/auto: "true"
+        configmap.reloader.stakater.com/auto: "true"
       labels:
         app: age-api
     spec:
       containers:
         - name: age-api
-          image: git.unkin.net/unkin/age-api:v0.1.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/age-api:v0.1.0
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8080

apps/base/artifactapi/api-deployment.yaml

@@ -5,7 +5,8 @@ metadata:
   name: api
   namespace: artifactapi
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "vault-ca-cert"
 spec:
   selector:
     matchLabels:

apps/base/artifactapi/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-artifactapi
   namespace: artifactapi
 spec:
+  placementTarget: ec
   bucketName: cnpg-artifactapi
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.

apps/base/artifactapi/httproute.yaml

@@ -1,30 +1,6 @@
 ---
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
-metadata:
-  name: http-redirect
-  namespace: artifactapi
-spec:
-  hostnames:
-    - artifactapi.k8s.syd1.au.unkin.net
-  parentRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: artifactapi
-      sectionName: http
-  rules:
-    - filters:
-        - type: RequestRedirect
-          requestRedirect:
-            scheme: https
-            statusCode: 301
-      matches:
-        - path:
-            type: PathPrefix
-            value: /
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
 metadata:
   name: api-route
   namespace: artifactapi
@@ -32,6 +8,12 @@ spec:
   hostnames:
     - artifactapi.k8s.syd1.au.unkin.net
   parentRefs:
+    # Early-boot clients (anaconda/kickstart, yum in %post, PXE) need plain HTTP
+    # for the rpm repos; serve the app directly on port 80 instead of redirecting.
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: artifactapi
+      sectionName: http
     - group: gateway.networking.k8s.io
       kind: Gateway
       name: artifactapi

apps/base/artifactapi/ui-deployment.yaml

@@ -5,7 +5,7 @@ metadata:
   name: ui
   namespace: artifactapi
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
 spec:
   selector:
     matchLabels:

apps/base/authentik/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-authentik
   namespace: authentik
 spec:
+  placementTarget: ec
   bucketName: cnpg-authentik
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.

apps/base/bind-external/agent-dns-rolebinding.yaml

@@ -0,0 +1,16 @@
+---
+# Confines the agent-dns service account (in bind-system) to the agent-dns
+# ClusterRole within this namespace.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: agent-dns
+  namespace: bind-external
+subjects:
+  - kind: ServiceAccount
+    name: agent-dns
+    namespace: bind-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: agent-dns

apps/base/bind-external/cluster.yaml

@@ -0,0 +1,52 @@
+---
+# Externally-reachable authoritative BIND for zones we delegate to ourselves.
+# First tenant: acme.unkin.net, the DNS-01 challenge zone Let's Encrypt validates
+# via a one-time _acme-challenge.unkin.net CNAME. Authoritative-only, recursion
+# off, no forwarding, no open transfers -- the primaryService is the single
+# dmz-pinned LoadBalancer that public NAT targets and that cert-manager writes to.
+apiVersion: bind.unkin.net/v1alpha1
+kind: BindCluster
+metadata:
+  name: bind-external
+  namespace: bind-external
+spec:
+  mode: authoritative
+  recursion: false
+  replicas: 2
+  storageClassName: cephrbd-fast-delete
+  storageSize: 1Gi
+  # Public server: answer queries from anywhere (Let's Encrypt validates over the
+  # internet), deny recursion and open zone transfers. localhost + pod net are
+  # implied by "any" and cover in-pod nsupdate and secondary SOA refresh; per-zone
+  # allow-transfer (catalog + acme zone) still permits key-authenticated AXFR.
+  extraOptions:
+    - "allow-query { any; }"
+    - "allow-transfer { none; }"
+  service:
+    type: ClusterIP
+  primaryService:
+    type: LoadBalancer
+    externalTrafficPolicy: Local
+    annotations:
+      purelb.io/service-group: dmz
+      purelb.io/addresses: 198.18.199.53
+      external-dns.alpha.kubernetes.io/hostname: bind-external-primary.k8s.syd1.au.unkin.net
+  resources:
+    requests:
+      cpu: 20m
+      memory: 128Mi
+    limits:
+      cpu: "1"
+      memory: 512Mi
+---
+# Catalog zone so the acme zone replicates onto the secondary (AXFR/IXFR keyed
+# with the certmanager TSIG key, reused here as the transfer key).
+apiVersion: bind.unkin.net/v1alpha1
+kind: BindCatalogZone
+metadata:
+  name: bind-external-catalog
+  namespace: bind-external
+spec:
+  clusterRef: bind-external
+  zoneName: catalog.external
+  transferKeyRef: certmanager

apps/base/bind-external/kustomization.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - cluster.yaml
+  - tsigkey.yaml
+  - zones.yaml
+  - agent-dns-rolebinding.yaml

apps/base/bind-external/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: bind-external

apps/base/bind-external/tsigkey.yaml

@@ -0,0 +1,20 @@
+---
+# TSIG key cert-manager uses to send RFC2136 dynamic updates (the DNS-01 TXT
+# records) to the primary, and that the secondary reuses for AXFR. The operator
+# generates the material into Secret certmanager-tsig in this namespace;
+# secretTemplate stamps emberstack reflector hints so the Secret is mirrored into
+# the cert-manager namespace, where the rfc2136 solver reads its "secret" key.
+apiVersion: bind.unkin.net/v1alpha1
+kind: BindTSIGKey
+metadata:
+  name: certmanager
+  namespace: bind-external
+spec:
+  clusterRef: bind-external
+  algorithm: hmac-sha256
+  secretTemplate:
+    annotations:
+      reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cert-manager"
+      reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cert-manager"

apps/base/bind-external/zones.yaml

@@ -0,0 +1,19 @@
+---
+# Self-delegated ACME challenge zone. Google Cloud DNS holds a one-time
+# _acme-challenge.unkin.net CNAME -> _acme-challenge.acme.unkin.net and an
+# acme.unkin.net NS delegation pointing here; cert-manager writes the challenge
+# TXT records via RFC2136 authenticated with the certmanager key.
+apiVersion: bind.unkin.net/v1alpha1
+kind: BindZone
+metadata:
+  name: acme-unkin-net
+  namespace: bind-external
+spec:
+  clusterRef: bind-external
+  zoneName: acme.unkin.net
+  type: primary
+  defaultTTL: 60
+  dynamicUpdate: true
+  updateKeyRef: certmanager
+  allowTransfer:
+    - key certmanager

apps/base/bind-internal/agent-dns-rolebinding.yaml

@@ -0,0 +1,16 @@
+---
+# Confines the agent-dns service account (in bind-system) to the agent-dns
+# ClusterRole within this namespace.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: agent-dns
+  namespace: bind-internal
+subjects:
+  - kind: ServiceAccount
+    name: agent-dns
+    namespace: bind-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: agent-dns

apps/base/bind-internal/authoritative/cluster.yaml

@@ -18,6 +18,9 @@ spec:
   # without it every dynamic update is "denied due to allow-query".
   extraOptions:
     - "allow-query { localhost; auth-acl-main; 10.42.0.0/16; }"
+    # Enable query logging for the Tier-2 vector bind_query pipeline (see the
+    # resolvers cluster for the routing rationale).
+    - "querylog yes"
   service:
     type: LoadBalancer
     externalTrafficPolicy: Local

apps/base/bind-internal/authoritative/records.yaml

@@ -20,6 +20,32 @@ spec:
     # identity.unkin.net hostname there.
     - 198.18.200.4
 ---
+# PRODUCTION CUTOVER RECORD — intentionally commented out.
+# git.unkin.net currently resolves to the LIVE VM forge (HAProxy VRRP VIP
+# 198.18.19.17), which holds every repo the estate depends on. Uncommenting this
+# repoints the whole org's git.unkin.net at the new k8s Gitea gateway VIP, so it
+# is the FINAL step of the forge migration — gated on the data migration (gitea
+# dump/restore + SECRET_KEY copy) in argocd-apps docs/gitea-migration.md.
+# NOTE: the live git.unkin.net answer is served by the puppet DNS master today
+# (profiles::dns::master, records from PuppetDB); this k8s apex zone holds only
+# SOA+NS + a few DNSRecords so far. Confirm the k8s bind cluster is the live
+# authority for unkin.net (or update the puppet record instead) before relying
+# on this CR at cutover.
+# ---
+# apiVersion: bind.unkin.net/v1alpha1
+# kind: DNSRecord
+# metadata:
+#   name: git-dns-internal
+#   namespace: bind-internal
+# spec:
+#   zoneRef: unkin-net
+#   name: git
+#   type: A
+#   ttl: 600
+#   values:
+#     # traefik-internal gateway VIP; the gitea Gateway serves git.unkin.net there.
+#     - 198.18.200.4
+---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:

apps/base/bind-internal/kustomization.yaml

@@ -8,3 +8,4 @@ resources:
   - resolvers
   - externaldns
   - tsig-api
+  - agent-dns-rolebinding.yaml

apps/base/bind-internal/resolvers/cluster.yaml

@@ -30,6 +30,11 @@ spec:
   # (incl. k8s.syd1.au.unkin.net); 18.198.in-addr.arpa covers every reverse zone.
   extraOptions:
     - "validate-except { unkin.net; 18.198.in-addr.arpa; consul; }"
+    # Enable query logging so the Tier-2 vector bind_query pipeline can parse
+    # client/qname/qtype. Routes to the `queries` category which, with no explicit
+    # logging{} clause, follows the default category to the named foreground
+    # stderr channel -> pod stdout -> vector (subject logs.k8s.bind-internal.*).
+    - "querylog yes"
   resources:
     requests:
       cpu: 20m

apps/base/bind-internal/tsig-api/tsig-api.yaml

@@ -13,7 +13,7 @@ metadata:
   name: bind-tsig-api
   namespace: bind-internal
 spec:
-  image: git.unkin.net/unkin/bind-tsig-api:v0.2.3
+  image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-tsig-api:v0.2.3
   replicas: 1
   port: 8443
   # targetNamespace defaults to this resource's namespace (bind-internal), where

apps/base/bind-system/agent-dns-rbac.yaml

@@ -0,0 +1,38 @@
+---
+# Static service account that Vault's kubernetes secret engine mints scoped
+# tokens for (agent-dns role). RBAC is confined to the bind namespaces via the
+# per-namespace RoleBindings below, not a ClusterRoleBinding.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: agent-dns
+  namespace: bind-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: agent-dns
+rules:
+  - apiGroups: ["bind.unkin.net"]
+    resources: ["*"]
+    verbs: ["*"]
+  - apiGroups: [""]
+    resources: ["pods", "services", "configmaps", "events"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods/log"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: agent-dns
+  namespace: bind-system
+subjects:
+  - kind: ServiceAccount
+    name: agent-dns
+    namespace: bind-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: agent-dns

apps/base/bind-system/deployment.yaml

@@ -21,7 +21,7 @@ spec:
         runAsNonRoot: true
       containers:
         - name: operator
-          image: git.unkin.net/unkin/bind-operator:v0.2.6
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.2.6
           args:
             - --metrics-bind-address=:8080
             - --health-probe-bind-address=:8081

apps/base/bind-system/kustomization.yaml

@@ -8,5 +8,6 @@ resources:
   # vendored here, so they never drift from the operator.
   - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.2.6/config/crd/install.yaml
   - rbac.yaml
+  - agent-dns-rbac.yaml
   - deployment.yaml
   - vpa.yaml

apps/base/cephrgw-system/deployment.yaml

@@ -7,8 +7,10 @@ metadata:
   labels:
     app.kubernetes.io/name: cephrgw-operator
   annotations:
-    # Restart the operator when the credentials Secret rotates.
+    # Restart on internal CA rotation only; cephrgw-credentials is Vault-rotated
-    reloader.stakater.com/auto: "true"
+    # (VSO) and deliberately excluded so routine key rotation causes no restart.
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "vault-ca-cert"
 spec:
   replicas: 1
   selector:
@@ -24,7 +26,7 @@ spec:
         runAsNonRoot: true
       containers:
         - name: operator
-          image: git.unkin.net/unkin/cephrgw-operator:v0.3.1
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/cephrgw-operator:v0.4.0
           args:
             - --metrics-bind-address=:8080
             - --health-probe-bind-address=:8081

apps/base/cephrgw-system/kustomization.yaml

@@ -6,7 +6,7 @@ resources:
   - namespace.yaml
   # CRDs are pulled from the cephrgw-operator repo at the matching tag rather
   # than vendored here, so they never drift from the operator.
-  - https://git.unkin.net/unkin/cephrgw-operator/raw/tag/v0.3.1/config/crd/install.yaml
+  - https://git.unkin.net/unkin/cephrgw-operator/raw/tag/v0.4.0/config/crd/install.yaml
   - rbac.yaml
   - deployment.yaml
   - vaultauth.yaml

apps/base/cert-manager/clusterissuer_letsencrypt-staging.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-staging
+spec:
+  acme:
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    email: admin@unkin.net
+    privateKeySecretRef:
+      name: letsencrypt-staging-account-key
+    solvers:
+      - dns01:
+          cnameStrategy: Follow
+          rfc2136:
+            nameserver: "198.18.199.53:53"
+            tsigKeyName: certmanager
+            tsigAlgorithm: HMACSHA256
+            tsigSecretSecretRef:
+              name: certmanager-tsig
+              key: secret

apps/base/cert-manager/clusterissuer_letsencrypt.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: admin@unkin.net
+    privateKeySecretRef:
+      name: letsencrypt-account-key
+    solvers:
+      - dns01:
+          cnameStrategy: Follow
+          rfc2136:
+            nameserver: "198.18.199.53:53"
+            tsigKeyName: certmanager
+            tsigAlgorithm: HMACSHA256
+            tsigSecretSecretRef:
+              name: certmanager-tsig
+              key: secret

apps/base/cert-manager/clusterissuer_vault-issuer.yaml

@@ -0,0 +1,27 @@
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: vault-issuer
+  labels:
+    app.kubernetes.io/instance: cert-manager-config
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: cert-manager-config
+  annotations:
+    meta.helm.sh/release-name: cert-manager-clusterissuer
+    meta.helm.sh/release-namespace: cert-manager
+spec:
+  vault:
+    server: https://vault.service.consul:8200
+    path: pki_int/sign/servers_default
+    caBundleSecretRef:
+      key: ca.crt
+      name: vault-ca-cert
+    auth:
+      kubernetes:
+        mountPath: /v1/auth/k8s/au/syd1
+        role: cert_manager_issuer
+        serviceAccountRef:
+          name: cert-manager-vault-issuer
+          audiences:
+            - vault

apps/base/cert-manager/kustomization.yaml

@@ -7,4 +7,7 @@ resources:
   - serviceaccount.yaml
   - clusterrole.yaml
   - clusterrolebinding.yaml
+  - clusterissuer_vault-issuer.yaml
   - vmservicescrape.yaml
+  - clusterissuer_letsencrypt.yaml
+  - clusterissuer_letsencrypt-staging.yaml

apps/base/certificates/vault-ca-cert.yaml

@@ -1,4 +1,6 @@
 ---
+# pre-commit: allow-plain-secret -- public CA bundle; this secret bootstraps
+# trust in Vault itself and therefore cannot be Vault-sourced.
 apiVersion: v1
 kind: Secret
 metadata:
@@ -28,7 +30,7 @@ stringData:
     mitItX+RAgMBAAGjgewwgekwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB
     Af8wHQYDVR0OBBYEFEp/+grAdVqRSeb9xJjSeZYNW32MMB8GA1UdIwQYMBaAFBqc
     v6Y+hfHt4EjgKa/uoQGEHTknMEcGCCsGAQUFBwEBBDswOTA3BggrBgEFBQcwAoYr
-    aHR0cHM6Ly92YXVsdC5zZXJ2aWNlLmNvbnN1bC92MS9wa2lfcm9vdC9jYTA9BgNV
+    aHR0cHM6Ly92YXVsdC5zZXJ2dWNlLmNvbnN1bC92MS9wa2lfcm9vdC9jYTA9BgNV
     HR8ENjA0MDKgMKAuhixodHRwczovL3ZhdWx0LnNlcnZpY2UuY29uc3VsL3YxL3Br
     aV9yb290L2NybDANBgkqhkiG9w0BAQsFAAOCAQEAM0FS8tscZe7yly/gM7jO6lx5
     muMFusifjUIrcQGnZBkoECeuUVPNTs3e/Th+XaxjCnmSpqSNT3z9Irr6Hhxf7n03

apps/base/dhcp-system/cr/keaapi.yaml

@@ -0,0 +1,25 @@
+---
+# Terraform-friendly REST API for KeaSubnet/KeaClientClass CRUD. The bearer
+# token Secret is generated by the operator when absent (no plain Secret is
+# committed here); it can later be pre-seeded from Vault under the same name.
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaAPI
+metadata:
+  name: kea-api
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  replicas: 1
+  image: git.unkin.net/unkin/kea-api:v0.1.2
+  tokenSecretName: kea-api-token
+  service:
+    type: ClusterIP
+    port: 8080
+  resources:
+    requests:
+      cpu: 100m
+      memory: 64Mi
+    limits:
+      cpu: "1"
+      memory: 256Mi

apps/base/dhcp-system/cr/keaclientclasses.yaml

@@ -0,0 +1,27 @@
+# PXE boot classes matching client architecture (option 93), replacing the
+# legacy dhcpd "Legacy" and "UEFI-64" classes. Object names are lowercased to
+# satisfy RFC1123 (the operator renders the kea class name from metadata.name).
+---
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaClientClass
+metadata:
+  name: legacy
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  clusterRef: kea
+  archHex: ["0x0000"]
+  bootFileName: /undionly.kpxe
+---
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaClientClass
+metadata:
+  name: uefi-64
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  clusterRef: kea
+  archHex: ["0x0007", "0x0009"]
+  bootFileName: /ipxe.efi

apps/base/dhcp-system/cr/keacluster.yaml

@@ -0,0 +1,35 @@
+---
+# HA pair fronted by a PureLB anycast Service on a NEW, unused common-pool IP
+# (198.18.200.10). This is intentionally NOT the current isc-dhcpd anycast
+# address (198.18.19.18) -- the production cutover is a separate later task.
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaCluster
+metadata:
+  name: kea
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  replicas: 2
+  image: git.unkin.net/unkin/kea:v0.1.2
+  domainName: main.unkin.net
+  defaultLeaseTime: 1200
+  maxLeaseTime: 86400
+  # No ntpServers: DHCP option 42 (ntp-servers) carries IPv4 addresses only, so
+  # the rotating AU pool.ntp.org hostnames cannot be delivered this way (kea
+  # rejects them at config load). Add concrete NTP server IPs here if needed.
+  ha:
+    mode: hot-standby
+  service:
+    type: LoadBalancer
+    ipAddressPool: common
+    loadBalancerIP: 198.18.200.10
+    annotations:
+      purelb.io/addresses: 198.18.200.10
+  resources:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: "1"
+      memory: 512Mi

apps/base/dhcp-system/cr/keasubnets.yaml

@@ -0,0 +1,90 @@
+# Translation of the legacy ISC dhcpd pools (puppet
+# roles/infra/dhcp/server.yaml): 198.18.13-17.0/24, each a .200-.220 pool,
+# next-server 198.18.19.19. Gateways per the original config:
+# .13/.14/.15/.16 -> .254, .17 -> .1. DNS points at the in-cluster
+# bind-resolvers PureLB IP (198.18.200.7), not the legacy 198.18.19.15.
+---
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaSubnet
+metadata:
+  name: net-198-18-13
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  clusterRef: kea
+  subnet: 198.18.13.0/24
+  pools:
+    - 198.18.13.200 - 198.18.13.220
+  routers: [198.18.13.254]
+  dnsServers: [198.18.200.7]
+  domainName: main.unkin.net
+  nextServer: 198.18.19.19
+---
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaSubnet
+metadata:
+  name: net-198-18-14
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  clusterRef: kea
+  subnet: 198.18.14.0/24
+  pools:
+    - 198.18.14.200 - 198.18.14.220
+  routers: [198.18.14.254]
+  dnsServers: [198.18.200.7]
+  domainName: main.unkin.net
+  nextServer: 198.18.19.19
+---
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaSubnet
+metadata:
+  name: net-198-18-15
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  clusterRef: kea
+  subnet: 198.18.15.0/24
+  pools:
+    - 198.18.15.200 - 198.18.15.220
+  routers: [198.18.15.254]
+  dnsServers: [198.18.200.7]
+  domainName: main.unkin.net
+  nextServer: 198.18.19.19
+---
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaSubnet
+metadata:
+  name: net-198-18-16
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  clusterRef: kea
+  subnet: 198.18.16.0/24
+  pools:
+    - 198.18.16.200 - 198.18.16.220
+  routers: [198.18.16.254]
+  dnsServers: [198.18.200.7]
+  domainName: main.unkin.net
+  nextServer: 198.18.19.19
+---
+apiVersion: kea.unkin.net/v1alpha1
+kind: KeaSubnet
+metadata:
+  name: net-198-18-17
+  namespace: dhcp-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  clusterRef: kea
+  subnet: 198.18.17.0/24
+  pools:
+    - 198.18.17.200 - 198.18.17.220
+  routers: [198.18.17.1]
+  dnsServers: [198.18.200.7]
+  domainName: main.unkin.net
+  nextServer: 198.18.19.19

apps/base/dhcp-system/deployment.yaml

@@ -0,0 +1,56 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kea-operator
+  namespace: dhcp-system
+  labels:
+    app.kubernetes.io/name: kea-operator
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kea-operator
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kea-operator
+    spec:
+      serviceAccountName: kea-operator
+      securityContext:
+        runAsNonRoot: true
+      containers:
+        - name: operator
+          image: git.unkin.net/unkin/kea-operator:v0.1.2
+          args:
+            - --metrics-bind-address=:8080
+            - --health-probe-bind-address=:8081
+          ports:
+            - containerPort: 8080
+              name: metrics
+            - containerPort: 8081
+              name: health
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: 8081
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8081
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 500m
+              memory: 256Mi

apps/base/dhcp-system/kustomization.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  # CRDs are pulled from the kea-operator repo at the matching tag rather than
+  # vendored here, so they never drift from the operator.
+  - https://git.unkin.net/unkin/kea-operator/raw/tag/v0.1.0/config/crd/install.yaml
+  - rbac.yaml
+  - deployment.yaml
+  - vpa.yaml
+  # CRs (sync-wave 1) reconcile after the operator + CRDs are established.
+  - cr/keacluster.yaml
+  - cr/keasubnets.yaml
+  - cr/keaclientclasses.yaml
+  - cr/keaapi.yaml

apps/base/dhcp-system/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: dhcp-system

apps/base/dhcp-system/rbac.yaml

@@ -0,0 +1,46 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kea-operator
+  namespace: dhcp-system
+---
+# Sourced from the kea-operator repo config/rbac/role.yaml (v0.1.0). Leader
+# election is disabled so no coordination.k8s.io/leases grant is needed.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kea-operator
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps", "secrets", "serviceaccounts", "services"]
+    verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments", "statefulsets"]
+    verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
+  - apiGroups: ["kea.unkin.net"]
+    resources: ["keaapis", "keaclientclasses", "keaclusters", "keasubnets"]
+    verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
+  - apiGroups: ["kea.unkin.net"]
+    resources:
+      ["keaapis/status", "keaclientclasses/status", "keaclusters/status", "keasubnets/status"]
+    verbs: ["get", "patch", "update"]
+  - apiGroups: ["rbac.authorization.k8s.io"]
+    resources: ["rolebindings", "roles"]
+    verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kea-operator
+subjects:
+  - kind: ServiceAccount
+    name: kea-operator
+    namespace: dhcp-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kea-operator

apps/base/dhcp-system/vpa.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: autoscaling.k8s.io/v1
+kind: VerticalPodAutoscaler
+metadata:
+  name: kea-operator-vpa
+  namespace: dhcp-system
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: kea-operator
+  updatePolicy:
+    updateMode: "Off"

apps/base/encapi/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-encapi
   namespace: encapi
 spec:
+  placementTarget: ec
   bucketName: cnpg-encapi
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.

apps/base/encapi/deployment.yaml

@@ -5,7 +5,7 @@ metadata:
   name: encapi
   namespace: encapi
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
 spec:
   replicas: 2
   selector:
@@ -23,7 +23,7 @@ spec:
       automountServiceAccountToken: true
       containers:
         - name: encapi
-          image: git.unkin.net/unkin/encapi:v0.1.1
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/encapi:v0.1.1
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000

apps/base/externaldns/agent-dns-rolebinding.yaml

@@ -0,0 +1,16 @@
+---
+# Confines the agent-dns service account (in bind-system) to the agent-dns
+# ClusterRole within this namespace.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: agent-dns
+  namespace: externaldns
+subjects:
+  - kind: ServiceAccount
+    name: agent-dns
+    namespace: bind-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: agent-dns

apps/base/externaldns/kustomization.yaml

@@ -6,3 +6,4 @@ resources:
   - namespace.yaml
   - vaultauth.yaml
   - vaultstaticsecret.yaml
+  - agent-dns-rolebinding.yaml

apps/base/gitea/cnpg_backup.yaml

@@ -0,0 +1,46 @@
+---
+# Ceph RGW (S3) backup target for the gitea CNPG cluster, provisioned by the
+# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: cnpg-gitea-backup
+  namespace: gitea
+spec:
+  displayName: "CNPG backup owner (gitea)"
+  # RGW users are global; keep the uid namespace-qualified so it never collides.
+  uid: cnpg-gitea-backup
+  maxBuckets: 5
+  # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
+  # S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it.
+  secretName: cnpg-gitea-backup-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: cnpg-gitea
+  namespace: gitea
+spec:
+  bucketName: cnpg-gitea
+  ownerRef: cnpg-gitea-backup
+  versioning: false
+  tags:
+    app: gitea
+    purpose: cnpg-backup
+  retainOnDelete: true
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-gitea-nightly
+  namespace: gitea
+spec:
+  # 6-field CNPG cron (seconds first). 04:00 — next free slot after netbox
+  # (03:40), keeping the estate's 20-minute stagger.
+  schedule: "0 0 4 * * *"
+  immediate: false
+  backupOwnerReference: self
+  method: barmanObjectStore
+  cluster:
+    name: gitea-postgres

apps/base/gitea/cnpg_cluster.yaml

@@ -0,0 +1,90 @@
+---
+# Postgres for the k8s Gitea (replaces the Patroni-shared DB the VM uses). Gitea
+# already runs on Postgres, so cutover is a plain pg dump/restore (no engine
+# conversion). App-user creds come from the postgres-credentials Vault secret.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: gitea-postgres
+  namespace: gitea
+spec:
+  affinity:
+    podAntiAffinityType: preferred
+  backup:
+    # 30-day retention. Enforced by CNPG against the object store on each
+    # successful base backup.
+    retentionPolicy: 30d
+    barmanObjectStore:
+      # Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it).
+      destinationPath: s3://cnpg-gitea
+      endpointURL: https://s3.ceph.unkin.net
+      # radosgw serves a Vault-PKI cert; trust the internal CA (reflected into
+      # every namespace as the vault-ca-cert Secret).
+      endpointCA:
+        name: vault-ca-cert
+        key: ca.crt
+      # Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded.
+      s3Credentials:
+        accessKeyId:
+          name: cnpg-gitea-backup-s3
+          key: AWS_ACCESS_KEY_ID
+        secretAccessKey:
+          name: cnpg-gitea-backup-s3
+          key: AWS_SECRET_ACCESS_KEY
+      # Path prefix within the bucket; keep stable across restores (see docs).
+      serverName: gitea
+      data:
+        compression: bzip2
+        jobs: 2
+      wal:
+        compression: zstd
+        maxParallel: 2
+  bootstrap:
+    initdb:
+      database: gitea
+      encoding: UTF8
+      localeCType: C
+      localeCollate: C
+      owner: gitea
+      secret:
+        name: postgres-credentials
+  enablePDB: true
+  enableSuperuserAccess: false
+  failoverDelay: 0
+  imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie
+  instances: 2
+  logLevel: info
+  monitoring:
+    customQueriesConfigMap:
+      - key: queries
+        name: cnpg-default-monitoring
+    disableDefaultQueries: false
+    enablePodMonitor: false
+  postgresql:
+    parameters:
+      max_connections: "200"
+      shared_buffers: 256MB
+  primaryUpdateMethod: restart
+  primaryUpdateStrategy: unsupervised
+  replicationSlots:
+    highAvailability:
+      enabled: true
+      slotPrefix: _cnpg_
+    synchronizeReplicas:
+      enabled: true
+    updateInterval: 30
+  resources:
+    limits:
+      cpu: "2"
+      memory: 2Gi
+    requests:
+      cpu: 250m
+      memory: 512Mi
+  smartShutdownTimeout: 180
+  startDelay: 3600
+  stopDelay: 1800
+  storage:
+    resizeInUseVolumes: true
+    size: 20Gi
+    storageClass: cephrbd-fast-delete
+  switchoverDelay: 3600

apps/base/gitea/cnpg_pooler.yaml

@@ -0,0 +1,36 @@
+---
+# pgbouncer in front of the primary. Gitea opens a connection per request and
+# benefits from pooling under multiple app replicas. Session mode keeps Gitea's
+# occasional session-scoped state (advisory locks, LISTEN/NOTIFY) working.
+apiVersion: postgresql.cnpg.io/v1
+kind: Pooler
+metadata:
+  name: gitea-postgres-pooler-rw
+  namespace: gitea
+spec:
+  cluster:
+    name: gitea-postgres
+  instances: 2
+  pgbouncer:
+    parameters:
+      default_pool_size: "50"
+      max_client_conn: "200"
+    paused: false
+    poolMode: session
+  template:
+    metadata:
+      labels:
+        app: pooler-rw
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - pooler-rw
+              topologyKey: kubernetes.io/hostname
+      containers: []
+  type: rw

apps/base/gitea/gateway.yaml

@@ -0,0 +1,69 @@
+---
+# HTTPS front for the k8s Gitea, served on two names:
+#   git.unkin.net              — canonical/production (apex, bind-operator zone;
+#                                DNS flip is the gated cutover step, see the doc)
+#   git.k8s.syd1.au.unkin.net  — admin/backup route (external-dns k8s.syd1 zone),
+#                                same dual-name pattern as identity.unkin.net.
+# The cert-manager Certificate (vault-issuer) takes CN git.unkin.net and gets a
+# DNS SAN for each TLS listener hostname automatically.
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: gitea
+  namespace: gitea
+  labels:
+    app.kubernetes.io/name: gitea
+    app.kubernetes.io/instance: gitea
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: git.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    # Only the k8s admin route is published by external-dns (it owns just the
+    # k8s.syd1.au.unkin.net zone). git.unkin.net lives in the apex zone and is
+    # flipped at cutover — NOT managed here.
+    external-dns.alpha.kubernetes.io/hostname: git.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - name: http-primary
+      port: 80
+      protocol: HTTP
+      hostname: git.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https-primary
+      port: 443
+      protocol: HTTPS
+      hostname: git.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: gitea-tls
+    - name: http-admin
+      port: 80
+      protocol: HTTP
+      hostname: git.k8s.syd1.au.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https-admin
+      port: 443
+      protocol: HTTPS
+      hostname: git.k8s.syd1.au.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: gitea-tls

apps/base/gitea/httproute.yaml

@@ -0,0 +1,65 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: gitea-http-redirect
+  namespace: gitea
+  labels:
+    app.kubernetes.io/name: gitea
+    app.kubernetes.io/instance: gitea
+spec:
+  hostnames:
+    - git.unkin.net
+    - git.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gitea
+      sectionName: http-primary
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gitea
+      sectionName: http-admin
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: gitea
+  namespace: gitea
+  labels:
+    app.kubernetes.io/name: gitea
+    app.kubernetes.io/instance: gitea
+spec:
+  hostnames:
+    - git.unkin.net
+    - git.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gitea
+      sectionName: https-primary
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gitea
+      sectionName: https-admin
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: gitea-http
+          port: 3000
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/gitea/kustomization.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - cnpg_cluster.yaml
+  - cnpg_backup.yaml
+  - cnpg_pooler.yaml
+  - valkey-deployment.yaml
+  - valkey-pvc.yaml
+  - valkey-service.yaml
+  - vaultauth.yaml
+  - vaultstaticsecret.yaml
+  - gateway.yaml
+  - httproute.yaml

apps/base/gitea/namespace.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    app.kubernetes.io/name: gitea
+  name: gitea

apps/base/gitea/valkey-deployment.yaml

@@ -0,0 +1,89 @@
+---
+# Standalone Valkey (Redis-compatible) for Gitea's session store, cache and
+# queue. The Gitea chart bundles a redis-cluster subchart, but we run our own
+# standalone Valkey here: it keeps image control in-estate (valkey/valkey,
+# already allowlisted through the artifactapi dockerhub mirror) and matches the
+# standalone-cache pattern used by litellm/netbox. One instance serves three
+# logical DBs: DB 0 = session, DB 1 = cache, DB 2 = queue. AOF persistence is
+# enabled so queued actions/webhook deliveries survive a restart.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gitea-valkey
+  namespace: gitea
+  labels:
+    app.kubernetes.io/name: gitea
+    app.kubernetes.io/component: valkey
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: gitea-valkey
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: gitea-valkey
+        app.kubernetes.io/name: gitea
+        app.kubernetes.io/component: valkey
+    spec:
+      securityContext:
+        fsGroup: 999
+      containers:
+        - name: valkey
+          image: valkey/valkey:8-alpine
+          imagePullPolicy: IfNotPresent
+          command:
+            - valkey-server
+            - --appendonly
+            - "yes"
+            - --save
+            - "60"
+            - "1"
+          ports:
+            - containerPort: 6379
+              name: valkey
+              protocol: TCP
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsNonRoot: true
+            runAsUser: 999
+            capabilities:
+              drop:
+                - ALL
+          livenessProbe:
+            exec:
+              command:
+                - valkey-cli
+                - ping
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            successThreshold: 1
+            timeoutSeconds: 5
+          readinessProbe:
+            exec:
+              command:
+                - valkey-cli
+                - ping
+            failureThreshold: 3
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 5
+          resources:
+            limits:
+              cpu: 500m
+              memory: 512Mi
+            requests:
+              cpu: 50m
+              memory: 128Mi
+          volumeMounts:
+            - mountPath: /data
+              name: data
+      restartPolicy: Always
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: gitea-valkey-data

apps/base/gitea/valkey-pvc.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: gitea-valkey-data
+  namespace: gitea
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+  storageClassName: cephrbd-fast-delete
+  volumeMode: Filesystem

apps/base/gitea/valkey-service.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: gitea-valkey
+  namespace: gitea
+  labels:
+    app.kubernetes.io/name: gitea
+    app.kubernetes.io/component: valkey
+spec:
+  internalTrafficPolicy: Cluster
+  ports:
+    - name: valkey
+      port: 6379
+      protocol: TCP
+      targetPort: valkey
+  selector:
+    app: gitea-valkey
+  sessionAffinity: None
+  type: ClusterIP

apps/base/gitea/vaultauth.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: default
+  namespace: gitea
+spec:
+  allowedNamespaces:
+    - gitea
+  kubernetes:
+    audiences:
+      - vault
+    role: default
+    serviceAccount: default
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default

apps/base/gitea/vaultstaticsecret.yaml

@@ -0,0 +1,83 @@
+---
+# CNPG app-user credentials (keys: username, password). Consumed by the Cluster
+# bootstrap (initdb.secret) AND by Gitea (gitea.config.database.PASSWD via the
+# chart's existingSecret wiring). One-time Vault seed — see the PR description.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: postgres-credentials
+  namespace: gitea
+spec:
+  destination:
+    create: true
+    name: postgres-credentials
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/gitea/default/postgres-credentials
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+# Initial Gitea admin (keys: username, password, email). Applied by the chart's
+# init job on first boot (gitea.admin.existingSecret). Local fallback account
+# that survives the Authentik OIDC cutover. One-time Vault seed.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: gitea-admin
+  namespace: gitea
+spec:
+  destination:
+    create: true
+    name: gitea-admin
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/gitea/default/gitea-admin
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+# Gitea internal secrets (keys: SECRET_KEY, INTERNAL_TOKEN). Pinned here rather
+# than chart-generated so all replicas share identical values AND so the data
+# cutover can replace them with the VM's app.ini values (SECRET_KEY encrypts
+# 2FA/mirror/oauth secrets in the DB — it MUST match the restored database).
+# One-time Vault seed.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: gitea-inner
+  namespace: gitea
+spec:
+  destination:
+    create: true
+    name: gitea-inner
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/gitea/default/gitea-inner
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+# Authentik OIDC client secret (key: client_secret). Read by the
+# terraform-authentik provider runner (policy already grants
+# kv/.../namespace/+/default/oauth-credentials) AND mounted into Gitea to
+# register the OIDC login source. One-time Vault seed.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: oauth-credentials
+  namespace: gitea
+spec:
+  destination:
+    create: true
+    name: oauth-credentials
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/gitea/default/oauth-credentials
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/base/grafana/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-grafana
   namespace: grafana
 spec:
+  placementTarget: ec
   bucketName: cnpg-grafana
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.

apps/base/grafana/grafana.yaml

@@ -26,6 +26,13 @@ spec:
                     secretKeyRef:
                       name: oauth-credentials
                       key: client_secret
+              # identity.unkin.net is served by the internal unkin.net CA, which
+              # the stock Grafana image doesn't trust. Mount the reflected
+              # vault-ca-cert and point generic_oauth's tls_client_ca at it.
+              volumeMounts:
+                - name: vault-ca-cert
+                  mountPath: /etc/grafana/vault-ca
+                  readOnly: true
               resources:
                 requests:
                   cpu: 100m
@@ -33,6 +40,13 @@ spec:
                 limits:
                   cpu: "1"
                   memory: 1Gi
+          volumes:
+            - name: vault-ca-cert
+              secret:
+                secretName: vault-ca-cert
+                items:
+                  - key: ca.crt
+                    path: ca.crt
   config:
     server:
       root_url: "https://grafana.k8s.syd1.au.unkin.net"
@@ -57,6 +71,9 @@ spec:
       auth_url: "https://identity.unkin.net/application/o/authorize/"
       token_url: "https://identity.unkin.net/application/o/token/"
       api_url: "https://identity.unkin.net/application/o/userinfo/"
+      # Trust the internal unkin.net CA that signs identity.unkin.net's cert
+      # (mounted from the reflected vault-ca-cert Secret).
+      tls_client_ca: "/etc/grafana/vault-ca/ca.crt"
       # Authentik permission groups -> Grafana roles. akP-grafana-admin is granted
       # to akR-global-admin members (and direct members) via terraform-authentik.
       role_attribute_path: "contains(ak_groups[*], 'akP-grafana-admin') && 'Admin' || 'Viewer'"

apps/base/kanidm/statefulset.yaml

@@ -5,7 +5,8 @@ metadata:
   name: kanidm
   namespace: kanidm
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "kanidm-tls"
   labels:
     app.kubernetes.io/name: kanidm
     app.kubernetes.io/instance: kanidm

apps/base/litellm/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-litellm
   namespace: litellm
 spec:
+  placementTarget: ec
   bucketName: cnpg-litellm
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.

apps/base/litellm/cnpg_cluster.yaml

@@ -48,7 +48,7 @@ spec:
   enablePDB: true
   enableSuperuserAccess: false
   failoverDelay: 0
-  imageName: ghcr.io/cloudnative-pg/postgresql:17-minimal-trixie
+  imageName: ghcr.io/cloudnative-pg/postgresql:17-system-trixie
   instances: 3
   logLevel: info
   maxSyncReplicas: 0

apps/base/litellm/deployment.yaml

@@ -11,10 +11,28 @@ spec:
   template:
     metadata:
       annotations:
-        reloader.stakater.com/auto: "true"
+        configmap.reloader.stakater.com/auto: "true"
+        secret.reloader.stakater.com/reload: "vault-ca-cert"
       labels:
         app: litellm
     spec:
+      # LiteLLM's SSO client reaches identity.unkin.net, whose cert is signed by
+      # the internal unkin.net CA. Combine the image's public roots with the
+      # reflected vault-ca-cert into one bundle (SSL_CERT_FILE/REQUESTS_CA_BUNDLE
+      # in litellm-env point at it) so internal OIDC and public HTTPS both work.
+      initContainers:
+        - name: combine-certs
+          image: alpine:3
+          command:
+            - sh
+            - -c
+            - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt
+          volumeMounts:
+            - name: vault-ca-cert
+              mountPath: /custom-ca
+              readOnly: true
+            - name: combined-certs
+              mountPath: /combined-certs
       containers:
         - name: litellm
           image: docker.litellm.ai/berriai/litellm-database:main-stable
@@ -72,8 +90,19 @@ spec:
             - mountPath: /app/config.yaml
               name: config
               subPath: config.yaml
+            - name: combined-certs
+              mountPath: /etc/ssl/combined
+              readOnly: true
       restartPolicy: Always
       volumes:
         - name: config
           configMap:
             name: litellm-config
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: combined-certs
+          emptyDir: {}

apps/base/litellm/kustomization.yaml

@@ -27,6 +27,9 @@ configMapGenerator:
   - name: litellm-env
     literals:
       - STORE_MODEL_IN_DB=True
+      # Emit structured JSON logs so the Tier-2 vector litellm pipeline can parse
+      # model/tokens/latency/key/status (logs.k8s.litellm.*).
+      - JSON_LOGS=True
       # Authentik OIDC SSO (generic). Client secret is injected from the
       # oauth-credentials Secret in the Deployment; endpoints match the other
       # apps (identity.unkin.net). PROXY_BASE_URL is required for SSO.
@@ -39,5 +42,9 @@ configMapGenerator:
       - GENERIC_SCOPE=openid email profile litellm_role
       - GENERIC_USER_ROLE_ATTRIBUTE=litellm_role
       - PROXY_BASE_URL=https://litellm.k8s.syd1.au.unkin.net
+      # Trust the internal unkin.net CA (identity.unkin.net) via the combined
+      # bundle assembled by the combine-certs init container.
+      - SSL_CERT_FILE=/etc/ssl/combined/ca-certificates.crt
+      - REQUESTS_CA_BUNDLE=/etc/ssl/combined/ca-certificates.crt
     options:
       disableNameSuffixHash: true

apps/base/logging/configmap_logarchiver.yaml

@@ -0,0 +1,40 @@
+---
+# logarchiver non-secret config. Secrets (NATS/S3/ClickHouse creds) and the
+# subject filter come from env; everything else uses the binary's built-in
+# defaults, which already target this stack. ack_wait MUST exceed batch.max_age
+# so unacked messages in an open batch are not redelivered mid-batch.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logarchiver-config
+  namespace: logging
+data:
+  config.yaml: |
+    nats:
+      ack_wait: 5m
+      fetch_batch: 512
+    batch:
+      max_bytes: 67108864
+      max_events: 200000
+      max_age: 2m
+    # Pin the proven RGW endpoint/bucket; ignore the secret's S3_ENDPOINT/BUCKET_NAME
+    # (AWS creds still come from the secret env). endpoint_env/bucket_env off.
+    s3:
+      endpoint: "https://s3.ceph.unkin.net"
+      bucket: "logs-archive"
+      region: "us-east-1"
+      path_style: true
+      ca_file: /etc/vault-ca/ca.crt
+      endpoint_env: ""
+      bucket_env: ""
+    crypto:
+      key_name: logarchive
+      pubkey_source: vault
+      vault:
+        address: "https://vault.service.consul:8200"
+        mount: gpg
+        auth_method: kubernetes
+        k8s_mount: k8s/au/syd1
+        k8s_role: logging_logarchiver
+        k8s_jwt_path: /var/run/secrets/vault/token
+        ca_file: /etc/vault-ca/ca.crt

apps/base/logging/deployment_logarchiver.yaml

@@ -0,0 +1,134 @@
+---
+# logarchiver — replaces the vector-archiver leg. Independent JetStream durable
+# consumer (archiver) that seals raw logs to S3 as zstd + OpenPGP objects and
+# indexes each object in ClickHouse (logs.archive_index). Acks only after the
+# object is in S3 AND indexed. Reuses the same NATS/S3/CA wiring the Vector
+# archiver used; the OpenPGP public key is delivered as a mounted file.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: logarchiver
+  namespace: logging
+  annotations:
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "vault-ca-cert"
+  labels:
+    app.kubernetes.io/name: logarchiver
+    app.kubernetes.io/component: archiver
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: logarchiver
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: logarchiver
+        vector.dev/exclude: "true"
+    spec:
+      # Dedicated SA whose projected vault-audience token authenticates the
+      # k8s-auth login used to fetch the logarchive public key from the gpg engine.
+      serviceAccountName: logarchiver
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: logarchiver
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/logarchiver:v0.1.0
+          imagePullPolicy: IfNotPresent
+          args: ["run"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          ports:
+            - containerPort: 9090
+              name: metrics
+              protocol: TCP
+          env:
+            - name: LOGARCHIVER_CONFIG
+              value: /etc/logarchiver/config.yaml
+            # Server-side subject filter; must match the archiver consumer's filter.
+            - name: ARCHIVE_SUBJECTS
+              value: "logs.k8s.vault.>"
+            - name: NATS_CONSUMER_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: nats-auth
+                  key: consumer_password
+            - name: CLICKHOUSE_USER
+              valueFrom:
+                secretKeyRef:
+                  name: clickhouse-credentials
+                  key: username
+            - name: CLICKHOUSE_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: clickhouse-credentials
+                  key: password
+          # S3 creds + S3_ENDPOINT + BUCKET_NAME from the cephrgw BucketAccess Secret.
+          envFrom:
+            - secretRef:
+                name: logs-archive-s3
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 15
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: "1"
+              memory: 1Gi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/logarchiver/config.yaml
+              subPath: config.yaml
+              readOnly: true
+            - name: vault-token
+              mountPath: /var/run/secrets/vault
+              readOnly: true
+            - name: vault-ca-cert
+              mountPath: /etc/vault-ca/ca.crt
+              subPath: ca.crt
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: logarchiver-config
+        # Projected SA token with audience "vault" for the gpg-engine k8s login.
+        - name: vault-token
+          projected:
+            sources:
+              - serviceAccountToken:
+                  path: token
+                  audience: vault
+                  expirationSeconds: 600
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+        - name: tmp
+          emptyDir: {}

apps/base/logging/job_clickhouse-schema.yaml

@@ -88,6 +88,32 @@ spec:
               ORDER BY (source, namespace, host, timestamp)
               TTL toDateTime(timestamp) + INTERVAL 3 DAY
               SETTINGS index_granularity = 8192;
+
+              -- One row per archived S3 object (written by logarchiver). No TTL:
+              -- the index must outlive logs.raw so the long-term S3 archive stays
+              -- searchable. Keep in sync with logarchiver internal/index/ddl.go.
+              CREATE TABLE IF NOT EXISTS logs.archive_index
+              (
+                  object_key       String,
+                  bucket           LowCardinality(String),
+                  subject          LowCardinality(String),
+                  hosts            Array(LowCardinality(String)),
+                  min_ts           DateTime64(3),
+                  max_ts           DateTime64(3),
+                  event_count      UInt64,
+                  raw_bytes        UInt64,
+                  stored_bytes     UInt64,
+                  compression      LowCardinality(String),
+                  cipher           LowCardinality(String),
+                  container_format LowCardinality(String),
+                  key_name         LowCardinality(String),
+                  key_fingerprint  String,
+                  created_at       DateTime64(3) DEFAULT now64(3),
+                  INDEX idx_hosts hosts TYPE bloom_filter GRANULARITY 1
+              )
+              ENGINE = MergeTree
+              PARTITION BY toYYYYMM(min_ts)
+              ORDER BY (subject, min_ts, object_key);
               EOSQL
               echo "Schema applied."
           resources:

apps/base/logging/kustomization.yaml

@@ -12,6 +12,9 @@ resources:
   - cephrgw.yaml
   - gateway.yaml
   - httproute.yaml
+  - serviceaccount_logarchiver.yaml
+  - configmap_logarchiver.yaml
+  - deployment_logarchiver.yaml
 
 # Vector pipelines are the single source of truth (also validated by
 # `vector test` in CI). Mounted into each tier via `existingConfigMaps`.
@@ -44,8 +47,3 @@ configMapGenerator:
       - vm-ingest.yaml=vector/vm-ingest.yaml
     options:
       disableNameSuffixHash: true
-  - name: vector-archiver-config
-    files:
-      - archiver.yaml=vector/archiver.yaml
-    options:
-      disableNameSuffixHash: true

apps/base/logging/nats-bootstrap-job.yaml

@@ -59,6 +59,11 @@ spec:
       containers:
         - name: nats-bootstrap
           image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/natsio/nats-box:0.18.0
+          # nats CLI stats the working directory when loading its response
+          # schemas; under readOnlyRootFilesystem + runAsUser 1000 the image's
+          # default WORKDIR is not accessible ("stat .: permission denied"), so
+          # run from the writable /tmp emptyDir.
+          workingDir: /tmp
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true

apps/base/logging/serviceaccount_logarchiver.yaml

@@ -0,0 +1,9 @@
+---
+# Dedicated SA for logarchiver's Vault k8s-auth login (role logging_logarchiver,
+# terraform-vault). Only used to fetch the logarchive public key.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: logarchiver
+  namespace: logging
+automountServiceAccountToken: false

apps/base/logging/vector/aggregator-tests.yaml

@@ -61,3 +61,608 @@ tests:
               assert_eq!(.severity, "info")
               assert_eq!(.message, "sshd started")
               assert_eq!(.labels.role, "database")
+
+  # --- catch-all preservation: an unclaimed k8s event still flows app_route ->
+  # generic route -> k8s_shape (proves the two-stage chain keeps the fallback) ---
+  - name: unclaimed_k8s_falls_through_to_generic
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.shop.web"
+          message: "plain app log"
+    outputs:
+      - extract_from: route.k8s
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.message, "plain app log")
+
+  # --- Tier-1: Authentik SSO (k8s, LIVE NOW) ---
+  - name: authentik_routes_by_subject
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.authentik.server"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.authentik
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: authentik_parse_extracts_event
+    inputs:
+      - insert_at: authentik_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.authentik.server"
+          stream: "stdout"
+          kubernetes.pod_namespace: "authentik"
+          kubernetes.container_name: "server"
+          kubernetes.pod_node_name: "node-2"
+          message: '{"event":"login","action":"login","user":"alice","client_ip":"203.0.113.9","result":"success","level":"info","logger":"authentik.events","timestamp":"2026-07-27T00:00:00Z"}'
+    outputs:
+      - extract_from: authentik_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "authentik")
+              assert_eq!(.container, "server")
+              assert_eq!(.severity, "info")
+              assert_eq!(.message, "login")
+              assert_eq!(.labels.app, "authentik")
+              assert_eq!(.fields.event, "login")
+              assert_eq!(.fields.action, "login")
+              assert_eq!(.fields.user, "alice")
+              assert_eq!(.fields.client_ip, "203.0.113.9")
+              assert_eq!(.fields.result, "success")
+
+  # --- Tier-1: Traefik ingress (k8s, JSON access logs) ---
+  - name: traefik_routes_by_subject
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.traefik-system.traefik"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.traefik
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: traefik_parse_extracts_access_fields
+    inputs:
+      - insert_at: traefik_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.traefik-system.traefik"
+          kubernetes.pod_namespace: "traefik-system"
+          kubernetes.container_name: "traefik"
+          kubernetes.pod_node_name: "node-3"
+          message: '{"RouterName":"web@kubernetes","ServiceName":"shop-svc@kubernetes","RequestMethod":"GET","RequestPath":"/api","RequestHost":"shop.example.net","RequestProtocol":"HTTP/1.1","DownstreamStatus":200,"Duration":5000000,"ClientHost":"203.0.113.5","StartUTC":"2026-07-27T00:00:00Z"}'
+    outputs:
+      - extract_from: traefik_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "traefik-system")
+              assert_eq!(.labels.app, "traefik")
+              assert_eq!(.message, "GET /api 200")
+              assert_eq!(.fields.route, "web@kubernetes")
+              assert_eq!(.fields.service, "shop-svc@kubernetes")
+              assert_eq!(.fields.method, "GET")
+              assert_eq!(.fields.path, "/api")
+              assert_eq!(.fields.host, "shop.example.net")
+              assert_eq!(.fields.status, "200")
+              assert_eq!(.fields.duration_ms, "5")
+              assert_eq!(.fields.client_ip, "203.0.113.5")
+
+  # --- Tier-1: Vault/OpenBao file audit (VM, awaiting VM vector) ---
+  - name: vault_routes_by_file
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.vm.vault1_syd1"
+          file: "/var/log/vault_audit.log"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.vault
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: vault_parse_extracts_request
+    inputs:
+      - insert_at: vault_parse
+        type: log
+        log_fields:
+          subject: "logs.vm.vault1_syd1"
+          host: "vault1"
+          file: "/var/log/vault_audit.log"
+          message: '{"time":"2026-07-27T00:00:00Z","type":"response","auth":{"display_name":"token"},"request":{"operation":"read","path":"secret/data/app","remote_address":"10.0.0.9"},"error":""}'
+    outputs:
+      - extract_from: vault_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "vm")
+              assert_eq!(.host, "vault1")
+              assert_eq!(.labels.app, "vault")
+              assert_eq!(.message, "read secret/data/app")
+              assert_eq!(.fields.type, "response")
+              assert_eq!(.fields.display_name, "token")
+              assert_eq!(.fields.operation, "read")
+              assert_eq!(.fields.path, "secret/data/app")
+              assert_eq!(.fields.remote_address, "10.0.0.9")
+
+  # --- Tier-1: nginx access (VM, awaiting VM vector) ---
+  - name: nginx_access_routes_by_file
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.vm.web1_syd1"
+          file: "/var/log/nginx/shop_access.log"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.nginx_access
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: nginx_access_parse_extracts_combined
+    inputs:
+      - insert_at: nginx_access_parse
+        type: log
+        log_fields:
+          subject: "logs.vm.web1_syd1"
+          host: "web1"
+          file: "/var/log/nginx/shop_access.log"
+          message: '192.0.2.10 - - [27/Jul/2026:00:00:00 +0000] "GET /index.html HTTP/1.1" 200 1024 "https://ref.example/" "Mozilla/5.0" 0.012'
+    outputs:
+      - extract_from: nginx_access_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "vm")
+              assert_eq!(.stream, "access")
+              assert_eq!(.labels.log_type, "access")
+              assert_eq!(.fields.client_ip, "192.0.2.10")
+              assert_eq!(.fields.method, "GET")
+              assert_eq!(.fields.path, "/index.html")
+              assert_eq!(.fields.status, "200")
+              assert_eq!(.fields.bytes, "1024")
+              assert_eq!(.fields.referer, "https://ref.example/")
+              assert_eq!(.fields.user_agent, "Mozilla/5.0")
+              assert_eq!(.fields.request_time, "0.012")
+
+  # --- Tier-1: nginx error (VM, awaiting VM vector) ---
+  - name: nginx_error_routes_by_file
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.vm.web1_syd1"
+          file: "/var/log/nginx/shop_error.log"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.nginx_error
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: nginx_error_parse_extracts_fields
+    inputs:
+      - insert_at: nginx_error_parse
+        type: log
+        log_fields:
+          subject: "logs.vm.web1_syd1"
+          host: "web1"
+          file: "/var/log/nginx/shop_error.log"
+          message: '2026/07/27 00:00:00 [error] 1234#0: *5 open() "/var/www/x" failed (2: No such file or directory), client: 192.0.2.20, server: shop, request: "GET / HTTP/1.1", host: "shop"'
+    outputs:
+      - extract_from: nginx_error_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.stream, "error")
+              assert_eq!(.severity, "error")
+              assert_eq!(.labels.log_type, "error")
+              assert_eq!(.fields.level, "error")
+              assert_eq!(.fields.pid, "1234")
+              assert_eq!(.fields.cid, "5")
+              assert_eq!(.fields.client_ip, "192.0.2.20")
+
+  # --- Tier-1: HAProxy httplog (VM journald, awaiting VM vector) ---
+  - name: haproxy_routes_by_identifier
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.vm.halb1_syd1"
+          SYSLOG_IDENTIFIER: "haproxy"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.haproxy
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: haproxy_parse_extracts_timers
+    inputs:
+      - insert_at: haproxy_parse
+        type: log
+        log_fields:
+          subject: "logs.vm.halb1_syd1"
+          host: "halb1"
+          SYSLOG_IDENTIFIER: "haproxy"
+          message: '192.0.2.30:54321 [27/Jul/2026:00:00:00.123] fe_http be_app/app1 10/0/1/2/13 200 512 - - ---- 5/4/3/2/0 0/0 "GET /health HTTP/1.1"'
+    outputs:
+      - extract_from: haproxy_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "vm")
+              assert_eq!(.labels.app, "haproxy")
+              assert_eq!(.fields.client_ip, "192.0.2.30")
+              assert_eq!(.fields.frontend, "fe_http")
+              assert_eq!(.fields.backend, "be_app")
+              assert_eq!(.fields.server, "app1")
+              assert_eq!(.fields.tq, "10")
+              assert_eq!(.fields.tw, "0")
+              assert_eq!(.fields.tc, "1")
+              assert_eq!(.fields.tr, "2")
+              assert_eq!(.fields.tt, "13")
+              assert_eq!(.fields.termination_state, "----")
+              assert_eq!(.fields.retries, "0")
+              assert_eq!(.fields.status, "200")
+              assert_eq!(.fields.bytes, "512")
+
+  # --- Tier-1: glauth LDAP (VM, awaiting VM vector) ---
+  - name: glauth_routes_by_identifier
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.vm.ldap1_syd1"
+          SYSLOG_IDENTIFIER: "glauth"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.glauth
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: glauth_parse_extracts_bind
+    inputs:
+      - insert_at: glauth_parse
+        type: log
+        log_fields:
+          subject: "logs.vm.ldap1_syd1"
+          host: "ldap1"
+          SYSLOG_IDENTIFIER: "glauth"
+          message: '{"level":"info","msg":"Bind success as user","bindDN":"cn=admin,dc=example,dc=com","src":"192.0.2.40:1234","time":"2026-07-27T00:00:00Z"}'
+    outputs:
+      - extract_from: glauth_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "vm")
+              assert_eq!(.host, "ldap1")
+              assert_eq!(.severity, "info")
+              assert_eq!(.labels.app, "glauth")
+              assert_eq!(.fields.bindDN, "cn=admin,dc=example,dc=com")
+              assert_eq!(.fields.remote, "192.0.2.40:1234")
+              assert_eq!(.fields.success, "true")
+
+  # ================= Tier-2 (stacks on #318) =================
+
+  # --- BIND query logs (k8s bind-* + VM named) ---
+  - name: bind_routes_k8s_by_subject
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.bind-internal.named"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.bind_query
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: bind_routes_vm_by_identifier
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.vm.dns1_syd1"
+          SYSLOG_IDENTIFIER: "named"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.bind_query
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: bind_parse_extracts_query
+    inputs:
+      - insert_at: bind_query_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.bind-internal.named"
+          kubernetes.pod_namespace: "bind-internal"
+          kubernetes.container_name: "named"
+          kubernetes.pod_node_name: "node-4"
+          message: '02-Aug-2026 00:00:00.123 client @0x7f 192.0.2.1#40426 (www.example.com): view internal: query: www.example.com IN A +E(0)K (198.18.200.7)'
+    outputs:
+      - extract_from: bind_query_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "bind-internal")
+              assert_eq!(.labels.app, "bind")
+              assert_eq!(.message, "query www.example.com A")
+              assert_eq!(.fields.client_ip, "192.0.2.1")
+              assert_eq!(.fields.qname, "www.example.com")
+              assert_eq!(.fields.qclass, "IN")
+              assert_eq!(.fields.qtype, "A")
+              assert_eq!(.fields.view, "internal")
+
+  # --- Rancher audit (k8s, cattle-system sidecar) ---
+  - name: rancher_routes_by_subject
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.cattle-system.rancher-audit-log"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.rancher_audit
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: rancher_parse_extracts_audit
+    inputs:
+      - insert_at: rancher_audit_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.cattle-system.rancher-audit-log"
+          kubernetes.pod_namespace: "cattle-system"
+          kubernetes.container_name: "rancher-audit-log"
+          kubernetes.pod_node_name: "node-5"
+          message: '{"auditID":"abc-123","requestURI":"/v3/tokens","user":{"name":"u-alice","group":["admins"]},"method":"GET","remoteAddr":"10.42.0.9:1234","responseCode":200,"requestTimestamp":"2026-08-01T00:00:00Z"}'
+    outputs:
+      - extract_from: rancher_audit_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "cattle-system")
+              assert_eq!(.labels.app, "rancher")
+              assert_eq!(.labels.log_type, "audit")
+              assert_eq!(.message, "GET /v3/tokens 200")
+              assert_eq!(.fields.user, "u-alice")
+              assert_eq!(.fields.verb, "GET")
+              assert_eq!(.fields.uri, "/v3/tokens")
+              assert_eq!(.fields.status, "200")
+
+  # --- CNPG Postgres (ONE transform, all clusters) ---
+  - name: cnpg_routes_by_postgres_container
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.litellm.postgres"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.cnpg_pg
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  # mutual exclusivity: an app-namespace CNPG pod (authentik) is claimed by
+  # cnpg_pg, NOT the authentik app route (which now carves out .postgres).
+  - name: cnpg_authentik_postgres_routes_to_cnpg
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.authentik.postgres"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.cnpg_pg
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: cnpg_parse_extracts_record
+    inputs:
+      - insert_at: cnpg_pg_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.litellm.postgres"
+          kubernetes.pod_namespace: "litellm"
+          kubernetes.container_name: "postgres"
+          kubernetes.pod_node_name: "node-6"
+          kubernetes.pod_labels."cnpg.io/cluster": "litellm-postgres"
+          message: '{"level":"info","ts":"2026-08-01T00:00:00Z","logger":"postgres","msg":"record","record":{"user_name":"litellm","database_name":"litellm","error_severity":"LOG","message":"duration: 12.345 ms  statement: SELECT 1","query":""}}'
+    outputs:
+      - extract_from: cnpg_pg_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "litellm")
+              assert_eq!(.severity, "LOG")
+              assert_eq!(.labels.app, "cnpg")
+              assert_eq!(.labels.cluster, "litellm-postgres")
+              assert_eq!(.fields.error_severity, "LOG")
+              assert_eq!(.fields.duration_ms, "12.345")
+              assert_eq!(.fields.user, "litellm")
+              assert_eq!(.fields.database, "litellm")
+
+  # --- Gitea router/access (k8s + VM) ---
+  - name: gitea_routes_k8s_by_subject
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.gitea.gitea"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.gitea
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: gitea_parse_router_line
+    inputs:
+      - insert_at: gitea_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.gitea.gitea"
+          kubernetes.pod_namespace: "gitea"
+          kubernetes.container_name: "gitea"
+          kubernetes.pod_node_name: "node-7"
+          message: '2026/08/01 00:00:00 .../router.go:100:func() [I] router: completed GET /user/login for 10.0.0.1:0, 200 OK in 12.3ms @ web/base.go:1'
+    outputs:
+      - extract_from: gitea_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "gitea")
+              assert_eq!(.labels.app, "gitea")
+              assert_eq!(.message, "GET /user/login 200")
+              assert_eq!(.fields.method, "GET")
+              assert_eq!(.fields.path, "/user/login")
+              assert_eq!(.fields.status, "200")
+              assert_eq!(.fields.latency, "12.3ms")
+  - name: gitea_parse_access_line
+    inputs:
+      - insert_at: gitea_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.gitea.gitea"
+          kubernetes.pod_namespace: "gitea"
+          kubernetes.container_name: "gitea"
+          message: '10.0.0.5 - alice [01/Aug/2026:00:00:00 +0000] "POST /repo/foo HTTP/1.1" 201 512 "-" "git/2.0"'
+    outputs:
+      - extract_from: gitea_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.fields.method, "POST")
+              assert_eq!(.fields.path, "/repo/foo")
+              assert_eq!(.fields.status, "201")
+              assert_eq!(.fields.user, "alice")
+              assert_eq!(.fields.client_ip, "10.0.0.5")
+
+  # --- PuppetServer / PuppetDB (k8s stdout) ---
+  - name: puppet_routes_by_subject
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.puppet.puppetserver"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.puppet
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: puppet_parse_logback_line
+    inputs:
+      - insert_at: puppet_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.puppet.puppetserver"
+          kubernetes.pod_namespace: "puppet"
+          kubernetes.container_name: "puppetserver"
+          kubernetes.pod_node_name: "node-8"
+          message: '2026-08-01 00:00:00,123 INFO  [qtp123-45] [puppetserver] Compiled catalog for web01.unkin.net in environment production in 1.23 seconds'
+    outputs:
+      - extract_from: puppet_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "puppet")
+              assert_eq!(.severity, "INFO")
+              assert_eq!(.labels.app, "puppet")
+              assert_eq!(.fields.level, "INFO")
+              assert_eq!(.fields.logger, "puppetserver")
+              assert_eq!(.fields.node, "web01.unkin.net")
+
+  # --- LiteLLM request logs (k8s JSON) ---
+  - name: litellm_routes_by_subject
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.k8s.litellm.litellm"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.litellm
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: litellm_parse_extracts_request
+    inputs:
+      - insert_at: litellm_parse
+        type: log
+        log_fields:
+          subject: "logs.k8s.litellm.litellm"
+          kubernetes.pod_namespace: "litellm"
+          kubernetes.container_name: "litellm"
+          kubernetes.pod_node_name: "node-9"
+          message: '{"message":"Request completed","level":"info","model":"gpt-4o","total_tokens":1234,"response_time":0.532,"api_key":"sk-abc","status":"success","timestamp":"2026-08-01T00:00:00Z"}'
+    outputs:
+      - extract_from: litellm_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "k8s")
+              assert_eq!(.namespace, "litellm")
+              assert_eq!(.severity, "info")
+              assert_eq!(.message, "Request completed")
+              assert_eq!(.labels.app, "litellm")
+              assert_eq!(.fields.model, "gpt-4o")
+              assert_eq!(.fields.tokens, "1234")
+              assert_eq!(.fields.latency, "0.532")
+              assert_eq!(.fields.key, "sk-abc")
+              assert_eq!(.fields.status, "success")
+
+  # --- Postfix maillog (VM, per-line best-effort) ---
+  - name: postfix_routes_by_identifier
+    inputs:
+      - insert_at: app_route
+        type: log
+        log_fields:
+          subject: "logs.vm.mail1_syd1"
+          SYSLOG_IDENTIFIER: "postfix/qmgr"
+          message: "routed"
+    outputs:
+      - extract_from: app_route.postfix
+        conditions:
+          - type: vrl
+            source: 'assert_eq!(.message, "routed")'
+  - name: postfix_parse_extracts_line
+    inputs:
+      - insert_at: postfix_parse
+        type: log
+        log_fields:
+          subject: "logs.vm.mail1_syd1"
+          host: "mail1"
+          SYSLOG_IDENTIFIER: "postfix/smtp"
+          message: 'ABC123DEF: to=<rcpt@example.com>, relay=mx.example.com[1.2.3.4]:25, delay=1.2, delays=0.1/0/0.5/0.6, dsn=2.0.0, status=sent (250 OK)'
+    outputs:
+      - extract_from: postfix_parse
+        conditions:
+          - type: vrl
+            source: |
+              assert_eq!(.source, "vm")
+              assert_eq!(.host, "mail1")
+              assert_eq!(.labels.app, "postfix")
+              assert_eq!(.fields.qid, "ABC123DEF")
+              assert_eq!(.fields.to, "rcpt@example.com")
+              assert_eq!(.fields.relay, "mx.example.com[1.2.3.4]:25")
+              assert_eq!(.fields.delay, "1.2")
+              assert_eq!(.fields.status, "sent")
+              assert_eq!(.fields.program, "postfix/smtp")

apps/base/logging/vector/aggregator.yaml

@@ -3,9 +3,25 @@
 # by `vector test` in CI. Consumes the whole log stream from JetStream via the
 # durable `transform` consumer (at-least-once; durable offsets tracked by
 # JetStream), routes by subject, normalises into the logs.raw columns, and is
-# the ONLY ClickHouse writer. Per-app parsing is added here as follow-ups:
+# the ONLY ClickHouse writer.
-# insert a transform and append its id to the clickhouse sink `inputs` — no edge
+#
-# or VM rollout required.
+# Routing model (two stages):
+#   1. app_route — peels off Tier-1 per-app streams by subject / source tag and
+#      hands each to a dedicated parse transform that emits the full logs.raw
+#      shape plus structured .fields. Conditions are MUTUALLY EXCLUSIVE, so an
+#      event is claimed by at most one app (no double-insert).
+#   2. route (generic catch-all) — everything app_route did NOT claim
+#      (app_route._unmatched) is split k8s/vm and shaped generically. This is the
+#      fallback for all un-parsed traffic and MUST stay intact.
+# Add a new per-app pipeline by appending a mutually-exclusive route to
+# app_route, a parse transform, and its id to the clickhouse sink `inputs`.
+#
+# Structured fields go into the logs.raw `fields Map(String,String)` column — no
+# DDL change is needed (values are stringified; empties are compacted away).
+#
+# VM source-tag convention (the puppet-side vector rollout MUST follow it so
+# these transforms light up): file sources set `.file` (absolute log path);
+# journald sources set `.SYSLOG_IDENTIFIER` (falls back to `.program`/`.appname`).
 #
 # Durability model: JetStream (3d / 130 GiB, S2-compressed) is the SOLE
 # durability layer and the replay window. This
@@ -39,10 +55,68 @@ sources:
       codec: json
 
 transforms:
-  route:
+  # Stage 1: peel off Tier-1 per-app streams. Mutually exclusive conditions;
+  # anything unmatched falls through to the generic `route` below.
+  app_route:
     type: route
     inputs:
       - js_in
+    route:
+      # k8s: authentik SSO — structlog JSON on stdout. The `.postgres` container
+      # is the authentik-namespace CNPG cluster; carve it out so it is claimed by
+      # the single `cnpg_pg` route below (keeps app_route mutually exclusive).
+      authentik: 'starts_with(to_string(.subject) ?? "", "logs.k8s.authentik.") && !ends_with(to_string(.subject) ?? "", ".postgres")'
+      # k8s: Traefik ingress — JSON access logs (requires logs.access.format=json,
+      # flipped in the traefik-system overlay values in this same change).
+      traefik: 'starts_with(to_string(.subject) ?? "", "logs.k8s.traefik-system.")'
+      # VM: Vault/OpenBao file audit device (/var/log/vault_audit.log), JSON.
+      vault: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && contains(to_string(.file) ?? "", "vault_audit")'
+      # VM: nginx combined access log (/var/log/nginx/<vhost>_access.log).
+      nginx_access: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && contains(to_string(.file) ?? "", "nginx") && ends_with(to_string(.file) ?? "", "access.log")'
+      # VM: nginx error log (/var/log/nginx/<vhost>_error.log).
+      nginx_error: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && contains(to_string(.file) ?? "", "nginx") && ends_with(to_string(.file) ?? "", "error.log")'
+      # VM: HAProxy httplog via journald.
+      haproxy: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && ((to_string(.SYSLOG_IDENTIFIER) ?? "") == "haproxy" || (to_string(.program) ?? "") == "haproxy" || (to_string(.appname) ?? "") == "haproxy")'
+      # VM: glauth LDAP — structuredlog (logrus) JSON.
+      glauth: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && (contains(to_string(.file) ?? "", "glauth") || (to_string(.SYSLOG_IDENTIFIER) ?? "") == "glauth" || (to_string(.program) ?? "") == "glauth" || (to_string(.appname) ?? "") == "glauth")'
+
+      # --- Tier-2 (stacks on #318) ---
+      # BIND query logs, k8s + VM. k8s: any bind-* namespace (bind-internal DNS
+      # servers, bind-system operator) — query logging enabled via `querylog yes`
+      # in the BindCluster extraOptions in this change. VM: puppet-managed named
+      # (file /var/log/named/*.log or journald `named`) — puppet-side enable is a
+      # required follow-up (profiles/dns/server.pp).
+      bind_query: 'starts_with(to_string(.subject) ?? "", "logs.k8s.bind") || (starts_with(to_string(.subject) ?? "", "logs.vm.") && (contains(to_string(.file) ?? "", "named") || (to_string(.SYSLOG_IDENTIFIER) ?? "") == "named" || (to_string(.program) ?? "") == "named" || (to_string(.appname) ?? "") == "named"))'
+      # k8s: Rancher audit log — JSON, emitted by the `rancher-audit-log` sidecar
+      # (auditLog.enabled level 1, already on in the cattle-system overlay).
+      rancher_audit: 'starts_with(to_string(.subject) ?? "", "logs.k8s.cattle-system.rancher-audit-log")'
+      # k8s: CNPG Postgres — ONE route for ALL clusters. The CNPG main container is
+      # always named `postgres`, so logs.k8s.<ns>.postgres uniquely identifies every
+      # cluster across all namespaces (authentik/litellm/artifactapi/woodpecker/
+      # puppet/paperclip/grafana/netbox/gitea/encapi). Mutually exclusive because the
+      # app routes above/below carve out `.postgres`.
+      cnpg_pg: 'starts_with(to_string(.subject) ?? "", "logs.k8s.") && ends_with(to_string(.subject) ?? "", ".postgres")'
+      # Gitea router/access logs. k8s: the new k8s gitea (ns gitea) with router +
+      # access logging enabled in the overlay values in this change — carve out
+      # `.postgres` (gitea-namespace CNPG). VM: puppet-managed gitea (file or
+      # journald `gitea`) — puppet-side log-format enable is a follow-up.
+      gitea: '(starts_with(to_string(.subject) ?? "", "logs.k8s.gitea.") && !ends_with(to_string(.subject) ?? "", ".postgres")) || (starts_with(to_string(.subject) ?? "", "logs.vm.") && (contains(to_string(.file) ?? "", "gitea") || (to_string(.SYSLOG_IDENTIFIER) ?? "") == "gitea" || (to_string(.program) ?? "") == "gitea" || (to_string(.appname) ?? "") == "gitea"))'
+      # PuppetServer / PuppetDB. k8s: openvoxserver/openvoxdb stdout (ns puppet) —
+      # carve out `.postgres` (puppet-namespace CNPG). VM file logs (multiline
+      # logback + puppetserver-access.log) are a puppet-side vector concern (the
+      # multiline join must happen at the edge) — follow-up.
+      puppet: 'starts_with(to_string(.subject) ?? "", "logs.k8s.puppet.") && !ends_with(to_string(.subject) ?? "", ".postgres")'
+      # k8s: LiteLLM request logs — JSON once JSON_LOGS=True (flipped in the litellm
+      # env in this change). Carve out `.postgres` (litellm-namespace CNPG).
+      litellm: 'starts_with(to_string(.subject) ?? "", "logs.k8s.litellm.") && !ends_with(to_string(.subject) ?? "", ".postgres")'
+      # VM: Postfix maillog — journald (SYSLOG_IDENTIFIER postfix/*) or file maillog.
+      postfix: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && (starts_with(to_string(.SYSLOG_IDENTIFIER) ?? "", "postfix") || starts_with(to_string(.program) ?? "", "postfix") || starts_with(to_string(.appname) ?? "", "postfix") || contains(to_string(.file) ?? "", "maillog"))'
+
+  # Stage 2: generic catch-all for everything app_route did not claim.
+  route:
+    type: route
+    inputs:
+      - app_route._unmatched
     route:
       k8s: 'starts_with(to_string(.subject) ?? "", "logs.k8s.")'
       vm: 'starts_with(to_string(.subject) ?? "", "logs.vm.")'
@@ -102,12 +176,707 @@ transforms:
         "fields": {}
       }
 
+  # --- Tier-1 per-app parse transforms (each emits the full logs.raw shape) ---
+
+  # Authentik SSO (k8s, ns authentik) — structlog JSON on stdout.
+  # LIVE NOW: authentik pods already stream to logs.k8s.authentik.*.
+  authentik_parse:
+    type: remap
+    inputs:
+      - app_route.authentik
+    source: |
+      node = to_string(.kubernetes.pod_node_name || "") ?? ""
+      pod = to_string(.kubernetes.pod_name || "") ?? ""
+      container = to_string(.kubernetes.container_name || "") ?? ""
+      strm = to_string(.stream || "") ?? ""
+      raw = to_string(.message || "") ?? ""
+      ev = object(parse_json(raw) ?? {}) ?? {}
+      ts = ev.timestamp || .timestamp || now()
+      user = ""
+      if is_string(ev.user) {
+        user = to_string(ev.user) ?? ""
+      } else if is_object(ev.user) {
+        user = to_string(ev.user.username) ?? ""
+      }
+      fields = compact({
+        "event": to_string(ev.event) ?? "",
+        "action": to_string(ev.action) ?? "",
+        "user": user,
+        "client_ip": to_string(ev.client_ip) ?? "",
+        "result": to_string(ev.result) ?? "",
+        "logger": to_string(ev.logger) ?? ""
+      }, string: true)
+      sev = to_string(ev.level) ?? ""
+      msg = raw
+      if ev.event != null {
+        msg = to_string(ev.event) ?? raw
+      }
+      . = {
+        "timestamp": ts,
+        "host": node,
+        "source": "k8s",
+        "namespace": "authentik",
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": sev,
+        "message": msg,
+        "labels": {"app": "authentik"},
+        "fields": fields
+      }
+
+  # Traefik ingress (k8s, ns traefik-system) — JSON access logs. Non-access
+  # traefik lines (app logs) simply parse to no access fields and keep .message.
+  # geoip on client_ip is a PREREQUISITE (no enrichment table yet — see PR note).
+  traefik_parse:
+    type: remap
+    inputs:
+      - app_route.traefik
+    source: |
+      node = to_string(.kubernetes.pod_node_name || "") ?? ""
+      pod = to_string(.kubernetes.pod_name || "") ?? ""
+      container = to_string(.kubernetes.container_name || "") ?? ""
+      strm = to_string(.stream || "") ?? ""
+      raw = to_string(.message || "") ?? ""
+      ev = object(parse_json(raw) ?? {}) ?? {}
+      ts = ev.StartUTC || ev.time || .timestamp || now()
+      status = ""
+      if ev.DownstreamStatus != null {
+        status = to_string(ev.DownstreamStatus) ?? ""
+      }
+      dur_ns = to_int(ev.Duration) ?? 0
+      dur_ms = ""
+      if dur_ns > 0 {
+        dur_ms = to_string(dur_ns / 1000000)
+      }
+      method = to_string(ev.RequestMethod) ?? ""
+      path = to_string(ev.RequestPath) ?? ""
+      fields = compact({
+        "route": to_string(ev.RouterName) ?? "",
+        "service": to_string(ev.ServiceName) ?? "",
+        "method": method,
+        "path": path,
+        "host": to_string(ev.RequestHost) ?? "",
+        "status": status,
+        "duration_ms": dur_ms,
+        "client_ip": to_string(ev.ClientHost) ?? "",
+        "protocol": to_string(ev.RequestProtocol) ?? ""
+      }, string: true)
+      msg = raw
+      if method != "" {
+        msg = method + " " + path + " " + status
+      }
+      . = {
+        "timestamp": ts,
+        "host": node,
+        "source": "k8s",
+        "namespace": "traefik-system",
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": "",
+        "message": msg,
+        "labels": {"app": "traefik"},
+        "fields": fields
+      }
+
+  # Vault/OpenBao file audit device (VM, /var/log/vault_audit.log) — JSON, one
+  # object per request/response. AWAITING VM VECTOR (in-cluster vault is quiet;
+  # lights up when the puppet vector rollout ships logs.vm.* with .file set).
+  vault_parse:
+    type: remap
+    inputs:
+      - app_route.vault
+    source: |
+      host = to_string(.host || .hostname || "") ?? ""
+      raw = to_string(.message || .msg || "") ?? ""
+      ev = object(parse_json(raw) ?? {}) ?? {}
+      ts = ev.time || .timestamp || .ts || now()
+      auth = object(ev.auth) ?? {}
+      req = object(ev.request) ?? {}
+      fields = compact({
+        "type": to_string(ev.type) ?? "",
+        "display_name": to_string(auth.display_name) ?? "",
+        "operation": to_string(req.operation) ?? "",
+        "path": to_string(req.path) ?? "",
+        "remote_address": to_string(req.remote_address) ?? "",
+        "error": to_string(ev.error) ?? ""
+      }, string: true)
+      op = to_string(req.operation) ?? ""
+      pth = to_string(req.path) ?? ""
+      msg = raw
+      if op != "" || pth != "" {
+        msg = op + " " + pth
+      }
+      . = {
+        "timestamp": ts,
+        "host": host,
+        "source": "vm",
+        "namespace": "",
+        "pod": "",
+        "container": "",
+        "stream": "",
+        "severity": "",
+        "message": msg,
+        "labels": {"app": "vault"},
+        "fields": fields
+      }
+
+  # nginx access log (VM) — combined/CLF + optional trailing request_time.
+  # AWAITING VM VECTOR. geoip on client_ip is a PREREQUISITE (see PR note).
+  nginx_access_parse:
+    type: remap
+    inputs:
+      - app_route.nginx_access
+    source: |
+      host = to_string(.host || .hostname || "") ?? ""
+      raw = to_string(.message || .msg || "") ?? ""
+      ts = .timestamp || .ts || now()
+      m = parse_regex(raw, r'^(?P<client_ip>\S+) \S+ (?P<user>\S+) \[(?P<time_local>[^\]]+)\] "(?P<method>\S+) (?P<path>\S+) (?P<protocol>[^"]*)" (?P<status>\d{3}) (?P<bytes>\d+|-) "(?P<referer>[^"]*)" "(?P<user_agent>[^"]*)"(?: (?P<request_time>[\d.]+))?') ?? {}
+      fields = compact({
+        "client_ip": to_string(m.client_ip),
+        "method": to_string(m.method),
+        "path": to_string(m.path),
+        "status": to_string(m.status),
+        "bytes": to_string(m.bytes),
+        "referer": to_string(m.referer),
+        "user_agent": to_string(m.user_agent),
+        "request_time": to_string(m.request_time)
+      }, string: true)
+      . = {
+        "timestamp": ts,
+        "host": host,
+        "source": "vm",
+        "namespace": "",
+        "pod": "",
+        "container": "",
+        "stream": "access",
+        "severity": "",
+        "message": raw,
+        "labels": {"app": "nginx", "log_type": "access"},
+        "fields": fields
+      }
+
+  # nginx error log (VM). AWAITING VM VECTOR.
+  nginx_error_parse:
+    type: remap
+    inputs:
+      - app_route.nginx_error
+    source: |
+      host = to_string(.host || .hostname || "") ?? ""
+      raw = to_string(.message || .msg || "") ?? ""
+      ts = .timestamp || .ts || now()
+      m = parse_regex(raw, r'^(?P<time_local>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?P<level>\w+)\] (?P<pid>\d+)#(?P<tid>\d+): (?:\*(?P<cid>\d+) )?(?P<err>.*)$') ?? {}
+      c = parse_regex(raw, r'client: (?P<client_ip>[0-9a-fA-F:.]+)') ?? {}
+      lvl = to_string(m.level)
+      err = to_string(m.err)
+      fields = compact({
+        "level": lvl,
+        "pid": to_string(m.pid),
+        "cid": to_string(m.cid),
+        "client_ip": to_string(c.client_ip),
+        "error": err
+      }, string: true)
+      msg = raw
+      if err != "" {
+        msg = err
+      }
+      . = {
+        "timestamp": ts,
+        "host": host,
+        "source": "vm",
+        "namespace": "",
+        "pod": "",
+        "container": "",
+        "stream": "error",
+        "severity": lvl,
+        "message": msg,
+        "labels": {"app": "nginx", "log_type": "error"},
+        "fields": fields
+      }
+
+  # HAProxy httplog (VM, journald). AWAITING VM VECTOR.
+  # httplog: %ci:%cp [%tr] %ft %b/%s %Tq/%Tw/%Tc/%Tr/%Tt %ST %B %CC %CS %tsc
+  #          %ac/%fc/%bc/%sc/%rc %sq/%bq {hdrs} "%r"
+  haproxy_parse:
+    type: remap
+    inputs:
+      - app_route.haproxy
+    source: |
+      host = to_string(.host || .hostname || "") ?? ""
+      raw = to_string(.message || .msg || "") ?? ""
+      ts = .timestamp || .ts || now()
+      m = parse_regex(raw, r'(?P<client_ip>\d{1,3}(?:\.\d{1,3}){3}):(?P<client_port>\d+) \[(?P<accept_date>[^\]]+)\] (?P<frontend>\S+) (?P<backend>[^/ ]+)/(?P<server>\S+) (?P<tq>-?\d+)/(?P<tw>-?\d+)/(?P<tc>-?\d+)/(?P<tr>-?\d+)/(?P<tt>[+-]?\d+) (?P<status>\d{3}) (?P<bytes>\d+) \S+ \S+ (?P<termination_state>\S{4}) (?P<actconn>\d+)/(?P<feconn>\d+)/(?P<beconn>\d+)/(?P<srvconn>\d+)/(?P<retries>\d+) (?P<srv_queue>\d+)/(?P<backend_queue>\d+)') ?? {}
+      fields = compact({
+        "client_ip": to_string(m.client_ip),
+        "frontend": to_string(m.frontend),
+        "backend": to_string(m.backend),
+        "server": to_string(m.server),
+        "tq": to_string(m.tq),
+        "tw": to_string(m.tw),
+        "tc": to_string(m.tc),
+        "tr": to_string(m.tr),
+        "tt": to_string(m.tt),
+        "termination_state": to_string(m.termination_state),
+        "retries": to_string(m.retries),
+        "status": to_string(m.status),
+        "bytes": to_string(m.bytes)
+      }, string: true)
+      . = {
+        "timestamp": ts,
+        "host": host,
+        "source": "vm",
+        "namespace": "",
+        "pod": "",
+        "container": "",
+        "stream": "",
+        "severity": "",
+        "message": raw,
+        "labels": {"app": "haproxy"},
+        "fields": fields
+      }
+
+  # glauth LDAP (VM) — structuredlog (logrus) JSON. AWAITING VM VECTOR.
+  glauth_parse:
+    type: remap
+    inputs:
+      - app_route.glauth
+    source: |
+      host = to_string(.host || .hostname || "") ?? ""
+      raw = to_string(.message || .msg || "") ?? ""
+      ev = object(parse_json(raw) ?? {}) ?? {}
+      ts = ev.time || .timestamp || .ts || now()
+      binddn = to_string(ev.bindDN) ?? ""
+      if binddn == "" {
+        binddn = to_string(ev.binddn) ?? ""
+      }
+      remote = to_string(ev.src) ?? ""
+      if remote == "" {
+        remote = to_string(ev.remoteAddr) ?? ""
+      }
+      lvl = to_string(ev.level) ?? ""
+      gmsg = to_string(ev.msg) ?? ""
+      success = "false"
+      if contains(downcase(gmsg), "success") || (lvl == "info" && contains(downcase(gmsg), "bind")) {
+        success = "true"
+      }
+      fields = compact({
+        "bindDN": binddn,
+        "remote": remote,
+        "success": success,
+        "level": lvl,
+        "msg": gmsg
+      }, string: true)
+      msg = gmsg
+      if msg == "" {
+        msg = raw
+      }
+      . = {
+        "timestamp": ts,
+        "host": host,
+        "source": "vm",
+        "namespace": "",
+        "pod": "",
+        "container": "",
+        "stream": "",
+        "severity": lvl,
+        "message": msg,
+        "labels": {"app": "glauth"},
+        "fields": fields
+      }
+
+  # --- Tier-2 per-app parse transforms (stacks on #318) ---
+
+  # BIND query logs (k8s bind-* namespaces + VM named). LIVE on k8s once the
+  # `querylog yes` extraOptions (this change) roll out; VM AWAITS the puppet-side
+  # enable (profiles/dns/server.pp). rcode is NOT present in standard query-log
+  # lines (that needs response logging / dnstap) — extracted only if a
+  # response-style `status:` line is seen. Non-query lines keep .message.
+  bind_query_parse:
+    type: remap
+    inputs:
+      - app_route.bind_query
+    source: |
+      subj = to_string(.subject) ?? ""
+      is_k8s = starts_with(subj, "logs.k8s.")
+      raw = to_string(.message || .msg || "") ?? ""
+      ts = .timestamp || .ts || now()
+      node = ""
+      ns = ""
+      pod = ""
+      container = ""
+      strm = ""
+      hostv = ""
+      src = "vm"
+      if is_k8s {
+        src = "k8s"
+        node = to_string(.kubernetes.pod_node_name || "") ?? ""
+        ns = to_string(.kubernetes.pod_namespace || "") ?? ""
+        pod = to_string(.kubernetes.pod_name || "") ?? ""
+        container = to_string(.kubernetes.container_name || "") ?? ""
+        strm = to_string(.stream || "") ?? ""
+        hostv = node
+      } else {
+        hostv = to_string(.host || .hostname || "") ?? ""
+      }
+      m = parse_regex(raw, r'client\s+(?:@\S+\s+)?(?P<client_ip>[0-9a-fA-F:.]+)#(?P<port>\d+)(?:\s+\([^)]*\))?:\s+(?:view\s+(?P<view>\S+):\s+)?query:\s+(?P<qname>\S+)\s+(?P<qclass>\S+)\s+(?P<qtype>\S+)(?:\s+(?P<flags>\S+))?') ?? {}
+      rc = parse_regex(raw, r'status:\s+(?P<rcode>\w+)') ?? {}
+      fields = compact({
+        "client_ip": to_string(m.client_ip),
+        "qname": to_string(m.qname),
+        "qtype": to_string(m.qtype),
+        "qclass": to_string(m.qclass),
+        "view": to_string(m.view),
+        "flags": to_string(m.flags),
+        "rcode": to_string(rc.rcode)
+      }, string: true)
+      qn = to_string(m.qname)
+      msg = raw
+      if qn != "" {
+        msg = "query " + qn + " " + to_string(m.qtype)
+      }
+      . = {
+        "timestamp": ts,
+        "host": hostv,
+        "source": src,
+        "namespace": ns,
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": "",
+        "message": msg,
+        "labels": {"app": "bind"},
+        "fields": fields
+      }
+
+  # Rancher audit log (k8s, cattle-system rancher-audit-log sidecar) — JSON,
+  # auditLog level 1 (already enabled in the overlay). LIVE NOW.
+  rancher_audit_parse:
+    type: remap
+    inputs:
+      - app_route.rancher_audit
+    source: |
+      node = to_string(.kubernetes.pod_node_name || "") ?? ""
+      pod = to_string(.kubernetes.pod_name || "") ?? ""
+      container = to_string(.kubernetes.container_name || "") ?? ""
+      strm = to_string(.stream || "") ?? ""
+      raw = to_string(.message || "") ?? ""
+      ev = object(parse_json(raw) ?? {}) ?? {}
+      ts = ev.requestTimestamp || ev.time || .timestamp || now()
+      user = ""
+      if is_object(ev.user) {
+        user = to_string(ev.user.name) ?? ""
+      } else if is_string(ev.user) {
+        user = to_string(ev.user) ?? ""
+      }
+      verb = to_string(ev.method) ?? ""
+      if verb == "" { verb = to_string(ev.verb) ?? "" }
+      uri = to_string(ev.requestURI) ?? ""
+      if uri == "" { uri = to_string(ev.uri) ?? "" }
+      status = ""
+      if ev.responseCode != null { status = to_string(ev.responseCode) ?? "" }
+      if status == "" && is_object(ev.responseStatus) { status = to_string(ev.responseStatus.code) ?? "" }
+      fields = compact({
+        "user": user,
+        "verb": verb,
+        "uri": uri,
+        "status": status,
+        "auditID": to_string(ev.auditID) ?? "",
+        "remote_addr": to_string(ev.remoteAddr) ?? ""
+      }, string: true)
+      msg = raw
+      if verb != "" || uri != "" {
+        msg = verb + " " + uri + " " + status
+      }
+      . = {
+        "timestamp": ts,
+        "host": node,
+        "source": "k8s",
+        "namespace": "cattle-system",
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": "",
+        "message": msg,
+        "labels": {"app": "rancher", "log_type": "audit"},
+        "fields": fields
+      }
+
+  # CNPG Postgres — ONE transform for ALL clusters (10 namespaces). The instance
+  # manager wraps postgres logs as JSON on stdout; the postgres CSV columns nest
+  # under `.record` (logger == "postgres"). Non-postgres lines (instance-manager
+  # operator logs) keep .message and set no PG fields. LIVE NOW.
+  cnpg_pg_parse:
+    type: remap
+    inputs:
+      - app_route.cnpg_pg
+    source: |
+      node = to_string(.kubernetes.pod_node_name || "") ?? ""
+      ns = to_string(.kubernetes.pod_namespace || "") ?? ""
+      pod = to_string(.kubernetes.pod_name || "") ?? ""
+      container = to_string(.kubernetes.container_name || "") ?? ""
+      strm = to_string(.stream || "") ?? ""
+      raw = to_string(.message || "") ?? ""
+      cluster = to_string(.kubernetes.pod_labels."cnpg.io/cluster" || "") ?? ""
+      ev = object(parse_json(raw) ?? {}) ?? {}
+      ts = .timestamp || now()
+      rec = object(ev.record) ?? {}
+      logger = to_string(ev.logger) ?? ""
+      sev = ""
+      pgmsg = ""
+      fields = {}
+      if logger == "postgres" {
+        sev = to_string(rec.error_severity) ?? ""
+        pgmsg = to_string(rec.message) ?? ""
+        dm = parse_regex(pgmsg, r'duration:\s+(?P<ms>[0-9.]+)\s+ms') ?? {}
+        fields = compact({
+          "error_severity": sev,
+          "message": pgmsg,
+          "query": to_string(rec.query) ?? "",
+          "duration_ms": to_string(dm.ms),
+          "user": to_string(rec.user_name) ?? "",
+          "database": to_string(rec.database_name) ?? ""
+        }, string: true)
+      }
+      lbls = {"app": "cnpg"}
+      if cluster != "" {
+        lbls = {"app": "cnpg", "cluster": cluster}
+      }
+      msg = raw
+      if pgmsg != "" { msg = pgmsg }
+      . = {
+        "timestamp": ts,
+        "host": node,
+        "source": "k8s",
+        "namespace": ns,
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": sev,
+        "message": msg,
+        "labels": lbls,
+        "fields": fields
+      }
+
+  # Gitea router/access logs (k8s gitea + VM gitea). Router "completed" lines give
+  # method/path/status/latency; NCSA access lines give method/path/status/user.
+  # k8s LIVE once the overlay log config (this change) rolls out; VM AWAITS the
+  # puppet-side log-format enable.
+  gitea_parse:
+    type: remap
+    inputs:
+      - app_route.gitea
+    source: |
+      subj = to_string(.subject) ?? ""
+      is_k8s = starts_with(subj, "logs.k8s.")
+      raw = to_string(.message || .msg || "") ?? ""
+      ts = .timestamp || .ts || now()
+      node = ""
+      ns = ""
+      pod = ""
+      container = ""
+      strm = ""
+      hostv = ""
+      src = "vm"
+      if is_k8s {
+        src = "k8s"
+        node = to_string(.kubernetes.pod_node_name || "") ?? ""
+        ns = to_string(.kubernetes.pod_namespace || "") ?? ""
+        pod = to_string(.kubernetes.pod_name || "") ?? ""
+        container = to_string(.kubernetes.container_name || "") ?? ""
+        strm = to_string(.stream || "") ?? ""
+        hostv = node
+      } else {
+        hostv = to_string(.host || .hostname || "") ?? ""
+      }
+      r = parse_regex(raw, r'completed (?P<method>\S+) (?P<path>\S+) for (?P<client>\S+), (?P<status>\d{3}) [^ ]+ in (?P<latency>[0-9.]+\w+)') ?? {}
+      a = parse_regex(raw, r'^(?P<client_ip>\S+) \S+ (?P<user>\S+) \[[^\]]+\] "(?P<method>\S+) (?P<path>\S+) [^"]*" (?P<status>\d{3})') ?? {}
+      method = to_string(r.method)
+      if method == "" { method = to_string(a.method) }
+      path = to_string(r.path)
+      if path == "" { path = to_string(a.path) }
+      status = to_string(r.status)
+      if status == "" { status = to_string(a.status) }
+      user = to_string(a.user)
+      if user == "-" { user = "" }
+      fields = compact({
+        "method": method,
+        "path": path,
+        "status": status,
+        "latency": to_string(r.latency),
+        "user": user,
+        "client_ip": to_string(a.client_ip)
+      }, string: true)
+      msg = raw
+      if method != "" {
+        msg = method + " " + path + " " + status
+      }
+      . = {
+        "timestamp": ts,
+        "host": hostv,
+        "source": src,
+        "namespace": ns,
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": "",
+        "message": msg,
+        "labels": {"app": "gitea"},
+        "fields": fields
+      }
+
+  # PuppetServer / PuppetDB (k8s openvoxserver/openvoxdb stdout, ns puppet). Per
+  # line logback parse (level/logger/message + node) and an access-log line
+  # (method/status/node) where present. VM multiline stacktrace join +
+  # puppetserver-access.log are a puppet-side edge concern (follow-up). LIVE NOW.
+  puppet_parse:
+    type: remap
+    inputs:
+      - app_route.puppet
+    source: |
+      node = to_string(.kubernetes.pod_node_name || "") ?? ""
+      pod = to_string(.kubernetes.pod_name || "") ?? ""
+      container = to_string(.kubernetes.container_name || "") ?? ""
+      strm = to_string(.stream || "") ?? ""
+      raw = to_string(.message || "") ?? ""
+      ts = .timestamp || now()
+      lb = parse_regex(raw, r'^(?P<ts>\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}[,.]\d+)\s+(?P<level>[A-Z]+)\s+\[(?P<thread>[^\]]*)\]\s+\[(?P<logger>[^\]]*)\]\s+(?P<msg>.*)$') ?? {}
+      ac = parse_regex(raw, r'^(?P<client_ip>\S+) \S+ \S+ \[[^\]]+\] "(?P<method>\S+) (?P<path>\S+) [^"]*" (?P<status>\d{3})') ?? {}
+      nd = parse_regex(raw, r'(?:catalog for|for node)\s+(?P<node>[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+)') ?? {}
+      lvl = to_string(lb.level)
+      pmsg = to_string(lb.msg)
+      err = ""
+      if lvl == "ERROR" { err = pmsg }
+      fields = compact({
+        "level": lvl,
+        "logger": to_string(lb.logger),
+        "node": to_string(nd.node),
+        "method": to_string(ac.method),
+        "status": to_string(ac.status),
+        "path": to_string(ac.path),
+        "client_ip": to_string(ac.client_ip),
+        "error": err
+      }, string: true)
+      msg = raw
+      if pmsg != "" { msg = pmsg }
+      . = {
+        "timestamp": ts,
+        "host": node,
+        "source": "k8s",
+        "namespace": "puppet",
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": lvl,
+        "message": msg,
+        "labels": {"app": "puppet"},
+        "fields": fields
+      }
+
+  # LiteLLM request logs (k8s) — JSON once JSON_LOGS=True (flipped in the litellm
+  # env this change). parse_json -> model/tokens/latency/key/status (best-effort
+  # against litellm's JSON schema); non-JSON lines keep .message. Field keys light
+  # up once the env flip rolls out.
+  litellm_parse:
+    type: remap
+    inputs:
+      - app_route.litellm
+    source: |
+      node = to_string(.kubernetes.pod_node_name || "") ?? ""
+      pod = to_string(.kubernetes.pod_name || "") ?? ""
+      container = to_string(.kubernetes.container_name || "") ?? ""
+      strm = to_string(.stream || "") ?? ""
+      raw = to_string(.message || "") ?? ""
+      ev = object(parse_json(raw) ?? {}) ?? {}
+      ts = ev.timestamp || .timestamp || now()
+      sev = to_string(ev.level) ?? ""
+      lmsg = to_string(ev.message) ?? ""
+      fields = compact({
+        "model": to_string(ev.model) ?? "",
+        "tokens": to_string(ev.total_tokens) ?? "",
+        "latency": to_string(ev.response_time) ?? "",
+        "key": to_string(ev.api_key) ?? "",
+        "status": to_string(ev.status) ?? "",
+        "user": to_string(ev.user) ?? ""
+      }, string: true)
+      msg = raw
+      if lmsg != "" { msg = lmsg }
+      . = {
+        "timestamp": ts,
+        "host": node,
+        "source": "k8s",
+        "namespace": "litellm",
+        "pod": pod,
+        "container": container,
+        "stream": strm,
+        "severity": sev,
+        "message": msg,
+        "labels": {"app": "litellm"},
+        "fields": fields
+      }
+
+  # Postfix maillog (VM) — best-effort PER-LINE parse (qid + from/to/status/relay/
+  # delay). Full qid-lifecycle correlation is a query-time GROUP BY qid in
+  # ClickHouse, NOT a stateless-aggregator job (stitching the multi-line lifecycle
+  # needs a stateful reduce). AWAITS VM VECTOR.
+  postfix_parse:
+    type: remap
+    inputs:
+      - app_route.postfix
+    source: |
+      host = to_string(.host || .hostname || "") ?? ""
+      raw = to_string(.message || .msg || "") ?? ""
+      ts = .timestamp || .ts || now()
+      prog = to_string(.SYSLOG_IDENTIFIER || .program || .appname || "") ?? ""
+      q = parse_regex(raw, r'^(?P<qid>[0-9A-F]{6,}):') ?? {}
+      frm = parse_regex(raw, r'from=<(?P<from>[^>]*)>') ?? {}
+      rcpt = parse_regex(raw, r'to=<(?P<to>[^>]*)>') ?? {}
+      st = parse_regex(raw, r'status=(?P<status>\w+)') ?? {}
+      rel = parse_regex(raw, r'relay=(?P<relay>[^,]+)') ?? {}
+      dly = parse_regex(raw, r'delay=(?P<delay>[0-9.]+)') ?? {}
+      fields = compact({
+        "qid": to_string(q.qid),
+        "from": to_string(frm.from),
+        "to": to_string(rcpt.to),
+        "status": to_string(st.status),
+        "relay": to_string(rel.relay),
+        "delay": to_string(dly.delay),
+        "program": prog
+      }, string: true)
+      . = {
+        "timestamp": ts,
+        "host": host,
+        "source": "vm",
+        "namespace": "",
+        "pod": "",
+        "container": "",
+        "stream": "",
+        "severity": "",
+        "message": raw,
+        "labels": {"app": "postfix"},
+        "fields": fields
+      }
+
 sinks:
   clickhouse:
     type: clickhouse
     inputs:
       - k8s_shape
       - vm_shape
+      - authentik_parse
+      - traefik_parse
+      - vault_parse
+      - nginx_access_parse
+      - nginx_error_parse
+      - haproxy_parse
+      - glauth_parse
+      - bind_query_parse
+      - rancher_audit_parse
+      - cnpg_pg_parse
+      - gitea_parse
+      - puppet_parse
+      - litellm_parse
+      - postfix_parse
     endpoint: http://clickhouse-logs.logging.svc.cluster.local:8123
     database: logs
     table: raw

apps/base/logging/vector/archiver.yaml

@@ -1,62 +0,0 @@
----
-# Vector ARCHIVER tier — long-term raw-log backup to S3 (Ceph RGW). Independent
-# durable JetStream consumer (`archiver`) so its offsets/lag are fully isolated
-# from the ClickHouse transform path (archive lag can never stall ingest — true
-# fan-out). Writes RAW, pre-transform events (as they sit in JetStream) as
-# gzipped NDJSON, partitioned by subject + date. This is the long-horizon replay
-# source beyond JetStream's 3d retention window.
-data_dir: /vector-data-dir
-
-api:
-  enabled: true
-  address: 0.0.0.0:8686
-
-sources:
-  js_archive:
-    type: nats
-    url: nats://nats.logging.svc.cluster.local:4222
-    connection_name: vector-archiver
-    subject: "logs.>"
-    jetstream:
-      stream: LOGS
-      consumer: archiver
-    auth:
-      strategy: user_password
-      user_password:
-        user: log-consumer
-        password: ${NATS_CONSUMER_PASSWORD}
-    decoding:
-      codec: json
-
-sinks:
-  s3:
-    type: aws_s3
-    inputs:
-      - js_archive
-    bucket: logs-archive
-    endpoint: https://s3.ceph.unkin.net
-    region: us-east-1
-    force_path_style: true
-    tls:
-      ca_file: /etc/vault-ca/ca.crt
-    # AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY come from the logs-archive-s3
-    # Secret (cephrgw-operator) via envFrom on the deployment.
-    key_prefix: "raw/{{ subject }}/%Y/%m/%d/"
-    compression: gzip
-    encoding:
-      codec: json
-    framing:
-      method: newline_delimited
-    filename_time_format: "%Y%m%dT%H%M%SZ"
-    filename_append_uuid: true
-    batch:
-      max_bytes: 134217728
-      timeout_secs: 300
-    buffer:
-      type: memory
-      max_events: 5000
-      when_full: block
-    # Disabled so slow BucketAccess credential propagation doesn't crash-loop
-    # the pod; RGW reachability is proven by the operator's own health.
-    healthcheck:
-      enabled: false

apps/base/netbox/cnpg_backup.yaml

@@ -25,6 +25,7 @@ metadata:
   name: cnpg-netbox
   namespace: netbox
 spec:
+  placementTarget: ec
   bucketName: cnpg-netbox
   ownerRef: cnpg-netbox-backup
   versioning: false

apps/base/paperclip/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-paperclip
   namespace: paperclip
 spec:
+  placementTarget: ec
   bucketName: cnpg-paperclip
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.

apps/base/paperclip/cnpg_cluster.yaml

@@ -48,7 +48,7 @@ spec:
   enablePDB: true
   enableSuperuserAccess: false
   failoverDelay: 0
-  imageName: ghcr.io/cloudnative-pg/postgresql:17-minimal-trixie
+  imageName: ghcr.io/cloudnative-pg/postgresql:17-system-trixie
   instances: 3
   logLevel: info
   maxSyncReplicas: 0

apps/base/pdbmux/deployment.yaml

@@ -5,7 +5,7 @@ metadata:
   name: pdbmux
   namespace: pdbmux
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
 spec:
   replicas: 2
   selector:
@@ -25,7 +25,7 @@ spec:
         - name: pdbmux
           # Image is published by the pdbmux repo's .woodpecker/docker.yaml on
           # a v* tag. It only exists after that tag is cut (see PR merge gates).
-          image: git.unkin.net/unkin/pdbmux:v0.1.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/pdbmux:v0.1.0
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8080

apps/base/puppet/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-puppet
   namespace: puppet
 spec:
+  placementTarget: ec
   bucketName: cnpg-puppet
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.

apps/base/puppet/cnpg_cluster.yaml

@@ -69,7 +69,7 @@ spec:
   enablePDB: true
   enableSuperuserAccess: false
   failoverDelay: 0
-  imageName: ghcr.io/cloudnative-pg/postgresql:17-minimal-trixie
+  imageName: ghcr.io/cloudnative-pg/postgresql:17-system-trixie
   instances: 3
   logLevel: info
   maxSyncReplicas: 0

apps/base/puppet/cronjob_generate-types.yaml

@@ -52,20 +52,16 @@ spec:
               securityContext:
                 runAsUser: 0
                 runAsNonRoot: false
+                allowPrivilegeEscalation: false
+                # Root to `gem install` into the image's root-owned gem dir and
+                # to `runuser` for `puppet generate types` (SETUID/SETGID).
                 capabilities:
                   add:
-                    - CAP_CHOWN
-                    - CAP_SETUID
-                    - CAP_SETGID
-                    - CAP_DAC_OVERRIDE
-                    - CAP_AUDIT_WRITE
-                    - CAP_FOWNER
                     - CHOWN
-                    - SETUID
-                    - SETGID
                     - DAC_OVERRIDE
-                    - AUDIT_WRITE
                     - FOWNER
+                    - SETGID
+                    - SETUID
                   drop:
                     - all
               volumeMounts:
@@ -76,6 +72,8 @@ spec:
           restartPolicy: OnFailure
           securityContext:
             fsGroup: 999
+            seccompProfile:
+              type: RuntimeDefault
           volumes:
             - name: puppet-code-volume
               persistentVolumeClaim:

apps/base/puppet/deployment_puppetboard.yaml

@@ -21,7 +21,7 @@ spec:
   template:
     metadata:
       annotations:
-        reloader.stakater.com/auto: "true"
+        configmap.reloader.stakater.com/auto: "true"
       labels:
         app.kubernetes.io/component: puppetboard
         app.kubernetes.io/instance: puppetserver
@@ -29,6 +29,11 @@ spec:
         app.kubernetes.io/version: 8.8.0
     spec:
       enableServiceLinks: false
+      securityContext:
+        fsGroup: 1000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
       initContainers:
         - name: wait-puppetserver
           image: curlimages/curl:8.11.1
@@ -115,9 +120,6 @@ spec:
               chmod 600 ${CERT_DIR}/${HOSTNAME}.key
               chmod 644 ${CERT_DIR}/ca.pem
 
-              # Change ownership to puppetboard user (1000:1000)
-              chown -R 1000:1000 ${CERT_DIR}
-
               echo "Certificate generation completed for ${HOSTNAME}"
           volumeMounts:
             - name: puppetboard-certs
@@ -130,9 +132,13 @@ spec:
               cpu: 50m
               memory: 64Mi
           securityContext:
-            runAsUser: 0
+            runAsUser: 1000
-            runAsGroup: 0
+            runAsGroup: 1000
-            allowPrivilegeEscalation: true
+            runAsNonRoot: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - all
       containers:
         - name: puppetboard
           image: ghcr.io/voxpupuli/puppetboard:7.0.1

apps/base/puppet/deployment_puppetdb.yaml

@@ -20,7 +20,7 @@ spec:
   template:
     metadata:
       annotations:
-        reloader.stakater.com/auto: "true"
+        configmap.reloader.stakater.com/auto: "true"
       labels:
         app.kubernetes.io/component: puppetdb
         app.kubernetes.io/instance: puppetserver
@@ -49,6 +49,10 @@ spec:
             - configMapRef:
                 name: puppetdb-config
           env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
             - name: OPENVOXDB_POSTGRES_PASSWORD
               valueFrom:
                 secretKeyRef:
@@ -71,23 +75,24 @@ spec:
                   name: postgres-read-credentials
           securityContext:
             allowPrivilegeEscalation: false
+            # Root entrypoint drops to the puppetdb user via `runuser` (needs
+            # SETUID/SETGID) after chowning SSL/data dirs (CHOWN). Cannot run
+            # non-root: the image entrypoint requires a root start.
             capabilities:
               add:
-                - CAP_FOWNER
-                - CAP_CHOWN
-                - CAP_SETUID
-                - CAP_SETGID
-                - CAP_DAC_OVERRIDE
-                - FOWNER
                 - CHOWN
-                - SETUID
-                - SETGID
                 - DAC_OVERRIDE
+                - FOWNER
+                - SETGID
+                - SETUID
               drop:
                 - all
           volumeMounts:
             - mountPath: /opt/puppetlabs/server/data/puppetdb
               name: puppetdb-storage
+            - mountPath: /opt/puppetlabs/server/data/puppetdb/stockpile
+              name: puppetdb-storage
+              subPathExpr: stockpile/$(POD_NAME)
             - mountPath: /etc/puppetlabs/puppetdb/conf.d/read-database.conf
               name: puppetdb-read-database-conf
               subPath: read-database.conf
@@ -98,7 +103,12 @@ spec:
             - sh
             - -c
           args:
-            - mkdir -p /opt/puppetlabs/server/data/puppetdb/logs && chown 999:999 /opt/puppetlabs/server/data/puppetdb/logs
+            - mkdir -p /opt/puppetlabs/server/data/puppetdb/logs /opt/puppetlabs/server/data/puppetdb/stockpile
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
           resources:
             limits:
               cpu: 20m
@@ -107,10 +117,19 @@ spec:
               cpu: 20m
               memory: 32Mi
           securityContext:
-            runAsUser: 0
+            runAsUser: 999
+            runAsGroup: 999
+            runAsNonRoot: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - all
           volumeMounts:
             - mountPath: /opt/puppetlabs/server/data/puppetdb
               name: puppetdb-storage
+            - mountPath: /opt/puppetlabs/server/data/puppetdb/stockpile
+              name: puppetdb-storage
+              subPathExpr: stockpile/$(POD_NAME)
 
         - name: pgchecker
           image: docker.io/busybox:1.37
@@ -163,6 +182,11 @@ spec:
             runAsGroup: 1000
             runAsNonRoot: true
             allowPrivilegeEscalation: false
+      securityContext:
+        fsGroup: 999
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
       volumes:
         - name: puppetdb-storage
           persistentVolumeClaim:

apps/base/puppet/deployment_puppetserver-compiler.yaml

@@ -2,7 +2,8 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "vault-ca-cert"
   labels:
     app.kubernetes.io/component: puppetserver-compilers
     app.kubernetes.io/instance: puppetserver
@@ -65,20 +66,16 @@ spec:
           timeoutSeconds: 20
         securityContext:
           allowPrivilegeEscalation: false
+          # Root entrypoint chowns baked-in dirs (CHOWN) then drops the JVM to
+          # the puppet user via `runuser` (needs SETUID/SETGID). Cannot run
+          # non-root: the image entrypoint requires a root start.
           capabilities:
             add:
-            - CAP_CHOWN
-            - CAP_SETUID
-            - CAP_SETGID
-            - CAP_DAC_OVERRIDE
-            - CAP_AUDIT_WRITE
-            - CAP_FOWNER
             - CHOWN
-            - SETUID
-            - SETGID
             - DAC_OVERRIDE
-            - AUDIT_WRITE
             - FOWNER
+            - SETGID
+            - SETUID
             drop:
             - all
         startupProbe:
@@ -158,19 +155,13 @@ spec:
         securityContext:
           runAsUser: 0
           runAsNonRoot: false
+          allowPrivilegeEscalation: false
+          # Runs as root to chown the mounted PVC dirs to puppet:puppet before
+          # the main container starts (CHOWN); does not drop privileges itself.
           capabilities:
             add:
-            - CAP_CHOWN
-            - CAP_SETUID
-            - CAP_SETGID
-            - CAP_DAC_OVERRIDE
-            - CAP_AUDIT_WRITE
-            - CAP_FOWNER
             - CHOWN
-            - SETUID
-            - SETGID
             - DAC_OVERRIDE
-            - AUDIT_WRITE
             - FOWNER
             drop:
             - all
@@ -211,6 +202,8 @@ spec:
           name: puppet-shared-bins
       securityContext:
         fsGroup: 999
+        seccompProfile:
+          type: RuntimeDefault
       volumes:
       - name: puppet-code-volume
         persistentVolumeClaim:

apps/base/puppet/deployment_puppetserver-master.yaml

@@ -2,7 +2,8 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "vault-ca-cert"
   labels:
     app.kubernetes.io/component: puppetserver
     app.kubernetes.io/instance: puppetserver
@@ -11,16 +12,17 @@ metadata:
   name: puppetserver-master
   namespace: puppet
 spec:
+  replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/component: puppetserver
       app.kubernetes.io/name: puppetserver
   strategy:
-    type: RollingUpdate
+    type: Recreate
   template:
     metadata:
       annotations:
-        reloader.stakater.com/auto: "true"
+        configmap.reloader.stakater.com/auto: "true"
       labels:
         app.kubernetes.io/component: puppetserver
         app.kubernetes.io/instance: puppetserver
@@ -64,20 +66,16 @@ spec:
             timeoutSeconds: 20
           securityContext:
             allowPrivilegeEscalation: false
+            # Root entrypoint chowns baked-in dirs (CHOWN) then drops the JVM to
+            # the puppet user via `runuser` (needs SETUID/SETGID). Cannot run
+            # non-root: the image entrypoint requires a root start.
             capabilities:
               add:
-                - CAP_CHOWN
-                - CAP_SETUID
-                - CAP_SETGID
-                - CAP_DAC_OVERRIDE
-                - CAP_AUDIT_WRITE
-                - CAP_FOWNER
                 - CHOWN
-                - SETUID
-                - SETGID
                 - DAC_OVERRIDE
-                - AUDIT_WRITE
                 - FOWNER
+                - SETGID
+                - SETUID
               drop:
                 - all
           startupProbe:
@@ -131,19 +129,13 @@ spec:
           securityContext:
             runAsUser: 0
             runAsNonRoot: false
+            allowPrivilegeEscalation: false
+            # Runs as root to chown the mounted PVC dirs to puppet:puppet before
+            # the main container starts (CHOWN); does not drop privileges itself.
             capabilities:
               add:
-                - CAP_CHOWN
-                - CAP_SETUID
-                - CAP_SETGID
-                - CAP_DAC_OVERRIDE
-                - CAP_AUDIT_WRITE
-                - CAP_FOWNER
                 - CHOWN
-                - SETUID
-                - SETGID
                 - DAC_OVERRIDE
-                - AUDIT_WRITE
                 - FOWNER
               drop:
                 - all
@@ -155,6 +147,8 @@ spec:
               subPath: check_for_masters.sh
       securityContext:
         fsGroup: 999
+        seccompProfile:
+          type: RuntimeDefault
       volumes:
         - name: puppet-ca-storage
           persistentVolumeClaim:

apps/base/puppet/horizontalpodautoscaler_puppetserver-masters-autoscaler.yaml

@@ -1,37 +0,0 @@
-apiVersion: autoscaling/v2
-kind: HorizontalPodAutoscaler
-metadata:
-  labels:
-    app.kubernetes.io/component: puppetserver
-    app.kubernetes.io/instance: puppetserver
-    app.kubernetes.io/name: puppetserver
-    app.kubernetes.io/version: 8.8.0
-  name: puppetserver-masters-autoscaler
-  namespace: puppet
-spec:
-  scaleTargetRef:
-    apiVersion: apps/v1
-    kind: Deployment
-    name: puppetserver-master
-  minReplicas: 2
-  maxReplicas: 5
-  metrics:
-  - resource:
-      name: cpu
-      target:
-        averageUtilization: 75
-        type: Utilization
-    type: Resource
-  behavior:
-    scaleUp:
-      stabilizationWindowSeconds: 60
-      policies:
-      - type: Percent
-        value: 50
-        periodSeconds: 15
-    scaleDown:
-      stabilizationWindowSeconds: 300
-      policies:
-      - type: Percent
-        value: 25
-        periodSeconds: 60

apps/base/puppet/kustomization.yaml

@@ -23,7 +23,6 @@ resources:
   - deployment_puppetdb.yaml
   - deployment_puppetserver-master.yaml
   - horizontalpodautoscaler_puppetserver-compilers-autoscaler.yaml
-  - horizontalpodautoscaler_puppetserver-masters-autoscaler.yaml
   - horizontalpodautoscaler_puppetserver-puppetboard-autoscaler.yaml
   - horizontalpodautoscaler_puppetserver-puppetdb-autoscaler.yaml
   - gateway_puppetboard.yaml

apps/base/puppet/vpa.yaml

@@ -52,9 +52,9 @@ kind: VerticalPodAutoscaler
 metadata:
   name: puppetserver-master-vpa
   namespace: puppet
-  # NOTE: this workload also has an HPA. updateMode Off is recommendation-only
+  # NOTE: the master is a pinned single replica (Recreate, no HPA) so the CA/master
-  # and does not act, so there is no HPA/VPA conflict today. Do not flip to Auto/
+  # never coexists. updateMode Off keeps this recommendation-only; do not flip to
-  # Initial without first moving the HPA off CPU/memory (VPA owns those under Auto).
+  # Auto/Initial, which would evict and briefly recreate the singleton pod.
 spec:
   targetRef:
     apiVersion: apps/v1

apps/base/woodpecker/cnpg_backup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: cnpg-woodpecker
   namespace: woodpecker
 spec:
+  placementTarget: ec
   bucketName: cnpg-woodpecker
   # The owner user has full control of its own bucket (read + write), which is
   # all the backup/restore identity needs — no extra BucketAccess grant.

apps/base/woodpecker/kustomization.yaml

@@ -7,10 +7,12 @@ resources:
   - cnpg_cluster.yaml
   - cnpg_backup.yaml
   - cnpg_pooler.yaml
+  - serviceaccount_kea_operator_ci.yaml
   - serviceaccount_terraform_artifactapi.yaml
   - serviceaccount_terraform_authentik.yaml
   - serviceaccount_terraform_enc.yaml
   - serviceaccount_terraform_git.yaml
+  - serviceaccount_terraform_infra.yaml
   - serviceaccount_terraform_prowlarr.yaml
   - serviceaccount_terraform_rancher.yaml
   - serviceaccount_terraform_radarr.yaml

apps/base/woodpecker/serviceaccount_kea_operator_ci.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kea-operator-ci
+  namespace: woodpecker

apps/base/woodpecker/serviceaccount_terraform_infra.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: terraform-infra
+  namespace: woodpecker

apps/overlays/au-syd1/authentik/values.yaml

@@ -75,7 +75,7 @@ global:
 server:
   replicas: 3
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
   ingress:
     enabled: false
   resources:
@@ -89,7 +89,7 @@ server:
 worker:
   replicas: 2
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
   resources:
     limits:
       cpu: "2"

apps/overlays/au-syd1/bind-external/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/bind-external

apps/overlays/au-syd1/cert-manager/values.yaml

@@ -5,6 +5,11 @@ config:
   apiVersion: controller.config.cert-manager.io/v1alpha1
   kind: ControllerConfiguration
   enableGatewayAPI: true
+  acmeDNS01:
+    recursiveNameservers:
+      - "8.8.8.8:53"
+      - "1.1.1.1:53"
+    recursiveNameserversOnly: true
 
 replicaCount: 2
 

apps/overlays/au-syd1/clickhouse-system/values.yaml

@@ -6,6 +6,12 @@
 # upstream). Upstream official images are used; no Docker Hardened Image variant
 # is adopted (DHI is subscription-gated and served from a private org namespace
 # not reachable via the anonymous artifactapi dockerhub proxy).
+#
+# Watch the logging namespace where the ClickHouseInstallation lives. The chart
+# default (watchNamespaces: []) makes the operator watch ONLY its own namespace
+# (clickhouse-system), so the logs CHI was never reconciled — set it explicitly.
+watchNamespaces:
+  - logging
 crdHook:
   image:
     repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/bitnami/kubectl

apps/overlays/au-syd1/dhcp-system/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/dhcp-system

apps/overlays/au-syd1/gitea/kustomization.yaml

@@ -0,0 +1,51 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../base/gitea
+
+helmCharts:
+  - name: gitea
+    repo: oci://docker.gitea.com/charts
+    version: "12.6.0"
+    releaseName: gitea
+    namespace: gitea
+    valuesFile: values.yaml
+
+# The chart renders a `helm.sh/hook: test` connection Pod (busybox). We deploy
+# via kustomize+ArgoCD (not `helm test`), so drop it rather than leave an orphan
+# Pod pulling an unallowlisted image.
+patches:
+  - target:
+      version: v1
+      kind: Pod
+      name: gitea-test-connection
+    patch: |-
+      $patch: delete
+      apiVersion: v1
+      kind: Pod
+      metadata:
+        name: gitea-test-connection
+  # The chart renders the http Service with `targetPort: null` (it defaults the
+  # target to the port name, which kustomize drops); pin it to 3000 so the
+  # manifest is valid and the HTTPRoute backend resolves.
+  - target:
+      version: v1
+      kind: Service
+      name: gitea-http
+    patch: |-
+      - op: replace
+        path: /spec/ports/0/targetPort
+        value: 3000
+  # SSH is disabled (DISABLE_SSH); drop the ssh Service the chart still renders.
+  - target:
+      version: v1
+      kind: Service
+      name: gitea-ssh
+    patch: |-
+      $patch: delete
+      apiVersion: v1
+      kind: Service
+      metadata:
+        name: gitea-ssh

apps/overlays/au-syd1/gitea/values.yaml

@@ -0,0 +1,193 @@
+# Gitea helm-gitea chart 12.6.0 (appVersion 1.26.1), app pinned to 1.26.2 to
+# match the VM being replaced. HA-shaped: 2 replicas on shared RWX CephFS, with
+# Postgres (CNPG), cache/session/queue (Valkey), and OIDC all externalised —
+# exactly the shape the chart's docs/ha-setup.md requires for replicaCount > 1.
+replicaCount: 2
+
+image:
+  registry: docker.io
+  repository: gitea/gitea
+  tag: "1.26.2"       # chart appends "-rootless" because rootless: true below
+  rootless: true
+  pullPolicy: IfNotPresent
+
+# All bundled stateful subcharts OFF — we run CNPG Postgres + standalone Valkey.
+postgresql-ha:
+  enabled: false
+postgresql:
+  enabled: false
+valkey-cluster:
+  enabled: false
+valkey:
+  enabled: false
+
+# Shared repo/LFS/attachment storage. RWX is mandatory for >1 replica so every
+# pod sees the same /data (repos, lfs, avatars, attachments).
+persistence:
+  enabled: true
+  create: true
+  mount: true
+  accessModes:
+    - ReadWriteMany
+  storageClass: cephfs-raid6-delete
+  size: 20Gi
+
+service:
+  http:
+    type: ClusterIP
+    port: 3000
+  # git-over-ssh is disabled — HTTPS clones only (the estate norm). DISABLE_SSH
+  # below stops the in-pod SSH server; the chart still renders a gitea-ssh
+  # Service, which the overlay kustomization $patch-deletes.
+  ssh:
+    type: ClusterIP
+    port: 22
+
+# Gateway API (HTTPRoute in the base) fronts HTTP; the chart Ingress is unused.
+ingress:
+  enabled: false
+
+# Rolling replicas on slow shared storage: keep one old pod up and give new pods
+# generous startup headroom so migrations on boot don't trip the probes.
+strategy:
+  type: RollingUpdate
+  rollingUpdate:
+    maxSurge: "100%"
+    maxUnavailable: 0
+
+resources:
+  requests:
+    cpu: "500m"
+    memory: 512Mi
+  limits:
+    cpu: "2"
+    memory: 2Gi
+
+# Trust the internal unkin.net CA that signs identity.unkin.net's cert. Gitea is
+# Go, so SSL_CERT_DIR adds the mounted vault-ca-cert to the system trust pool
+# (additive — public roots stay via the default bundle). Needed for the OIDC
+# discovery/JWKS fetch at login.
+deployment:
+  env:
+    - name: SSL_CERT_DIR
+      value: /etc/gitea/tls-ca
+extraVolumes:
+  - name: vault-ca-cert
+    secret:
+      secretName: vault-ca-cert
+      items:
+        - key: ca.crt
+          path: ca.crt
+extraContainerVolumeMounts:
+  - name: vault-ca-cert
+    mountPath: /etc/gitea/tls-ca
+    readOnly: true
+extraInitVolumeMounts:
+  - name: vault-ca-cert
+    mountPath: /etc/gitea/tls-ca
+    readOnly: true
+
+gitea:
+  # Local admin fallback (survives the OIDC cutover). Secret keys: username,
+  # password. Seeded in Vault -> synced to the gitea-admin Secret by VSO.
+  admin:
+    existingSecret: gitea-admin
+    email: "benvin@unkin.net"
+    passwordMode: keepUpdated
+
+  # DB password (and Gitea internal secrets) injected as env from VSO Secrets,
+  # never rendered into app.ini in git. GITEA__<section>__<KEY> maps to app.ini.
+  additionalConfigFromEnvs:
+    - name: GITEA__database__PASSWD
+      valueFrom:
+        secretKeyRef:
+          name: postgres-credentials
+          key: password
+    - name: GITEA__security__SECRET_KEY
+      valueFrom:
+        secretKeyRef:
+          name: gitea-inner
+          key: SECRET_KEY
+    - name: GITEA__security__INTERNAL_TOKEN
+      valueFrom:
+        secretKeyRef:
+          name: gitea-inner
+          key: INTERNAL_TOKEN
+
+  # Authentik OIDC login source, registered declaratively on boot. existingSecret
+  # must carry keys `key` (client id) and `secret` (client secret) — seeded at the
+  # same Vault path terraform-authentik reads client_secret from.
+  oauth:
+    - name: "authentik"
+      provider: "openidConnect"
+      existingSecret: oauth-credentials
+      autoDiscoverUrl: "https://identity.unkin.net/application/o/gitea/.well-known/openid-configuration"
+
+  config:
+    server:
+      DOMAIN: git.k8s.syd1.au.unkin.net
+      ROOT_URL: https://git.k8s.syd1.au.unkin.net/
+      SSH_DOMAIN: git.k8s.syd1.au.unkin.net
+      DISABLE_SSH: true
+      START_SSH_SERVER: false
+      LFS_START_SERVER: true
+
+    database:
+      DB_TYPE: postgres
+      HOST: gitea-postgres-pooler-rw:5432
+      NAME: gitea
+      USER: gitea
+      SSL_MODE: disable
+
+    # Standalone Valkey: session on db 0, cache on db 1, queue on db 2.
+    session:
+      PROVIDER: redis
+      PROVIDER_CONFIG: "redis://gitea-valkey:6379/0"
+    cache:
+      ENABLED: true
+      ADAPTER: redis
+      HOST: "redis://gitea-valkey:6379/1"
+    queue:
+      TYPE: redis
+      CONN_STR: "redis://gitea-valkey:6379/2"
+
+    # DB-backed issue indexer is replica-safe on shared storage (matches the VM);
+    # avoids the on-disk bleve indexer that HA can't share.
+    indexer:
+      ISSUE_INDEXER_TYPE: db
+      REPO_INDEXER_ENABLED: false
+
+    # CI stays on Woodpecker — Gitea Actions disabled.
+    actions:
+      ENABLED: false
+
+    # Router + NCSA access logs to stdout so the Tier-2 vector gitea pipeline can
+    # parse method/path/status/latency/user (subject logs.k8s.gitea.*).
+    log:
+      LEVEL: Info
+      MODE: console
+      ROUTER: console
+      ENABLE_ACCESS_LOG: true
+      ACCESS: console
+
+    repository:
+      DEFAULT_BRANCH: main
+      DEFAULT_PRIVATE: last
+
+    # OIDC auto-registration + account linking so existing local users (benvin,
+    # bots) link to their Authentik identity by matching username on first login.
+    service:
+      ENABLE_AUTO_REGISTRATION: true
+    oauth2_client:
+      ENABLE_AUTO_REGISTRATION: true
+      ACCOUNT_LINKING: auto
+      USERNAME: preferred_username
+      UPDATE_AVATAR: true
+
+  # Startup headroom for migrations on RWX storage across a rolling update.
+  startupProbe:
+    enabled: true
+  livenessProbe:
+    enabled: true
+  readinessProbe:
+    enabled: true

apps/overlays/au-syd1/logging/kustomization.yaml

@@ -36,10 +36,4 @@ helmCharts:
     releaseName: vector-aggregator
     namespace: logging
     valuesFile: values-vector-aggregator.yaml
-  # Archiver (Deployment): independent JetStream consumer -> raw logs to S3.
+  # Archiver: replaced by the logarchiver Deployment (apps/base/logging).
-  - name: vector
-    repo: https://helm.vector.dev
-    version: "0.57.0"
-    releaseName: vector-archiver
-    namespace: logging
-    valuesFile: values-vector-archiver.yaml

apps/overlays/au-syd1/logging/values-nats.yaml

@@ -22,16 +22,19 @@ config:
         size: 180Gi
         storageClassName: cephrbd-fast-delete
   # Per-user auth with publish/subscribe separation. Passwords are injected as
-  # env vars from the Vault-synced nats-auth Secret (NATS expands $VAR in config).
+  # env vars from the Vault-synced nats-auth Secret. The `<< $VAR >>` wrapping is
+  # REQUIRED by this chart: it renders the value UNQUOTED in nats.conf so the
+  # NATS server expands the env var. A plain `$VAR` is JSON-quoted ("$VAR") and
+  # NATS then treats it as a literal string — which broke auth for every client.
   merge:
     authorization:
       users:
         # Bootstrap Job (stream/consumer management) — full JetStream API.
         - user: log-admin
-          password: $NATS_ADMIN_PASSWORD
+          password: << $NATS_ADMIN_PASSWORD >>
         # Edge publishers (k8s DaemonSet + VM ingest) — publish only.
         - user: log-producer
-          password: $NATS_PRODUCER_PASSWORD
+          password: << $NATS_PRODUCER_PASSWORD >>
           permissions:
             publish:
               allow:
@@ -42,7 +45,7 @@ config:
         # Consumers (transform tier + archiver) — pull + ack only, no publish
         # to log subjects.
         - user: log-consumer
-          password: $NATS_CONSUMER_PASSWORD
+          password: << $NATS_CONSUMER_PASSWORD >>
           permissions:
             publish:
               allow:
@@ -83,12 +86,13 @@ container:
       cpu: "2"
       memory: 4Gi
 
-# Roll the StatefulSet when nats-auth changes.
+# Roll the StatefulSet on config changes only; nats-auth is Vault-rotated (VSO)
+# so it is deliberately not watched here (no restart on routine key rotation).
 podTemplate:
   merge:
     metadata:
       annotations:
-        reloader.stakater.com/auto: "true"
+        configmap.reloader.stakater.com/auto: "true"
 
 # Config-reloader sidecar image, also through artifactapi.
 reloader:

apps/overlays/au-syd1/logging/values-vector-agent.yaml

@@ -23,6 +23,9 @@ tolerations:
   - operator: Exists
 
 env:
+  # Vector 0.57 disables ${VAR} config interpolation by default; auth needs it.
+  - name: VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION
+    value: "true"
   - name: NATS_PRODUCER_PASSWORD
     valueFrom:
       secretKeyRef:
@@ -47,4 +50,4 @@ existingConfigMaps:
   - vector-agent-config
 
 workloadResourceAnnotations:
-  reloader.stakater.com/auto: "true"
+  configmap.reloader.stakater.com/auto: "true"

apps/overlays/au-syd1/logging/values-vector-aggregator.yaml

@@ -26,7 +26,7 @@ autoscaling:
   targetCPUUtilizationPercentage: 70
 
 workloadResourceAnnotations:
-  reloader.stakater.com/auto: "true"
+  configmap.reloader.stakater.com/auto: "true"
 
 podLabels:
   vector.dev/exclude: "true"
@@ -40,6 +40,10 @@ existingConfigMaps:
 
 # The ONLY place ClickHouse + NATS-consumer creds are consumed.
 env:
+  # Vector 0.57 disables ${VAR} config interpolation by default; the pipeline's
+  # auth (${CLICKHOUSE_*}, ${NATS_CONSUMER_PASSWORD}) needs it enabled.
+  - name: VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION
+    value: "true"
   - name: CLICKHOUSE_USER
     valueFrom:
       secretKeyRef:

apps/overlays/au-syd1/logging/values-vector-archiver.yaml

@@ -1,56 +0,0 @@
-# Vector ARCHIVER tier (Deployment) — independent JetStream consumer writing raw
-# logs to S3 (Ceph RGW). Isolated from the ClickHouse path (own durable
-# consumer). Pipeline: apps/base/logging/vector/archiver.yaml.
-role: Stateless-Aggregator
-fullnameOverride: vector-archiver
-replicas: 1
-
-image:
-  repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector
-  tag: 0.57.0-distroless-libc
-
-workloadResourceAnnotations:
-  reloader.stakater.com/auto: "true"
-
-podLabels:
-  vector.dev/exclude: "true"
-
-dataDir: /vector-data-dir
-existingConfigMaps:
-  - vector-archiver-config
-
-env:
-  - name: NATS_CONSUMER_PASSWORD
-    valueFrom:
-      secretKeyRef:
-        name: nats-auth
-        key: consumer_password
-
-# S3 creds (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY) from the cephrgw-operator
-# BucketAccess Secret.
-envFrom:
-  - secretRef:
-      name: logs-archive-s3
-
-# Trust the internal unkin.net Vault-PKI CA to verify s3.ceph.unkin.net.
-# vault-ca-cert is reflected into every namespace from the certificates ns.
-extraVolumes:
-  - name: vault-ca-cert
-    secret:
-      secretName: vault-ca-cert
-extraVolumeMounts:
-  - name: vault-ca-cert
-    mountPath: /etc/vault-ca/ca.crt
-    subPath: ca.crt
-    readOnly: true
-
-service:
-  enabled: false
-
-resources:
-  requests:
-    cpu: 100m
-    memory: 256Mi
-  limits:
-    cpu: "1"
-    memory: 1Gi