add internal mail edge A records

Recursing public names in-cluster SERVFAILs a few percent of cold lookups and loses the cache on every restart, which breaks CI. DNSBLs refuse or poison queries relayed via public resolvers, so rspamd's RBL lookups must keep resolving directly.

- add global forwarders 1.1.1.1 and 8.8.8.8 (forward first) to bind-resolvers
- add empty-forwarder zones to the openforwarder view for the RBL domains rspamd queries, so they resolve iteratively

Reviewed-on: #555
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

.woodpecker/vector-test.yaml

@@ -1,29 +0,0 @@
-when:
-  - event: pull_request
-
-steps:
-  - name: vector-test
-    image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector:0.57.0-debian
-    commands:
-      # Dummy creds + writable dirs so the full topologies build; the unit tests
-      # only exercise the transforms (sources are not started).
-      - export CLICKHOUSE_USER=ci CLICKHOUSE_PASSWORD=ci
-      - export NATS_PRODUCER_PASSWORD=ci NATS_CONSUMER_PASSWORD=ci
-      - mkdir -p /vector-data-dir /etc/vault-ca
-      - cp /etc/ssl/certs/ca-certificates.crt /etc/vault-ca/ca.crt
-      # Transform tier + VM ingest: unit-tested transforms.
-      - vector test apps/base/logging/vector/aggregator.yaml apps/base/logging/vector/aggregator-tests.yaml
-      - vector test apps/base/logging/vector/vm-ingest.yaml apps/base/logging/vector/vm-ingest-tests.yaml
-      # Agent has no transforms to unit-test; validate it builds. (The archiver
-      # leg is now the logarchiver service, not a Vector pipeline.)
-      - vector validate --no-environment apps/base/logging/vector/agent.yaml
-    backend_options:
-      kubernetes:
-        serviceAccountName: default
-        resources:
-          requests:
-            memory: 256Mi
-            cpu: 250m
-          limits:
-            memory: 1Gi
-            cpu: 1

apps/base/artifactapi/api-deployment.yaml

@@ -36,7 +36,7 @@ spec:
               mountPath: /combined-certs
       containers:
         - name: api
-          image: git.unkin.net/unkin/artifactapi:v3.11.2
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.14.0
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000

apps/base/artifactapi/configmap.yaml

@@ -5,6 +5,8 @@ metadata:
   name: api-env
   namespace: artifactapi
 data:
+  ARTIFACTAPI_OIDC_AUDIENCE: artifactapi
+  ARTIFACTAPI_OIDC_ISSUER: https://vault.service.consul:8200/v1/identity/oidc
   DBHOST: postgres-pooler
   DBNAME: artifacts
   DBPORT: "5432"

apps/base/artifactapi/image-keeper.yaml

@@ -0,0 +1,105 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: image-keeper
+  namespace: artifactapi
+spec:
+  selector:
+    matchLabels:
+      app: image-keeper
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: image-keeper
+    spec:
+      automountServiceAccountToken: false
+      priorityClassName: low
+      tolerations:
+        - operator: Exists
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        # artifactapi is distroless with no exit-0 flag, so run a static busybox as `true`
+        - name: copy-true
+          image: busybox:1.37.0-musl
+          imagePullPolicy: IfNotPresent
+          command: ["cp", "/bin/busybox", "/keeper/true"]
+          volumeMounts:
+            - name: keeper
+              mountPath: /keeper
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+        - name: api
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.14.0
+          imagePullPolicy: IfNotPresent
+          command: ["/keeper/true"]
+          volumeMounts:
+            - name: keeper
+              mountPath: /keeper
+              readOnly: true
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+        - name: ui
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.14.0
+          imagePullPolicy: IfNotPresent
+          command: ["true"]
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+      containers:
+        - name: pause
+          image: rancher/mirrored-pause:3.6
+          imagePullPolicy: IfNotPresent
+          resources:
+            limits:
+              cpu: 10m
+              memory: 16Mi
+            requests:
+              cpu: 1m
+              memory: 4Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+      volumes:
+        - name: keeper
+          emptyDir: {}

apps/base/artifactapi/kustomization.yaml

@@ -11,6 +11,7 @@ resources:
   - cnpg_pooler.yaml
   - gateway.yaml
   - httproute.yaml
+  - image-keeper.yaml
   - namespace.yaml
   - oauth2-proxy-configmap.yaml
   - oauth2-proxy-deployment.yaml

apps/base/artifactapi/oauth2-proxy-configmap.yaml

@@ -39,3 +39,8 @@ data:
   OAUTH2_PROXY_REVERSE_PROXY: "true"
   OAUTH2_PROXY_CODE_CHALLENGE_METHOD: "S256"
   OAUTH2_PROXY_SKIP_PROVIDER_BUTTON: "true"
+  # Back-channel discovery/token calls resolve the issuer inside the cluster,
+  # where it is served under the internal unkin.net CA rather than the publicly
+  # trusted cert the browser sees. Trust the bundle the combine-certs init
+  # container assembles, as every other oauth2-proxy in the estate does.
+  OAUTH2_PROXY_PROVIDER_CA_FILES: "/etc/ssl/combined/ca-certificates.crt"

apps/base/artifactapi/oauth2-proxy-deployment.yaml

@@ -6,7 +6,7 @@ metadata:
   namespace: artifactapi
   annotations:
     configmap.reloader.stakater.com/auto: "true"
-    secret.reloader.stakater.com/reload: "oauth-credentials"
+    secret.reloader.stakater.com/reload: "oauth-credentials,vault-ca-cert"
 spec:
   replicas: 2
   selector:
@@ -30,6 +30,36 @@ spec:
         fsGroup: 65532
         seccompProfile:
           type: RuntimeDefault
+      initContainers:
+        # The Authentik issuer is served behind the internal unkin.net CA;
+        # combine the system roots with it so oauth2-proxy's OIDC HTTP client
+        # trusts the discovery endpoint.
+        - name: combine-certs
+          image: docker.io/library/alpine:3
+          imagePullPolicy: IfNotPresent
+          command:
+            - sh
+            - -c
+            - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt
+          volumeMounts:
+            - name: vault-ca-cert
+              mountPath: /custom-ca
+              readOnly: true
+            - name: combined-certs
+              mountPath: /combined-certs
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 50m
+              memory: 32Mi
+            limits:
+              cpu: 200m
+              memory: 64Mi
       containers:
         - name: oauth2-proxy
           image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.3
@@ -83,6 +113,10 @@ spec:
             capabilities:
               drop:
                 - ALL
+          volumeMounts:
+            - name: combined-certs
+              mountPath: /etc/ssl/combined
+              readOnly: true
           resources:
             requests:
               cpu: 50m
@@ -90,4 +124,13 @@ spec:
             limits:
               cpu: 500m
               memory: 256Mi
+      volumes:
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: combined-certs
+          emptyDir: {}
       restartPolicy: Always

apps/base/artifactapi/ui-deployment.yaml

@@ -22,7 +22,7 @@ spec:
       automountServiceAccountToken: true
       containers:
         - name: ui
-          image: git.unkin.net/unkin/artifactapi-ui:v3.11.2
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.14.0
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 80

apps/base/authentik/cnpg_cluster.yaml

@@ -64,8 +64,12 @@ spec:
       archive_mode: "on"
       archive_timeout: 5min
       dynamic_shared_memory_type: posix
-      effective_cache_size: 256MB
+      effective_cache_size: 1536MB
       full_page_writes: "on"
+      # Replicas report their oldest xmin to the primary, so multi-second reads on
+      # a hot standby stop exhausting max_standby_streaming_delay and being
+      # cancelled. Retained-dead-tuple cost is negligible on a ~155MB database.
+      hot_standby_feedback: "on"
       log_destination: csvlog
       log_directory: /controller/log
       log_filename: postgres
@@ -77,7 +81,12 @@ spec:
       max_parallel_workers: "16"
       max_replication_slots: "16"
       max_worker_processes: "16"
-      shared_buffers: 128MB
+      # A pg_stat_statements.* parameter is what makes CNPG treat the extension as
+      # managed and run CREATE EXTENSION in every database; preloading alone does
+      # not create it.
+      pg_stat_statements.max: "10000"
+      pg_stat_statements.track: top
+      shared_buffers: 512MB
       shared_memory_type: mmap
       ssl_max_protocol_version: TLSv1.3
       ssl_min_protocol_version: TLSv1.3
@@ -86,6 +95,9 @@ spec:
       wal_log_hints: "on"
       wal_receiver_timeout: 5s
       wal_sender_timeout: 5s
+    # CNPG merges this with the libraries it manages itself.
+    shared_preload_libraries:
+      - pg_stat_statements
     syncReplicaElectionConstraint:
       enabled: false
   primaryUpdateMethod: restart
@@ -105,13 +117,16 @@ spec:
     updateInterval: 30
   resources:
     limits:
-      cpu: 500m
+      # 500m is a 50ms CFS quota per 100ms period, exhausted by bursts even at
+      # ~0.01 cores average, so every query pays throttle latency.
+      cpu: "2"
       # 512Mi OOMKilled replicas under load (shared_buffers 128MB +
       # max_connections 200 leave no headroom) — see incident 2026-07-28.
-      memory: 1Gi
+      # shared_buffers 512MB needs the same headroom multiple, hence 2Gi.
+      memory: 2Gi
     requests:
-      cpu: 50m
-      memory: 512Mi
+      cpu: 500m
+      memory: 1Gi
   smartShutdownTimeout: 180
   startDelay: 3600
   stopDelay: 1800

apps/base/authentik/httproute.yaml

@@ -37,6 +37,22 @@ spec:
       name: authentik
       sectionName: https
   rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: authentik-server
+          port: 80
+          weight: 1
+      filters:
+        - type: URLRewrite
+          urlRewrite:
+            path:
+              type: ReplaceFullPath
+              replaceFullPath: /application/o/token/
+      matches:
+        - path:
+            type: Exact
+            value: /application/o/token
     - backendRefs:
         - group: ""
           kind: Service
@@ -86,6 +102,22 @@ spec:
       name: authentik-internal
       sectionName: https
   rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: authentik-server
+          port: 80
+          weight: 1
+      filters:
+        - type: URLRewrite
+          urlRewrite:
+            path:
+              type: ReplaceFullPath
+              replaceFullPath: /application/o/token/
+      matches:
+        - path:
+            type: Exact
+            value: /application/o/token
     - backendRefs:
         - group: ""
           kind: Service

apps/base/authentik/kustomization.yaml

@@ -19,6 +19,7 @@ resources:
   - redis-deployment.yaml
   - redis-pvc.yaml
   - redis-service.yaml
+  - server-vmpodscrape.yaml
   - vaultauth.yaml
   - vaultstaticsecret.yaml
   - vmpodscrape.yaml

apps/base/authentik/server-vmpodscrape.yaml

@@ -0,0 +1,16 @@
+---
+# Scrape the authentik server's django_prometheus endpoint (:9300). Picked up
+# by the observability VMAgent (selectAllByDefault).
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: authentik-server
+  namespace: authentik
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: authentik
+      app.kubernetes.io/component: server
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics

apps/base/bind-external/acme-unkin-net/a/kustomization.yaml

@@ -3,4 +3,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-  - namespace.yaml
+  - ns1.yaml

apps/base/bind-external/acme-unkin-net/a/ns1.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: acme-ns1-a
+  namespace: bind-external
+spec:
+  zoneRef: acme-unkin-net
+  name: ns1
+  type: A
+  ttl: 3600
+  values:
+    # Public address of this cluster's external BIND, same target as
+    # acme-ns1.unkin.net. Resolvers that cached the seeded ns1.acme.unkin.net
+    # NS name must still reach the zone.
+    - 103.216.191.185

apps/base/bind-external/acme-unkin-net/kustomization.yaml

@@ -0,0 +1,11 @@
+---
+# Authoritative delegation records for acme.unkin.net. Without these the zone
+# only holds the operator's seed apex (NS ns1.acme.unkin.net glued to the
+# primary pod IP), which is unroutable off-cluster and goes stale on
+# reschedule. DNSRecords must live in the same namespace as their BindZone.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ns
+  - a

apps/base/bind-external/acme-unkin-net/ns/apex.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: acme-apex-ns
+  namespace: bind-external
+spec:
+  zoneRef: acme-unkin-net
+  # "@" is the zone apex.
+  name: "@"
+  type: NS
+  ttl: 3600
+  values:
+    # Matches the parent delegation in Google Cloud DNS. Out of zone, so the
+    # child needs no glue of its own.
+    - acme-ns1.unkin.net.

apps/base/bind-external/acme-unkin-net/ns/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - apex.yaml

apps/base/bind-external/kustomization.yaml

@@ -7,4 +7,5 @@ resources:
   - cluster.yaml
   - tsigkey.yaml
   - zones.yaml
+  - acme-unkin-net
   - agent-dns-rolebinding.yaml

apps/base/bind-external/zones.yaml

@@ -17,3 +17,14 @@ spec:
   updateKeyRef: certmanager
   allowTransfer:
     - key certmanager
+  # Published apex NS. acme-ns1 is what the parent delegates to and glues; ns1 is
+  # in-zone, so its address is declared below or a reseed would glue it to the
+  # primary pod IP.
+  nameservers:
+    - acme-ns1.unkin.net.
+    - ns1.acme.unkin.net.
+  records:
+    - name: ns1
+      type: A
+      ttl: 3600
+      values: ["103.216.191.185"]

apps/base/bind-internal/authoritative/ceph-unkin-net/cname/dashboard.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: dashboard-ceph-cname
+  namespace: bind-internal
+spec:
+  zoneRef: ceph-unkin-net
+  name: dashboard
+  type: CNAME
+  ttl: 600
+  values:
+    # Ceph mgr dashboard, reached via lb1. Lets in-cluster clients (the
+    # cephrgw-operator) resolve dashboard.ceph.unkin.net.
+    - lb1.unkin.net.

apps/base/bind-internal/authoritative/ceph-unkin-net/cname/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - dashboard.yaml
+  - s3.yaml

apps/base/bind-internal/authoritative/ceph-unkin-net/cname/s3.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: s3-ceph-cname
+  namespace: bind-internal
+spec:
+  zoneRef: ceph-unkin-net
+  name: s3
+  type: CNAME
+  ttl: 600
+  values:
+    # radosgw S3 endpoint. Points at the Consul service for now; the real
+    # target will be changed later.
+    - radosgw.service.consul.

apps/base/bind-internal/authoritative/ceph-unkin-net/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - cname

apps/base/bind-internal/authoritative/kustomization.yaml

@@ -2,9 +2,15 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+# Individually-managed authoritative records live under <zone>/<type>/<record>.yaml.
+# DNSRecords must live in the same namespace as their BindZone (the operator
+# resolves zoneRef/clusterRef/updateKeyRef within the record's namespace), so
+# these sit alongside the zone in bind-internal, not in the app namespace.
 resources:
   - cluster.yaml
   - tsigkey.yaml
   - zones.yaml
-  - records.yaml
+  - unkin-net
+  - main-unkin-net
+  - ceph-unkin-net
   - acls.yaml

apps/base/bind-internal/authoritative/main-unkin-net/a/jellyfin.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: jellyfin-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: main-unkin-net
+  name: jellyfin
+  type: A
+  ttl: 600
+  values:
+    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
+    - 198.18.199.0

apps/base/bind-internal/authoritative/main-unkin-net/a/kustomization.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - jellyfin.yaml
+  - mail-in.yaml
+  - lidarr.yaml
+  - nzbget.yaml
+  - prowlarr.yaml
+  - radarr.yaml
+  - readarr.yaml
+  - sonarr.yaml

apps/base/bind-internal/authoritative/main-unkin-net/a/lidarr.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: lidarr-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: main-unkin-net
+  name: lidarr
+  type: A
+  ttl: 600
+  values:
+    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
+    - 198.18.199.0

apps/base/bind-internal/authoritative/main-unkin-net/a/mail-in.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: mail-in-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: main-unkin-net
+  name: mail-in
+  type: A
+  ttl: 600
+  values:
+    # home WAN (public path for edge to smarthost relay), mirrors public DNS.
+    - 103.216.191.185

apps/base/bind-internal/authoritative/main-unkin-net/a/nzbget.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: nzbget-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: main-unkin-net
+  name: nzbget
+  type: A
+  ttl: 600
+  values:
+    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
+    - 198.18.199.0

apps/base/bind-internal/authoritative/main-unkin-net/a/prowlarr.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: prowlarr-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: main-unkin-net
+  name: prowlarr
+  type: A
+  ttl: 600
+  values:
+    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
+    - 198.18.199.0

apps/base/bind-internal/authoritative/main-unkin-net/a/radarr.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: radarr-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: main-unkin-net
+  name: radarr
+  type: A
+  ttl: 600
+  values:
+    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
+    - 198.18.199.0

apps/base/bind-internal/authoritative/main-unkin-net/a/readarr.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: readarr-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: main-unkin-net
+  name: readarr
+  type: A
+  ttl: 600
+  values:
+    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
+    - 198.18.199.0

apps/base/bind-internal/authoritative/main-unkin-net/a/sonarr.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: sonarr-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: main-unkin-net
+  name: sonarr
+  type: A
+  ttl: 600
+  values:
+    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
+    - 198.18.199.0

apps/base/bind-internal/authoritative/main-unkin-net/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - a

apps/base/bind-internal/authoritative/records.yaml

@@ -1,164 +0,0 @@
-# Individually-managed authoritative records for the unkin.net zone.
-# DNSRecords must live in the same namespace as their BindZone (the operator
-# resolves zoneRef/clusterRef/updateKeyRef within the record's namespace), so
-# these sit alongside the zone in bind-internal, not in the app namespace.
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  # "internal" in the name distinguishes this from the external DNS that
-  # Authentik will manage its own records from later.
-  name: identity-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: identity
-  type: A
-  ttl: 600
-  values:
-    # traefik-EXTERNAL (DMZ) gateway VIP; the authentik Gateway serves the
-    # identity.unkin.net hostname there.
-    - 198.18.199.0
----
-# PRODUCTION CUTOVER RECORD — intentionally commented out.
-# git.unkin.net currently resolves to the LIVE VM forge (HAProxy VRRP VIP
-# 198.18.19.17), which holds every repo the estate depends on. Uncommenting this
-# repoints the whole org's git.unkin.net at the new k8s Gitea gateway VIP, so it
-# is the FINAL step of the forge migration — gated on the data migration (gitea
-# dump/restore + SECRET_KEY copy) in argocd-apps docs/gitea-migration.md.
-# NOTE: the live git.unkin.net answer is served by the puppet DNS master today
-# (profiles::dns::master, records from PuppetDB); this k8s apex zone holds only
-# SOA+NS + a few DNSRecords so far. Confirm the k8s bind cluster is the live
-# authority for unkin.net (or update the puppet record instead) before relying
-# on this CR at cutover.
-# ---
-# apiVersion: bind.unkin.net/v1alpha1
-# kind: DNSRecord
-# metadata:
-#   name: git-dns-internal
-#   namespace: bind-internal
-# spec:
-#   zoneRef: unkin-net
-#   name: git
-#   type: A
-#   ttl: 600
-#   values:
-#     # traefik-internal gateway VIP; the gitea Gateway serves git.unkin.net there.
-#     - 198.18.200.4
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: s3-ceph-cname
-  namespace: bind-internal
-spec:
-  zoneRef: ceph-unkin-net
-  name: s3
-  type: CNAME
-  ttl: 600
-  values:
-    # radosgw S3 endpoint. Points at the Consul service for now; the real
-    # target will be changed later.
-    - radosgw.service.consul.
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: dashboard-ceph-cname
-  namespace: bind-internal
-spec:
-  zoneRef: ceph-unkin-net
-  name: dashboard
-  type: CNAME
-  ttl: 600
-  values:
-    # Ceph mgr dashboard, reached via lb1. Lets in-cluster clients (the
-    # cephrgw-operator) resolve dashboard.ceph.unkin.net.
-    - lb1.unkin.net.
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: lb1-unkin-net
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: lb1
-  type: A
-  ttl: 600
-  values:
-    - 103.216.191.185
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: ghp-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: ghp
-  type: A
-  ttl: 600
-  values:
-    # traefik-internal gateway VIP; the ghp Gateway serves ghp.unkin.net there.
-    - 198.18.200.4
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: arrstack-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: arrstack
-  type: A
-  ttl: 600
-  values:
-    # traefik-EXTERNAL (DMZ) gateway VIP; the arrproxy Gateway serves the
-    # arrstack.unkin.net front door (oauth2-proxy) there.
-    - 198.18.199.0
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: logviewer-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: logviewer
-  type: A
-  ttl: 600
-  values:
-    # traefik-internal gateway VIP; the logviewer Gateway serves
-    # logviewer.unkin.net there.
-    - 198.18.200.4
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: cheeztv-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: cheeztv
-  type: A
-  ttl: 600
-  values:
-    # traefik-internal gateway VIP; the cheeztv Gateway serves cheeztv.unkin.net
-    # there.
-    - 198.18.200.4
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: watchstate-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: watchstate
-  type: A
-  ttl: 600
-  values:
-    # traefik-EXTERNAL (DMZ) gateway VIP; the watchstate-external Gateway serves
-    # the watchstate.unkin.net front door (oauth2-proxy) there.
-    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/a/arrstack.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: arrstack-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: arrstack
+  type: A
+  ttl: 600
+  values:
+    # traefik-EXTERNAL (DMZ) gateway VIP; the arrproxy Gateway serves the
+    # arrstack.unkin.net front door (oauth2-proxy) there.
+    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/a/auth.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: auth-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: auth
+  type: A
+  ttl: 600
+  values:
+    # traefik-internal gateway VIP; the kanidm Gateway serves auth.unkin.net there.
+    - 198.18.200.4

apps/base/bind-internal/authoritative/unkin-net/a/cheeztv.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: cheeztv-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: cheeztv
+  type: A
+  ttl: 600
+  values:
+    # traefik-internal gateway VIP; the cheeztv Gateway serves cheeztv.unkin.net
+    # there.
+    - 198.18.200.4

apps/base/bind-internal/authoritative/unkin-net/a/fafflix.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: fafflix-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: fafflix
+  type: A
+  ttl: 600
+  values:
+    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
+    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/a/ghp.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: ghp-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: ghp
+  type: A
+  ttl: 600
+  values:
+    # traefik-internal gateway VIP; the ghp Gateway serves ghp.unkin.net there.
+    - 198.18.200.4

apps/base/bind-internal/authoritative/unkin-net/a/git-edge.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: git-edge-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: git
+  type: A
+  ttl: 600
+  values:
+    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
+    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/a/git.yaml

@@ -0,0 +1,23 @@
+---
+# PRODUCTION CUTOVER RECORD — intentionally commented out.
+# git.unkin.net currently resolves to the haproxy edge (git-edge.yaml), which
+# forwards to the LIVE VM forge holding every repo. Uncommenting this
+# repoints the whole org's git.unkin.net at the new k8s Gitea gateway VIP, so it
+# is the FINAL step of the forge migration — gated on the data migration (gitea
+# dump/restore + SECRET_KEY copy) in argocd-apps docs/gitea-migration.md.
+# Uncomment this record AND its entry in kustomization.yaml (removing
+# git-edge.yaml there) to activate it.
+# ---
+# apiVersion: bind.unkin.net/v1alpha1
+# kind: DNSRecord
+# metadata:
+#   name: git-dns-internal
+#   namespace: bind-internal
+# spec:
+#   zoneRef: unkin-net
+#   name: git
+#   type: A
+#   ttl: 600
+#   values:
+#     # traefik-internal gateway VIP; the gitea Gateway serves git.unkin.net there.
+#     - 198.18.200.4

apps/base/bind-internal/authoritative/unkin-net/a/grafana.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: grafana-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: grafana
+  type: A
+  ttl: 600
+  values:
+    # traefik-EXTERNAL (DMZ) gateway VIP; the grafana-external Gateway serves
+    # grafana.unkin.net there.
+    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/a/identity.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  # "internal" in the name distinguishes this from the external DNS that
+  # Authentik will manage its own records from later.
+  name: identity-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: identity
+  type: A
+  ttl: 600
+  values:
+    # traefik-EXTERNAL (DMZ) gateway VIP; the authentik Gateway serves the
+    # identity.unkin.net hostname there.
+    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/a/kustomization.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - arrstack.yaml
+  - auth.yaml
+  - cheeztv.yaml
+  - fafflix.yaml
+  # PRODUCTION CUTOVER RECORD — see git.yaml. Uncomment together with the
+  # record itself, and remove git-edge.yaml.
+  # - git.yaml
+  - ghp.yaml
+  - git-edge.yaml
+  - grafana.yaml
+  - identity.yaml
+  - lb1.yaml
+  - logviewer.yaml
+  - mail.yaml
+  - mta1-edge.yaml
+  - mta2-edge.yaml
+  - vlogs.yaml
+  - watchstate.yaml

apps/base/bind-internal/authoritative/unkin-net/a/lb1.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: lb1-unkin-net
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: lb1
+  type: A
+  ttl: 600
+  values:
+    - 103.216.191.185

apps/base/bind-internal/authoritative/unkin-net/a/logviewer.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: logviewer-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: logviewer
+  type: A
+  ttl: 600
+  values:
+    # traefik-internal gateway VIP; the logviewer Gateway serves
+    # logviewer.unkin.net there.
+    - 198.18.200.4

apps/base/bind-internal/authoritative/unkin-net/a/mail.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: mail-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: mail
+  type: A
+  ttl: 600
+  values:
+    # stalwart-lb PureLB VIP (IMAPS, submission, HTTPS).
+    - 198.18.200.18

apps/base/bind-internal/authoritative/unkin-net/a/mta1-edge.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: mta1-edge-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: mta1.edge
+  type: A
+  ttl: 600
+  values:
+    # edge MTA 1 public IP, mirrors public DNS.
+    - 66.226.148.207

apps/base/bind-internal/authoritative/unkin-net/a/mta2-edge.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: mta2-edge-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: mta2.edge
+  type: A
+  ttl: 600
+  values:
+    # edge MTA 2 public IP, mirrors public DNS.
+    - 66.226.149.50

apps/base/bind-internal/authoritative/unkin-net/a/vlogs.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: vlogs-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: vlogs
+  type: A
+  ttl: 600
+  values:
+    # traefik-EXTERNAL (DMZ) gateway VIP; the vlogs-external Gateway serves the
+    # vlogs.unkin.net front door (oauth2-proxy) there.
+    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/a/watchstate.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: watchstate-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: watchstate
+  type: A
+  ttl: 600
+  values:
+    # traefik-EXTERNAL (DMZ) gateway VIP; the watchstate-external Gateway serves
+    # the watchstate.unkin.net front door (oauth2-proxy) there.
+    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - a

apps/base/bind-internal/resolvers/acls.yaml

@@ -38,6 +38,9 @@ spec:
   clusterRef: bind-resolvers
   entries:
     - 198.18.24.0/24
+    - 172.18.30.0/24  # BinaryLane DMZ VPC syd (mail edge, via VPN)
+    - 172.18.31.0/24  # BinaryLane DMZ VPC mel (mail edge, via VPN)
+    - 172.18.32.0/24  # BinaryLane DMZ VPC bne (mail edge, via VPN)
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: BindACL

apps/base/bind-internal/resolvers/cluster.yaml

@@ -18,9 +18,10 @@ spec:
       purelb.io/service-group: common
       purelb.io/addresses: 198.18.200.7
       external-dns.alpha.kubernetes.io/hostname: bind-resolvers.k8s.syd1.au.unkin.net
+  # Public names go via global forwarders; RBL zones bypass them (view.yaml).
   forwarders:
-    - 8.8.8.8
     - 1.1.1.1
+    - 8.8.8.8
   # The internal split-horizon zones are served UNSIGNED by the in-cluster
   # authoritative, but their public parents publish DS records (e.g. unkin.net
   # is DNSSEC-signed on the Internet). With dnssec-validation on, the validator

apps/base/bind-internal/resolvers/view.yaml

@@ -21,3 +21,16 @@ spec:
   extraOptions:
     - "allow-recursion { any; }"
     - "allow-query-cache { any; }"
+    # pods have no IPv6 route; v6 NS attempts burn the recursion-query budget
+    - "server ::/0 { bogus yes; }"
+    # RBL zones resolve directly: DNSBLs refuse queries relayed via public resolvers.
+    - 'zone "spamhaus.org" { type forward; forwarders { }; }'
+    - 'zone "mailspike.net" { type forward; forwarders { }; }'
+    - 'zone "spameatingmonkey.net" { type forward; forwarders { }; }'
+    - 'zone "dnswl.org" { type forward; forwarders { }; }'
+    - 'zone "virusfree.cz" { type forward; forwarders { }; }'
+    - 'zone "blocklist.de" { type forward; forwarders { }; }'
+    - 'zone "msbl.org" { type forward; forwarders { }; }'
+    - 'zone "surbl.org" { type forward; forwarders { }; }'
+    - 'zone "uribl.com" { type forward; forwarders { }; }'
+    - 'zone "rspamd.com" { type forward; forwarders { }; }'

apps/base/bind-system/deployment.yaml

@@ -21,7 +21,7 @@ spec:
         runAsNonRoot: true
       containers:
         - name: operator
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.2.6
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.3.1
           args:
             - --metrics-bind-address=:8080
             - --health-probe-bind-address=:8081

apps/base/bind-system/kustomization.yaml

@@ -6,7 +6,7 @@ resources:
   - namespace.yaml
   # CRDs are pulled from the bind-operator repo at the matching tag rather than
   # vendored here, so they never drift from the operator.
-  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.2.6/config/crd/install.yaml
+  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.3.1/config/crd/install.yaml
   - rbac.yaml
   - agent-dns-rbac.yaml
   - deployment.yaml

apps/base/cert-manager/certificate_wildcard-ceph-unkin-net.yaml

@@ -0,0 +1,26 @@
+---
+# Let's Encrypt *.ceph.unkin.net wildcard for the haproxy edge (ceph dashboard).
+# DNS-01 needs the delegated _acme-challenge.ceph.unkin.net CNAME in the public
+# unkin.net zone.
+# _acme-challenge.ceph.unkin.net. CNAME _acme-challenge.ceph.acme.unkin.net.
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: wildcard-ceph-unkin-net
+  namespace: cert-manager
+spec:
+  secretName: wildcard-ceph-unkin-net-tls
+  secretTemplate:
+    annotations:
+      reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy"
+      reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy"
+  privateKey:
+    size: 4096
+  dnsNames:
+    - "*.ceph.unkin.net"
+  issuerRef:
+    name: letsencrypt
+    kind: ClusterIssuer
+    group: cert-manager.io

apps/base/cert-manager/certificate_wildcard-main-unkin-net.yaml

@@ -0,0 +1,26 @@
+---
+# Let's Encrypt *.main.unkin.net wildcard for the haproxy edge (pve, arr stack,
+# jellyfin, stalwart webadmin/autoconfig) and the mail smarthost. DNS-01 needs the delegated
+# _acme-challenge.main.unkin.net CNAME in the public unkin.net zone.
+# _acme-challenge.main.unkin.net. CNAME _acme-challenge.main.acme.unkin.net.
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: wildcard-main-unkin-net
+  namespace: cert-manager
+spec:
+  secretName: wildcard-main-unkin-net-tls
+  secretTemplate:
+    annotations:
+      reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy,mail"
+      reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy,mail"
+  privateKey:
+    size: 4096
+  dnsNames:
+    - "*.main.unkin.net"
+  issuerRef:
+    name: letsencrypt
+    kind: ClusterIssuer
+    group: cert-manager.io

apps/base/cert-manager/certificate_wildcard-unkin-net.yaml

@@ -14,9 +14,9 @@ spec:
   secretTemplate:
     annotations:
       reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
-      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
       reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
-      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
   privateKey:
     size: 4096
   dnsNames:

apps/base/cert-manager/kustomization.yaml

@@ -12,3 +12,5 @@ resources:
   - clusterissuer_letsencrypt.yaml
   - clusterissuer_letsencrypt-staging.yaml
   - certificate_wildcard-unkin-net.yaml
+  - certificate_wildcard-main-unkin-net.yaml
+  - certificate_wildcard-ceph-unkin-net.yaml

apps/base/cheeztv/plugin-configmap.yaml

@@ -26,7 +26,7 @@ data:
           </key>
           <value>
             <PluginConfiguration>
-              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
               <OidClientId>jellyfin</OidClientId>
               <OidSecret>@@CLIENT_SECRET@@</OidSecret>
               <Enabled>true</Enabled>

apps/base/cheeztv/service.yaml

@@ -13,6 +13,4 @@ spec:
       targetPort: http
   selector:
     app: cheeztv
-  # Pin each client to one replica to reduce transcode-session churn/takeover.
-  sessionAffinity: ClientIP
   type: ClusterIP

apps/base/cheeztv/statefulset.yaml

@@ -4,6 +4,8 @@ kind: StatefulSet
 metadata:
   name: cheeztv
   namespace: cheeztv
+  annotations:
+    configmap.reloader.stakater.com/auto: "true"
 spec:
   # HA: two replicas coordinate transcode session ownership through Valkey and
   # resume each other's HLS segments off the shared RWX transcode PVC. Stable
@@ -162,7 +164,7 @@ spec:
               readOnly: true
       containers:
         - name: cheeztv
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.4.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

apps/base/clickhouse-system/namespace.yaml

@@ -1,7 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  labels:
-    app.kubernetes.io/name: clickhouse-system
-  name: clickhouse-system

apps/base/dhcp-system/cr/keaapi.yaml

@@ -11,7 +11,7 @@ metadata:
     argocd.argoproj.io/sync-wave: "1"
 spec:
   replicas: 1
-  image: git.unkin.net/unkin/kea-api:v0.1.3
+  image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/kea-api:v0.1.7
   tokenSecretName: kea-api-token
   service:
     type: ClusterIP

apps/base/dhcp-system/cr/keacluster.yaml

@@ -11,7 +11,7 @@ metadata:
     argocd.argoproj.io/sync-wave: "1"
 spec:
   replicas: 2
-  image: git.unkin.net/unkin/kea:v0.1.3
+  image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/kea:v0.1.7
   domainName: main.unkin.net
   defaultLeaseTime: 1200
   maxLeaseTime: 86400

apps/base/dhcp-system/deployment.yaml

@@ -21,7 +21,7 @@ spec:
         runAsNonRoot: true
       containers:
         - name: operator
-          image: git.unkin.net/unkin/kea-operator:v0.1.5
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/kea-operator:v0.1.7
           args:
             - --metrics-bind-address=:8080
             - --health-probe-bind-address=:8081

apps/base/fafflix/plugin-configmap.yaml

@@ -26,7 +26,7 @@ data:
           </key>
           <value>
             <PluginConfiguration>
-              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
               <OidClientId>jellyfin</OidClientId>
               <OidSecret>@@CLIENT_SECRET@@</OidSecret>
               <Enabled>true</Enabled>

apps/base/fafflix/service.yaml

@@ -13,6 +13,4 @@ spec:
       targetPort: http
   selector:
     app: fafflix
-  # Pin each client to one replica to reduce transcode-session churn/takeover.
-  sessionAffinity: ClientIP
   type: ClusterIP

apps/base/fafflix/statefulset.yaml

@@ -4,6 +4,8 @@ kind: StatefulSet
 metadata:
   name: fafflix
   namespace: fafflix
+  annotations:
+    configmap.reloader.stakater.com/auto: "true"
 spec:
   # HA: two replicas coordinate transcode session ownership through Valkey and
   # resume each other's HLS segments off the shared RWX transcode PVC. Stable
@@ -162,7 +164,7 @@ spec:
               readOnly: true
       containers:
         - name: fafflix
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.4.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

apps/base/gitea/gateway.yaml

@@ -1,11 +1,6 @@
 ---
-# HTTPS front for the k8s Gitea, served on two names:
-#   git.unkin.net              — canonical/production (apex, bind-operator zone;
-#                                DNS flip is the gated cutover step, see the doc)
-#   git.k8s.syd1.au.unkin.net  — admin/backup route (external-dns k8s.syd1 zone),
-#                                same dual-name pattern as identity.unkin.net.
-# The cert-manager Certificate (vault-issuer) takes CN git.unkin.net and gets a
-# DNS SAN for each TLS listener hostname automatically.
+# HTTPS front for the k8s Gitea on git.k8s.syd1.au.unkin.net (external-dns
+# k8s.syd1 zone). git.unkin.net stays on the VM haproxy until cutover.
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
@@ -17,36 +12,13 @@ metadata:
     traefik.io/instance: internal
   annotations:
     cert-manager.io/cluster-issuer: vault-issuer
-    cert-manager.io/common-name: git.unkin.net
+    cert-manager.io/common-name: git.k8s.syd1.au.unkin.net
     cert-manager.io/private-key-size: "4096"
-    # Only the k8s admin route is published by external-dns (it owns just the
-    # k8s.syd1.au.unkin.net zone). git.unkin.net lives in the apex zone and is
-    # flipped at cutover — NOT managed here.
     external-dns.alpha.kubernetes.io/hostname: git.k8s.syd1.au.unkin.net
     external-dns.alpha.kubernetes.io/target: 198.18.200.4
 spec:
   gatewayClassName: traefik-internal
   listeners:
-    - name: http-primary
-      port: 80
-      protocol: HTTP
-      hostname: git.unkin.net
-      allowedRoutes:
-        namespaces:
-          from: Same
-    - name: https-primary
-      port: 443
-      protocol: HTTPS
-      hostname: git.unkin.net
-      allowedRoutes:
-        namespaces:
-          from: Same
-      tls:
-        mode: Terminate
-        certificateRefs:
-          - group: ""
-            kind: Secret
-            name: gitea-tls
     - name: http-admin
       port: 80
       protocol: HTTP

apps/base/gitea/httproute.yaml

@@ -9,13 +9,8 @@ metadata:
     app.kubernetes.io/instance: gitea
 spec:
   hostnames:
-    - git.unkin.net
     - git.k8s.syd1.au.unkin.net
   parentRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: gitea
-      sectionName: http-primary
     - group: gateway.networking.k8s.io
       kind: Gateway
       name: gitea
@@ -41,13 +36,8 @@ metadata:
     app.kubernetes.io/instance: gitea
 spec:
   hostnames:
-    - git.unkin.net
     - git.k8s.syd1.au.unkin.net
   parentRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: gitea
-      sectionName: https-primary
     - group: gateway.networking.k8s.io
       kind: Gateway
       name: gitea

apps/base/grafana/dashboards/k8up.yaml

@@ -0,0 +1,15 @@
+---
+# k8up dashboard (M0NsTeRRR/grafana-dashboards), datasource resolved to the
+# in-cluster VictoriaMetrics. Backed by the k8up VMServiceScrape.
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: k8up
+  namespace: grafana
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  resyncPeriod: 5m
+  allowCrossNamespaceImport: false
+  gzipJson: H4sIAAAAAAACA+2ce3PbNhLAvwqHc3eTzlixqIdfSTvjptc2M7Hjq9Pc3NUeDUhCEiIIYAHQsqLRffZbAHzAop0qaSRHKf4SCZLAcrH7wwJYcREixrhCinAmw5NgEVIiFRz8tgjjnFD1ksFJtBeEKVJI8lwk2Nym5pk+CEcCDRFDIdyRk1SXtFrBT7YwaLXCJVzADMVU361EjuF8TFLnjCScveCUC/2wGMXoSXsv6ETQZqff3wuib3TdDE1Nc6e1tME/glOKhZL6eilOiuQ45kik4fJaN51imQiS6Qf05clRngXVPcFYqUye7O/r4qeE64pwStRdaYdEJoj+ByNxqZBQZ5ypMVwFIcMRw+qlfulOOzo42DPayMZvOKeKZMU9RikspxSOKWETreXfrs3JDT7nMzgdIip1SxlimEqr/IRTijKJU+f6SJD0gttuGhfdop/v9ODgtmhvrn+XZcPdO9XqZhVR5u3C1xkWSIHaHfUJkAeeXTzU25ngU6zGOJdOh6dYdeb9foqS3rtJYnr8QUlXBI0qQSOtPp7Vdpjw1LZNERvlaGTbp4gwhW+Vbl6O+ewVYfg8n8ZYSEdR+soZYeQMZWWpbggMTWGmrTvUz09tC+EUiUnKZ8xIntF8RNhbqK8wmaj9tPO0G7qaOwdjlBkC3TiqM1JtQ3edhu46R7uju8tkjNOcYhm847F8FAX2GgrstndHgW+AfxSqJRSn29dht9DhQUOHx7ujwu9RMoFxQMJAkj+O+o6b/NshE3wxxsnkMdUXNRmo44Vd0d+pSMYw+D+qBvtNDXZ2R4O/YAmRy+Nq8Kipwe7uaPBC5OxR9deJmvrr7V4cs00V9h6Ioju1Bg+3qsHnZqoEKphT/O1VOCOpGp9E7fbfnwVjTEZjZU9iLlIsTtrZbSA5JWkQUxiBn12F312JKxYEz1Va1aHFaiFKRuwkSKAZLJ7dwCSPwBSsLJ6SNKX4A7U+z6rqhiBsa2ZlCWJOU339b6wM4J/vZ6UM+yo1h3CgX+o71zrGakr/yDIEzjAyWsG3GRcKpwPmzhPs9R+IwEk5G73ZiNUMCabpC86GZGSeT/EQ5VSV9mAn2ovq3dRYYOhpmkrz9BRlGWGjerpYX3afQjHoPFfmzaTCWT1rtfP4mObm2g3SB3YCbGfkOSNGS4wzbFrk0L9g6Ng2uY7ddxp232lGnyDIWSGsFaKcnpelRgAofJdLRYbzshjlyiwEcEHA/FDZV2WxAK9P8Gu3KUQTIzv4mFTnXJ3Du4bXZU/IEjVGCul6ku7yhozgRPgVmvNcFUsQa9ndPXaFxAirol/+pEnpRZFanXpdCDzPrJbc4inQRDjUAOs3FiDz6RO9rjKQ5YxvMEL5CC+a7vHt/67C2ievwqVZ8iEM0Go4U1ZN8Qiz9EcuptbRBoOqV4BuzpKNwEOzLBOehstrx8c0q79AH2P8beEmYTv8dJ8bCYxZw+n2HIm09bo3RJ/NJXsNl+xG3iW35pJmmP+wPxpf1OsFA2VWEFrBfd7513K8Bxwtw0KHHmj0oKulEC62Vpyp6W0Nh4Ro6K7HFS2Z049zvNWFGGcsXPG7KVgpef+ziYGg4FDP/GzZv3W8VhZtwbl0QPmmVPkrFNtVaTfcrK6egYJtNFpYliTvoecqydYOxtaLwbbgmNWoGMTz4ElzEPwmeFK7qMyTBEs5zOkg4bmOgtcYNvf0cuAbEBfi3tgscemhNNgP1mzUcOGT29kQOSyVPDo2gI7jBjoOPTo8OhK9urt5clTNfOXgME+fEXaGbp3X+wpxEjWn5UeeJ54nyO52bJ4oTkM+GNk5evQb9Oh5enh6CLvTt3l6OA15euwcPY4a9Djw9PD0yPQu9+bZUTXjybFr5KjSEGpy9D05PDnKfZHNw8NtafP8+FN55Qd1okn/4cTyi1yOg5+QwjM030Ry+UcyLEMUK4VbCXibJLaKJIdgb2puQ7dEfm9SSS7Hd7Lx9YUXJgMFp//Fgq9ecrczy2QafcH4ZenDuuCCQp9Pi5SZsudiJE51OktRrjWcCjS71Hkr+j5K7FbokFD6GoyGqHn5pwOBUk2a1U1T/a+KH0XxVtZkHJFV9aeEsuSGvHfZAmaHhUGRyxQtxkutg4zTCm26EJn9f31UQlBbTcYJU5fkPS4wqv85gX8gUgkS58XjdYcX9ZRku9APS1dJ4C1sVSJwjmRiSbYAXfA8swbvpOtU+9j1+FTqtTYLPhyGy8+Y8XL/7vv1cr1h6ahwuWg1t+uw3tdbzRCu+tgZUvQLpERmFFW75+YPPYb2jhnGXGkHKPO/yprKze3aWmoFSFASta8PdKv1vNwq/w1r7a7TQE/ZSDJAN4hQnbM1kAzQNuZKLixE70ktWY+wi6qC5XIdzlaZvaUoQSVKADFKUNZ2J+GLgFQYggbpqeip+Fen4gKiGjmHKqa/4KHJuYQrl8Y9TPcZkRSEStaMrRvEc/NXrHA1YC+Ew7cJzS0+DAJMzD3B84RyNAlawX553Mq4VCCqbKX5NDOheAY+Tm6NEikNoCacqRMbAqP0NaPzkpYmkBY800mq5atY6aw7Pa0s0NHCR5iibeR6ud6wUW3COONG5MeNe8eNZKypng5SM53R0zc5SHMB4hY3fs5RBMxtschuko8bT15YEQNHxMCKGFgR/ejiRxc/unzmmDvq1/Dsenh+EJ7gKx+HzWGFR42oQcGoLfPUSO1J6knqSboRktZ/0HVQ2vMo/bwo9cT0xPTE/Lpiz063BmbfA/NeYDI8+5In7ed45ifsHpoemtsLM11qHnhqemp6anpqemp+ONTsOl8QPPTQvBeaOQMRCWDhi94m+rWS0iPUI9QjdHuBp8vQjmfoHzH0i1zidOj5qauc17Z+bZolW3Ra7xTVydXdY9MPjiWbT7Wo0lnq724nuRDWDBbLhz+8vW53AX8JIxUcNP4GNrN8NW95757k5rD+XLf5mDUzGTWnlNZ6pSVR73wWeQrMJ/VN5fe772ST185iFPJ7jsXcvOPvYv7GvmbklH+S8FWvX1T6eosE0Tl6/9L1/tNYc+tOmdFb3Z2RORvh26prJyT7VdDLOUtc9FmvazsmYiW3qTvaXMzLDe0oAN45ax2af0AoXpyH5Y0ZSSY206ksea99uWi/slytA6f34yg6Oj6O41a/E0et3rCNWqjX7bXifq/TTnv9XoRiw8DKKPVceYbxxHzZ3FS//D9scYEQEl4AAA==

apps/base/grafana/gateway.yaml

@@ -37,3 +37,36 @@ spec:
           - group: ""
             kind: Secret
             name: grafana-tls
+---
+# Public grafana.unkin.net via the external Traefik; TLS uses the reflected
+# Let's Encrypt *.unkin.net wildcard, DNS lives in the bind-operator zone.
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: external
+  name: grafana-external
+  namespace: grafana
+spec:
+  gatewayClassName: traefik-external
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: grafana.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: grafana.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: wildcard-unkin-net-tls
+        mode: Terminate

apps/base/grafana/grafana.yaml

@@ -9,8 +9,16 @@ metadata:
 spec:
   deployment:
     spec:
+      replicas: 3
       template:
         spec:
+          topologySpreadConstraints:
+            - maxSkew: 1
+              topologyKey: kubernetes.io/hostname
+              whenUnsatisfiable: ScheduleAnyway
+              labelSelector:
+                matchLabels:
+                  app: grafana
           containers:
             - name: grafana
               env:
@@ -31,11 +39,11 @@ spec:
                   cpu: 100m
                   memory: 256Mi
                 limits:
-                  cpu: "1"
-                  memory: 1Gi
+                  cpu: "2"
+                  memory: 4Gi
   config:
     server:
-      root_url: "https://grafana.k8s.syd1.au.unkin.net"
+      root_url: "https://grafana.unkin.net"
     database:
       type: "postgres"
       host: "postgres-pooler-rw.grafana.svc.cluster.local:5432"
@@ -61,3 +69,8 @@ spec:
       # to akR-global-admin members (and direct members) via terraform-authentik.
       role_attribute_path: "contains(ak_groups[*], 'akP-grafana-admin') && 'Admin' || 'Viewer'"
       role_attribute_strict: "false"
+    # Gossip between replicas so each alert notification is sent once.
+    unified_alerting:
+      ha_peers: "grafana-alerting.grafana.svc.cluster.local:9094"
+      ha_listen_address: "${POD_IP}:9094"
+      ha_advertise_address: "${POD_IP}:9094"

apps/base/grafana/grafanadatasource.yaml

@@ -20,3 +20,22 @@ spec:
     jsonData:
       timeInterval: "15s"
       httpMethod: "POST"
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDatasource
+metadata:
+  name: victorialogs
+  namespace: grafana
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  plugins:
+    - name: victoriametrics-logs-datasource
+      version: 0.32.0
+  datasource:
+    name: "VictoriaLogs"
+    type: "victoriametrics-logs-datasource"
+    uid: "victorialogs"
+    access: "proxy"
+    url: "http://vlselect-logs.logging.svc.cluster.local:9471"

apps/base/grafana/httproute.yaml

@@ -53,3 +53,58 @@ spec:
         - path:
             type: PathPrefix
             value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: grafana-external-http-redirect
+  namespace: grafana
+  labels:
+    app.kubernetes.io/name: grafana
+    app.kubernetes.io/instance: grafana
+spec:
+  hostnames:
+    - grafana.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: grafana-external
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: grafana-external
+  namespace: grafana
+  labels:
+    app.kubernetes.io/name: grafana
+    app.kubernetes.io/instance: grafana
+spec:
+  hostnames:
+    - grafana.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: grafana-external
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: grafana-service
+          port: 3000
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/grafana/kustomization.yaml

@@ -18,6 +18,7 @@ resources:
   - dashboards/frr-ospf-route-metrics.yaml
   - dashboards/gitea.yaml
   - dashboards/haproxy.yaml
+  - dashboards/k8up.yaml
   - dashboards/media-dashboard.yaml
   - dashboards/node-exporter-full.yaml
   - dashboards/nzbget.yaml

apps/base/haproxy/backendtlspolicy.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: BackendTLSPolicy
+metadata:
+  name: haproxy
+  namespace: haproxy
+  labels:
+    app: haproxy
+spec:
+  targetRefs:
+    - group: ""
+      kind: Service
+      name: haproxy
+  validation:
+    wellKnownCACertificates: System
+    hostname: git.unkin.net

apps/base/haproxy/configmap.yaml

@@ -0,0 +1,256 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: haproxy-config
+  namespace: haproxy
+data:
+  certificate.list: |
+    # First entry is the default cert for non-matching SNI.
+    /etc/haproxy/certs/unkin-net/tls.crt
+    /etc/haproxy/certs/main-unkin-net/tls.crt
+    /etc/haproxy/certs/ceph-unkin-net/tls.crt
+
+  fe_https.map: |
+    sonarr.main.unkin.net be_sonarr
+    radarr.main.unkin.net be_radarr
+    lidarr.main.unkin.net be_lidarr
+    readarr.main.unkin.net be_readarr
+    prowlarr.main.unkin.net be_prowlarr
+    nzbget.main.unkin.net be_nzbget
+    jellyfin.main.unkin.net be_jellyfin
+    fafflix.unkin.net be_jellyfin
+    git.unkin.net be_gitea
+    dashboard.ceph.unkin.net be_ceph_dashboard
+    auth.unkin.net be_k8s_kanidm
+
+  haproxy.cfg: |
+    global
+      log stdout format raw local0
+      log stdout format raw local1 notice
+      maxconn  4000
+      hard-stop-after  2m
+      ssl-default-bind-ciphers  EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH
+      ssl-default-bind-options  ssl-min-ver TLSv1.2 ssl-max-ver TLSv1.3
+      ssl-default-server-ciphers  kEECDH+aRSA+AES:kRSA+AES:+AES256:RC4-SHA:!kEDH:!LOW:!EXP:!MD5:!aNULL:!eNULL
+      ssl-default-server-options  no-sslv3
+      stats  timeout 30s
+      stats  socket /var/lib/haproxy/stats
+      stats  socket /var/lib/haproxy/admin.sock mode 660 level admin
+      tune.ssl.default-dh-param  2048
+
+    defaults
+      log  global
+      maxconn  5000
+      mode  http
+      option  httplog
+      option  dontlognull
+      option  http-server-close
+      option  forwardfor except 127.0.0.0/8
+      option  redispatch
+      retries  3
+      stats  enable
+      timeout  http-request 10s
+      timeout  queue 1m
+      timeout  connect 10s
+      timeout  client 5m
+      timeout  server 5m
+      timeout  http-keep-alive 10s
+      timeout  check 10s
+
+    frontend fe_https
+      bind 0.0.0.0:443 ssl crt-list /usr/local/etc/haproxy/certificate.list ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH force-tlsv12
+      mode http
+      description Global HTTPS Frontend
+      http-request set-header X-Forwarded-Proto https
+      http-request set-header X-Real-IP %[src]
+      http-response set-header X-Content-Type-Options nosniff
+      http-response set-header X-XSS-Protection 1;mode=block
+      use_backend %[req.hdr(host),lower,map(/usr/local/etc/haproxy/fe_https.map,be_default)]
+
+    frontend fe_metrics
+      bind 0.0.0.0:8405
+      mode http
+      description Metrics Frontend
+      http-request set-header X-Forwarded-Proto https
+      http-request set-header X-Real-IP %[src]
+      http-request use-service prometheus-exporter if { path /metrics }
+
+    backend be_ceph_dashboard
+      description Backend for Ceph Dashboard from Mgr instances
+      balance roundrobin
+      cookie SRVNAME insert indirect nocache
+      http-check expect status 200
+      http-request set-header X-Forwarded-Port %[dst_port]
+      http-request add-header X-Forwarded-Proto https if { dst_port 9443 }
+      http-reuse always
+      option httpchk GET /
+      option forwardfor
+      option http-keep-alive
+      option prefer-last-server
+      redirect scheme https if !{ ssl_fc }
+      stick-table type ip size 200k expire 30m
+      server prodnxsr0009 198.18.23.9:9443 check cookie prodnxsr0009 fall 2 inter 2s rise 3 ssl verify none
+      server prodnxsr0010 198.18.23.10:9443 check cookie prodnxsr0010 fall 2 inter 2s rise 3 ssl verify none
+      server prodnxsr0011 198.18.23.11:9443 check cookie prodnxsr0011 fall 2 inter 2s rise 3 ssl verify none
+      server prodnxsr0012 198.18.23.12:9443 check cookie prodnxsr0012 fall 2 inter 2s rise 3 ssl verify none
+      server prodnxsr0013 198.18.23.13:9443 check cookie prodnxsr0013 fall 2 inter 2s rise 3 ssl verify none
+
+    backend be_default
+      description Backend for unmatched HTTP traffic
+      balance roundrobin
+      cookie SRVNAME insert
+      http-request set-header X-Forwarded-Port %[dst_port]
+      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
+      option httpchk GET /
+      option forwardfor
+
+    backend be_gitea
+      description Backend for gitea cluster
+      balance roundrobin
+      cookie SRVNAME insert indirect nocache
+      http-request set-header X-Forwarded-Port %[dst_port]
+      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
+      http-reuse always
+      option httpchk GET /
+      option forwardfor
+      option http-keep-alive
+      option prefer-last-server
+      redirect scheme https if !{ ssl_fc }
+      stick on src
+      stick-table type ip size 200k expire 30m
+      server ausyd1nxvm2080 198.18.26.18:443 check cookie ausyd1nxvm2080 fall 2 inter 2s rise 3 ssl verify none
+      server ausyd1nxvm2081 198.18.27.117:443 check cookie ausyd1nxvm2081 fall 2 inter 2s rise 3 ssl verify none
+      server ausyd1nxvm2082 198.18.28.71:443 check cookie ausyd1nxvm2082 fall 2 inter 2s rise 3 ssl verify none
+
+    backend be_jellyfin
+      description Backend for au-syd1 jellyfin
+      balance roundrobin
+      cookie SRVNAME insert indirect nocache
+      http-request set-header X-Forwarded-Port %[dst_port]
+      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
+      http-reuse always
+      option httpchk GET /
+      option forwardfor
+      option http-keep-alive
+      option prefer-last-server
+      redirect scheme https if !{ ssl_fc }
+      server ausyd1nxvm2051 198.18.25.164:443 check cookie ausyd1nxvm2051 fall 2 inter 2s rise 3 ssl verify none
+
+    backend be_k8s_kanidm
+      description Backend for Kanidm (auth.unkin.net via Kubernetes internal Traefik)
+      balance roundrobin
+      http-reuse always
+      http-request set-header X-Forwarded-Port %[dst_port]
+      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
+      redirect scheme https if !{ ssl_fc }
+      option httpchk
+      option forwardfor
+      option http-keep-alive
+      option prefer-last-server
+      http-check connect ssl sni auth.unkin.net
+      http-check send meth GET uri /status ver HTTP/1.1 hdr Host auth.unkin.net
+      http-check expect status 200
+      server k8s-traefik-internal 198.18.200.4:443 ssl verify none check inter 2s rise 3 fall 2 sni str(auth.unkin.net)
+
+    backend be_lidarr
+      description Backend for au-syd1 lidarr
+      balance roundrobin
+      cookie SRVNAME insert indirect nocache
+      http-request set-header X-Forwarded-Port %[dst_port]
+      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
+      http-reuse always
+      option httpchk GET /consul/health
+      option forwardfor
+      option http-keep-alive
+      option prefer-last-server
+      redirect scheme https if !{ ssl_fc }
+      server ausyd1nxvm2048 198.18.28.165:443 check cookie ausyd1nxvm2048 fall 2 inter 2s rise 3 ssl verify none
+
+    backend be_nzbget
+      description Backend for au-syd1 nzbget
+      balance roundrobin
+      cookie SRVNAME insert indirect nocache
+      http-request set-header X-Forwarded-Port %[dst_port]
+      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
+      http-reuse always
+      option httpchk GET /consul/health
+      option forwardfor
+      option http-keep-alive
+      option prefer-last-server
+      redirect scheme https if !{ ssl_fc }
+      server ausyd1nxvm2045 198.18.25.44:443 check cookie ausyd1nxvm2045 fall 2 inter 2s rise 3 ssl verify none
+
+    backend be_prowlarr
+      description Backend for au-syd1 prowlarr
+      balance roundrobin
+      cookie SRVNAME insert indirect nocache
+      http-request set-header X-Forwarded-Port %[dst_port]
+      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
+      http-reuse always
+      option httpchk GET /consul/health
+      option forwardfor
+      option http-keep-alive
+      option prefer-last-server
+      redirect scheme https if !{ ssl_fc }
+      server ausyd1nxvm2050 198.18.25.66:443 check cookie ausyd1nxvm2050 fall 2 inter 2s rise 3 ssl verify none
+
+    backend be_radarr
+      description Backend for au-syd1 radarr
+      balance roundrobin
+      cookie SRVNAME insert indirect nocache
+      http-request set-header X-Forwarded-Port %[dst_port]
+      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
+      http-reuse always
+      option httpchk GET /consul/health
+      option forwardfor
+      option http-keep-alive
+      option prefer-last-server
+      redirect scheme https if !{ ssl_fc }
+      server ausyd1nxvm2047 198.18.27.131:443 check cookie ausyd1nxvm2047 fall 2 inter 2s rise 3 ssl verify none
+
+    backend be_readarr
+      description Backend for au-syd1 readarr
+      balance roundrobin
+      cookie SRVNAME insert indirect nocache
+      http-request set-header X-Forwarded-Port %[dst_port]
+      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
+      http-reuse always
+      option httpchk GET /consul/health
+      option forwardfor
+      option http-keep-alive
+      option prefer-last-server
+      redirect scheme https if !{ ssl_fc }
+      server ausyd1nxvm2049 198.18.29.32:443 check cookie ausyd1nxvm2049 fall 2 inter 2s rise 3 ssl verify none
+
+    backend be_sonarr
+      description Backend for au-syd1 sonarr
+      balance roundrobin
+      cookie SRVNAME insert indirect nocache
+      http-request set-header X-Forwarded-Port %[dst_port]
+      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
+      http-reuse always
+      option httpchk GET /consul/health
+      option forwardfor
+      option http-keep-alive
+      option prefer-last-server
+      redirect scheme https if !{ ssl_fc }
+      server ausyd1nxvm2046 198.18.26.161:443 check cookie ausyd1nxvm2046 fall 2 inter 2s rise 3 ssl verify none
+
+    # The `peers au-syd1-prod` section is dropped: peer names must be static and a
+    # Deployment cannot provide them. Behind the external Traefik's TLS
+    # passthrough `src` is a Traefik pod, so X-Real-IP, forwardfor and the
+    # `stick on src` tables all key on that; the SRVNAME cookie carries real
+    # session persistence. Traefik cannot emit PROXY protocol to a TLSRoute
+    # backend, so there is nothing to bind `accept-proxy` to.
+
+    listen health
+      bind 0.0.0.0:8404
+      mode http
+      monitor-uri /healthz
+
+    listen stats
+      bind 127.0.0.1:9090
+      mode http
+      stats uri /
+      stats auth admin:admin

apps/base/haproxy/deployment.yaml

@@ -0,0 +1,148 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: haproxy
+  namespace: haproxy
+  annotations:
+    reloader.stakater.com/auto: "true"
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: haproxy
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        app: haproxy
+    spec:
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 150
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: haproxy
+              topologyKey: kubernetes.io/hostname
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 99
+        runAsGroup: 99
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: haproxy
+          image: haproxy:3.2.24-alpine
+          imagePullPolicy: IfNotPresent
+          command:
+            - haproxy
+            - -W
+            - -db
+            - -f
+            - /usr/local/etc/haproxy/haproxy.cfg
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: [ALL]
+              # fe_https binds the privileged port 443 as uid 99, and the
+              # dst_port ACLs need the real port.
+              add: [NET_BIND_SERVICE]
+          ports:
+            - name: https
+              containerPort: 443
+              protocol: TCP
+            - name: health
+              containerPort: 8404
+              protocol: TCP
+            - name: metrics
+              containerPort: 8405
+              protocol: TCP
+            - name: stats
+              containerPort: 9090
+              protocol: TCP
+          lifecycle:
+            preStop:
+              exec:
+                # SIGUSR1 to the master soft-stops the workers; hard-stop-after
+                # caps the drain. Wait so kubelet holds SIGTERM until it is done.
+                command:
+                  - /bin/sh
+                  - -c
+                  - kill -s USR1 1; while kill -0 1 2>/dev/null; do sleep 1; done
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: health
+            initialDelaySeconds: 15
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: health
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            timeoutSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 200m
+              memory: 256Mi
+            limits:
+              cpu: 2
+              memory: 1Gi
+          volumeMounts:
+            - name: config
+              mountPath: /usr/local/etc/haproxy
+              readOnly: true
+            - name: cert-unkin-net
+              mountPath: /etc/haproxy/certs/unkin-net
+              readOnly: true
+            - name: cert-main-unkin-net
+              mountPath: /etc/haproxy/certs/main-unkin-net
+              readOnly: true
+            - name: cert-ceph-unkin-net
+              mountPath: /etc/haproxy/certs/ceph-unkin-net
+              readOnly: true
+            - name: run
+              mountPath: /var/lib/haproxy
+      volumes:
+        - name: config
+          configMap:
+            name: haproxy-config
+        # ssl-load-extra-files loads <crtfile>.key by default, so the key is
+        # projected next to the cert as tls.crt.key.
+        - name: cert-unkin-net
+          secret:
+            secretName: wildcard-unkin-net-tls
+            items:
+              - key: tls.crt
+                path: tls.crt
+              - key: tls.key
+                path: tls.crt.key
+        - name: cert-main-unkin-net
+          secret:
+            secretName: wildcard-main-unkin-net-tls
+            items:
+              - key: tls.crt
+                path: tls.crt
+              - key: tls.key
+                path: tls.crt.key
+        - name: cert-ceph-unkin-net
+          secret:
+            secretName: wildcard-ceph-unkin-net-tls
+            items:
+              - key: tls.crt
+                path: tls.crt
+              - key: tls.key
+                path: tls.crt.key
+        - name: run
+          emptyDir: {}
+      restartPolicy: Always

apps/base/haproxy/gateway.yaml

@@ -0,0 +1,39 @@
+---
+# External (DMZ) front for the haproxy edge on the traefik-external LB VIP
+# 198.18.199.0. Traefik terminates TLS so HTTP/2 connections coalesced from
+# other *.unkin.net hosts still route by Host; the hop to haproxy is re-encrypted via
+# BackendTLSPolicy. Routes carry the explicit hostname list.
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: haproxy
+  namespace: haproxy
+  labels:
+    traefik.io/instance: external
+spec:
+  gatewayClassName: traefik-external
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: wildcard-unkin-net-tls
+          - group: ""
+            kind: Secret
+            name: wildcard-main-unkin-net-tls
+          - group: ""
+            kind: Secret
+            name: wildcard-ceph-unkin-net-tls
+      allowedRoutes:
+        namespaces:
+          from: Same

apps/base/haproxy/httproute.yaml

@@ -0,0 +1,69 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: haproxy-http-redirect
+  namespace: haproxy
+  labels:
+    app: haproxy
+spec:
+  hostnames:
+    - sonarr.main.unkin.net
+    - radarr.main.unkin.net
+    - lidarr.main.unkin.net
+    - readarr.main.unkin.net
+    - prowlarr.main.unkin.net
+    - nzbget.main.unkin.net
+    - jellyfin.main.unkin.net
+    - fafflix.unkin.net
+    - git.unkin.net
+    - dashboard.ceph.unkin.net
+    - auth.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: haproxy
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: haproxy
+  namespace: haproxy
+  labels:
+    app: haproxy
+spec:
+  hostnames:
+    - sonarr.main.unkin.net
+    - radarr.main.unkin.net
+    - lidarr.main.unkin.net
+    - readarr.main.unkin.net
+    - prowlarr.main.unkin.net
+    - nzbget.main.unkin.net
+    - jellyfin.main.unkin.net
+    - fafflix.unkin.net
+    - git.unkin.net
+    - dashboard.ceph.unkin.net
+    - auth.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: haproxy
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: haproxy
+          port: 443
+          weight: 1

apps/base/haproxy/kustomization.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - configmap.yaml
+  - deployment.yaml
+  - service.yaml
+  - gateway.yaml
+  - backendtlspolicy.yaml
+  - httproute.yaml
+  - pdb.yaml
+  - vpa.yaml
+  - vmpodscrape.yaml

apps/base/haproxy/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: haproxy

apps/base/haproxy/pdb.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: haproxy
+  namespace: haproxy
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: haproxy

apps/base/haproxy/service.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: haproxy
+  namespace: haproxy
+spec:
+  type: ClusterIP
+  # Reached only by the external Traefik (re-encrypted HTTPRoute), so the
+  # peer address here is a Traefik pod, not the client. sessionAffinity is
+  # deliberately absent: keyed on ClientIP it would pin whole Traefik pods,
+  # not clients. Backend persistence rests on the per-backend SRVNAME cookie.
+  selector:
+    app: haproxy
+  ports:
+    - name: https
+      port: 443
+      protocol: TCP
+      targetPort: https

apps/base/haproxy/vmpodscrape.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: haproxy
+  namespace: haproxy
+spec:
+  selector:
+    matchLabels:
+      app: haproxy
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics

apps/base/haproxy/vpa.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: autoscaling.k8s.io/v1
+kind: VerticalPodAutoscaler
+metadata:
+  name: haproxy-vpa
+  namespace: haproxy
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: haproxy
+  updatePolicy:
+    updateMode: "Off"

apps/base/k8up-system/kustomization.yaml

@@ -4,3 +4,4 @@ kind: Kustomization
 
 resources:
   - namespace.yaml
+  - vmservicescrape.yaml

apps/base/k8up-system/vmservicescrape.yaml

@@ -0,0 +1,15 @@
+---
+# Scrape the k8up operator metrics (k8up-metrics Service, port http :8080).
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMServiceScrape
+metadata:
+  name: k8up
+  namespace: k8up-system
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: k8up
+      app.kubernetes.io/instance: k8up
+  endpoints:
+    - port: http
+      path: /metrics

apps/base/logging/cephrgw.yaml

@@ -1,43 +0,0 @@
----
-# S3 bucket (Ceph RGW) for the long-term raw-log archive, provisioned by the
-# in-estate cephrgw-operator. The archiver Vector deployment writes here.
-apiVersion: ceph.unkin.net/v1alpha1
-kind: ObjectStoreUser
-metadata:
-  name: logs-archive-owner
-  namespace: logging
-spec:
-  displayName: "Logging raw-archive bucket owner"
-  maxBuckets: 5
-  quota:
-    enabled: true
-    # 5 TiB soft cap; real retention is enforced RGW-side by a bucket lifecycle
-    # policy (see PR notes) — the operator does not manage lifecycle.
-    maxSizeBytes: 5497558138880
----
-apiVersion: ceph.unkin.net/v1alpha1
-kind: Bucket
-metadata:
-  name: logs-archive
-  namespace: logging
-spec:
-  bucketName: logs-archive
-  ownerRef: logs-archive-owner
-  versioning: false
-  tags:
-    app: logging
-    purpose: raw-log-archive
-  # Keep the bucket (and its objects) if this CR is ever deleted.
-  retainOnDelete: true
----
-apiVersion: ceph.unkin.net/v1alpha1
-kind: BucketAccess
-metadata:
-  name: logs-archive-writer
-  namespace: logging
-spec:
-  bucketRef: logs-archive
-  level: read-write
-  # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
-  # S3_ENDPOINT, BUCKET_NAME) into this Secret; the archiver consumes it.
-  secretName: logs-archive-s3

apps/base/logging/clickhouseinstallation.yaml

@@ -1,97 +0,0 @@
----
-apiVersion: clickhouse.altinity.com/v1
-kind: ClickHouseInstallation
-metadata:
-  name: logs
-  namespace: logging
-spec:
-  defaults:
-    templates:
-      dataVolumeClaimTemplate: data-volume
-      serviceTemplate: chi-service
-      podTemplate: clickhouse
-  configuration:
-    users:
-      # Password hash is sourced from the Vault-synced clickhouse-credentials
-      # Secret; the plaintext never lands in git or the ClickHouse config.
-      vector/password_sha256_hex:
-        valueFrom:
-          secretKeyRef:
-            name: clickhouse-credentials
-            key: password_sha256_hex
-      vector/networks/ip:
-        - "::/0"
-      vector/profile: default
-      vector/quota: default
-      # Allow the vector user to create the logs database/table (bootstrap Job)
-      # and to INSERT. Restrict the built-in default user to loopback only.
-      vector/access_management: "1"
-      default/networks/ip:
-        - "127.0.0.1"
-        - "::1"
-      # Read-only user for CLI tools + the logviewer UI. Hash sourced from the
-      # Vault-synced clickhouse-logreader Secret, same mechanism as vector.
-      # Scoped to the logs database only (unlike vector, which bootstraps it).
-      logreader/password_sha256_hex:
-        valueFrom:
-          secretKeyRef:
-            name: clickhouse-logreader
-            key: password_sha256_hex
-      logreader/networks/ip:
-        - "::/0"
-      logreader/profile: readonly
-      logreader/quota: default
-      logreader/allow_databases/database:
-        - "logs"
-    profiles:
-      default/max_memory_usage: "10000000000"
-      default/max_execution_time: "120"
-      readonly/readonly: "2"
-      readonly/max_memory_usage: "10000000000"
-      readonly/max_execution_time: "120"
-    clusters:
-      - name: logs
-        layout:
-          shardsCount: 1
-          replicasCount: 1
-  templates:
-    volumeClaimTemplates:
-      - name: data-volume
-        spec:
-          storageClassName: cephrbd-fast-delete
-          accessModes:
-            - ReadWriteOnce
-          resources:
-            requests:
-              # 3d TTL on logs.raw. At ~130 GiB/day raw, ClickHouse LZ4/ZSTD
-              # (~6x on log text) stores ~20-25 GiB/day => ~60-75 GiB/3d, plus
-              # merge headroom (~2x peak). logs.raw is the only table. 150Gi
-              # gives comfortable headroom; long-term data lives in S3, not here.
-              storage: 150Gi
-    serviceTemplates:
-      - name: chi-service
-        generateName: "clickhouse-{chi}"
-        spec:
-          type: ClusterIP
-          ports:
-            - name: http
-              port: 8123
-            - name: tcp
-              port: 9000
-    podTemplates:
-      - name: clickhouse
-        spec:
-          securityContext:
-            fsGroup: 101
-            runAsUser: 101
-            runAsGroup: 101
-          containers:
-            - name: clickhouse
-              image: docker.io/clickhouse/clickhouse-server:24.8
-              resources:
-                requests:
-                  cpu: 500m
-                  memory: 2Gi
-                limits:
-                  cpu: "2"
-                  memory: 8Gi

apps/base/logging/configmap_logarchiver.yaml

@@ -1,40 +0,0 @@
----
-# logarchiver non-secret config. Secrets (NATS/S3/ClickHouse creds) and the
-# subject filter come from env; everything else uses the binary's built-in
-# defaults, which already target this stack. ack_wait MUST exceed batch.max_age
-# so unacked messages in an open batch are not redelivered mid-batch.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: logarchiver-config
-  namespace: logging
-data:
-  config.yaml: |
-    nats:
-      ack_wait: 5m
-      fetch_batch: 512
-    batch:
-      max_bytes: 67108864
-      max_events: 200000
-      max_age: 2m
-    # Pin the proven RGW endpoint/bucket; ignore the secret's S3_ENDPOINT/BUCKET_NAME
-    # (AWS creds still come from the secret env). endpoint_env/bucket_env off.
-    s3:
-      endpoint: "https://s3.ceph.unkin.net"
-      bucket: "logs-archive"
-      region: "us-east-1"
-      path_style: true
-      ca_file: /etc/vault-ca/ca.crt
-      endpoint_env: ""
-      bucket_env: ""
-    crypto:
-      key_name: logarchive
-      pubkey_source: vault
-      vault:
-        address: "https://vault.service.consul:8200"
-        mount: gpg
-        auth_method: kubernetes
-        k8s_mount: k8s/au/syd1
-        k8s_role: logging_logarchiver
-        k8s_jwt_path: /var/run/secrets/vault/token
-        ca_file: /etc/vault-ca/ca.crt

apps/base/logging/deployment_logarchiver.yaml

@@ -1,134 +0,0 @@
----
-# logarchiver — replaces the vector-archiver leg. Independent JetStream durable
-# consumer (archiver) that seals raw logs to S3 as zstd + OpenPGP objects and
-# indexes each object in ClickHouse (logs.archive_index). Acks only after the
-# object is in S3 AND indexed. Reuses the same NATS/S3/CA wiring the Vector
-# archiver used; the OpenPGP public key is delivered as a mounted file.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: logarchiver
-  namespace: logging
-  annotations:
-    configmap.reloader.stakater.com/auto: "true"
-    secret.reloader.stakater.com/reload: "vault-ca-cert"
-  labels:
-    app.kubernetes.io/name: logarchiver
-    app.kubernetes.io/component: archiver
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: logarchiver
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: logarchiver
-        vector.dev/exclude: "true"
-    spec:
-      # Dedicated SA whose projected vault-audience token authenticates the
-      # k8s-auth login used to fetch the logarchive public key from the gpg engine.
-      serviceAccountName: logarchiver
-      automountServiceAccountToken: false
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65532
-        runAsGroup: 65532
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: logarchiver
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/logarchiver:v0.1.0
-          imagePullPolicy: IfNotPresent
-          args: ["run"]
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop:
-                - ALL
-          ports:
-            - containerPort: 9090
-              name: metrics
-              protocol: TCP
-          env:
-            - name: LOGARCHIVER_CONFIG
-              value: /etc/logarchiver/config.yaml
-            # Server-side subject filter; must match the archiver consumer's filter.
-            - name: ARCHIVE_SUBJECTS
-              value: "logs.k8s.vault.>"
-            - name: NATS_CONSUMER_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: nats-auth
-                  key: consumer_password
-            - name: CLICKHOUSE_USER
-              valueFrom:
-                secretKeyRef:
-                  name: clickhouse-credentials
-                  key: username
-            - name: CLICKHOUSE_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: clickhouse-credentials
-                  key: password
-          # S3 creds + S3_ENDPOINT + BUCKET_NAME from the cephrgw BucketAccess Secret.
-          envFrom:
-            - secretRef:
-                name: logs-archive-s3
-          livenessProbe:
-            httpGet:
-              path: /healthz
-              port: metrics
-            initialDelaySeconds: 15
-            periodSeconds: 30
-            timeoutSeconds: 5
-            failureThreshold: 3
-          readinessProbe:
-            httpGet:
-              path: /healthz
-              port: metrics
-            initialDelaySeconds: 5
-            periodSeconds: 10
-            timeoutSeconds: 5
-            failureThreshold: 3
-          resources:
-            requests:
-              cpu: 100m
-              memory: 256Mi
-            limits:
-              cpu: "1"
-              memory: 1Gi
-          volumeMounts:
-            - name: config
-              mountPath: /etc/logarchiver/config.yaml
-              subPath: config.yaml
-              readOnly: true
-            - name: vault-token
-              mountPath: /var/run/secrets/vault
-              readOnly: true
-            - name: vault-ca-cert
-              mountPath: /etc/vault-ca/ca.crt
-              subPath: ca.crt
-              readOnly: true
-            - name: tmp
-              mountPath: /tmp
-      volumes:
-        - name: config
-          configMap:
-            name: logarchiver-config
-        # Projected SA token with audience "vault" for the gpg-engine k8s login.
-        - name: vault-token
-          projected:
-            sources:
-              - serviceAccountToken:
-                  path: token
-                  audience: vault
-                  expirationSeconds: 600
-        - name: vault-ca-cert
-          secret:
-            secretName: vault-ca-cert
-        - name: tmp
-          emptyDir: {}

apps/base/logging/gateway.yaml

@@ -1,16 +1,13 @@
 ---
-# Log ingestion endpoint for puppet-managed VMs (and any non-k8s client).
-# Reuses the internal Traefik gateway + cert-manager + external-dns pattern so
-# VMs reach the Vector aggregator's HTTP source over TLS at a DNS name they can
-# resolve. The puppet-side Vector rollout ships NDJSON to
-# https://logs-ingest.k8s.syd1.au.unkin.net/ (a later task).
+# Log ingestion endpoint for puppet-managed VMs (and any non-k8s client):
+# fronts the VLCluster vlinsert service over TLS at a name VMs can resolve.
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
   name: logs-ingest
   namespace: logging
   labels:
-    app.kubernetes.io/name: vector-aggregator
+    app.kubernetes.io/name: victorialogs
     app.kubernetes.io/component: ingest
     traefik.io/instance: internal
   annotations:
@@ -42,3 +39,43 @@ spec:
           - group: ""
             kind: Secret
             name: logs-ingest-tls
+---
+# Unauthenticated LogsQL query endpoint fronting the VLCluster vlselect service.
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: vlselect
+  namespace: logging
+  labels:
+    app.kubernetes.io/name: victorialogs
+    app.kubernetes.io/component: query
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: vlselect.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    external-dns.alpha.kubernetes.io/hostname: vlselect.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      hostname: vlselect.k8s.syd1.au.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https
+      port: 443
+      protocol: HTTPS
+      hostname: vlselect.k8s.syd1.au.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: vlselect-tls

apps/base/logging/httproute.yaml

@@ -5,7 +5,7 @@ metadata:
   name: logs-ingest-http-redirect
   namespace: logging
   labels:
-    app.kubernetes.io/name: vector-aggregator
+    app.kubernetes.io/name: victorialogs
     app.kubernetes.io/component: ingest
 spec:
   hostnames:
@@ -32,7 +32,7 @@ metadata:
   name: logs-ingest
   namespace: logging
   labels:
-    app.kubernetes.io/name: vector-aggregator
+    app.kubernetes.io/name: victorialogs
     app.kubernetes.io/component: ingest
 spec:
   hostnames:
@@ -46,8 +46,63 @@ spec:
     - backendRefs:
         - group: ""
           kind: Service
-          name: vector-vm-ingest
-          port: 8080
+          name: vlinsert-logs
+          port: 9481
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: vlselect-http-redirect
+  namespace: logging
+  labels:
+    app.kubernetes.io/name: victorialogs
+    app.kubernetes.io/component: query
+spec:
+  hostnames:
+    - vlselect.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: vlselect
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: vlselect
+  namespace: logging
+  labels:
+    app.kubernetes.io/name: victorialogs
+    app.kubernetes.io/component: query
+spec:
+  hostnames:
+    - vlselect.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: vlselect
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: vlselect-logs
+          port: 9471
           weight: 1
       matches:
         - path:

apps/base/logging/job_clickhouse-schema.yaml

@@ -1,131 +0,0 @@
----
-# Declarative ClickHouse schema bootstrap. Runs as an ArgoCD PostSync hook so it
-# executes after the ClickHouseInstallation is reconciled, and re-runs on every
-# sync (idempotent CREATE ... IF NOT EXISTS). Edit the DDL here to evolve the
-# schema; the Vector aggregator writes to logs.raw with skip_unknown_fields, so
-# adding columns is backward-compatible.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: clickhouse-schema
-  namespace: logging
-  annotations:
-    argocd.argoproj.io/hook: PostSync
-    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
-  labels:
-    app.kubernetes.io/name: clickhouse-schema
-    app.kubernetes.io/component: bootstrap
-spec:
-  backoffLimit: 20
-  activeDeadlineSeconds: 1800
-  ttlSecondsAfterFinished: 3600
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: clickhouse-schema
-        vector.dev/exclude: "true"
-    spec:
-      restartPolicy: OnFailure
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 101
-        runAsGroup: 101
-      containers:
-        - name: clickhouse-schema
-          image: docker.io/clickhouse/clickhouse-server:24.8
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop:
-                - ALL
-          env:
-            - name: HOME
-              value: /tmp
-            - name: CLICKHOUSE_USER
-              valueFrom:
-                secretKeyRef:
-                  name: clickhouse-credentials
-                  key: username
-            - name: CLICKHOUSE_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: clickhouse-credentials
-                  key: password
-          command:
-            - /bin/bash
-            - -ec
-            - |
-              host=clickhouse-logs.logging.svc.cluster.local
-              echo "Waiting for ClickHouse at ${host}:9000 ..."
-              until clickhouse-client --host "$host" --port 9000 \
-                  --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \
-                  --query "SELECT 1" >/dev/null 2>&1; do
-                echo "  not ready, retrying in 5s"; sleep 5
-              done
-              echo "Applying schema ..."
-              clickhouse-client --host "$host" --port 9000 \
-                --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \
-                --multiquery <<'EOSQL'
-              CREATE DATABASE IF NOT EXISTS logs;
-
-              CREATE TABLE IF NOT EXISTS logs.raw
-              (
-                  timestamp  DateTime64(3)           DEFAULT now64(3),
-                  host       LowCardinality(String)  DEFAULT '',
-                  source     LowCardinality(String)  DEFAULT '',
-                  namespace  LowCardinality(String)  DEFAULT '',
-                  pod        String                  DEFAULT '',
-                  container  LowCardinality(String)  DEFAULT '',
-                  stream     LowCardinality(String)  DEFAULT '',
-                  severity   LowCardinality(String)  DEFAULT '',
-                  message    String                  DEFAULT '',
-                  labels     Map(LowCardinality(String), String),
-                  fields     Map(LowCardinality(String), String)
-              )
-              ENGINE = MergeTree
-              PARTITION BY toDate(timestamp)
-              ORDER BY (source, namespace, host, timestamp)
-              TTL toDateTime(timestamp) + INTERVAL 3 DAY
-              SETTINGS index_granularity = 8192;
-
-              -- One row per archived S3 object (written by logarchiver). No TTL:
-              -- the index must outlive logs.raw so the long-term S3 archive stays
-              -- searchable. Keep in sync with logarchiver internal/index/ddl.go.
-              CREATE TABLE IF NOT EXISTS logs.archive_index
-              (
-                  object_key       String,
-                  bucket           LowCardinality(String),
-                  subject          LowCardinality(String),
-                  hosts            Array(LowCardinality(String)),
-                  min_ts           DateTime64(3),
-                  max_ts           DateTime64(3),
-                  event_count      UInt64,
-                  raw_bytes        UInt64,
-                  stored_bytes     UInt64,
-                  compression      LowCardinality(String),
-                  cipher           LowCardinality(String),
-                  container_format LowCardinality(String),
-                  key_name         LowCardinality(String),
-                  key_fingerprint  String,
-                  created_at       DateTime64(3) DEFAULT now64(3),
-                  INDEX idx_hosts hosts TYPE bloom_filter GRANULARITY 1
-              )
-              ENGINE = MergeTree
-              PARTITION BY toYYYYMM(min_ts)
-              ORDER BY (subject, min_ts, object_key);
-              EOSQL
-              echo "Schema applied."
-          resources:
-            requests:
-              cpu: 50m
-              memory: 128Mi
-            limits:
-              cpu: 500m
-              memory: 512Mi
-          volumeMounts:
-            - name: tmp
-              mountPath: /tmp
-      volumes:
-        - name: tmp
-          emptyDir: {}