add agent-observability service account and namespace-scoped RBAC

.pre-commit-config.yaml

@@ -42,9 +42,3 @@ repos:
         entry: ci/validate-no-secrets.sh
         language: system
         pass_filenames: false
-      - id: runner_hooks
-        name: test arc runner job-started hook
-        entry: ci/test-runner-hooks.sh
-        language: system
-        pass_filenames: false
-        files: ^(apps/base/iuclabs-runners/job-started\.sh|ci/test-runner-hooks\.sh|ci/fixtures/runner-hooks/)

.woodpecker/vector-test.yaml

@@ -0,0 +1,29 @@
+when:
+  - event: pull_request
+
+steps:
+  - name: vector-test
+    image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector:0.57.0-debian
+    commands:
+      # Dummy creds + writable dirs so the full topologies build; the unit tests
+      # only exercise the transforms (sources are not started).
+      - export CLICKHOUSE_USER=ci CLICKHOUSE_PASSWORD=ci
+      - export NATS_PRODUCER_PASSWORD=ci NATS_CONSUMER_PASSWORD=ci
+      - mkdir -p /vector-data-dir /etc/vault-ca
+      - cp /etc/ssl/certs/ca-certificates.crt /etc/vault-ca/ca.crt
+      # Transform tier + VM ingest: unit-tested transforms.
+      - vector test apps/base/logging/vector/aggregator.yaml apps/base/logging/vector/aggregator-tests.yaml
+      - vector test apps/base/logging/vector/vm-ingest.yaml apps/base/logging/vector/vm-ingest-tests.yaml
+      # Agent has no transforms to unit-test; validate it builds. (The archiver
+      # leg is now the logarchiver service, not a Vector pipeline.)
+      - vector validate --no-environment apps/base/logging/vector/agent.yaml
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 256Mi
+            cpu: 250m
+          limits:
+            memory: 1Gi
+            cpu: 1

apps/base/arc-system/kustomization.yaml

@@ -1,8 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - namespace.yaml
-  - pdb.yaml
-  - vmpodscrape.yaml

apps/base/arc-system/namespace.yaml

@@ -1,5 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: arc-system

apps/base/arc-system/pdb.yaml

@@ -1,12 +0,0 @@
----
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
-  name: arc-controller
-  namespace: arc-system
-spec:
-  minAvailable: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: gha-rs-controller
-      app.kubernetes.io/instance: arc

apps/base/arc-system/vmpodscrape.yaml

@@ -1,40 +0,0 @@
----
-# Controller metrics. gha_controller_*_ephemeral_runners carry the runner
-# namespace, which the dashboard filters on, so restore it over the target's.
-apiVersion: operator.victoriametrics.com/v1beta1
-kind: VMPodScrape
-metadata:
-  name: arc-controller
-  namespace: arc-system
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: gha-rs-controller
-      app.kubernetes.io/instance: arc
-  podMetricsEndpoints:
-    - port: metrics
-      path: /metrics
-      metricRelabelConfigs:
-        - sourceLabels: [__name__, exported_namespace]
-          regex: gha_controller_.+_ephemeral_runners;(.+)
-          targetLabel: namespace
----
-# Listener metrics carry their own name/namespace (the runner scale set), and
-# the dashboard also expects the scale-set pod labels.
-apiVersion: operator.victoriametrics.com/v1beta1
-kind: VMPodScrape
-metadata:
-  name: arc-listener
-  namespace: arc-system
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/component: runner-scale-set-listener
-      app.kubernetes.io/part-of: gha-runner-scale-set
-  podMetricsEndpoints:
-    - port: metrics
-      path: /metrics
-      honorLabels: true
-      relabelConfigs:
-        - action: labelmap
-          regex: __meta_kubernetes_pod_label_(actions_github_com_scale_set_.+)

apps/base/artifactapi/api-deployment.yaml

@@ -36,7 +36,7 @@ spec:
               mountPath: /combined-certs
       containers:
         - name: api
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.25.2
+          image: git.unkin.net/unkin/artifactapi:v3.11.2
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000

apps/base/artifactapi/configmap.yaml

@@ -5,9 +5,6 @@ metadata:
   name: api-env
   namespace: artifactapi
 data:
-  ARTIFACTAPI_AUTH_UPLOADS: "true"
-  ARTIFACTAPI_OIDC_AUDIENCE: artifactapi
-  ARTIFACTAPI_OIDC_ISSUER: https://vault.service.consul:8200/v1/identity/oidc
   DBHOST: postgres-pooler
   DBNAME: artifacts
   DBPORT: "5432"

apps/base/artifactapi/image-keeper.yaml

@@ -1,105 +0,0 @@
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: image-keeper
-  namespace: artifactapi
-spec:
-  selector:
-    matchLabels:
-      app: image-keeper
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 25%
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        app: image-keeper
-    spec:
-      automountServiceAccountToken: false
-      priorityClassName: low
-      tolerations:
-        - operator: Exists
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65532
-        runAsGroup: 65532
-        seccompProfile:
-          type: RuntimeDefault
-      initContainers:
-        # artifactapi is distroless with no exit-0 flag, so run a static busybox as `true`
-        - name: copy-true
-          image: busybox:1.37.0-musl
-          imagePullPolicy: IfNotPresent
-          command: ["cp", "/bin/busybox", "/keeper/true"]
-          volumeMounts:
-            - name: keeper
-              mountPath: /keeper
-          resources:
-            limits:
-              cpu: 10m
-              memory: 16Mi
-            requests:
-              cpu: 1m
-              memory: 4Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-        - name: api
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.25.2
-          imagePullPolicy: IfNotPresent
-          command: ["/keeper/true"]
-          volumeMounts:
-            - name: keeper
-              mountPath: /keeper
-              readOnly: true
-          resources:
-            limits:
-              cpu: 10m
-              memory: 16Mi
-            requests:
-              cpu: 1m
-              memory: 4Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-        - name: ui
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.25.2
-          imagePullPolicy: IfNotPresent
-          command: ["true"]
-          resources:
-            limits:
-              cpu: 10m
-              memory: 16Mi
-            requests:
-              cpu: 1m
-              memory: 4Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-      containers:
-        - name: pause
-          image: rancher/mirrored-pause:3.6
-          imagePullPolicy: IfNotPresent
-          resources:
-            limits:
-              cpu: 10m
-              memory: 16Mi
-            requests:
-              cpu: 1m
-              memory: 4Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-      volumes:
-        - name: keeper
-          emptyDir: {}

apps/base/artifactapi/kustomization.yaml

@@ -11,7 +11,6 @@ resources:
   - cnpg_pooler.yaml
   - gateway.yaml
   - httproute.yaml
-  - image-keeper.yaml
   - namespace.yaml
   - oauth2-proxy-configmap.yaml
   - oauth2-proxy-deployment.yaml

apps/base/artifactapi/ui-deployment.yaml

@@ -22,7 +22,7 @@ spec:
       automountServiceAccountToken: true
       containers:
         - name: ui
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.25.2
+          image: git.unkin.net/unkin/artifactapi-ui:v3.11.2
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 80

apps/base/bind-external/acme-unkin-net/a/kustomization.yaml

@@ -1,6 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - ns1.yaml

apps/base/bind-external/acme-unkin-net/a/ns1.yaml

@@ -1,16 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: acme-ns1-a
-  namespace: bind-external
-spec:
-  zoneRef: acme-unkin-net
-  name: ns1
-  type: A
-  ttl: 3600
-  values:
-    # Public address of this cluster's external BIND, same target as
-    # acme-ns1.unkin.net. Resolvers that cached the seeded ns1.acme.unkin.net
-    # NS name must still reach the zone.
-    - 103.216.191.185

apps/base/bind-external/acme-unkin-net/kustomization.yaml

@@ -1,11 +0,0 @@
----
-# Authoritative delegation records for acme.unkin.net. Without these the zone
-# only holds the operator's seed apex (NS ns1.acme.unkin.net glued to the
-# primary pod IP), which is unroutable off-cluster and goes stale on
-# reschedule. DNSRecords must live in the same namespace as their BindZone.
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - ns
-  - a

apps/base/bind-external/acme-unkin-net/ns/apex.yaml

@@ -1,16 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: acme-apex-ns
-  namespace: bind-external
-spec:
-  zoneRef: acme-unkin-net
-  # "@" is the zone apex.
-  name: "@"
-  type: NS
-  ttl: 3600
-  values:
-    # Matches the parent delegation in Google Cloud DNS. Out of zone, so the
-    # child needs no glue of its own.
-    - acme-ns1.unkin.net.

apps/base/bind-external/acme-unkin-net/ns/kustomization.yaml

@@ -1,6 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - apex.yaml

apps/base/bind-external/kustomization.yaml

@@ -7,5 +7,5 @@ resources:
   - cluster.yaml
   - tsigkey.yaml
   - zones.yaml
-  - acme-unkin-net
+  - records.yaml
   - agent-dns-rolebinding.yaml

apps/base/bind-external/records.yaml

@@ -0,0 +1,36 @@
+# Authoritative delegation records for acme.unkin.net. Without these the zone
+# only holds the operator's seed apex (NS ns1.acme.unkin.net glued to the
+# primary pod IP), which is unroutable off-cluster and goes stale on
+# reschedule. DNSRecords must live in the same namespace as their BindZone.
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: acme-apex-ns
+  namespace: bind-external
+spec:
+  zoneRef: acme-unkin-net
+  # "@" is the zone apex.
+  name: "@"
+  type: NS
+  ttl: 3600
+  values:
+    # Matches the parent delegation in Google Cloud DNS. Out of zone, so the
+    # child needs no glue of its own.
+    - acme-ns1.unkin.net.
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: acme-ns1-a
+  namespace: bind-external
+spec:
+  zoneRef: acme-unkin-net
+  name: ns1
+  type: A
+  ttl: 3600
+  values:
+    # Public address of this cluster's external BIND, same target as
+    # acme-ns1.unkin.net. Resolvers that cached the seeded ns1.acme.unkin.net
+    # NS name must still reach the zone.
+    - 103.216.191.185

apps/base/bind-internal/authoritative/ceph-unkin-net/cname/dashboard.yaml

@@ -1,15 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: dashboard-ceph-cname
-  namespace: bind-internal
-spec:
-  zoneRef: ceph-unkin-net
-  name: dashboard
-  type: CNAME
-  ttl: 600
-  values:
-    # Ceph mgr dashboard, reached via lb1. Lets in-cluster clients (the
-    # cephrgw-operator) resolve dashboard.ceph.unkin.net.
-    - lb1.unkin.net.

apps/base/bind-internal/authoritative/ceph-unkin-net/cname/kustomization.yaml

@@ -1,7 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - dashboard.yaml
-  - s3.yaml

apps/base/bind-internal/authoritative/ceph-unkin-net/cname/s3.yaml

@@ -1,15 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: s3-ceph-cname
-  namespace: bind-internal
-spec:
-  zoneRef: ceph-unkin-net
-  name: s3
-  type: CNAME
-  ttl: 600
-  values:
-    # radosgw S3 endpoint. Points at the Consul service for now; the real
-    # target will be changed later.
-    - radosgw.service.consul.

apps/base/bind-internal/authoritative/ceph-unkin-net/kustomization.yaml

@@ -1,6 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - cname

apps/base/bind-internal/authoritative/kustomization.yaml

@@ -2,15 +2,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-# Individually-managed authoritative records live under <zone>/<type>/<record>.yaml.
-# DNSRecords must live in the same namespace as their BindZone (the operator
-# resolves zoneRef/clusterRef/updateKeyRef within the record's namespace), so
-# these sit alongside the zone in bind-internal, not in the app namespace.
 resources:
   - cluster.yaml
   - tsigkey.yaml
   - zones.yaml
-  - unkin-net
-  - main-unkin-net
-  - ceph-unkin-net
+  - records.yaml
   - acls.yaml

apps/base/bind-internal/authoritative/main-unkin-net/a/jellyfin.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: jellyfin-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: main-unkin-net
-  name: jellyfin
-  type: A
-  ttl: 600
-  values:
-    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
-    - 198.18.199.0

apps/base/bind-internal/authoritative/main-unkin-net/a/kustomization.yaml

@@ -1,13 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - jellyfin.yaml
-  - mail-in.yaml
-  - lidarr.yaml
-  - nzbget.yaml
-  - prowlarr.yaml
-  - radarr.yaml
-  - readarr.yaml
-  - sonarr.yaml

apps/base/bind-internal/authoritative/main-unkin-net/a/lidarr.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: lidarr-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: main-unkin-net
-  name: lidarr
-  type: A
-  ttl: 600
-  values:
-    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
-    - 198.18.199.0

apps/base/bind-internal/authoritative/main-unkin-net/a/mail-in.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: mail-in-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: main-unkin-net
-  name: mail-in
-  type: A
-  ttl: 600
-  values:
-    # home WAN (public path for edge to smarthost relay), mirrors public DNS.
-    - 103.216.191.185

apps/base/bind-internal/authoritative/main-unkin-net/a/nzbget.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: nzbget-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: main-unkin-net
-  name: nzbget
-  type: A
-  ttl: 600
-  values:
-    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
-    - 198.18.199.0

apps/base/bind-internal/authoritative/main-unkin-net/a/prowlarr.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: prowlarr-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: main-unkin-net
-  name: prowlarr
-  type: A
-  ttl: 600
-  values:
-    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
-    - 198.18.199.0

apps/base/bind-internal/authoritative/main-unkin-net/a/radarr.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: radarr-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: main-unkin-net
-  name: radarr
-  type: A
-  ttl: 600
-  values:
-    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
-    - 198.18.199.0

apps/base/bind-internal/authoritative/main-unkin-net/a/readarr.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: readarr-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: main-unkin-net
-  name: readarr
-  type: A
-  ttl: 600
-  values:
-    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
-    - 198.18.199.0

apps/base/bind-internal/authoritative/main-unkin-net/a/sonarr.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: sonarr-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: main-unkin-net
-  name: sonarr
-  type: A
-  ttl: 600
-  values:
-    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
-    - 198.18.199.0

apps/base/bind-internal/authoritative/records.yaml

@@ -0,0 +1,164 @@
+# Individually-managed authoritative records for the unkin.net zone.
+# DNSRecords must live in the same namespace as their BindZone (the operator
+# resolves zoneRef/clusterRef/updateKeyRef within the record's namespace), so
+# these sit alongside the zone in bind-internal, not in the app namespace.
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  # "internal" in the name distinguishes this from the external DNS that
+  # Authentik will manage its own records from later.
+  name: identity-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: identity
+  type: A
+  ttl: 600
+  values:
+    # traefik-EXTERNAL (DMZ) gateway VIP; the authentik Gateway serves the
+    # identity.unkin.net hostname there.
+    - 198.18.199.0
+---
+# PRODUCTION CUTOVER RECORD — intentionally commented out.
+# git.unkin.net currently resolves to the LIVE VM forge (HAProxy VRRP VIP
+# 198.18.19.17), which holds every repo the estate depends on. Uncommenting this
+# repoints the whole org's git.unkin.net at the new k8s Gitea gateway VIP, so it
+# is the FINAL step of the forge migration — gated on the data migration (gitea
+# dump/restore + SECRET_KEY copy) in argocd-apps docs/gitea-migration.md.
+# NOTE: the live git.unkin.net answer is served by the puppet DNS master today
+# (profiles::dns::master, records from PuppetDB); this k8s apex zone holds only
+# SOA+NS + a few DNSRecords so far. Confirm the k8s bind cluster is the live
+# authority for unkin.net (or update the puppet record instead) before relying
+# on this CR at cutover.
+# ---
+# apiVersion: bind.unkin.net/v1alpha1
+# kind: DNSRecord
+# metadata:
+#   name: git-dns-internal
+#   namespace: bind-internal
+# spec:
+#   zoneRef: unkin-net
+#   name: git
+#   type: A
+#   ttl: 600
+#   values:
+#     # traefik-internal gateway VIP; the gitea Gateway serves git.unkin.net there.
+#     - 198.18.200.4
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: s3-ceph-cname
+  namespace: bind-internal
+spec:
+  zoneRef: ceph-unkin-net
+  name: s3
+  type: CNAME
+  ttl: 600
+  values:
+    # radosgw S3 endpoint. Points at the Consul service for now; the real
+    # target will be changed later.
+    - radosgw.service.consul.
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: dashboard-ceph-cname
+  namespace: bind-internal
+spec:
+  zoneRef: ceph-unkin-net
+  name: dashboard
+  type: CNAME
+  ttl: 600
+  values:
+    # Ceph mgr dashboard, reached via lb1. Lets in-cluster clients (the
+    # cephrgw-operator) resolve dashboard.ceph.unkin.net.
+    - lb1.unkin.net.
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: lb1-unkin-net
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: lb1
+  type: A
+  ttl: 600
+  values:
+    - 103.216.191.185
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: ghp-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: ghp
+  type: A
+  ttl: 600
+  values:
+    # traefik-internal gateway VIP; the ghp Gateway serves ghp.unkin.net there.
+    - 198.18.200.4
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: arrstack-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: arrstack
+  type: A
+  ttl: 600
+  values:
+    # traefik-EXTERNAL (DMZ) gateway VIP; the arrproxy Gateway serves the
+    # arrstack.unkin.net front door (oauth2-proxy) there.
+    - 198.18.199.0
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: logviewer-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: logviewer
+  type: A
+  ttl: 600
+  values:
+    # traefik-internal gateway VIP; the logviewer Gateway serves
+    # logviewer.unkin.net there.
+    - 198.18.200.4
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: cheeztv-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: cheeztv
+  type: A
+  ttl: 600
+  values:
+    # traefik-internal gateway VIP; the cheeztv Gateway serves cheeztv.unkin.net
+    # there.
+    - 198.18.200.4
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: watchstate-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: watchstate
+  type: A
+  ttl: 600
+  values:
+    # traefik-EXTERNAL (DMZ) gateway VIP; the watchstate-external Gateway serves
+    # the watchstate.unkin.net front door (oauth2-proxy) there.
+    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/a/arrstack.yaml

@@ -1,15 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: arrstack-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: arrstack
-  type: A
-  ttl: 600
-  values:
-    # traefik-EXTERNAL (DMZ) gateway VIP; the arrproxy Gateway serves the
-    # arrstack.unkin.net front door (oauth2-proxy) there.
-    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/a/auth.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: auth-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: auth
-  type: A
-  ttl: 600
-  values:
-    # traefik-internal gateway VIP; the kanidm Gateway serves auth.unkin.net there.
-    - 198.18.200.4

apps/base/bind-internal/authoritative/unkin-net/a/cheeztv.yaml

@@ -1,15 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: cheeztv-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: cheeztv
-  type: A
-  ttl: 600
-  values:
-    # traefik-internal gateway VIP; the cheeztv Gateway serves cheeztv.unkin.net
-    # there.
-    - 198.18.200.4

apps/base/bind-internal/authoritative/unkin-net/a/fafflix.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: fafflix-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: fafflix
-  type: A
-  ttl: 600
-  values:
-    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
-    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/a/ghp.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: ghp-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: ghp
-  type: A
-  ttl: 600
-  values:
-    # traefik-internal gateway VIP; the ghp Gateway serves ghp.unkin.net there.
-    - 198.18.200.4

apps/base/bind-internal/authoritative/unkin-net/a/git-edge.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: git-edge-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: git
-  type: A
-  ttl: 600
-  values:
-    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
-    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/a/git.yaml

@@ -1,23 +0,0 @@
----
-# PRODUCTION CUTOVER RECORD — intentionally commented out.
-# git.unkin.net currently resolves to the haproxy edge (git-edge.yaml), which
-# forwards to the LIVE VM forge holding every repo. Uncommenting this
-# repoints the whole org's git.unkin.net at the new k8s Gitea gateway VIP, so it
-# is the FINAL step of the forge migration — gated on the data migration (gitea
-# dump/restore + SECRET_KEY copy) in argocd-apps docs/gitea-migration.md.
-# Uncomment this record AND its entry in kustomization.yaml (removing
-# git-edge.yaml there) to activate it.
-# ---
-# apiVersion: bind.unkin.net/v1alpha1
-# kind: DNSRecord
-# metadata:
-#   name: git-dns-internal
-#   namespace: bind-internal
-# spec:
-#   zoneRef: unkin-net
-#   name: git
-#   type: A
-#   ttl: 600
-#   values:
-#     # traefik-internal gateway VIP; the gitea Gateway serves git.unkin.net there.
-#     - 198.18.200.4

apps/base/bind-internal/authoritative/unkin-net/a/grafana.yaml

@@ -1,15 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: grafana-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: grafana
-  type: A
-  ttl: 600
-  values:
-    # traefik-EXTERNAL (DMZ) gateway VIP; the grafana-external Gateway serves
-    # grafana.unkin.net there.
-    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/a/identity.yaml

@@ -1,17 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  # "internal" in the name distinguishes this from the external DNS that
-  # Authentik will manage its own records from later.
-  name: identity-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: identity
-  type: A
-  ttl: 600
-  values:
-    # traefik-EXTERNAL (DMZ) gateway VIP; the authentik Gateway serves the
-    # identity.unkin.net hostname there.
-    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/a/kustomization.yaml

@@ -1,23 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - arrstack.yaml
-  - auth.yaml
-  - cheeztv.yaml
-  - fafflix.yaml
-  # PRODUCTION CUTOVER RECORD — see git.yaml. Uncomment together with the
-  # record itself, and remove git-edge.yaml.
-  # - git.yaml
-  - ghp.yaml
-  - git-edge.yaml
-  - grafana.yaml
-  - identity.yaml
-  - lb1.yaml
-  - logviewer.yaml
-  - mail.yaml
-  - mta1-edge.yaml
-  - mta2-edge.yaml
-  - vlogs.yaml
-  - watchstate.yaml

apps/base/bind-internal/authoritative/unkin-net/a/lb1.yaml

@@ -1,13 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: lb1-unkin-net
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: lb1
-  type: A
-  ttl: 600
-  values:
-    - 103.216.191.185

apps/base/bind-internal/authoritative/unkin-net/a/logviewer.yaml

@@ -1,15 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: logviewer-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: logviewer
-  type: A
-  ttl: 600
-  values:
-    # traefik-internal gateway VIP; the logviewer Gateway serves
-    # logviewer.unkin.net there.
-    - 198.18.200.4

apps/base/bind-internal/authoritative/unkin-net/a/mail.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: mail-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: mail
-  type: A
-  ttl: 600
-  values:
-    # stalwart-lb PureLB VIP (IMAPS, submission, HTTPS).
-    - 198.18.200.18

apps/base/bind-internal/authoritative/unkin-net/a/mta1-edge.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: mta1-edge-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: mta1.edge
-  type: A
-  ttl: 600
-  values:
-    # edge MTA 1 public IP, mirrors public DNS.
-    - 66.226.148.207

apps/base/bind-internal/authoritative/unkin-net/a/mta2-edge.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: mta2-edge-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: mta2.edge
-  type: A
-  ttl: 600
-  values:
-    # edge MTA 2 public IP, mirrors public DNS.
-    - 66.226.149.50

apps/base/bind-internal/authoritative/unkin-net/a/vlogs.yaml

@@ -1,15 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: vlogs-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: vlogs
-  type: A
-  ttl: 600
-  values:
-    # traefik-EXTERNAL (DMZ) gateway VIP; the vlogs-external Gateway serves the
-    # vlogs.unkin.net front door (oauth2-proxy) there.
-    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/a/watchstate.yaml

@@ -1,15 +0,0 @@
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: DNSRecord
-metadata:
-  name: watchstate-dns-internal
-  namespace: bind-internal
-spec:
-  zoneRef: unkin-net
-  name: watchstate
-  type: A
-  ttl: 600
-  values:
-    # traefik-EXTERNAL (DMZ) gateway VIP; the watchstate-external Gateway serves
-    # the watchstate.unkin.net front door (oauth2-proxy) there.
-    - 198.18.199.0

apps/base/bind-internal/authoritative/unkin-net/kustomization.yaml

@@ -1,6 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - a

apps/base/bind-internal/resolvers/acls.yaml

@@ -38,9 +38,6 @@ spec:
   clusterRef: bind-resolvers
   entries:
     - 198.18.24.0/24
-    - 172.18.30.0/24  # BinaryLane DMZ VPC syd (mail edge, via VPN)
-    - 172.18.31.0/24  # BinaryLane DMZ VPC mel (mail edge, via VPN)
-    - 172.18.32.0/24  # BinaryLane DMZ VPC bne (mail edge, via VPN)
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: BindACL

apps/base/bind-internal/resolvers/cluster.yaml

@@ -18,10 +18,9 @@ spec:
       purelb.io/service-group: common
       purelb.io/addresses: 198.18.200.7
       external-dns.alpha.kubernetes.io/hostname: bind-resolvers.k8s.syd1.au.unkin.net
-  # Public names go via global forwarders; RBL zones bypass them (view.yaml).
   forwarders:
-    - 1.1.1.1
     - 8.8.8.8
+    - 1.1.1.1
   # The internal split-horizon zones are served UNSIGNED by the in-cluster
   # authoritative, but their public parents publish DS records (e.g. unkin.net
   # is DNSSEC-signed on the Internet). With dnssec-validation on, the validator

apps/base/bind-internal/resolvers/kustomization.yaml

@@ -7,4 +7,3 @@ resources:
   - acls.yaml
   - view.yaml
   - forward-zones.yaml
-  - rbl-zones.yaml

apps/base/bind-internal/resolvers/rbl-zones.yaml

@@ -1,121 +0,0 @@
-# DNSBLs refuse queries relayed via public resolvers, so these zones resolve directly.
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: BindZone
-metadata:
-  name: rbl-spamhaus-org
-  namespace: bind-internal
-spec:
-  clusterRef: bind-resolvers
-  viewRef: openforwarder
-  zoneName: spamhaus.org
-  type: forward
-  catalog: false
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: BindZone
-metadata:
-  name: rbl-mailspike-net
-  namespace: bind-internal
-spec:
-  clusterRef: bind-resolvers
-  viewRef: openforwarder
-  zoneName: mailspike.net
-  type: forward
-  catalog: false
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: BindZone
-metadata:
-  name: rbl-spameatingmonkey-net
-  namespace: bind-internal
-spec:
-  clusterRef: bind-resolvers
-  viewRef: openforwarder
-  zoneName: spameatingmonkey.net
-  type: forward
-  catalog: false
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: BindZone
-metadata:
-  name: rbl-dnswl-org
-  namespace: bind-internal
-spec:
-  clusterRef: bind-resolvers
-  viewRef: openforwarder
-  zoneName: dnswl.org
-  type: forward
-  catalog: false
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: BindZone
-metadata:
-  name: rbl-virusfree-cz
-  namespace: bind-internal
-spec:
-  clusterRef: bind-resolvers
-  viewRef: openforwarder
-  zoneName: virusfree.cz
-  type: forward
-  catalog: false
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: BindZone
-metadata:
-  name: rbl-blocklist-de
-  namespace: bind-internal
-spec:
-  clusterRef: bind-resolvers
-  viewRef: openforwarder
-  zoneName: blocklist.de
-  type: forward
-  catalog: false
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: BindZone
-metadata:
-  name: rbl-msbl-org
-  namespace: bind-internal
-spec:
-  clusterRef: bind-resolvers
-  viewRef: openforwarder
-  zoneName: msbl.org
-  type: forward
-  catalog: false
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: BindZone
-metadata:
-  name: rbl-surbl-org
-  namespace: bind-internal
-spec:
-  clusterRef: bind-resolvers
-  viewRef: openforwarder
-  zoneName: surbl.org
-  type: forward
-  catalog: false
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: BindZone
-metadata:
-  name: rbl-uribl-com
-  namespace: bind-internal
-spec:
-  clusterRef: bind-resolvers
-  viewRef: openforwarder
-  zoneName: uribl.com
-  type: forward
-  catalog: false
----
-apiVersion: bind.unkin.net/v1alpha1
-kind: BindZone
-metadata:
-  name: rbl-rspamd-com
-  namespace: bind-internal
-spec:
-  clusterRef: bind-resolvers
-  viewRef: openforwarder
-  zoneName: rspamd.com
-  type: forward
-  catalog: false

apps/base/bind-internal/resolvers/view.yaml

@@ -21,5 +21,3 @@ spec:
   extraOptions:
     - "allow-recursion { any; }"
     - "allow-query-cache { any; }"
-    # pods have no IPv6 route; v6 NS attempts burn the recursion-query budget
-    - "server ::/0 { bogus yes; }"

apps/base/bind-system/deployment.yaml

@@ -21,7 +21,7 @@ spec:
         runAsNonRoot: true
       containers:
         - name: operator
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.4.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.3.0
           args:
             - --metrics-bind-address=:8080
             - --health-probe-bind-address=:8081

apps/base/bind-system/kustomization.yaml

@@ -6,7 +6,7 @@ resources:
   - namespace.yaml
   # CRDs are pulled from the bind-operator repo at the matching tag rather than
   # vendored here, so they never drift from the operator.
-  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.4.0/config/crd/install.yaml
+  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.3.0/config/crd/install.yaml
   - rbac.yaml
   - agent-dns-rbac.yaml
   - deployment.yaml

apps/base/cert-manager/certificate_wildcard-main-unkin-net.yaml

@@ -1,6 +1,6 @@
 ---
 # Let's Encrypt *.main.unkin.net wildcard for the haproxy edge (pve, arr stack,
-# jellyfin, stalwart webadmin/autoconfig) and the mail smarthost. DNS-01 needs the delegated
+# jellyfin, stalwart webadmin/autoconfig). DNS-01 needs the delegated
 # _acme-challenge.main.unkin.net CNAME in the public unkin.net zone.
 # _acme-challenge.main.unkin.net. CNAME _acme-challenge.main.acme.unkin.net.
 apiVersion: cert-manager.io/v1
@@ -13,9 +13,9 @@ spec:
   secretTemplate:
     annotations:
       reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
-      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy,mail"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy"
       reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
-      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy,mail"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy"
   privateKey:
     size: 4096
   dnsNames:

apps/base/cert-manager/certificate_wildcard-unkin-net.yaml

@@ -14,9 +14,9 @@ spec:
   secretTemplate:
     annotations:
       reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
-      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy"
       reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
-      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy"
   privateKey:
     size: 4096
   dnsNames:

apps/base/clickhouse-system/kustomization.yaml

@@ -3,4 +3,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-  - a
+  - namespace.yaml

apps/base/clickhouse-system/namespace.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    app.kubernetes.io/name: clickhouse-system
+  name: clickhouse-system

apps/base/dhcp-system/cr/keaapi.yaml

@@ -11,7 +11,7 @@ metadata:
     argocd.argoproj.io/sync-wave: "1"
 spec:
   replicas: 1
-  image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/kea-api:v0.1.7
+  image: git.unkin.net/unkin/kea-api:v0.1.3
   tokenSecretName: kea-api-token
   service:
     type: ClusterIP

apps/base/dhcp-system/cr/keacluster.yaml

@@ -11,7 +11,7 @@ metadata:
     argocd.argoproj.io/sync-wave: "1"
 spec:
   replicas: 2
-  image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/kea:v0.1.7
+  image: git.unkin.net/unkin/kea:v0.1.3
   domainName: main.unkin.net
   defaultLeaseTime: 1200
   maxLeaseTime: 86400

apps/base/dhcp-system/deployment.yaml

@@ -21,7 +21,7 @@ spec:
         runAsNonRoot: true
       containers:
         - name: operator
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/kea-operator:v0.1.7
+          image: git.unkin.net/unkin/kea-operator:v0.1.5
           args:
             - --metrics-bind-address=:8080
             - --health-probe-bind-address=:8081

apps/base/encapi/configmap.yaml

@@ -11,5 +11,3 @@ data:
   DBPORT: "5432"
   DBUSER: encapi
   DBSSL: require
-  ENCAPI_OIDC_ISSUER: https://vault.service.consul:8200/v1/identity/oidc
-  SSL_CERT_FILE: /etc/ssl/combined/ca-certificates.crt

apps/base/encapi/deployment.yaml

@@ -6,7 +6,6 @@ metadata:
   namespace: encapi
   annotations:
     configmap.reloader.stakater.com/auto: "true"
-    secret.reloader.stakater.com/reload: "vault-ca-cert"
 spec:
   replicas: 2
   selector:
@@ -22,22 +21,9 @@ spec:
         app: encapi
     spec:
       automountServiceAccountToken: true
-      initContainers:
-        - name: combine-certs
-          image: alpine:3
-          command:
-            - sh
-            - -c
-            - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt
-          volumeMounts:
-            - name: vault-ca-cert
-              mountPath: /custom-ca
-              readOnly: true
-            - name: combined-certs
-              mountPath: /combined-certs
       containers:
         - name: encapi
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/encapi:v0.3.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/encapi:v0.1.2
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000
@@ -52,10 +38,6 @@ spec:
             - secretRef:
                 name: environment
                 optional: false
-          volumeMounts:
-            - name: combined-certs
-              mountPath: /etc/ssl/combined
-              readOnly: true
           livenessProbe:
             httpGet:
               path: /healthz
@@ -79,13 +61,4 @@ spec:
             limits:
               cpu: 500m
               memory: 256Mi
-      volumes:
-        - name: vault-ca-cert
-          secret:
-            secretName: vault-ca-cert
-            items:
-              - key: ca.crt
-                path: ca.crt
-        - name: combined-certs
-          emptyDir: {}
       restartPolicy: Always

apps/base/gitea/gateway.yaml

@@ -1,6 +1,11 @@
 ---
-# HTTPS front for the k8s Gitea on git.k8s.syd1.au.unkin.net (external-dns
-# k8s.syd1 zone). git.unkin.net stays on the VM haproxy until cutover.
+# HTTPS front for the k8s Gitea, served on two names:
+#   git.unkin.net              — canonical/production (apex, bind-operator zone;
+#                                DNS flip is the gated cutover step, see the doc)
+#   git.k8s.syd1.au.unkin.net  — admin/backup route (external-dns k8s.syd1 zone),
+#                                same dual-name pattern as identity.unkin.net.
+# The cert-manager Certificate (vault-issuer) takes CN git.unkin.net and gets a
+# DNS SAN for each TLS listener hostname automatically.
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
@@ -12,13 +17,36 @@ metadata:
     traefik.io/instance: internal
   annotations:
     cert-manager.io/cluster-issuer: vault-issuer
-    cert-manager.io/common-name: git.k8s.syd1.au.unkin.net
+    cert-manager.io/common-name: git.unkin.net
     cert-manager.io/private-key-size: "4096"
+    # Only the k8s admin route is published by external-dns (it owns just the
+    # k8s.syd1.au.unkin.net zone). git.unkin.net lives in the apex zone and is
+    # flipped at cutover — NOT managed here.
     external-dns.alpha.kubernetes.io/hostname: git.k8s.syd1.au.unkin.net
     external-dns.alpha.kubernetes.io/target: 198.18.200.4
 spec:
   gatewayClassName: traefik-internal
   listeners:
+    - name: http-primary
+      port: 80
+      protocol: HTTP
+      hostname: git.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https-primary
+      port: 443
+      protocol: HTTPS
+      hostname: git.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: gitea-tls
     - name: http-admin
       port: 80
       protocol: HTTP

apps/base/gitea/httproute.yaml

@@ -9,8 +9,13 @@ metadata:
     app.kubernetes.io/instance: gitea
 spec:
   hostnames:
+    - git.unkin.net
     - git.k8s.syd1.au.unkin.net
   parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gitea
+      sectionName: http-primary
     - group: gateway.networking.k8s.io
       kind: Gateway
       name: gitea
@@ -36,8 +41,13 @@ metadata:
     app.kubernetes.io/instance: gitea
 spec:
   hostnames:
+    - git.unkin.net
     - git.k8s.syd1.au.unkin.net
   parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gitea
+      sectionName: https-primary
     - group: gateway.networking.k8s.io
       kind: Gateway
       name: gitea

apps/base/grafana/dashboards/k8up.yaml

@@ -1,15 +0,0 @@
----
-# k8up dashboard (M0NsTeRRR/grafana-dashboards), datasource resolved to the
-# in-cluster VictoriaMetrics. Backed by the k8up VMServiceScrape.
-apiVersion: grafana.integreatly.org/v1beta1
-kind: GrafanaDashboard
-metadata:
-  name: k8up
-  namespace: grafana
-spec:
-  instanceSelector:
-    matchLabels:
-      dashboards: "grafana"
-  resyncPeriod: 5m
-  allowCrossNamespaceImport: false
-  gzipJson: H4sIAAAAAAACA+2ce3PbNhLAvwqHc3eTzlixqIdfSTvjptc2M7Hjq9Pc3NUeDUhCEiIIYAHQsqLRffZbAHzAop0qaSRHKf4SCZLAcrH7wwJYcREixrhCinAmw5NgEVIiFRz8tgjjnFD1ksFJtBeEKVJI8lwk2Nym5pk+CEcCDRFDIdyRk1SXtFrBT7YwaLXCJVzADMVU361EjuF8TFLnjCScveCUC/2wGMXoSXsv6ETQZqff3wuib3TdDE1Nc6e1tME/glOKhZL6eilOiuQ45kik4fJaN51imQiS6Qf05clRngXVPcFYqUye7O/r4qeE64pwStRdaYdEJoj+ByNxqZBQZ5ypMVwFIcMRw+qlfulOOzo42DPayMZvOKeKZMU9RikspxSOKWETreXfrs3JDT7nMzgdIip1SxlimEqr/IRTijKJU+f6SJD0gttuGhfdop/v9ODgtmhvrn+XZcPdO9XqZhVR5u3C1xkWSIHaHfUJkAeeXTzU25ngU6zGOJdOh6dYdeb9foqS3rtJYnr8QUlXBI0qQSOtPp7Vdpjw1LZNERvlaGTbp4gwhW+Vbl6O+ewVYfg8n8ZYSEdR+soZYeQMZWWpbggMTWGmrTvUz09tC+EUiUnKZ8xIntF8RNhbqK8wmaj9tPO0G7qaOwdjlBkC3TiqM1JtQ3edhu46R7uju8tkjNOcYhm847F8FAX2GgrstndHgW+AfxSqJRSn29dht9DhQUOHx7ujwu9RMoFxQMJAkj+O+o6b/NshE3wxxsnkMdUXNRmo44Vd0d+pSMYw+D+qBvtNDXZ2R4O/YAmRy+Nq8Kipwe7uaPBC5OxR9deJmvrr7V4cs00V9h6Ioju1Bg+3qsHnZqoEKphT/O1VOCOpGp9E7fbfnwVjTEZjZU9iLlIsTtrZbSA5JWkQUxiBn12F312JKxYEz1Va1aHFaiFKRuwkSKAZLJ7dwCSPwBSsLJ6SNKX4A7U+z6rqhiBsa2ZlCWJOU339b6wM4J/vZ6UM+yo1h3CgX+o71zrGakr/yDIEzjAyWsG3GRcKpwPmzhPs9R+IwEk5G73ZiNUMCabpC86GZGSeT/EQ5VSV9mAn2ovq3dRYYOhpmkrz9BRlGWGjerpYX3afQjHoPFfmzaTCWT1rtfP4mObm2g3SB3YCbGfkOSNGS4wzbFrk0L9g6Ng2uY7ddxp232lGnyDIWSGsFaKcnpelRgAofJdLRYbzshjlyiwEcEHA/FDZV2WxAK9P8Gu3KUQTIzv4mFTnXJ3Du4bXZU/IEjVGCul6ku7yhozgRPgVmvNcFUsQa9ndPXaFxAirol/+pEnpRZFanXpdCDzPrJbc4inQRDjUAOs3FiDz6RO9rjKQ5YxvMEL5CC+a7vHt/67C2ievwqVZ8iEM0Go4U1ZN8Qiz9EcuptbRBoOqV4BuzpKNwEOzLBOehstrx8c0q79AH2P8beEmYTv8dJ8bCYxZw+n2HIm09bo3RJ/NJXsNl+xG3iW35pJmmP+wPxpf1OsFA2VWEFrBfd7513K8Bxwtw0KHHmj0oKulEC62Vpyp6W0Nh4Ro6K7HFS2Z049zvNWFGGcsXPG7KVgpef+ziYGg4FDP/GzZv3W8VhZtwbl0QPmmVPkrFNtVaTfcrK6egYJtNFpYliTvoecqydYOxtaLwbbgmNWoGMTz4ElzEPwmeFK7qMyTBEs5zOkg4bmOgtcYNvf0cuAbEBfi3tgscemhNNgP1mzUcOGT29kQOSyVPDo2gI7jBjoOPTo8OhK9urt5clTNfOXgME+fEXaGbp3X+wpxEjWn5UeeJ54nyO52bJ4oTkM+GNk5evQb9Oh5enh6CLvTt3l6OA15euwcPY4a9Djw9PD0yPQu9+bZUTXjybFr5KjSEGpy9D05PDnKfZHNw8NtafP8+FN55Qd1okn/4cTyi1yOg5+QwjM030Ry+UcyLEMUK4VbCXibJLaKJIdgb2puQ7dEfm9SSS7Hd7Lx9YUXJgMFp//Fgq9ecrczy2QafcH4ZenDuuCCQp9Pi5SZsudiJE51OktRrjWcCjS71Hkr+j5K7FbokFD6GoyGqHn5pwOBUk2a1U1T/a+KH0XxVtZkHJFV9aeEsuSGvHfZAmaHhUGRyxQtxkutg4zTCm26EJn9f31UQlBbTcYJU5fkPS4wqv85gX8gUgkS58XjdYcX9ZRku9APS1dJ4C1sVSJwjmRiSbYAXfA8swbvpOtU+9j1+FTqtTYLPhyGy8+Y8XL/7vv1cr1h6ahwuWg1t+uw3tdbzRCu+tgZUvQLpERmFFW75+YPPYb2jhnGXGkHKPO/yprKze3aWmoFSFASta8PdKv1vNwq/w1r7a7TQE/ZSDJAN4hQnbM1kAzQNuZKLixE70ktWY+wi6qC5XIdzlaZvaUoQSVKADFKUNZ2J+GLgFQYggbpqeip+Fen4gKiGjmHKqa/4KHJuYQrl8Y9TPcZkRSEStaMrRvEc/NXrHA1YC+Ew7cJzS0+DAJMzD3B84RyNAlawX553Mq4VCCqbKX5NDOheAY+Tm6NEikNoCacqRMbAqP0NaPzkpYmkBY800mq5atY6aw7Pa0s0NHCR5iibeR6ud6wUW3COONG5MeNe8eNZKypng5SM53R0zc5SHMB4hY3fs5RBMxtschuko8bT15YEQNHxMCKGFgR/ejiRxc/unzmmDvq1/Dsenh+EJ7gKx+HzWGFR42oQcGoLfPUSO1J6knqSboRktZ/0HVQ2vMo/bwo9cT0xPTE/Lpiz063BmbfA/NeYDI8+5In7ed45ifsHpoemtsLM11qHnhqemp6anpqemp+ONTsOl8QPPTQvBeaOQMRCWDhi94m+rWS0iPUI9QjdHuBp8vQjmfoHzH0i1zidOj5qauc17Z+bZolW3Ra7xTVydXdY9MPjiWbT7Wo0lnq724nuRDWDBbLhz+8vW53AX8JIxUcNP4GNrN8NW95757k5rD+XLf5mDUzGTWnlNZ6pSVR73wWeQrMJ/VN5fe772ST185iFPJ7jsXcvOPvYv7GvmbklH+S8FWvX1T6eosE0Tl6/9L1/tNYc+tOmdFb3Z2RORvh26prJyT7VdDLOUtc9FmvazsmYiW3qTvaXMzLDe0oAN45ax2af0AoXpyH5Y0ZSSY206ksea99uWi/slytA6f34yg6Oj6O41a/E0et3rCNWqjX7bXifq/TTnv9XoRiw8DKKPVceYbxxHzZ3FS//D9scYEQEl4AAA==

apps/base/grafana/gateway.yaml

@@ -37,36 +37,3 @@ spec:
           - group: ""
             kind: Secret
             name: grafana-tls
----
-# Public grafana.unkin.net via the external Traefik; TLS uses the reflected
-# Let's Encrypt *.unkin.net wildcard, DNS lives in the bind-operator zone.
-apiVersion: gateway.networking.k8s.io/v1
-kind: Gateway
-metadata:
-  labels:
-    traefik.io/instance: external
-  name: grafana-external
-  namespace: grafana
-spec:
-  gatewayClassName: traefik-external
-  listeners:
-    - allowedRoutes:
-        namespaces:
-          from: Same
-      hostname: grafana.unkin.net
-      name: http
-      port: 80
-      protocol: HTTP
-    - allowedRoutes:
-        namespaces:
-          from: Same
-      hostname: grafana.unkin.net
-      name: https
-      port: 443
-      protocol: HTTPS
-      tls:
-        certificateRefs:
-          - group: ""
-            kind: Secret
-            name: wildcard-unkin-net-tls
-        mode: Terminate

apps/base/grafana/grafana.yaml

@@ -9,16 +9,8 @@ metadata:
 spec:
   deployment:
     spec:
-      replicas: 3
       template:
         spec:
-          topologySpreadConstraints:
-            - maxSkew: 1
-              topologyKey: kubernetes.io/hostname
-              whenUnsatisfiable: ScheduleAnyway
-              labelSelector:
-                matchLabels:
-                  app: grafana
           containers:
             - name: grafana
               env:
@@ -39,11 +31,11 @@ spec:
                   cpu: 100m
                   memory: 256Mi
                 limits:
-                  cpu: "2"
-                  memory: 4Gi
+                  cpu: "1"
+                  memory: 1Gi
   config:
     server:
-      root_url: "https://grafana.unkin.net"
+      root_url: "https://grafana.k8s.syd1.au.unkin.net"
     database:
       type: "postgres"
       host: "postgres-pooler-rw.grafana.svc.cluster.local:5432"
@@ -69,8 +61,3 @@ spec:
       # to akR-global-admin members (and direct members) via terraform-authentik.
       role_attribute_path: "contains(ak_groups[*], 'akP-grafana-admin') && 'Admin' || 'Viewer'"
       role_attribute_strict: "false"
-    # Gossip between replicas so each alert notification is sent once.
-    unified_alerting:
-      ha_peers: "grafana-alerting.grafana.svc.cluster.local:9094"
-      ha_listen_address: "${POD_IP}:9094"
-      ha_advertise_address: "${POD_IP}:9094"

apps/base/grafana/httproute.yaml

@@ -53,58 +53,3 @@ spec:
         - path:
             type: PathPrefix
             value: /
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: grafana-external-http-redirect
-  namespace: grafana
-  labels:
-    app.kubernetes.io/name: grafana
-    app.kubernetes.io/instance: grafana
-spec:
-  hostnames:
-    - grafana.unkin.net
-  parentRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: grafana-external
-      sectionName: http
-  rules:
-    - filters:
-        - type: RequestRedirect
-          requestRedirect:
-            scheme: https
-            statusCode: 301
-      matches:
-        - path:
-            type: PathPrefix
-            value: /
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: grafana-external
-  namespace: grafana
-  labels:
-    app.kubernetes.io/name: grafana
-    app.kubernetes.io/instance: grafana
-spec:
-  hostnames:
-    - grafana.unkin.net
-  parentRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: grafana-external
-      sectionName: https
-  rules:
-    - backendRefs:
-        - group: ""
-          kind: Service
-          name: grafana-service
-          port: 3000
-          weight: 1
-      matches:
-        - path:
-            type: PathPrefix
-            value: /

apps/base/grafana/kustomization.yaml

@@ -18,7 +18,6 @@ resources:
   - dashboards/frr-ospf-route-metrics.yaml
   - dashboards/gitea.yaml
   - dashboards/haproxy.yaml
-  - dashboards/k8up.yaml
   - dashboards/media-dashboard.yaml
   - dashboards/node-exporter-full.yaml
   - dashboards/nzbget.yaml
@@ -30,4 +29,3 @@ resources:
   - dashboards/victoriametrics-cluster.yaml
   - dashboards/victoriametrics-vmagent.yaml
   - dashboards/cloudnativepg.yaml
-  - dashboards/arc-runner-scale-set.yaml

apps/base/haproxy/backendtlspolicy.yaml

@@ -1,16 +0,0 @@
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: BackendTLSPolicy
-metadata:
-  name: haproxy
-  namespace: haproxy
-  labels:
-    app: haproxy
-spec:
-  targetRefs:
-    - group: ""
-      kind: Service
-      name: haproxy
-  validation:
-    wellKnownCACertificates: System
-    hostname: git.unkin.net

apps/base/haproxy/configmap.yaml

@@ -21,6 +21,7 @@ data:
     jellyfin.main.unkin.net be_jellyfin
     fafflix.unkin.net be_jellyfin
     git.unkin.net be_gitea
+    grafana.unkin.net be_grafana
     dashboard.ceph.unkin.net be_ceph_dashboard
     auth.unkin.net be_k8s_kanidm
 
@@ -123,6 +124,23 @@ data:
       server ausyd1nxvm2081 198.18.27.117:443 check cookie ausyd1nxvm2081 fall 2 inter 2s rise 3 ssl verify none
       server ausyd1nxvm2082 198.18.28.71:443 check cookie ausyd1nxvm2082 fall 2 inter 2s rise 3 ssl verify none
 
+    backend be_grafana
+      description Backend for grafana nodes
+      balance roundrobin
+      cookie SRVNAME insert indirect nocache
+      http-request set-header X-Forwarded-Port %[dst_port]
+      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
+      http-reuse always
+      option httpchk GET /
+      option forwardfor
+      option http-keep-alive
+      option prefer-last-server
+      redirect scheme https if !{ ssl_fc }
+      stick on src
+      stick-table type ip size 200k expire 30m
+      server ausyd1nxvm2015 198.18.27.2:443 check cookie ausyd1nxvm2015 fall 2 inter 2s rise 3 ssl verify none
+      server ausyd1nxvm2016 198.18.28.189:443 check cookie ausyd1nxvm2016 fall 2 inter 2s rise 3 ssl verify none
+
     backend be_jellyfin
       description Backend for au-syd1 jellyfin
       balance roundrobin

apps/base/haproxy/gateway.yaml

@@ -1,8 +1,10 @@
 ---
 # External (DMZ) front for the haproxy edge on the traefik-external LB VIP
-# 198.18.199.0. Traefik terminates TLS so HTTP/2 connections coalesced from
-# other *.unkin.net hosts still route by Host; the hop to haproxy is re-encrypted via
-# BackendTLSPolicy. Routes carry the explicit hostname list.
+# 198.18.199.0. The :443 listener is TLS Passthrough: haproxy owns the three
+# wildcard certs and terminates behind Traefik, so there are no certificateRefs
+# here. Listener hostnames are deliberately unset and the routes carry the
+# explicit hostname list instead; allowedRoutes Same keeps other namespaces off
+# these listeners.
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
@@ -19,21 +21,11 @@ spec:
       allowedRoutes:
         namespaces:
           from: Same
-    - name: https
+    - name: https-passthrough
       port: 443
-      protocol: HTTPS
+      protocol: TLS
       tls:
-        mode: Terminate
-        certificateRefs:
-          - group: ""
-            kind: Secret
-            name: wildcard-unkin-net-tls
-          - group: ""
-            kind: Secret
-            name: wildcard-main-unkin-net-tls
-          - group: ""
-            kind: Secret
-            name: wildcard-ceph-unkin-net-tls
+        mode: Passthrough
       allowedRoutes:
         namespaces:
           from: Same

apps/base/haproxy/httproute.yaml

@@ -17,6 +17,7 @@ spec:
     - jellyfin.main.unkin.net
     - fafflix.unkin.net
     - git.unkin.net
+    - grafana.unkin.net
     - dashboard.ceph.unkin.net
     - auth.unkin.net
   parentRefs:
@@ -34,36 +35,3 @@ spec:
         - path:
             type: PathPrefix
             value: /
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: haproxy
-  namespace: haproxy
-  labels:
-    app: haproxy
-spec:
-  hostnames:
-    - sonarr.main.unkin.net
-    - radarr.main.unkin.net
-    - lidarr.main.unkin.net
-    - readarr.main.unkin.net
-    - prowlarr.main.unkin.net
-    - nzbget.main.unkin.net
-    - jellyfin.main.unkin.net
-    - fafflix.unkin.net
-    - git.unkin.net
-    - dashboard.ceph.unkin.net
-    - auth.unkin.net
-  parentRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: haproxy
-      sectionName: https
-  rules:
-    - backendRefs:
-        - group: ""
-          kind: Service
-          name: haproxy
-          port: 443
-          weight: 1

apps/base/haproxy/kustomization.yaml

@@ -8,7 +8,7 @@ resources:
   - deployment.yaml
   - service.yaml
   - gateway.yaml
-  - backendtlspolicy.yaml
+  - tlsroute.yaml
   - httproute.yaml
   - pdb.yaml
   - vpa.yaml

apps/base/haproxy/service.yaml

@@ -6,7 +6,7 @@ metadata:
   namespace: haproxy
 spec:
   type: ClusterIP
-  # Reached only by the external Traefik (re-encrypted HTTPRoute), so the
+  # Reached only by the external Traefik's TLS-passthrough TLSRoute, so the
   # peer address here is a Traefik pod, not the client. sessionAffinity is
   # deliberately absent: keyed on ClientIP it would pin whole Traefik pods,
   # not clients. Backend persistence rests on the per-backend SRVNAME cookie.

apps/base/haproxy/tlsroute.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: TLSRoute
+metadata:
+  name: haproxy
+  namespace: haproxy
+  labels:
+    app: haproxy
+spec:
+  hostnames:
+    - sonarr.main.unkin.net
+    - radarr.main.unkin.net
+    - lidarr.main.unkin.net
+    - readarr.main.unkin.net
+    - prowlarr.main.unkin.net
+    - nzbget.main.unkin.net
+    - jellyfin.main.unkin.net
+    - fafflix.unkin.net
+    - git.unkin.net
+    - grafana.unkin.net
+    - dashboard.ceph.unkin.net
+    - auth.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: haproxy
+      sectionName: https-passthrough
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: haproxy
+          port: 443
+          weight: 1

apps/base/iuclabs-runners/components/iuclabs-k8s/kustomization.yaml

@@ -1,7 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1alpha1
-kind: Component
-
-resources:
-  - scaleset.yaml
-  - networkpolicy.yaml

apps/base/iuclabs-runners/components/iuclabs-k8s/networkpolicy.yaml

@@ -1,45 +0,0 @@
----
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: egress-iuclabs-k8s
-  namespace: iuclabs-runners
-spec:
-  podSelector:
-    matchLabels:
-      runners.unkin.net/scale-set: iuclabs-k8s
-  policyTypes:
-    - Egress
-  egress:
-    - to:
-        - ipBlock:
-            cidr: 0.0.0.0/0
-            except:
-              - 10.0.0.0/8
-              - 172.16.0.0/12
-              - 192.168.0.0/16
-              - 198.18.0.0/15
-              - 0.0.0.0/8
-              - 100.64.0.0/10
-              - 127.0.0.0/8
-              - 169.254.0.0/16
-              - 224.0.0.0/4
-              - 240.0.0.0/4
-      ports:
-        - protocol: TCP
-          port: 443
-    # vault.service.consul servers, from `dig vault.service.consul`
-    - to:
-        - ipBlock:
-            cidr: 198.18.25.78/32
-        - ipBlock:
-            cidr: 198.18.26.197/32
-        - ipBlock:
-            cidr: 198.18.27.116/32
-        - ipBlock:
-            cidr: 198.18.28.248/32
-        - ipBlock:
-            cidr: 198.18.29.92/32
-      ports:
-        - protocol: TCP
-          port: 8200

apps/base/iuclabs-runners/components/iuclabs-k8s/scaleset.yaml

@@ -1,66 +0,0 @@
----
-apiVersion: actions.github.com/v1alpha1
-kind: AutoscalingRunnerSet
-metadata:
-  name: iuclabs-k8s
-  namespace: iuclabs-runners
-  labels:
-    app.kubernetes.io/version: "0.15.1"
-  annotations:
-    actions.github.com/cleanup-manager-role-name: arc-manager
-    actions.github.com/cleanup-manager-role-binding: arc-manager
-spec:
-  githubConfigUrl: https://github.com/IUCLabs
-  githubConfigSecret: github-app
-  runnerScaleSetName: iuclabs-k8s
-  minRunners: 0
-  maxRunners: 4
-  template:
-    metadata:
-      labels:
-        runners.unkin.net/scale-set: iuclabs-k8s
-    spec:
-      serviceAccountName: runner
-      automountServiceAccountToken: false
-      restartPolicy: Never
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 1001
-        runAsGroup: 1001
-        seccompProfile:
-          type: RuntimeDefault
-      topologySpreadConstraints:
-        - maxSkew: 1
-          topologyKey: kubernetes.io/hostname
-          whenUnsatisfiable: ScheduleAnyway
-          labelSelector:
-            matchLabels:
-              runners.unkin.net/scale-set: iuclabs-k8s
-      containers:
-        - name: runner
-          image: ghcr.io/actions/actions-runner:2.338.2
-          command: ["/home/runner/run.sh"]
-          env:
-            - name: ACTIONS_RUNNER_HOOK_JOB_STARTED
-              value: /hooks/job-started.sh
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop:
-                - ALL
-          resources:
-            requests:
-              cpu: 1
-              memory: 512Mi
-            limits:
-              cpu: 2
-              memory: 2Gi
-          volumeMounts:
-            - name: hooks
-              mountPath: /hooks
-              readOnly: true
-      volumes:
-        - name: hooks
-          configMap:
-            name: runner-hooks
-            defaultMode: 0555

apps/base/iuclabs-runners/job-started.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-set -eu
-
-event="${GITHUB_EVENT_NAME:-}"
-
-refuse() {
-  echo "job-started hook: refusing '${event}' job: $1" >&2
-  exit 1
-}
-
-[ -r "${GITHUB_EVENT_PATH:-}" ] || refuse "event payload missing"
-jq -e 'type == "object"' "${GITHUB_EVENT_PATH}" >/dev/null 2>&1 || refuse "event payload unparseable"
-
-case "${event}" in
-  push | workflow_dispatch | schedule | merge_group | release) ;;
-  pull_request)
-    trusted=$(jq -r '
-      .pull_request.head.repo.full_name as $head
-      | if $head == null then false
-        elif $head == .repository.full_name then true
-        else (.pull_request.author_association | IN("OWNER", "MEMBER", "COLLABORATOR"))
-        end' "${GITHUB_EVENT_PATH}") || refuse "event payload unreadable"
-    [ "${trusted}" = true ] || refuse "pull request from untrusted fork"
-    ;;
-  *) refuse "event not allowed on self-hosted runners" ;;
-esac

apps/base/iuclabs-runners/kustomization.yaml

@@ -1,22 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - namespace.yaml
-  - vaultauth.yaml
-  - vaultstaticsecret.yaml
-  - networkpolicy.yaml
-  - rbac.yaml
-
-components:
-  - components/iuclabs-k8s
-
-generatorOptions:
-  disableNameSuffixHash: true
-
-configMapGenerator:
-  - name: runner-hooks
-    namespace: iuclabs-runners
-    files:
-      - job-started.sh

apps/base/iuclabs-runners/namespace.yaml

@@ -1,5 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: iuclabs-runners

apps/base/iuclabs-runners/networkpolicy.yaml

@@ -1,34 +0,0 @@
----
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: default-deny
-  namespace: iuclabs-runners
-spec:
-  podSelector: {}
-  policyTypes:
-    - Ingress
-    - Egress
----
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: allow-dns
-  namespace: iuclabs-runners
-spec:
-  podSelector: {}
-  policyTypes:
-    - Egress
-  egress:
-    - to:
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: kube-system
-          podSelector:
-            matchLabels:
-              k8s-app: kube-dns
-      ports:
-        - protocol: UDP
-          port: 53
-        - protocol: TCP
-          port: 53

apps/base/iuclabs-runners/rbac.yaml

@@ -1,44 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: runner
-  namespace: iuclabs-runners
-automountServiceAccountToken: false
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: arc-manager
-  namespace: iuclabs-runners
-  finalizers:
-    - actions.github.com/cleanup-protection
-rules:
-  - apiGroups: [""]
-    resources: ["pods"]
-    verbs: ["create", "delete", "get"]
-  - apiGroups: [""]
-    resources: ["pods/status"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["secrets", "serviceaccounts"]
-    verbs: ["create", "delete", "get", "list", "patch", "update"]
-  - apiGroups: ["rbac.authorization.k8s.io"]
-    resources: ["roles", "rolebindings"]
-    verbs: ["create", "delete", "get", "patch", "update"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: arc-manager
-  namespace: iuclabs-runners
-  finalizers:
-    - actions.github.com/cleanup-protection
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: arc-manager
-subjects:
-  - kind: ServiceAccount
-    name: arc-gha-rs-controller
-    namespace: arc-system

apps/base/iuclabs-runners/vaultauth.yaml

@@ -1,18 +0,0 @@
----
-apiVersion: secrets.hashicorp.com/v1beta1
-kind: VaultAuth
-metadata:
-  name: default
-  namespace: iuclabs-runners
-spec:
-  allowedNamespaces:
-    - iuclabs-runners
-  kubernetes:
-    audiences:
-      - vault
-    role: default
-    serviceAccount: default
-    tokenExpirationSeconds: 600
-  method: kubernetes
-  mount: k8s/au/syd1
-  vaultConnectionRef: vso-system/default

apps/base/iuclabs-runners/vaultstaticsecret.yaml

@@ -1,17 +0,0 @@
----
-apiVersion: secrets.hashicorp.com/v1beta1
-kind: VaultStaticSecret
-metadata:
-  name: github-app
-  namespace: iuclabs-runners
-spec:
-  destination:
-    create: true
-    name: github-app
-    overwrite: true
-  hmacSecretData: true
-  mount: kv
-  path: kubernetes/namespace/iuclabs-runners/default/github-app
-  refreshAfter: 5m
-  type: kv-v2
-  vaultAuthRef: default

apps/base/k8up-system/kustomization.yaml

@@ -4,4 +4,3 @@ kind: Kustomization
 
 resources:
   - namespace.yaml
-  - vmservicescrape.yaml

apps/base/k8up-system/vmservicescrape.yaml

@@ -1,15 +0,0 @@
----
-# Scrape the k8up operator metrics (k8up-metrics Service, port http :8080).
-apiVersion: operator.victoriametrics.com/v1beta1
-kind: VMServiceScrape
-metadata:
-  name: k8up
-  namespace: k8up-system
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: k8up
-      app.kubernetes.io/instance: k8up
-  endpoints:
-    - port: http
-      path: /metrics

apps/base/logging/agent-observability-rolebinding.yaml

@@ -0,0 +1,16 @@
+---
+# Confines the agent-observability service account (in vm-system) to the
+# agent-observability ClusterRole within this namespace.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: agent-observability
+  namespace: logging
+subjects:
+  - kind: ServiceAccount
+    name: agent-observability
+    namespace: vm-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: agent-observability

apps/base/logging/cephrgw.yaml

@@ -0,0 +1,43 @@
+---
+# S3 bucket (Ceph RGW) for the long-term raw-log archive, provisioned by the
+# in-estate cephrgw-operator. The archiver Vector deployment writes here.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: logs-archive-owner
+  namespace: logging
+spec:
+  displayName: "Logging raw-archive bucket owner"
+  maxBuckets: 5
+  quota:
+    enabled: true
+    # 5 TiB soft cap; real retention is enforced RGW-side by a bucket lifecycle
+    # policy (see PR notes) — the operator does not manage lifecycle.
+    maxSizeBytes: 5497558138880
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: logs-archive
+  namespace: logging
+spec:
+  bucketName: logs-archive
+  ownerRef: logs-archive-owner
+  versioning: false
+  tags:
+    app: logging
+    purpose: raw-log-archive
+  # Keep the bucket (and its objects) if this CR is ever deleted.
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: BucketAccess
+metadata:
+  name: logs-archive-writer
+  namespace: logging
+spec:
+  bucketRef: logs-archive
+  level: read-write
+  # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
+  # S3_ENDPOINT, BUCKET_NAME) into this Secret; the archiver consumes it.
+  secretName: logs-archive-s3

apps/base/logging/clickhouseinstallation.yaml

@@ -0,0 +1,97 @@
+---
+apiVersion: clickhouse.altinity.com/v1
+kind: ClickHouseInstallation
+metadata:
+  name: logs
+  namespace: logging
+spec:
+  defaults:
+    templates:
+      dataVolumeClaimTemplate: data-volume
+      serviceTemplate: chi-service
+      podTemplate: clickhouse
+  configuration:
+    users:
+      # Password hash is sourced from the Vault-synced clickhouse-credentials
+      # Secret; the plaintext never lands in git or the ClickHouse config.
+      vector/password_sha256_hex:
+        valueFrom:
+          secretKeyRef:
+            name: clickhouse-credentials
+            key: password_sha256_hex
+      vector/networks/ip:
+        - "::/0"
+      vector/profile: default
+      vector/quota: default
+      # Allow the vector user to create the logs database/table (bootstrap Job)
+      # and to INSERT. Restrict the built-in default user to loopback only.
+      vector/access_management: "1"
+      default/networks/ip:
+        - "127.0.0.1"
+        - "::1"
+      # Read-only user for CLI tools + the logviewer UI. Hash sourced from the
+      # Vault-synced clickhouse-logreader Secret, same mechanism as vector.
+      # Scoped to the logs database only (unlike vector, which bootstraps it).
+      logreader/password_sha256_hex:
+        valueFrom:
+          secretKeyRef:
+            name: clickhouse-logreader
+            key: password_sha256_hex
+      logreader/networks/ip:
+        - "::/0"
+      logreader/profile: readonly
+      logreader/quota: default
+      logreader/allow_databases/database:
+        - "logs"
+    profiles:
+      default/max_memory_usage: "10000000000"
+      default/max_execution_time: "120"
+      readonly/readonly: "2"
+      readonly/max_memory_usage: "10000000000"
+      readonly/max_execution_time: "120"
+    clusters:
+      - name: logs
+        layout:
+          shardsCount: 1
+          replicasCount: 1
+  templates:
+    volumeClaimTemplates:
+      - name: data-volume
+        spec:
+          storageClassName: cephrbd-fast-delete
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              # 3d TTL on logs.raw. At ~130 GiB/day raw, ClickHouse LZ4/ZSTD
+              # (~6x on log text) stores ~20-25 GiB/day => ~60-75 GiB/3d, plus
+              # merge headroom (~2x peak). logs.raw is the only table. 150Gi
+              # gives comfortable headroom; long-term data lives in S3, not here.
+              storage: 150Gi
+    serviceTemplates:
+      - name: chi-service
+        generateName: "clickhouse-{chi}"
+        spec:
+          type: ClusterIP
+          ports:
+            - name: http
+              port: 8123
+            - name: tcp
+              port: 9000
+    podTemplates:
+      - name: clickhouse
+        spec:
+          securityContext:
+            fsGroup: 101
+            runAsUser: 101
+            runAsGroup: 101
+          containers:
+            - name: clickhouse
+              image: docker.io/clickhouse/clickhouse-server:24.8
+              resources:
+                requests:
+                  cpu: 500m
+                  memory: 2Gi
+                limits:
+                  cpu: "2"
+                  memory: 8Gi

apps/base/logging/configmap_logarchiver.yaml

@@ -0,0 +1,40 @@
+---
+# logarchiver non-secret config. Secrets (NATS/S3/ClickHouse creds) and the
+# subject filter come from env; everything else uses the binary's built-in
+# defaults, which already target this stack. ack_wait MUST exceed batch.max_age
+# so unacked messages in an open batch are not redelivered mid-batch.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logarchiver-config
+  namespace: logging
+data:
+  config.yaml: |
+    nats:
+      ack_wait: 5m
+      fetch_batch: 512
+    batch:
+      max_bytes: 67108864
+      max_events: 200000
+      max_age: 2m
+    # Pin the proven RGW endpoint/bucket; ignore the secret's S3_ENDPOINT/BUCKET_NAME
+    # (AWS creds still come from the secret env). endpoint_env/bucket_env off.
+    s3:
+      endpoint: "https://s3.ceph.unkin.net"
+      bucket: "logs-archive"
+      region: "us-east-1"
+      path_style: true
+      ca_file: /etc/vault-ca/ca.crt
+      endpoint_env: ""
+      bucket_env: ""
+    crypto:
+      key_name: logarchive
+      pubkey_source: vault
+      vault:
+        address: "https://vault.service.consul:8200"
+        mount: gpg
+        auth_method: kubernetes
+        k8s_mount: k8s/au/syd1
+        k8s_role: logging_logarchiver
+        k8s_jwt_path: /var/run/secrets/vault/token
+        ca_file: /etc/vault-ca/ca.crt

apps/base/logging/deployment_logarchiver.yaml

@@ -0,0 +1,134 @@
+---
+# logarchiver — replaces the vector-archiver leg. Independent JetStream durable
+# consumer (archiver) that seals raw logs to S3 as zstd + OpenPGP objects and
+# indexes each object in ClickHouse (logs.archive_index). Acks only after the
+# object is in S3 AND indexed. Reuses the same NATS/S3/CA wiring the Vector
+# archiver used; the OpenPGP public key is delivered as a mounted file.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: logarchiver
+  namespace: logging
+  annotations:
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "vault-ca-cert"
+  labels:
+    app.kubernetes.io/name: logarchiver
+    app.kubernetes.io/component: archiver
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: logarchiver
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: logarchiver
+        vector.dev/exclude: "true"
+    spec:
+      # Dedicated SA whose projected vault-audience token authenticates the
+      # k8s-auth login used to fetch the logarchive public key from the gpg engine.
+      serviceAccountName: logarchiver
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: logarchiver
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/logarchiver:v0.1.0
+          imagePullPolicy: IfNotPresent
+          args: ["run"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          ports:
+            - containerPort: 9090
+              name: metrics
+              protocol: TCP
+          env:
+            - name: LOGARCHIVER_CONFIG
+              value: /etc/logarchiver/config.yaml
+            # Server-side subject filter; must match the archiver consumer's filter.
+            - name: ARCHIVE_SUBJECTS
+              value: "logs.k8s.vault.>"
+            - name: NATS_CONSUMER_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: nats-auth
+                  key: consumer_password
+            - name: CLICKHOUSE_USER
+              valueFrom:
+                secretKeyRef:
+                  name: clickhouse-credentials
+                  key: username
+            - name: CLICKHOUSE_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: clickhouse-credentials
+                  key: password
+          # S3 creds + S3_ENDPOINT + BUCKET_NAME from the cephrgw BucketAccess Secret.
+          envFrom:
+            - secretRef:
+                name: logs-archive-s3
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 15
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: "1"
+              memory: 1Gi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/logarchiver/config.yaml
+              subPath: config.yaml
+              readOnly: true
+            - name: vault-token
+              mountPath: /var/run/secrets/vault
+              readOnly: true
+            - name: vault-ca-cert
+              mountPath: /etc/vault-ca/ca.crt
+              subPath: ca.crt
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: logarchiver-config
+        # Projected SA token with audience "vault" for the gpg-engine k8s login.
+        - name: vault-token
+          projected:
+            sources:
+              - serviceAccountToken:
+                  path: token
+                  audience: vault
+                  expirationSeconds: 600
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+        - name: tmp
+          emptyDir: {}

apps/base/logging/gateway.yaml

@@ -39,43 +39,3 @@ spec:
           - group: ""
             kind: Secret
             name: logs-ingest-tls
----
-# Unauthenticated LogsQL query endpoint fronting the VLCluster vlselect service.
-apiVersion: gateway.networking.k8s.io/v1
-kind: Gateway
-metadata:
-  name: vlselect
-  namespace: logging
-  labels:
-    app.kubernetes.io/name: victorialogs
-    app.kubernetes.io/component: query
-    traefik.io/instance: internal
-  annotations:
-    cert-manager.io/cluster-issuer: vault-issuer
-    cert-manager.io/common-name: vlselect.k8s.syd1.au.unkin.net
-    cert-manager.io/private-key-size: "4096"
-    external-dns.alpha.kubernetes.io/hostname: vlselect.k8s.syd1.au.unkin.net
-    external-dns.alpha.kubernetes.io/target: 198.18.200.4
-spec:
-  gatewayClassName: traefik-internal
-  listeners:
-    - name: http
-      port: 80
-      protocol: HTTP
-      hostname: vlselect.k8s.syd1.au.unkin.net
-      allowedRoutes:
-        namespaces:
-          from: Same
-    - name: https
-      port: 443
-      protocol: HTTPS
-      hostname: vlselect.k8s.syd1.au.unkin.net
-      allowedRoutes:
-        namespaces:
-          from: Same
-      tls:
-        mode: Terminate
-        certificateRefs:
-          - group: ""
-            kind: Secret
-            name: vlselect-tls