Raise puppet-postgres memory

bump artifactapi to v3.20.0 (#565 )
v3.20.0 adds virtual puppet repositories and filtered remote Forge release listings. - pin api, ui and image-keeper to v3.20.0 Reviewed-on: #565 Co-authored-by: unkin-agent <unkin-agent@unkin.net> Co-committed-by: unkin-agent <unkin-agent@unkin.net>
2026-10-22 21:34:23 +11:00 · 2026-10-20 22:25:18 +11:00 · 2026-10-19 22:07:09 +11:00 · 2026-10-18 19:03:40 +11:00 · 2026-10-18 02:49:49 +11:00 · 2026-10-17 22:00:21 +11:00
220 changed files with 4789 additions and 3407 deletions
@@ -1,29 +0,0 @@
 when:
  - event: pull_request
 steps:
  - name: vector-test
    image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector:0.57.0-debian
    commands:
      # Dummy creds + writable dirs so the full topologies build; the unit tests
      # only exercise the transforms (sources are not started).
      - export CLICKHOUSE_USER=ci CLICKHOUSE_PASSWORD=ci
      - export NATS_PRODUCER_PASSWORD=ci NATS_CONSUMER_PASSWORD=ci
      - mkdir -p /vector-data-dir /etc/vault-ca
      - cp /etc/ssl/certs/ca-certificates.crt /etc/vault-ca/ca.crt
      # Transform tier + VM ingest: unit-tested transforms.
      - vector test apps/base/logging/vector/aggregator.yaml apps/base/logging/vector/aggregator-tests.yaml
      - vector test apps/base/logging/vector/vm-ingest.yaml apps/base/logging/vector/vm-ingest-tests.yaml
      # Agent has no transforms to unit-test; validate it builds. (The archiver
      # leg is now the logarchiver service, not a Vector pipeline.)
      - vector validate --no-environment apps/base/logging/vector/agent.yaml
    backend_options:
      kubernetes:
        serviceAccountName: default
        resources:
          requests:
            memory: 256Mi
            cpu: 250m
          limits:
            memory: 1Gi
            cpu: 1
@@ -36,7 +36,7 @@ spec:
              mountPath: /combined-certs
      containers:
        - name: api
-          image: git.unkin.net/unkin/artifactapi:v3.11.2
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.20.0
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 8000
@@ -5,6 +5,8 @@ metadata:
  name: api-env
  namespace: artifactapi
 data:
  ARTIFACTAPI_OIDC_AUDIENCE: artifactapi
  ARTIFACTAPI_OIDC_ISSUER: https://vault.service.consul:8200/v1/identity/oidc
  DBHOST: postgres-pooler
  DBNAME: artifacts
  DBPORT: "5432"
@@ -1,4 +1,21 @@
 ---
 # Path split between the authenticated UI and the unauthenticated machine API.
 # Longest matching prefix wins, so the two UI rules take precedence over "/".
 #
 # AUTHENTICATED (oauth2 Service -> oauth2-proxy -> ui Service):
 #   /oauth2   oauth2-proxy sign_in / start / callback / sign_out
 #   /ui       the human-facing SPA
 #
 # NOT AUTHENTICATED (artifactapi Service, unchanged):
 #   /api/v1/{remote,local,virtual}/*   package proxy reads (yum/dnf, pip, ...)
 #   /api/v2/remotes|virtuals|locals/*  management API + the UI's own XHR calls
 #   /api/v2/remotes/{name}/files/*     CI publish uploads (PUT) and downloads
 #   /v2/*                              Docker Registry V2 (containerd, buildah)
 #   /terraform/v1/providers/*          Terraform provider registry
 #   /.well-known/terraform.json        Terraform service discovery
 #   /health, /version, /               probes and the redirect to /ui/
 # Those clients cannot complete a browser OIDC flow, so they must never be
 # routed through oauth2-proxy.
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:
@@ -22,7 +39,17 @@ spec:
    - backendRefs:
        - group: ""
          kind: Service
-          name: ui
+          name: oauth2
          port: 80
          weight: 1
      matches:
        - path:
            type: PathPrefix
            value: /oauth2
    - backendRefs:
        - group: ""
          kind: Service
          name: oauth2
          port: 80
          weight: 1
      matches:
@@ -0,0 +1,105 @@
 ---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
  name: image-keeper
  namespace: artifactapi
 spec:
  selector:
    matchLabels:
      app: image-keeper
  updateStrategy:
    rollingUpdate:
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: image-keeper
    spec:
      automountServiceAccountToken: false
      priorityClassName: low
      tolerations:
        - operator: Exists
      securityContext:
        runAsNonRoot: true
        runAsUser: 65532
        runAsGroup: 65532
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        # artifactapi is distroless with no exit-0 flag, so run a static busybox as `true`
        - name: copy-true
          image: busybox:1.37.0-musl
          imagePullPolicy: IfNotPresent
          command: ["cp", "/bin/busybox", "/keeper/true"]
          volumeMounts:
            - name: keeper
              mountPath: /keeper
          resources:
            limits:
              cpu: 10m
              memory: 16Mi
            requests:
              cpu: 1m
              memory: 4Mi
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop: ["ALL"]
        - name: api
          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi:v3.20.0
          imagePullPolicy: IfNotPresent
          command: ["/keeper/true"]
          volumeMounts:
            - name: keeper
              mountPath: /keeper
              readOnly: true
          resources:
            limits:
              cpu: 10m
              memory: 16Mi
            requests:
              cpu: 1m
              memory: 4Mi
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop: ["ALL"]
        - name: ui
          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.20.0
          imagePullPolicy: IfNotPresent
          command: ["true"]
          resources:
            limits:
              cpu: 10m
              memory: 16Mi
            requests:
              cpu: 1m
              memory: 4Mi
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop: ["ALL"]
      containers:
        - name: pause
          image: rancher/mirrored-pause:3.6
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              cpu: 10m
              memory: 16Mi
            requests:
              cpu: 1m
              memory: 4Mi
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop: ["ALL"]
      volumes:
        - name: keeper
          emptyDir: {}
@@ -11,7 +11,10 @@ resources:
  - cnpg_pooler.yaml
  - gateway.yaml
  - httproute.yaml
  - image-keeper.yaml
  - namespace.yaml
  - oauth2-proxy-configmap.yaml
  - oauth2-proxy-deployment.yaml
  - redis-deployment.yaml
  - services.yaml
  - ui-deployment.yaml
@@ -0,0 +1,46 @@
 ---
 # Non-secret oauth2-proxy configuration (client_id/secret/cookie_secret come
 # from the oauth-credentials Secret).
 #
 # SCOPE: this proxy fronts the artifactapi web UI ONLY. The HTTPRoute sends just
 # /ui and /oauth2 here; every machine surface (/api/v1, /api/v2, /v2 docker
 # registry, /terraform, /.well-known/terraform.json, /health, /version, /) goes
 # straight to the api Service and is NOT authenticated. yum/dnf, containerd
 # registry mirrors, docker/buildah, terraform init and Woodpecker publish steps
 # cannot complete a browser OIDC flow, so they must never reach this container.
 # Its only upstream is the ui Service -- there is deliberately no api upstream.
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: artifactapi-oauth2-env
  namespace: artifactapi
 data:
  OAUTH2_PROXY_HTTP_ADDRESS: "0.0.0.0:4180"
  OAUTH2_PROXY_METRICS_ADDRESS: "0.0.0.0:44180"
  OAUTH2_PROXY_PROVIDER: "oidc"
  # Publicly-trusted Authentik host: the authorize step is a browser redirect,
  # so the issuer must present a cert every user's browser already trusts (the
  # k8s host serves an internal-CA cert). Slug from terraform-authentik.
  OAUTH2_PROXY_OIDC_ISSUER_URL: "https://identity.unkin.net/application/o/artifactapi/"
  OAUTH2_PROXY_REDIRECT_URL: "https://artifactapi.k8s.syd1.au.unkin.net/oauth2/callback"
  OAUTH2_PROXY_UPSTREAMS: "http://ui.artifactapi.svc.cluster.local:80/"
  OAUTH2_PROXY_SCOPE: "openid email profile ak_groups"
  # Populate session.Groups from the Authentik hierarchical ak_groups claim.
  OAUTH2_PROXY_OIDC_GROUPS_CLAIM: "ak_groups"
  OAUTH2_PROXY_ALLOWED_GROUPS: "akP-artifactapi-admin"
  OAUTH2_PROXY_PASS_USER_HEADERS: "true"
  OAUTH2_PROXY_EMAIL_DOMAINS: "*"
  # Authentik hardcodes email_verified=false in the id_token; authorization is
  # enforced via ak_groups, so accepting the unverified email is safe.
  OAUTH2_PROXY_INSECURE_OIDC_ALLOW_UNVERIFIED_EMAIL: "true"
  OAUTH2_PROXY_COOKIE_SECURE: "true"
  OAUTH2_PROXY_COOKIE_DOMAINS: "artifactapi.k8s.syd1.au.unkin.net"
  OAUTH2_PROXY_WHITELIST_DOMAINS: "artifactapi.k8s.syd1.au.unkin.net"
  OAUTH2_PROXY_REVERSE_PROXY: "true"
  OAUTH2_PROXY_CODE_CHALLENGE_METHOD: "S256"
  OAUTH2_PROXY_SKIP_PROVIDER_BUTTON: "true"
  # Back-channel discovery/token calls resolve the issuer inside the cluster,
  # where it is served under the internal unkin.net CA rather than the publicly
  # trusted cert the browser sees. Trust the bundle the combine-certs init
  # container assembles, as every other oauth2-proxy in the estate does.
  OAUTH2_PROXY_PROVIDER_CA_FILES: "/etc/ssl/combined/ca-certificates.crt"
@@ -0,0 +1,136 @@
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: oauth2
  namespace: artifactapi
  annotations:
    configmap.reloader.stakater.com/auto: "true"
    secret.reloader.stakater.com/reload: "oauth-credentials,vault-ca-cert"
 spec:
  replicas: 2
  selector:
    matchLabels:
      app: oauth2
  strategy:
    rollingUpdate:
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: oauth2
    spec:
      serviceAccountName: default
      automountServiceAccountToken: false
      securityContext:
        runAsNonRoot: true
        runAsUser: 65532
        runAsGroup: 65532
        fsGroup: 65532
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        # The Authentik issuer is served behind the internal unkin.net CA;
        # combine the system roots with it so oauth2-proxy's OIDC HTTP client
        # trusts the discovery endpoint.
        - name: combine-certs
          image: docker.io/library/alpine:3
          imagePullPolicy: IfNotPresent
          command:
            - sh
            - -c
            - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt
          volumeMounts:
            - name: vault-ca-cert
              mountPath: /custom-ca
              readOnly: true
            - name: combined-certs
              mountPath: /combined-certs
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop:
                - ALL
          resources:
            requests:
              cpu: 50m
              memory: 32Mi
            limits:
              cpu: 200m
              memory: 64Mi
      containers:
        - name: oauth2-proxy
          image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.3
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 4180
              name: http
              protocol: TCP
            - containerPort: 44180
              name: metrics
              protocol: TCP
          envFrom:
            - configMapRef:
                name: artifactapi-oauth2-env
                optional: false
          env:
            - name: OAUTH2_PROXY_CLIENT_ID
              valueFrom:
                secretKeyRef:
                  name: oauth-credentials
                  key: client_id
            - name: OAUTH2_PROXY_CLIENT_SECRET
              valueFrom:
                secretKeyRef:
                  name: oauth-credentials
                  key: client_secret
            - name: OAUTH2_PROXY_COOKIE_SECRET
              valueFrom:
                secretKeyRef:
                  name: oauth-credentials
                  key: cookie_secret
          livenessProbe:
            httpGet:
              path: /ping
              port: http
            initialDelaySeconds: 10
            periodSeconds: 30
            timeoutSeconds: 5
            failureThreshold: 3
          readinessProbe:
            httpGet:
              path: /ready
              port: http
            initialDelaySeconds: 5
            periodSeconds: 10
            timeoutSeconds: 5
            failureThreshold: 3
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop:
                - ALL
          volumeMounts:
            - name: combined-certs
              mountPath: /etc/ssl/combined
              readOnly: true
          resources:
            requests:
              cpu: 50m
              memory: 64Mi
            limits:
              cpu: 500m
              memory: 256Mi
      volumes:
        - name: vault-ca-cert
          secret:
            secretName: vault-ca-cert
            items:
              - key: ca.crt
                path: ca.crt
        - name: combined-certs
          emptyDir: {}
      restartPolicy: Always
@@ -16,6 +16,26 @@ spec:
  sessionAffinity: None
  type: ClusterIP
 ---
 # Authenticated front door for the web UI only: api-route sends /ui and /oauth2
 # here, oauth2-proxy authenticates and forwards to the ui Service. Every other
 # path reaches the api Service above directly and stays unauthenticated.
 apiVersion: v1
 kind: Service
 metadata:
  name: oauth2
  namespace: artifactapi
 spec:
  internalTrafficPolicy: Cluster
  ports:
    - name: http
      port: 80
      protocol: TCP
      targetPort: http
  selector:
    app: oauth2
  sessionAffinity: None
  type: ClusterIP
 ---
 apiVersion: v1
 kind: Service
 metadata:
@@ -22,7 +22,7 @@ spec:
      automountServiceAccountToken: true
      containers:
        - name: ui
-          image: git.unkin.net/unkin/artifactapi-ui:v3.11.2
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui:v3.20.0
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 80
@@ -32,3 +32,26 @@ spec:
  refreshAfter: 5m
  type: kv-v2
  vaultAuthRef: default
 ---
 # Authentik OIDC client for the artifactapi UI front door (client_id,
 # client_secret, cookie_secret). Seeded out of band at
 # kv/kubernetes/namespace/artifactapi/default/oauth-credentials; the default
 # k8s auth role already grants the artifactapi/default ServiceAccount read on
 # kv/data/kubernetes/namespace/{{sa_namespace}}/{{sa_name}}/*, so no
 # terraform-vault change is needed. Consumed by the oauth2 Deployment.
 apiVersion: secrets.hashicorp.com/v1beta1
 kind: VaultStaticSecret
 metadata:
  name: oauth-credentials
  namespace: artifactapi
 spec:
  destination:
    create: true
    name: oauth-credentials
    overwrite: true
  hmacSecretData: true
  mount: kv
  path: kubernetes/namespace/artifactapi/default/oauth-credentials
  refreshAfter: 5m
  type: kv-v2
  vaultAuthRef: default
@@ -14,3 +14,17 @@ spec:
  podMetricsEndpoints:
    - port: metrics
      path: /metrics
 ---
 # Scrape the UI oauth2-proxy (:44180), which exposes sign-in/authz counters.
 apiVersion: operator.victoriametrics.com/v1beta1
 kind: VMPodScrape
 metadata:
  name: oauth2
  namespace: artifactapi
 spec:
  selector:
    matchLabels:
      app: oauth2
  podMetricsEndpoints:
    - port: metrics
      path: /metrics
@@ -64,8 +64,12 @@ spec:
      archive_mode: "on"
      archive_timeout: 5min
      dynamic_shared_memory_type: posix
-      effective_cache_size: 256MB
+      effective_cache_size: 1536MB
      full_page_writes: "on"
      # Replicas report their oldest xmin to the primary, so multi-second reads on
      # a hot standby stop exhausting max_standby_streaming_delay and being
      # cancelled. Retained-dead-tuple cost is negligible on a ~155MB database.
      hot_standby_feedback: "on"
      log_destination: csvlog
      log_directory: /controller/log
      log_filename: postgres
@@ -77,7 +81,12 @@ spec:
      max_parallel_workers: "16"
      max_replication_slots: "16"
      max_worker_processes: "16"
-      shared_buffers: 128MB
+      # A pg_stat_statements.* parameter is what makes CNPG treat the extension as
      # managed and run CREATE EXTENSION in every database; preloading alone does
      # not create it.
      pg_stat_statements.max: "10000"
      pg_stat_statements.track: top
      shared_buffers: 512MB
      shared_memory_type: mmap
      ssl_max_protocol_version: TLSv1.3
      ssl_min_protocol_version: TLSv1.3
@@ -86,6 +95,9 @@ spec:
      wal_log_hints: "on"
      wal_receiver_timeout: 5s
      wal_sender_timeout: 5s
    # CNPG merges this with the libraries it manages itself.
    shared_preload_libraries:
      - pg_stat_statements
    syncReplicaElectionConstraint:
      enabled: false
  primaryUpdateMethod: restart
@@ -105,13 +117,16 @@ spec:
    updateInterval: 30
  resources:
    limits:
-      cpu: 500m
+      # 500m is a 50ms CFS quota per 100ms period, exhausted by bursts even at
      # ~0.01 cores average, so every query pays throttle latency.
      cpu: "2"
      # 512Mi OOMKilled replicas under load (shared_buffers 128MB +
      # max_connections 200 leave no headroom) — see incident 2026-07-28.
-      memory: 1Gi
+      # shared_buffers 512MB needs the same headroom multiple, hence 2Gi.
      memory: 2Gi
    requests:
-      cpu: 50m
+      cpu: 500m
-      memory: 512Mi
+      memory: 1Gi
  smartShutdownTimeout: 180
  startDelay: 3600
  stopDelay: 1800
@@ -37,6 +37,22 @@ spec:
      name: authentik
      sectionName: https
  rules:
    - backendRefs:
        - group: ""
          kind: Service
          name: authentik-server
          port: 80
          weight: 1
      filters:
        - type: URLRewrite
          urlRewrite:
            path:
              type: ReplaceFullPath
              replaceFullPath: /application/o/token/
      matches:
        - path:
            type: Exact
            value: /application/o/token
    - backendRefs:
        - group: ""
          kind: Service
@@ -86,6 +102,22 @@ spec:
      name: authentik-internal
      sectionName: https
  rules:
    - backendRefs:
        - group: ""
          kind: Service
          name: authentik-server
          port: 80
          weight: 1
      filters:
        - type: URLRewrite
          urlRewrite:
            path:
              type: ReplaceFullPath
              replaceFullPath: /application/o/token/
      matches:
        - path:
            type: Exact
            value: /application/o/token
    - backendRefs:
        - group: ""
          kind: Service
@@ -19,6 +19,7 @@ resources:
  - redis-deployment.yaml
  - redis-pvc.yaml
  - redis-service.yaml
  - server-vmpodscrape.yaml
  - vaultauth.yaml
  - vaultstaticsecret.yaml
  - vmpodscrape.yaml
@@ -0,0 +1,16 @@
 ---
 # Scrape the authentik server's django_prometheus endpoint (:9300). Picked up
 # by the observability VMAgent (selectAllByDefault).
 apiVersion: operator.victoriametrics.com/v1beta1
 kind: VMPodScrape
 metadata:
  name: authentik-server
  namespace: authentik
 spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: authentik
      app.kubernetes.io/component: server
  podMetricsEndpoints:
    - port: metrics
      path: /metrics
@@ -3,4 +3,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - namespace.yaml
+  - ns1.yaml
@@ -0,0 +1,16 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: acme-ns1-a
  namespace: bind-external
 spec:
  zoneRef: acme-unkin-net
  name: ns1
  type: A
  ttl: 3600
  values:
    # Public address of this cluster's external BIND, same target as
    # acme-ns1.unkin.net. Resolvers that cached the seeded ns1.acme.unkin.net
    # NS name must still reach the zone.
    - 103.216.191.185
@@ -0,0 +1,11 @@
 ---
 # Authoritative delegation records for acme.unkin.net. Without these the zone
 # only holds the operator's seed apex (NS ns1.acme.unkin.net glued to the
 # primary pod IP), which is unroutable off-cluster and goes stale on
 # reschedule. DNSRecords must live in the same namespace as their BindZone.
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - ns
  - a
@@ -0,0 +1,16 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: acme-apex-ns
  namespace: bind-external
 spec:
  zoneRef: acme-unkin-net
  # "@" is the zone apex.
  name: "@"
  type: NS
  ttl: 3600
  values:
    # Matches the parent delegation in Google Cloud DNS. Out of zone, so the
    # child needs no glue of its own.
    - acme-ns1.unkin.net.
@@ -0,0 +1,6 @@
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - apex.yaml
@@ -7,4 +7,5 @@ resources:
  - cluster.yaml
  - tsigkey.yaml
  - zones.yaml
  - acme-unkin-net
  - agent-dns-rolebinding.yaml
@@ -17,3 +17,14 @@ spec:
  updateKeyRef: certmanager
  allowTransfer:
    - key certmanager
  # Published apex NS. acme-ns1 is what the parent delegates to and glues; ns1 is
  # in-zone, so its address is declared below or a reseed would glue it to the
  # primary pod IP.
  nameservers:
    - acme-ns1.unkin.net.
    - ns1.acme.unkin.net.
  records:
    - name: ns1
      type: A
      ttl: 3600
      values: ["103.216.191.185"]
@@ -0,0 +1,15 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: dashboard-ceph-cname
  namespace: bind-internal
 spec:
  zoneRef: ceph-unkin-net
  name: dashboard
  type: CNAME
  ttl: 600
  values:
    # Ceph mgr dashboard, reached via lb1. Lets in-cluster clients (the
    # cephrgw-operator) resolve dashboard.ceph.unkin.net.
    - lb1.unkin.net.
@@ -0,0 +1,7 @@
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - dashboard.yaml
  - s3.yaml
@@ -0,0 +1,15 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: s3-ceph-cname
  namespace: bind-internal
 spec:
  zoneRef: ceph-unkin-net
  name: s3
  type: CNAME
  ttl: 600
  values:
    # radosgw S3 endpoint. Points at the Consul service for now; the real
    # target will be changed later.
    - radosgw.service.consul.
@@ -0,0 +1,6 @@
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - cname
@@ -2,9 +2,15 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 # Individually-managed authoritative records live under <zone>/<type>/<record>.yaml.
 # DNSRecords must live in the same namespace as their BindZone (the operator
 # resolves zoneRef/clusterRef/updateKeyRef within the record's namespace), so
 # these sit alongside the zone in bind-internal, not in the app namespace.
 resources:
  - cluster.yaml
  - tsigkey.yaml
  - zones.yaml
-  - records.yaml
+  - unkin-net
  - main-unkin-net
  - ceph-unkin-net
  - acls.yaml
@@ -0,0 +1,14 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: jellyfin-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: main-unkin-net
  name: jellyfin
  type: A
  ttl: 600
  values:
    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
    - 198.18.199.0
@@ -0,0 +1,13 @@
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - jellyfin.yaml
  - mail-in.yaml
  - lidarr.yaml
  - nzbget.yaml
  - prowlarr.yaml
  - radarr.yaml
  - readarr.yaml
  - sonarr.yaml
@@ -0,0 +1,14 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: lidarr-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: main-unkin-net
  name: lidarr
  type: A
  ttl: 600
  values:
    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
    - 198.18.199.0
@@ -0,0 +1,14 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: mail-in-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: main-unkin-net
  name: mail-in
  type: A
  ttl: 600
  values:
    # home WAN (public path for edge to smarthost relay), mirrors public DNS.
    - 103.216.191.185
@@ -0,0 +1,14 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: nzbget-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: main-unkin-net
  name: nzbget
  type: A
  ttl: 600
  values:
    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
    - 198.18.199.0
@@ -0,0 +1,14 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: prowlarr-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: main-unkin-net
  name: prowlarr
  type: A
  ttl: 600
  values:
    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
    - 198.18.199.0
@@ -0,0 +1,14 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: radarr-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: main-unkin-net
  name: radarr
  type: A
  ttl: 600
  values:
    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
    - 198.18.199.0
@@ -0,0 +1,14 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: readarr-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: main-unkin-net
  name: readarr
  type: A
  ttl: 600
  values:
    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
    - 198.18.199.0
@@ -0,0 +1,14 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: sonarr-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: main-unkin-net
  name: sonarr
  type: A
  ttl: 600
  values:
    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
    - 198.18.199.0
@@ -0,0 +1,6 @@
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - a
@@ -1,164 +0,0 @@
 # Individually-managed authoritative records for the unkin.net zone.
 # DNSRecords must live in the same namespace as their BindZone (the operator
 # resolves zoneRef/clusterRef/updateKeyRef within the record's namespace), so
 # these sit alongside the zone in bind-internal, not in the app namespace.
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  # "internal" in the name distinguishes this from the external DNS that
  # Authentik will manage its own records from later.
  name: identity-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: identity
  type: A
  ttl: 600
  values:
    # traefik-EXTERNAL (DMZ) gateway VIP; the authentik Gateway serves the
    # identity.unkin.net hostname there.
    - 198.18.199.0
 ---
 # PRODUCTION CUTOVER RECORD — intentionally commented out.
 # git.unkin.net currently resolves to the LIVE VM forge (HAProxy VRRP VIP
 # 198.18.19.17), which holds every repo the estate depends on. Uncommenting this
 # repoints the whole org's git.unkin.net at the new k8s Gitea gateway VIP, so it
 # is the FINAL step of the forge migration — gated on the data migration (gitea
 # dump/restore + SECRET_KEY copy) in argocd-apps docs/gitea-migration.md.
 # NOTE: the live git.unkin.net answer is served by the puppet DNS master today
 # (profiles::dns::master, records from PuppetDB); this k8s apex zone holds only
 # SOA+NS + a few DNSRecords so far. Confirm the k8s bind cluster is the live
 # authority for unkin.net (or update the puppet record instead) before relying
 # on this CR at cutover.
 # ---
 # apiVersion: bind.unkin.net/v1alpha1
 # kind: DNSRecord
 # metadata:
 #   name: git-dns-internal
 #   namespace: bind-internal
 # spec:
 #   zoneRef: unkin-net
 #   name: git
 #   type: A
 #   ttl: 600
 #   values:
 #     # traefik-internal gateway VIP; the gitea Gateway serves git.unkin.net there.
 #     - 198.18.200.4
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: s3-ceph-cname
  namespace: bind-internal
 spec:
  zoneRef: ceph-unkin-net
  name: s3
  type: CNAME
  ttl: 600
  values:
    # radosgw S3 endpoint. Points at the Consul service for now; the real
    # target will be changed later.
    - radosgw.service.consul.
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: dashboard-ceph-cname
  namespace: bind-internal
 spec:
  zoneRef: ceph-unkin-net
  name: dashboard
  type: CNAME
  ttl: 600
  values:
    # Ceph mgr dashboard, reached via lb1. Lets in-cluster clients (the
    # cephrgw-operator) resolve dashboard.ceph.unkin.net.
    - lb1.unkin.net.
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: lb1-unkin-net
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: lb1
  type: A
  ttl: 600
  values:
    - 103.216.191.185
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: ghp-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: ghp
  type: A
  ttl: 600
  values:
    # traefik-internal gateway VIP; the ghp Gateway serves ghp.unkin.net there.
    - 198.18.200.4
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: arrstack-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: arrstack
  type: A
  ttl: 600
  values:
    # traefik-EXTERNAL (DMZ) gateway VIP; the arrproxy Gateway serves the
    # arrstack.unkin.net front door (oauth2-proxy) there.
    - 198.18.199.0
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: logviewer-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: logviewer
  type: A
  ttl: 600
  values:
    # traefik-internal gateway VIP; the logviewer Gateway serves
    # logviewer.unkin.net there.
    - 198.18.200.4
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: cheeztv-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: cheeztv
  type: A
  ttl: 600
  values:
    # traefik-internal gateway VIP; the cheeztv Gateway serves cheeztv.unkin.net
    # there.
    - 198.18.200.4
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: watchstate-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: watchstate
  type: A
  ttl: 600
  values:
    # traefik-EXTERNAL (DMZ) gateway VIP; the watchstate-external Gateway serves
    # the watchstate.unkin.net front door (oauth2-proxy) there.
    - 198.18.199.0
@@ -0,0 +1,15 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: arrstack-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: arrstack
  type: A
  ttl: 600
  values:
    # traefik-EXTERNAL (DMZ) gateway VIP; the arrproxy Gateway serves the
    # arrstack.unkin.net front door (oauth2-proxy) there.
    - 198.18.199.0
@@ -0,0 +1,14 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: auth-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: auth
  type: A
  ttl: 600
  values:
    # traefik-internal gateway VIP; the kanidm Gateway serves auth.unkin.net there.
    - 198.18.200.4
@@ -0,0 +1,15 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: cheeztv-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: cheeztv
  type: A
  ttl: 600
  values:
    # traefik-internal gateway VIP; the cheeztv Gateway serves cheeztv.unkin.net
    # there.
    - 198.18.200.4
@@ -0,0 +1,14 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: fafflix-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: fafflix
  type: A
  ttl: 600
  values:
    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
    - 198.18.199.0
@@ -0,0 +1,14 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: ghp-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: ghp
  type: A
  ttl: 600
  values:
    # traefik-internal gateway VIP; the ghp Gateway serves ghp.unkin.net there.
    - 198.18.200.4
@@ -0,0 +1,14 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: git-edge-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: git
  type: A
  ttl: 600
  values:
    # traefik-external haproxy edge Gateway VIP (TLS passthrough).
    - 198.18.199.0
@@ -0,0 +1,23 @@
 ---
 # PRODUCTION CUTOVER RECORD — intentionally commented out.
 # git.unkin.net currently resolves to the haproxy edge (git-edge.yaml), which
 # forwards to the LIVE VM forge holding every repo. Uncommenting this
 # repoints the whole org's git.unkin.net at the new k8s Gitea gateway VIP, so it
 # is the FINAL step of the forge migration — gated on the data migration (gitea
 # dump/restore + SECRET_KEY copy) in argocd-apps docs/gitea-migration.md.
 # Uncomment this record AND its entry in kustomization.yaml (removing
 # git-edge.yaml there) to activate it.
 # ---
 # apiVersion: bind.unkin.net/v1alpha1
 # kind: DNSRecord
 # metadata:
 #   name: git-dns-internal
 #   namespace: bind-internal
 # spec:
 #   zoneRef: unkin-net
 #   name: git
 #   type: A
 #   ttl: 600
 #   values:
 #     # traefik-internal gateway VIP; the gitea Gateway serves git.unkin.net there.
 #     - 198.18.200.4
@@ -0,0 +1,15 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: grafana-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: grafana
  type: A
  ttl: 600
  values:
    # traefik-EXTERNAL (DMZ) gateway VIP; the grafana-external Gateway serves
    # grafana.unkin.net there.
    - 198.18.199.0
@@ -0,0 +1,17 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  # "internal" in the name distinguishes this from the external DNS that
  # Authentik will manage its own records from later.
  name: identity-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: identity
  type: A
  ttl: 600
  values:
    # traefik-EXTERNAL (DMZ) gateway VIP; the authentik Gateway serves the
    # identity.unkin.net hostname there.
    - 198.18.199.0
@@ -0,0 +1,23 @@
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - arrstack.yaml
  - auth.yaml
  - cheeztv.yaml
  - fafflix.yaml
  # PRODUCTION CUTOVER RECORD — see git.yaml. Uncomment together with the
  # record itself, and remove git-edge.yaml.
  # - git.yaml
  - ghp.yaml
  - git-edge.yaml
  - grafana.yaml
  - identity.yaml
  - lb1.yaml
  - logviewer.yaml
  - mail.yaml
  - mta1-edge.yaml
  - mta2-edge.yaml
  - vlogs.yaml
  - watchstate.yaml
@@ -0,0 +1,13 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: lb1-unkin-net
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: lb1
  type: A
  ttl: 600
  values:
    - 103.216.191.185
@@ -0,0 +1,15 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: logviewer-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: logviewer
  type: A
  ttl: 600
  values:
    # traefik-internal gateway VIP; the logviewer Gateway serves
    # logviewer.unkin.net there.
    - 198.18.200.4
@@ -0,0 +1,14 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: mail-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: mail
  type: A
  ttl: 600
  values:
    # stalwart-lb PureLB VIP (IMAPS, submission, HTTPS).
    - 198.18.200.18
@@ -0,0 +1,14 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: mta1-edge-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: mta1.edge
  type: A
  ttl: 600
  values:
    # edge MTA 1 public IP, mirrors public DNS.
    - 66.226.148.207
@@ -0,0 +1,14 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: mta2-edge-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: mta2.edge
  type: A
  ttl: 600
  values:
    # edge MTA 2 public IP, mirrors public DNS.
    - 66.226.149.50
@@ -0,0 +1,15 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: vlogs-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: vlogs
  type: A
  ttl: 600
  values:
    # traefik-EXTERNAL (DMZ) gateway VIP; the vlogs-external Gateway serves the
    # vlogs.unkin.net front door (oauth2-proxy) there.
    - 198.18.199.0
@@ -0,0 +1,15 @@
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: DNSRecord
 metadata:
  name: watchstate-dns-internal
  namespace: bind-internal
 spec:
  zoneRef: unkin-net
  name: watchstate
  type: A
  ttl: 600
  values:
    # traefik-EXTERNAL (DMZ) gateway VIP; the watchstate-external Gateway serves
    # the watchstate.unkin.net front door (oauth2-proxy) there.
    - 198.18.199.0
@@ -0,0 +1,6 @@
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - a
@@ -38,6 +38,9 @@ spec:
  clusterRef: bind-resolvers
  entries:
    - 198.18.24.0/24
    - 172.18.30.0/24  # BinaryLane DMZ VPC syd (mail edge, via VPN)
    - 172.18.31.0/24  # BinaryLane DMZ VPC mel (mail edge, via VPN)
    - 172.18.32.0/24  # BinaryLane DMZ VPC bne (mail edge, via VPN)
 ---
 apiVersion: bind.unkin.net/v1alpha1
 kind: BindACL
@@ -18,9 +18,10 @@ spec:
      purelb.io/service-group: common
      purelb.io/addresses: 198.18.200.7
      external-dns.alpha.kubernetes.io/hostname: bind-resolvers.k8s.syd1.au.unkin.net
  # Public names go via global forwarders; RBL zones bypass them (view.yaml).
  forwarders:
    - 8.8.8.8
    - 1.1.1.1
    - 8.8.8.8
  # The internal split-horizon zones are served UNSIGNED by the in-cluster
  # authoritative, but their public parents publish DS records (e.g. unkin.net
  # is DNSSEC-signed on the Internet). With dnssec-validation on, the validator
@@ -21,3 +21,16 @@ spec:
  extraOptions:
    - "allow-recursion { any; }"
    - "allow-query-cache { any; }"
    # pods have no IPv6 route; v6 NS attempts burn the recursion-query budget
    - "server ::/0 { bogus yes; }"
    # RBL zones resolve directly: DNSBLs refuse queries relayed via public resolvers.
    - 'zone "spamhaus.org" { type forward; forwarders { }; }'
    - 'zone "mailspike.net" { type forward; forwarders { }; }'
    - 'zone "spameatingmonkey.net" { type forward; forwarders { }; }'
    - 'zone "dnswl.org" { type forward; forwarders { }; }'
    - 'zone "virusfree.cz" { type forward; forwarders { }; }'
    - 'zone "blocklist.de" { type forward; forwarders { }; }'
    - 'zone "msbl.org" { type forward; forwarders { }; }'
    - 'zone "surbl.org" { type forward; forwarders { }; }'
    - 'zone "uribl.com" { type forward; forwarders { }; }'
    - 'zone "rspamd.com" { type forward; forwarders { }; }'
@@ -21,7 +21,7 @@ spec:
        runAsNonRoot: true
      containers:
        - name: operator
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.2.6
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/bind-operator:v0.4.0
          args:
            - --metrics-bind-address=:8080
            - --health-probe-bind-address=:8081
@@ -6,7 +6,7 @@ resources:
  - namespace.yaml
  # CRDs are pulled from the bind-operator repo at the matching tag rather than
  # vendored here, so they never drift from the operator.
-  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.2.6/config/crd/install.yaml
+  - https://git.unkin.net/unkin/bind-operator/raw/tag/v0.4.0/config/crd/install.yaml
  - rbac.yaml
  - agent-dns-rbac.yaml
  - deployment.yaml
@@ -0,0 +1,26 @@
 ---
 # Let's Encrypt *.ceph.unkin.net wildcard for the haproxy edge (ceph dashboard).
 # DNS-01 needs the delegated _acme-challenge.ceph.unkin.net CNAME in the public
 # unkin.net zone.
 # _acme-challenge.ceph.unkin.net. CNAME _acme-challenge.ceph.acme.unkin.net.
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
  name: wildcard-ceph-unkin-net
  namespace: cert-manager
 spec:
  secretName: wildcard-ceph-unkin-net-tls
  secretTemplate:
    annotations:
      reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy"
      reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy"
  privateKey:
    size: 4096
  dnsNames:
    - "*.ceph.unkin.net"
  issuerRef:
    name: letsencrypt
    kind: ClusterIssuer
    group: cert-manager.io
@@ -0,0 +1,26 @@
 ---
 # Let's Encrypt *.main.unkin.net wildcard for the haproxy edge (pve, arr stack,
 # jellyfin, stalwart webadmin/autoconfig) and the mail smarthost. DNS-01 needs the delegated
 # _acme-challenge.main.unkin.net CNAME in the public unkin.net zone.
 # _acme-challenge.main.unkin.net. CNAME _acme-challenge.main.acme.unkin.net.
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
  name: wildcard-main-unkin-net
  namespace: cert-manager
 spec:
  secretName: wildcard-main-unkin-net-tls
  secretTemplate:
    annotations:
      reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "haproxy,mail"
      reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "haproxy,mail"
  privateKey:
    size: 4096
  dnsNames:
    - "*.main.unkin.net"
  issuerRef:
    name: letsencrypt
    kind: ClusterIssuer
    group: cert-manager.io
@@ -14,9 +14,9 @@ spec:
  secretTemplate:
    annotations:
      reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
-      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
      reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
-      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cheeztv,arrstack,authentik,gitea,watchstate,mediamark,repospawner,haproxy,vlogs,grafana,mail"
  privateKey:
    size: 4096
  dnsNames:
@@ -12,3 +12,5 @@ resources:
  - clusterissuer_letsencrypt.yaml
  - clusterissuer_letsencrypt-staging.yaml
  - certificate_wildcard-unkin-net.yaml
  - certificate_wildcard-main-unkin-net.yaml
  - certificate_wildcard-ceph-unkin-net.yaml
@@ -26,7 +26,7 @@ data:
          </key>
          <value>
            <PluginConfiguration>
-              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
              <OidClientId>jellyfin</OidClientId>
              <OidSecret>@@CLIENT_SECRET@@</OidSecret>
              <Enabled>true</Enabled>
@@ -13,6 +13,4 @@ spec:
      targetPort: http
  selector:
    app: cheeztv
  # Pin each client to one replica to reduce transcode-session churn/takeover.
  sessionAffinity: ClientIP
  type: ClusterIP
@@ -4,6 +4,8 @@ kind: StatefulSet
 metadata:
  name: cheeztv
  namespace: cheeztv
  annotations:
    configmap.reloader.stakater.com/auto: "true"
 spec:
  # HA: two replicas coordinate transcode session ownership through Valkey and
  # resume each other's HLS segments off the shared RWX transcode PVC. Stable
@@ -162,7 +164,7 @@ spec:
              readOnly: true
      containers:
        - name: cheeztv
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.4.0
          imagePullPolicy: IfNotPresent
          ports:
            - name: http
@@ -1,7 +0,0 @@
 ---
 apiVersion: v1
 kind: Namespace
 metadata:
  labels:
    app.kubernetes.io/name: clickhouse-system
  name: clickhouse-system
@@ -11,7 +11,7 @@ metadata:
    argocd.argoproj.io/sync-wave: "1"
 spec:
  replicas: 1
-  image: git.unkin.net/unkin/kea-api:v0.1.3
+  image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/kea-api:v0.1.7
  tokenSecretName: kea-api-token
  service:
    type: ClusterIP
@@ -11,7 +11,7 @@ metadata:
    argocd.argoproj.io/sync-wave: "1"
 spec:
  replicas: 2
-  image: git.unkin.net/unkin/kea:v0.1.3
+  image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/kea:v0.1.7
  domainName: main.unkin.net
  defaultLeaseTime: 1200
  maxLeaseTime: 86400
@@ -21,7 +21,7 @@ spec:
        runAsNonRoot: true
      containers:
        - name: operator
-          image: git.unkin.net/unkin/kea-operator:v0.1.5
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/kea-operator:v0.1.7
          args:
            - --metrics-bind-address=:8080
            - --health-probe-bind-address=:8081
@@ -26,7 +26,7 @@ data:
          </key>
          <value>
            <PluginConfiguration>
-              <OidEndpoint>https://identity.k8s.syd1.au.unkin.net/application/o/jellyfin/</OidEndpoint>
+              <OidEndpoint>https://identity.unkin.net/application/o/jellyfin/</OidEndpoint>
              <OidClientId>jellyfin</OidClientId>
              <OidSecret>@@CLIENT_SECRET@@</OidSecret>
              <Enabled>true</Enabled>
@@ -13,6 +13,4 @@ spec:
      targetPort: http
  selector:
    app: fafflix
  # Pin each client to one replica to reduce transcode-session churn/takeover.
  sessionAffinity: ClientIP
  type: ClusterIP
@@ -4,6 +4,8 @@ kind: StatefulSet
 metadata:
  name: fafflix
  namespace: fafflix
  annotations:
    configmap.reloader.stakater.com/auto: "true"
 spec:
  # HA: two replicas coordinate transcode session ownership through Valkey and
  # resume each other's HLS segments off the shared RWX transcode PVC. Stable
@@ -162,7 +164,7 @@ spec:
              readOnly: true
      containers:
        - name: fafflix
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.2.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/jellyfin-ha:v0.4.0
          imagePullPolicy: IfNotPresent
          ports:
            - name: http
@@ -1,11 +1,6 @@
 ---
-# HTTPS front for the k8s Gitea, served on two names:
+# HTTPS front for the k8s Gitea on git.k8s.syd1.au.unkin.net (external-dns
-#   git.unkin.net              — canonical/production (apex, bind-operator zone;
+# k8s.syd1 zone). git.unkin.net stays on the VM haproxy until cutover.
 #                                DNS flip is the gated cutover step, see the doc)
 #   git.k8s.syd1.au.unkin.net  — admin/backup route (external-dns k8s.syd1 zone),
 #                                same dual-name pattern as identity.unkin.net.
 # The cert-manager Certificate (vault-issuer) takes CN git.unkin.net and gets a
 # DNS SAN for each TLS listener hostname automatically.
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
@@ -17,36 +12,13 @@ metadata:
    traefik.io/instance: internal
  annotations:
    cert-manager.io/cluster-issuer: vault-issuer
-    cert-manager.io/common-name: git.unkin.net
+    cert-manager.io/common-name: git.k8s.syd1.au.unkin.net
    cert-manager.io/private-key-size: "4096"
    # Only the k8s admin route is published by external-dns (it owns just the
    # k8s.syd1.au.unkin.net zone). git.unkin.net lives in the apex zone and is
    # flipped at cutover — NOT managed here.
    external-dns.alpha.kubernetes.io/hostname: git.k8s.syd1.au.unkin.net
    external-dns.alpha.kubernetes.io/target: 198.18.200.4
 spec:
  gatewayClassName: traefik-internal
  listeners:
    - name: http-primary
      port: 80
      protocol: HTTP
      hostname: git.unkin.net
      allowedRoutes:
        namespaces:
          from: Same
    - name: https-primary
      port: 443
      protocol: HTTPS
      hostname: git.unkin.net
      allowedRoutes:
        namespaces:
          from: Same
      tls:
        mode: Terminate
        certificateRefs:
          - group: ""
            kind: Secret
            name: gitea-tls
    - name: http-admin
      port: 80
      protocol: HTTP
@@ -9,13 +9,8 @@ metadata:
    app.kubernetes.io/instance: gitea
 spec:
  hostnames:
    - git.unkin.net
    - git.k8s.syd1.au.unkin.net
  parentRefs:
    - group: gateway.networking.k8s.io
      kind: Gateway
      name: gitea
      sectionName: http-primary
    - group: gateway.networking.k8s.io
      kind: Gateway
      name: gitea
@@ -41,13 +36,8 @@ metadata:
    app.kubernetes.io/instance: gitea
 spec:
  hostnames:
    - git.unkin.net
    - git.k8s.syd1.au.unkin.net
  parentRefs:
    - group: gateway.networking.k8s.io
      kind: Gateway
      name: gitea
      sectionName: https-primary
    - group: gateway.networking.k8s.io
      kind: Gateway
      name: gitea
@@ -0,0 +1,15 @@
 ---
 # k8up dashboard (M0NsTeRRR/grafana-dashboards), datasource resolved to the
 # in-cluster VictoriaMetrics. Backed by the k8up VMServiceScrape.
 apiVersion: grafana.integreatly.org/v1beta1
 kind: GrafanaDashboard
 metadata:
  name: k8up
  namespace: grafana
 spec:
  instanceSelector:
    matchLabels:
      dashboards: "grafana"
  resyncPeriod: 5m
  allowCrossNamespaceImport: false
  gzipJson: H4sIAAAAAAACA+2ce3PbNhLAvwqHc3eTzlixqIdfSTvjptc2M7Hjq9Pc3NUeDUhCEiIIYAHQsqLRffZbAHzAop0qaSRHKf4SCZLAcrH7wwJYcREixrhCinAmw5NgEVIiFRz8tgjjnFD1ksFJtBeEKVJI8lwk2Nym5pk+CEcCDRFDIdyRk1SXtFrBT7YwaLXCJVzADMVU361EjuF8TFLnjCScveCUC/2wGMXoSXsv6ETQZqff3wuib3TdDE1Nc6e1tME/glOKhZL6eilOiuQ45kik4fJaN51imQiS6Qf05clRngXVPcFYqUye7O/r4qeE64pwStRdaYdEJoj+ByNxqZBQZ5ypMVwFIcMRw+qlfulOOzo42DPayMZvOKeKZMU9RikspxSOKWETreXfrs3JDT7nMzgdIip1SxlimEqr/IRTijKJU+f6SJD0gttuGhfdop/v9ODgtmhvrn+XZcPdO9XqZhVR5u3C1xkWSIHaHfUJkAeeXTzU25ngU6zGOJdOh6dYdeb9foqS3rtJYnr8QUlXBI0qQSOtPp7Vdpjw1LZNERvlaGTbp4gwhW+Vbl6O+ewVYfg8n8ZYSEdR+soZYeQMZWWpbggMTWGmrTvUz09tC+EUiUnKZ8xIntF8RNhbqK8wmaj9tPO0G7qaOwdjlBkC3TiqM1JtQ3edhu46R7uju8tkjNOcYhm847F8FAX2GgrstndHgW+AfxSqJRSn29dht9DhQUOHx7ujwu9RMoFxQMJAkj+O+o6b/NshE3wxxsnkMdUXNRmo44Vd0d+pSMYw+D+qBvtNDXZ2R4O/YAmRy+Nq8Kipwe7uaPBC5OxR9deJmvrr7V4cs00V9h6Ioju1Bg+3qsHnZqoEKphT/O1VOCOpGp9E7fbfnwVjTEZjZU9iLlIsTtrZbSA5JWkQUxiBn12F312JKxYEz1Va1aHFaiFKRuwkSKAZLJ7dwCSPwBSsLJ6SNKX4A7U+z6rqhiBsa2ZlCWJOU339b6wM4J/vZ6UM+yo1h3CgX+o71zrGakr/yDIEzjAyWsG3GRcKpwPmzhPs9R+IwEk5G73ZiNUMCabpC86GZGSeT/EQ5VSV9mAn2ovq3dRYYOhpmkrz9BRlGWGjerpYX3afQjHoPFfmzaTCWT1rtfP4mObm2g3SB3YCbGfkOSNGS4wzbFrk0L9g6Ng2uY7ddxp232lGnyDIWSGsFaKcnpelRgAofJdLRYbzshjlyiwEcEHA/FDZV2WxAK9P8Gu3KUQTIzv4mFTnXJ3Du4bXZU/IEjVGCul6ku7yhozgRPgVmvNcFUsQa9ndPXaFxAirol/+pEnpRZFanXpdCDzPrJbc4inQRDjUAOs3FiDz6RO9rjKQ5YxvMEL5CC+a7vHt/67C2ievwqVZ8iEM0Go4U1ZN8Qiz9EcuptbRBoOqV4BuzpKNwEOzLBOehstrx8c0q79AH2P8beEmYTv8dJ8bCYxZw+n2HIm09bo3RJ/NJXsNl+xG3iW35pJmmP+wPxpf1OsFA2VWEFrBfd7513K8Bxwtw0KHHmj0oKulEC62Vpyp6W0Nh4Ro6K7HFS2Z049zvNWFGGcsXPG7KVgpef+ziYGg4FDP/GzZv3W8VhZtwbl0QPmmVPkrFNtVaTfcrK6egYJtNFpYliTvoecqydYOxtaLwbbgmNWoGMTz4ElzEPwmeFK7qMyTBEs5zOkg4bmOgtcYNvf0cuAbEBfi3tgscemhNNgP1mzUcOGT29kQOSyVPDo2gI7jBjoOPTo8OhK9urt5clTNfOXgME+fEXaGbp3X+wpxEjWn5UeeJ54nyO52bJ4oTkM+GNk5evQb9Oh5enh6CLvTt3l6OA15euwcPY4a9Djw9PD0yPQu9+bZUTXjybFr5KjSEGpy9D05PDnKfZHNw8NtafP8+FN55Qd1okn/4cTyi1yOg5+QwjM030Ry+UcyLEMUK4VbCXibJLaKJIdgb2puQ7dEfm9SSS7Hd7Lx9YUXJgMFp//Fgq9ecrczy2QafcH4ZenDuuCCQp9Pi5SZsudiJE51OktRrjWcCjS71Hkr+j5K7FbokFD6GoyGqHn5pwOBUk2a1U1T/a+KH0XxVtZkHJFV9aeEsuSGvHfZAmaHhUGRyxQtxkutg4zTCm26EJn9f31UQlBbTcYJU5fkPS4wqv85gX8gUgkS58XjdYcX9ZRku9APS1dJ4C1sVSJwjmRiSbYAXfA8swbvpOtU+9j1+FTqtTYLPhyGy8+Y8XL/7vv1cr1h6ahwuWg1t+uw3tdbzRCu+tgZUvQLpERmFFW75+YPPYb2jhnGXGkHKPO/yprKze3aWmoFSFASta8PdKv1vNwq/w1r7a7TQE/ZSDJAN4hQnbM1kAzQNuZKLixE70ktWY+wi6qC5XIdzlaZvaUoQSVKADFKUNZ2J+GLgFQYggbpqeip+Fen4gKiGjmHKqa/4KHJuYQrl8Y9TPcZkRSEStaMrRvEc/NXrHA1YC+Ew7cJzS0+DAJMzD3B84RyNAlawX553Mq4VCCqbKX5NDOheAY+Tm6NEikNoCacqRMbAqP0NaPzkpYmkBY800mq5atY6aw7Pa0s0NHCR5iibeR6ud6wUW3COONG5MeNe8eNZKypng5SM53R0zc5SHMB4hY3fs5RBMxtschuko8bT15YEQNHxMCKGFgR/ejiRxc/unzmmDvq1/Dsenh+EJ7gKx+HzWGFR42oQcGoLfPUSO1J6knqSboRktZ/0HVQ2vMo/bwo9cT0xPTE/Lpiz063BmbfA/NeYDI8+5In7ed45ifsHpoemtsLM11qHnhqemp6anpqemp+ONTsOl8QPPTQvBeaOQMRCWDhi94m+rWS0iPUI9QjdHuBp8vQjmfoHzH0i1zidOj5qauc17Z+bZolW3Ra7xTVydXdY9MPjiWbT7Wo0lnq724nuRDWDBbLhz+8vW53AX8JIxUcNP4GNrN8NW95757k5rD+XLf5mDUzGTWnlNZ6pSVR73wWeQrMJ/VN5fe772ST185iFPJ7jsXcvOPvYv7GvmbklH+S8FWvX1T6eosE0Tl6/9L1/tNYc+tOmdFb3Z2RORvh26prJyT7VdDLOUtc9FmvazsmYiW3qTvaXMzLDe0oAN45ax2af0AoXpyH5Y0ZSSY206ksea99uWi/slytA6f34yg6Oj6O41a/E0et3rCNWqjX7bXifq/TTnv9XoRiw8DKKPVceYbxxHzZ3FS//D9scYEQEl4AAA==
@@ -37,3 +37,36 @@ spec:
          - group: ""
            kind: Secret
            name: grafana-tls
 ---
 # Public grafana.unkin.net via the external Traefik; TLS uses the reflected
 # Let's Encrypt *.unkin.net wildcard, DNS lives in the bind-operator zone.
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
  labels:
    traefik.io/instance: external
  name: grafana-external
  namespace: grafana
 spec:
  gatewayClassName: traefik-external
  listeners:
    - allowedRoutes:
        namespaces:
          from: Same
      hostname: grafana.unkin.net
      name: http
      port: 80
      protocol: HTTP
    - allowedRoutes:
        namespaces:
          from: Same
      hostname: grafana.unkin.net
      name: https
      port: 443
      protocol: HTTPS
      tls:
        certificateRefs:
          - group: ""
            kind: Secret
            name: wildcard-unkin-net-tls
        mode: Terminate
@@ -9,8 +9,16 @@ metadata:
 spec:
  deployment:
    spec:
      replicas: 3
      template:
        spec:
          topologySpreadConstraints:
            - maxSkew: 1
              topologyKey: kubernetes.io/hostname
              whenUnsatisfiable: ScheduleAnyway
              labelSelector:
                matchLabels:
                  app: grafana
          containers:
            - name: grafana
              env:
@@ -31,11 +39,11 @@ spec:
                  cpu: 100m
                  memory: 256Mi
                limits:
-                  cpu: "1"
+                  cpu: "2"
-                  memory: 1Gi
+                  memory: 4Gi
  config:
    server:
-      root_url: "https://grafana.k8s.syd1.au.unkin.net"
+      root_url: "https://grafana.unkin.net"
    database:
      type: "postgres"
      host: "postgres-pooler-rw.grafana.svc.cluster.local:5432"
@@ -61,3 +69,8 @@ spec:
      # to akR-global-admin members (and direct members) via terraform-authentik.
      role_attribute_path: "contains(ak_groups[*], 'akP-grafana-admin') && 'Admin' || 'Viewer'"
      role_attribute_strict: "false"
    # Gossip between replicas so each alert notification is sent once.
    unified_alerting:
      ha_peers: "grafana-alerting.grafana.svc.cluster.local:9094"
      ha_listen_address: "${POD_IP}:9094"
      ha_advertise_address: "${POD_IP}:9094"
@@ -20,3 +20,22 @@ spec:
    jsonData:
      timeInterval: "15s"
      httpMethod: "POST"
 ---
 apiVersion: grafana.integreatly.org/v1beta1
 kind: GrafanaDatasource
 metadata:
  name: victorialogs
  namespace: grafana
 spec:
  instanceSelector:
    matchLabels:
      dashboards: "grafana"
  plugins:
    - name: victoriametrics-logs-datasource
      version: 0.32.0
  datasource:
    name: "VictoriaLogs"
    type: "victoriametrics-logs-datasource"
    uid: "victorialogs"
    access: "proxy"
    url: "http://vlselect-logs.logging.svc.cluster.local:9471"
@@ -53,3 +53,58 @@ spec:
        - path:
            type: PathPrefix
            value: /
 ---
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:
  name: grafana-external-http-redirect
  namespace: grafana
  labels:
    app.kubernetes.io/name: grafana
    app.kubernetes.io/instance: grafana
 spec:
  hostnames:
    - grafana.unkin.net
  parentRefs:
    - group: gateway.networking.k8s.io
      kind: Gateway
      name: grafana-external
      sectionName: http
  rules:
    - filters:
        - type: RequestRedirect
          requestRedirect:
            scheme: https
            statusCode: 301
      matches:
        - path:
            type: PathPrefix
            value: /
 ---
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:
  name: grafana-external
  namespace: grafana
  labels:
    app.kubernetes.io/name: grafana
    app.kubernetes.io/instance: grafana
 spec:
  hostnames:
    - grafana.unkin.net
  parentRefs:
    - group: gateway.networking.k8s.io
      kind: Gateway
      name: grafana-external
      sectionName: https
  rules:
    - backendRefs:
        - group: ""
          kind: Service
          name: grafana-service
          port: 3000
          weight: 1
      matches:
        - path:
            type: PathPrefix
            value: /
@@ -18,6 +18,7 @@ resources:
  - dashboards/frr-ospf-route-metrics.yaml
  - dashboards/gitea.yaml
  - dashboards/haproxy.yaml
  - dashboards/k8up.yaml
  - dashboards/media-dashboard.yaml
  - dashboards/node-exporter-full.yaml
  - dashboards/nzbget.yaml
@@ -0,0 +1,16 @@
 ---
 apiVersion: gateway.networking.k8s.io/v1
 kind: BackendTLSPolicy
 metadata:
  name: haproxy
  namespace: haproxy
  labels:
    app: haproxy
 spec:
  targetRefs:
    - group: ""
      kind: Service
      name: haproxy
  validation:
    wellKnownCACertificates: System
    hostname: git.unkin.net
@@ -0,0 +1,256 @@
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: haproxy-config
  namespace: haproxy
 data:
  certificate.list: |
    # First entry is the default cert for non-matching SNI.
    /etc/haproxy/certs/unkin-net/tls.crt
    /etc/haproxy/certs/main-unkin-net/tls.crt
    /etc/haproxy/certs/ceph-unkin-net/tls.crt
  fe_https.map: |
    sonarr.main.unkin.net be_sonarr
    radarr.main.unkin.net be_radarr
    lidarr.main.unkin.net be_lidarr
    readarr.main.unkin.net be_readarr
    prowlarr.main.unkin.net be_prowlarr
    nzbget.main.unkin.net be_nzbget
    jellyfin.main.unkin.net be_jellyfin
    fafflix.unkin.net be_jellyfin
    git.unkin.net be_gitea
    dashboard.ceph.unkin.net be_ceph_dashboard
    auth.unkin.net be_k8s_kanidm
  haproxy.cfg: |
    global
      log stdout format raw local0
      log stdout format raw local1 notice
      maxconn  4000
      hard-stop-after  2m
      ssl-default-bind-ciphers  EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH
      ssl-default-bind-options  ssl-min-ver TLSv1.2 ssl-max-ver TLSv1.3
      ssl-default-server-ciphers  kEECDH+aRSA+AES:kRSA+AES:+AES256:RC4-SHA:!kEDH:!LOW:!EXP:!MD5:!aNULL:!eNULL
      ssl-default-server-options  no-sslv3
      stats  timeout 30s
      stats  socket /var/lib/haproxy/stats
      stats  socket /var/lib/haproxy/admin.sock mode 660 level admin
      tune.ssl.default-dh-param  2048
    defaults
      log  global
      maxconn  5000
      mode  http
      option  httplog
      option  dontlognull
      option  http-server-close
      option  forwardfor except 127.0.0.0/8
      option  redispatch
      retries  3
      stats  enable
      timeout  http-request 10s
      timeout  queue 1m
      timeout  connect 10s
      timeout  client 5m
      timeout  server 5m
      timeout  http-keep-alive 10s
      timeout  check 10s
    frontend fe_https
      bind 0.0.0.0:443 ssl crt-list /usr/local/etc/haproxy/certificate.list ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH force-tlsv12
      mode http
      description Global HTTPS Frontend
      http-request set-header X-Forwarded-Proto https
      http-request set-header X-Real-IP %[src]
      http-response set-header X-Content-Type-Options nosniff
      http-response set-header X-XSS-Protection 1;mode=block
      use_backend %[req.hdr(host),lower,map(/usr/local/etc/haproxy/fe_https.map,be_default)]
    frontend fe_metrics
      bind 0.0.0.0:8405
      mode http
      description Metrics Frontend
      http-request set-header X-Forwarded-Proto https
      http-request set-header X-Real-IP %[src]
      http-request use-service prometheus-exporter if { path /metrics }
    backend be_ceph_dashboard
      description Backend for Ceph Dashboard from Mgr instances
      balance roundrobin
      cookie SRVNAME insert indirect nocache
      http-check expect status 200
      http-request set-header X-Forwarded-Port %[dst_port]
      http-request add-header X-Forwarded-Proto https if { dst_port 9443 }
      http-reuse always
      option httpchk GET /
      option forwardfor
      option http-keep-alive
      option prefer-last-server
      redirect scheme https if !{ ssl_fc }
      stick-table type ip size 200k expire 30m
      server prodnxsr0009 198.18.23.9:9443 check cookie prodnxsr0009 fall 2 inter 2s rise 3 ssl verify none
      server prodnxsr0010 198.18.23.10:9443 check cookie prodnxsr0010 fall 2 inter 2s rise 3 ssl verify none
      server prodnxsr0011 198.18.23.11:9443 check cookie prodnxsr0011 fall 2 inter 2s rise 3 ssl verify none
      server prodnxsr0012 198.18.23.12:9443 check cookie prodnxsr0012 fall 2 inter 2s rise 3 ssl verify none
      server prodnxsr0013 198.18.23.13:9443 check cookie prodnxsr0013 fall 2 inter 2s rise 3 ssl verify none
    backend be_default
      description Backend for unmatched HTTP traffic
      balance roundrobin
      cookie SRVNAME insert
      http-request set-header X-Forwarded-Port %[dst_port]
      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
      option httpchk GET /
      option forwardfor
    backend be_gitea
      description Backend for gitea cluster
      balance roundrobin
      cookie SRVNAME insert indirect nocache
      http-request set-header X-Forwarded-Port %[dst_port]
      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
      http-reuse always
      option httpchk GET /
      option forwardfor
      option http-keep-alive
      option prefer-last-server
      redirect scheme https if !{ ssl_fc }
      stick on src
      stick-table type ip size 200k expire 30m
      server ausyd1nxvm2080 198.18.26.18:443 check cookie ausyd1nxvm2080 fall 2 inter 2s rise 3 ssl verify none
      server ausyd1nxvm2081 198.18.27.117:443 check cookie ausyd1nxvm2081 fall 2 inter 2s rise 3 ssl verify none
      server ausyd1nxvm2082 198.18.28.71:443 check cookie ausyd1nxvm2082 fall 2 inter 2s rise 3 ssl verify none
    backend be_jellyfin
      description Backend for au-syd1 jellyfin
      balance roundrobin
      cookie SRVNAME insert indirect nocache
      http-request set-header X-Forwarded-Port %[dst_port]
      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
      http-reuse always
      option httpchk GET /
      option forwardfor
      option http-keep-alive
      option prefer-last-server
      redirect scheme https if !{ ssl_fc }
      server ausyd1nxvm2051 198.18.25.164:443 check cookie ausyd1nxvm2051 fall 2 inter 2s rise 3 ssl verify none
    backend be_k8s_kanidm
      description Backend for Kanidm (auth.unkin.net via Kubernetes internal Traefik)
      balance roundrobin
      http-reuse always
      http-request set-header X-Forwarded-Port %[dst_port]
      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
      redirect scheme https if !{ ssl_fc }
      option httpchk
      option forwardfor
      option http-keep-alive
      option prefer-last-server
      http-check connect ssl sni auth.unkin.net
      http-check send meth GET uri /status ver HTTP/1.1 hdr Host auth.unkin.net
      http-check expect status 200
      server k8s-traefik-internal 198.18.200.4:443 ssl verify none check inter 2s rise 3 fall 2 sni str(auth.unkin.net)
    backend be_lidarr
      description Backend for au-syd1 lidarr
      balance roundrobin
      cookie SRVNAME insert indirect nocache
      http-request set-header X-Forwarded-Port %[dst_port]
      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
      http-reuse always
      option httpchk GET /consul/health
      option forwardfor
      option http-keep-alive
      option prefer-last-server
      redirect scheme https if !{ ssl_fc }
      server ausyd1nxvm2048 198.18.28.165:443 check cookie ausyd1nxvm2048 fall 2 inter 2s rise 3 ssl verify none
    backend be_nzbget
      description Backend for au-syd1 nzbget
      balance roundrobin
      cookie SRVNAME insert indirect nocache
      http-request set-header X-Forwarded-Port %[dst_port]
      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
      http-reuse always
      option httpchk GET /consul/health
      option forwardfor
      option http-keep-alive
      option prefer-last-server
      redirect scheme https if !{ ssl_fc }
      server ausyd1nxvm2045 198.18.25.44:443 check cookie ausyd1nxvm2045 fall 2 inter 2s rise 3 ssl verify none
    backend be_prowlarr
      description Backend for au-syd1 prowlarr
      balance roundrobin
      cookie SRVNAME insert indirect nocache
      http-request set-header X-Forwarded-Port %[dst_port]
      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
      http-reuse always
      option httpchk GET /consul/health
      option forwardfor
      option http-keep-alive
      option prefer-last-server
      redirect scheme https if !{ ssl_fc }
      server ausyd1nxvm2050 198.18.25.66:443 check cookie ausyd1nxvm2050 fall 2 inter 2s rise 3 ssl verify none
    backend be_radarr
      description Backend for au-syd1 radarr
      balance roundrobin
      cookie SRVNAME insert indirect nocache
      http-request set-header X-Forwarded-Port %[dst_port]
      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
      http-reuse always
      option httpchk GET /consul/health
      option forwardfor
      option http-keep-alive
      option prefer-last-server
      redirect scheme https if !{ ssl_fc }
      server ausyd1nxvm2047 198.18.27.131:443 check cookie ausyd1nxvm2047 fall 2 inter 2s rise 3 ssl verify none
    backend be_readarr
      description Backend for au-syd1 readarr
      balance roundrobin
      cookie SRVNAME insert indirect nocache
      http-request set-header X-Forwarded-Port %[dst_port]
      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
      http-reuse always
      option httpchk GET /consul/health
      option forwardfor
      option http-keep-alive
      option prefer-last-server
      redirect scheme https if !{ ssl_fc }
      server ausyd1nxvm2049 198.18.29.32:443 check cookie ausyd1nxvm2049 fall 2 inter 2s rise 3 ssl verify none
    backend be_sonarr
      description Backend for au-syd1 sonarr
      balance roundrobin
      cookie SRVNAME insert indirect nocache
      http-request set-header X-Forwarded-Port %[dst_port]
      http-request add-header X-Forwarded-Proto https if { dst_port 443 }
      http-reuse always
      option httpchk GET /consul/health
      option forwardfor
      option http-keep-alive
      option prefer-last-server
      redirect scheme https if !{ ssl_fc }
      server ausyd1nxvm2046 198.18.26.161:443 check cookie ausyd1nxvm2046 fall 2 inter 2s rise 3 ssl verify none
    # The `peers au-syd1-prod` section is dropped: peer names must be static and a
    # Deployment cannot provide them. Behind the external Traefik's TLS
    # passthrough `src` is a Traefik pod, so X-Real-IP, forwardfor and the
    # `stick on src` tables all key on that; the SRVNAME cookie carries real
    # session persistence. Traefik cannot emit PROXY protocol to a TLSRoute
    # backend, so there is nothing to bind `accept-proxy` to.
    listen health
      bind 0.0.0.0:8404
      mode http
      monitor-uri /healthz
    listen stats
      bind 127.0.0.1:9090
      mode http
      stats uri /
      stats auth admin:admin
@@ -0,0 +1,148 @@
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: haproxy
  namespace: haproxy
  annotations:
    reloader.stakater.com/auto: "true"
 spec:
  replicas: 3
  selector:
    matchLabels:
      app: haproxy
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  template:
    metadata:
      labels:
        app: haproxy
    spec:
      automountServiceAccountToken: false
      terminationGracePeriodSeconds: 150
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchLabels:
                  app: haproxy
              topologyKey: kubernetes.io/hostname
      securityContext:
        runAsNonRoot: true
        runAsUser: 99
        runAsGroup: 99
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: haproxy
          image: haproxy:3.2.24-alpine
          imagePullPolicy: IfNotPresent
          command:
            - haproxy
            - -W
            - -db
            - -f
            - /usr/local/etc/haproxy/haproxy.cfg
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop: [ALL]
              # fe_https binds the privileged port 443 as uid 99, and the
              # dst_port ACLs need the real port.
              add: [NET_BIND_SERVICE]
          ports:
            - name: https
              containerPort: 443
              protocol: TCP
            - name: health
              containerPort: 8404
              protocol: TCP
            - name: metrics
              containerPort: 8405
              protocol: TCP
            - name: stats
              containerPort: 9090
              protocol: TCP
          lifecycle:
            preStop:
              exec:
                # SIGUSR1 to the master soft-stops the workers; hard-stop-after
                # caps the drain. Wait so kubelet holds SIGTERM until it is done.
                command:
                  - /bin/sh
                  - -c
                  - kill -s USR1 1; while kill -0 1 2>/dev/null; do sleep 1; done
          livenessProbe:
            httpGet:
              path: /healthz
              port: health
            initialDelaySeconds: 15
            periodSeconds: 30
            timeoutSeconds: 5
            failureThreshold: 3
          readinessProbe:
            httpGet:
              path: /healthz
              port: health
            initialDelaySeconds: 5
            periodSeconds: 5
            timeoutSeconds: 5
            failureThreshold: 3
          resources:
            requests:
              cpu: 200m
              memory: 256Mi
            limits:
              cpu: 2
              memory: 1Gi
          volumeMounts:
            - name: config
              mountPath: /usr/local/etc/haproxy
              readOnly: true
            - name: cert-unkin-net
              mountPath: /etc/haproxy/certs/unkin-net
              readOnly: true
            - name: cert-main-unkin-net
              mountPath: /etc/haproxy/certs/main-unkin-net
              readOnly: true
            - name: cert-ceph-unkin-net
              mountPath: /etc/haproxy/certs/ceph-unkin-net
              readOnly: true
            - name: run
              mountPath: /var/lib/haproxy
      volumes:
        - name: config
          configMap:
            name: haproxy-config
        # ssl-load-extra-files loads <crtfile>.key by default, so the key is
        # projected next to the cert as tls.crt.key.
        - name: cert-unkin-net
          secret:
            secretName: wildcard-unkin-net-tls
            items:
              - key: tls.crt
                path: tls.crt
              - key: tls.key
                path: tls.crt.key
        - name: cert-main-unkin-net
          secret:
            secretName: wildcard-main-unkin-net-tls
            items:
              - key: tls.crt
                path: tls.crt
              - key: tls.key
                path: tls.crt.key
        - name: cert-ceph-unkin-net
          secret:
            secretName: wildcard-ceph-unkin-net-tls
            items:
              - key: tls.crt
                path: tls.crt
              - key: tls.key
                path: tls.crt.key
        - name: run
          emptyDir: {}
      restartPolicy: Always
@@ -0,0 +1,39 @@
 ---
 # External (DMZ) front for the haproxy edge on the traefik-external LB VIP
 # 198.18.199.0. Traefik terminates TLS so HTTP/2 connections coalesced from
 # other *.unkin.net hosts still route by Host; the hop to haproxy is re-encrypted via
 # BackendTLSPolicy. Routes carry the explicit hostname list.
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
  name: haproxy
  namespace: haproxy
  labels:
    traefik.io/instance: external
 spec:
  gatewayClassName: traefik-external
  listeners:
    - name: http
      port: 80
      protocol: HTTP
      allowedRoutes:
        namespaces:
          from: Same
    - name: https
      port: 443
      protocol: HTTPS
      tls:
        mode: Terminate
        certificateRefs:
          - group: ""
            kind: Secret
            name: wildcard-unkin-net-tls
          - group: ""
            kind: Secret
            name: wildcard-main-unkin-net-tls
          - group: ""
            kind: Secret
            name: wildcard-ceph-unkin-net-tls
      allowedRoutes:
        namespaces:
          from: Same
@@ -0,0 +1,69 @@
 ---
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:
  name: haproxy-http-redirect
  namespace: haproxy
  labels:
    app: haproxy
 spec:
  hostnames:
    - sonarr.main.unkin.net
    - radarr.main.unkin.net
    - lidarr.main.unkin.net
    - readarr.main.unkin.net
    - prowlarr.main.unkin.net
    - nzbget.main.unkin.net
    - jellyfin.main.unkin.net
    - fafflix.unkin.net
    - git.unkin.net
    - dashboard.ceph.unkin.net
    - auth.unkin.net
  parentRefs:
    - group: gateway.networking.k8s.io
      kind: Gateway
      name: haproxy
      sectionName: http
  rules:
    - filters:
        - type: RequestRedirect
          requestRedirect:
            scheme: https
            statusCode: 301
      matches:
        - path:
            type: PathPrefix
            value: /
 ---
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:
  name: haproxy
  namespace: haproxy
  labels:
    app: haproxy
 spec:
  hostnames:
    - sonarr.main.unkin.net
    - radarr.main.unkin.net
    - lidarr.main.unkin.net
    - readarr.main.unkin.net
    - prowlarr.main.unkin.net
    - nzbget.main.unkin.net
    - jellyfin.main.unkin.net
    - fafflix.unkin.net
    - git.unkin.net
    - dashboard.ceph.unkin.net
    - auth.unkin.net
  parentRefs:
    - group: gateway.networking.k8s.io
      kind: Gateway
      name: haproxy
      sectionName: https
  rules:
    - backendRefs:
        - group: ""
          kind: Service
          name: haproxy
          port: 443
          weight: 1
@@ -0,0 +1,15 @@
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - namespace.yaml
  - configmap.yaml
  - deployment.yaml
  - service.yaml
  - gateway.yaml
  - backendtlspolicy.yaml
  - httproute.yaml
  - pdb.yaml
  - vpa.yaml
  - vmpodscrape.yaml
@@ -0,0 +1,5 @@
 ---
 apiVersion: v1
 kind: Namespace
 metadata:
  name: haproxy
@@ -0,0 +1,11 @@
 ---
 apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:
  name: haproxy
  namespace: haproxy
 spec:
  maxUnavailable: 1
  selector:
    matchLabels:
      app: haproxy
@@ -0,0 +1,19 @@
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: haproxy
  namespace: haproxy
 spec:
  type: ClusterIP
  # Reached only by the external Traefik (re-encrypted HTTPRoute), so the
  # peer address here is a Traefik pod, not the client. sessionAffinity is
  # deliberately absent: keyed on ClientIP it would pin whole Traefik pods,
  # not clients. Backend persistence rests on the per-backend SRVNAME cookie.
  selector:
    app: haproxy
  ports:
    - name: https
      port: 443
      protocol: TCP
      targetPort: https
@@ -0,0 +1,13 @@
 ---
 apiVersion: operator.victoriametrics.com/v1beta1
 kind: VMPodScrape
 metadata:
  name: haproxy
  namespace: haproxy
 spec:
  selector:
    matchLabels:
      app: haproxy
  podMetricsEndpoints:
    - port: metrics
      path: /metrics
@@ -0,0 +1,13 @@
 ---
 apiVersion: autoscaling.k8s.io/v1
 kind: VerticalPodAutoscaler
 metadata:
  name: haproxy-vpa
  namespace: haproxy
 spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: haproxy
  updatePolicy:
    updateMode: "Off"
@@ -4,3 +4,4 @@ kind: Kustomization
 resources:
  - namespace.yaml
  - vmservicescrape.yaml
@@ -0,0 +1,15 @@
 ---
 # Scrape the k8up operator metrics (k8up-metrics Service, port http :8080).
 apiVersion: operator.victoriametrics.com/v1beta1
 kind: VMServiceScrape
 metadata:
  name: k8up
  namespace: k8up-system
 spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: k8up
      app.kubernetes.io/instance: k8up
  endpoints:
    - port: http
      path: /metrics
@@ -1,43 +0,0 @@
 ---
 # S3 bucket (Ceph RGW) for the long-term raw-log archive, provisioned by the
 # in-estate cephrgw-operator. The archiver Vector deployment writes here.
 apiVersion: ceph.unkin.net/v1alpha1
 kind: ObjectStoreUser
 metadata:
  name: logs-archive-owner
  namespace: logging
 spec:
  displayName: "Logging raw-archive bucket owner"
  maxBuckets: 5
  quota:
    enabled: true
    # 5 TiB soft cap; real retention is enforced RGW-side by a bucket lifecycle
    # policy (see PR notes) — the operator does not manage lifecycle.
    maxSizeBytes: 5497558138880
 ---
 apiVersion: ceph.unkin.net/v1alpha1
 kind: Bucket
 metadata:
  name: logs-archive
  namespace: logging
 spec:
  bucketName: logs-archive
  ownerRef: logs-archive-owner
  versioning: false
  tags:
    app: logging
    purpose: raw-log-archive
  # Keep the bucket (and its objects) if this CR is ever deleted.
  retainOnDelete: true
 ---
 apiVersion: ceph.unkin.net/v1alpha1
 kind: BucketAccess
 metadata:
  name: logs-archive-writer
  namespace: logging
 spec:
  bucketRef: logs-archive
  level: read-write
  # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
  # S3_ENDPOINT, BUCKET_NAME) into this Secret; the archiver consumes it.
  secretName: logs-archive-s3
@@ -1,97 +0,0 @@
 ---
 apiVersion: clickhouse.altinity.com/v1
 kind: ClickHouseInstallation
 metadata:
  name: logs
  namespace: logging
 spec:
  defaults:
    templates:
      dataVolumeClaimTemplate: data-volume
      serviceTemplate: chi-service
      podTemplate: clickhouse
  configuration:
    users:
      # Password hash is sourced from the Vault-synced clickhouse-credentials
      # Secret; the plaintext never lands in git or the ClickHouse config.
      vector/password_sha256_hex:
        valueFrom:
          secretKeyRef:
            name: clickhouse-credentials
            key: password_sha256_hex
      vector/networks/ip:
        - "::/0"
      vector/profile: default
      vector/quota: default
      # Allow the vector user to create the logs database/table (bootstrap Job)
      # and to INSERT. Restrict the built-in default user to loopback only.
      vector/access_management: "1"
      default/networks/ip:
        - "127.0.0.1"
        - "::1"
      # Read-only user for CLI tools + the logviewer UI. Hash sourced from the
      # Vault-synced clickhouse-logreader Secret, same mechanism as vector.
      # Scoped to the logs database only (unlike vector, which bootstraps it).
      logreader/password_sha256_hex:
        valueFrom:
          secretKeyRef:
            name: clickhouse-logreader
            key: password_sha256_hex
      logreader/networks/ip:
        - "::/0"
      logreader/profile: readonly
      logreader/quota: default
      logreader/allow_databases/database:
        - "logs"
    profiles:
      default/max_memory_usage: "10000000000"
      default/max_execution_time: "120"
      readonly/readonly: "2"
      readonly/max_memory_usage: "10000000000"
      readonly/max_execution_time: "120"
    clusters:
      - name: logs
        layout:
          shardsCount: 1
          replicasCount: 1
  templates:
    volumeClaimTemplates:
      - name: data-volume
        spec:
          storageClassName: cephrbd-fast-delete
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              # 3d TTL on logs.raw. At ~130 GiB/day raw, ClickHouse LZ4/ZSTD
              # (~6x on log text) stores ~20-25 GiB/day => ~60-75 GiB/3d, plus
              # merge headroom (~2x peak). logs.raw is the only table. 150Gi
              # gives comfortable headroom; long-term data lives in S3, not here.
              storage: 150Gi
    serviceTemplates:
      - name: chi-service
        generateName: "clickhouse-{chi}"
        spec:
          type: ClusterIP
          ports:
            - name: http
              port: 8123
            - name: tcp
              port: 9000
    podTemplates:
      - name: clickhouse
        spec:
          securityContext:
            fsGroup: 101
            runAsUser: 101
            runAsGroup: 101
          containers:
            - name: clickhouse
              image: docker.io/clickhouse/clickhouse-server:24.8
              resources:
                requests:
                  cpu: 500m
                  memory: 2Gi
                limits:
                  cpu: "2"
                  memory: 8Gi
@@ -1,40 +0,0 @@
 ---
 # logarchiver non-secret config. Secrets (NATS/S3/ClickHouse creds) and the
 # subject filter come from env; everything else uses the binary's built-in
 # defaults, which already target this stack. ack_wait MUST exceed batch.max_age
 # so unacked messages in an open batch are not redelivered mid-batch.
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: logarchiver-config
  namespace: logging
 data:
  config.yaml: |
    nats:
      ack_wait: 5m
      fetch_batch: 512
    batch:
      max_bytes: 67108864
      max_events: 200000
      max_age: 2m
    # Pin the proven RGW endpoint/bucket; ignore the secret's S3_ENDPOINT/BUCKET_NAME
    # (AWS creds still come from the secret env). endpoint_env/bucket_env off.
    s3:
      endpoint: "https://s3.ceph.unkin.net"
      bucket: "logs-archive"
      region: "us-east-1"
      path_style: true
      ca_file: /etc/vault-ca/ca.crt
      endpoint_env: ""
      bucket_env: ""
    crypto:
      key_name: logarchive
      pubkey_source: vault
      vault:
        address: "https://vault.service.consul:8200"
        mount: gpg
        auth_method: kubernetes
        k8s_mount: k8s/au/syd1
        k8s_role: logging_logarchiver
        k8s_jwt_path: /var/run/secrets/vault/token
        ca_file: /etc/vault-ca/ca.crt
--- a/Show More
+++ b/Show More