logging: set clickhouse readonly profile to readonly=2 (#409 )

## Why The `readonly` profile used by `logreader` sets `readonly=1`, which rejects any per-query settings change. HTTP clients (logviewer, chlog) send settings params like `max_execution_time` with their queries, so every query fails with `Code: 164 DB::Exception: Cannot modify 'max_execution_time' setting in readonly mode` — logviewer healthz returns 503 and CrashLoops, chlog queries fail. `readonly=2` keeps data access read-only while permitting settings changes, which is what HTTP clients need. ## Changes - Changes the `readonly` profile in `apps/base/logging/clickhouseinstallation.yaml` from `readonly: "1"` to `readonly: "2"` Reviewed-on: #409 Co-authored-by: unkin-agent <unkin-agent@unkin.net> Co-committed-by: unkin-agent <unkin-agent@unkin.net>
arrstack: raise arrstack-postgres resources after OOMKills (#408 )
2026-08-23 22:36:52 +10:00 · 2026-08-23 22:01:25 +10:00 · 2026-08-23 21:56:45 +10:00 · 2026-08-23 21:38:43 +10:00 · 2026-08-23 21:38:27 +10:00 · 2026-08-23 21:38:06 +10:00
83 changed files with 2563 additions and 191 deletions
@@ -7,7 +7,7 @@ metadata:
  annotations:
    # Wave 2: serve only after the wave-1 migrate Job completes.
    argocd.argoproj.io/sync-wave: "2"
-    secret.reloader.stakater.com/reload: "arrproxy-pepper,arrproxy-db-app,sonarr-apikey,radarr-apikey,prowlarr-apikey"
+    secret.reloader.stakater.com/reload: "arrproxy-pepper,arrproxy-admin-token,arrproxy-db-app,sonarr-apikey,radarr-apikey,prowlarr-apikey"
 spec:
  replicas: 2
  selector:
@@ -33,7 +33,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: api
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/arrproxy-api:v0.1.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/arrproxy-api:v0.3.0
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 8080
@@ -59,6 +59,15 @@ spec:
                secretKeyRef:
                  name: arrproxy-pepper
                  key: pepper
+            # Machine-mint admin bearer, synced from Vault KV via the
+            # arrproxy-admin-token VSO. Gates the /api/admin/ route that
+            # oauth2-proxy intentionally skip-auths so OpenBao on the VMs can
+            # mint tokens against arrproxy's own bearer.
+            - name: ARRPROXY_ADMIN_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: arrproxy-admin-token
+                  key: token
            # DSN assembled from the CNPG-generated arrproxy-db-app Secret;
            # $(VAR) expansion resolves the two env entries defined above it.
            - name: ARRPROXY_DB_USER
@@ -33,10 +33,20 @@ data:
  # set-xauthrequest is intentionally NOT set -- it only populates auth_request
  # *response* headers, which never reach an --upstreams-proxied backend.
  OAUTH2_PROXY_PASS_USER_HEADERS: "true"
-  # Bypass auth ONLY for the *arr proxy API (/<app>/api...): this matches
-  # /sonarr/api but NOT /api/tokens or /api/me (which stay authenticated).
-  OAUTH2_PROXY_SKIP_AUTH_REGEX: "^/[^/]+/api"
+  # Bypass auth for the *arr proxy API (/<app>/api...) and the machine-mint
+  # admin route (/api/admin/...). The first matches /sonarr/api; the second
+  # matches /api/admin/ only -- both routed to the arrproxy-api upstream by the
+  # catch-all /api/ prefix above. /api/admin/ is protected by arrproxy's OWN
+  # ARRPROXY_ADMIN_TOKEN bearer (OpenBao on the VMs reaches it via the ingress),
+  # so it is intentionally oauth-skipped. /api/tokens and /api/me are NOT
+  # matched and stay oauth-authenticated.
+  OAUTH2_PROXY_SKIP_AUTH_REGEX: "^/[^/]+/api,^/api/admin/"
  OAUTH2_PROXY_EMAIL_DOMAINS: "*"
+  # Authentik hardcodes email_verified=false in the id_token; without this
+  # oauth2-proxy rejects the session ("email ... isn't verified") -> 500 on
+  # /oauth2/callback. Authorization is enforced downstream via ak_groups, so
+  # accepting the unverified email here is safe.
+  OAUTH2_PROXY_INSECURE_OIDC_ALLOW_UNVERIFIED_EMAIL: "true"
  OAUTH2_PROXY_COOKIE_SECURE: "true"
  OAUTH2_PROXY_COOKIE_DOMAINS: "arrstack.unkin.net"
  OAUTH2_PROXY_WHITELIST_DOMAINS: "arrstack.unkin.net"
@@ -62,7 +62,7 @@ spec:
              memory: 64Mi
      containers:
        - name: oauth2-proxy
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/oauth2-proxy:v7.15.3
+          image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.3
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 4180
@@ -31,7 +31,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: ui
-          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/arrproxy-ui:v0.1.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/arrproxy-ui:v0.3.0
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 8080
@@ -25,6 +25,33 @@ spec:
  type: kv-v2
  vaultAuthRef: default
 ---
+# Machine-mint admin bearer token. Seeded (openssl rand) at
+# kv/kubernetes/namespace/arrstack/default/arrproxy-admin-token (key: token) and
+# shared as the source of truth with the future Vault engine. The default k8s
+# role's templated policy already grants read on
+# kv/data/kubernetes/namespace/{{sa_namespace}}/{{sa_name}}/* for the
+# arrstack/default ServiceAccount, so no terraform-vault change is needed. VSO
+# syncs it into the arrproxy-admin-token Secret consumed by arrproxy-api as
+# ARRPROXY_ADMIN_TOKEN to gate the bearer-protected /api/admin/ route.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: arrproxy-admin-token
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  destination:
+    create: true
+    name: arrproxy-admin-token
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/arrstack/default/arrproxy-admin-token
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
 # Authentik OIDC client for the arrstack front door (client_id, client_secret,
 # cookie_secret), created by terraform-authentik at
 # kv/kubernetes/namespace/arrstack/default/oauth-credentials. VSO syncs it into
@@ -0,0 +1,42 @@
+---
+# Shared Ceph RGW (S3) bucket for arrstack application backups (the periodic
+# radarr/sonarr/prowlarr config+database zip backups), provisioned by the
+# in-estate cephrgw-operator. Backups otherwise land on each replica's local
+# /config (an emptyDir) and are lost when that pod is rescheduled; routing them
+# to S3 lets any replica write and restore them.
+#
+# Backups are kept in their own bucket (not the arrstack-media one) because they
+# carry secrets (Config.xml holds API keys, the DB has all app state) and thus
+# warrant separate credentials, lifecycle and retention from the public-ish
+# poster art. The three apps share this one bucket, isolated by a per-app key
+# prefix (radarr/, sonarr/, prowlarr/) set via <App>__BackupS3__Prefix.
+#
+# The operator mints the S3 credential Secret (arrstack-backups-s3) in this
+# namespace with keys AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and S3_ENDPOINT,
+# so no Vault KV seeding is required.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: arrstack-backups
+  namespace: arrstack
+spec:
+  displayName: "arrstack backups owner"
+  uid: arrstack-backups
+  maxBuckets: 5
+  secretName: arrstack-backups-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: arrstack-backups
+  namespace: arrstack
+spec:
+  placementTarget: ec
+  bucketName: arrstack-backups
+  ownerRef: arrstack-backups
+  versioning: false
+  tags:
+    app: arrstack
+    purpose: backups
+  retainOnDelete: true
@@ -7,10 +7,17 @@ resources:
  - vaultauth.yaml
  - pv-media-tv.yaml
  - pv-media-movies.yaml
+  - pv-mediafs.yaml
  - pvc-media-tv.yaml
  - pvc-media-movies.yaml
+  - pvc-mediafs.yaml
+  - media-bucket.yaml
+  - backups-bucket.yaml
+  - postgres
+  - valkey
  - sonarr
  - radarr
  - prowlarr
  - nzbget
  - arrproxy
+  - mediamover
@@ -0,0 +1,34 @@
+---
+# Shared Ceph RGW (S3) bucket for arrstack media assets (Servarr MediaCover
+# posters/fanart and, later, application backups), provisioned by the in-estate
+# cephrgw-operator. Moving these off each replica's local /config (an emptyDir)
+# lets any radarr/sonarr/prowlarr replica serve covers and store backups without
+# leader-local state. The operator mints the S3 credential Secret
+# (arrstack-media-s3) in this namespace with keys AWS_ACCESS_KEY_ID,
+# AWS_SECRET_ACCESS_KEY and S3_ENDPOINT, so no Vault KV seeding is required.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: arrstack-media
+  namespace: arrstack
+spec:
+  displayName: "arrstack media assets owner"
+  uid: arrstack-media
+  maxBuckets: 5
+  secretName: arrstack-media-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: arrstack-media
+  namespace: arrstack
+spec:
+  placementTarget: ec
+  bucketName: arrstack-media
+  ownerRef: arrstack-media
+  versioning: false
+  tags:
+    app: arrstack
+    purpose: media-assets
+  retainOnDelete: true
@@ -0,0 +1,95 @@
+---
+# mediamover server: REST API + UI on :8080. Spawns one worker Job per queued
+# file using this same image with `worker` args (Job spec lives in code; workers
+# run as the default ServiceAccount and mount the same PVCs). The queue is
+# in-memory, so keep a single replica; a restart just loses queued entries.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mediamover
+  namespace: arrstack
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mediamover
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: mediamover
+    spec:
+      serviceAccountName: mediamover
+      automountServiceAccountToken: true
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: server
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/mediamover:v0.1.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - server
+            - --src-root=/srv/src
+            - --src-pvc=mediafs
+            - --dst-roots=movies=/srv/dst/movies,tv=/srv/dst/tv
+            - --dst-pvc=movies=media-movies,tv=media-tv
+            - --namespace=arrstack
+            - --image=artifactapi.k8s.syd1.au.unkin.net/docker-internal/mediamover:v0.1.0
+          ports:
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+          volumeMounts:
+            # RW: move mode deletes the source file after a successful copy.
+            - name: src
+              mountPath: /srv/src
+            - name: dst-movies
+              mountPath: /srv/dst/movies
+            - name: dst-tv
+              mountPath: /srv/dst/tv
+          livenessProbe:
+            httpGet:
+              path: /api/limit
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /api/limit
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+      volumes:
+        - name: src
+          persistentVolumeClaim:
+            claimName: mediafs
+        - name: dst-movies
+          persistentVolumeClaim:
+            claimName: media-movies
+        - name: dst-tv
+          persistentVolumeClaim:
+            claimName: media-tv
+      restartPolicy: Always
@@ -0,0 +1,40 @@
+---
+# Internal front for mediamover. The existing arrstack (arrproxy) Gateway is
+# external and hostname-locked to arrstack.unkin.net, so this tool gets its own
+# internal Gateway following the cluster convention (cf. pdbmux).
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: mediamover.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    external-dns.alpha.kubernetes.io/hostname: mediamover.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+  name: mediamover
+  namespace: arrstack
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: mediamover.k8s.syd1.au.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: mediamover.k8s.syd1.au.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: mediamover-tls
+        mode: Terminate
@@ -0,0 +1,49 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: mediamover-http-redirect
+  namespace: arrstack
+spec:
+  hostnames:
+    - mediamover.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: mediamover
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: mediamover
+  namespace: arrstack
+spec:
+  hostnames:
+    - mediamover.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: mediamover
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: mediamover
+          port: 8080
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
@@ -0,0 +1,11 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - serviceaccount.yaml
+  - rbac.yaml
+  - deployment.yaml
+  - service.yaml
+  - gateway.yaml
+  - httproute.yaml
@@ -0,0 +1,48 @@
+---
+# The server creates one worker Job per queued file and polls Job/Pod state to
+# track progress and clean up.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mediamover
+  namespace: arrstack
+rules:
+  - apiGroups:
+      - batch
+    resources:
+      - jobs
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+      - delete
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods/log
+    verbs:
+      - get
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mediamover
+  namespace: arrstack
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mediamover
+subjects:
+  - kind: ServiceAccount
+    name: mediamover
+    namespace: arrstack
@@ -0,0 +1,19 @@
+---
+# Must stay named "mediamover" on port 8080: worker Jobs default their progress
+# callback URL to http://mediamover.<namespace>.svc:8080.
+apiVersion: v1
+kind: Service
+metadata:
+  name: mediamover
+  namespace: arrstack
+spec:
+  internalTrafficPolicy: Cluster
+  ports:
+    - name: http
+      port: 8080
+      protocol: TCP
+      targetPort: http
+  selector:
+    app: mediamover
+  sessionAffinity: None
+  type: ClusterIP
@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mediamover
+  namespace: arrstack
@@ -0,0 +1,45 @@
+---
+# Ceph RGW (S3) backup target for the shared arrstack CNPG cluster, provisioned
+# by the in-estate cephrgw-operator: one dedicated bucket + owner user. CNPG
+# reads the S3 credential Secret (cnpg-arrstack-backup-s3) from this namespace.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: cnpg-arrstack-backup
+  namespace: arrstack
+spec:
+  displayName: "CNPG backup owner (arrstack)"
+  uid: cnpg-arrstack-backup
+  maxBuckets: 5
+  secretName: cnpg-arrstack-backup-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: cnpg-arrstack
+  namespace: arrstack
+spec:
+  placementTarget: ec
+  bucketName: cnpg-arrstack
+  ownerRef: cnpg-arrstack-backup
+  versioning: false
+  tags:
+    app: arrstack
+    purpose: cnpg-backup
+  retainOnDelete: true
+---
+# Nightly base backup on top of always-on WAL archiving. Staggered from the
+# other CNPG clusters (6-field cron, seconds first).
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-arrstack-nightly
+  namespace: arrstack
+spec:
+  schedule: "0 45 3 * * *"
+  immediate: false
+  backupOwnerReference: self
+  method: barmanObjectStore
+  cluster:
+    name: arrstack-postgres
@@ -0,0 +1,161 @@
+---
+# Shared PostgreSQL backend for the -unkin2 fork sonarr/radarr/prowlarr, whose
+# Npgsql/EF Core provider moves each *arr off SQLite into Postgres and makes the
+# shared-nothing, active-active multi-replica deployment possible. One cluster,
+# one throwaway initdb owner ("app"), and three managed login roles — one per
+# app — each with its own per-app database (see database-*.yaml). Role passwords
+# come from the VSO-synced <app>-db Secrets (vaultstaticsecret.yaml), so no
+# credential is rendered into git.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: arrstack-postgres
+  namespace: arrstack
+spec:
+  inheritedMetadata:
+    annotations:
+      k8up.io/backup: "false"
+  affinity:
+    podAntiAffinityType: preferred
+  backup:
+    retentionPolicy: 30d
+    barmanObjectStore:
+      destinationPath: s3://cnpg-arrstack
+      endpointURL: https://s3.ceph.unkin.net
+      endpointCA:
+        name: vault-ca-cert
+        key: ca.crt
+      s3Credentials:
+        accessKeyId:
+          name: cnpg-arrstack-backup-s3
+          key: AWS_ACCESS_KEY_ID
+        secretAccessKey:
+          name: cnpg-arrstack-backup-s3
+          key: AWS_SECRET_ACCESS_KEY
+      serverName: arrstack
+      data:
+        compression: bzip2
+        jobs: 2
+      wal:
+        compression: zstd
+        maxParallel: 2
+  bootstrap:
+    initdb:
+      # Throwaway owner + database: the real per-app databases are provisioned by
+      # the CNPG Database CRDs, owned by the managed roles below.
+      database: app
+      encoding: UTF8
+      localeCType: C
+      localeCollate: C
+      owner: app
+  managed:
+    roles:
+      - name: sonarr
+        ensure: present
+        comment: Sonarr application role (owns sonarr-main)
+        login: true
+        superuser: false
+        createdb: false
+        createrole: false
+        inherit: true
+        replication: false
+        connectionLimit: -1
+        passwordSecret:
+          name: sonarr-db
+      - name: radarr
+        ensure: present
+        comment: Radarr application role (owns radarr-main)
+        login: true
+        superuser: false
+        createdb: false
+        createrole: false
+        inherit: true
+        replication: false
+        connectionLimit: -1
+        passwordSecret:
+          name: radarr-db
+      - name: prowlarr
+        ensure: present
+        comment: Prowlarr application role (owns prowlarr-main)
+        login: true
+        superuser: false
+        createdb: false
+        createrole: false
+        inherit: true
+        replication: false
+        connectionLimit: -1
+        passwordSecret:
+          name: prowlarr-db
+  enablePDB: true
+  enableSuperuserAccess: false
+  failoverDelay: 0
+  imageName: ghcr.io/cloudnative-pg/postgresql:17-system-trixie
+  instances: 3
+  logLevel: info
+  maxSyncReplicas: 0
+  minSyncReplicas: 0
+  monitoring:
+    customQueriesConfigMap:
+      - key: queries
+        name: cnpg-default-monitoring
+    disableDefaultQueries: false
+    enablePodMonitor: false
+  postgresql:
+    parameters:
+      archive_mode: "on"
+      archive_timeout: 5min
+      dynamic_shared_memory_type: posix
+      effective_cache_size: 256MB
+      full_page_writes: "on"
+      log_destination: csvlog
+      log_directory: /controller/log
+      log_filename: postgres
+      log_rotation_age: "0"
+      log_rotation_size: "0"
+      log_truncate_on_rotation: "false"
+      logging_collector: "on"
+      max_connections: "200"
+      max_parallel_workers: "16"
+      max_replication_slots: "16"
+      max_worker_processes: "16"
+      shared_buffers: 128MB
+      shared_memory_type: mmap
+      ssl_max_protocol_version: TLSv1.3
+      ssl_min_protocol_version: TLSv1.3
+      wal_keep_size: 256MB
+      wal_level: logical
+      wal_log_hints: "on"
+      wal_receiver_timeout: 5s
+      wal_sender_timeout: 5s
+    syncReplicaElectionConstraint:
+      enabled: false
+  primaryUpdateMethod: restart
+  primaryUpdateStrategy: unsupervised
+  probes:
+    liveness:
+      isolationCheck:
+        connectionTimeout: 1000
+        enabled: true
+        requestTimeout: 1000
+  replicationSlots:
+    highAvailability:
+      enabled: true
+      slotPrefix: _cnpg_
+    synchronizeReplicas:
+      enabled: true
+    updateInterval: 30
+  resources:
+    limits:
+      cpu: "1"
+      memory: 2Gi
+    requests:
+      cpu: 250m
+      memory: 1Gi
+  smartShutdownTimeout: 180
+  startDelay: 3600
+  stopDelay: 1800
+  storage:
+    resizeInUseVolumes: true
+    size: 10Gi
+    storageClass: cephrbd-fast-delete
+  switchoverDelay: 3600
@@ -0,0 +1,15 @@
+---
+# Per-app database owned by the prowlarr managed role. The fork's provider runs
+# its own schema migrations on first start (advisory-locked, so only one replica
+# migrates). retain: the database survives a Database CRD delete.
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: prowlarr-main
+  namespace: arrstack
+spec:
+  cluster:
+    name: arrstack-postgres
+  name: prowlarr-main
+  owner: prowlarr
+  databaseReclaimPolicy: retain
@@ -0,0 +1,15 @@
+---
+# Per-app database owned by the radarr managed role. The fork's provider runs its
+# own schema migrations on first start (advisory-locked, so only one replica
+# migrates). retain: the database survives a Database CRD delete.
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: radarr-main
+  namespace: arrstack
+spec:
+  cluster:
+    name: arrstack-postgres
+  name: radarr-main
+  owner: radarr
+  databaseReclaimPolicy: retain
@@ -0,0 +1,15 @@
+---
+# Per-app database owned by the sonarr managed role. The fork's provider runs its
+# own schema migrations on first start (advisory-locked, so only one replica
+# migrates). retain: the database survives a Database CRD delete.
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: sonarr-main
+  namespace: arrstack
+spec:
+  cluster:
+    name: arrstack-postgres
+  name: sonarr-main
+  owner: sonarr
+  databaseReclaimPolicy: retain
@@ -0,0 +1,11 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - vaultstaticsecret.yaml
+  - cnpg_cluster.yaml
+  - cnpg_backup.yaml
+  - database-sonarr.yaml
+  - database-radarr.yaml
+  - database-prowlarr.yaml
@@ -0,0 +1,60 @@
+---
+# Per-app Postgres role credentials. Each is seeded out-of-band at
+# kv/kubernetes/namespace/arrstack/default/<app>-db (keys: username, password);
+# the default k8s role's templated policy already grants read on
+# kv/data/kubernetes/namespace/{{sa_namespace}}/{{sa_name}}/* for the
+# arrstack/default ServiceAccount, so no terraform-vault change is needed. VSO
+# syncs each into the <app>-db Secret, which is both the CNPG managed role's
+# passwordSecret (cnpg_cluster.yaml) and the source of the app Deployment's
+# <App>__Postgres__User/__Password env. Wave 0: must exist before the Cluster
+# (wave 1) reconciles the roles.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: sonarr-db
+  namespace: arrstack
+spec:
+  destination:
+    create: true
+    name: sonarr-db
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/arrstack/default/sonarr-db
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: radarr-db
+  namespace: arrstack
+spec:
+  destination:
+    create: true
+    name: radarr-db
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/arrstack/default/radarr-db
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: prowlarr-db
+  namespace: arrstack
+spec:
+  destination:
+    create: true
+    name: prowlarr-db
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/arrstack/default/prowlarr-db
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
@@ -0,0 +1,33 @@
+---
+# Non-secret env for the -unkin2 fork. The fork reads Servarr config from
+# Prowlarr__<Section>__<Key> env (no config.xml edits, no s6/PUID). Postgres
+# wiring points every replica at the same shared DB (arrstack-postgres-rw /
+# prowlarr-main); Auth__Method=External defers UI auth to arrproxy/oauth2-proxy;
+# Server__UrlBase keeps the /prowlarr prefix so arrproxy path-routing works;
+# App__InstanceName is identical across replicas (shared session-cookie name).
+# User/Password/ApiKey come from Secrets (see deployment.yaml), not here.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prowlarr-env
+  namespace: arrstack
+data:
+  Prowlarr__Postgres__Host: arrstack-postgres-rw.arrstack.svc.cluster.local
+  Prowlarr__Postgres__Port: "5432"
+  Prowlarr__Postgres__MainDb: prowlarr-main
+  Prowlarr__Log__DbEnabled: "false"
+  Prowlarr__Auth__Method: External
+  Prowlarr__Auth__Required: DisabledForLocalAddresses
+  Prowlarr__App__InstanceName: Prowlarr
+  Prowlarr__Server__Port: "9696"
+  Prowlarr__Server__UrlBase: /prowlarr
+  Prowlarr__Update__Mechanism: External
+  # Shared arrstack Valkey (valkey-operator). Setting Host is what activates the
+  # fork's #14 Redis features (SignalR backplane, cross-replica cache-invalidation
+  # bus, distributed rate limiter): RedisOptions.IsConfigured gates purely on a
+  # non-empty Host, so there is no separate Enabled flag. The operator leaves the
+  # default user passwordless (jellyfin parity), so no Password/Ssl is wired.
+  # Channels/keys are namespaced by this fork's prowlarr:ratelimit: prefix, so the
+  # one cluster is safe to share with sonarr/radarr.
+  Prowlarr__Redis__Host: valkey-arrstack-valkey.arrstack.svc.cluster.local
+  Prowlarr__Redis__Port: "6379"
@@ -4,11 +4,21 @@ kind: Deployment
 metadata:
  name: prowlarr
  namespace: arrstack
+  annotations:
+    # prowlarr-env is a plain (unhashed) ConfigMap consumed by fixed-name envFrom,
+    # so editing it does not roll the Deployment on its own. Reloader watches the
+    # referenced ConfigMap and triggers a rolling restart on change, so adding the
+    # Redis env activates the #14 features on the next ArgoCD sync without a manual
+    # `rollout restart`.
+    configmap.reloader.stakater.com/auto: "true"
 spec:
-  replicas: 1
+  # Active-active: the -unkin2 fork keeps all state in the shared Postgres
+  # (arrstack-postgres) and coordinates via Postgres advisory locks, so N
+  # replicas run concurrently behind the prowlarr Service. RollingUpdate is safe
+  # — no SQLite, no RWO lock.
+  replicas: 3
  strategy:
-    # RWO config PVC + single stateful SQLite DB: never run two pods at once.
-    type: Recreate
+    type: RollingUpdate
  selector:
    matchLabels:
      app: prowlarr
@@ -18,74 +28,114 @@ spec:
        app: prowlarr
    spec:
      securityContext:
+        # Fork image has no USER (runs as root by default); pin it to a non-root
+        # UID and group-write the shared RWX CephFS /config. OnRootMismatch
+        # avoids a recursive chown of the whole volume.
+        runAsUser: 1000
+        runAsGroup: 1000
        fsGroup: 1000
        fsGroupChangePolicy: OnRootMismatch
      initContainers:
-        # Enforce the Vault-sourced API key in /config/config.xml before the app
-        # starts. Vault is source of truth (override bootstrap): the key is minted
-        # in Vault, synced by VSO into the prowlarr-apikey Secret, and written here.
-        # Runs as root to fix ownership; touches only the <ApiKey> element.
-        - name: apikey-init
-          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/busybox:1.37.0
-          imagePullPolicy: IfNotPresent
-          securityContext:
-            runAsUser: 0
+        # Gate the app on its own Postgres database+role being reachable, instead
+        # of relying on ArgoCD sync-waves (which deadlock if apps aren't Healthy).
+        # waitfordb reads the PG* env as a libpq fallback, so the password never lands in argv.
+        - name: wait-for-db
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/waitfordb:v0.1.0
          env:
-            - name: API_KEY
+            - name: WAITFORDB_TIMEOUT
+              value: 5m
+            - name: WAITFORDB_SSLMODE
+              value: disable
+            - name: PGHOST
+              value: arrstack-postgres-rw.arrstack.svc.cluster.local
+            - name: PGPORT
+              value: "5432"
+            - name: PGDATABASE
+              value: prowlarr-main
+            - name: PGUSER
              valueFrom:
                secretKeyRef:
-                  name: prowlarr-apikey
-                  key: apitoken
-          command:
-            - sh
-            - -c
-            - |
-              set -eu
-              case "$API_KEY" in
-                "" | *[!0-9a-fA-F]*)
-                  echo "apikey-init: API_KEY missing or not hex; refusing" >&2
-                  exit 1
-                  ;;
-              esac
-              CFG=/config/config.xml
-              if [ ! -f "$CFG" ]; then
-                printf '<Config>\n  <ApiKey>%s</ApiKey>\n</Config>\n' "$API_KEY" > "$CFG"
-              elif grep -q '<ApiKey>' "$CFG"; then
-                sed -i "s|<ApiKey>[^<]*</ApiKey>|<ApiKey>${API_KEY}</ApiKey>|" "$CFG"
-              else
-                sed -i "s|<Config>|<Config>\n  <ApiKey>${API_KEY}</ApiKey>|" "$CFG"
-              fi
-              chown 1000:1000 "$CFG"
-              chmod 600 "$CFG"
-              echo "apikey-init: <ApiKey> enforced from Vault"
+                  name: prowlarr-db
+                  key: username
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: prowlarr-db
+                  key: password
          resources:
            requests:
-              cpu: 50m
+              cpu: 10m
              memory: 32Mi
            limits:
-              cpu: 200m
+              cpu: 100m
              memory: 64Mi
-          volumeMounts:
-            - name: config
-              mountPath: /config
      containers:
        - name: prowlarr
-          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/linuxserver/prowlarr:2.5.2
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/prowlarr:v2.6.2-unkin6
          imagePullPolicy: IfNotPresent
+          command:
+            - /app/Prowlarr
+          args:
+            - -nobrowser
+            - -data=/config
+            # Required: bypass the single-instance guard so multiple replicas
+            # can share one /config. Cross-replica safety is the Postgres layer,
+            # not a local lock file.
+            - -nosingleinstancecheck
          ports:
            - name: http
              containerPort: 9696
              protocol: TCP
+          envFrom:
+            - configMapRef:
+                name: prowlarr-env
          env:
-            - name: PUID
-              value: "1000"
-            - name: PGID
-              value: "1000"
-            - name: TZ
-              value: Australia/Sydney
+            - name: Prowlarr__Postgres__User
+              valueFrom:
+                secretKeyRef:
+                  name: prowlarr-db
+                  key: username
+            - name: Prowlarr__Postgres__Password
+              valueFrom:
+                secretKeyRef:
+                  name: prowlarr-db
+                  key: password
+            - name: Prowlarr__Auth__ApiKey
+              valueFrom:
+                secretKeyRef:
+                  name: prowlarr-apikey
+                  key: apitoken
+            # Backup object store (shared arrstack-backups Ceph RGW bucket,
+            # per-app key prefix). Routes the periodic config+DB zip backups off
+            # the ephemeral /config so any replica can write and restore them.
+            # Consumed by the -unkin3+ image; older images ignore these unknown
+            # config keys. Creds Secret minted by cephrgw-operator.
+            - name: Prowlarr__BackupS3__Endpoint
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: S3_ENDPOINT
+            - name: Prowlarr__BackupS3__AccessKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: AWS_ACCESS_KEY_ID
+            - name: Prowlarr__BackupS3__SecretKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: Prowlarr__BackupS3__Bucket
+              value: arrstack-backups
+            - name: Prowlarr__BackupS3__Prefix
+              value: prowlarr
+            - name: Prowlarr__BackupS3__ForcePathStyle
+              value: "true"
+            - name: Prowlarr__BackupS3__CaCertPath
+              value: /etc/ssl/vault-ca/ca.crt
          livenessProbe:
            httpGet:
-              path: /ping
+              path: /prowlarr/ping
              port: http
            initialDelaySeconds: 30
            periodSeconds: 30
@@ -93,7 +143,7 @@ spec:
            failureThreshold: 3
          readinessProbe:
            httpGet:
-              path: /ping
+              path: /prowlarr/ping
              port: http
            initialDelaySeconds: 10
            periodSeconds: 10
@@ -105,11 +155,20 @@ spec:
              memory: 256Mi
            limits:
              cpu: "1"
-              memory: 512Mi
+              memory: 1Gi
          volumeMounts:
            - name: config
              mountPath: /config
+            - name: vault-ca
+              mountPath: /etc/ssl/vault-ca
+              readOnly: true
      volumes:
        - name: config
-          persistentVolumeClaim:
-            claimName: prowlarr-config
+          emptyDir: {}
+        # Estate CA for validating the Ceph RGW (s3.ceph.unkin.net) TLS cert.
+        - name: vault-ca
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
@@ -3,8 +3,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization

 resources:
-  - pvc-config.yaml
  - vaultstaticsecret.yaml
+  - configmap.yaml
  - deployment.yaml
  - service.yaml
  - gateway.yaml
@@ -1,6 +1,7 @@
 ---
-# Prowlarr config + SQLite DB. RWO on cephrbd (block) — the arr apps' SQLite
-# does not tolerate CephFS locking. Retain: this is state.
+# Prowlarr /config. RWX on CephFS so all replicas share it (the -unkin2 fork
+# keeps the database in Postgres; /config now holds only config.xml + assets,
+# which tolerate — and want — shared access). Retain: this is state.
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -8,9 +9,9 @@ metadata:
  namespace: arrstack
 spec:
  accessModes:
-    - ReadWriteOnce
+    - ReadWriteMany
  resources:
    requests:
      storage: 5Gi
-  storageClassName: cephrbd-fast-retain
+  storageClassName: cephfs-raid5-retain
  volumeMode: Filesystem
@@ -0,0 +1,31 @@
+---
+# Static PV exposing the legacy mediafs CephFS filesystem root for the upcoming
+# mediamover tool. clusterID only selects the monitor set from the csi config;
+# fsName selects the actual filesystem, so the ssd_ec clusterID still reaches
+# mediafs on the same cluster. Staged with the dedicated ceph-mediafs client.
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: arrstack-mediafs
+spec:
+  capacity:
+    storage: 10Ti
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: ""
+  volumeMode: Filesystem
+  claimRef:
+    namespace: arrstack
+    name: mediafs
+  csi:
+    driver: cephfs.csi.ceph.com
+    volumeHandle: arrstack-mediafs-static
+    nodeStageSecretRef:
+      name: ceph-mediafs-secret
+      namespace: csi-cephfs
+    volumeAttributes:
+      staticVolume: "true"
+      clusterID: cephfs_csi_ssd_ec_4_1
+      fsName: mediafs
+      rootPath: /
@@ -0,0 +1,20 @@
+---
+# Legacy mediafs filesystem root, statically bound to the arrstack-mediafs PV
+# for the upcoming mediamover tool. storageClassName "" + volumeName disables
+# dynamic provisioning and binds the pre-created static PV.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: mediafs
+  namespace: arrstack
+  annotations:
+    k8up.io/backup: "false"
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 10Ti
+  storageClassName: ""
+  volumeName: arrstack-mediafs
+  volumeMode: Filesystem
@@ -0,0 +1,33 @@
+---
+# Non-secret env for the -unkin2 fork. The fork reads Servarr config from
+# Radarr__<Section>__<Key> env (no config.xml edits, no s6/PUID). Postgres wiring
+# points every replica at the same shared DB (arrstack-postgres-rw / radarr-main);
+# Auth__Method=External defers UI auth to arrproxy/oauth2-proxy; Server__UrlBase
+# keeps the /radarr prefix so arrproxy path-routing works; App__InstanceName is
+# identical across replicas (shared session-cookie name). User/Password/ApiKey
+# come from Secrets (see deployment.yaml), not here.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: radarr-env
+  namespace: arrstack
+data:
+  Radarr__Postgres__Host: arrstack-postgres-rw.arrstack.svc.cluster.local
+  Radarr__Postgres__Port: "5432"
+  Radarr__Postgres__MainDb: radarr-main
+  Radarr__Log__DbEnabled: "false"
+  Radarr__Auth__Method: External
+  Radarr__Auth__Required: DisabledForLocalAddresses
+  Radarr__App__InstanceName: Radarr
+  Radarr__Server__Port: "7878"
+  Radarr__Server__UrlBase: /radarr
+  Radarr__Update__Mechanism: External
+  # Shared arrstack Valkey (valkey-operator). Setting Host is what activates the
+  # fork's #14 Redis features (SignalR backplane, cross-replica cache-invalidation
+  # bus, distributed rate limiter): RedisOptions.IsConfigured gates purely on a
+  # non-empty Host, so there is no separate Enabled flag. The operator leaves the
+  # default user passwordless (jellyfin parity), so no Password/Ssl is wired.
+  # Channels/keys are namespaced by this fork's radarr:ratelimit: prefix, so the
+  # one cluster is safe to share with sonarr/prowlarr.
+  Radarr__Redis__Host: valkey-arrstack-valkey.arrstack.svc.cluster.local
+  Radarr__Redis__Port: "6379"
@@ -4,11 +4,21 @@ kind: Deployment
 metadata:
  name: radarr
  namespace: arrstack
+  annotations:
+    # radarr-env is a plain (unhashed) ConfigMap consumed by fixed-name envFrom,
+    # so editing it does not roll the Deployment on its own. Reloader watches the
+    # referenced ConfigMap and triggers a rolling restart on change, so adding the
+    # Redis env activates the #14 features on the next ArgoCD sync without a manual
+    # `rollout restart`.
+    configmap.reloader.stakater.com/auto: "true"
 spec:
-  replicas: 1
+  # Active-active: the -unkin2 fork keeps all state in the shared Postgres
+  # (arrstack-postgres) and coordinates via Postgres advisory locks, so N
+  # replicas run concurrently behind the radarr Service. RollingUpdate is safe —
+  # no SQLite, no RWO lock.
+  replicas: 3
  strategy:
-    # RWO config PVC + single stateful SQLite DB: never run two pods at once.
-    type: Recreate
+    type: RollingUpdate
  selector:
    matchLabels:
      app: radarr
@@ -18,74 +28,139 @@ spec:
        app: radarr
    spec:
      securityContext:
+        # Fork image has no USER (runs as root by default); pin it to a non-root
+        # UID and group-write the shared RWX CephFS /config (MediaCover etc.).
+        # OnRootMismatch avoids a recursive chown of the whole media tree.
+        runAsUser: 1000
+        runAsGroup: 1000
        fsGroup: 1000
        fsGroupChangePolicy: OnRootMismatch
      initContainers:
-        # Enforce the Vault-sourced API key in /config/config.xml before the app
-        # starts. Vault is source of truth (override bootstrap): the key is minted
-        # in Vault, synced by VSO into the radarr-apikey Secret, and written here.
-        # Runs as root to fix ownership; touches only the <ApiKey> element.
-        - name: apikey-init
-          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/busybox:1.37.0
-          imagePullPolicy: IfNotPresent
-          securityContext:
-            runAsUser: 0
+        # Gate the app on its own Postgres database+role being reachable, instead
+        # of relying on ArgoCD sync-waves (which deadlock if apps aren't Healthy).
+        # waitfordb reads the PG* env as a libpq fallback, so the password never lands in argv.
+        - name: wait-for-db
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/waitfordb:v0.1.0
          env:
-            - name: API_KEY
+            - name: WAITFORDB_TIMEOUT
+              value: 5m
+            - name: WAITFORDB_SSLMODE
+              value: disable
+            - name: PGHOST
+              value: arrstack-postgres-rw.arrstack.svc.cluster.local
+            - name: PGPORT
+              value: "5432"
+            - name: PGDATABASE
+              value: radarr-main
+            - name: PGUSER
              valueFrom:
                secretKeyRef:
-                  name: radarr-apikey
-                  key: apitoken
-          command:
-            - sh
-            - -c
-            - |
-              set -eu
-              case "$API_KEY" in
-                "" | *[!0-9a-fA-F]*)
-                  echo "apikey-init: API_KEY missing or not hex; refusing" >&2
-                  exit 1
-                  ;;
-              esac
-              CFG=/config/config.xml
-              if [ ! -f "$CFG" ]; then
-                printf '<Config>\n  <ApiKey>%s</ApiKey>\n</Config>\n' "$API_KEY" > "$CFG"
-              elif grep -q '<ApiKey>' "$CFG"; then
-                sed -i "s|<ApiKey>[^<]*</ApiKey>|<ApiKey>${API_KEY}</ApiKey>|" "$CFG"
-              else
-                sed -i "s|<Config>|<Config>\n  <ApiKey>${API_KEY}</ApiKey>|" "$CFG"
-              fi
-              chown 1000:1000 "$CFG"
-              chmod 600 "$CFG"
-              echo "apikey-init: <ApiKey> enforced from Vault"
+                  name: radarr-db
+                  key: username
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: radarr-db
+                  key: password
          resources:
            requests:
-              cpu: 50m
+              cpu: 10m
              memory: 32Mi
            limits:
-              cpu: 200m
+              cpu: 100m
              memory: 64Mi
-          volumeMounts:
-            - name: config
-              mountPath: /config
      containers:
        - name: radarr
-          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/linuxserver/radarr:6.3.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/radarr:v6.4.2-unkin7
          imagePullPolicy: IfNotPresent
+          command:
+            - /app/Radarr
+          args:
+            - -nobrowser
+            - -data=/config
+            # Required: bypass the single-instance guard so multiple replicas
+            # can share one /config. Cross-replica safety is the Postgres layer,
+            # not a local lock file.
+            - -nosingleinstancecheck
          ports:
            - name: http
              containerPort: 7878
              protocol: TCP
+          envFrom:
+            - configMapRef:
+                name: radarr-env
          env:
-            - name: PUID
-              value: "1000"
-            - name: PGID
-              value: "1000"
-            - name: TZ
-              value: Australia/Sydney
+            - name: Radarr__Postgres__User
+              valueFrom:
+                secretKeyRef:
+                  name: radarr-db
+                  key: username
+            - name: Radarr__Postgres__Password
+              valueFrom:
+                secretKeyRef:
+                  name: radarr-db
+                  key: password
+            - name: Radarr__Auth__ApiKey
+              valueFrom:
+                secretKeyRef:
+                  name: radarr-apikey
+                  key: apitoken
+            # MediaCover object store (shared Ceph RGW bucket). Serves posters/fanart
+            # from S3 so any replica can render them instead of the leader-local
+            # emptyDir /config. Consumed by the -unkin3+ image; older images ignore
+            # these unknown config keys. Creds Secret is minted by cephrgw-operator.
+            - name: Radarr__MediaCover__S3__Endpoint
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-media-s3
+                  key: S3_ENDPOINT
+            - name: Radarr__MediaCover__S3__AccessKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-media-s3
+                  key: AWS_ACCESS_KEY_ID
+            - name: Radarr__MediaCover__S3__SecretKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-media-s3
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: Radarr__MediaCover__S3__Bucket
+              value: arrstack-media
+            - name: Radarr__MediaCover__S3__Prefix
+              value: radarr
+            - name: Radarr__MediaCover__S3__ForcePathStyle
+              value: "true"
+            - name: Radarr__MediaCover__S3__CaCertPath
+              value: /etc/ssl/vault-ca/ca.crt
+            # Backup object store (shared arrstack-backups Ceph RGW bucket,
+            # per-app key prefix). Routes the periodic config+DB zip backups off
+            # the ephemeral /config so any replica can write and restore them.
+            - name: Radarr__BackupS3__Endpoint
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: S3_ENDPOINT
+            - name: Radarr__BackupS3__AccessKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: AWS_ACCESS_KEY_ID
+            - name: Radarr__BackupS3__SecretKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: Radarr__BackupS3__Bucket
+              value: arrstack-backups
+            - name: Radarr__BackupS3__Prefix
+              value: radarr
+            - name: Radarr__BackupS3__ForcePathStyle
+              value: "true"
+            - name: Radarr__BackupS3__CaCertPath
+              value: /etc/ssl/vault-ca/ca.crt
          livenessProbe:
            httpGet:
-              path: /ping
+              path: /radarr/ping
              port: http
            initialDelaySeconds: 30
            periodSeconds: 30
@@ -93,7 +168,7 @@ spec:
            failureThreshold: 3
          readinessProbe:
            httpGet:
-              path: /ping
+              path: /radarr/ping
              port: http
            initialDelaySeconds: 10
            periodSeconds: 10
@@ -111,10 +186,64 @@ spec:
              mountPath: /config
            - name: media-movies
              mountPath: /media/movies
+            - name: vault-ca
+              mountPath: /etc/ssl/vault-ca
+              readOnly: true
+        # exportarr sidecar: polls the local replica's API and exposes Prometheus
+        # metrics on :9708 (scraped by the radarr-exportarr VMPodScrape).
+        - name: exportarr
+          image: artifactapi.k8s.syd1.au.unkin.net/ghcr/onedr0p/exportarr:v2.3.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - radarr
+          env:
+            - name: PORT
+              value: "9708"
+            # URL includes the /radarr UrlBase (Radarr__Server__UrlBase).
+            - name: URL
+              value: http://localhost:7878/radarr
+            - name: APIKEY
+              valueFrom:
+                secretKeyRef:
+                  name: radarr-apikey
+                  key: apitoken
+          ports:
+            - name: metrics
+              containerPort: 9708
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 15
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 25m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 128Mi
      volumes:
        - name: config
-          persistentVolumeClaim:
-            claimName: radarr-config
+          emptyDir: {}
        - name: media-movies
          persistentVolumeClaim:
            claimName: media-movies
+        # Estate CA for validating the Ceph RGW (s3.ceph.unkin.net) TLS cert.
+        - name: vault-ca
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
@@ -3,9 +3,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization

 resources:
-  - pvc-config.yaml
  - vaultstaticsecret.yaml
+  - configmap.yaml
  - deployment.yaml
  - service.yaml
  - gateway.yaml
  - httproute.yaml
+  - vmpodscrape.yaml
@@ -1,6 +1,7 @@
 ---
-# Radarr config + SQLite DB. RWO on cephrbd (block) — the arr apps' SQLite
-# does not tolerate CephFS locking. Retain: this is state.
+# Radarr /config. RWX on CephFS so all replicas share it (the -unkin2 fork keeps
+# the database in Postgres; /config now holds only config.xml + MediaCover, which
+# tolerate — and want — shared access). Retain: this is state.
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -8,9 +9,9 @@ metadata:
  namespace: arrstack
 spec:
  accessModes:
-    - ReadWriteOnce
+    - ReadWriteMany
  resources:
    requests:
      storage: 5Gi
-  storageClassName: cephrbd-fast-retain
+  storageClassName: cephfs-raid5-retain
  volumeMode: Filesystem
@@ -0,0 +1,16 @@
+---
+# Scrape the exportarr sidecar (:9708) on every radarr pod. Picked up by the
+# observability VMAgent (selectAllByDefault). Pod-level rather than
+# VMServiceScrape because the radarr Service doesn't expose the metrics port.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: radarr-exportarr
+  namespace: arrstack
+spec:
+  selector:
+    matchLabels:
+      app: radarr
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
@@ -0,0 +1,33 @@
+---
+# Non-secret env for the -unkin2 fork. The fork reads Servarr config from
+# Sonarr__<Section>__<Key> env (no config.xml edits, no s6/PUID). Postgres wiring
+# points every replica at the same shared DB (arrstack-postgres-rw / sonarr-main);
+# Auth__Method=External defers UI auth to arrproxy/oauth2-proxy; Server__UrlBase
+# keeps the /sonarr prefix so arrproxy path-routing works; App__InstanceName is
+# identical across replicas (shared session-cookie name). User/Password/ApiKey
+# come from Secrets (see deployment.yaml), not here.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sonarr-env
+  namespace: arrstack
+data:
+  Sonarr__Postgres__Host: arrstack-postgres-rw.arrstack.svc.cluster.local
+  Sonarr__Postgres__Port: "5432"
+  Sonarr__Postgres__MainDb: sonarr-main
+  Sonarr__Log__DbEnabled: "false"
+  Sonarr__Auth__Method: External
+  Sonarr__Auth__Required: DisabledForLocalAddresses
+  Sonarr__App__InstanceName: Sonarr
+  Sonarr__Server__Port: "8989"
+  Sonarr__Server__UrlBase: /sonarr
+  Sonarr__Update__Mechanism: External
+  # Shared arrstack Valkey (valkey-operator). Setting Host is what activates the
+  # fork's #14 Redis features (SignalR backplane, cross-replica cache-invalidation
+  # bus, distributed rate limiter): RedisOptions.IsConfigured gates purely on a
+  # non-empty Host, so there is no separate Enabled flag. The operator leaves the
+  # default user passwordless (jellyfin parity), so no Password/Ssl is wired.
+  # Channels/keys are namespaced by this fork's sonarr:ratelimit: prefix, so the
+  # one cluster is safe to share with radarr/prowlarr.
+  Sonarr__Redis__Host: valkey-arrstack-valkey.arrstack.svc.cluster.local
+  Sonarr__Redis__Port: "6379"
@@ -4,11 +4,21 @@ kind: Deployment
 metadata:
  name: sonarr
  namespace: arrstack
+  annotations:
+    # sonarr-env is a plain (unhashed) ConfigMap consumed by fixed-name envFrom,
+    # so editing it does not roll the Deployment on its own. Reloader watches the
+    # referenced ConfigMap and triggers a rolling restart on change, so adding the
+    # Redis env activates the #14 features on the next ArgoCD sync without a manual
+    # `rollout restart`.
+    configmap.reloader.stakater.com/auto: "true"
 spec:
-  replicas: 1
+  # Active-active: the -unkin2 fork keeps all state in the shared Postgres
+  # (arrstack-postgres) and coordinates via Postgres advisory locks, so N
+  # replicas run concurrently behind the sonarr Service. RollingUpdate is safe —
+  # no SQLite, no RWO lock.
+  replicas: 3
  strategy:
-    # RWO config PVC + single stateful SQLite DB: never run two pods at once.
-    type: Recreate
+    type: RollingUpdate
  selector:
    matchLabels:
      app: sonarr
@@ -18,77 +28,141 @@ spec:
        app: sonarr
    spec:
      securityContext:
-        # LinuxServer images init as root via s6 then step down to PUID/PGID.
-        # fsGroup makes the shared CephFS group-writable for that user;
+        # Fork image has no USER (runs as root by default); pin it to a non-root
+        # UID and group-write the shared RWX CephFS /config (MediaCover etc.).
        # OnRootMismatch avoids a recursive chown of the whole media tree.
+        runAsUser: 1000
+        runAsGroup: 1000
        fsGroup: 1000
        fsGroupChangePolicy: OnRootMismatch
      initContainers:
-        # Enforce the Vault-sourced API key in /config/config.xml before the app
-        # starts. Vault is source of truth (override bootstrap): the key is minted
-        # in Vault, synced by VSO into the sonarr-apikey Secret, and written here.
-        # Runs as root to fix ownership; touches only the <ApiKey> element.
-        - name: apikey-init
-          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/busybox:1.37.0
-          imagePullPolicy: IfNotPresent
-          securityContext:
-            runAsUser: 0
+        # Gate the app on its own Postgres database+role being reachable, instead
+        # of relying on ArgoCD sync-waves (which deadlock if apps aren't Healthy).
+        # waitfordb reads the PG* env as a libpq fallback, so the password never lands in argv.
+        - name: wait-for-db
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/waitfordb:v0.1.0
          env:
-            - name: API_KEY
+            - name: WAITFORDB_TIMEOUT
+              value: 5m
+            - name: WAITFORDB_SSLMODE
+              value: disable
+            - name: PGHOST
+              value: arrstack-postgres-rw.arrstack.svc.cluster.local
+            - name: PGPORT
+              value: "5432"
+            - name: PGDATABASE
+              value: sonarr-main
+            - name: PGUSER
              valueFrom:
                secretKeyRef:
-                  name: sonarr-apikey
-                  key: apitoken
-          command:
-            - sh
-            - -c
-            - |
-              set -eu
-              case "$API_KEY" in
-                "" | *[!0-9a-fA-F]*)
-                  echo "apikey-init: API_KEY missing or not hex; refusing" >&2
-                  exit 1
-                  ;;
-              esac
-              CFG=/config/config.xml
-              if [ ! -f "$CFG" ]; then
-                printf '<Config>\n  <ApiKey>%s</ApiKey>\n</Config>\n' "$API_KEY" > "$CFG"
-              elif grep -q '<ApiKey>' "$CFG"; then
-                sed -i "s|<ApiKey>[^<]*</ApiKey>|<ApiKey>${API_KEY}</ApiKey>|" "$CFG"
-              else
-                sed -i "s|<Config>|<Config>\n  <ApiKey>${API_KEY}</ApiKey>|" "$CFG"
-              fi
-              chown 1000:1000 "$CFG"
-              chmod 600 "$CFG"
-              echo "apikey-init: <ApiKey> enforced from Vault"
+                  name: sonarr-db
+                  key: username
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: sonarr-db
+                  key: password
          resources:
            requests:
-              cpu: 50m
+              cpu: 10m
              memory: 32Mi
            limits:
-              cpu: 200m
+              cpu: 100m
              memory: 64Mi
-          volumeMounts:
-            - name: config
-              mountPath: /config
      containers:
        - name: sonarr
-          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/linuxserver/sonarr:4.0.19
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/sonarr:v5.0.0-unkin6
          imagePullPolicy: IfNotPresent
+          command:
+            - /app/Sonarr
+          args:
+            - -nobrowser
+            - -data=/config
+            # Required: bypass the single-instance guard so multiple replicas
+            # can share one /config. Cross-replica safety is the Postgres layer,
+            # not a local lock file.
+            - -nosingleinstancecheck
          ports:
            - name: http
              containerPort: 8989
              protocol: TCP
+          envFrom:
+            - configMapRef:
+                name: sonarr-env
          env:
-            - name: PUID
-              value: "1000"
-            - name: PGID
-              value: "1000"
-            - name: TZ
-              value: Australia/Sydney
+            - name: Sonarr__Postgres__User
+              valueFrom:
+                secretKeyRef:
+                  name: sonarr-db
+                  key: username
+            - name: Sonarr__Postgres__Password
+              valueFrom:
+                secretKeyRef:
+                  name: sonarr-db
+                  key: password
+            - name: Sonarr__Auth__ApiKey
+              valueFrom:
+                secretKeyRef:
+                  name: sonarr-apikey
+                  key: apitoken
+            # MediaCover object store (shared arrstack-media Ceph RGW bucket,
+            # partitioned from radarr by the sonarr key prefix). Serves
+            # posters/fanart from S3 so any replica renders them instead of the
+            # leader-local emptyDir /config. Consumed by the -unkin3+ image;
+            # older images ignore these unknown config keys. Creds Secret minted
+            # by cephrgw-operator.
+            - name: Sonarr__MediaCoverS3__Endpoint
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-media-s3
+                  key: S3_ENDPOINT
+            - name: Sonarr__MediaCoverS3__AccessKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-media-s3
+                  key: AWS_ACCESS_KEY_ID
+            - name: Sonarr__MediaCoverS3__SecretKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-media-s3
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: Sonarr__MediaCoverS3__Bucket
+              value: arrstack-media
+            - name: Sonarr__MediaCoverS3__Prefix
+              value: sonarr
+            - name: Sonarr__MediaCoverS3__ForcePathStyle
+              value: "true"
+            - name: Sonarr__MediaCoverS3__CaCertPath
+              value: /etc/ssl/vault-ca/ca.crt
+            # Backup object store (shared arrstack-backups Ceph RGW bucket,
+            # per-app key prefix). Routes the periodic config+DB zip backups off
+            # the ephemeral /config so any replica can write and restore them.
+            - name: Sonarr__BackupS3__Endpoint
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: S3_ENDPOINT
+            - name: Sonarr__BackupS3__AccessKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: AWS_ACCESS_KEY_ID
+            - name: Sonarr__BackupS3__SecretKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: Sonarr__BackupS3__Bucket
+              value: arrstack-backups
+            - name: Sonarr__BackupS3__Prefix
+              value: sonarr
+            - name: Sonarr__BackupS3__ForcePathStyle
+              value: "true"
+            - name: Sonarr__BackupS3__CaCertPath
+              value: /etc/ssl/vault-ca/ca.crt
          livenessProbe:
            httpGet:
-              path: /ping
+              path: /sonarr/ping
              port: http
            initialDelaySeconds: 30
            periodSeconds: 30
@@ -96,7 +170,7 @@ spec:
            failureThreshold: 3
          readinessProbe:
            httpGet:
-              path: /ping
+              path: /sonarr/ping
              port: http
            initialDelaySeconds: 10
            periodSeconds: 10
@@ -114,10 +188,64 @@ spec:
              mountPath: /config
            - name: media-tv
              mountPath: /media/tv
+            - name: vault-ca
+              mountPath: /etc/ssl/vault-ca
+              readOnly: true
+        # exportarr sidecar: polls the local replica's API and exposes Prometheus
+        # metrics on :9707 (scraped by the sonarr-exportarr VMPodScrape).
+        - name: exportarr
+          image: artifactapi.k8s.syd1.au.unkin.net/ghcr/onedr0p/exportarr:v2.3.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - sonarr
+          env:
+            - name: PORT
+              value: "9707"
+            # URL includes the /sonarr UrlBase (Sonarr__Server__UrlBase).
+            - name: URL
+              value: http://localhost:8989/sonarr
+            - name: APIKEY
+              valueFrom:
+                secretKeyRef:
+                  name: sonarr-apikey
+                  key: apitoken
+          ports:
+            - name: metrics
+              containerPort: 9707
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 15
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 25m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 128Mi
      volumes:
        - name: config
-          persistentVolumeClaim:
-            claimName: sonarr-config
+          emptyDir: {}
        - name: media-tv
          persistentVolumeClaim:
            claimName: media-tv
+        # Estate CA for validating the Ceph RGW (s3.ceph.unkin.net) TLS cert.
+        - name: vault-ca
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
@@ -3,9 +3,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization

 resources:
-  - pvc-config.yaml
  - vaultstaticsecret.yaml
+  - configmap.yaml
  - deployment.yaml
  - service.yaml
  - gateway.yaml
  - httproute.yaml
+  - vmpodscrape.yaml
@@ -1,6 +1,7 @@
 ---
-# Sonarr config + SQLite DB. RWO on cephrbd (block) — the arr apps' SQLite
-# does not tolerate CephFS locking. Retain: this is state.
+# Sonarr /config. RWX on CephFS so all replicas share it (the -unkin2 fork keeps
+# the database in Postgres; /config now holds only config.xml + MediaCover, which
+# tolerate — and want — shared access). Retain: this is state.
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -8,9 +9,9 @@ metadata:
  namespace: arrstack
 spec:
  accessModes:
-    - ReadWriteOnce
+    - ReadWriteMany
  resources:
    requests:
      storage: 5Gi
-  storageClassName: cephrbd-fast-retain
+  storageClassName: cephfs-raid5-retain
  volumeMode: Filesystem
@@ -0,0 +1,16 @@
+---
+# Scrape the exportarr sidecar (:9707) on every sonarr pod. Picked up by the
+# observability VMAgent (selectAllByDefault). Pod-level rather than
+# VMServiceScrape because the sonarr Service doesn't expose the metrics port.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: sonarr-exportarr
+  namespace: arrstack
+spec:
+  selector:
+    matchLabels:
+      app: sonarr
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - valkeycluster.yaml
@@ -0,0 +1,47 @@
+---
+# Single shared HA Valkey for the arr apps (sonarr/radarr/prowlarr), managed by
+# valkey-operator. It activates the fork's #14 Redis features: the SignalR
+# backplane, the cross-replica cache-invalidation bus, and the distributed rate
+# limiter. One cluster is safe for all three because each fork namespaces its
+# keys and pub/sub channels by a per-app prefix (sonarr:ratelimit: /
+# radarr:ratelimit: / prowlarr:ratelimit:), so their state never collides.
+#
+# Modeled on jellyfin-valkey: shards:1 + replicas:2 is one primary with two
+# replicas in a single shard group (three ValkeyNodes total); losing the primary
+# triggers an automatic failover so a node/pod loss no longer drops the shared
+# state the app replicas coordinate through. The operator runs Valkey
+# cluster-mode-enabled with protected-mode off and leaves the built-in `default`
+# user passwordless, so clients connect with no auth/TLS; StackExchange.Redis
+# seeds off the single service and auto-discovers topology plus failovers.
+# scheduling.node.spread.shard:Required keeps the three nodes on distinct hosts,
+# so one host loss removes at most one node; podDisruptionBudget.mode:Cluster
+# lets the operator manage a quorum-aware PDB. Persistence is omitted (/data is an
+# emptyDir): the coordination state is ephemeral (short TTLs / transient pub/sub),
+# replication+failover already provide redundancy, and an operator-managed PVC
+# cannot carry the k8up.io/backup:"false" annotation the namespace k8up Schedule
+# needs to skip in-use RWO volumes.
+apiVersion: valkey.io/v1alpha1
+kind: ValkeyCluster
+metadata:
+  name: arrstack-valkey
+  namespace: arrstack
+spec:
+  shards: 1
+  replicas: 2
+  image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/valkey/valkey:9.0.0
+  exporter:
+    enabled: false
+  scheduling:
+    node:
+      spread:
+        shard:
+          mode: Required
+  podDisruptionBudget:
+    mode: Cluster
+  resources:
+    requests:
+      cpu: 50m
+      memory: 128Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
@@ -18,4 +18,5 @@ resources:
  - ui-hpa.yaml
  - vaultauth.yaml
  - vaultstaticsecret.yaml
+  - vmpodscrape.yaml
  - vpa.yaml
@@ -53,4 +53,18 @@ spec:
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 5
+        - name: metrics-exporter
+          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/oliver006/redis_exporter:v1.89.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 9121
+              name: metrics
+              protocol: TCP
+          resources:
+            limits:
+              cpu: 100m
+              memory: 64Mi
+            requests:
+              cpu: 25m
+              memory: 32Mi
      restartPolicy: Always
@@ -0,0 +1,16 @@
+---
+# Scrape the redis_exporter sidecar (:9121) on the redis pod. Picked up by the
+# observability VMAgent (selectAllByDefault). Pod-level rather than
+# VMServiceScrape because the redis Service doesn't expose the metrics port.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: redis-exporter
+  namespace: artifactapi
+spec:
+  selector:
+    matchLabels:
+      app: redis
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
@@ -18,4 +18,5 @@ resources:
  - redis-service.yaml
  - vaultauth.yaml
  - vaultstaticsecret.yaml
+  - vmpodscrape.yaml
  - vpa.yaml
@@ -52,6 +52,20 @@ spec:
          volumeMounts:
            - mountPath: /data
              name: redis-data
+        - name: metrics-exporter
+          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/oliver006/redis_exporter:v1.89.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 9121
+              name: metrics
+              protocol: TCP
+          resources:
+            limits:
+              cpu: 100m
+              memory: 64Mi
+            requests:
+              cpu: 25m
+              memory: 32Mi
      volumes:
        - name: redis-data
          persistentVolumeClaim:
@@ -0,0 +1,16 @@
+---
+# Scrape the redis_exporter sidecar (:9121) on the redis pod. Picked up by the
+# observability VMAgent (selectAllByDefault). Pod-level rather than
+# VMServiceScrape because the redis Service doesn't expose the metrics port.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: redis-exporter
+  namespace: authentik
+spec:
+  selector:
+    matchLabels:
+      app: redis
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
@@ -117,3 +117,18 @@ spec:
    # traefik-EXTERNAL (DMZ) gateway VIP; the arrproxy Gateway serves the
    # arrstack.unkin.net front door (oauth2-proxy) there.
    - 198.18.199.0
+---
+apiVersion: bind.unkin.net/v1alpha1
+kind: DNSRecord
+metadata:
+  name: logviewer-dns-internal
+  namespace: bind-internal
+spec:
+  zoneRef: unkin-net
+  name: logviewer
+  type: A
+  ttl: 600
+  values:
+    # traefik-internal gateway VIP; the logviewer Gateway serves
+    # logviewer.unkin.net there.
+    - 198.18.200.4
@@ -6,5 +6,6 @@ resources:
  - namespace.yaml
  - vaultauth.yaml
  - vaultstaticsecret.yaml
+  - vaultstaticsecret-mediafs.yaml
  - storageclass.yaml
  - vmservicescrape.yaml
@@ -0,0 +1,15 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: ceph-mediafs-secret
+  namespace: csi-cephfs
+spec:
+  vaultAuthRef: ceph-csi-cephfs
+  mount: kv
+  type: kv-v2
+  path: service/kubernetes/au/syd1/csi/ceph-mediafs-secret
+  refreshAfter: 5m
+  destination:
+    name: ceph-mediafs-secret
+    create: true
@@ -10,6 +10,7 @@ resources:
  - valkey-deployment.yaml
  - valkey-pvc.yaml
  - valkey-service.yaml
+  - vmpodscrape.yaml
  - vaultauth.yaml
  - vaultstaticsecret.yaml
  - gateway.yaml
@@ -82,6 +82,26 @@ spec:
          volumeMounts:
            - mountPath: /data
              name: data
+        - name: metrics-exporter
+          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/oliver006/redis_exporter:v1.89.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 9121
+              name: metrics
+              protocol: TCP
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsNonRoot: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 100m
+              memory: 64Mi
+            requests:
+              cpu: 25m
+              memory: 32Mi
      restartPolicy: Always
      volumes:
        - name: data
@@ -0,0 +1,16 @@
+---
+# Scrape the redis_exporter sidecar (:9121) on the gitea-valkey pod. Picked up
+# by the observability VMAgent (selectAllByDefault). Pod-level rather than
+# VMServiceScrape because the valkey Service doesn't expose the metrics port.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: valkey-exporter
+  namespace: gitea
+spec:
+  selector:
+    matchLabels:
+      app: gitea-valkey
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
@@ -0,0 +1,13 @@
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: redis
+  namespace: grafana
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  resyncPeriod: 5m
+  allowCrossNamespaceImport: false
+  gzipJson: H4sIAAAAAAAC/+1cbVPbOhb+K17vnZ2yY6jtkCYw0w9Ay1xmC+1d2u7sQiej2EqiYlteWYaETO5vv+dIsmPzmgQItM0HTCzr5eg55zxHL7bGdqdDIxrTRGb29nji2CRJuCSS8QQT7Ihl0t4+GdvdnEXyILG3PccOiSQZz0VAMYscpfC/mujYOQshqS9IjyTEhmppQroRZJMip449YGH5mwU82eMRF1BA9LvkletYvufBpdl0LG8NaktIjC3sTEWz/mHtRFSA0M60/WzQ5USE9uQbNBjSLBAsxdzw8N80ZJn1rshi9biwPgkeUzmgeWbpx++HKReSCsvbGEK9kCarQvdYFpDov5SIY0mEPOSJHNjbrmP3EyoPoL+tNw0H+5wOPnMeSZYqsBCJJI8iB7BMzgDUk2/485we8Qt7u0eiDOpOSUKjTAF9I7hpKWsJbkilP2o2QxJsfj8LEOMeo1G4x5Me62PRkPZIHim92oEGeAx5hjSswP2qAUh7PmDubTUQ65ijanQ+W+EYsJigbNDTmKQpS/paUJ6WVhITGQAWNvYT6hA0g4aV+HQI/+2j1zv2ZFKqKkuhUhKBoiBpALkHPAp1Rbp10s14lEu0pEzSVLdn+gAIU5rAk3MS5VRjO3GmjwXIXT5su8oY8oShGAo8+HkMejRqhYf8nAoBBqlUM0ENsvATV+JAn1qODWoCxQ4VAiO4TrRSt2oajcnwHWjuE2fKlTwXMlcQUtId6t4lPMGeKUsxSURQAknf80yy3qhIzCWHRC4YuCcxljyA20swPaKBDvOAfqw0Q6IABbIjkskjLo9QId+MacADu4AmM6YHfQH0Lz5RsLdE7g1I0qelVaL26rJcAE4fyIjnsoQvjfI+S75SkWkBPXejsdFAvySiT+VDjZoOU9QqwPsK/jqorI5kMX0l0Gc7eYo3HZZ0MgpEEmZjlmSSJAF9++ep/Vtxc2pPTn7rQDZwb+j/tzU0dSABMFyoXFWRUcAZJSkyabSKu30SSLQvH7RO+zQJ94vS6DRUChbo34L2kAzsHWO8YAtt11WmziSanX1IhtYXJfeUvUBMaaMZr9z/ydzf+N1iDOArBmiUDIAh7DOo8KMpW4YzUI7nr6hhedSQ5bEhAyCAhAaShp0gQmBuJ4MZ3P8BDu9rb4/pPnQD+x3bU+/f06It3fWNb1UcbgnuDaZvDD5miWpkfodXg8ImUlTLd6xNGBS6G1uttTsoAAv4jRYy2xaUcLFEe2utxgvX8mPFzU39Bw1Ucm81qyySan944FCiORuRePMQCUD8lV3+Tll/AIK2mo5J+Q8LcaSKCcsgjM+Fhj+QrhrUGsqoPTsk4gx4oehsxi7B8goyWSp1AIrWPy3DIDGNuRh18gw4pDuS9HYGsazXVq0Mjk7uLrL2jKxzqIS0vmSkXxl19EkOt0vjnhR8RUq6HoBFZUyVDiCsobRjmwxZtstFSMXxoDIzwuQ9iujQ8H9U8PqDSuhUZKSTleFpmPD2U0QCNcWdBqsuETsR6yc6FdwoFOTiWI4UVuBxiFGPRdHHlARMjpAxVFQO0X3qsRr9VyM+NiqbBsliGmjuz9nl1FXAMGACi041dRFs+QD7mvKocFJMI8LWD40rAy2kSALH7JIqLsGZKX0Hc3XBurkuWGrRVFCEc0MedkKBcNB4YO5ppDC+KElwprxxDF3meWqsLC66DQYZYXVTKjfIlYrmvZ6KCDXuKoPLsmd9QHOL03Vb0XW7NvPz2rWuVSi4MIApe8JzIIk0IsXQTa2oIMFNjbLLJTqBVtCHqg1NKkZU4hSDfzHMjf3aVisdCu0lDreIpOWYK44JzLs6EtZmotvp8sSLYdZldUfWqyAOH3H6NR5bUKE1mVQJsiZbB3QRGQFvZs7N2vzsM2a09kzHgOdhajnlTJTWCLsizvuJ03tRxOk/K3Em9IG0ufgg+nnp03tT58+XRJ9aLUujT1bhzjM6AiMKaGfA5H0EetIE/rydNfWKuvGGeSkUm3csINKivStsehtpFtBglodx4V3gxCzL6HPCowWYEaDdWwGqRJjfAXCIK4eqYgtmlNbxKsSsQsxdIaaYRP+UMUbNm++NMWYliiqzVRbfHR0RtYpexhC14oRkLji4lWSmoMqtBSlFuLIa/vfdfW/Xda+tgYOctwU3z69sC7VMcGstIbgZcF5abJt3DeUhyyFY/YuIWXMuAs0Zo+6ISmR4Q/9vDEPXJzc3Lwet4s4q7jzX1OblhZt7WB//VWgfjeFX5f0rS0LwOgzstKe51FQ4y8j9QSvjigShuToD1pd3nKfpJux2PnY/51j30q3PSPxHVF5wcWYdvP64YvwFGb+1YvwfdRdgppc/bpphfFW11RidIxh9JaLeNcW2YY35IFO2WUrgTm6biSh73yiNpjIniSpUX7EdfVsxnVmnJd6mCVDN2ePT1R3fny1YFbu9YVetMt2+Yau2K8KuUw6yH5e/wy5w9x2D+J1Z1pL0gP5A0livJL3bXdH7it5Xm7yLD+lLyvQav/SYfj6aLCnSWreuF+5AtUyAbcxay+MRLXw0YBWtL0S2zpPA+YyI3IHGvVsa1dDz3tRjnWcWDBfWy/t/Qf9WUWi1rPTjLis9atiZaeOCnrNAZgvuXbS33H3Xv2nvwplt14TGAYFMYYfhQLKjhIG7K0sapza8g7vhtTbcDW97y2u6p7bznXchfeq/uMb9PJ0QFLydxYti2Nh/0240Ft7/8T0zamivFgJNpFNxBqxIhbtZVsdmDXgP2s43Ur2IzaOrgBm3e3TA5sZIC7LIBtN7De/r97qK1Vjg5xkLeD/4qw0vYjAw2xy0jCb46twvHU6e7ssu7xrtlc3UAsDjMb3qSzfiwdlT98Q0UiPsKkvvFT19vatzWte+S1tR9VxUvfpA5Mch6l9xI8hvm4jiv6Q3FZ7h4xV209crYS6UPxXHCFwZfVtv/7SujL/r37OcJvBB4Glya/1zV/eSP5DZAU+BF8cs/GzVOk7BDFBw88HM6nuZ1YeGP/4e1JOfKHNluF+Qs7e5Iud5yfna0kidS62/vbXcl/+94dxk+k1VjeatRc9gvBKTUkmNLVSP8oO6FpSs+FoHrPSm6P3aA8uTvoJcCGU8eBzY4koFfmbgGYZbkCg7+rSA8uQex8KDvdRHPmvlop6LOgiiPKQ78PJJwVfaMIs7cx5YWbo6KEOv+H9OxWjGVksIffzdp0MD5hlLv4joeJQE0wUK5RWeglUNBrM/TEO2hrp6q5HSkjj4tvpnpQtDyktEeVx2V/lJeQeOUn+vZQb8NW8b+Iuyc6E/szTPohcka3UuE24O6FMUEn6x7m8OsCjXt7bJlDKYN6tRh5G1PF5K+VwT1ea5eG2oqzqPoYkXT10brvqNVft48ULbHODQmeJZLaCyvlFZVQEtlo/BqRWqCqEGI9wljhmA/gW/yFT0LchmkZPwtPFR1213g0Zvve0Rf33T7W2td0nPXyeNRi9sw0kyLcVU5wUBgeIuKD1Th+WpBfO/AMd2wkZtUAAA
@@ -0,0 +1,13 @@
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: traefik
+  namespace: grafana
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  resyncPeriod: 5m
+  allowCrossNamespaceImport: false
+  gzipJson: H4sIAAAAAAAC/+2deXPjthXA/9enYDk7HTuRvKJk+chMOnV2k83O7NXaTaddux6IhCTEFMEFIR+rqp+9D7wP8JBM+djAk6xNAMTD8fDe74HXsqNpOnIcyhEn1PH0H7QlJEGiTTwOR5/9Iy1M9XPGC2Lztw5kGt0k1UIceXTBTBzXEebwO1ek6VOGJshBejeduSCWyOv1tDdBttbr6XGBVUoAdtDYFhVxtsCp9BmxJKnEpM4ralMmamfTMdrpd7WBYcA/o1FXM3ZTzdAdNPdbeJIMhPZn7cTGjHvpchyxKeb5/tlkTkSi0e9nujZH3JydOHeQNUG2hzOZHE3FYH++6MqGykLebEwRs+RDUVps5f++6ITFdQt7JiOu6JEo/3EyISZBthafqE0o0045cixkUwdrZwzhCbnyO61ji/BwzOMO6BPimcj+F0YMTmP8PXX4DAr4XdenDuZvxYQah8P9gyCJIXd2RqnNiRuXs4lzlXQfDq/xB3qTkeMiB9terIHRiOsmtW3ketjKDytIItYn6mWmR5/l1FQIGeynEm6jRoXHYrr6ndyY676WHseHSePi+dM54f5Q6W+wgxmy9SQnnC4GXeyk6o37VLJ24hNdRueYz/Aio43h0rEwH9yNRhYy93+/MvVC03M6kDRrQrBtvaLOhEyzYi08QQube3lFN8P1lE4Uek79BajzGcPejNqWp6cKrHJrwnWJI1X95OxSCWjsUXvBccaEQL7HseuljFX0s8wdp/oA2oKxk6vIL3GN7IWQ5ixsO5e76q4hgIGKVlR/1M9Xnjm+6Mhy0kaAXmMGGo+DsSxMe+lqGORWw6hmMRjyxWAM42PqZn1HMg7vw4kLOp2q2LcKUS5iOO0V9N8XHieTuzh7wWk6mzKCncBMy7Jh3Bcm/ihpk2gVss2ipug28vgHyj/AlKd1N6uh/nrxMmsomVEvMkdyg41veaY7hTF17cWUOL9h5oXdOt4b7hkpG+K7nmzTMz0r9b91dqTelhSXsfAMyeya4nc2/9ZlQc7C4Ts8cCqXpm9rLhm2KbK8Sw6+1l5GmcTxwA2Z+Mf/nesvooNzfbWbrdnGU+xYv1AG3lVIuLzMzb/QAORMi0jgq8bE9076ScqvdnJznRjy0Blqb8PWeEWTDhn8W7bpLrIx57hnwgrxSIVGmLBm6bxYEbol3itYrhiW5b8xozIYioqlLYZYMbqk0Ds0xnZhDYaZn2xk4jlIk9iFAF0RO7HJ1AnL9HPZFkM3p/wumH1glIKjmRDb/ugik/A7yelg1CxhmqI+OLRYg6DVX5hkpGLdlo+Q0JwYouT51+RrwQhJvJbfsbdiRlxqx1ZUJCKmS4r+k1g8R1GBxaLE4afkK876kMAhg53FryGCYGS8CEUUuhuthFB0ZZuBDG4+CYFe2dR6wGTCfnslGgYr1bwC9JC1ZMrowg3sQmFU5+nJrGxjQjCRDi3LKqOTSa4uRUytElM2xnT8+Azq/+J6eqswdZSDqcMcTI2awdRhNUvFhqEJy8wxys8SKNFtOddYxHNtFONWEPJlSrhpuzqmXFj6TAGxPN9FjSy4XY8y/pPovf4e2lHIeg3OLDxNDk+x3ct0P1LxObg1Uqw1cKOeWfT0q2cOVd5ivsMQxzFXwcSwO98cA1t9ARblDeGqm5zp5ySHAF6foSA4CVh9F7u72vhO20mya6BsuUyKrlbbgbO/hz3VXMy0n2NxRUTjZI49DDGD9/igNuPc9X54+XIOc72Y75l0/vKv4CPFfFweHwwH+y8TaT3kWvi2h8QuFDiAnmUcov7BCA+Prb4CPgV8CvgU8Cng28oWWZ7qjEEO6zIJFVx3oLhOcV1TrttpAHaX1oL5JvzSw7CdBrto44V5hXkt6dn4x3O9vzeEP0UL4GDQ769NgGJAqLWrfa+dO9rWm2vsDVpp7q72UhvA/5u22d/FrG3y/dt57tRhdVBwW0h9IoBT80zK8NPG6PdgCrUMbTxdHn4M/tuALba4R3Kw5gWnYckFp/1NnGnOCfp3NZT7QEamM76OC4yv/eScNQSlpqizI8OV9KC6BJ9FK4Tg+17H8hlhKxewNvbV6A+xBQOW8ZqYeN39l/A0Pzn8e+87SIcOcQqWCfyJ2DKocBpdTbRyt6nj+LxcivKr1cV2XMiv0FrtFUjQih4E9NucwS0j6nqZ2j5R2ydq+0Rtnzzx62VbvlZW2FXJg+CxHAQHw2e0q7I2UaY2VfJNU7sqG6Aap+7VjjHqQvAPP7Zwh0Bp/hztBGnip4TjinsBQH4tUl2wKaGt3YpmOxLN29FNWnGuh0WhIBy8MILfmcSdz//pXXzf8//dFVXC1gX813z/IsFQjTrachnWvq2NjTPqap5Nb7RQjve0tzcUnio8VXiq8FTh6WPfzlV/4a+EUUcKURWitouoQJ7+vl/IMP71pt0MPj7qluRfwH+2QJHVBPnA3PieelwLxw9b96DH9ENr6Ua1+MyacVhy2cSQPLX2BBZHGfNWcm8V+1bwb1MGlrq4MhZeg4cbM3EtFzdm4wZ83IiR6zm5KSvX8XIDZm7CzZXsLJ3gtRi6jqPrWLoxT9cydVlfatm6CV/XMXYdZ9ewdlnb65m7mruLGClb0KUMXsfhTVi8ksflyCzjckmhVSHtorarESuH16P9w8qoR0rORest81syipaStMyHJX7sqFSo72CGmSQqvTxeRtgVlF1G2jLaLhB3M+puQt719F1z510lhEvDXCmM1wB5BZRXh78yQJctiioWacYjzZhEvmpqwD0D70VcN7SetpOj9d0AjSsJvY2b5UroPcfqL/PBRBRHrNm+e21DZ9sk5CdBxEbhQz56KNmDlscMlXFDaeyQV/bcjk4UR5yGgQN4WgKufKqdvvuo0QkYxX5/nlXc0pAip6kK3BW4K3BX4K7AXYH7w4N7IbEJue8rclfkrsi95KkcRe7PidyHa4J7+TUGqHCNN8U9xIWEo5ILCQfqQoKKR1Q8ouIRFY8883iku0Ft8rt5qu/ouW/wk7vDZ1thTyGauZHEQqXXK46rox7j8BsKeyTvCFg/7pG1WQU+Dxj4bHa/Ufi84QPfduQ/VQ+Jg729R7oJqZVb2bcapAxubzVhB7V4LNQlhW8I4d+A7zqd+a9Hl2mOAn0F+gr0Fegr0G8P9I82Af0jBfoK9BXo3xP0Rwr0S0F/pEBfgb4CfQX6CvQV6CvQbwH0jYNNSP9Ykb4ifUX69yD9PwVb+v9VtF9O+x9BoZj/ekdPUb+ifkX9ivoV9Svqf4bUPybOTw+B/fd68HhoVFP/oK+eX1DQ/61A/1q8P77j+Dm+P6iG8R+N7ONv1/lIoGBewbyCeQXzCuYVzCuYb+lh5FqaV08jK5r/g9G854KKY4Xz7eN8OLKK59Xjs4rUFakrUlek/gRIHUaPPfm3Bg1rbrYZGArU1WuDnhCni29/RtRMXezAu3QcB5u+WtbidNV3oYPPeiZJ+e9Bl+BvcsJDMu+rpNMavKNM+zluRStvxfn17OyT9hpzeOuOV/V6nE54tuiKcBdpLwrufYbn6DfMvMC1Dw+D5AizLMSu/MpBS1NeCIBw7gr8SHnc4MPBiRKnXrJjLhgLlvcy/ZGLUn2u0+Q6Hc4IAWUDYx+Bkh8zBl/zrfjAeRj6dbW8sqajLZ8Qc28HIo5pLyx8YtsFhQpk+9/aCirV3oaVpusM7ErhXAfN/QEhklMSe59GBB06wu7yAxsltjoQ6WVyCrmgNNZviBFh7f/mC5TOTKKPg0zqFN/mwgnduyLuP5h9eueYRQiMjG76NU6RAn1Ji4+Fp1TTh87CcIvU4AVgYk308zlz4iuTkb4nNK3kme+yYBuMAJYGCkFcJUPfmHj0F5eXvsxo5yD+Qz6oEp1MdKdwa1xadzplnifdhaKNLO/Dmr3IOZ3yNsi+dRU1wpiXNgGy2pU1Kpc1aluW0a/oWL9tacMKacPWpRmz8q7NWpY1KJc1aFvWfrms/bZlHZXLOsrK6kioJXELxrw7mndBo7owz11j1h3Muvuzbqb6EsNdbaMjg5xf9zKb/MRxYV2iXXW1FLHeAyAiNsiPbWTgsQwwW8SD1jp+b2AwWgGG/ibA8MSVc7MHEWCmopelblE/QxHbUc42O/5sgLYT7XatgqiMzBP10yfB5jXsad70DiIDrvu4K9L0zGkugbfxslir/bSvYjM0anjqU8HBkAapofaGw9yDbU5ikpAx9es4rgzCyhuMr2A8gy0SvbP6PwInZHFbpwAA
@@ -23,6 +23,8 @@ resources:
  - dashboards/nzbget.yaml
  - dashboards/postgresql-database.yaml
  - dashboards/puppet-report.yaml
+  - dashboards/redis.yaml
+  - dashboards/traefik.yaml
  - dashboards/victorialogs-cluster.yaml
  - dashboards/victoriametrics-cluster.yaml
  - dashboards/victoriametrics-vmagent.yaml
@@ -21,5 +21,6 @@ resources:
  - pdb.yaml
  - service.yaml
  - valkey.yaml
+  - vmpodscrape.yaml
  - gateway.yaml
  - httproute.yaml
@@ -25,8 +25,14 @@ spec:
  shards: 1
  replicas: 2
  image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/valkey/valkey:9.0.0
+  # redis_exporter sidecar (:9121, port name `metrics`) on every ValkeyNode pod;
+  # the operator manages a dedicated _exporter ACL user for it. Image overridden
+  # from the operator default (bare dockerhub oliver006/redis_exporter:v1.80.0)
+  # to the artifactapi-proxied pin. Scraped by the valkey-exporter VMPodScrape
+  # in vmpodscrape.yaml alongside this file.
  exporter:
-    enabled: false
+    enabled: true
+    image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/oliver006/redis_exporter:v1.89.0
  scheduling:
    node:
      spread:
@@ -0,0 +1,19 @@
+---
+# Scrape the operator-injected redis_exporter sidecar (:9121, port name
+# `metrics`) on the jellyfin-valkey ValkeyNode pods. The valkey-operator gives
+# its pods fixed labels only (no pod-label passthrough on the ValkeyCluster CR),
+# so select on the operator-managed labels. Picked up by the observability
+# VMAgent (selectAllByDefault).
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: valkey-exporter
+  namespace: jellyfin
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/managed-by: valkey-operator
+      app.kubernetes.io/component: valkey-node
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
@@ -16,6 +16,7 @@ resources:
  - services.yaml
  - vaultauth.yaml
  - vaultstaticsecret.yaml
+  - vmpodscrape.yaml
  - vpa.yaml

 configMapGenerator:
@@ -60,6 +60,20 @@ spec:
            - mountPath: /data
              mountPropagation: None
              name: data
+        - name: metrics-exporter
+          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/oliver006/redis_exporter:v1.89.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 9121
+              name: metrics
+              protocol: TCP
+          resources:
+            limits:
+              cpu: 100m
+              memory: 64Mi
+            requests:
+              cpu: 25m
+              memory: 32Mi
      restartPolicy: Always
      volumes:
        - name: data
@@ -0,0 +1,16 @@
+---
+# Scrape the redis_exporter sidecar (:9121) on the redis pod. Picked up by the
+# observability VMAgent (selectAllByDefault). Pod-level rather than
+# VMServiceScrape because the redis Service doesn't expose the metrics port.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: redis-exporter
+  namespace: litellm
+spec:
+  selector:
+    matchLabels:
+      app: redis
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
@@ -29,9 +29,26 @@ spec:
      default/networks/ip:
        - "127.0.0.1"
        - "::1"
+      # Read-only user for CLI tools + the logviewer UI. Hash sourced from the
+      # Vault-synced clickhouse-logreader Secret, same mechanism as vector.
+      # Scoped to the logs database only (unlike vector, which bootstraps it).
+      logreader/password_sha256_hex:
+        valueFrom:
+          secretKeyRef:
+            name: clickhouse-logreader
+            key: password_sha256_hex
+      logreader/networks/ip:
+        - "::/0"
+      logreader/profile: readonly
+      logreader/quota: default
+      logreader/allow_databases/database:
+        - "logs"
    profiles:
      default/max_memory_usage: "10000000000"
      default/max_execution_time: "120"
+      readonly/readonly: "2"
+      readonly/max_memory_usage: "10000000000"
+      readonly/max_execution_time: "120"
    clusters:
      - name: logs
        layout:
@@ -15,6 +15,7 @@ resources:
  - serviceaccount_logarchiver.yaml
  - configmap_logarchiver.yaml
  - deployment_logarchiver.yaml
+  - logviewer

 # Vector pipelines are the single source of truth (also validated by
 # `vector test` in CI). Mounted into each tier via `existingConfigMaps`.
@@ -0,0 +1,80 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: logviewer
+  namespace: logging
+  annotations:
+    secret.reloader.stakater.com/reload: "clickhouse-logreader"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: logviewer
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: logviewer
+    spec:
+      serviceAccountName: default
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: logviewer
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/logviewer:v0.1.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+          env:
+            - name: CH_URL
+              value: http://clickhouse-logs.logging.svc.cluster.local:8123
+            - name: CH_USER
+              valueFrom:
+                secretKeyRef:
+                  name: clickhouse-logreader
+                  key: username
+            - name: CH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: clickhouse-logreader
+                  key: password
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+      restartPolicy: Always
@@ -0,0 +1,38 @@
+---
+# Internal front for the logviewer UI (cf. mediamover/pdbmux).
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: logviewer.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    external-dns.alpha.kubernetes.io/hostname: logviewer.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+  name: logviewer
+  namespace: logging
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: logviewer.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: logviewer.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: logviewer-tls
+        mode: Terminate
@@ -0,0 +1,49 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: logviewer-http-redirect
+  namespace: logging
+spec:
+  hostnames:
+    - logviewer.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: logviewer
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: logviewer
+  namespace: logging
+spec:
+  hostnames:
+    - logviewer.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: logviewer
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: logviewer-oauth2
+          port: 80
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
@@ -0,0 +1,12 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - vaultstaticsecret.yaml
+  - deployment.yaml
+  - oauth2-proxy-configmap.yaml
+  - oauth2-proxy-deployment.yaml
+  - service.yaml
+  - gateway.yaml
+  - httproute.yaml
@@ -0,0 +1,32 @@
+---
+# Non-secret oauth2-proxy configuration (client_id/secret/cookie_secret come
+# from the logviewer-oauth-credentials Secret). Single auth front for the
+# logviewer UI: everything requires an Authentik session in akP-logviewer-admin.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logviewer-oauth2-env
+  namespace: logging
+data:
+  OAUTH2_PROXY_HTTP_ADDRESS: "0.0.0.0:4180"
+  OAUTH2_PROXY_PROVIDER: "oidc"
+  # Authentik logviewer app discovery issuer (terraform-authentik PR #21).
+  OAUTH2_PROXY_OIDC_ISSUER_URL: "https://identity.unkin.net/application/o/logviewer/"
+  OAUTH2_PROXY_REDIRECT_URL: "https://logviewer.unkin.net/oauth2/callback"
+  OAUTH2_PROXY_UPSTREAMS: "http://logviewer.logging.svc.cluster.local:8080/"
+  OAUTH2_PROXY_SCOPE: "openid email profile ak_groups"
+  # Populate session.Groups from the Authentik ak_groups claim.
+  OAUTH2_PROXY_OIDC_GROUPS_CLAIM: "ak_groups"
+  OAUTH2_PROXY_ALLOWED_GROUPS: "akP-logviewer-admin"
+  OAUTH2_PROXY_PASS_USER_HEADERS: "true"
+  OAUTH2_PROXY_EMAIL_DOMAINS: "*"
+  # Authentik hardcodes email_verified=false in the id_token; authorization is
+  # enforced via ak_groups, so accepting the unverified email is safe.
+  OAUTH2_PROXY_INSECURE_OIDC_ALLOW_UNVERIFIED_EMAIL: "true"
+  OAUTH2_PROXY_COOKIE_SECURE: "true"
+  OAUTH2_PROXY_COOKIE_DOMAINS: "logviewer.unkin.net"
+  OAUTH2_PROXY_WHITELIST_DOMAINS: "logviewer.unkin.net"
+  OAUTH2_PROXY_REVERSE_PROXY: "true"
+  OAUTH2_PROXY_PROVIDER_CA_FILES: "/etc/ssl/combined/ca-certificates.crt"
+  OAUTH2_PROXY_CODE_CHALLENGE_METHOD: "S256"
+  OAUTH2_PROXY_SKIP_PROVIDER_BUTTON: "true"
@@ -0,0 +1,132 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: logviewer-oauth2
+  namespace: logging
+  annotations:
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "logviewer-oauth-credentials,vault-ca-cert"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: logviewer-oauth2
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: logviewer-oauth2
+    spec:
+      serviceAccountName: default
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        # identity.unkin.net serves a Vault-PKI cert; combine the system roots
+        # with the internal CA so oauth2-proxy's OIDC HTTP client trusts it.
+        - name: combine-certs
+          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/alpine:3
+          imagePullPolicy: IfNotPresent
+          command:
+            - sh
+            - -c
+            - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt
+          volumeMounts:
+            - name: vault-ca-cert
+              mountPath: /custom-ca
+              readOnly: true
+            - name: combined-certs
+              mountPath: /combined-certs
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 50m
+              memory: 32Mi
+            limits:
+              cpu: 200m
+              memory: 64Mi
+      containers:
+        - name: oauth2-proxy
+          image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.3
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 4180
+              name: http
+              protocol: TCP
+          envFrom:
+            - configMapRef:
+                name: logviewer-oauth2-env
+                optional: false
+          env:
+            - name: OAUTH2_PROXY_CLIENT_ID
+              valueFrom:
+                secretKeyRef:
+                  name: logviewer-oauth-credentials
+                  key: client_id
+            - name: OAUTH2_PROXY_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: logviewer-oauth-credentials
+                  key: client_secret
+            - name: OAUTH2_PROXY_COOKIE_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: logviewer-oauth-credentials
+                  key: cookie_secret
+          volumeMounts:
+            - name: combined-certs
+              mountPath: /etc/ssl/combined
+              readOnly: true
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 500m
+              memory: 256Mi
+      volumes:
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: combined-certs
+          emptyDir: {}
+      restartPolicy: Always
@@ -0,0 +1,36 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: logviewer
+  namespace: logging
+spec:
+  internalTrafficPolicy: Cluster
+  ports:
+    - name: http
+      port: 8080
+      protocol: TCP
+      targetPort: http
+  selector:
+    app: logviewer
+  sessionAffinity: None
+  type: ClusterIP
+---
+# Front-door entry Service: the HTTPRoute for logviewer.unkin.net targets this;
+# all traffic enters via oauth2-proxy.
+apiVersion: v1
+kind: Service
+metadata:
+  name: logviewer-oauth2
+  namespace: logging
+spec:
+  internalTrafficPolicy: Cluster
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: http
+  selector:
+    app: logviewer-oauth2
+  sessionAffinity: None
+  type: ClusterIP
@@ -0,0 +1,21 @@
+---
+# Authentik OIDC client for logviewer (client_id, client_secret, cookie_secret)
+# seeded at kv/kubernetes/namespace/logging/default/oauth-credentials; the
+# logging/default templated policy already grants read, so no terraform-vault
+# change is needed.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: logviewer-oauth-credentials
+  namespace: logging
+spec:
+  destination:
+    create: true
+    name: logviewer-oauth-credentials
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/logging/default/oauth-credentials
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
@@ -27,6 +27,29 @@ spec:
  type: kv-v2
  vaultAuthRef: default
 ---
+# ClickHouse credentials for the read-only `logreader` user (CLI tools +
+# logviewer UI). Seeded the same way as clickhouse-credentials above:
+#   PW=$(openssl rand -hex 24)
+#   HASH=$(printf '%s' "$PW" | sha256sum | cut -d' ' -f1)
+#   vault kv put kv/kubernetes/namespace/logging/default/clickhouse-logreader \
+#     username=logreader password="$PW" password_sha256_hex="$HASH"
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: clickhouse-logreader
+  namespace: logging
+spec:
+  destination:
+    create: true
+    name: clickhouse-logreader
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/logging/default/clickhouse-logreader
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
 # NATS JetStream auth. Distinct passwords for the producer (edge), consumer
 # (transform tier + archiver) and admin (bootstrap Job) users. Seed once:
 #   for k in admin producer consumer; do declare P_$k=$(openssl rand -base64 24); done
@@ -10,6 +10,7 @@ resources:
  - valkey-deployment.yaml
  - valkey-pvc.yaml
  - valkey-service.yaml
+  - vmpodscrape.yaml
  - vaultauth.yaml
  - vaultstaticsecret.yaml
  - gateway.yaml
@@ -82,6 +82,26 @@ spec:
          volumeMounts:
            - mountPath: /data
              name: data
+        - name: metrics-exporter
+          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/oliver006/redis_exporter:v1.89.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 9121
+              name: metrics
+              protocol: TCP
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsNonRoot: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 100m
+              memory: 64Mi
+            requests:
+              cpu: 25m
+              memory: 32Mi
      restartPolicy: Always
      volumes:
        - name: data
@@ -0,0 +1,16 @@
+---
+# Scrape the redis_exporter sidecar (:9121) on the netbox-valkey pod. Picked up
+# by the observability VMAgent (selectAllByDefault). Pod-level rather than
+# VMServiceScrape because the valkey Service doesn't expose the metrics port.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: valkey-exporter
+  namespace: netbox
+spec:
+  selector:
+    matchLabels:
+      app: netbox-valkey
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
@@ -0,0 +1,201 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-state-metrics
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: kube-state-metrics
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics
+  labels:
+    app.kubernetes.io/name: kube-state-metrics
+rules:
+  - apiGroups: [""]
+    resources:
+      - configmaps
+      - secrets
+      - nodes
+      - pods
+      - services
+      - serviceaccounts
+      - resourcequotas
+      - replicationcontrollers
+      - limitranges
+      - persistentvolumeclaims
+      - persistentvolumes
+      - namespaces
+      - endpoints
+    verbs: [list, watch]
+  - apiGroups: [apps]
+    resources:
+      - statefulsets
+      - daemonsets
+      - deployments
+      - replicasets
+    verbs: [list, watch]
+  - apiGroups: [batch]
+    resources:
+      - cronjobs
+      - jobs
+    verbs: [list, watch]
+  - apiGroups: [autoscaling]
+    resources:
+      - horizontalpodautoscalers
+    verbs: [list, watch]
+  - apiGroups: [authentication.k8s.io]
+    resources:
+      - tokenreviews
+    verbs: [create]
+  - apiGroups: [authorization.k8s.io]
+    resources:
+      - subjectaccessreviews
+    verbs: [create]
+  - apiGroups: [policy]
+    resources:
+      - poddisruptionbudgets
+    verbs: [list, watch]
+  - apiGroups: [certificates.k8s.io]
+    resources:
+      - certificatesigningrequests
+    verbs: [list, watch]
+  - apiGroups: [discovery.k8s.io]
+    resources:
+      - endpointslices
+    verbs: [list, watch]
+  - apiGroups: [storage.k8s.io]
+    resources:
+      - storageclasses
+      - volumeattachments
+    verbs: [list, watch]
+  - apiGroups: [admissionregistration.k8s.io]
+    resources:
+      - mutatingwebhookconfigurations
+      - validatingwebhookconfigurations
+    verbs: [list, watch]
+  - apiGroups: [networking.k8s.io]
+    resources:
+      - networkpolicies
+      - ingressclasses
+      - ingresses
+    verbs: [list, watch]
+  - apiGroups: [coordination.k8s.io]
+    resources:
+      - leases
+    verbs: [list, watch]
+  - apiGroups: [rbac.authorization.k8s.io]
+    resources:
+      - clusterrolebindings
+      - clusterroles
+      - rolebindings
+      - roles
+    verbs: [list, watch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-state-metrics
+  labels:
+    app.kubernetes.io/name: kube-state-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-state-metrics
+subjects:
+  - kind: ServiceAccount
+    name: kube-state-metrics
+    namespace: observability
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-state-metrics
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: kube-state-metrics
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kube-state-metrics
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kube-state-metrics
+    spec:
+      serviceAccountName: kube-state-metrics
+      automountServiceAccountToken: true
+      containers:
+        - name: kube-state-metrics
+          image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.20.0
+          ports:
+            - name: http-metrics
+              containerPort: 8080
+            - name: telemetry
+              containerPort: 8081
+          livenessProbe:
+            httpGet:
+              path: /livez
+              port: http-metrics
+            initialDelaySeconds: 5
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: telemetry
+            initialDelaySeconds: 5
+            timeoutSeconds: 5
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 65534
+            seccompProfile:
+              type: RuntimeDefault
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-state-metrics
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: kube-state-metrics
+spec:
+  clusterIP: None
+  selector:
+    app.kubernetes.io/name: kube-state-metrics
+  ports:
+    - name: http-metrics
+      port: 8080
+      targetPort: http-metrics
+    - name: telemetry
+      port: 8081
+      targetPort: telemetry
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMServiceScrape
+metadata:
+  name: kube-state-metrics
+  namespace: observability
+spec:
+  # jobLabel yields job="kube-state-metrics" as kube-prometheus selectors expect.
+  jobLabel: app.kubernetes.io/name
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kube-state-metrics
+  endpoints:
+    - port: http-metrics
+      honorLabels: true
+    - port: telemetry
@@ -6,6 +6,10 @@ resources:
  - namespace.yaml
  - vmcluster.yaml
  - vmagent.yaml
+  - vmalert.yaml
+  - vmrule-k8s.yaml
+  - kube-state-metrics.yaml
  - gateway.yaml
  - httproute.yaml
  - vmpodscrape-cnpg.yaml
+  - vmpodscrape-traefik.yaml
@@ -49,7 +49,8 @@ spec:
            - __meta_kubernetes_endpoint_port_name
          action: keep
          regex: default;kubernetes;https
-    - job_name: "kubernetes-nodes"
+    # Only the node name goes onto series: a blanket node-label labelmap pushes past vminsert's 40-label limit and drops everything.
+    - job_name: "kubelet"
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
@@ -58,9 +59,11 @@ spec:
      kubernetes_sd_configs:
        - role: node
      relabel_configs:
-        - action: labelmap
-          regex: __meta_kubernetes_node_label_(.+)
-    - job_name: "kubernetes-nodes-cadvisor"
+        - source_labels: [__meta_kubernetes_node_name]
+          target_label: node
+        - source_labels: [__metrics_path__]
+          target_label: metrics_path
+    - job_name: "kubelet-cadvisor"
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
@@ -70,10 +73,13 @@ spec:
        - role: node
      metrics_path: /metrics/cadvisor
      relabel_configs:
-        - action: labelmap
-          regex: __meta_kubernetes_node_label_(.+)
+        - source_labels: [__meta_kubernetes_node_name]
+          target_label: node
        - source_labels: [__metrics_path__]
          target_label: metrics_path
+        # job_name must be unique; kube-prometheus selectors expect job="kubelet"
+        - target_label: job
+          replacement: kubelet
      metric_relabel_configs:
        - action: replace
          source_labels: [pod]
@@ -0,0 +1,27 @@
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMAlert
+metadata:
+  name: main
+  namespace: observability
+spec:
+  replicaCount: 1
+  selectAllByDefault: true
+  evaluationInterval: 30s
+  datasource:
+    url: http://vmselect-main.observability.svc.cluster.local:8481/select/0/prometheus/
+  remoteWrite:
+    url: http://vminsert-main.observability.svc.cluster.local:8480/insert/0/prometheus/
+  remoteRead:
+    url: http://vmselect-main.observability.svc.cluster.local:8481/select/0/prometheus/
+  extraArgs:
+    loggerFormat: json
+    # recording rules only for now; no alertmanager deployed
+    notifier.blackhole: "true"
+  resources:
+    requests:
+      cpu: 50m
+      memory: 128Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
@@ -0,0 +1,23 @@
+---
+# Scrape metrics (:9100) from the traefik-internal and traefik-external gateway
+# pods. Picked up by the observability VMAgent (selectAllByDefault). The
+# app.kubernetes.io/instance pod label is kept as traefik_instance so series
+# from the two releases stay distinguishable.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: traefik
+  namespace: observability
+spec:
+  namespaceSelector:
+    matchNames:
+      - traefik-system
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: traefik
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
+      relabelConfigs:
+        - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
+          targetLabel: traefik_instance
@@ -0,0 +1,133 @@
+---
+# kube-prometheus k8s.rules recording rules (dashboard queries depend on these).
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  name: k8s-rules
+  namespace: observability
+spec:
+  groups:
+    - name: k8s.rules
+      rules:
+        - record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate
+          expr: |
+            sum by (cluster, namespace, pod, container) (
+              irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m])
+            ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (
+              1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
+            )
+        - record: node_namespace_pod_container:container_memory_working_set_bytes
+          expr: |
+            container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
+            * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (
+              1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
+            )
+        - record: node_namespace_pod_container:container_memory_rss
+          expr: |
+            container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
+            * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (
+              1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
+            )
+        - record: node_namespace_pod_container:container_memory_cache
+          expr: |
+            container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
+            * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (
+              1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
+            )
+        - record: node_namespace_pod_container:container_memory_swap
+          expr: |
+            container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
+            * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (
+              1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
+            )
+        - record: namespace_memory:kube_pod_container_resource_requests:sum
+          expr: |
+            sum by (namespace, cluster) (
+              sum by (namespace, pod, cluster) (
+                max by (namespace, pod, container, cluster) (
+                  kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"}
+                ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
+                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
+                )
+              )
+            )
+        - record: namespace_cpu:kube_pod_container_resource_requests:sum
+          expr: |
+            sum by (namespace, cluster) (
+              sum by (namespace, pod, cluster) (
+                max by (namespace, pod, container, cluster) (
+                  kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"}
+                ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
+                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
+                )
+              )
+            )
+        - record: namespace_memory:kube_pod_container_resource_limits:sum
+          expr: |
+            sum by (namespace, cluster) (
+              sum by (namespace, pod, cluster) (
+                max by (namespace, pod, container, cluster) (
+                  kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"}
+                ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
+                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
+                )
+              )
+            )
+        - record: namespace_cpu:kube_pod_container_resource_limits:sum
+          expr: |
+            sum by (namespace, cluster) (
+              sum by (namespace, pod, cluster) (
+                max by (namespace, pod, container, cluster) (
+                  kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"}
+                ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
+                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
+                )
+              )
+            )
+        - record: namespace_workload_pod:kube_pod_owner:relabel
+          labels:
+            workload_type: deployment
+          expr: |
+            max by (cluster, namespace, workload, pod) (
+              label_replace(
+                label_replace(
+                  kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"},
+                  "replicaset", "$1", "owner_name", "(.*)"
+                ) * on (replicaset, namespace) group_left(owner_name) topk by (replicaset, namespace) (
+                  1, max by (replicaset, namespace, owner_name) (
+                    kube_replicaset_owner{job="kube-state-metrics"}
+                  )
+                ),
+                "workload", "$1", "owner_name", "(.*)"
+              )
+            )
+        - record: namespace_workload_pod:kube_pod_owner:relabel
+          labels:
+            workload_type: daemonset
+          expr: |
+            max by (cluster, namespace, workload, pod) (
+              label_replace(
+                kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"},
+                "workload", "$1", "owner_name", "(.*)"
+              )
+            )
+        - record: namespace_workload_pod:kube_pod_owner:relabel
+          labels:
+            workload_type: statefulset
+          expr: |
+            max by (cluster, namespace, workload, pod) (
+              label_replace(
+                kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"},
+                "workload", "$1", "owner_name", "(.*)"
+              )
+            )
+        - record: namespace_workload_pod:kube_pod_owner:relabel
+          labels:
+            workload_type: job
+          expr: |
+            max by (cluster, namespace, workload, pod) (
+              label_replace(
+                kube_pod_owner{job="kube-state-metrics", owner_kind="Job"},
+                "workload", "$1", "owner_name", "(.*)"
+              )
+            )
@@ -29,6 +29,11 @@ providers:
    nativeLBByDefault: false
    labelSelector: "traefik.io/instance=external"

+metrics:
+  prometheus:
+    addEntryPointsLabels: true
+    addRoutersLabels: true
+
 logs:
  access:
    enabled: true
@@ -29,6 +29,11 @@ providers:
    nativeLBByDefault: false
    labelSelector: "traefik.io/instance=internal"

+metrics:
+  prometheus:
+    addEntryPointsLabels: true
+    addRoutersLabels: true
+
 logs:
  access:
    enabled: true