logging: set clickhouse readonly profile to readonly=2 (#409)

## Why

The `readonly` profile used by `logreader` sets `readonly=1`, which rejects any per-query settings change. HTTP clients (logviewer, chlog) send settings params like `max_execution_time` with their queries, so every query fails with `Code: 164 DB::Exception: Cannot modify 'max_execution_time' setting in readonly mode` — logviewer healthz returns 503 and CrashLoops, chlog queries fail. `readonly=2` keeps data access read-only while permitting settings changes, which is what HTTP clients need.

## Changes

- Changes the `readonly` profile in `apps/base/logging/clickhouseinstallation.yaml` from `readonly: "1"` to `readonly: "2"`

Reviewed-on: #409
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

.woodpecker/vector-test.yaml

@@ -0,0 +1,29 @@
+when:
+  - event: pull_request
+
+steps:
+  - name: vector-test
+    image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector:0.57.0-debian
+    commands:
+      # Dummy creds + writable dirs so the full topologies build; the unit tests
+      # only exercise the transforms (sources are not started).
+      - export CLICKHOUSE_USER=ci CLICKHOUSE_PASSWORD=ci
+      - export NATS_PRODUCER_PASSWORD=ci NATS_CONSUMER_PASSWORD=ci
+      - mkdir -p /vector-data-dir /etc/vault-ca
+      - cp /etc/ssl/certs/ca-certificates.crt /etc/vault-ca/ca.crt
+      # Transform tier + VM ingest: unit-tested transforms.
+      - vector test apps/base/logging/vector/aggregator.yaml apps/base/logging/vector/aggregator-tests.yaml
+      - vector test apps/base/logging/vector/vm-ingest.yaml apps/base/logging/vector/vm-ingest-tests.yaml
+      # Agent has no transforms to unit-test; validate it builds. (The archiver
+      # leg is now the logarchiver service, not a Vector pipeline.)
+      - vector validate --no-environment apps/base/logging/vector/agent.yaml
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 256Mi
+            cpu: 250m
+          limits:
+            memory: 1Gi
+            cpu: 1

apps/base/age-api/deployment.yaml

@@ -12,13 +12,13 @@ spec:
   template:
     metadata:
       annotations:
-        reloader.stakater.com/auto: "true"
+        configmap.reloader.stakater.com/auto: "true"
       labels:
         app: age-api
     spec:
       containers:
         - name: age-api
-          image: git.unkin.net/unkin/age-api:v0.1.0
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/age-api:v0.1.0
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8080

apps/base/age-api/kustomization.yaml

@@ -8,6 +8,7 @@ resources:
   - httproute.yaml
   - namespace.yaml
   - service.yaml
+  - vpa.yaml
 
 configMapGenerator:
   - name: age-api-config

apps/base/age-api/vpa.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: autoscaling.k8s.io/v1
+kind: VerticalPodAutoscaler
+metadata:
+  name: age-api-vpa
+  namespace: age-api
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: age-api
+  updatePolicy:
+    updateMode: "Off"

apps/base/arrstack/arrproxy/api-deployment.yaml

@@ -0,0 +1,145 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: arrproxy-api
+  namespace: arrstack
+  annotations:
+    # Wave 2: serve only after the wave-1 migrate Job completes.
+    argocd.argoproj.io/sync-wave: "2"
+    secret.reloader.stakater.com/reload: "arrproxy-pepper,arrproxy-admin-token,arrproxy-db-app,sonarr-apikey,radarr-apikey,prowlarr-apikey"
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: arrproxy-api
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: arrproxy-api
+    spec:
+      serviceAccountName: default
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: api
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/arrproxy-api:v0.3.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+          env:
+            - name: ARRPROXY_ADDR
+              value: ":8080"
+            # oauth2-proxy --pass-user-headers forwards identity to the upstream as
+            # X-Forwarded-{User,Email,Groups} (a single comma-joined Groups value).
+            # Email/User already match the api defaults; override the groups header
+            # (default X-Auth-Request-Groups is auth_request-response-only and never
+            # reaches this upstream) so group-based authorization works.
+            - name: ARRPROXY_GROUPS_HEADER
+              value: X-Forwarded-Groups
+            # Real per-app *arr keys, projected one file per app under this dir
+            # (sourced from the existing <app>-apikey Secrets). The api injects
+            # them server-side and redacts them from every proxied response.
+            - name: ARRPROXY_KEYS_DIR
+              value: /etc/arrproxy/keys
+            - name: ARRPROXY_PEPPER
+              valueFrom:
+                secretKeyRef:
+                  name: arrproxy-pepper
+                  key: pepper
+            # Machine-mint admin bearer, synced from Vault KV via the
+            # arrproxy-admin-token VSO. Gates the /api/admin/ route that
+            # oauth2-proxy intentionally skip-auths so OpenBao on the VMs can
+            # mint tokens against arrproxy's own bearer.
+            - name: ARRPROXY_ADMIN_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: arrproxy-admin-token
+                  key: token
+            # DSN assembled from the CNPG-generated arrproxy-db-app Secret;
+            # $(VAR) expansion resolves the two env entries defined above it.
+            - name: ARRPROXY_DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: arrproxy-db-app
+                  key: username
+            - name: ARRPROXY_DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: arrproxy-db-app
+                  key: password
+            - name: DATABASE_URL
+              value: "postgres://$(ARRPROXY_DB_USER):$(ARRPROXY_DB_PASSWORD)@arrproxy-db-rw.arrstack.svc.cluster.local:5432/arrproxy?sslmode=require"
+          volumeMounts:
+            - name: arr-keys
+              mountPath: /etc/arrproxy/keys
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+          livenessProbe:
+            httpGet:
+              path: /livez
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: "1"
+              memory: 512Mi
+      volumes:
+        # Real *arr API keys, one file per app named exactly <app> so the api
+        # reads /etc/arrproxy/keys/{sonarr,radarr,prowlarr}. Reuses the same
+        # <app>-apikey Secrets the *arr Deployments already consume.
+        - name: arr-keys
+          projected:
+            sources:
+              - secret:
+                  name: sonarr-apikey
+                  items:
+                    - key: apitoken
+                      path: sonarr
+              - secret:
+                  name: radarr-apikey
+                  items:
+                    - key: apitoken
+                      path: radarr
+              - secret:
+                  name: prowlarr-apikey
+                  items:
+                    - key: apitoken
+                      path: prowlarr
+        - name: tmp
+          emptyDir:
+            sizeLimit: 64Mi
+      restartPolicy: Always

apps/base/arrstack/arrproxy/cnpg_backup.yaml

@@ -0,0 +1,51 @@
+---
+# Ceph RGW (S3) backup target for the arrproxy token-store CNPG cluster,
+# provisioned by the in-estate cephrgw-operator. One dedicated bucket + owner
+# user per cluster.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: cnpg-arrproxy-backup
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  displayName: "CNPG backup owner (arrproxy)"
+  uid: cnpg-arrproxy-backup
+  maxBuckets: 5
+  secretName: cnpg-arrproxy-backup-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: cnpg-arrproxy
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  placementTarget: ec
+  bucketName: cnpg-arrproxy
+  ownerRef: cnpg-arrproxy-backup
+  versioning: false
+  tags:
+    app: arrproxy
+    purpose: cnpg-backup
+  retainOnDelete: true
+---
+# Nightly base backup; continuous WAL archiving is always-on via the Cluster's
+# spec.backup.barmanObjectStore. Staggered off other clusters' schedules.
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-arrproxy-nightly
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  schedule: "0 20 2 * * *"
+  immediate: false
+  backupOwnerReference: self
+  method: barmanObjectStore
+  cluster:
+    name: arrproxy-db

apps/base/arrstack/arrproxy/cnpg_cluster.yaml

@@ -0,0 +1,117 @@
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: arrproxy-db
+  namespace: arrstack
+  annotations:
+    # Wave 0: DB (and the generated arrproxy-db-app Secret) must be Healthy before
+    # the wave-1 migrate Job runs. ArgoCD gates on the Cluster's health status.
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  affinity:
+    podAntiAffinityType: preferred
+  backup:
+    retentionPolicy: 30d
+    barmanObjectStore:
+      destinationPath: s3://cnpg-arrproxy
+      endpointURL: https://s3.ceph.unkin.net
+      endpointCA:
+        name: vault-ca-cert
+        key: ca.crt
+      s3Credentials:
+        accessKeyId:
+          name: cnpg-arrproxy-backup-s3
+          key: AWS_ACCESS_KEY_ID
+        secretAccessKey:
+          name: cnpg-arrproxy-backup-s3
+          key: AWS_SECRET_ACCESS_KEY
+      serverName: arrproxy
+      data:
+        compression: bzip2
+        jobs: 2
+      wal:
+        compression: zstd
+        maxParallel: 2
+  bootstrap:
+    initdb:
+      # No secret ref: CNPG mints the owner credentials and publishes them in the
+      # generated "arrproxy-db-app" Secret, which the api reads to build the DSN.
+      database: arrproxy
+      encoding: UTF8
+      localeCType: C
+      localeCollate: C
+      owner: arrproxy
+  enablePDB: true
+  enableSuperuserAccess: false
+  failoverDelay: 0
+  imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie
+  instances: 2
+  logLevel: info
+  maxSyncReplicas: 0
+  minSyncReplicas: 0
+  monitoring:
+    customQueriesConfigMap:
+      - key: queries
+        name: cnpg-default-monitoring
+    disableDefaultQueries: false
+    enablePodMonitor: false
+  postgresql:
+    parameters:
+      archive_mode: "on"
+      archive_timeout: 5min
+      dynamic_shared_memory_type: posix
+      effective_cache_size: 256MB
+      full_page_writes: "on"
+      log_destination: csvlog
+      log_directory: /controller/log
+      log_filename: postgres
+      log_rotation_age: "0"
+      log_rotation_size: "0"
+      log_truncate_on_rotation: "false"
+      logging_collector: "on"
+      max_connections: "200"
+      max_parallel_workers: "16"
+      max_replication_slots: "16"
+      max_worker_processes: "16"
+      shared_buffers: 128MB
+      shared_memory_type: mmap
+      ssl_max_protocol_version: TLSv1.3
+      ssl_min_protocol_version: TLSv1.3
+      wal_keep_size: 256MB
+      wal_level: logical
+      wal_log_hints: "on"
+      wal_receiver_timeout: 5s
+      wal_sender_timeout: 5s
+    syncReplicaElectionConstraint:
+      enabled: false
+  primaryUpdateMethod: restart
+  primaryUpdateStrategy: unsupervised
+  probes:
+    liveness:
+      isolationCheck:
+        connectionTimeout: 1000
+        enabled: true
+        requestTimeout: 1000
+  replicationSlots:
+    highAvailability:
+      enabled: true
+      slotPrefix: _cnpg_
+    synchronizeReplicas:
+      enabled: true
+    updateInterval: 30
+  resources:
+    limits:
+      cpu: 500m
+      memory: 512Mi
+    requests:
+      cpu: 50m
+      memory: 256Mi
+  smartShutdownTimeout: 180
+  startDelay: 3600
+  stopDelay: 1800
+  storage:
+    resizeInUseVolumes: true
+    size: 10Gi
+    storageClass: cephrbd-fast-delete
+  switchoverDelay: 3600

apps/base/arrstack/arrproxy/gateway.yaml

@@ -0,0 +1,41 @@
+---
+# External (DMZ) front for the arrstack, served on arrstack.unkin.net via the
+# external Traefik (LB VIP 198.18.199.0). cert-manager mints arrproxy-gateway-tls
+# (CN arrstack.unkin.net) off the internal Vault-PKI CA. The apex arrstack.unkin.net
+# A record lives in the bind-operator unkin.net zone (bind-internal/authoritative),
+# NOT external-dns, so no external-dns annotation here.
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: external
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: arrstack.unkin.net
+    cert-manager.io/private-key-size: "4096"
+  name: arrproxy
+  namespace: arrstack
+spec:
+  gatewayClassName: traefik-external
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      hostname: arrstack.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https
+      port: 443
+      protocol: HTTPS
+      hostname: arrstack.unkin.net
+      allowedRoutes:
+        namespaces:
+          from: Same
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: arrproxy-gateway-tls

apps/base/arrstack/arrproxy/httproute.yaml

@@ -0,0 +1,58 @@
+---
+# Redirect plain HTTP to HTTPS.
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: arrproxy-http-redirect
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+spec:
+  hostnames:
+    - arrstack.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: arrproxy
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+# All HTTPS traffic enters via oauth2-proxy (the arrproxy entry Service), which
+# authenticates and path-routes to arrproxy-ui / arrproxy-api. The UI-vs-api and
+# the /<app>/api oauth-bypass split is done inside oauth2-proxy (upstreams +
+# skip-auth-regex), so a single backend here is sufficient.
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: arrproxy-route
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+spec:
+  hostnames:
+    - arrstack.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: arrproxy
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: arrproxy
+          port: 80
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/arrstack/arrproxy/kustomization.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - cnpg_cluster.yaml
+  - cnpg_backup.yaml
+  - migrations-configmap.yaml
+  - migrate-job.yaml
+  - vaultstaticsecret.yaml
+  - oauth2-proxy-configmap.yaml
+  - oauth2-proxy-deployment.yaml
+  - api-deployment.yaml
+  - ui-deployment.yaml
+  - services.yaml
+  - gateway.yaml
+  - httproute.yaml

apps/base/arrstack/arrproxy/migrate-job.yaml

@@ -0,0 +1,92 @@
+---
+# Applies the arrproxy schema once per sync, before the api rolls, so the serve
+# replicas never race migrations (arrproxy-api does not self-migrate). Runs as the
+# CNPG-minted app user so the tokens table is owned by that role.
+#
+# Sync-phase hook at wave 1 (NOT PreSync): the CNPG Cluster + generated
+# arrproxy-db-app Secret apply at wave 0 and ArgoCD waits for the Cluster to be
+# Healthy before starting wave 1, so Postgres exists before migrate connects.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: arrproxy-migrate
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/hook: Sync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  backoffLimit: 6
+  ttlSecondsAfterFinished: 600
+  template:
+    metadata:
+      labels:
+        app: arrproxy-migrate
+    spec:
+      serviceAccountName: default
+      automountServiceAccountToken: false
+      restartPolicy: Never
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: migrate
+          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/postgres:18-alpine
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: HOME
+              value: /tmp
+            - name: PGUSER
+              valueFrom:
+                secretKeyRef:
+                  name: arrproxy-db-app
+                  key: username
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: arrproxy-db-app
+                  key: password
+            - name: PGHOST
+              value: arrproxy-db-rw.arrstack.svc.cluster.local
+            - name: PGPORT
+              value: "5432"
+            - name: PGDATABASE
+              value: arrproxy
+            - name: PGSSLMODE
+              value: require
+          command:
+            - psql
+            - -v
+            - ON_ERROR_STOP=1
+            - -f
+            - /migrations/0001_init.sql
+          volumeMounts:
+            - name: migrations
+              mountPath: /migrations
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 256Mi
+      volumes:
+        - name: migrations
+          configMap:
+            name: arrproxy-migrations
+        - name: tmp
+          emptyDir:
+            sizeLimit: 64Mi

apps/base/arrstack/arrproxy/migrations-configmap.yaml

@@ -0,0 +1,29 @@
+---
+# arrproxy schema, mirrored from the arrproxy repo migrations/0001_init.sql
+# (v0.1.0). arrproxy-api does NOT self-migrate, so the wave-1 migrate Job applies
+# this once per sync as the app user. Keep in sync with the repo on schema bumps.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: arrproxy-migrations
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+data:
+  0001_init.sql: |
+    -- arrproxy token store. Only token hashes are persisted; plaintext is shown
+    -- once at mint time and never recoverable.
+    CREATE TABLE IF NOT EXISTS tokens (
+        id           TEXT PRIMARY KEY,
+        subject      TEXT        NOT NULL,
+        label        TEXT        NOT NULL DEFAULT '',
+        token_hash   TEXT        NOT NULL UNIQUE,
+        apps         TEXT[]      NOT NULL DEFAULT '{}',
+        created_at   TIMESTAMPTZ NOT NULL DEFAULT now(),
+        expires_at   TIMESTAMPTZ,
+        disabled     BOOLEAN     NOT NULL DEFAULT false,
+        last_used_at TIMESTAMPTZ
+    );
+
+    CREATE INDEX IF NOT EXISTS tokens_subject_idx ON tokens (subject);
+    CREATE INDEX IF NOT EXISTS tokens_token_hash_idx ON tokens (token_hash);

apps/base/arrstack/arrproxy/oauth2-proxy-configmap.yaml

@@ -0,0 +1,56 @@
+---
+# Non-secret oauth2-proxy configuration (client_id/secret/cookie_secret come from
+# the oauth-credentials Secret). oauth2-proxy is the single auth front for the
+# arrstack: it authenticates the UI and the token API against Authentik, and path-
+# routes to the arrproxy-ui / arrproxy-api upstreams. /<app>/api is exempted from
+# auth (SKIP_AUTH_REGEX) so *arr clients presenting a per-user token reach the api
+# directly; the api then validates the token. Everything else requires the oauth
+# session and receives identity via X-Forwarded-* / X-Auth-Request-* headers.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: arrproxy-oauth2-env
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+data:
+  OAUTH2_PROXY_HTTP_ADDRESS: "0.0.0.0:4180"
+  OAUTH2_PROXY_PROVIDER: "oidc"
+  # Authentik arrstack app discovery issuer (served by the internal unkin.net CA;
+  # trusted via PROVIDER_CA_FILES below). CONFIRM the slug matches the Authentik
+  # application (terraform-authentik PR #18).
+  OAUTH2_PROXY_OIDC_ISSUER_URL: "https://identity.unkin.net/application/o/arrstack/"
+  OAUTH2_PROXY_REDIRECT_URL: "https://arrstack.unkin.net/oauth2/callback"
+  # Longest-prefix wins: /api and /<app> go to arrproxy-api, everything else
+  # (the SPA + static assets) to arrproxy-ui.
+  OAUTH2_PROXY_UPSTREAMS: "http://arrproxy-ui.arrstack.svc.cluster.local:8080/,http://arrproxy-api.arrstack.svc.cluster.local:8080/api/,http://arrproxy-api.arrstack.svc.cluster.local:8080/sonarr/,http://arrproxy-api.arrstack.svc.cluster.local:8080/radarr/,http://arrproxy-api.arrstack.svc.cluster.local:8080/prowlarr/"
+  OAUTH2_PROXY_SCOPE: "openid email profile ak_groups"
+  # Populate session.Groups from the Authentik ak_groups claim; pass-user-headers
+  # then emits it as a single comma-joined X-Forwarded-Groups header.
+  OAUTH2_PROXY_OIDC_GROUPS_CLAIM: "ak_groups"
+  # Forward identity + groups to arrproxy-api as X-Forwarded-{User,Email,Groups}
+  # (the api reads these; ARRPROXY_GROUPS_HEADER=X-Forwarded-Groups). NOTE:
+  # set-xauthrequest is intentionally NOT set -- it only populates auth_request
+  # *response* headers, which never reach an --upstreams-proxied backend.
+  OAUTH2_PROXY_PASS_USER_HEADERS: "true"
+  # Bypass auth for the *arr proxy API (/<app>/api...) and the machine-mint
+  # admin route (/api/admin/...). The first matches /sonarr/api; the second
+  # matches /api/admin/ only -- both routed to the arrproxy-api upstream by the
+  # catch-all /api/ prefix above. /api/admin/ is protected by arrproxy's OWN
+  # ARRPROXY_ADMIN_TOKEN bearer (OpenBao on the VMs reaches it via the ingress),
+  # so it is intentionally oauth-skipped. /api/tokens and /api/me are NOT
+  # matched and stay oauth-authenticated.
+  OAUTH2_PROXY_SKIP_AUTH_REGEX: "^/[^/]+/api,^/api/admin/"
+  OAUTH2_PROXY_EMAIL_DOMAINS: "*"
+  # Authentik hardcodes email_verified=false in the id_token; without this
+  # oauth2-proxy rejects the session ("email ... isn't verified") -> 500 on
+  # /oauth2/callback. Authorization is enforced downstream via ak_groups, so
+  # accepting the unverified email here is safe.
+  OAUTH2_PROXY_INSECURE_OIDC_ALLOW_UNVERIFIED_EMAIL: "true"
+  OAUTH2_PROXY_COOKIE_SECURE: "true"
+  OAUTH2_PROXY_COOKIE_DOMAINS: "arrstack.unkin.net"
+  OAUTH2_PROXY_WHITELIST_DOMAINS: "arrstack.unkin.net"
+  OAUTH2_PROXY_REVERSE_PROXY: "true"
+  OAUTH2_PROXY_PROVIDER_CA_FILES: "/etc/ssl/combined/ca-certificates.crt"
+  OAUTH2_PROXY_CODE_CHALLENGE_METHOD: "S256"
+  OAUTH2_PROXY_SKIP_PROVIDER_BUTTON: "true"

apps/base/arrstack/arrproxy/oauth2-proxy-deployment.yaml

@@ -0,0 +1,133 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: arrproxy-oauth2
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "oauth-credentials,vault-ca-cert"
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: arrproxy-oauth2
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: arrproxy-oauth2
+    spec:
+      serviceAccountName: default
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        # identity.unkin.net serves a Vault-PKI cert; combine the system roots
+        # with the internal CA so oauth2-proxy's OIDC HTTP client trusts it.
+        - name: combine-certs
+          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/alpine:3
+          imagePullPolicy: IfNotPresent
+          command:
+            - sh
+            - -c
+            - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt
+          volumeMounts:
+            - name: vault-ca-cert
+              mountPath: /custom-ca
+              readOnly: true
+            - name: combined-certs
+              mountPath: /combined-certs
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 50m
+              memory: 32Mi
+            limits:
+              cpu: 200m
+              memory: 64Mi
+      containers:
+        - name: oauth2-proxy
+          image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.3
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 4180
+              name: http
+              protocol: TCP
+          envFrom:
+            - configMapRef:
+                name: arrproxy-oauth2-env
+                optional: false
+          env:
+            - name: OAUTH2_PROXY_CLIENT_ID
+              valueFrom:
+                secretKeyRef:
+                  name: oauth-credentials
+                  key: client_id
+            - name: OAUTH2_PROXY_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: oauth-credentials
+                  key: client_secret
+            - name: OAUTH2_PROXY_COOKIE_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: oauth-credentials
+                  key: cookie_secret
+          volumeMounts:
+            - name: combined-certs
+              mountPath: /etc/ssl/combined
+              readOnly: true
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 500m
+              memory: 256Mi
+      volumes:
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt
+        - name: combined-certs
+          emptyDir: {}
+      restartPolicy: Always

apps/base/arrstack/arrproxy/services.yaml

@@ -0,0 +1,59 @@
+---
+# Front-door entry Service: the HTTPRoute for arrstack.unkin.net targets this.
+# All traffic (UI, token API, and the *arr proxy) enters via oauth2-proxy.
+apiVersion: v1
+kind: Service
+metadata:
+  name: arrproxy
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+spec:
+  internalTrafficPolicy: Cluster
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: http
+  selector:
+    app: arrproxy-oauth2
+  sessionAffinity: None
+  type: ClusterIP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: arrproxy-api
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+spec:
+  internalTrafficPolicy: Cluster
+  ports:
+    - name: http
+      port: 8080
+      protocol: TCP
+      targetPort: http
+  selector:
+    app: arrproxy-api
+  sessionAffinity: None
+  type: ClusterIP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: arrproxy-ui
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+spec:
+  internalTrafficPolicy: Cluster
+  ports:
+    - name: http
+      port: 8080
+      protocol: TCP
+      targetPort: http
+  selector:
+    app: arrproxy-ui
+  sessionAffinity: None
+  type: ClusterIP

apps/base/arrstack/arrproxy/ui-deployment.yaml

@@ -0,0 +1,72 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: arrproxy-ui
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: arrproxy-ui
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: arrproxy-ui
+    spec:
+      serviceAccountName: default
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: ui
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/arrproxy-ui:v0.3.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+          env:
+            - name: ARRPROXY_UI_ADDR
+              value: ":8080"
+          livenessProbe:
+            httpGet:
+              path: /livez
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 50m
+              memory: 32Mi
+            limits:
+              cpu: 200m
+              memory: 128Mi
+      restartPolicy: Always

apps/base/arrstack/arrproxy/vaultstaticsecret.yaml

@@ -0,0 +1,76 @@
+---
+# Per-deployment token-hash pepper. Seeded (openssl rand) at
+# kv/kubernetes/namespace/arrstack/default/arrproxy-pepper (key: pepper); the
+# default k8s role's templated policy already grants read on
+# kv/data/kubernetes/namespace/{{sa_namespace}}/{{sa_name}}/* for the
+# arrstack/default ServiceAccount, so no terraform-vault change is needed. VSO
+# syncs it into the arrproxy-pepper Secret consumed by arrproxy-api as
+# ARRPROXY_PEPPER.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: arrproxy-pepper
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  destination:
+    create: true
+    name: arrproxy-pepper
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/arrstack/default/arrproxy-pepper
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+# Machine-mint admin bearer token. Seeded (openssl rand) at
+# kv/kubernetes/namespace/arrstack/default/arrproxy-admin-token (key: token) and
+# shared as the source of truth with the future Vault engine. The default k8s
+# role's templated policy already grants read on
+# kv/data/kubernetes/namespace/{{sa_namespace}}/{{sa_name}}/* for the
+# arrstack/default ServiceAccount, so no terraform-vault change is needed. VSO
+# syncs it into the arrproxy-admin-token Secret consumed by arrproxy-api as
+# ARRPROXY_ADMIN_TOKEN to gate the bearer-protected /api/admin/ route.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: arrproxy-admin-token
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  destination:
+    create: true
+    name: arrproxy-admin-token
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/arrstack/default/arrproxy-admin-token
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+# Authentik OIDC client for the arrstack front door (client_id, client_secret,
+# cookie_secret), created by terraform-authentik at
+# kv/kubernetes/namespace/arrstack/default/oauth-credentials. VSO syncs it into
+# the oauth-credentials Secret consumed by the oauth2-proxy Deployment.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: oauth-credentials
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  destination:
+    create: true
+    name: oauth-credentials
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/arrstack/default/oauth-credentials
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/base/arrstack/backups-bucket.yaml

@@ -0,0 +1,42 @@
+---
+# Shared Ceph RGW (S3) bucket for arrstack application backups (the periodic
+# radarr/sonarr/prowlarr config+database zip backups), provisioned by the
+# in-estate cephrgw-operator. Backups otherwise land on each replica's local
+# /config (an emptyDir) and are lost when that pod is rescheduled; routing them
+# to S3 lets any replica write and restore them.
+#
+# Backups are kept in their own bucket (not the arrstack-media one) because they
+# carry secrets (Config.xml holds API keys, the DB has all app state) and thus
+# warrant separate credentials, lifecycle and retention from the public-ish
+# poster art. The three apps share this one bucket, isolated by a per-app key
+# prefix (radarr/, sonarr/, prowlarr/) set via <App>__BackupS3__Prefix.
+#
+# The operator mints the S3 credential Secret (arrstack-backups-s3) in this
+# namespace with keys AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and S3_ENDPOINT,
+# so no Vault KV seeding is required.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: arrstack-backups
+  namespace: arrstack
+spec:
+  displayName: "arrstack backups owner"
+  uid: arrstack-backups
+  maxBuckets: 5
+  secretName: arrstack-backups-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: arrstack-backups
+  namespace: arrstack
+spec:
+  placementTarget: ec
+  bucketName: arrstack-backups
+  ownerRef: arrstack-backups
+  versioning: false
+  tags:
+    app: arrstack
+    purpose: backups
+  retainOnDelete: true

apps/base/arrstack/kustomization.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - vaultauth.yaml
+  - pv-media-tv.yaml
+  - pv-media-movies.yaml
+  - pv-mediafs.yaml
+  - pvc-media-tv.yaml
+  - pvc-media-movies.yaml
+  - pvc-mediafs.yaml
+  - media-bucket.yaml
+  - backups-bucket.yaml
+  - postgres
+  - valkey
+  - sonarr
+  - radarr
+  - prowlarr
+  - nzbget
+  - arrproxy
+  - mediamover

apps/base/arrstack/media-bucket.yaml

@@ -0,0 +1,34 @@
+---
+# Shared Ceph RGW (S3) bucket for arrstack media assets (Servarr MediaCover
+# posters/fanart and, later, application backups), provisioned by the in-estate
+# cephrgw-operator. Moving these off each replica's local /config (an emptyDir)
+# lets any radarr/sonarr/prowlarr replica serve covers and store backups without
+# leader-local state. The operator mints the S3 credential Secret
+# (arrstack-media-s3) in this namespace with keys AWS_ACCESS_KEY_ID,
+# AWS_SECRET_ACCESS_KEY and S3_ENDPOINT, so no Vault KV seeding is required.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: arrstack-media
+  namespace: arrstack
+spec:
+  displayName: "arrstack media assets owner"
+  uid: arrstack-media
+  maxBuckets: 5
+  secretName: arrstack-media-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: arrstack-media
+  namespace: arrstack
+spec:
+  placementTarget: ec
+  bucketName: arrstack-media
+  ownerRef: arrstack-media
+  versioning: false
+  tags:
+    app: arrstack
+    purpose: media-assets
+  retainOnDelete: true

apps/base/arrstack/mediamover/deployment.yaml

@@ -0,0 +1,95 @@
+---
+# mediamover server: REST API + UI on :8080. Spawns one worker Job per queued
+# file using this same image with `worker` args (Job spec lives in code; workers
+# run as the default ServiceAccount and mount the same PVCs). The queue is
+# in-memory, so keep a single replica; a restart just loses queued entries.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mediamover
+  namespace: arrstack
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mediamover
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: mediamover
+    spec:
+      serviceAccountName: mediamover
+      automountServiceAccountToken: true
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: server
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/mediamover:v0.1.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - server
+            - --src-root=/srv/src
+            - --src-pvc=mediafs
+            - --dst-roots=movies=/srv/dst/movies,tv=/srv/dst/tv
+            - --dst-pvc=movies=media-movies,tv=media-tv
+            - --namespace=arrstack
+            - --image=artifactapi.k8s.syd1.au.unkin.net/docker-internal/mediamover:v0.1.0
+          ports:
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+          volumeMounts:
+            # RW: move mode deletes the source file after a successful copy.
+            - name: src
+              mountPath: /srv/src
+            - name: dst-movies
+              mountPath: /srv/dst/movies
+            - name: dst-tv
+              mountPath: /srv/dst/tv
+          livenessProbe:
+            httpGet:
+              path: /api/limit
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /api/limit
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+      volumes:
+        - name: src
+          persistentVolumeClaim:
+            claimName: mediafs
+        - name: dst-movies
+          persistentVolumeClaim:
+            claimName: media-movies
+        - name: dst-tv
+          persistentVolumeClaim:
+            claimName: media-tv
+      restartPolicy: Always

apps/base/arrstack/mediamover/gateway.yaml

@@ -0,0 +1,40 @@
+---
+# Internal front for mediamover. The existing arrstack (arrproxy) Gateway is
+# external and hostname-locked to arrstack.unkin.net, so this tool gets its own
+# internal Gateway following the cluster convention (cf. pdbmux).
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: mediamover.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    external-dns.alpha.kubernetes.io/hostname: mediamover.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+  name: mediamover
+  namespace: arrstack
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: mediamover.k8s.syd1.au.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: mediamover.k8s.syd1.au.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: mediamover-tls
+        mode: Terminate

apps/base/arrstack/mediamover/httproute.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: mediamover-http-redirect
+  namespace: arrstack
+spec:
+  hostnames:
+    - mediamover.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: mediamover
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: mediamover
+  namespace: arrstack
+spec:
+  hostnames:
+    - mediamover.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: mediamover
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: mediamover
+          port: 8080
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/arrstack/mediamover/kustomization.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - serviceaccount.yaml
+  - rbac.yaml
+  - deployment.yaml
+  - service.yaml
+  - gateway.yaml
+  - httproute.yaml

apps/base/arrstack/mediamover/rbac.yaml

@@ -0,0 +1,48 @@
+---
+# The server creates one worker Job per queued file and polls Job/Pod state to
+# track progress and clean up.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mediamover
+  namespace: arrstack
+rules:
+  - apiGroups:
+      - batch
+    resources:
+      - jobs
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+      - delete
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods/log
+    verbs:
+      - get
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mediamover
+  namespace: arrstack
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mediamover
+subjects:
+  - kind: ServiceAccount
+    name: mediamover
+    namespace: arrstack

apps/base/arrstack/mediamover/service.yaml

@@ -0,0 +1,19 @@
+---
+# Must stay named "mediamover" on port 8080: worker Jobs default their progress
+# callback URL to http://mediamover.<namespace>.svc:8080.
+apiVersion: v1
+kind: Service
+metadata:
+  name: mediamover
+  namespace: arrstack
+spec:
+  internalTrafficPolicy: Cluster
+  ports:
+    - name: http
+      port: 8080
+      protocol: TCP
+      targetPort: http
+  selector:
+    app: mediamover
+  sessionAffinity: None
+  type: ClusterIP

apps/base/arrstack/mediamover/serviceaccount.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mediamover
+  namespace: arrstack

apps/base/arrstack/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: arrstack

apps/base/arrstack/nzbget/deployment.yaml

@@ -0,0 +1,137 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nzbget
+  namespace: arrstack
+spec:
+  replicas: 1
+  strategy:
+    # RWO config PVC + single queue state: never run two pods at once.
+    type: Recreate
+  selector:
+    matchLabels:
+      app: nzbget
+  template:
+    metadata:
+      labels:
+        app: nzbget
+    spec:
+      securityContext:
+        fsGroup: 1000
+        fsGroupChangePolicy: OnRootMismatch
+      initContainers:
+        # Seed download layout onto the shared media PVCs (not /config or an
+        # emptyDir) so completed downloads land beside the arr libraries and
+        # imports are same-filesystem hardlink moves. Reuses the image's own
+        # template (inherits correct WebDir/ConfigTemplate) and appends the
+        # path/category overrides once; nzbget honours the last value for a
+        # repeated option, and the grep guard keeps re-runs idempotent so admin
+        # UI edits to the persisted /config/nzbget.conf survive restarts.
+        - name: seed-config
+          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/linuxserver/nzbget:version-v26.2
+          command:
+            - sh
+            - -c
+            - |
+              set -e
+              if [ ! -f /config/nzbget.conf ]; then
+                cp /app/nzbget/share/nzbget/nzbget.conf /config/nzbget.conf
+              fi
+              if ! grep -q '# arrstack-managed' /config/nzbget.conf; then
+                cat >> /config/nzbget.conf << 'CONF'
+
+              # arrstack-managed download layout (appended once; last value wins).
+              # Downloads land on the shared media PVCs by category so sonarr/radarr
+              # import with atomic hardlink moves (download dir + library share one
+              # filesystem per media type). InterDir is empty: nzbget writes each
+              # download straight into its category DestDir, so BOTH tv and movies
+              # stay on their own PVC with no cross-filesystem intermediate copy.
+              MainDir=/media/tv
+              InterDir=
+              DestDir=/media/tv/downloads
+              NzbDir=/config/nzb
+              QueueDir=/config/queue
+              TempDir=/config/tmp
+              ControlIP=0.0.0.0
+              ControlPort=6789
+              Category1.Name=tv
+              Category1.DestDir=/media/tv/downloads
+              Category2.Name=movies
+              Category2.DestDir=/media/movies/downloads
+              CONF
+              fi
+              mkdir -p /media/tv/series /media/tv/downloads /media/movies/films /media/movies/downloads
+              chown 1000:1000 /config/nzbget.conf \
+                /media/tv /media/tv/series /media/tv/downloads \
+                /media/movies /media/movies/films /media/movies/downloads
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 200m
+              memory: 128Mi
+          volumeMounts:
+            - name: config
+              mountPath: /config
+            - name: media-tv
+              mountPath: /media/tv
+            - name: media-movies
+              mountPath: /media/movies
+      containers:
+        - name: nzbget
+          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/linuxserver/nzbget:version-v26.2
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 6789
+              protocol: TCP
+          env:
+            - name: PUID
+              value: "1000"
+            - name: PGID
+              value: "1000"
+            - name: TZ
+              value: Australia/Sydney
+          livenessProbe:
+            # nzbget's root path requires auth (401); a TCP check is the
+            # dependency-free liveness signal for the web/JSON-RPC server.
+            tcpSocket:
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            tcpSocket:
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 200m
+              memory: 256Mi
+            limits:
+              # Headroom for par2 repair + unpack of large downloads.
+              cpu: "2"
+              memory: 2Gi
+          volumeMounts:
+            - name: config
+              mountPath: /config
+            - name: media-tv
+              mountPath: /media/tv
+            - name: media-movies
+              mountPath: /media/movies
+      volumes:
+        - name: config
+          persistentVolumeClaim:
+            claimName: nzbget-config
+        - name: media-tv
+          persistentVolumeClaim:
+            claimName: media-tv
+        - name: media-movies
+          persistentVolumeClaim:
+            claimName: media-movies

apps/base/arrstack/nzbget/gateway.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: nzbget.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    external-dns.alpha.kubernetes.io/hostname: nzbget.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+  name: nzbget
+  namespace: arrstack
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: nzbget.k8s.syd1.au.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: nzbget.k8s.syd1.au.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: nzbget-tls
+        mode: Terminate

apps/base/arrstack/nzbget/httproute.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: nzbget-http-redirect
+  namespace: arrstack
+spec:
+  hostnames:
+    - nzbget.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: nzbget
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: nzbget-route
+  namespace: arrstack
+spec:
+  hostnames:
+    - nzbget.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: nzbget
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: nzbget
+          port: 6789
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/arrstack/nzbget/kustomization.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - pvc-config.yaml
+  - deployment.yaml
+  - service.yaml
+  - gateway.yaml
+  - httproute.yaml

apps/base/arrstack/nzbget/pvc-config.yaml

@@ -0,0 +1,16 @@
+---
+# NZBGet config + queue/temp state. RWO on cephrbd (block). Retain: this is
+# state. The download data itself lives on the shared media PVCs, not here.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: nzbget-config
+  namespace: arrstack
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+  storageClassName: cephrbd-fast-retain
+  volumeMode: Filesystem

apps/base/arrstack/nzbget/service.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nzbget
+  namespace: arrstack
+spec:
+  ports:
+    - name: http
+      port: 6789
+      protocol: TCP
+      targetPort: http
+  selector:
+    app: nzbget
+  type: ClusterIP

apps/base/arrstack/postgres/cnpg_backup.yaml

@@ -0,0 +1,45 @@
+---
+# Ceph RGW (S3) backup target for the shared arrstack CNPG cluster, provisioned
+# by the in-estate cephrgw-operator: one dedicated bucket + owner user. CNPG
+# reads the S3 credential Secret (cnpg-arrstack-backup-s3) from this namespace.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: cnpg-arrstack-backup
+  namespace: arrstack
+spec:
+  displayName: "CNPG backup owner (arrstack)"
+  uid: cnpg-arrstack-backup
+  maxBuckets: 5
+  secretName: cnpg-arrstack-backup-s3
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: cnpg-arrstack
+  namespace: arrstack
+spec:
+  placementTarget: ec
+  bucketName: cnpg-arrstack
+  ownerRef: cnpg-arrstack-backup
+  versioning: false
+  tags:
+    app: arrstack
+    purpose: cnpg-backup
+  retainOnDelete: true
+---
+# Nightly base backup on top of always-on WAL archiving. Staggered from the
+# other CNPG clusters (6-field cron, seconds first).
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-arrstack-nightly
+  namespace: arrstack
+spec:
+  schedule: "0 45 3 * * *"
+  immediate: false
+  backupOwnerReference: self
+  method: barmanObjectStore
+  cluster:
+    name: arrstack-postgres

apps/base/arrstack/postgres/cnpg_cluster.yaml

@@ -0,0 +1,161 @@
+---
+# Shared PostgreSQL backend for the -unkin2 fork sonarr/radarr/prowlarr, whose
+# Npgsql/EF Core provider moves each *arr off SQLite into Postgres and makes the
+# shared-nothing, active-active multi-replica deployment possible. One cluster,
+# one throwaway initdb owner ("app"), and three managed login roles — one per
+# app — each with its own per-app database (see database-*.yaml). Role passwords
+# come from the VSO-synced <app>-db Secrets (vaultstaticsecret.yaml), so no
+# credential is rendered into git.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: arrstack-postgres
+  namespace: arrstack
+spec:
+  inheritedMetadata:
+    annotations:
+      k8up.io/backup: "false"
+  affinity:
+    podAntiAffinityType: preferred
+  backup:
+    retentionPolicy: 30d
+    barmanObjectStore:
+      destinationPath: s3://cnpg-arrstack
+      endpointURL: https://s3.ceph.unkin.net
+      endpointCA:
+        name: vault-ca-cert
+        key: ca.crt
+      s3Credentials:
+        accessKeyId:
+          name: cnpg-arrstack-backup-s3
+          key: AWS_ACCESS_KEY_ID
+        secretAccessKey:
+          name: cnpg-arrstack-backup-s3
+          key: AWS_SECRET_ACCESS_KEY
+      serverName: arrstack
+      data:
+        compression: bzip2
+        jobs: 2
+      wal:
+        compression: zstd
+        maxParallel: 2
+  bootstrap:
+    initdb:
+      # Throwaway owner + database: the real per-app databases are provisioned by
+      # the CNPG Database CRDs, owned by the managed roles below.
+      database: app
+      encoding: UTF8
+      localeCType: C
+      localeCollate: C
+      owner: app
+  managed:
+    roles:
+      - name: sonarr
+        ensure: present
+        comment: Sonarr application role (owns sonarr-main)
+        login: true
+        superuser: false
+        createdb: false
+        createrole: false
+        inherit: true
+        replication: false
+        connectionLimit: -1
+        passwordSecret:
+          name: sonarr-db
+      - name: radarr
+        ensure: present
+        comment: Radarr application role (owns radarr-main)
+        login: true
+        superuser: false
+        createdb: false
+        createrole: false
+        inherit: true
+        replication: false
+        connectionLimit: -1
+        passwordSecret:
+          name: radarr-db
+      - name: prowlarr
+        ensure: present
+        comment: Prowlarr application role (owns prowlarr-main)
+        login: true
+        superuser: false
+        createdb: false
+        createrole: false
+        inherit: true
+        replication: false
+        connectionLimit: -1
+        passwordSecret:
+          name: prowlarr-db
+  enablePDB: true
+  enableSuperuserAccess: false
+  failoverDelay: 0
+  imageName: ghcr.io/cloudnative-pg/postgresql:17-system-trixie
+  instances: 3
+  logLevel: info
+  maxSyncReplicas: 0
+  minSyncReplicas: 0
+  monitoring:
+    customQueriesConfigMap:
+      - key: queries
+        name: cnpg-default-monitoring
+    disableDefaultQueries: false
+    enablePodMonitor: false
+  postgresql:
+    parameters:
+      archive_mode: "on"
+      archive_timeout: 5min
+      dynamic_shared_memory_type: posix
+      effective_cache_size: 256MB
+      full_page_writes: "on"
+      log_destination: csvlog
+      log_directory: /controller/log
+      log_filename: postgres
+      log_rotation_age: "0"
+      log_rotation_size: "0"
+      log_truncate_on_rotation: "false"
+      logging_collector: "on"
+      max_connections: "200"
+      max_parallel_workers: "16"
+      max_replication_slots: "16"
+      max_worker_processes: "16"
+      shared_buffers: 128MB
+      shared_memory_type: mmap
+      ssl_max_protocol_version: TLSv1.3
+      ssl_min_protocol_version: TLSv1.3
+      wal_keep_size: 256MB
+      wal_level: logical
+      wal_log_hints: "on"
+      wal_receiver_timeout: 5s
+      wal_sender_timeout: 5s
+    syncReplicaElectionConstraint:
+      enabled: false
+  primaryUpdateMethod: restart
+  primaryUpdateStrategy: unsupervised
+  probes:
+    liveness:
+      isolationCheck:
+        connectionTimeout: 1000
+        enabled: true
+        requestTimeout: 1000
+  replicationSlots:
+    highAvailability:
+      enabled: true
+      slotPrefix: _cnpg_
+    synchronizeReplicas:
+      enabled: true
+    updateInterval: 30
+  resources:
+    limits:
+      cpu: "1"
+      memory: 2Gi
+    requests:
+      cpu: 250m
+      memory: 1Gi
+  smartShutdownTimeout: 180
+  startDelay: 3600
+  stopDelay: 1800
+  storage:
+    resizeInUseVolumes: true
+    size: 10Gi
+    storageClass: cephrbd-fast-delete
+  switchoverDelay: 3600

apps/base/arrstack/postgres/database-prowlarr.yaml

@@ -0,0 +1,15 @@
+---
+# Per-app database owned by the prowlarr managed role. The fork's provider runs
+# its own schema migrations on first start (advisory-locked, so only one replica
+# migrates). retain: the database survives a Database CRD delete.
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: prowlarr-main
+  namespace: arrstack
+spec:
+  cluster:
+    name: arrstack-postgres
+  name: prowlarr-main
+  owner: prowlarr
+  databaseReclaimPolicy: retain

apps/base/arrstack/postgres/database-radarr.yaml

@@ -0,0 +1,15 @@
+---
+# Per-app database owned by the radarr managed role. The fork's provider runs its
+# own schema migrations on first start (advisory-locked, so only one replica
+# migrates). retain: the database survives a Database CRD delete.
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: radarr-main
+  namespace: arrstack
+spec:
+  cluster:
+    name: arrstack-postgres
+  name: radarr-main
+  owner: radarr
+  databaseReclaimPolicy: retain

apps/base/arrstack/postgres/database-sonarr.yaml

@@ -0,0 +1,15 @@
+---
+# Per-app database owned by the sonarr managed role. The fork's provider runs its
+# own schema migrations on first start (advisory-locked, so only one replica
+# migrates). retain: the database survives a Database CRD delete.
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: sonarr-main
+  namespace: arrstack
+spec:
+  cluster:
+    name: arrstack-postgres
+  name: sonarr-main
+  owner: sonarr
+  databaseReclaimPolicy: retain

apps/base/arrstack/postgres/kustomization.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - vaultstaticsecret.yaml
+  - cnpg_cluster.yaml
+  - cnpg_backup.yaml
+  - database-sonarr.yaml
+  - database-radarr.yaml
+  - database-prowlarr.yaml

apps/base/arrstack/postgres/vaultstaticsecret.yaml

@@ -0,0 +1,60 @@
+---
+# Per-app Postgres role credentials. Each is seeded out-of-band at
+# kv/kubernetes/namespace/arrstack/default/<app>-db (keys: username, password);
+# the default k8s role's templated policy already grants read on
+# kv/data/kubernetes/namespace/{{sa_namespace}}/{{sa_name}}/* for the
+# arrstack/default ServiceAccount, so no terraform-vault change is needed. VSO
+# syncs each into the <app>-db Secret, which is both the CNPG managed role's
+# passwordSecret (cnpg_cluster.yaml) and the source of the app Deployment's
+# <App>__Postgres__User/__Password env. Wave 0: must exist before the Cluster
+# (wave 1) reconciles the roles.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: sonarr-db
+  namespace: arrstack
+spec:
+  destination:
+    create: true
+    name: sonarr-db
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/arrstack/default/sonarr-db
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: radarr-db
+  namespace: arrstack
+spec:
+  destination:
+    create: true
+    name: radarr-db
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/arrstack/default/radarr-db
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: prowlarr-db
+  namespace: arrstack
+spec:
+  destination:
+    create: true
+    name: prowlarr-db
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/arrstack/default/prowlarr-db
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/base/arrstack/prowlarr/configmap.yaml

@@ -0,0 +1,33 @@
+---
+# Non-secret env for the -unkin2 fork. The fork reads Servarr config from
+# Prowlarr__<Section>__<Key> env (no config.xml edits, no s6/PUID). Postgres
+# wiring points every replica at the same shared DB (arrstack-postgres-rw /
+# prowlarr-main); Auth__Method=External defers UI auth to arrproxy/oauth2-proxy;
+# Server__UrlBase keeps the /prowlarr prefix so arrproxy path-routing works;
+# App__InstanceName is identical across replicas (shared session-cookie name).
+# User/Password/ApiKey come from Secrets (see deployment.yaml), not here.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prowlarr-env
+  namespace: arrstack
+data:
+  Prowlarr__Postgres__Host: arrstack-postgres-rw.arrstack.svc.cluster.local
+  Prowlarr__Postgres__Port: "5432"
+  Prowlarr__Postgres__MainDb: prowlarr-main
+  Prowlarr__Log__DbEnabled: "false"
+  Prowlarr__Auth__Method: External
+  Prowlarr__Auth__Required: DisabledForLocalAddresses
+  Prowlarr__App__InstanceName: Prowlarr
+  Prowlarr__Server__Port: "9696"
+  Prowlarr__Server__UrlBase: /prowlarr
+  Prowlarr__Update__Mechanism: External
+  # Shared arrstack Valkey (valkey-operator). Setting Host is what activates the
+  # fork's #14 Redis features (SignalR backplane, cross-replica cache-invalidation
+  # bus, distributed rate limiter): RedisOptions.IsConfigured gates purely on a
+  # non-empty Host, so there is no separate Enabled flag. The operator leaves the
+  # default user passwordless (jellyfin parity), so no Password/Ssl is wired.
+  # Channels/keys are namespaced by this fork's prowlarr:ratelimit: prefix, so the
+  # one cluster is safe to share with sonarr/radarr.
+  Prowlarr__Redis__Host: valkey-arrstack-valkey.arrstack.svc.cluster.local
+  Prowlarr__Redis__Port: "6379"

apps/base/arrstack/prowlarr/deployment.yaml

@@ -0,0 +1,174 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prowlarr
+  namespace: arrstack
+  annotations:
+    # prowlarr-env is a plain (unhashed) ConfigMap consumed by fixed-name envFrom,
+    # so editing it does not roll the Deployment on its own. Reloader watches the
+    # referenced ConfigMap and triggers a rolling restart on change, so adding the
+    # Redis env activates the #14 features on the next ArgoCD sync without a manual
+    # `rollout restart`.
+    configmap.reloader.stakater.com/auto: "true"
+spec:
+  # Active-active: the -unkin2 fork keeps all state in the shared Postgres
+  # (arrstack-postgres) and coordinates via Postgres advisory locks, so N
+  # replicas run concurrently behind the prowlarr Service. RollingUpdate is safe
+  # — no SQLite, no RWO lock.
+  replicas: 3
+  strategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: prowlarr
+  template:
+    metadata:
+      labels:
+        app: prowlarr
+    spec:
+      securityContext:
+        # Fork image has no USER (runs as root by default); pin it to a non-root
+        # UID and group-write the shared RWX CephFS /config. OnRootMismatch
+        # avoids a recursive chown of the whole volume.
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+        fsGroupChangePolicy: OnRootMismatch
+      initContainers:
+        # Gate the app on its own Postgres database+role being reachable, instead
+        # of relying on ArgoCD sync-waves (which deadlock if apps aren't Healthy).
+        # waitfordb reads the PG* env as a libpq fallback, so the password never lands in argv.
+        - name: wait-for-db
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/waitfordb:v0.1.0
+          env:
+            - name: WAITFORDB_TIMEOUT
+              value: 5m
+            - name: WAITFORDB_SSLMODE
+              value: disable
+            - name: PGHOST
+              value: arrstack-postgres-rw.arrstack.svc.cluster.local
+            - name: PGPORT
+              value: "5432"
+            - name: PGDATABASE
+              value: prowlarr-main
+            - name: PGUSER
+              valueFrom:
+                secretKeyRef:
+                  name: prowlarr-db
+                  key: username
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: prowlarr-db
+                  key: password
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 64Mi
+      containers:
+        - name: prowlarr
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/prowlarr:v2.6.2-unkin6
+          imagePullPolicy: IfNotPresent
+          command:
+            - /app/Prowlarr
+          args:
+            - -nobrowser
+            - -data=/config
+            # Required: bypass the single-instance guard so multiple replicas
+            # can share one /config. Cross-replica safety is the Postgres layer,
+            # not a local lock file.
+            - -nosingleinstancecheck
+          ports:
+            - name: http
+              containerPort: 9696
+              protocol: TCP
+          envFrom:
+            - configMapRef:
+                name: prowlarr-env
+          env:
+            - name: Prowlarr__Postgres__User
+              valueFrom:
+                secretKeyRef:
+                  name: prowlarr-db
+                  key: username
+            - name: Prowlarr__Postgres__Password
+              valueFrom:
+                secretKeyRef:
+                  name: prowlarr-db
+                  key: password
+            - name: Prowlarr__Auth__ApiKey
+              valueFrom:
+                secretKeyRef:
+                  name: prowlarr-apikey
+                  key: apitoken
+            # Backup object store (shared arrstack-backups Ceph RGW bucket,
+            # per-app key prefix). Routes the periodic config+DB zip backups off
+            # the ephemeral /config so any replica can write and restore them.
+            # Consumed by the -unkin3+ image; older images ignore these unknown
+            # config keys. Creds Secret minted by cephrgw-operator.
+            - name: Prowlarr__BackupS3__Endpoint
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: S3_ENDPOINT
+            - name: Prowlarr__BackupS3__AccessKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: AWS_ACCESS_KEY_ID
+            - name: Prowlarr__BackupS3__SecretKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: Prowlarr__BackupS3__Bucket
+              value: arrstack-backups
+            - name: Prowlarr__BackupS3__Prefix
+              value: prowlarr
+            - name: Prowlarr__BackupS3__ForcePathStyle
+              value: "true"
+            - name: Prowlarr__BackupS3__CaCertPath
+              value: /etc/ssl/vault-ca/ca.crt
+          livenessProbe:
+            httpGet:
+              path: /prowlarr/ping
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /prowlarr/ping
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: "1"
+              memory: 1Gi
+          volumeMounts:
+            - name: config
+              mountPath: /config
+            - name: vault-ca
+              mountPath: /etc/ssl/vault-ca
+              readOnly: true
+      volumes:
+        - name: config
+          emptyDir: {}
+        # Estate CA for validating the Ceph RGW (s3.ceph.unkin.net) TLS cert.
+        - name: vault-ca
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt

apps/base/arrstack/prowlarr/gateway.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: prowlarr.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    external-dns.alpha.kubernetes.io/hostname: prowlarr.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+  name: prowlarr
+  namespace: arrstack
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: prowlarr.k8s.syd1.au.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: prowlarr.k8s.syd1.au.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: prowlarr-tls
+        mode: Terminate

apps/base/arrstack/prowlarr/httproute.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: prowlarr-http-redirect
+  namespace: arrstack
+spec:
+  hostnames:
+    - prowlarr.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: prowlarr
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: prowlarr-route
+  namespace: arrstack
+spec:
+  hostnames:
+    - prowlarr.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: prowlarr
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: prowlarr
+          port: 9696
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/arrstack/prowlarr/kustomization.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - vaultstaticsecret.yaml
+  - configmap.yaml
+  - deployment.yaml
+  - service.yaml
+  - gateway.yaml
+  - httproute.yaml

apps/base/arrstack/prowlarr/pvc-config.yaml

@@ -0,0 +1,17 @@
+---
+# Prowlarr /config. RWX on CephFS so all replicas share it (the -unkin2 fork
+# keeps the database in Postgres; /config now holds only config.xml + assets,
+# which tolerate — and want — shared access). Retain: this is state.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: prowlarr-config
+  namespace: arrstack
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 5Gi
+  storageClassName: cephfs-raid5-retain
+  volumeMode: Filesystem

apps/base/arrstack/prowlarr/service.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: prowlarr
+  namespace: arrstack
+spec:
+  ports:
+    - name: http
+      port: 9696
+      protocol: TCP
+      targetPort: http
+  selector:
+    app: prowlarr
+  type: ClusterIP

apps/base/arrstack/prowlarr/vaultstaticsecret.yaml

@@ -0,0 +1,25 @@
+---
+# prowlarr API key. Seeded at kv/kubernetes/namespace/arrstack/default/prowlarr
+# (key: apitoken); the default k8s role's templated policy already grants read
+# on kv/data/kubernetes/namespace/{{sa_namespace}}/{{sa_name}}/* for the
+# arrstack/default ServiceAccount, so no terraform-vault change is needed. VSO
+# syncs it into the prowlarr-apikey Secret that the apikey-init initContainer reads
+# to enforce <ApiKey> in /config/config.xml (Vault is source of truth).
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: prowlarr-apikey
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  destination:
+    create: true
+    name: prowlarr-apikey
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/arrstack/default/prowlarr
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/base/arrstack/pv-media-movies.yaml

@@ -0,0 +1,31 @@
+---
+# Static PV for the shared MOVIES CephFS subvolume. Same rootPath as jellyfin's
+# movies PV so radarr/nzbget write and jellyfin reads the identical library
+# tree; each namespace gets its own PV (unique name + volumeHandle) pinned by
+# claimRef.
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: arrstack-media-movies
+spec:
+  capacity:
+    storage: 1Ti
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: ""
+  volumeMode: Filesystem
+  claimRef:
+    namespace: arrstack
+    name: media-movies
+  csi:
+    driver: cephfs.csi.ceph.com
+    volumeHandle: arrstack-media-movies-static
+    nodeStageSecretRef:
+      name: csi-cephfs-secret
+      namespace: csi-cephfs
+    volumeAttributes:
+      staticVolume: "true"
+      clusterID: cephfs_csi_ssd_ec_4_1
+      fsName: cephfs
+      rootPath: /volumes/csi_ssd_ec_4_1/media-movies/e95d8ace-c736-465a-acc3-0c3e46dcede9

apps/base/arrstack/pv-media-tv.yaml

@@ -0,0 +1,30 @@
+---
+# Static PV for the shared TV CephFS subvolume. Same rootPath as jellyfin's TV
+# PV so sonarr/nzbget write and jellyfin reads the identical library tree; each
+# namespace gets its own PV (unique name + volumeHandle) pinned by claimRef.
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: arrstack-media-tv
+spec:
+  capacity:
+    storage: 1Ti
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: ""
+  volumeMode: Filesystem
+  claimRef:
+    namespace: arrstack
+    name: media-tv
+  csi:
+    driver: cephfs.csi.ceph.com
+    volumeHandle: arrstack-media-tv-static
+    nodeStageSecretRef:
+      name: csi-cephfs-secret
+      namespace: csi-cephfs
+    volumeAttributes:
+      staticVolume: "true"
+      clusterID: cephfs_csi_ssd_ec_4_1
+      fsName: cephfs
+      rootPath: /volumes/csi_ssd_ec_4_1/media-tv/4692957d-f5df-4f72-b9c9-56e4ee6d1333

apps/base/arrstack/pv-mediafs.yaml

@@ -0,0 +1,31 @@
+---
+# Static PV exposing the legacy mediafs CephFS filesystem root for the upcoming
+# mediamover tool. clusterID only selects the monitor set from the csi config;
+# fsName selects the actual filesystem, so the ssd_ec clusterID still reaches
+# mediafs on the same cluster. Staged with the dedicated ceph-mediafs client.
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: arrstack-mediafs
+spec:
+  capacity:
+    storage: 10Ti
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: ""
+  volumeMode: Filesystem
+  claimRef:
+    namespace: arrstack
+    name: mediafs
+  csi:
+    driver: cephfs.csi.ceph.com
+    volumeHandle: arrstack-mediafs-static
+    nodeStageSecretRef:
+      name: ceph-mediafs-secret
+      namespace: csi-cephfs
+    volumeAttributes:
+      staticVolume: "true"
+      clusterID: cephfs_csi_ssd_ec_4_1
+      fsName: mediafs
+      rootPath: /

apps/base/arrstack/pvc-media-movies.yaml

@@ -0,0 +1,22 @@
+---
+# Movies library + downloads, shared RWX across radarr and nzbget. Statically
+# bound to the arrstack-media-movies PV (same CephFS subvolume jellyfin mounts
+# read-only). storageClassName "" + volumeName disables dynamic provisioning and
+# binds the pre-created static PV. Downloads and library live on one filesystem
+# so import is an atomic hardlink move.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: media-movies
+  namespace: arrstack
+  annotations:
+    k8up.io/backup: "false"
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Ti
+  storageClassName: ""
+  volumeName: arrstack-media-movies
+  volumeMode: Filesystem

apps/base/arrstack/pvc-media-tv.yaml

@@ -0,0 +1,22 @@
+---
+# TV library + downloads, shared RWX across sonarr and nzbget. Statically bound
+# to the arrstack-media-tv PV (same CephFS subvolume jellyfin mounts read-only).
+# storageClassName "" + volumeName disables dynamic provisioning and binds the
+# pre-created static PV. Downloads and library live on one filesystem so import
+# is an atomic hardlink move.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: media-tv
+  namespace: arrstack
+  annotations:
+    k8up.io/backup: "false"
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Ti
+  storageClassName: ""
+  volumeName: arrstack-media-tv
+  volumeMode: Filesystem

apps/base/arrstack/pvc-mediafs.yaml

@@ -0,0 +1,20 @@
+---
+# Legacy mediafs filesystem root, statically bound to the arrstack-mediafs PV
+# for the upcoming mediamover tool. storageClassName "" + volumeName disables
+# dynamic provisioning and binds the pre-created static PV.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: mediafs
+  namespace: arrstack
+  annotations:
+    k8up.io/backup: "false"
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 10Ti
+  storageClassName: ""
+  volumeName: arrstack-mediafs
+  volumeMode: Filesystem

apps/base/arrstack/radarr/configmap.yaml

@@ -0,0 +1,33 @@
+---
+# Non-secret env for the -unkin2 fork. The fork reads Servarr config from
+# Radarr__<Section>__<Key> env (no config.xml edits, no s6/PUID). Postgres wiring
+# points every replica at the same shared DB (arrstack-postgres-rw / radarr-main);
+# Auth__Method=External defers UI auth to arrproxy/oauth2-proxy; Server__UrlBase
+# keeps the /radarr prefix so arrproxy path-routing works; App__InstanceName is
+# identical across replicas (shared session-cookie name). User/Password/ApiKey
+# come from Secrets (see deployment.yaml), not here.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: radarr-env
+  namespace: arrstack
+data:
+  Radarr__Postgres__Host: arrstack-postgres-rw.arrstack.svc.cluster.local
+  Radarr__Postgres__Port: "5432"
+  Radarr__Postgres__MainDb: radarr-main
+  Radarr__Log__DbEnabled: "false"
+  Radarr__Auth__Method: External
+  Radarr__Auth__Required: DisabledForLocalAddresses
+  Radarr__App__InstanceName: Radarr
+  Radarr__Server__Port: "7878"
+  Radarr__Server__UrlBase: /radarr
+  Radarr__Update__Mechanism: External
+  # Shared arrstack Valkey (valkey-operator). Setting Host is what activates the
+  # fork's #14 Redis features (SignalR backplane, cross-replica cache-invalidation
+  # bus, distributed rate limiter): RedisOptions.IsConfigured gates purely on a
+  # non-empty Host, so there is no separate Enabled flag. The operator leaves the
+  # default user passwordless (jellyfin parity), so no Password/Ssl is wired.
+  # Channels/keys are namespaced by this fork's radarr:ratelimit: prefix, so the
+  # one cluster is safe to share with sonarr/prowlarr.
+  Radarr__Redis__Host: valkey-arrstack-valkey.arrstack.svc.cluster.local
+  Radarr__Redis__Port: "6379"

apps/base/arrstack/radarr/deployment.yaml

@@ -0,0 +1,249 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: radarr
+  namespace: arrstack
+  annotations:
+    # radarr-env is a plain (unhashed) ConfigMap consumed by fixed-name envFrom,
+    # so editing it does not roll the Deployment on its own. Reloader watches the
+    # referenced ConfigMap and triggers a rolling restart on change, so adding the
+    # Redis env activates the #14 features on the next ArgoCD sync without a manual
+    # `rollout restart`.
+    configmap.reloader.stakater.com/auto: "true"
+spec:
+  # Active-active: the -unkin2 fork keeps all state in the shared Postgres
+  # (arrstack-postgres) and coordinates via Postgres advisory locks, so N
+  # replicas run concurrently behind the radarr Service. RollingUpdate is safe —
+  # no SQLite, no RWO lock.
+  replicas: 3
+  strategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: radarr
+  template:
+    metadata:
+      labels:
+        app: radarr
+    spec:
+      securityContext:
+        # Fork image has no USER (runs as root by default); pin it to a non-root
+        # UID and group-write the shared RWX CephFS /config (MediaCover etc.).
+        # OnRootMismatch avoids a recursive chown of the whole media tree.
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+        fsGroupChangePolicy: OnRootMismatch
+      initContainers:
+        # Gate the app on its own Postgres database+role being reachable, instead
+        # of relying on ArgoCD sync-waves (which deadlock if apps aren't Healthy).
+        # waitfordb reads the PG* env as a libpq fallback, so the password never lands in argv.
+        - name: wait-for-db
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/waitfordb:v0.1.0
+          env:
+            - name: WAITFORDB_TIMEOUT
+              value: 5m
+            - name: WAITFORDB_SSLMODE
+              value: disable
+            - name: PGHOST
+              value: arrstack-postgres-rw.arrstack.svc.cluster.local
+            - name: PGPORT
+              value: "5432"
+            - name: PGDATABASE
+              value: radarr-main
+            - name: PGUSER
+              valueFrom:
+                secretKeyRef:
+                  name: radarr-db
+                  key: username
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: radarr-db
+                  key: password
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 64Mi
+      containers:
+        - name: radarr
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/radarr:v6.4.2-unkin7
+          imagePullPolicy: IfNotPresent
+          command:
+            - /app/Radarr
+          args:
+            - -nobrowser
+            - -data=/config
+            # Required: bypass the single-instance guard so multiple replicas
+            # can share one /config. Cross-replica safety is the Postgres layer,
+            # not a local lock file.
+            - -nosingleinstancecheck
+          ports:
+            - name: http
+              containerPort: 7878
+              protocol: TCP
+          envFrom:
+            - configMapRef:
+                name: radarr-env
+          env:
+            - name: Radarr__Postgres__User
+              valueFrom:
+                secretKeyRef:
+                  name: radarr-db
+                  key: username
+            - name: Radarr__Postgres__Password
+              valueFrom:
+                secretKeyRef:
+                  name: radarr-db
+                  key: password
+            - name: Radarr__Auth__ApiKey
+              valueFrom:
+                secretKeyRef:
+                  name: radarr-apikey
+                  key: apitoken
+            # MediaCover object store (shared Ceph RGW bucket). Serves posters/fanart
+            # from S3 so any replica can render them instead of the leader-local
+            # emptyDir /config. Consumed by the -unkin3+ image; older images ignore
+            # these unknown config keys. Creds Secret is minted by cephrgw-operator.
+            - name: Radarr__MediaCover__S3__Endpoint
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-media-s3
+                  key: S3_ENDPOINT
+            - name: Radarr__MediaCover__S3__AccessKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-media-s3
+                  key: AWS_ACCESS_KEY_ID
+            - name: Radarr__MediaCover__S3__SecretKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-media-s3
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: Radarr__MediaCover__S3__Bucket
+              value: arrstack-media
+            - name: Radarr__MediaCover__S3__Prefix
+              value: radarr
+            - name: Radarr__MediaCover__S3__ForcePathStyle
+              value: "true"
+            - name: Radarr__MediaCover__S3__CaCertPath
+              value: /etc/ssl/vault-ca/ca.crt
+            # Backup object store (shared arrstack-backups Ceph RGW bucket,
+            # per-app key prefix). Routes the periodic config+DB zip backups off
+            # the ephemeral /config so any replica can write and restore them.
+            - name: Radarr__BackupS3__Endpoint
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: S3_ENDPOINT
+            - name: Radarr__BackupS3__AccessKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: AWS_ACCESS_KEY_ID
+            - name: Radarr__BackupS3__SecretKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: Radarr__BackupS3__Bucket
+              value: arrstack-backups
+            - name: Radarr__BackupS3__Prefix
+              value: radarr
+            - name: Radarr__BackupS3__ForcePathStyle
+              value: "true"
+            - name: Radarr__BackupS3__CaCertPath
+              value: /etc/ssl/vault-ca/ca.crt
+          livenessProbe:
+            httpGet:
+              path: /radarr/ping
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /radarr/ping
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: "1"
+              memory: 1Gi
+          volumeMounts:
+            - name: config
+              mountPath: /config
+            - name: media-movies
+              mountPath: /media/movies
+            - name: vault-ca
+              mountPath: /etc/ssl/vault-ca
+              readOnly: true
+        # exportarr sidecar: polls the local replica's API and exposes Prometheus
+        # metrics on :9708 (scraped by the radarr-exportarr VMPodScrape).
+        - name: exportarr
+          image: artifactapi.k8s.syd1.au.unkin.net/ghcr/onedr0p/exportarr:v2.3.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - radarr
+          env:
+            - name: PORT
+              value: "9708"
+            # URL includes the /radarr UrlBase (Radarr__Server__UrlBase).
+            - name: URL
+              value: http://localhost:7878/radarr
+            - name: APIKEY
+              valueFrom:
+                secretKeyRef:
+                  name: radarr-apikey
+                  key: apitoken
+          ports:
+            - name: metrics
+              containerPort: 9708
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 15
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 25m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 128Mi
+      volumes:
+        - name: config
+          emptyDir: {}
+        - name: media-movies
+          persistentVolumeClaim:
+            claimName: media-movies
+        # Estate CA for validating the Ceph RGW (s3.ceph.unkin.net) TLS cert.
+        - name: vault-ca
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt

apps/base/arrstack/radarr/gateway.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: radarr.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    external-dns.alpha.kubernetes.io/hostname: radarr.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+  name: radarr
+  namespace: arrstack
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: radarr.k8s.syd1.au.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: radarr.k8s.syd1.au.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: radarr-tls
+        mode: Terminate

apps/base/arrstack/radarr/httproute.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: radarr-http-redirect
+  namespace: arrstack
+spec:
+  hostnames:
+    - radarr.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: radarr
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: radarr-route
+  namespace: arrstack
+spec:
+  hostnames:
+    - radarr.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: radarr
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: radarr
+          port: 7878
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/arrstack/radarr/kustomization.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - vaultstaticsecret.yaml
+  - configmap.yaml
+  - deployment.yaml
+  - service.yaml
+  - gateway.yaml
+  - httproute.yaml
+  - vmpodscrape.yaml

apps/base/arrstack/radarr/pvc-config.yaml

@@ -0,0 +1,17 @@
+---
+# Radarr /config. RWX on CephFS so all replicas share it (the -unkin2 fork keeps
+# the database in Postgres; /config now holds only config.xml + MediaCover, which
+# tolerate — and want — shared access). Retain: this is state.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: radarr-config
+  namespace: arrstack
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 5Gi
+  storageClassName: cephfs-raid5-retain
+  volumeMode: Filesystem

apps/base/arrstack/radarr/service.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: radarr
+  namespace: arrstack
+spec:
+  ports:
+    - name: http
+      port: 7878
+      protocol: TCP
+      targetPort: http
+  selector:
+    app: radarr
+  type: ClusterIP

apps/base/arrstack/radarr/vaultstaticsecret.yaml

@@ -0,0 +1,25 @@
+---
+# radarr API key. Seeded at kv/kubernetes/namespace/arrstack/default/radarr
+# (key: apitoken); the default k8s role's templated policy already grants read
+# on kv/data/kubernetes/namespace/{{sa_namespace}}/{{sa_name}}/* for the
+# arrstack/default ServiceAccount, so no terraform-vault change is needed. VSO
+# syncs it into the radarr-apikey Secret that the apikey-init initContainer reads
+# to enforce <ApiKey> in /config/config.xml (Vault is source of truth).
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: radarr-apikey
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  destination:
+    create: true
+    name: radarr-apikey
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/arrstack/default/radarr
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/base/arrstack/radarr/vmpodscrape.yaml

@@ -0,0 +1,16 @@
+---
+# Scrape the exportarr sidecar (:9708) on every radarr pod. Picked up by the
+# observability VMAgent (selectAllByDefault). Pod-level rather than
+# VMServiceScrape because the radarr Service doesn't expose the metrics port.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: radarr-exportarr
+  namespace: arrstack
+spec:
+  selector:
+    matchLabels:
+      app: radarr
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics

apps/base/arrstack/sonarr/configmap.yaml

@@ -0,0 +1,33 @@
+---
+# Non-secret env for the -unkin2 fork. The fork reads Servarr config from
+# Sonarr__<Section>__<Key> env (no config.xml edits, no s6/PUID). Postgres wiring
+# points every replica at the same shared DB (arrstack-postgres-rw / sonarr-main);
+# Auth__Method=External defers UI auth to arrproxy/oauth2-proxy; Server__UrlBase
+# keeps the /sonarr prefix so arrproxy path-routing works; App__InstanceName is
+# identical across replicas (shared session-cookie name). User/Password/ApiKey
+# come from Secrets (see deployment.yaml), not here.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sonarr-env
+  namespace: arrstack
+data:
+  Sonarr__Postgres__Host: arrstack-postgres-rw.arrstack.svc.cluster.local
+  Sonarr__Postgres__Port: "5432"
+  Sonarr__Postgres__MainDb: sonarr-main
+  Sonarr__Log__DbEnabled: "false"
+  Sonarr__Auth__Method: External
+  Sonarr__Auth__Required: DisabledForLocalAddresses
+  Sonarr__App__InstanceName: Sonarr
+  Sonarr__Server__Port: "8989"
+  Sonarr__Server__UrlBase: /sonarr
+  Sonarr__Update__Mechanism: External
+  # Shared arrstack Valkey (valkey-operator). Setting Host is what activates the
+  # fork's #14 Redis features (SignalR backplane, cross-replica cache-invalidation
+  # bus, distributed rate limiter): RedisOptions.IsConfigured gates purely on a
+  # non-empty Host, so there is no separate Enabled flag. The operator leaves the
+  # default user passwordless (jellyfin parity), so no Password/Ssl is wired.
+  # Channels/keys are namespaced by this fork's sonarr:ratelimit: prefix, so the
+  # one cluster is safe to share with radarr/prowlarr.
+  Sonarr__Redis__Host: valkey-arrstack-valkey.arrstack.svc.cluster.local
+  Sonarr__Redis__Port: "6379"

apps/base/arrstack/sonarr/deployment.yaml

@@ -0,0 +1,251 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: sonarr
+  namespace: arrstack
+  annotations:
+    # sonarr-env is a plain (unhashed) ConfigMap consumed by fixed-name envFrom,
+    # so editing it does not roll the Deployment on its own. Reloader watches the
+    # referenced ConfigMap and triggers a rolling restart on change, so adding the
+    # Redis env activates the #14 features on the next ArgoCD sync without a manual
+    # `rollout restart`.
+    configmap.reloader.stakater.com/auto: "true"
+spec:
+  # Active-active: the -unkin2 fork keeps all state in the shared Postgres
+  # (arrstack-postgres) and coordinates via Postgres advisory locks, so N
+  # replicas run concurrently behind the sonarr Service. RollingUpdate is safe —
+  # no SQLite, no RWO lock.
+  replicas: 3
+  strategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: sonarr
+  template:
+    metadata:
+      labels:
+        app: sonarr
+    spec:
+      securityContext:
+        # Fork image has no USER (runs as root by default); pin it to a non-root
+        # UID and group-write the shared RWX CephFS /config (MediaCover etc.).
+        # OnRootMismatch avoids a recursive chown of the whole media tree.
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+        fsGroupChangePolicy: OnRootMismatch
+      initContainers:
+        # Gate the app on its own Postgres database+role being reachable, instead
+        # of relying on ArgoCD sync-waves (which deadlock if apps aren't Healthy).
+        # waitfordb reads the PG* env as a libpq fallback, so the password never lands in argv.
+        - name: wait-for-db
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/waitfordb:v0.1.0
+          env:
+            - name: WAITFORDB_TIMEOUT
+              value: 5m
+            - name: WAITFORDB_SSLMODE
+              value: disable
+            - name: PGHOST
+              value: arrstack-postgres-rw.arrstack.svc.cluster.local
+            - name: PGPORT
+              value: "5432"
+            - name: PGDATABASE
+              value: sonarr-main
+            - name: PGUSER
+              valueFrom:
+                secretKeyRef:
+                  name: sonarr-db
+                  key: username
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: sonarr-db
+                  key: password
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 64Mi
+      containers:
+        - name: sonarr
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/sonarr:v5.0.0-unkin6
+          imagePullPolicy: IfNotPresent
+          command:
+            - /app/Sonarr
+          args:
+            - -nobrowser
+            - -data=/config
+            # Required: bypass the single-instance guard so multiple replicas
+            # can share one /config. Cross-replica safety is the Postgres layer,
+            # not a local lock file.
+            - -nosingleinstancecheck
+          ports:
+            - name: http
+              containerPort: 8989
+              protocol: TCP
+          envFrom:
+            - configMapRef:
+                name: sonarr-env
+          env:
+            - name: Sonarr__Postgres__User
+              valueFrom:
+                secretKeyRef:
+                  name: sonarr-db
+                  key: username
+            - name: Sonarr__Postgres__Password
+              valueFrom:
+                secretKeyRef:
+                  name: sonarr-db
+                  key: password
+            - name: Sonarr__Auth__ApiKey
+              valueFrom:
+                secretKeyRef:
+                  name: sonarr-apikey
+                  key: apitoken
+            # MediaCover object store (shared arrstack-media Ceph RGW bucket,
+            # partitioned from radarr by the sonarr key prefix). Serves
+            # posters/fanart from S3 so any replica renders them instead of the
+            # leader-local emptyDir /config. Consumed by the -unkin3+ image;
+            # older images ignore these unknown config keys. Creds Secret minted
+            # by cephrgw-operator.
+            - name: Sonarr__MediaCoverS3__Endpoint
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-media-s3
+                  key: S3_ENDPOINT
+            - name: Sonarr__MediaCoverS3__AccessKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-media-s3
+                  key: AWS_ACCESS_KEY_ID
+            - name: Sonarr__MediaCoverS3__SecretKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-media-s3
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: Sonarr__MediaCoverS3__Bucket
+              value: arrstack-media
+            - name: Sonarr__MediaCoverS3__Prefix
+              value: sonarr
+            - name: Sonarr__MediaCoverS3__ForcePathStyle
+              value: "true"
+            - name: Sonarr__MediaCoverS3__CaCertPath
+              value: /etc/ssl/vault-ca/ca.crt
+            # Backup object store (shared arrstack-backups Ceph RGW bucket,
+            # per-app key prefix). Routes the periodic config+DB zip backups off
+            # the ephemeral /config so any replica can write and restore them.
+            - name: Sonarr__BackupS3__Endpoint
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: S3_ENDPOINT
+            - name: Sonarr__BackupS3__AccessKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: AWS_ACCESS_KEY_ID
+            - name: Sonarr__BackupS3__SecretKey
+              valueFrom:
+                secretKeyRef:
+                  name: arrstack-backups-s3
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: Sonarr__BackupS3__Bucket
+              value: arrstack-backups
+            - name: Sonarr__BackupS3__Prefix
+              value: sonarr
+            - name: Sonarr__BackupS3__ForcePathStyle
+              value: "true"
+            - name: Sonarr__BackupS3__CaCertPath
+              value: /etc/ssl/vault-ca/ca.crt
+          livenessProbe:
+            httpGet:
+              path: /sonarr/ping
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /sonarr/ping
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: "1"
+              memory: 1Gi
+          volumeMounts:
+            - name: config
+              mountPath: /config
+            - name: media-tv
+              mountPath: /media/tv
+            - name: vault-ca
+              mountPath: /etc/ssl/vault-ca
+              readOnly: true
+        # exportarr sidecar: polls the local replica's API and exposes Prometheus
+        # metrics on :9707 (scraped by the sonarr-exportarr VMPodScrape).
+        - name: exportarr
+          image: artifactapi.k8s.syd1.au.unkin.net/ghcr/onedr0p/exportarr:v2.3.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - sonarr
+          env:
+            - name: PORT
+              value: "9707"
+            # URL includes the /sonarr UrlBase (Sonarr__Server__UrlBase).
+            - name: URL
+              value: http://localhost:8989/sonarr
+            - name: APIKEY
+              valueFrom:
+                secretKeyRef:
+                  name: sonarr-apikey
+                  key: apitoken
+          ports:
+            - name: metrics
+              containerPort: 9707
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 15
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 25m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 128Mi
+      volumes:
+        - name: config
+          emptyDir: {}
+        - name: media-tv
+          persistentVolumeClaim:
+            claimName: media-tv
+        # Estate CA for validating the Ceph RGW (s3.ceph.unkin.net) TLS cert.
+        - name: vault-ca
+          secret:
+            secretName: vault-ca-cert
+            items:
+              - key: ca.crt
+                path: ca.crt

apps/base/arrstack/sonarr/gateway.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  labels:
+    traefik.io/instance: internal
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+    cert-manager.io/common-name: sonarr.k8s.syd1.au.unkin.net
+    cert-manager.io/private-key-size: "4096"
+    external-dns.alpha.kubernetes.io/hostname: sonarr.k8s.syd1.au.unkin.net
+    external-dns.alpha.kubernetes.io/target: 198.18.200.4
+  name: sonarr
+  namespace: arrstack
+spec:
+  gatewayClassName: traefik-internal
+  listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: sonarr.k8s.syd1.au.unkin.net
+      name: http
+      port: 80
+      protocol: HTTP
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      hostname: sonarr.k8s.syd1.au.unkin.net
+      name: https
+      port: 443
+      protocol: HTTPS
+      tls:
+        certificateRefs:
+          - group: ""
+            kind: Secret
+            name: sonarr-tls
+        mode: Terminate

apps/base/arrstack/sonarr/httproute.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: sonarr-http-redirect
+  namespace: arrstack
+spec:
+  hostnames:
+    - sonarr.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: sonarr
+      sectionName: http
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+      matches:
+        - path:
+            type: PathPrefix
+            value: /
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: sonarr-route
+  namespace: arrstack
+spec:
+  hostnames:
+    - sonarr.k8s.syd1.au.unkin.net
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: sonarr
+      sectionName: https
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: sonarr
+          port: 8989
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

apps/base/arrstack/sonarr/kustomization.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - vaultstaticsecret.yaml
+  - configmap.yaml
+  - deployment.yaml
+  - service.yaml
+  - gateway.yaml
+  - httproute.yaml
+  - vmpodscrape.yaml

apps/base/arrstack/sonarr/pvc-config.yaml

@@ -0,0 +1,17 @@
+---
+# Sonarr /config. RWX on CephFS so all replicas share it (the -unkin2 fork keeps
+# the database in Postgres; /config now holds only config.xml + MediaCover, which
+# tolerate — and want — shared access). Retain: this is state.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: sonarr-config
+  namespace: arrstack
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 5Gi
+  storageClassName: cephfs-raid5-retain
+  volumeMode: Filesystem

apps/base/arrstack/sonarr/service.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: sonarr
+  namespace: arrstack
+spec:
+  ports:
+    - name: http
+      port: 8989
+      protocol: TCP
+      targetPort: http
+  selector:
+    app: sonarr
+  type: ClusterIP

apps/base/arrstack/sonarr/vaultstaticsecret.yaml

@@ -0,0 +1,25 @@
+---
+# sonarr API key. Seeded at kv/kubernetes/namespace/arrstack/default/sonarr
+# (key: apitoken); the default k8s role's templated policy already grants read
+# on kv/data/kubernetes/namespace/{{sa_namespace}}/{{sa_name}}/* for the
+# arrstack/default ServiceAccount, so no terraform-vault change is needed. VSO
+# syncs it into the sonarr-apikey Secret that the apikey-init initContainer reads
+# to enforce <ApiKey> in /config/config.xml (Vault is source of truth).
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: sonarr-apikey
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  destination:
+    create: true
+    name: sonarr-apikey
+    overwrite: true
+  hmacSecretData: true
+  mount: kv
+  path: kubernetes/namespace/arrstack/default/sonarr
+  refreshAfter: 5m
+  type: kv-v2
+  vaultAuthRef: default

apps/base/arrstack/sonarr/vmpodscrape.yaml

@@ -0,0 +1,16 @@
+---
+# Scrape the exportarr sidecar (:9707) on every sonarr pod. Picked up by the
+# observability VMAgent (selectAllByDefault). Pod-level rather than
+# VMServiceScrape because the sonarr Service doesn't expose the metrics port.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: sonarr-exportarr
+  namespace: arrstack
+spec:
+  selector:
+    matchLabels:
+      app: sonarr
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics

apps/base/arrstack/valkey/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - valkeycluster.yaml

apps/base/arrstack/valkey/valkeycluster.yaml

@@ -0,0 +1,47 @@
+---
+# Single shared HA Valkey for the arr apps (sonarr/radarr/prowlarr), managed by
+# valkey-operator. It activates the fork's #14 Redis features: the SignalR
+# backplane, the cross-replica cache-invalidation bus, and the distributed rate
+# limiter. One cluster is safe for all three because each fork namespaces its
+# keys and pub/sub channels by a per-app prefix (sonarr:ratelimit: /
+# radarr:ratelimit: / prowlarr:ratelimit:), so their state never collides.
+#
+# Modeled on jellyfin-valkey: shards:1 + replicas:2 is one primary with two
+# replicas in a single shard group (three ValkeyNodes total); losing the primary
+# triggers an automatic failover so a node/pod loss no longer drops the shared
+# state the app replicas coordinate through. The operator runs Valkey
+# cluster-mode-enabled with protected-mode off and leaves the built-in `default`
+# user passwordless, so clients connect with no auth/TLS; StackExchange.Redis
+# seeds off the single service and auto-discovers topology plus failovers.
+# scheduling.node.spread.shard:Required keeps the three nodes on distinct hosts,
+# so one host loss removes at most one node; podDisruptionBudget.mode:Cluster
+# lets the operator manage a quorum-aware PDB. Persistence is omitted (/data is an
+# emptyDir): the coordination state is ephemeral (short TTLs / transient pub/sub),
+# replication+failover already provide redundancy, and an operator-managed PVC
+# cannot carry the k8up.io/backup:"false" annotation the namespace k8up Schedule
+# needs to skip in-use RWO volumes.
+apiVersion: valkey.io/v1alpha1
+kind: ValkeyCluster
+metadata:
+  name: arrstack-valkey
+  namespace: arrstack
+spec:
+  shards: 1
+  replicas: 2
+  image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/valkey/valkey:9.0.0
+  exporter:
+    enabled: false
+  scheduling:
+    node:
+      spread:
+        shard:
+          mode: Required
+  podDisruptionBudget:
+    mode: Cluster
+  resources:
+    requests:
+      cpu: 50m
+      memory: 128Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi

apps/base/arrstack/vaultauth.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: default
+  namespace: arrstack
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  allowedNamespaces:
+    - arrstack
+  kubernetes:
+    audiences:
+      - vault
+    role: default
+    serviceAccount: default
+    tokenExpirationSeconds: 600
+  method: kubernetes
+  mount: k8s/au/syd1
+  vaultConnectionRef: vso-system/default

apps/base/artifactapi/api-deployment.yaml

@@ -5,7 +5,8 @@ metadata:
   name: api
   namespace: artifactapi
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
+    secret.reloader.stakater.com/reload: "vault-ca-cert"
 spec:
   selector:
     matchLabels:
@@ -35,7 +36,7 @@ spec:
               mountPath: /combined-certs
       containers:
         - name: api
-          image: git.unkin.net/unkin/artifactapi:v3.7.6
+          image: git.unkin.net/unkin/artifactapi:v3.11.1
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000

apps/base/artifactapi/cnpg_backup.yaml

@@ -0,0 +1,56 @@
+---
+# Ceph RGW (S3) backup target for the artifactapi CNPG cluster, provisioned by the
+# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster:
+# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from
+# its own namespace, so backups are per-database rather than one shared bucket.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: cnpg-artifactapi-backup
+  namespace: artifactapi
+spec:
+  displayName: "CNPG backup owner (artifactapi)"
+  # RGW users are global; keep the uid namespace-qualified so it never collides.
+  uid: cnpg-artifactapi-backup
+  maxBuckets: 5
+  # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
+  # S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it.
+  secretName: cnpg-artifactapi-backup-s3
+  # Keep the RGW user (and thus the keys) if this CR is ever deleted, so an
+  # in-flight restore can still reach the archive.
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: cnpg-artifactapi
+  namespace: artifactapi
+spec:
+  placementTarget: ec
+  bucketName: cnpg-artifactapi
+  # The owner user has full control of its own bucket (read + write), which is
+  # all the backup/restore identity needs — no extra BucketAccess grant.
+  ownerRef: cnpg-artifactapi-backup
+  versioning: false
+  tags:
+    app: artifactapi
+    purpose: cnpg-backup
+  # Never drop the backups if the CR is removed; retire buckets by hand.
+  retainOnDelete: true
+---
+# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's
+# spec.backup.barmanObjectStore; this schedules the periodic full backup that
+# WAL is layered on top of. Schedules are staggered across clusters so the 8
+# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first).
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-artifactapi-nightly
+  namespace: artifactapi
+spec:
+  schedule: "0 40 1 * * *"
+  immediate: false
+  backupOwnerReference: self
+  method: barmanObjectStore
+  cluster:
+    name: postgres

apps/base/artifactapi/cnpg_cluster.yaml

@@ -7,6 +7,35 @@ metadata:
 spec:
   affinity:
     podAntiAffinityType: preferred
+  backup:
+    # 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG
+    # against the object store on each successful base backup.
+    retentionPolicy: 30d
+    barmanObjectStore:
+      # Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it).
+      destinationPath: s3://cnpg-artifactapi
+      endpointURL: https://s3.ceph.unkin.net
+      # radosgw serves a Vault-PKI cert; trust the internal CA (reflected into
+      # every namespace as the vault-ca-cert Secret).
+      endpointCA:
+        name: vault-ca-cert
+        key: ca.crt
+      # Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded.
+      s3Credentials:
+        accessKeyId:
+          name: cnpg-artifactapi-backup-s3
+          key: AWS_ACCESS_KEY_ID
+        secretAccessKey:
+          name: cnpg-artifactapi-backup-s3
+          key: AWS_SECRET_ACCESS_KEY
+      # Path prefix within the bucket; keep stable across restores (see docs).
+      serverName: artifactapi
+      data:
+        compression: bzip2
+        jobs: 2
+      wal:
+        compression: zstd
+        maxParallel: 2
   bootstrap:
     initdb:
       database: artifacts

apps/base/artifactapi/httproute.yaml

@@ -1,30 +1,6 @@
 ---
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
-metadata:
-  name: http-redirect
-  namespace: artifactapi
-spec:
-  hostnames:
-    - artifactapi.k8s.syd1.au.unkin.net
-  parentRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: artifactapi
-      sectionName: http
-  rules:
-    - filters:
-        - type: RequestRedirect
-          requestRedirect:
-            scheme: https
-            statusCode: 301
-      matches:
-        - path:
-            type: PathPrefix
-            value: /
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
 metadata:
   name: api-route
   namespace: artifactapi
@@ -32,6 +8,12 @@ spec:
   hostnames:
     - artifactapi.k8s.syd1.au.unkin.net
   parentRefs:
+    # Early-boot clients (anaconda/kickstart, yum in %post, PXE) need plain HTTP
+    # for the rpm repos; serve the app directly on port 80 instead of redirecting.
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: artifactapi
+      sectionName: http
     - group: gateway.networking.k8s.io
       kind: Gateway
       name: artifactapi

apps/base/artifactapi/kustomization.yaml

@@ -7,6 +7,7 @@ resources:
   - api-hpa.yaml
   - configmap.yaml
   - cnpg_cluster.yaml
+  - cnpg_backup.yaml
   - cnpg_pooler.yaml
   - gateway.yaml
   - httproute.yaml
@@ -17,3 +18,5 @@ resources:
   - ui-hpa.yaml
   - vaultauth.yaml
   - vaultstaticsecret.yaml
+  - vmpodscrape.yaml
+  - vpa.yaml

apps/base/artifactapi/redis-deployment.yaml

@@ -53,4 +53,18 @@ spec:
             periodSeconds: 10
             successThreshold: 1
             timeoutSeconds: 5
+        - name: metrics-exporter
+          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/oliver006/redis_exporter:v1.89.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 9121
+              name: metrics
+              protocol: TCP
+          resources:
+            limits:
+              cpu: 100m
+              memory: 64Mi
+            requests:
+              cpu: 25m
+              memory: 32Mi
       restartPolicy: Always

apps/base/artifactapi/ui-deployment.yaml

@@ -5,7 +5,7 @@ metadata:
   name: ui
   namespace: artifactapi
   annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/auto: "true"
 spec:
   selector:
     matchLabels:
@@ -22,7 +22,7 @@ spec:
       automountServiceAccountToken: true
       containers:
         - name: ui
-          image: git.unkin.net/unkin/artifactapi-ui:v3.7.6
+          image: git.unkin.net/unkin/artifactapi-ui:v3.11.1
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 80

apps/base/artifactapi/vmpodscrape.yaml

@@ -0,0 +1,16 @@
+---
+# Scrape the redis_exporter sidecar (:9121) on the redis pod. Picked up by the
+# observability VMAgent (selectAllByDefault). Pod-level rather than
+# VMServiceScrape because the redis Service doesn't expose the metrics port.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: redis-exporter
+  namespace: artifactapi
+spec:
+  selector:
+    matchLabels:
+      app: redis
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics

apps/base/artifactapi/vpa.yaml

@@ -0,0 +1,45 @@
+---
+apiVersion: autoscaling.k8s.io/v1
+kind: VerticalPodAutoscaler
+metadata:
+  name: api-vpa
+  namespace: artifactapi
+  # NOTE: this workload also has an HPA. updateMode Off is recommendation-only
+  # and does not act, so there is no HPA/VPA conflict today. Do not flip to Auto/
+  # Initial without first moving the HPA off CPU/memory (VPA owns those under Auto).
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: api
+  updatePolicy:
+    updateMode: "Off"
+---
+apiVersion: autoscaling.k8s.io/v1
+kind: VerticalPodAutoscaler
+metadata:
+  name: redis-vpa
+  namespace: artifactapi
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: redis
+  updatePolicy:
+    updateMode: "Off"
+---
+apiVersion: autoscaling.k8s.io/v1
+kind: VerticalPodAutoscaler
+metadata:
+  name: ui-vpa
+  namespace: artifactapi
+  # NOTE: this workload also has an HPA. updateMode Off is recommendation-only
+  # and does not act, so there is no HPA/VPA conflict today. Do not flip to Auto/
+  # Initial without first moving the HPA off CPU/memory (VPA owns those under Auto).
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: ui
+  updatePolicy:
+    updateMode: "Off"

apps/base/authentik/cnpg_backup.yaml

@@ -0,0 +1,56 @@
+---
+# Ceph RGW (S3) backup target for the authentik CNPG cluster, provisioned by the
+# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster:
+# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from
+# its own namespace, so backups are per-database rather than one shared bucket.
+apiVersion: ceph.unkin.net/v1alpha1
+kind: ObjectStoreUser
+metadata:
+  name: cnpg-authentik-backup
+  namespace: authentik
+spec:
+  displayName: "CNPG backup owner (authentik)"
+  # RGW users are global; keep the uid namespace-qualified so it never collides.
+  uid: cnpg-authentik-backup
+  maxBuckets: 5
+  # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID,
+  # S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it.
+  secretName: cnpg-authentik-backup-s3
+  # Keep the RGW user (and thus the keys) if this CR is ever deleted, so an
+  # in-flight restore can still reach the archive.
+  retainOnDelete: true
+---
+apiVersion: ceph.unkin.net/v1alpha1
+kind: Bucket
+metadata:
+  name: cnpg-authentik
+  namespace: authentik
+spec:
+  placementTarget: ec
+  bucketName: cnpg-authentik
+  # The owner user has full control of its own bucket (read + write), which is
+  # all the backup/restore identity needs — no extra BucketAccess grant.
+  ownerRef: cnpg-authentik-backup
+  versioning: false
+  tags:
+    app: authentik
+    purpose: cnpg-backup
+  # Never drop the backups if the CR is removed; retire buckets by hand.
+  retainOnDelete: true
+---
+# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's
+# spec.backup.barmanObjectStore; this schedules the periodic full backup that
+# WAL is layered on top of. Schedules are staggered across clusters so the 8
+# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first).
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cnpg-authentik-nightly
+  namespace: authentik
+spec:
+  schedule: "0 0 1 * * *"
+  immediate: false
+  backupOwnerReference: self
+  method: barmanObjectStore
+  cluster:
+    name: postgres

apps/base/authentik/cnpg_cluster.yaml

@@ -7,6 +7,35 @@ metadata:
 spec:
   affinity:
     podAntiAffinityType: preferred
+  backup:
+    # 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG
+    # against the object store on each successful base backup.
+    retentionPolicy: 30d
+    barmanObjectStore:
+      # Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it).
+      destinationPath: s3://cnpg-authentik
+      endpointURL: https://s3.ceph.unkin.net
+      # radosgw serves a Vault-PKI cert; trust the internal CA (reflected into
+      # every namespace as the vault-ca-cert Secret).
+      endpointCA:
+        name: vault-ca-cert
+        key: ca.crt
+      # Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded.
+      s3Credentials:
+        accessKeyId:
+          name: cnpg-authentik-backup-s3
+          key: AWS_ACCESS_KEY_ID
+        secretAccessKey:
+          name: cnpg-authentik-backup-s3
+          key: AWS_SECRET_ACCESS_KEY
+      # Path prefix within the bucket; keep stable across restores (see docs).
+      serverName: authentik
+      data:
+        compression: bzip2
+        jobs: 2
+      wal:
+        compression: zstd
+        maxParallel: 2
   bootstrap:
     initdb:
       database: authentik
@@ -77,10 +106,12 @@ spec:
   resources:
     limits:
       cpu: 500m
-      memory: 512Mi
+      # 512Mi OOMKilled replicas under load (shared_buffers 128MB +
+      # max_connections 200 leave no headroom) — see incident 2026-07-28.
+      memory: 1Gi
     requests:
       cpu: 50m
-      memory: 256Mi
+      memory: 512Mi
   smartShutdownTimeout: 180
   startDelay: 3600
   stopDelay: 1800

apps/base/authentik/kustomization.yaml

@@ -4,6 +4,7 @@ kind: Kustomization
 
 resources:
   - cnpg_cluster.yaml
+  - cnpg_backup.yaml
   - cnpg_pooler.yaml
   - gateway.yaml
   - httproute.yaml
@@ -17,3 +18,5 @@ resources:
   - redis-service.yaml
   - vaultauth.yaml
   - vaultstaticsecret.yaml
+  - vmpodscrape.yaml
+  - vpa.yaml

apps/base/authentik/redis-deployment.yaml

@@ -52,6 +52,20 @@ spec:
           volumeMounts:
             - mountPath: /data
               name: redis-data
+        - name: metrics-exporter
+          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/oliver006/redis_exporter:v1.89.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 9121
+              name: metrics
+              protocol: TCP
+          resources:
+            limits:
+              cpu: 100m
+              memory: 64Mi
+            requests:
+              cpu: 25m
+              memory: 32Mi
       volumes:
         - name: redis-data
           persistentVolumeClaim:

apps/base/authentik/vmpodscrape.yaml

@@ -0,0 +1,16 @@
+---
+# Scrape the redis_exporter sidecar (:9121) on the redis pod. Picked up by the
+# observability VMAgent (selectAllByDefault). Pod-level rather than
+# VMServiceScrape because the redis Service doesn't expose the metrics port.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: redis-exporter
+  namespace: authentik
+spec:
+  selector:
+    matchLabels:
+      app: redis
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics

apps/base/authentik/vpa.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: autoscaling.k8s.io/v1
+kind: VerticalPodAutoscaler
+metadata:
+  name: redis-vpa
+  namespace: authentik
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: redis
+  updatePolicy:
+    updateMode: "Off"

apps/base/bind-external/agent-dns-rolebinding.yaml

@@ -0,0 +1,16 @@
+---
+# Confines the agent-dns service account (in bind-system) to the agent-dns
+# ClusterRole within this namespace.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: agent-dns
+  namespace: bind-external
+subjects:
+  - kind: ServiceAccount
+    name: agent-dns
+    namespace: bind-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: agent-dns

apps/base/bind-external/cluster.yaml

@@ -0,0 +1,52 @@
+---
+# Externally-reachable authoritative BIND for zones we delegate to ourselves.
+# First tenant: acme.unkin.net, the DNS-01 challenge zone Let's Encrypt validates
+# via a one-time _acme-challenge.unkin.net CNAME. Authoritative-only, recursion
+# off, no forwarding, no open transfers -- the primaryService is the single
+# dmz-pinned LoadBalancer that public NAT targets and that cert-manager writes to.
+apiVersion: bind.unkin.net/v1alpha1
+kind: BindCluster
+metadata:
+  name: bind-external
+  namespace: bind-external
+spec:
+  mode: authoritative
+  recursion: false
+  replicas: 2
+  storageClassName: cephrbd-fast-delete
+  storageSize: 1Gi
+  # Public server: answer queries from anywhere (Let's Encrypt validates over the
+  # internet), deny recursion and open zone transfers. localhost + pod net are
+  # implied by "any" and cover in-pod nsupdate and secondary SOA refresh; per-zone
+  # allow-transfer (catalog + acme zone) still permits key-authenticated AXFR.
+  extraOptions:
+    - "allow-query { any; }"
+    - "allow-transfer { none; }"
+  service:
+    type: ClusterIP
+  primaryService:
+    type: LoadBalancer
+    externalTrafficPolicy: Local
+    annotations:
+      purelb.io/service-group: dmz
+      purelb.io/addresses: 198.18.199.53
+      external-dns.alpha.kubernetes.io/hostname: bind-external-primary.k8s.syd1.au.unkin.net
+  resources:
+    requests:
+      cpu: 20m
+      memory: 128Mi
+    limits:
+      cpu: "1"
+      memory: 512Mi
+---
+# Catalog zone so the acme zone replicates onto the secondary (AXFR/IXFR keyed
+# with the certmanager TSIG key, reused here as the transfer key).
+apiVersion: bind.unkin.net/v1alpha1
+kind: BindCatalogZone
+metadata:
+  name: bind-external-catalog
+  namespace: bind-external
+spec:
+  clusterRef: bind-external
+  zoneName: catalog.external
+  transferKeyRef: certmanager

apps/base/bind-external/kustomization.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - cluster.yaml
+  - tsigkey.yaml
+  - zones.yaml
+  - agent-dns-rolebinding.yaml

apps/base/bind-external/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: bind-external

apps/base/bind-external/tsigkey.yaml

@@ -0,0 +1,20 @@
+---
+# TSIG key cert-manager uses to send RFC2136 dynamic updates (the DNS-01 TXT
+# records) to the primary, and that the secondary reuses for AXFR. The operator
+# generates the material into Secret certmanager-tsig in this namespace;
+# secretTemplate stamps emberstack reflector hints so the Secret is mirrored into
+# the cert-manager namespace, where the rfc2136 solver reads its "secret" key.
+apiVersion: bind.unkin.net/v1alpha1
+kind: BindTSIGKey
+metadata:
+  name: certmanager
+  namespace: bind-external
+spec:
+  clusterRef: bind-external
+  algorithm: hmac-sha256
+  secretTemplate:
+    annotations:
+      reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "cert-manager"
+      reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "cert-manager"

apps/base/bind-external/zones.yaml

@@ -0,0 +1,19 @@
+---
+# Self-delegated ACME challenge zone. Google Cloud DNS holds a one-time
+# _acme-challenge.unkin.net CNAME -> _acme-challenge.acme.unkin.net and an
+# acme.unkin.net NS delegation pointing here; cert-manager writes the challenge
+# TXT records via RFC2136 authenticated with the certmanager key.
+apiVersion: bind.unkin.net/v1alpha1
+kind: BindZone
+metadata:
+  name: acme-unkin-net
+  namespace: bind-external
+spec:
+  clusterRef: bind-external
+  zoneName: acme.unkin.net
+  type: primary
+  defaultTTL: 60
+  dynamicUpdate: true
+  updateKeyRef: certmanager
+  allowTransfer:
+    - key certmanager

apps/base/bind-internal/agent-dns-rolebinding.yaml

@@ -0,0 +1,16 @@
+---
+# Confines the agent-dns service account (in bind-system) to the agent-dns
+# ClusterRole within this namespace.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: agent-dns
+  namespace: bind-internal
+subjects:
+  - kind: ServiceAccount
+    name: agent-dns
+    namespace: bind-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: agent-dns

apps/base/bind-internal/authoritative/acls.yaml

@@ -24,3 +24,6 @@ spec:
     - 198.18.27.0/24
     - 198.18.28.0/24
     - 198.18.29.0/24
+    # Admin/management access (individual hosts, not whole subnets)
+    - 10.10.12.200/32   # benvin workstation (wireguard)
+    - 198.18.21.160/32  # benvin router

apps/base/bind-internal/authoritative/cluster.yaml

@@ -13,9 +13,14 @@ spec:
   storageSize: 2Gi
   # Restrict queries to internal networks (puppet acl-main.unkin.net).
   # 10.42.0.0/16 (pod net) is required so secondaries can SOA-refresh
-  # from the primary during catalog replication.
+  # from the primary during catalog replication. localhost is required so the
+  # operator's in-pod `nsupdate` (sent to 127.0.0.1) passes query-authorization;
+  # without it every dynamic update is "denied due to allow-query".
   extraOptions:
-    - "allow-query { auth-acl-main; 10.42.0.0/16; }"
+    - "allow-query { localhost; auth-acl-main; 10.42.0.0/16; }"
+    # Enable query logging for the Tier-2 vector bind_query pipeline (see the
+    # resolvers cluster for the routing rationale).
+    - "querylog yes"
   service:
     type: LoadBalancer
     externalTrafficPolicy: Local