arrstack: run sonarr/radarr/prowlarr active-active on shared Postgres (#arrstack)

Convert the three *arr apps from single-replica upstream LinuxServer/SQLite to
our -unkin2 forks running active-active (replicas 3) on a shared CNPG Postgres
cluster.

- new apps/base/arrstack/postgres/: CNPG Cluster arrstack-postgres (3 instances,
  3 managed login roles sonarr/radarr/prowlarr, throwaway initdb owner), per-app
  Database CRDs (<app>-main), cephrgw backup bucket + nightly ScheduledBackup,
  and 3 VaultStaticSecrets syncing the <app>-db role credentials.
- per app: image -> docker-internal/<app>:v<base>-unkin2; replicas 3 +
  RollingUpdate; drop the apikey-init initContainer + PUID/PGID/TZ; run the
  binary directly with -nosingleinstancecheck; env from a new ConfigMap plus
  Postgres user/password and the API key from Secrets; /config PVC -> RWX
  cephfs-raid5-retain (shared).

apps/base/arrstack/prowlarr/configmap.yaml

@@ -0,0 +1,24 @@
+---
+# Non-secret env for the -unkin2 fork. The fork reads Servarr config from
+# Prowlarr__<Section>__<Key> env (no config.xml edits, no s6/PUID). Postgres
+# wiring points every replica at the same shared DB (arrstack-postgres-rw /
+# prowlarr-main); Auth__Method=External defers UI auth to arrproxy/oauth2-proxy;
+# Server__UrlBase keeps the /prowlarr prefix so arrproxy path-routing works;
+# App__InstanceName is identical across replicas (shared session-cookie name).
+# User/Password/ApiKey come from Secrets (see deployment.yaml), not here.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prowlarr-env
+  namespace: arrstack
+data:
+  Prowlarr__Postgres__Host: arrstack-postgres-rw.arrstack.svc.cluster.local
+  Prowlarr__Postgres__Port: "5432"
+  Prowlarr__Postgres__MainDb: prowlarr-main
+  Prowlarr__Log__DbEnabled: "false"
+  Prowlarr__Auth__Method: External
+  Prowlarr__Auth__Required: DisabledForLocalAddresses
+  Prowlarr__App__InstanceName: Prowlarr
+  Prowlarr__Server__Port: "9696"
+  Prowlarr__Server__UrlBase: /prowlarr
+  Prowlarr__Update__Mechanism: External

apps/base/arrstack/prowlarr/deployment.yaml

@@ -5,10 +5,13 @@ metadata:
   name: prowlarr
   namespace: arrstack
 spec:
-  replicas: 1
+  # Active-active: the -unkin2 fork keeps all state in the shared Postgres
+  # (arrstack-postgres) and coordinates via Postgres advisory locks, so N
+  # replicas run concurrently behind the prowlarr Service. RollingUpdate is safe
+  # — no SQLite, no RWO lock.
+  replicas: 3
   strategy:
-    # RWO config PVC + single stateful SQLite DB: never run two pods at once.
-    type: Recreate
+    type: RollingUpdate
   selector:
     matchLabels:
       app: prowlarr
@@ -18,96 +21,49 @@ spec:
         app: prowlarr
     spec:
       securityContext:
+        # Fork image has no USER (runs as root by default); pin it to a non-root
+        # UID and group-write the shared RWX CephFS /config. OnRootMismatch
+        # avoids a recursive chown of the whole volume.
+        runAsUser: 1000
+        runAsGroup: 1000
         fsGroup: 1000
         fsGroupChangePolicy: OnRootMismatch
-      initContainers:
-        # Enforce the Vault-sourced API key and the reverse-proxy URL base in
-        # /config/config.xml before the app starts. Vault is source of truth
-        # (override bootstrap): the key is minted in Vault, synced by VSO into the
-        # prowlarr-apikey Secret, and written here. UrlBase=/prowlarr lets arrproxy
-        # forward arrstack.unkin.net/prowlarr/... with the prefix preserved (no 307).
-        # Runs as root to fix ownership; touches only <ApiKey>, <UrlBase>, and
-        # <AuthenticationMethod>=External + <AuthenticationRequired>. External makes
-        # the *arr defer UI login to arrproxy/oauth2-proxy (same no-auth request
-        # handler as None, but permits remote access without prompting).
-        - name: apikey-init
-          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/busybox:1.37.0
-          imagePullPolicy: IfNotPresent
-          securityContext:
-            runAsUser: 0
-          env:
-            - name: API_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: prowlarr-apikey
-                  key: apitoken
-          command:
-            - sh
-            - -c
-            - |
-              set -eu
-              case "$API_KEY" in
-                "" | *[!0-9a-fA-F]*)
-                  echo "config-init: API_KEY missing or not hex; refusing" >&2
-                  exit 1
-                  ;;
-              esac
-              URL_BASE=/prowlarr
-              AUTH_METHOD=External
-              AUTH_REQUIRED=Enabled
-              CFG=/config/config.xml
-              if [ ! -f "$CFG" ]; then
-                printf '<Config>\n  <ApiKey>%s</ApiKey>\n  <UrlBase>%s</UrlBase>\n  <AuthenticationMethod>%s</AuthenticationMethod>\n  <AuthenticationRequired>%s</AuthenticationRequired>\n</Config>\n' "$API_KEY" "$URL_BASE" "$AUTH_METHOD" "$AUTH_REQUIRED" > "$CFG"
-              else
-                if grep -q '<ApiKey>' "$CFG"; then
-                  sed -i "s|<ApiKey>[^<]*</ApiKey>|<ApiKey>${API_KEY}</ApiKey>|" "$CFG"
-                else
-                  sed -i "s|<Config>|<Config>\n  <ApiKey>${API_KEY}</ApiKey>|" "$CFG"
-                fi
-                if grep -q '<UrlBase>' "$CFG"; then
-                  sed -i "s|<UrlBase>[^<]*</UrlBase>|<UrlBase>${URL_BASE}</UrlBase>|" "$CFG"
-                else
-                  sed -i "s|<Config>|<Config>\n  <UrlBase>${URL_BASE}</UrlBase>|" "$CFG"
-                fi
-                if grep -q '<AuthenticationMethod>' "$CFG"; then
-                  sed -i "s|<AuthenticationMethod>[^<]*</AuthenticationMethod>|<AuthenticationMethod>${AUTH_METHOD}</AuthenticationMethod>|" "$CFG"
-                else
-                  sed -i "s|<Config>|<Config>\n  <AuthenticationMethod>${AUTH_METHOD}</AuthenticationMethod>|" "$CFG"
-                fi
-                if grep -q '<AuthenticationRequired>' "$CFG"; then
-                  sed -i "s|<AuthenticationRequired>[^<]*</AuthenticationRequired>|<AuthenticationRequired>${AUTH_REQUIRED}</AuthenticationRequired>|" "$CFG"
-                else
-                  sed -i "s|<Config>|<Config>\n  <AuthenticationRequired>${AUTH_REQUIRED}</AuthenticationRequired>|" "$CFG"
-                fi
-              fi
-              chown 1000:1000 "$CFG"
-              chmod 600 "$CFG"
-              echo "config-init: <ApiKey>, <UrlBase>=${URL_BASE}, <AuthenticationMethod>=${AUTH_METHOD}, <AuthenticationRequired>=${AUTH_REQUIRED} enforced from Vault"
-          resources:
-            requests:
-              cpu: 50m
-              memory: 32Mi
-            limits:
-              cpu: 200m
-              memory: 64Mi
-          volumeMounts:
-            - name: config
-              mountPath: /config
       containers:
         - name: prowlarr
-          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/linuxserver/prowlarr:2.5.2
+          image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/prowlarr:v2.6.2-unkin2
           imagePullPolicy: IfNotPresent
+          command:
+            - /app/Prowlarr
+          args:
+            - -nobrowser
+            - -data=/config
+            # Required: bypass the single-instance guard so multiple replicas
+            # can share one /config. Cross-replica safety is the Postgres layer,
+            # not a local lock file.
+            - -nosingleinstancecheck
           ports:
             - name: http
               containerPort: 9696
               protocol: TCP
+          envFrom:
+            - configMapRef:
+                name: prowlarr-env
           env:
-            - name: PUID
-              value: "1000"
-            - name: PGID
-              value: "1000"
-            - name: TZ
-              value: Australia/Sydney
+            - name: Prowlarr__Postgres__User
+              valueFrom:
+                secretKeyRef:
+                  name: prowlarr-db
+                  key: username
+            - name: Prowlarr__Postgres__Password
+              valueFrom:
+                secretKeyRef:
+                  name: prowlarr-db
+                  key: password
+            - name: Prowlarr__Auth__ApiKey
+              valueFrom:
+                secretKeyRef:
+                  name: prowlarr-apikey
+                  key: apitoken
           livenessProbe:
             httpGet:
               path: /prowlarr/ping
@@ -130,7 +86,7 @@ spec:
               memory: 256Mi
             limits:
               cpu: "1"
-              memory: 512Mi
+              memory: 1Gi
           volumeMounts:
             - name: config
               mountPath: /config

apps/base/arrstack/prowlarr/kustomization.yaml

@@ -5,6 +5,7 @@ kind: Kustomization
 resources:
   - pvc-config.yaml
   - vaultstaticsecret.yaml
+  - configmap.yaml
   - deployment.yaml
   - service.yaml
   - gateway.yaml

apps/base/arrstack/prowlarr/pvc-config.yaml

@@ -1,6 +1,7 @@
 ---
-# Prowlarr config + SQLite DB. RWO on cephrbd (block) — the arr apps' SQLite
-# does not tolerate CephFS locking. Retain: this is state.
+# Prowlarr /config. RWX on CephFS so all replicas share it (the -unkin2 fork
+# keeps the database in Postgres; /config now holds only config.xml + assets,
+# which tolerate — and want — shared access). Retain: this is state.
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -8,9 +9,9 @@ metadata:
   namespace: arrstack
 spec:
   accessModes:
-    - ReadWriteOnce
+    - ReadWriteMany
   resources:
     requests:
       storage: 5Gi
-  storageClassName: cephrbd-fast-retain
+  storageClassName: cephfs-raid5-retain
   volumeMode: Filesystem