From 8446c314148ad5e9412a5fac4319214882137ea0 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 22 Aug 2026 12:47:38 +1000 Subject: [PATCH] arrstack: fix sync-wave deadlock + config multi-attach (#385) ## Why The active-active arrstack deploy (#383) is stuck in-cluster: the app Deployments (sync-wave 0) can never become Healthy without their Postgres DB, so ArgoCD never advances to create the shared `arrstack-postgres` cluster (wave 1). Separately, `/config` is an immutable RWO cephrbd PVC, so 3 replicas fail with Multi-Attach. ## How - Order DB infra before the apps: `-db` VaultStaticSecrets + backup ObjectStoreUser/Bucket at sync-wave -3, CNPG `Cluster` -2, `Database` CRDs -1; app Deployments stay at default 0. - Switch each app's `/config` to an emptyDir per-pod volume (library DB is in Postgres, config is env-driven; MediaCover regenerates). Old RWO config PVCs orphan. Validated with kustomize build (62 resources). arrproxy/nzbget untouched. Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/385 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- apps/base/arrstack/postgres/cnpg_cluster.yaml | 5 --- .../arrstack/postgres/database-prowlarr.yaml | 2 - .../arrstack/postgres/database-radarr.yaml | 2 - .../arrstack/postgres/database-sonarr.yaml | 2 - .../arrstack/postgres/vaultstaticsecret.yaml | 6 --- apps/base/arrstack/prowlarr/deployment.yaml | 41 ++++++++++++++++++- .../base/arrstack/prowlarr/kustomization.yaml | 1 - apps/base/arrstack/radarr/deployment.yaml | 41 ++++++++++++++++++- apps/base/arrstack/radarr/kustomization.yaml | 1 - apps/base/arrstack/sonarr/deployment.yaml | 41 ++++++++++++++++++- apps/base/arrstack/sonarr/kustomization.yaml | 1 - 11 files changed, 117 insertions(+), 26 deletions(-) diff --git a/apps/base/arrstack/postgres/cnpg_cluster.yaml b/apps/base/arrstack/postgres/cnpg_cluster.yaml index ebe748c..a27ded6 100644 --- a/apps/base/arrstack/postgres/cnpg_cluster.yaml +++ b/apps/base/arrstack/postgres/cnpg_cluster.yaml @@ -11,11 +11,6 @@ kind: Cluster metadata: name: arrstack-postgres namespace: arrstack - annotations: - # Wave 1: the per-app -db Secrets (wave 0) must exist first — CNPG reads - # them as the managed roles' passwordSecret. ArgoCD gates dependents on the - # Cluster's health status. - argocd.argoproj.io/sync-wave: "1" spec: inheritedMetadata: annotations: diff --git a/apps/base/arrstack/postgres/database-prowlarr.yaml b/apps/base/arrstack/postgres/database-prowlarr.yaml index 2dbd313..6d5c0f8 100644 --- a/apps/base/arrstack/postgres/database-prowlarr.yaml +++ b/apps/base/arrstack/postgres/database-prowlarr.yaml @@ -7,8 +7,6 @@ kind: Database metadata: name: prowlarr-main namespace: arrstack - annotations: - argocd.argoproj.io/sync-wave: "2" spec: cluster: name: arrstack-postgres diff --git a/apps/base/arrstack/postgres/database-radarr.yaml b/apps/base/arrstack/postgres/database-radarr.yaml index 56164fd..e30da8f 100644 --- a/apps/base/arrstack/postgres/database-radarr.yaml +++ b/apps/base/arrstack/postgres/database-radarr.yaml @@ -7,8 +7,6 @@ kind: Database metadata: name: radarr-main namespace: arrstack - annotations: - argocd.argoproj.io/sync-wave: "2" spec: cluster: name: arrstack-postgres diff --git a/apps/base/arrstack/postgres/database-sonarr.yaml b/apps/base/arrstack/postgres/database-sonarr.yaml index 57a7258..aa9f521 100644 --- a/apps/base/arrstack/postgres/database-sonarr.yaml +++ b/apps/base/arrstack/postgres/database-sonarr.yaml @@ -7,8 +7,6 @@ kind: Database metadata: name: sonarr-main namespace: arrstack - annotations: - argocd.argoproj.io/sync-wave: "2" spec: cluster: name: arrstack-postgres diff --git a/apps/base/arrstack/postgres/vaultstaticsecret.yaml b/apps/base/arrstack/postgres/vaultstaticsecret.yaml index 4f5c147..101a6ad 100644 --- a/apps/base/arrstack/postgres/vaultstaticsecret.yaml +++ b/apps/base/arrstack/postgres/vaultstaticsecret.yaml @@ -13,8 +13,6 @@ kind: VaultStaticSecret metadata: name: sonarr-db namespace: arrstack - annotations: - argocd.argoproj.io/sync-wave: "0" spec: destination: create: true @@ -32,8 +30,6 @@ kind: VaultStaticSecret metadata: name: radarr-db namespace: arrstack - annotations: - argocd.argoproj.io/sync-wave: "0" spec: destination: create: true @@ -51,8 +47,6 @@ kind: VaultStaticSecret metadata: name: prowlarr-db namespace: arrstack - annotations: - argocd.argoproj.io/sync-wave: "0" spec: destination: create: true diff --git a/apps/base/arrstack/prowlarr/deployment.yaml b/apps/base/arrstack/prowlarr/deployment.yaml index 5f24632..6ee4d6a 100644 --- a/apps/base/arrstack/prowlarr/deployment.yaml +++ b/apps/base/arrstack/prowlarr/deployment.yaml @@ -28,6 +28,44 @@ spec: runAsGroup: 1000 fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch + initContainers: + # Gate the app on its own Postgres database+role being reachable, instead + # of relying on ArgoCD sync-waves (which deadlock if apps aren't Healthy). + # libpq reads PG* from env, so the password never lands in argv. + - name: wait-for-db + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/postgres:17-alpine + command: + - sh + - -c + - | + until psql -tAc 'select 1' >/dev/null 2>&1; do + echo "waiting for $PGDATABASE on $PGHOST..."; sleep 3 + done + echo "database ready" + env: + - name: PGHOST + value: arrstack-postgres-rw.arrstack.svc.cluster.local + - name: PGPORT + value: "5432" + - name: PGDATABASE + value: prowlarr-main + - name: PGUSER + valueFrom: + secretKeyRef: + name: prowlarr-db + key: username + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: prowlarr-db + key: password + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi containers: - name: prowlarr image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/prowlarr:v2.6.2-unkin2 @@ -92,5 +130,4 @@ spec: mountPath: /config volumes: - name: config - persistentVolumeClaim: - claimName: prowlarr-config + emptyDir: {} diff --git a/apps/base/arrstack/prowlarr/kustomization.yaml b/apps/base/arrstack/prowlarr/kustomization.yaml index b1d64b3..faed5e3 100644 --- a/apps/base/arrstack/prowlarr/kustomization.yaml +++ b/apps/base/arrstack/prowlarr/kustomization.yaml @@ -3,7 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - pvc-config.yaml - vaultstaticsecret.yaml - configmap.yaml - deployment.yaml diff --git a/apps/base/arrstack/radarr/deployment.yaml b/apps/base/arrstack/radarr/deployment.yaml index 5a8f133..f52555f 100644 --- a/apps/base/arrstack/radarr/deployment.yaml +++ b/apps/base/arrstack/radarr/deployment.yaml @@ -28,6 +28,44 @@ spec: runAsGroup: 1000 fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch + initContainers: + # Gate the app on its own Postgres database+role being reachable, instead + # of relying on ArgoCD sync-waves (which deadlock if apps aren't Healthy). + # libpq reads PG* from env, so the password never lands in argv. + - name: wait-for-db + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/postgres:17-alpine + command: + - sh + - -c + - | + until psql -tAc 'select 1' >/dev/null 2>&1; do + echo "waiting for $PGDATABASE on $PGHOST..."; sleep 3 + done + echo "database ready" + env: + - name: PGHOST + value: arrstack-postgres-rw.arrstack.svc.cluster.local + - name: PGPORT + value: "5432" + - name: PGDATABASE + value: radarr-main + - name: PGUSER + valueFrom: + secretKeyRef: + name: radarr-db + key: username + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: radarr-db + key: password + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi containers: - name: radarr image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/radarr:v6.4.2-unkin2 @@ -94,8 +132,7 @@ spec: mountPath: /media/movies volumes: - name: config - persistentVolumeClaim: - claimName: radarr-config + emptyDir: {} - name: media-movies persistentVolumeClaim: claimName: media-movies diff --git a/apps/base/arrstack/radarr/kustomization.yaml b/apps/base/arrstack/radarr/kustomization.yaml index b1d64b3..faed5e3 100644 --- a/apps/base/arrstack/radarr/kustomization.yaml +++ b/apps/base/arrstack/radarr/kustomization.yaml @@ -3,7 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - pvc-config.yaml - vaultstaticsecret.yaml - configmap.yaml - deployment.yaml diff --git a/apps/base/arrstack/sonarr/deployment.yaml b/apps/base/arrstack/sonarr/deployment.yaml index 4be6748..9330c47 100644 --- a/apps/base/arrstack/sonarr/deployment.yaml +++ b/apps/base/arrstack/sonarr/deployment.yaml @@ -28,6 +28,44 @@ spec: runAsGroup: 1000 fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch + initContainers: + # Gate the app on its own Postgres database+role being reachable, instead + # of relying on ArgoCD sync-waves (which deadlock if apps aren't Healthy). + # libpq reads PG* from env, so the password never lands in argv. + - name: wait-for-db + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/postgres:17-alpine + command: + - sh + - -c + - | + until psql -tAc 'select 1' >/dev/null 2>&1; do + echo "waiting for $PGDATABASE on $PGHOST..."; sleep 3 + done + echo "database ready" + env: + - name: PGHOST + value: arrstack-postgres-rw.arrstack.svc.cluster.local + - name: PGPORT + value: "5432" + - name: PGDATABASE + value: sonarr-main + - name: PGUSER + valueFrom: + secretKeyRef: + name: sonarr-db + key: username + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: sonarr-db + key: password + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi containers: - name: sonarr image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/sonarr:v5.0.0-unkin2 @@ -94,8 +132,7 @@ spec: mountPath: /media/tv volumes: - name: config - persistentVolumeClaim: - claimName: sonarr-config + emptyDir: {} - name: media-tv persistentVolumeClaim: claimName: media-tv diff --git a/apps/base/arrstack/sonarr/kustomization.yaml b/apps/base/arrstack/sonarr/kustomization.yaml index b1d64b3..faed5e3 100644 --- a/apps/base/arrstack/sonarr/kustomization.yaml +++ b/apps/base/arrstack/sonarr/kustomization.yaml @@ -3,7 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - pvc-config.yaml - vaultstaticsecret.yaml - configmap.yaml - deployment.yaml