From df687e310bf3ddf2423be9ba6baeba6f61cd3600 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 22 Aug 2026 12:21:25 +1000 Subject: [PATCH 1/2] arrstack: fix sync-wave deadlock + config multi-attach The active-active arrstack deploy (#383) was stuck: the app Deployments (wave 0) can't become Healthy without their database, so ArgoCD never advanced to wave 1 to create the shared Postgres cluster. And /config was an RWO cephrbd PVC, so 3 replicas hit Multi-Attach. - move DB infra ahead of the apps: VSS + backup ObjectStoreUser/Bucket wave -3, CNPG Cluster -2, Databases -1 (apps stay at default 0) - switch each app's /config to emptyDir (per-pod, stateless; library is in Postgres, config is env-driven; MediaCover regenerates). Old RWO config PVCs orphan. --- apps/base/arrstack/postgres/cnpg_backup.yaml | 5 +++++ apps/base/arrstack/postgres/cnpg_cluster.yaml | 2 +- apps/base/arrstack/postgres/database-prowlarr.yaml | 2 +- apps/base/arrstack/postgres/database-radarr.yaml | 2 +- apps/base/arrstack/postgres/database-sonarr.yaml | 2 +- apps/base/arrstack/postgres/vaultstaticsecret.yaml | 6 +++--- apps/base/arrstack/prowlarr/deployment.yaml | 3 +-- apps/base/arrstack/prowlarr/kustomization.yaml | 1 - apps/base/arrstack/radarr/deployment.yaml | 3 +-- apps/base/arrstack/radarr/kustomization.yaml | 1 - apps/base/arrstack/sonarr/deployment.yaml | 3 +-- apps/base/arrstack/sonarr/kustomization.yaml | 1 - 12 files changed, 15 insertions(+), 16 deletions(-) diff --git a/apps/base/arrstack/postgres/cnpg_backup.yaml b/apps/base/arrstack/postgres/cnpg_backup.yaml index fce9a69..acabcd3 100644 --- a/apps/base/arrstack/postgres/cnpg_backup.yaml +++ b/apps/base/arrstack/postgres/cnpg_backup.yaml @@ -7,6 +7,9 @@ kind: ObjectStoreUser metadata: name: cnpg-arrstack-backup namespace: arrstack + annotations: + # S3 creds Secret must exist before the Cluster (wave -2) reconciles its backup. + argocd.argoproj.io/sync-wave: "-3" spec: displayName: "CNPG backup owner (arrstack)" uid: cnpg-arrstack-backup @@ -19,6 +22,8 @@ kind: Bucket metadata: name: cnpg-arrstack namespace: arrstack + annotations: + argocd.argoproj.io/sync-wave: "-3" spec: placementTarget: ec bucketName: cnpg-arrstack diff --git a/apps/base/arrstack/postgres/cnpg_cluster.yaml b/apps/base/arrstack/postgres/cnpg_cluster.yaml index ebe748c..fc2813f 100644 --- a/apps/base/arrstack/postgres/cnpg_cluster.yaml +++ b/apps/base/arrstack/postgres/cnpg_cluster.yaml @@ -15,7 +15,7 @@ metadata: # Wave 1: the per-app -db Secrets (wave 0) must exist first — CNPG reads # them as the managed roles' passwordSecret. ArgoCD gates dependents on the # Cluster's health status. - argocd.argoproj.io/sync-wave: "1" + argocd.argoproj.io/sync-wave: "-2" spec: inheritedMetadata: annotations: diff --git a/apps/base/arrstack/postgres/database-prowlarr.yaml b/apps/base/arrstack/postgres/database-prowlarr.yaml index 2dbd313..56febfd 100644 --- a/apps/base/arrstack/postgres/database-prowlarr.yaml +++ b/apps/base/arrstack/postgres/database-prowlarr.yaml @@ -8,7 +8,7 @@ metadata: name: prowlarr-main namespace: arrstack annotations: - argocd.argoproj.io/sync-wave: "2" + argocd.argoproj.io/sync-wave: "-1" spec: cluster: name: arrstack-postgres diff --git a/apps/base/arrstack/postgres/database-radarr.yaml b/apps/base/arrstack/postgres/database-radarr.yaml index 56164fd..fa205c1 100644 --- a/apps/base/arrstack/postgres/database-radarr.yaml +++ b/apps/base/arrstack/postgres/database-radarr.yaml @@ -8,7 +8,7 @@ metadata: name: radarr-main namespace: arrstack annotations: - argocd.argoproj.io/sync-wave: "2" + argocd.argoproj.io/sync-wave: "-1" spec: cluster: name: arrstack-postgres diff --git a/apps/base/arrstack/postgres/database-sonarr.yaml b/apps/base/arrstack/postgres/database-sonarr.yaml index 57a7258..91e3d62 100644 --- a/apps/base/arrstack/postgres/database-sonarr.yaml +++ b/apps/base/arrstack/postgres/database-sonarr.yaml @@ -8,7 +8,7 @@ metadata: name: sonarr-main namespace: arrstack annotations: - argocd.argoproj.io/sync-wave: "2" + argocd.argoproj.io/sync-wave: "-1" spec: cluster: name: arrstack-postgres diff --git a/apps/base/arrstack/postgres/vaultstaticsecret.yaml b/apps/base/arrstack/postgres/vaultstaticsecret.yaml index 4f5c147..1965408 100644 --- a/apps/base/arrstack/postgres/vaultstaticsecret.yaml +++ b/apps/base/arrstack/postgres/vaultstaticsecret.yaml @@ -14,7 +14,7 @@ metadata: name: sonarr-db namespace: arrstack annotations: - argocd.argoproj.io/sync-wave: "0" + argocd.argoproj.io/sync-wave: "-3" spec: destination: create: true @@ -33,7 +33,7 @@ metadata: name: radarr-db namespace: arrstack annotations: - argocd.argoproj.io/sync-wave: "0" + argocd.argoproj.io/sync-wave: "-3" spec: destination: create: true @@ -52,7 +52,7 @@ metadata: name: prowlarr-db namespace: arrstack annotations: - argocd.argoproj.io/sync-wave: "0" + argocd.argoproj.io/sync-wave: "-3" spec: destination: create: true diff --git a/apps/base/arrstack/prowlarr/deployment.yaml b/apps/base/arrstack/prowlarr/deployment.yaml index 5f24632..597786a 100644 --- a/apps/base/arrstack/prowlarr/deployment.yaml +++ b/apps/base/arrstack/prowlarr/deployment.yaml @@ -92,5 +92,4 @@ spec: mountPath: /config volumes: - name: config - persistentVolumeClaim: - claimName: prowlarr-config + emptyDir: {} diff --git a/apps/base/arrstack/prowlarr/kustomization.yaml b/apps/base/arrstack/prowlarr/kustomization.yaml index b1d64b3..faed5e3 100644 --- a/apps/base/arrstack/prowlarr/kustomization.yaml +++ b/apps/base/arrstack/prowlarr/kustomization.yaml @@ -3,7 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - pvc-config.yaml - vaultstaticsecret.yaml - configmap.yaml - deployment.yaml diff --git a/apps/base/arrstack/radarr/deployment.yaml b/apps/base/arrstack/radarr/deployment.yaml index 5a8f133..50e5e7f 100644 --- a/apps/base/arrstack/radarr/deployment.yaml +++ b/apps/base/arrstack/radarr/deployment.yaml @@ -94,8 +94,7 @@ spec: mountPath: /media/movies volumes: - name: config - persistentVolumeClaim: - claimName: radarr-config + emptyDir: {} - name: media-movies persistentVolumeClaim: claimName: media-movies diff --git a/apps/base/arrstack/radarr/kustomization.yaml b/apps/base/arrstack/radarr/kustomization.yaml index b1d64b3..faed5e3 100644 --- a/apps/base/arrstack/radarr/kustomization.yaml +++ b/apps/base/arrstack/radarr/kustomization.yaml @@ -3,7 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - pvc-config.yaml - vaultstaticsecret.yaml - configmap.yaml - deployment.yaml diff --git a/apps/base/arrstack/sonarr/deployment.yaml b/apps/base/arrstack/sonarr/deployment.yaml index 4be6748..1f9084b 100644 --- a/apps/base/arrstack/sonarr/deployment.yaml +++ b/apps/base/arrstack/sonarr/deployment.yaml @@ -94,8 +94,7 @@ spec: mountPath: /media/tv volumes: - name: config - persistentVolumeClaim: - claimName: sonarr-config + emptyDir: {} - name: media-tv persistentVolumeClaim: claimName: media-tv diff --git a/apps/base/arrstack/sonarr/kustomization.yaml b/apps/base/arrstack/sonarr/kustomization.yaml index b1d64b3..faed5e3 100644 --- a/apps/base/arrstack/sonarr/kustomization.yaml +++ b/apps/base/arrstack/sonarr/kustomization.yaml @@ -3,7 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - pvc-config.yaml - vaultstaticsecret.yaml - configmap.yaml - deployment.yaml -- 2.47.3 From 60309fc5c26ac5410cf5b87b61890dedca6cfcc9 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 22 Aug 2026 12:38:02 +1000 Subject: [PATCH 2/2] arrstack: gate apps on DB via initContainer instead of sync-waves Per review: drop the ArgoCD sync-wave annotations (their health-gating is what deadlocked the first deploy) and instead add a wait-for-db init container to each app that blocks until its own Postgres database+role is reachable (libpq PG* env, password never in argv). ArgoCD applies everything at once; the app container only starts once its DB is ready. --- apps/base/arrstack/postgres/cnpg_backup.yaml | 5 --- apps/base/arrstack/postgres/cnpg_cluster.yaml | 5 --- .../arrstack/postgres/database-prowlarr.yaml | 2 - .../arrstack/postgres/database-radarr.yaml | 2 - .../arrstack/postgres/database-sonarr.yaml | 2 - .../arrstack/postgres/vaultstaticsecret.yaml | 6 --- apps/base/arrstack/prowlarr/deployment.yaml | 38 +++++++++++++++++++ apps/base/arrstack/radarr/deployment.yaml | 38 +++++++++++++++++++ apps/base/arrstack/sonarr/deployment.yaml | 38 +++++++++++++++++++ 9 files changed, 114 insertions(+), 22 deletions(-) diff --git a/apps/base/arrstack/postgres/cnpg_backup.yaml b/apps/base/arrstack/postgres/cnpg_backup.yaml index acabcd3..fce9a69 100644 --- a/apps/base/arrstack/postgres/cnpg_backup.yaml +++ b/apps/base/arrstack/postgres/cnpg_backup.yaml @@ -7,9 +7,6 @@ kind: ObjectStoreUser metadata: name: cnpg-arrstack-backup namespace: arrstack - annotations: - # S3 creds Secret must exist before the Cluster (wave -2) reconciles its backup. - argocd.argoproj.io/sync-wave: "-3" spec: displayName: "CNPG backup owner (arrstack)" uid: cnpg-arrstack-backup @@ -22,8 +19,6 @@ kind: Bucket metadata: name: cnpg-arrstack namespace: arrstack - annotations: - argocd.argoproj.io/sync-wave: "-3" spec: placementTarget: ec bucketName: cnpg-arrstack diff --git a/apps/base/arrstack/postgres/cnpg_cluster.yaml b/apps/base/arrstack/postgres/cnpg_cluster.yaml index fc2813f..a27ded6 100644 --- a/apps/base/arrstack/postgres/cnpg_cluster.yaml +++ b/apps/base/arrstack/postgres/cnpg_cluster.yaml @@ -11,11 +11,6 @@ kind: Cluster metadata: name: arrstack-postgres namespace: arrstack - annotations: - # Wave 1: the per-app -db Secrets (wave 0) must exist first — CNPG reads - # them as the managed roles' passwordSecret. ArgoCD gates dependents on the - # Cluster's health status. - argocd.argoproj.io/sync-wave: "-2" spec: inheritedMetadata: annotations: diff --git a/apps/base/arrstack/postgres/database-prowlarr.yaml b/apps/base/arrstack/postgres/database-prowlarr.yaml index 56febfd..6d5c0f8 100644 --- a/apps/base/arrstack/postgres/database-prowlarr.yaml +++ b/apps/base/arrstack/postgres/database-prowlarr.yaml @@ -7,8 +7,6 @@ kind: Database metadata: name: prowlarr-main namespace: arrstack - annotations: - argocd.argoproj.io/sync-wave: "-1" spec: cluster: name: arrstack-postgres diff --git a/apps/base/arrstack/postgres/database-radarr.yaml b/apps/base/arrstack/postgres/database-radarr.yaml index fa205c1..e30da8f 100644 --- a/apps/base/arrstack/postgres/database-radarr.yaml +++ b/apps/base/arrstack/postgres/database-radarr.yaml @@ -7,8 +7,6 @@ kind: Database metadata: name: radarr-main namespace: arrstack - annotations: - argocd.argoproj.io/sync-wave: "-1" spec: cluster: name: arrstack-postgres diff --git a/apps/base/arrstack/postgres/database-sonarr.yaml b/apps/base/arrstack/postgres/database-sonarr.yaml index 91e3d62..aa9f521 100644 --- a/apps/base/arrstack/postgres/database-sonarr.yaml +++ b/apps/base/arrstack/postgres/database-sonarr.yaml @@ -7,8 +7,6 @@ kind: Database metadata: name: sonarr-main namespace: arrstack - annotations: - argocd.argoproj.io/sync-wave: "-1" spec: cluster: name: arrstack-postgres diff --git a/apps/base/arrstack/postgres/vaultstaticsecret.yaml b/apps/base/arrstack/postgres/vaultstaticsecret.yaml index 1965408..101a6ad 100644 --- a/apps/base/arrstack/postgres/vaultstaticsecret.yaml +++ b/apps/base/arrstack/postgres/vaultstaticsecret.yaml @@ -13,8 +13,6 @@ kind: VaultStaticSecret metadata: name: sonarr-db namespace: arrstack - annotations: - argocd.argoproj.io/sync-wave: "-3" spec: destination: create: true @@ -32,8 +30,6 @@ kind: VaultStaticSecret metadata: name: radarr-db namespace: arrstack - annotations: - argocd.argoproj.io/sync-wave: "-3" spec: destination: create: true @@ -51,8 +47,6 @@ kind: VaultStaticSecret metadata: name: prowlarr-db namespace: arrstack - annotations: - argocd.argoproj.io/sync-wave: "-3" spec: destination: create: true diff --git a/apps/base/arrstack/prowlarr/deployment.yaml b/apps/base/arrstack/prowlarr/deployment.yaml index 597786a..6ee4d6a 100644 --- a/apps/base/arrstack/prowlarr/deployment.yaml +++ b/apps/base/arrstack/prowlarr/deployment.yaml @@ -28,6 +28,44 @@ spec: runAsGroup: 1000 fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch + initContainers: + # Gate the app on its own Postgres database+role being reachable, instead + # of relying on ArgoCD sync-waves (which deadlock if apps aren't Healthy). + # libpq reads PG* from env, so the password never lands in argv. + - name: wait-for-db + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/postgres:17-alpine + command: + - sh + - -c + - | + until psql -tAc 'select 1' >/dev/null 2>&1; do + echo "waiting for $PGDATABASE on $PGHOST..."; sleep 3 + done + echo "database ready" + env: + - name: PGHOST + value: arrstack-postgres-rw.arrstack.svc.cluster.local + - name: PGPORT + value: "5432" + - name: PGDATABASE + value: prowlarr-main + - name: PGUSER + valueFrom: + secretKeyRef: + name: prowlarr-db + key: username + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: prowlarr-db + key: password + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi containers: - name: prowlarr image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/prowlarr:v2.6.2-unkin2 diff --git a/apps/base/arrstack/radarr/deployment.yaml b/apps/base/arrstack/radarr/deployment.yaml index 50e5e7f..f52555f 100644 --- a/apps/base/arrstack/radarr/deployment.yaml +++ b/apps/base/arrstack/radarr/deployment.yaml @@ -28,6 +28,44 @@ spec: runAsGroup: 1000 fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch + initContainers: + # Gate the app on its own Postgres database+role being reachable, instead + # of relying on ArgoCD sync-waves (which deadlock if apps aren't Healthy). + # libpq reads PG* from env, so the password never lands in argv. + - name: wait-for-db + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/postgres:17-alpine + command: + - sh + - -c + - | + until psql -tAc 'select 1' >/dev/null 2>&1; do + echo "waiting for $PGDATABASE on $PGHOST..."; sleep 3 + done + echo "database ready" + env: + - name: PGHOST + value: arrstack-postgres-rw.arrstack.svc.cluster.local + - name: PGPORT + value: "5432" + - name: PGDATABASE + value: radarr-main + - name: PGUSER + valueFrom: + secretKeyRef: + name: radarr-db + key: username + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: radarr-db + key: password + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi containers: - name: radarr image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/radarr:v6.4.2-unkin2 diff --git a/apps/base/arrstack/sonarr/deployment.yaml b/apps/base/arrstack/sonarr/deployment.yaml index 1f9084b..9330c47 100644 --- a/apps/base/arrstack/sonarr/deployment.yaml +++ b/apps/base/arrstack/sonarr/deployment.yaml @@ -28,6 +28,44 @@ spec: runAsGroup: 1000 fsGroup: 1000 fsGroupChangePolicy: OnRootMismatch + initContainers: + # Gate the app on its own Postgres database+role being reachable, instead + # of relying on ArgoCD sync-waves (which deadlock if apps aren't Healthy). + # libpq reads PG* from env, so the password never lands in argv. + - name: wait-for-db + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/postgres:17-alpine + command: + - sh + - -c + - | + until psql -tAc 'select 1' >/dev/null 2>&1; do + echo "waiting for $PGDATABASE on $PGHOST..."; sleep 3 + done + echo "database ready" + env: + - name: PGHOST + value: arrstack-postgres-rw.arrstack.svc.cluster.local + - name: PGPORT + value: "5432" + - name: PGDATABASE + value: sonarr-main + - name: PGUSER + valueFrom: + secretKeyRef: + name: sonarr-db + key: username + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: sonarr-db + key: password + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi containers: - name: sonarr image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/sonarr:v5.0.0-unkin2 -- 2.47.3