From bdf6114794e8dd5602ff7586cb0afdf8ef4e7919 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 22 Aug 2026 18:39:40 +1000 Subject: [PATCH] arrstack: S3 buckets + wiring for app backups and sonarr MediaCover (#389) ## Why Servarr backups (periodic config+DB zips) and sonarr MediaCover posters still live on each replica's local `/config` emptyDir: backups are leader-local and lost on reschedule, and sonarr covers render inconsistently across the 3 active-active replicas. #387 moved radarr covers to a shared Ceph RGW bucket; this extends that pattern to backups for all three apps and to sonarr's covers. ## Changes - Add an `arrstack-backups` ObjectStoreUser + Bucket (cephrgw-operator); the operator mints the `arrstack-backups-s3` credential Secret in-namespace (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / S3_ENDPOINT), so no Vault KV seeding is required. Backups get their own bucket (not `arrstack-media`) because they carry secrets (Config.xml API keys, the DB) and warrant separate credentials/lifecycle from poster art; the three apps share it, isolated by a per-app key prefix. - Wire sonarr to the shared `arrstack-media` bucket via `Sonarr__MediaCoverS3__*` (prefix `sonarr`, partitioned from radarr's `radarr` prefix to avoid movie/series id collisions), plus the estate CA mount at `/etc/ssl/vault-ca/ca.crt` for RGW TLS validation. - Wire radarr/sonarr/prowlarr backups via `__BackupS3__*` to `arrstack-backups` (per-app prefix); add the CA mount to sonarr and prowlarr. ## Notes - Credentials come only from the operator-minted Secret via env; no secrets in manifests. - The env keys are consumed by the -unkin3+ images (separate app PRs) and ignored by older images, so this is safe to apply ahead of the image bump. - `kubectl kustomize apps/base/arrstack` builds clean. Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/389 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- apps/base/arrstack/backups-bucket.yaml | 42 +++++++++++++ apps/base/arrstack/kustomization.yaml | 1 + apps/base/arrstack/prowlarr/deployment.yaml | 38 ++++++++++++ apps/base/arrstack/radarr/deployment.yaml | 26 +++++++++ apps/base/arrstack/sonarr/deployment.yaml | 65 +++++++++++++++++++++ 5 files changed, 172 insertions(+) create mode 100644 apps/base/arrstack/backups-bucket.yaml diff --git a/apps/base/arrstack/backups-bucket.yaml b/apps/base/arrstack/backups-bucket.yaml new file mode 100644 index 0000000..76beff3 --- /dev/null +++ b/apps/base/arrstack/backups-bucket.yaml @@ -0,0 +1,42 @@ +--- +# Shared Ceph RGW (S3) bucket for arrstack application backups (the periodic +# radarr/sonarr/prowlarr config+database zip backups), provisioned by the +# in-estate cephrgw-operator. Backups otherwise land on each replica's local +# /config (an emptyDir) and are lost when that pod is rescheduled; routing them +# to S3 lets any replica write and restore them. +# +# Backups are kept in their own bucket (not the arrstack-media one) because they +# carry secrets (Config.xml holds API keys, the DB has all app state) and thus +# warrant separate credentials, lifecycle and retention from the public-ish +# poster art. The three apps share this one bucket, isolated by a per-app key +# prefix (radarr/, sonarr/, prowlarr/) set via __BackupS3__Prefix. +# +# The operator mints the S3 credential Secret (arrstack-backups-s3) in this +# namespace with keys AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and S3_ENDPOINT, +# so no Vault KV seeding is required. +apiVersion: ceph.unkin.net/v1alpha1 +kind: ObjectStoreUser +metadata: + name: arrstack-backups + namespace: arrstack +spec: + displayName: "arrstack backups owner" + uid: arrstack-backups + maxBuckets: 5 + secretName: arrstack-backups-s3 + retainOnDelete: true +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: Bucket +metadata: + name: arrstack-backups + namespace: arrstack +spec: + placementTarget: ec + bucketName: arrstack-backups + ownerRef: arrstack-backups + versioning: false + tags: + app: arrstack + purpose: backups + retainOnDelete: true diff --git a/apps/base/arrstack/kustomization.yaml b/apps/base/arrstack/kustomization.yaml index 2f16ec7..d676b39 100644 --- a/apps/base/arrstack/kustomization.yaml +++ b/apps/base/arrstack/kustomization.yaml @@ -10,6 +10,7 @@ resources: - pvc-media-tv.yaml - pvc-media-movies.yaml - media-bucket.yaml + - backups-bucket.yaml - postgres - sonarr - radarr diff --git a/apps/base/arrstack/prowlarr/deployment.yaml b/apps/base/arrstack/prowlarr/deployment.yaml index 5b40242..e43806b 100644 --- a/apps/base/arrstack/prowlarr/deployment.yaml +++ b/apps/base/arrstack/prowlarr/deployment.yaml @@ -98,6 +98,34 @@ spec: secretKeyRef: name: prowlarr-apikey key: apitoken + # Backup object store (shared arrstack-backups Ceph RGW bucket, + # per-app key prefix). Routes the periodic config+DB zip backups off + # the ephemeral /config so any replica can write and restore them. + # Consumed by the -unkin3+ image; older images ignore these unknown + # config keys. Creds Secret minted by cephrgw-operator. + - name: Prowlarr__BackupS3__Endpoint + valueFrom: + secretKeyRef: + name: arrstack-backups-s3 + key: S3_ENDPOINT + - name: Prowlarr__BackupS3__AccessKey + valueFrom: + secretKeyRef: + name: arrstack-backups-s3 + key: AWS_ACCESS_KEY_ID + - name: Prowlarr__BackupS3__SecretKey + valueFrom: + secretKeyRef: + name: arrstack-backups-s3 + key: AWS_SECRET_ACCESS_KEY + - name: Prowlarr__BackupS3__Bucket + value: arrstack-backups + - name: Prowlarr__BackupS3__Prefix + value: prowlarr + - name: Prowlarr__BackupS3__ForcePathStyle + value: "true" + - name: Prowlarr__BackupS3__CaCertPath + value: /etc/ssl/vault-ca/ca.crt livenessProbe: httpGet: path: /prowlarr/ping @@ -124,6 +152,16 @@ spec: volumeMounts: - name: config mountPath: /config + - name: vault-ca + mountPath: /etc/ssl/vault-ca + readOnly: true volumes: - name: config emptyDir: {} + # Estate CA for validating the Ceph RGW (s3.ceph.unkin.net) TLS cert. + - name: vault-ca + secret: + secretName: vault-ca-cert + items: + - key: ca.crt + path: ca.crt diff --git a/apps/base/arrstack/radarr/deployment.yaml b/apps/base/arrstack/radarr/deployment.yaml index e8c3628..395bff3 100644 --- a/apps/base/arrstack/radarr/deployment.yaml +++ b/apps/base/arrstack/radarr/deployment.yaml @@ -125,6 +125,32 @@ spec: value: "true" - name: Radarr__MediaCover__S3__CaCertPath value: /etc/ssl/vault-ca/ca.crt + # Backup object store (shared arrstack-backups Ceph RGW bucket, + # per-app key prefix). Routes the periodic config+DB zip backups off + # the ephemeral /config so any replica can write and restore them. + - name: Radarr__BackupS3__Endpoint + valueFrom: + secretKeyRef: + name: arrstack-backups-s3 + key: S3_ENDPOINT + - name: Radarr__BackupS3__AccessKey + valueFrom: + secretKeyRef: + name: arrstack-backups-s3 + key: AWS_ACCESS_KEY_ID + - name: Radarr__BackupS3__SecretKey + valueFrom: + secretKeyRef: + name: arrstack-backups-s3 + key: AWS_SECRET_ACCESS_KEY + - name: Radarr__BackupS3__Bucket + value: arrstack-backups + - name: Radarr__BackupS3__Prefix + value: radarr + - name: Radarr__BackupS3__ForcePathStyle + value: "true" + - name: Radarr__BackupS3__CaCertPath + value: /etc/ssl/vault-ca/ca.crt livenessProbe: httpGet: path: /radarr/ping diff --git a/apps/base/arrstack/sonarr/deployment.yaml b/apps/base/arrstack/sonarr/deployment.yaml index ded5b99..d04d9fe 100644 --- a/apps/base/arrstack/sonarr/deployment.yaml +++ b/apps/base/arrstack/sonarr/deployment.yaml @@ -98,6 +98,61 @@ spec: secretKeyRef: name: sonarr-apikey key: apitoken + # MediaCover object store (shared arrstack-media Ceph RGW bucket, + # partitioned from radarr by the sonarr key prefix). Serves + # posters/fanart from S3 so any replica renders them instead of the + # leader-local emptyDir /config. Consumed by the -unkin3+ image; + # older images ignore these unknown config keys. Creds Secret minted + # by cephrgw-operator. + - name: Sonarr__MediaCoverS3__Endpoint + valueFrom: + secretKeyRef: + name: arrstack-media-s3 + key: S3_ENDPOINT + - name: Sonarr__MediaCoverS3__AccessKey + valueFrom: + secretKeyRef: + name: arrstack-media-s3 + key: AWS_ACCESS_KEY_ID + - name: Sonarr__MediaCoverS3__SecretKey + valueFrom: + secretKeyRef: + name: arrstack-media-s3 + key: AWS_SECRET_ACCESS_KEY + - name: Sonarr__MediaCoverS3__Bucket + value: arrstack-media + - name: Sonarr__MediaCoverS3__Prefix + value: sonarr + - name: Sonarr__MediaCoverS3__ForcePathStyle + value: "true" + - name: Sonarr__MediaCoverS3__CaCertPath + value: /etc/ssl/vault-ca/ca.crt + # Backup object store (shared arrstack-backups Ceph RGW bucket, + # per-app key prefix). Routes the periodic config+DB zip backups off + # the ephemeral /config so any replica can write and restore them. + - name: Sonarr__BackupS3__Endpoint + valueFrom: + secretKeyRef: + name: arrstack-backups-s3 + key: S3_ENDPOINT + - name: Sonarr__BackupS3__AccessKey + valueFrom: + secretKeyRef: + name: arrstack-backups-s3 + key: AWS_ACCESS_KEY_ID + - name: Sonarr__BackupS3__SecretKey + valueFrom: + secretKeyRef: + name: arrstack-backups-s3 + key: AWS_SECRET_ACCESS_KEY + - name: Sonarr__BackupS3__Bucket + value: arrstack-backups + - name: Sonarr__BackupS3__Prefix + value: sonarr + - name: Sonarr__BackupS3__ForcePathStyle + value: "true" + - name: Sonarr__BackupS3__CaCertPath + value: /etc/ssl/vault-ca/ca.crt livenessProbe: httpGet: path: /sonarr/ping @@ -126,9 +181,19 @@ spec: mountPath: /config - name: media-tv mountPath: /media/tv + - name: vault-ca + mountPath: /etc/ssl/vault-ca + readOnly: true volumes: - name: config emptyDir: {} - name: media-tv persistentVolumeClaim: claimName: media-tv + # Estate CA for validating the Ceph RGW (s3.ceph.unkin.net) TLS cert. + - name: vault-ca + secret: + secretName: vault-ca-cert + items: + - key: ca.crt + path: ca.crt