From 3c2bdf307a2c3463e2be0af5e29a42f922b66664 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Mon, 27 Jul 2026 23:57:08 +1000 Subject: [PATCH] Add S3 backups to all CNPG Postgres clusters (#298) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why None of the 8 CNPG Postgres clusters in this repo had **any** backup configured. A lost PVC, a fat-fingered migration, or a bad app deploy meant permanent, unrecoverable data loss for authentik, litellm, artifactapi, woodpecker, puppet, encapi, paperclip and grafana. This adds continuous WAL archiving + a nightly base backup to Ceph RGW for every cluster, plus code-forward restore docs. ## What - **`spec.backup.barmanObjectStore`** on each `cnpg_cluster.yaml` — turns on continuous WAL archiving to `s3://cnpg-`, WAL compressed with zstd, base backups with bzip2, 30-day retention. TLS to `s3.ceph.unkin.net` is trusted via the reflected `vault-ca-cert` (`endpointCA`). - **`cnpg_backup.yaml`** per app — a cephrgw `ObjectStoreUser` + `Bucket` (operator provisions the bucket and mints the S3 key into `cnpg--backup-s3`; **nothing is hardcoded**) and a staggered nightly `ScheduledBackup`. - **`schemas/ceph.unkin.net/*.json`** — the three cephrgw CRD schemas so kubeconform can validate the new CRs. - **`docs/`** — new docs folder (README index + `cnpg-backups.md` + `cnpg-restore.md`). ## Design decisions (answers to the open questions) **One bucket for all, or per-database?** → **Per-database (one bucket + owner user per cluster).** The cephrgw CRDs are namespace-scoped (`BucketRef`/`OwnerRef` resolve only *within the same namespace*), and CNPG reads its S3 credential Secret from its *own* namespace. A single shared bucket would require either cross-namespace bucket refs (unsupported) or hand-copying the S3 secret into all 8 namespaces (defeats "operator mints the keys"). Per-namespace `s3://cnpg-` with a dedicated owner user is the simplest correct topology and needs zero manual seeding. Each user owns exactly one bucket, so owner-level (full) access is already tightly scoped — no extra `BucketAccess` grant needed. **Backup mechanism.** The deployed CNPG operator is **v1.28** (helm chart `cloudnative-pg-0.27.0`, appVersion 1.28.0). 1.26+ deprecates the in-tree `barmanObjectStore` in favour of the Barman Cloud Plugin, but the plugin is **not deployed**, and `barmanObjectStore` is still fully functional on 1.28. So this uses the in-tree mechanism. Migrating to the plugin is a follow-up (noted in `docs/cnpg-backups.md`). ## Schedule / retention (defaults — Ben to adjust) | App | Cluster | Bucket | Nightly base backup | | --- | --- | --- | --- | | authentik | postgres | cnpg-authentik | 01:00 | | litellm | litellm-postgres | cnpg-litellm | 01:20 | | artifactapi | postgres | cnpg-artifactapi | 01:40 | | woodpecker | woodpecker-postgres | cnpg-woodpecker | 02:00 | | puppet | puppet-postgres | cnpg-puppet | 02:20 | | encapi | postgres | cnpg-encapi | 02:40 | | paperclip | paperclip-postgres | cnpg-paperclip | 03:00 | | grafana | postgres | cnpg-grafana | 03:20 | Retention is **30d** across the board — flagged as a default to tune per cluster. Schedules are staggered 20 min apart so 8 base backups don't hit RGW at once. ## Validation - `kustomize build --enable-helm` + `kubeconform` (repo CI args, incl. the new ceph schemas) pass on all 8 affected overlays (paperclip validated at base — it has no overlay yet). ceph CRs resolve their schemas (`Skipped: 0`). - `pre-commit run` passes on all changed files (yamllint, no-plain-secrets, etc.). - Note: a full `ci/validate-apps.sh` run aborts locally on the unrelated `cattle-system` overlay (`chart requires kubeVersion < 1.35 vs host helm v1.36.0`) — pre-existing, reproduces on `origin/main`, unrelated to this change. ## Notes / caveats - No overlap with the woodpecker chart-bump PR (#297, overlay files only) or the logging PR (#296) beyond the three **identical** generated `schemas/ceph.unkin.net/*.json` files, which merge cleanly whichever lands first. - Credentials: no manual seeding — the cephrgw-operator mints the RGW user + keys. The only prerequisite is the operator being healthy (it is, in `cephrgw-system`). ## Follow-ups - Barman Cloud Plugin migration (deploy plugin, move clusters to `ObjectStore` CRs). - Tune per-cluster retention / schedule if the defaults don't fit. https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/298 Co-authored-by: Ben Vincent Co-committed-by: Ben Vincent --- apps/base/artifactapi/cnpg_backup.yaml | 55 +++++ apps/base/artifactapi/cnpg_cluster.yaml | 29 +++ apps/base/artifactapi/kustomization.yaml | 1 + apps/base/authentik/cnpg_backup.yaml | 55 +++++ apps/base/authentik/cnpg_cluster.yaml | 29 +++ apps/base/authentik/kustomization.yaml | 1 + apps/base/encapi/cnpg_backup.yaml | 55 +++++ apps/base/encapi/cnpg_cluster.yaml | 29 +++ apps/base/encapi/kustomization.yaml | 1 + apps/base/grafana/cnpg_backup.yaml | 55 +++++ apps/base/grafana/cnpg_cluster.yaml | 29 +++ apps/base/grafana/kustomization.yaml | 1 + apps/base/litellm/cnpg_backup.yaml | 55 +++++ apps/base/litellm/cnpg_cluster.yaml | 29 +++ apps/base/litellm/kustomization.yaml | 1 + apps/base/paperclip/cnpg_backup.yaml | 55 +++++ apps/base/paperclip/cnpg_cluster.yaml | 29 +++ apps/base/paperclip/kustomization.yaml | 1 + apps/base/puppet/cnpg_backup.yaml | 55 +++++ apps/base/puppet/cnpg_cluster.yaml | 29 +++ apps/base/puppet/kustomization.yaml | 1 + apps/base/woodpecker/cnpg_backup.yaml | 55 +++++ apps/base/woodpecker/cnpg_cluster.yaml | 29 +++ apps/base/woodpecker/kustomization.yaml | 1 + docs/README.md | 8 + docs/cnpg-backups.md | 146 +++++++++++++ docs/cnpg-restore.md | 131 ++++++++++++ schemas/ceph.unkin.net/bucket_v1alpha1.json | 176 ++++++++++++++++ .../ceph.unkin.net/bucketaccess_v1alpha1.json | 191 ++++++++++++++++++ .../objectstoreuser_v1alpha1.json | 132 ++++++++++++ 30 files changed, 1464 insertions(+) create mode 100644 apps/base/artifactapi/cnpg_backup.yaml create mode 100644 apps/base/authentik/cnpg_backup.yaml create mode 100644 apps/base/encapi/cnpg_backup.yaml create mode 100644 apps/base/grafana/cnpg_backup.yaml create mode 100644 apps/base/litellm/cnpg_backup.yaml create mode 100644 apps/base/paperclip/cnpg_backup.yaml create mode 100644 apps/base/puppet/cnpg_backup.yaml create mode 100644 apps/base/woodpecker/cnpg_backup.yaml create mode 100644 docs/README.md create mode 100644 docs/cnpg-backups.md create mode 100644 docs/cnpg-restore.md create mode 100644 schemas/ceph.unkin.net/bucket_v1alpha1.json create mode 100644 schemas/ceph.unkin.net/bucketaccess_v1alpha1.json create mode 100644 schemas/ceph.unkin.net/objectstoreuser_v1alpha1.json diff --git a/apps/base/artifactapi/cnpg_backup.yaml b/apps/base/artifactapi/cnpg_backup.yaml new file mode 100644 index 0000000..ae94c34 --- /dev/null +++ b/apps/base/artifactapi/cnpg_backup.yaml @@ -0,0 +1,55 @@ +--- +# Ceph RGW (S3) backup target for the artifactapi CNPG cluster, provisioned by the +# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster: +# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from +# its own namespace, so backups are per-database rather than one shared bucket. +apiVersion: ceph.unkin.net/v1alpha1 +kind: ObjectStoreUser +metadata: + name: cnpg-artifactapi-backup + namespace: artifactapi +spec: + displayName: "CNPG backup owner (artifactapi)" + # RGW users are global; keep the uid namespace-qualified so it never collides. + uid: cnpg-artifactapi-backup + maxBuckets: 5 + # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID, + # S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it. + secretName: cnpg-artifactapi-backup-s3 + # Keep the RGW user (and thus the keys) if this CR is ever deleted, so an + # in-flight restore can still reach the archive. + retainOnDelete: true +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: Bucket +metadata: + name: cnpg-artifactapi + namespace: artifactapi +spec: + bucketName: cnpg-artifactapi + # The owner user has full control of its own bucket (read + write), which is + # all the backup/restore identity needs — no extra BucketAccess grant. + ownerRef: cnpg-artifactapi-backup + versioning: false + tags: + app: artifactapi + purpose: cnpg-backup + # Never drop the backups if the CR is removed; retire buckets by hand. + retainOnDelete: true +--- +# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's +# spec.backup.barmanObjectStore; this schedules the periodic full backup that +# WAL is layered on top of. Schedules are staggered across clusters so the 8 +# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first). +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: cnpg-artifactapi-nightly + namespace: artifactapi +spec: + schedule: "0 40 1 * * *" + immediate: false + backupOwnerReference: self + method: barmanObjectStore + cluster: + name: postgres diff --git a/apps/base/artifactapi/cnpg_cluster.yaml b/apps/base/artifactapi/cnpg_cluster.yaml index f2bbd70..08c8323 100644 --- a/apps/base/artifactapi/cnpg_cluster.yaml +++ b/apps/base/artifactapi/cnpg_cluster.yaml @@ -7,6 +7,35 @@ metadata: spec: affinity: podAntiAffinityType: preferred + backup: + # 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG + # against the object store on each successful base backup. + retentionPolicy: 30d + barmanObjectStore: + # Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it). + destinationPath: s3://cnpg-artifactapi + endpointURL: https://s3.ceph.unkin.net + # radosgw serves a Vault-PKI cert; trust the internal CA (reflected into + # every namespace as the vault-ca-cert Secret). + endpointCA: + name: vault-ca-cert + key: ca.crt + # Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded. + s3Credentials: + accessKeyId: + name: cnpg-artifactapi-backup-s3 + key: AWS_ACCESS_KEY_ID + secretAccessKey: + name: cnpg-artifactapi-backup-s3 + key: AWS_SECRET_ACCESS_KEY + # Path prefix within the bucket; keep stable across restores (see docs). + serverName: artifactapi + data: + compression: bzip2 + jobs: 2 + wal: + compression: zstd + maxParallel: 2 bootstrap: initdb: database: artifacts diff --git a/apps/base/artifactapi/kustomization.yaml b/apps/base/artifactapi/kustomization.yaml index e7a8b79..e338494 100644 --- a/apps/base/artifactapi/kustomization.yaml +++ b/apps/base/artifactapi/kustomization.yaml @@ -7,6 +7,7 @@ resources: - api-hpa.yaml - configmap.yaml - cnpg_cluster.yaml + - cnpg_backup.yaml - cnpg_pooler.yaml - gateway.yaml - httproute.yaml diff --git a/apps/base/authentik/cnpg_backup.yaml b/apps/base/authentik/cnpg_backup.yaml new file mode 100644 index 0000000..243ec38 --- /dev/null +++ b/apps/base/authentik/cnpg_backup.yaml @@ -0,0 +1,55 @@ +--- +# Ceph RGW (S3) backup target for the authentik CNPG cluster, provisioned by the +# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster: +# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from +# its own namespace, so backups are per-database rather than one shared bucket. +apiVersion: ceph.unkin.net/v1alpha1 +kind: ObjectStoreUser +metadata: + name: cnpg-authentik-backup + namespace: authentik +spec: + displayName: "CNPG backup owner (authentik)" + # RGW users are global; keep the uid namespace-qualified so it never collides. + uid: cnpg-authentik-backup + maxBuckets: 5 + # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID, + # S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it. + secretName: cnpg-authentik-backup-s3 + # Keep the RGW user (and thus the keys) if this CR is ever deleted, so an + # in-flight restore can still reach the archive. + retainOnDelete: true +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: Bucket +metadata: + name: cnpg-authentik + namespace: authentik +spec: + bucketName: cnpg-authentik + # The owner user has full control of its own bucket (read + write), which is + # all the backup/restore identity needs — no extra BucketAccess grant. + ownerRef: cnpg-authentik-backup + versioning: false + tags: + app: authentik + purpose: cnpg-backup + # Never drop the backups if the CR is removed; retire buckets by hand. + retainOnDelete: true +--- +# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's +# spec.backup.barmanObjectStore; this schedules the periodic full backup that +# WAL is layered on top of. Schedules are staggered across clusters so the 8 +# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first). +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: cnpg-authentik-nightly + namespace: authentik +spec: + schedule: "0 0 1 * * *" + immediate: false + backupOwnerReference: self + method: barmanObjectStore + cluster: + name: postgres diff --git a/apps/base/authentik/cnpg_cluster.yaml b/apps/base/authentik/cnpg_cluster.yaml index 103cfe6..350c55a 100644 --- a/apps/base/authentik/cnpg_cluster.yaml +++ b/apps/base/authentik/cnpg_cluster.yaml @@ -7,6 +7,35 @@ metadata: spec: affinity: podAntiAffinityType: preferred + backup: + # 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG + # against the object store on each successful base backup. + retentionPolicy: 30d + barmanObjectStore: + # Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it). + destinationPath: s3://cnpg-authentik + endpointURL: https://s3.ceph.unkin.net + # radosgw serves a Vault-PKI cert; trust the internal CA (reflected into + # every namespace as the vault-ca-cert Secret). + endpointCA: + name: vault-ca-cert + key: ca.crt + # Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded. + s3Credentials: + accessKeyId: + name: cnpg-authentik-backup-s3 + key: AWS_ACCESS_KEY_ID + secretAccessKey: + name: cnpg-authentik-backup-s3 + key: AWS_SECRET_ACCESS_KEY + # Path prefix within the bucket; keep stable across restores (see docs). + serverName: authentik + data: + compression: bzip2 + jobs: 2 + wal: + compression: zstd + maxParallel: 2 bootstrap: initdb: database: authentik diff --git a/apps/base/authentik/kustomization.yaml b/apps/base/authentik/kustomization.yaml index 0ba3537..173bd5e 100644 --- a/apps/base/authentik/kustomization.yaml +++ b/apps/base/authentik/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - cnpg_cluster.yaml + - cnpg_backup.yaml - cnpg_pooler.yaml - gateway.yaml - httproute.yaml diff --git a/apps/base/encapi/cnpg_backup.yaml b/apps/base/encapi/cnpg_backup.yaml new file mode 100644 index 0000000..630a7f6 --- /dev/null +++ b/apps/base/encapi/cnpg_backup.yaml @@ -0,0 +1,55 @@ +--- +# Ceph RGW (S3) backup target for the encapi CNPG cluster, provisioned by the +# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster: +# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from +# its own namespace, so backups are per-database rather than one shared bucket. +apiVersion: ceph.unkin.net/v1alpha1 +kind: ObjectStoreUser +metadata: + name: cnpg-encapi-backup + namespace: encapi +spec: + displayName: "CNPG backup owner (encapi)" + # RGW users are global; keep the uid namespace-qualified so it never collides. + uid: cnpg-encapi-backup + maxBuckets: 5 + # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID, + # S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it. + secretName: cnpg-encapi-backup-s3 + # Keep the RGW user (and thus the keys) if this CR is ever deleted, so an + # in-flight restore can still reach the archive. + retainOnDelete: true +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: Bucket +metadata: + name: cnpg-encapi + namespace: encapi +spec: + bucketName: cnpg-encapi + # The owner user has full control of its own bucket (read + write), which is + # all the backup/restore identity needs — no extra BucketAccess grant. + ownerRef: cnpg-encapi-backup + versioning: false + tags: + app: encapi + purpose: cnpg-backup + # Never drop the backups if the CR is removed; retire buckets by hand. + retainOnDelete: true +--- +# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's +# spec.backup.barmanObjectStore; this schedules the periodic full backup that +# WAL is layered on top of. Schedules are staggered across clusters so the 8 +# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first). +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: cnpg-encapi-nightly + namespace: encapi +spec: + schedule: "0 40 2 * * *" + immediate: false + backupOwnerReference: self + method: barmanObjectStore + cluster: + name: postgres diff --git a/apps/base/encapi/cnpg_cluster.yaml b/apps/base/encapi/cnpg_cluster.yaml index d96ee53..07dc3bf 100644 --- a/apps/base/encapi/cnpg_cluster.yaml +++ b/apps/base/encapi/cnpg_cluster.yaml @@ -7,6 +7,35 @@ metadata: spec: affinity: podAntiAffinityType: preferred + backup: + # 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG + # against the object store on each successful base backup. + retentionPolicy: 30d + barmanObjectStore: + # Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it). + destinationPath: s3://cnpg-encapi + endpointURL: https://s3.ceph.unkin.net + # radosgw serves a Vault-PKI cert; trust the internal CA (reflected into + # every namespace as the vault-ca-cert Secret). + endpointCA: + name: vault-ca-cert + key: ca.crt + # Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded. + s3Credentials: + accessKeyId: + name: cnpg-encapi-backup-s3 + key: AWS_ACCESS_KEY_ID + secretAccessKey: + name: cnpg-encapi-backup-s3 + key: AWS_SECRET_ACCESS_KEY + # Path prefix within the bucket; keep stable across restores (see docs). + serverName: encapi + data: + compression: bzip2 + jobs: 2 + wal: + compression: zstd + maxParallel: 2 bootstrap: initdb: database: encapi diff --git a/apps/base/encapi/kustomization.yaml b/apps/base/encapi/kustomization.yaml index a307335..ffda4df 100644 --- a/apps/base/encapi/kustomization.yaml +++ b/apps/base/encapi/kustomization.yaml @@ -10,6 +10,7 @@ resources: - gateway.yaml - httproute.yaml - cnpg_cluster.yaml + - cnpg_backup.yaml - cnpg_pooler.yaml - vaultauth.yaml - vaultstaticsecret.yaml diff --git a/apps/base/grafana/cnpg_backup.yaml b/apps/base/grafana/cnpg_backup.yaml new file mode 100644 index 0000000..cdf5f7b --- /dev/null +++ b/apps/base/grafana/cnpg_backup.yaml @@ -0,0 +1,55 @@ +--- +# Ceph RGW (S3) backup target for the grafana CNPG cluster, provisioned by the +# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster: +# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from +# its own namespace, so backups are per-database rather than one shared bucket. +apiVersion: ceph.unkin.net/v1alpha1 +kind: ObjectStoreUser +metadata: + name: cnpg-grafana-backup + namespace: grafana +spec: + displayName: "CNPG backup owner (grafana)" + # RGW users are global; keep the uid namespace-qualified so it never collides. + uid: cnpg-grafana-backup + maxBuckets: 5 + # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID, + # S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it. + secretName: cnpg-grafana-backup-s3 + # Keep the RGW user (and thus the keys) if this CR is ever deleted, so an + # in-flight restore can still reach the archive. + retainOnDelete: true +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: Bucket +metadata: + name: cnpg-grafana + namespace: grafana +spec: + bucketName: cnpg-grafana + # The owner user has full control of its own bucket (read + write), which is + # all the backup/restore identity needs — no extra BucketAccess grant. + ownerRef: cnpg-grafana-backup + versioning: false + tags: + app: grafana + purpose: cnpg-backup + # Never drop the backups if the CR is removed; retire buckets by hand. + retainOnDelete: true +--- +# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's +# spec.backup.barmanObjectStore; this schedules the periodic full backup that +# WAL is layered on top of. Schedules are staggered across clusters so the 8 +# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first). +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: cnpg-grafana-nightly + namespace: grafana +spec: + schedule: "0 20 3 * * *" + immediate: false + backupOwnerReference: self + method: barmanObjectStore + cluster: + name: postgres diff --git a/apps/base/grafana/cnpg_cluster.yaml b/apps/base/grafana/cnpg_cluster.yaml index 3e7201b..f6e1fc7 100644 --- a/apps/base/grafana/cnpg_cluster.yaml +++ b/apps/base/grafana/cnpg_cluster.yaml @@ -7,6 +7,35 @@ metadata: spec: affinity: podAntiAffinityType: preferred + backup: + # 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG + # against the object store on each successful base backup. + retentionPolicy: 30d + barmanObjectStore: + # Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it). + destinationPath: s3://cnpg-grafana + endpointURL: https://s3.ceph.unkin.net + # radosgw serves a Vault-PKI cert; trust the internal CA (reflected into + # every namespace as the vault-ca-cert Secret). + endpointCA: + name: vault-ca-cert + key: ca.crt + # Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded. + s3Credentials: + accessKeyId: + name: cnpg-grafana-backup-s3 + key: AWS_ACCESS_KEY_ID + secretAccessKey: + name: cnpg-grafana-backup-s3 + key: AWS_SECRET_ACCESS_KEY + # Path prefix within the bucket; keep stable across restores (see docs). + serverName: grafana + data: + compression: bzip2 + jobs: 2 + wal: + compression: zstd + maxParallel: 2 bootstrap: initdb: database: grafana diff --git a/apps/base/grafana/kustomization.yaml b/apps/base/grafana/kustomization.yaml index 359a736..08f0a2f 100644 --- a/apps/base/grafana/kustomization.yaml +++ b/apps/base/grafana/kustomization.yaml @@ -5,6 +5,7 @@ kind: Kustomization resources: - namespace.yaml - cnpg_cluster.yaml + - cnpg_backup.yaml - cnpg_pooler.yaml - vaultauth.yaml - vaultstaticsecret.yaml diff --git a/apps/base/litellm/cnpg_backup.yaml b/apps/base/litellm/cnpg_backup.yaml new file mode 100644 index 0000000..4da6a3f --- /dev/null +++ b/apps/base/litellm/cnpg_backup.yaml @@ -0,0 +1,55 @@ +--- +# Ceph RGW (S3) backup target for the litellm CNPG cluster, provisioned by the +# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster: +# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from +# its own namespace, so backups are per-database rather than one shared bucket. +apiVersion: ceph.unkin.net/v1alpha1 +kind: ObjectStoreUser +metadata: + name: cnpg-litellm-backup + namespace: litellm +spec: + displayName: "CNPG backup owner (litellm)" + # RGW users are global; keep the uid namespace-qualified so it never collides. + uid: cnpg-litellm-backup + maxBuckets: 5 + # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID, + # S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it. + secretName: cnpg-litellm-backup-s3 + # Keep the RGW user (and thus the keys) if this CR is ever deleted, so an + # in-flight restore can still reach the archive. + retainOnDelete: true +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: Bucket +metadata: + name: cnpg-litellm + namespace: litellm +spec: + bucketName: cnpg-litellm + # The owner user has full control of its own bucket (read + write), which is + # all the backup/restore identity needs — no extra BucketAccess grant. + ownerRef: cnpg-litellm-backup + versioning: false + tags: + app: litellm + purpose: cnpg-backup + # Never drop the backups if the CR is removed; retire buckets by hand. + retainOnDelete: true +--- +# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's +# spec.backup.barmanObjectStore; this schedules the periodic full backup that +# WAL is layered on top of. Schedules are staggered across clusters so the 8 +# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first). +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: cnpg-litellm-nightly + namespace: litellm +spec: + schedule: "0 20 1 * * *" + immediate: false + backupOwnerReference: self + method: barmanObjectStore + cluster: + name: litellm-postgres diff --git a/apps/base/litellm/cnpg_cluster.yaml b/apps/base/litellm/cnpg_cluster.yaml index 4dff798..1148308 100644 --- a/apps/base/litellm/cnpg_cluster.yaml +++ b/apps/base/litellm/cnpg_cluster.yaml @@ -7,6 +7,35 @@ metadata: spec: affinity: podAntiAffinityType: preferred + backup: + # 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG + # against the object store on each successful base backup. + retentionPolicy: 30d + barmanObjectStore: + # Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it). + destinationPath: s3://cnpg-litellm + endpointURL: https://s3.ceph.unkin.net + # radosgw serves a Vault-PKI cert; trust the internal CA (reflected into + # every namespace as the vault-ca-cert Secret). + endpointCA: + name: vault-ca-cert + key: ca.crt + # Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded. + s3Credentials: + accessKeyId: + name: cnpg-litellm-backup-s3 + key: AWS_ACCESS_KEY_ID + secretAccessKey: + name: cnpg-litellm-backup-s3 + key: AWS_SECRET_ACCESS_KEY + # Path prefix within the bucket; keep stable across restores (see docs). + serverName: litellm + data: + compression: bzip2 + jobs: 2 + wal: + compression: zstd + maxParallel: 2 bootstrap: initdb: database: litellm diff --git a/apps/base/litellm/kustomization.yaml b/apps/base/litellm/kustomization.yaml index c73e836..32f4171 100644 --- a/apps/base/litellm/kustomization.yaml +++ b/apps/base/litellm/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - cnpg_cluster.yaml + - cnpg_backup.yaml - cnpg_pooler.yaml - deployment.yaml - hpa.yaml diff --git a/apps/base/paperclip/cnpg_backup.yaml b/apps/base/paperclip/cnpg_backup.yaml new file mode 100644 index 0000000..75f6fb7 --- /dev/null +++ b/apps/base/paperclip/cnpg_backup.yaml @@ -0,0 +1,55 @@ +--- +# Ceph RGW (S3) backup target for the paperclip CNPG cluster, provisioned by the +# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster: +# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from +# its own namespace, so backups are per-database rather than one shared bucket. +apiVersion: ceph.unkin.net/v1alpha1 +kind: ObjectStoreUser +metadata: + name: cnpg-paperclip-backup + namespace: paperclip +spec: + displayName: "CNPG backup owner (paperclip)" + # RGW users are global; keep the uid namespace-qualified so it never collides. + uid: cnpg-paperclip-backup + maxBuckets: 5 + # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID, + # S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it. + secretName: cnpg-paperclip-backup-s3 + # Keep the RGW user (and thus the keys) if this CR is ever deleted, so an + # in-flight restore can still reach the archive. + retainOnDelete: true +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: Bucket +metadata: + name: cnpg-paperclip + namespace: paperclip +spec: + bucketName: cnpg-paperclip + # The owner user has full control of its own bucket (read + write), which is + # all the backup/restore identity needs — no extra BucketAccess grant. + ownerRef: cnpg-paperclip-backup + versioning: false + tags: + app: paperclip + purpose: cnpg-backup + # Never drop the backups if the CR is removed; retire buckets by hand. + retainOnDelete: true +--- +# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's +# spec.backup.barmanObjectStore; this schedules the periodic full backup that +# WAL is layered on top of. Schedules are staggered across clusters so the 8 +# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first). +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: cnpg-paperclip-nightly + namespace: paperclip +spec: + schedule: "0 0 3 * * *" + immediate: false + backupOwnerReference: self + method: barmanObjectStore + cluster: + name: paperclip-postgres diff --git a/apps/base/paperclip/cnpg_cluster.yaml b/apps/base/paperclip/cnpg_cluster.yaml index 01ab1e7..0703eae 100644 --- a/apps/base/paperclip/cnpg_cluster.yaml +++ b/apps/base/paperclip/cnpg_cluster.yaml @@ -7,6 +7,35 @@ metadata: spec: affinity: podAntiAffinityType: preferred + backup: + # 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG + # against the object store on each successful base backup. + retentionPolicy: 30d + barmanObjectStore: + # Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it). + destinationPath: s3://cnpg-paperclip + endpointURL: https://s3.ceph.unkin.net + # radosgw serves a Vault-PKI cert; trust the internal CA (reflected into + # every namespace as the vault-ca-cert Secret). + endpointCA: + name: vault-ca-cert + key: ca.crt + # Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded. + s3Credentials: + accessKeyId: + name: cnpg-paperclip-backup-s3 + key: AWS_ACCESS_KEY_ID + secretAccessKey: + name: cnpg-paperclip-backup-s3 + key: AWS_SECRET_ACCESS_KEY + # Path prefix within the bucket; keep stable across restores (see docs). + serverName: paperclip + data: + compression: bzip2 + jobs: 2 + wal: + compression: zstd + maxParallel: 2 bootstrap: initdb: database: paperclip diff --git a/apps/base/paperclip/kustomization.yaml b/apps/base/paperclip/kustomization.yaml index d03f2a5..c37e54f 100644 --- a/apps/base/paperclip/kustomization.yaml +++ b/apps/base/paperclip/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - cnpg_cluster.yaml + - cnpg_backup.yaml - cnpg_pooler.yaml - deployment.yaml - gateway.yaml diff --git a/apps/base/puppet/cnpg_backup.yaml b/apps/base/puppet/cnpg_backup.yaml new file mode 100644 index 0000000..b1c4384 --- /dev/null +++ b/apps/base/puppet/cnpg_backup.yaml @@ -0,0 +1,55 @@ +--- +# Ceph RGW (S3) backup target for the puppet CNPG cluster, provisioned by the +# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster: +# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from +# its own namespace, so backups are per-database rather than one shared bucket. +apiVersion: ceph.unkin.net/v1alpha1 +kind: ObjectStoreUser +metadata: + name: cnpg-puppet-backup + namespace: puppet +spec: + displayName: "CNPG backup owner (puppet)" + # RGW users are global; keep the uid namespace-qualified so it never collides. + uid: cnpg-puppet-backup + maxBuckets: 5 + # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID, + # S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it. + secretName: cnpg-puppet-backup-s3 + # Keep the RGW user (and thus the keys) if this CR is ever deleted, so an + # in-flight restore can still reach the archive. + retainOnDelete: true +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: Bucket +metadata: + name: cnpg-puppet + namespace: puppet +spec: + bucketName: cnpg-puppet + # The owner user has full control of its own bucket (read + write), which is + # all the backup/restore identity needs — no extra BucketAccess grant. + ownerRef: cnpg-puppet-backup + versioning: false + tags: + app: puppet + purpose: cnpg-backup + # Never drop the backups if the CR is removed; retire buckets by hand. + retainOnDelete: true +--- +# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's +# spec.backup.barmanObjectStore; this schedules the periodic full backup that +# WAL is layered on top of. Schedules are staggered across clusters so the 8 +# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first). +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: cnpg-puppet-nightly + namespace: puppet +spec: + schedule: "0 20 2 * * *" + immediate: false + backupOwnerReference: self + method: barmanObjectStore + cluster: + name: puppet-postgres diff --git a/apps/base/puppet/cnpg_cluster.yaml b/apps/base/puppet/cnpg_cluster.yaml index cd5afdd..7087ef1 100644 --- a/apps/base/puppet/cnpg_cluster.yaml +++ b/apps/base/puppet/cnpg_cluster.yaml @@ -7,6 +7,35 @@ metadata: spec: affinity: podAntiAffinityType: preferred + backup: + # 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG + # against the object store on each successful base backup. + retentionPolicy: 30d + barmanObjectStore: + # Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it). + destinationPath: s3://cnpg-puppet + endpointURL: https://s3.ceph.unkin.net + # radosgw serves a Vault-PKI cert; trust the internal CA (reflected into + # every namespace as the vault-ca-cert Secret). + endpointCA: + name: vault-ca-cert + key: ca.crt + # Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded. + s3Credentials: + accessKeyId: + name: cnpg-puppet-backup-s3 + key: AWS_ACCESS_KEY_ID + secretAccessKey: + name: cnpg-puppet-backup-s3 + key: AWS_SECRET_ACCESS_KEY + # Path prefix within the bucket; keep stable across restores (see docs). + serverName: puppet + data: + compression: bzip2 + jobs: 2 + wal: + compression: zstd + maxParallel: 2 bootstrap: initdb: database: puppetdb diff --git a/apps/base/puppet/kustomization.yaml b/apps/base/puppet/kustomization.yaml index 69fe8ce..bb889ad 100644 --- a/apps/base/puppet/kustomization.yaml +++ b/apps/base/puppet/kustomization.yaml @@ -5,6 +5,7 @@ kind: Kustomization resources: - namespace.yaml - cnpg_cluster.yaml + - cnpg_backup.yaml - cnpg_pooler.yaml - cronjob_g10k-code.yaml - cronjob_generate-types.yaml diff --git a/apps/base/woodpecker/cnpg_backup.yaml b/apps/base/woodpecker/cnpg_backup.yaml new file mode 100644 index 0000000..4606822 --- /dev/null +++ b/apps/base/woodpecker/cnpg_backup.yaml @@ -0,0 +1,55 @@ +--- +# Ceph RGW (S3) backup target for the woodpecker CNPG cluster, provisioned by the +# in-estate cephrgw-operator. One dedicated bucket + owner user per cluster: +# cephrgw CRs are namespace-scoped and CNPG reads its S3 credential Secret from +# its own namespace, so backups are per-database rather than one shared bucket. +apiVersion: ceph.unkin.net/v1alpha1 +kind: ObjectStoreUser +metadata: + name: cnpg-woodpecker-backup + namespace: woodpecker +spec: + displayName: "CNPG backup owner (woodpecker)" + # RGW users are global; keep the uid namespace-qualified so it never collides. + uid: cnpg-woodpecker-backup + maxBuckets: 5 + # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID, + # S3_ENDPOINT) into this Secret; the Cluster's barmanObjectStore consumes it. + secretName: cnpg-woodpecker-backup-s3 + # Keep the RGW user (and thus the keys) if this CR is ever deleted, so an + # in-flight restore can still reach the archive. + retainOnDelete: true +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: Bucket +metadata: + name: cnpg-woodpecker + namespace: woodpecker +spec: + bucketName: cnpg-woodpecker + # The owner user has full control of its own bucket (read + write), which is + # all the backup/restore identity needs — no extra BucketAccess grant. + ownerRef: cnpg-woodpecker-backup + versioning: false + tags: + app: woodpecker + purpose: cnpg-backup + # Never drop the backups if the CR is removed; retire buckets by hand. + retainOnDelete: true +--- +# Nightly base backup. Continuous WAL archiving is always-on via the Cluster's +# spec.backup.barmanObjectStore; this schedules the periodic full backup that +# WAL is layered on top of. Schedules are staggered across clusters so the 8 +# base backups do not hit RGW at once (CNPG cron is 6-field, seconds first). +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: cnpg-woodpecker-nightly + namespace: woodpecker +spec: + schedule: "0 0 2 * * *" + immediate: false + backupOwnerReference: self + method: barmanObjectStore + cluster: + name: woodpecker-postgres diff --git a/apps/base/woodpecker/cnpg_cluster.yaml b/apps/base/woodpecker/cnpg_cluster.yaml index 254bd7c..b9f11ae 100644 --- a/apps/base/woodpecker/cnpg_cluster.yaml +++ b/apps/base/woodpecker/cnpg_cluster.yaml @@ -7,6 +7,35 @@ metadata: spec: affinity: podAntiAffinityType: preferred + backup: + # 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG + # against the object store on each successful base backup. + retentionPolicy: 30d + barmanObjectStore: + # Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it). + destinationPath: s3://cnpg-woodpecker + endpointURL: https://s3.ceph.unkin.net + # radosgw serves a Vault-PKI cert; trust the internal CA (reflected into + # every namespace as the vault-ca-cert Secret). + endpointCA: + name: vault-ca-cert + key: ca.crt + # Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded. + s3Credentials: + accessKeyId: + name: cnpg-woodpecker-backup-s3 + key: AWS_ACCESS_KEY_ID + secretAccessKey: + name: cnpg-woodpecker-backup-s3 + key: AWS_SECRET_ACCESS_KEY + # Path prefix within the bucket; keep stable across restores (see docs). + serverName: woodpecker + data: + compression: bzip2 + jobs: 2 + wal: + compression: zstd + maxParallel: 2 bootstrap: initdb: database: woodpecker diff --git a/apps/base/woodpecker/kustomization.yaml b/apps/base/woodpecker/kustomization.yaml index 0289803..281b61d 100644 --- a/apps/base/woodpecker/kustomization.yaml +++ b/apps/base/woodpecker/kustomization.yaml @@ -5,6 +5,7 @@ kind: Kustomization resources: - namespace.yaml - cnpg_cluster.yaml + - cnpg_backup.yaml - cnpg_pooler.yaml - serviceaccount_terraform_artifactapi.yaml - serviceaccount_terraform_authentik.yaml diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..51f96d1 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,8 @@ +# argocd-apps docs + +Operational notes for the manifests in this repo. + +| Doc | What it covers | +| --- | --- | +| [cnpg-backups.md](cnpg-backups.md) | How CNPG Postgres backups (WAL archiving + nightly base backups) to Ceph RGW are configured. | +| [cnpg-restore.md](cnpg-restore.md) | Restoring a CNPG cluster: full recovery, point-in-time recovery, cutover, and gotchas. | diff --git a/docs/cnpg-backups.md b/docs/cnpg-backups.md new file mode 100644 index 0000000..d79edd6 --- /dev/null +++ b/docs/cnpg-backups.md @@ -0,0 +1,146 @@ +# CNPG backups + +Every CNPG Postgres cluster in this repo backs up to a dedicated Ceph RGW (S3) +bucket: continuous WAL archiving plus a staggered nightly base backup, 30-day +retention, compressed. Buckets and their S3 keys are provisioned by the in-estate +`cephrgw-operator` — nothing is seeded by hand. Because the operator's CRs are +namespace-scoped and CNPG reads its credential Secret from its own namespace, the +topology is **one bucket per cluster** (`s3://cnpg-`), not one shared bucket. + +The backup mechanism is CNPG's in-tree `barmanObjectStore` (the deployed operator +is v1.28; the Barman Cloud Plugin is not deployed — see the migration note at the +bottom). + +## Where it lives + +Per cluster, two files under `apps/base//`: + +- `cnpg_cluster.yaml` — the `spec.backup` stanza (WAL archiving + retention). +- `cnpg_backup.yaml` — the `ObjectStoreUser` + `Bucket` (RGW provisioning) and the + nightly `ScheduledBackup`. + +## The backup stanza (`spec.backup` in the Cluster) + +```yaml +spec: + backup: + retentionPolicy: 30d # default; adjust per cluster + barmanObjectStore: + destinationPath: s3://cnpg- + endpointURL: https://s3.ceph.unkin.net + endpointCA: # trust the internal Vault PKI CA + name: vault-ca-cert # reflected into every namespace + key: ca.crt + s3Credentials: + accessKeyId: + name: cnpg--backup-s3 # minted by the ObjectStoreUser + key: AWS_ACCESS_KEY_ID + secretAccessKey: + name: cnpg--backup-s3 + key: AWS_SECRET_ACCESS_KEY + serverName: # path prefix inside the bucket + data: + compression: bzip2 # base backup (zstd not supported here) + jobs: 2 + wal: + compression: zstd # WAL segments + maxParallel: 2 +``` + +Setting `spec.backup.barmanObjectStore` turns on **continuous WAL archiving** +immediately (CNPG points `archive_command` at the object store). The nightly +base backup is a separate object: + +## The nightly base backup (`ScheduledBackup`) + +```yaml +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: cnpg--nightly + namespace: +spec: + schedule: "0 0 1 * * *" # 6-field cron, SECONDS first (01:00:00 daily) + immediate: false + backupOwnerReference: self + method: barmanObjectStore + cluster: + name: +``` + +Schedules are staggered so the base backups don't hit RGW at once: + +| App | Cluster | Bucket | Nightly (local) | +| --- | --- | --- | --- | +| authentik | postgres | cnpg-authentik | 01:00 | +| litellm | litellm-postgres | cnpg-litellm | 01:20 | +| artifactapi | postgres | cnpg-artifactapi | 01:40 | +| woodpecker | woodpecker-postgres | cnpg-woodpecker | 02:00 | +| puppet | puppet-postgres | cnpg-puppet | 02:20 | +| encapi | postgres | cnpg-encapi | 02:40 | +| paperclip | paperclip-postgres | cnpg-paperclip | 03:00 | +| grafana | postgres | cnpg-grafana | 03:20 | + +## Where the credentials come from + +The `ObjectStoreUser` in `cnpg_backup.yaml` tells `cephrgw-operator` to mint an RGW +user and write its keys into a Secret; the `Bucket` makes that user the bucket owner +(full read/write on its own bucket). No keys are ever committed. + +```yaml +apiVersion: ceph.unkin.net/v1alpha1 +kind: ObjectStoreUser +metadata: + name: cnpg--backup + namespace: +spec: + uid: cnpg--backup # RGW users are global; keep it unique + secretName: cnpg--backup-s3 # -> AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY + retainOnDelete: true +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: Bucket +metadata: + name: cnpg- + namespace: +spec: + bucketName: cnpg- + ownerRef: cnpg--backup + retainOnDelete: true +``` + +Confirm the operator provisioned everything: + +```bash +kubectl -n get objectstoreuser,bucket +kubectl -n get secret cnpg--backup-s3 \ + -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d; echo +``` + +## Checking backup status + +```bash +# Cluster health + first-recoverability point (WAL archiving working?) +kubectl -n get cluster \ + -o jsonpath='{.status.firstRecoverabilityPoint}{"\n"}' + +# Backups taken so far +kubectl -n get backups.postgresql.cnpg.io + +# With the cnpg kubectl plugin (richer view, shows archiving + last backup) +kubectl cnpg status -n + +# Kick a one-off backup right now (verify the whole path end to end) +kubectl cnpg backup -n + +# Look at what actually landed in the bucket +aws --endpoint-url https://s3.ceph.unkin.net s3 ls s3://cnpg-// +``` + +## Follow-up: Barman Cloud Plugin migration + +CNPG 1.26+ deprecates the in-tree `barmanObjectStore` in favour of the Barman Cloud +Plugin (removal is planned for a future major). The plugin is **not** deployed today, +so this repo stays on the in-tree mechanism, which is fully functional on 1.28. +Migrating means deploying the plugin and moving each cluster's config to an +`ObjectStore` CR + `plugins:` reference — track that as separate work. diff --git a/docs/cnpg-restore.md b/docs/cnpg-restore.md new file mode 100644 index 0000000..226e9fb --- /dev/null +++ b/docs/cnpg-restore.md @@ -0,0 +1,131 @@ +# CNPG restore + +Recovery is always into a **new** Cluster that bootstraps from the object store — +CNPG never restores in place. The source backups live in `s3://cnpg-` under +`serverName: ` (see [cnpg-backups.md](cnpg-backups.md)). Do all of this in the +source cluster's namespace so the `cnpg--backup-s3` Secret and `vault-ca-cert` +are present. + +## (a) Full restore into a new cluster + +Recover the latest available state into a fresh cluster named `postgres-restore`. +The `externalClusters` entry points at the **existing** backup path; `serverName` +under `barmanObjectStore` (the new cluster's own archive target) MUST differ from the +source, or the restored cluster will overwrite the archive it just recovered from. + +```yaml +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: postgres-restore + namespace: +spec: + instances: 3 + imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie # match source + storage: + size: 20Gi + storageClass: cephrbd-fast-delete + bootstrap: + recovery: + source: source-archive # references externalClusters below + + # New archive target — DIFFERENT serverName from the source (avoids collision). + backup: + retentionPolicy: 30d + barmanObjectStore: + destinationPath: s3://cnpg- + endpointURL: https://s3.ceph.unkin.net + endpointCA: {name: vault-ca-cert, key: ca.crt} + s3Credentials: + accessKeyId: {name: cnpg--backup-s3, key: AWS_ACCESS_KEY_ID} + secretAccessKey: {name: cnpg--backup-s3, key: AWS_SECRET_ACCESS_KEY} + serverName: -restored # NOT "" + wal: {compression: zstd} + data: {compression: bzip2} + + externalClusters: + - name: source-archive + barmanObjectStore: + destinationPath: s3://cnpg- + endpointURL: https://s3.ceph.unkin.net + endpointCA: {name: vault-ca-cert, key: ca.crt} + s3Credentials: + accessKeyId: {name: cnpg--backup-s3, key: AWS_ACCESS_KEY_ID} + secretAccessKey: {name: cnpg--backup-s3, key: AWS_SECRET_ACCESS_KEY} + serverName: # the SOURCE archive to read from +``` + +```bash +kubectl apply -f postgres-restore.yaml +kubectl -n get cluster postgres-restore -w # wait for Cluster in healthy state +``` + +## (b) Point-in-time recovery (PITR) + +Same as above, but add `recoveryTarget` to stop replay at a timestamp. WAL is +replayed from the most recent base backup up to `targetTime`. + +```yaml + bootstrap: + recovery: + source: source-archive + recoveryTarget: + # RFC3339 with timezone. Also valid: targetLSN, targetXID, targetName. + targetTime: "2026-07-26 14:30:00.000000+00" +``` + +```bash +# List backups to pick a base that precedes your target time +kubectl -n get backups.postgresql.cnpg.io \ + -o custom-columns=NAME:.metadata.name,START:.status.startedAt,STOP:.status.stoppedAt +``` + +To recover from one **specific** base backup instead of the newest, point the +source at a `Backup` object: + +```yaml + externalClusters: + - name: source-archive + # ...barmanObjectStore as above... + bootstrap: + recovery: + backup: + name: + recoveryTarget: + targetTime: "2026-07-26 14:30:00+00" +``` + +## (c) Verify, then cut over + +```bash +# 1. Sanity-check the recovered data before touching production. +kubectl cnpg psql postgres-restore -n -- -c '\l' +kubectl cnpg psql postgres-restore -n -d -- \ + -c 'select max(id), count(*) from ;' + +# 2. Confirm the restored cluster is archiving to its NEW serverName. +kubectl cnpg status postgres-restore -n +``` + +Cutover = repoint the app at the new cluster. CNPG service names track the Cluster +name (`-rw` / `-ro` / `-r`), so update whatever the app connects through — +the CNPG `Pooler` (`cnpg_pooler.yaml`) `cluster.name`, or the app's DB host env — to +`postgres-restore`, then retire the old cluster. There is no in-place rename; the new +name is the cluster's identity. If you truly need the old name back, restore again +with `metadata.name` set to the original (after deleting the old one). + +## (d) Gotchas + +- **serverName collision.** The new cluster's `spec.backup...serverName` must differ + from the source's, or it re-uses the same path and corrupts/overwrites the source + archive on its first WAL push. Use `-restored` (or similar) as above. +- **Secrets must exist in the target namespace.** `cnpg--backup-s3` and + `vault-ca-cert` are referenced by both `externalClusters` and `backup`. Restoring + into a *different* namespace means recreating (or reflecting) those first — the + `ObjectStoreUser`/`Bucket` CRs are namespace-scoped. +- **Match the image major.** Bootstrap-recovery replays WAL; use the same + `imageName` Postgres major as the source (mismatched majors will refuse to start). +- **PITR base must precede the target.** `targetTime` has to fall after a completed + base backup's start; otherwise there's nothing to replay onto. +- **`recoveryTarget` is one-shot.** It only applies during bootstrap. Once promoted, + the cluster is a normal primary — you can't "re-PITR" it; start a new restore. diff --git a/schemas/ceph.unkin.net/bucket_v1alpha1.json b/schemas/ceph.unkin.net/bucket_v1alpha1.json new file mode 100644 index 0000000..37cdf6f --- /dev/null +++ b/schemas/ceph.unkin.net/bucket_v1alpha1.json @@ -0,0 +1,176 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "apiVersion": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "metadata": { + "type": "object" + }, + "spec": { + "properties": { + "bucketName": { + "type": "string" + }, + "managePolicy": { + "default": true, + "type": "boolean" + }, + "objectLock": { + "properties": { + "days": { + "format": "int32", + "type": "integer" + }, + "enabled": { + "type": "boolean" + }, + "mode": { + "enum": [ + "GOVERNANCE", + "COMPLIANCE" + ], + "type": "string" + }, + "years": { + "format": "int32", + "type": "integer" + } + }, + "required": [ + "enabled" + ], + "type": "object" + }, + "ownerRef": { + "type": "string" + }, + "placementTarget": { + "type": "string" + }, + "purgeOnDelete": { + "type": "boolean" + }, + "quota": { + "properties": { + "enabled": { + "default": true, + "type": "boolean" + }, + "maxObjects": { + "format": "int64", + "type": "integer" + }, + "maxSizeBytes": { + "format": "int64", + "type": "integer" + } + }, + "type": "object" + }, + "retainOnDelete": { + "type": "boolean" + }, + "tags": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "versioning": { + "type": "boolean" + }, + "zonegroup": { + "type": "string" + } + }, + "required": [ + "ownerRef" + ], + "type": "object" + }, + "status": { + "properties": { + "adopted": { + "type": "boolean" + }, + "bucketID": { + "type": "string" + }, + "bucketName": { + "type": "string" + }, + "conditions": { + "items": { + "properties": { + "lastTransitionTime": { + "format": "date-time", + "type": "string" + }, + "message": { + "maxLength": 32768, + "type": "string" + }, + "observedGeneration": { + "format": "int64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "maxLength": 1024, + "minLength": 1, + "pattern": "^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$", + "type": "string" + }, + "status": { + "enum": [ + "True", + "False", + "Unknown" + ], + "type": "string" + }, + "type": { + "maxLength": 316, + "pattern": "^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$", + "type": "string" + } + }, + "required": [ + "lastTransitionTime", + "message", + "reason", + "status", + "type" + ], + "type": "object" + }, + "type": "array", + "x-kubernetes-list-map-keys": [ + "type" + ], + "x-kubernetes-list-type": "map" + }, + "observedGeneration": { + "format": "int64", + "type": "integer" + }, + "owner": { + "type": "string" + }, + "phase": { + "type": "string" + }, + "policyPrincipals": { + "format": "int32", + "type": "integer" + } + }, + "type": "object" + } + }, + "type": "object" +} diff --git a/schemas/ceph.unkin.net/bucketaccess_v1alpha1.json b/schemas/ceph.unkin.net/bucketaccess_v1alpha1.json new file mode 100644 index 0000000..9e783e2 --- /dev/null +++ b/schemas/ceph.unkin.net/bucketaccess_v1alpha1.json @@ -0,0 +1,191 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "apiVersion": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "metadata": { + "type": "object" + }, + "spec": { + "properties": { + "actions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "bucketRef": { + "type": "string" + }, + "conditions": { + "properties": { + "secureTransportOnly": { + "type": "boolean" + }, + "sourceIPs": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "level": { + "enum": [ + "read-only", + "read-write", + "full" + ], + "type": "string" + }, + "paths": { + "items": { + "type": "string" + }, + "type": "array" + }, + "rawStatements": { + "items": { + "properties": { + "actions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "conditions": { + "additionalProperties": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": "object" + }, + "type": "object" + }, + "effect": { + "default": "Allow", + "enum": [ + "Allow", + "Deny" + ], + "type": "string" + }, + "resources": { + "items": { + "type": "string" + }, + "type": "array" + }, + "sid": { + "type": "string" + } + }, + "required": [ + "actions" + ], + "type": "object" + }, + "type": "array" + }, + "retainOnDelete": { + "type": "boolean" + }, + "secretName": { + "type": "string" + }, + "uid": { + "type": "string" + }, + "userRef": { + "type": "string" + } + }, + "required": [ + "bucketRef", + "level" + ], + "type": "object" + }, + "status": { + "properties": { + "bound": { + "type": "boolean" + }, + "conditions": { + "items": { + "properties": { + "lastTransitionTime": { + "format": "date-time", + "type": "string" + }, + "message": { + "maxLength": 32768, + "type": "string" + }, + "observedGeneration": { + "format": "int64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "maxLength": 1024, + "minLength": 1, + "pattern": "^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$", + "type": "string" + }, + "status": { + "enum": [ + "True", + "False", + "Unknown" + ], + "type": "string" + }, + "type": { + "maxLength": 316, + "pattern": "^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$", + "type": "string" + } + }, + "required": [ + "lastTransitionTime", + "message", + "reason", + "status", + "type" + ], + "type": "object" + }, + "type": "array", + "x-kubernetes-list-map-keys": [ + "type" + ], + "x-kubernetes-list-type": "map" + }, + "observedGeneration": { + "format": "int64", + "type": "integer" + }, + "phase": { + "type": "string" + }, + "secretName": { + "type": "string" + }, + "uid": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" +} diff --git a/schemas/ceph.unkin.net/objectstoreuser_v1alpha1.json b/schemas/ceph.unkin.net/objectstoreuser_v1alpha1.json new file mode 100644 index 0000000..3648da2 --- /dev/null +++ b/schemas/ceph.unkin.net/objectstoreuser_v1alpha1.json @@ -0,0 +1,132 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "apiVersion": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "metadata": { + "type": "object" + }, + "spec": { + "properties": { + "displayName": { + "type": "string" + }, + "email": { + "type": "string" + }, + "maxBuckets": { + "default": 1000, + "format": "int32", + "type": "integer" + }, + "quota": { + "properties": { + "enabled": { + "default": true, + "type": "boolean" + }, + "maxObjects": { + "format": "int64", + "type": "integer" + }, + "maxSizeBytes": { + "format": "int64", + "type": "integer" + } + }, + "type": "object" + }, + "retainOnDelete": { + "type": "boolean" + }, + "secretName": { + "type": "string" + }, + "suspended": { + "type": "boolean" + }, + "uid": { + "type": "string" + } + }, + "type": "object" + }, + "status": { + "properties": { + "adopted": { + "type": "boolean" + }, + "conditions": { + "items": { + "properties": { + "lastTransitionTime": { + "format": "date-time", + "type": "string" + }, + "message": { + "maxLength": 32768, + "type": "string" + }, + "observedGeneration": { + "format": "int64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "maxLength": 1024, + "minLength": 1, + "pattern": "^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$", + "type": "string" + }, + "status": { + "enum": [ + "True", + "False", + "Unknown" + ], + "type": "string" + }, + "type": { + "maxLength": 316, + "pattern": "^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$", + "type": "string" + } + }, + "required": [ + "lastTransitionTime", + "message", + "reason", + "status", + "type" + ], + "type": "object" + }, + "type": "array", + "x-kubernetes-list-map-keys": [ + "type" + ], + "x-kubernetes-list-type": "map" + }, + "observedGeneration": { + "format": "int64", + "type": "integer" + }, + "phase": { + "type": "string" + }, + "secretName": { + "type": "string" + }, + "uid": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" +}