unkinben
e0eeeb6b04
## Why The CNPG buckets are empty after PR #298. Live diagnosis found **two** independent causes; this PR fixes one of them. `litellm`, `puppet` and `paperclip` run the CNPG `17-minimal-trixie` image, which omits the `barman-cloud` CLI. In-tree `barmanObjectStore` archiving shells out to `barman-cloud-wal-archive`, so their archiver dies immediately: ``` ContinuousArchiving=False :: unexpected failure invoking barman-cloud-wal-archive: exec: "barman-cloud-wal-archive": executable file not found in $PATH ``` (verified on the live `puppet-postgres` primary: `which barman-cloud-wal-archive` → not found; on a `-system` cluster it resolves to `/usr/local/bin/barman-cloud-wal-archive`). ## How Switch those three clusters from `17-minimal-trixie` to `17-system-trixie` — the `-system` variant already used by the other six clusters, which bundles the barman-cloud tools. Tag confirmed present upstream (ghcr manifest HTTP 200). ``` - imageName: ghcr.io/cloudnative-pg/postgresql:17-minimal-trixie + imageName: ghcr.io/cloudnative-pg/postgresql:17-system-trixie ``` CNPG applies this as a rolling image update (switchover, no data change). ## Not fixed here (separate, primary blocker) The other six clusters (full image, barman present) fail with a **TLS trust** error — the reflected `vault-ca-cert` bundle carries a **stale intermediate CA**, so barman can't verify `s3.ceph.unkin.net`: ``` SSL: CERTIFICATE_VERIFY_FAILED ... certificate signature failure ``` That is a shared trust-anchor refresh (likely owned by terraform-k8s / `config/certificates/secret.yaml`, `managed-by: terragrunt`), handled separately — it also gates litellm/puppet once they have barman. See the investigation report. ## Validation - `kustomize build --enable-helm` + `kubeconform` pass on `litellm`, `puppet` overlays and the `paperclip` base (paperclip has no overlay yet). - `pre-commit run` passes on all changed files. ## Follow-ups - Longer term, the Barman Cloud Plugin (sidecar) would let minimal images keep their size while still archiving — track with the plugin migration. https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv Reviewed-on: #304 Co-authored-by: Ben Vincent <ben@unkin.net> Co-committed-by: Ben Vincent <ben@unkin.net>
121 lines
3.2 KiB
YAML
121 lines
3.2 KiB
YAML
---
|
|
apiVersion: postgresql.cnpg.io/v1
|
|
kind: Cluster
|
|
metadata:
|
|
name: litellm-postgres
|
|
namespace: litellm
|
|
spec:
|
|
affinity:
|
|
podAntiAffinityType: preferred
|
|
backup:
|
|
# 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG
|
|
# against the object store on each successful base backup.
|
|
retentionPolicy: 30d
|
|
barmanObjectStore:
|
|
# Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it).
|
|
destinationPath: s3://cnpg-litellm
|
|
endpointURL: https://s3.ceph.unkin.net
|
|
# radosgw serves a Vault-PKI cert; trust the internal CA (reflected into
|
|
# every namespace as the vault-ca-cert Secret).
|
|
endpointCA:
|
|
name: vault-ca-cert
|
|
key: ca.crt
|
|
# Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded.
|
|
s3Credentials:
|
|
accessKeyId:
|
|
name: cnpg-litellm-backup-s3
|
|
key: AWS_ACCESS_KEY_ID
|
|
secretAccessKey:
|
|
name: cnpg-litellm-backup-s3
|
|
key: AWS_SECRET_ACCESS_KEY
|
|
# Path prefix within the bucket; keep stable across restores (see docs).
|
|
serverName: litellm
|
|
data:
|
|
compression: bzip2
|
|
jobs: 2
|
|
wal:
|
|
compression: zstd
|
|
maxParallel: 2
|
|
bootstrap:
|
|
initdb:
|
|
database: litellm
|
|
encoding: UTF8
|
|
localeCType: C
|
|
localeCollate: C
|
|
owner: litellm
|
|
secret:
|
|
name: postgres-credentials
|
|
enablePDB: true
|
|
enableSuperuserAccess: false
|
|
failoverDelay: 0
|
|
imageName: ghcr.io/cloudnative-pg/postgresql:17-system-trixie
|
|
instances: 3
|
|
logLevel: info
|
|
maxSyncReplicas: 0
|
|
minSyncReplicas: 0
|
|
monitoring:
|
|
customQueriesConfigMap:
|
|
- key: queries
|
|
name: cnpg-default-monitoring
|
|
disableDefaultQueries: false
|
|
enablePodMonitor: false
|
|
postgresql:
|
|
parameters:
|
|
archive_mode: "on"
|
|
archive_timeout: 5min
|
|
dynamic_shared_memory_type: posix
|
|
effective_cache_size: 256MB
|
|
full_page_writes: "on"
|
|
log_destination: csvlog
|
|
log_directory: /controller/log
|
|
log_filename: postgres
|
|
log_rotation_age: "0"
|
|
log_rotation_size: "0"
|
|
log_truncate_on_rotation: "false"
|
|
logging_collector: "on"
|
|
max_connections: "200"
|
|
max_parallel_workers: "16"
|
|
max_replication_slots: "16"
|
|
max_worker_processes: "16"
|
|
shared_buffers: 128MB
|
|
shared_memory_type: mmap
|
|
ssl_max_protocol_version: TLSv1.3
|
|
ssl_min_protocol_version: TLSv1.3
|
|
wal_keep_size: 256MB
|
|
wal_level: logical
|
|
wal_log_hints: "on"
|
|
wal_receiver_timeout: 5s
|
|
wal_sender_timeout: 5s
|
|
syncReplicaElectionConstraint:
|
|
enabled: false
|
|
primaryUpdateMethod: restart
|
|
primaryUpdateStrategy: unsupervised
|
|
probes:
|
|
liveness:
|
|
isolationCheck:
|
|
connectionTimeout: 1000
|
|
enabled: true
|
|
requestTimeout: 1000
|
|
replicationSlots:
|
|
highAvailability:
|
|
enabled: true
|
|
slotPrefix: _cnpg_
|
|
synchronizeReplicas:
|
|
enabled: true
|
|
updateInterval: 30
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 1Gi
|
|
requests:
|
|
cpu: 50m
|
|
memory: 512Mi
|
|
smartShutdownTimeout: 180
|
|
startDelay: 3600
|
|
stopDelay: 1800
|
|
storage:
|
|
resizeInUseVolumes: true
|
|
size: 10Gi
|
|
storageClass: cephrbd-fast-delete
|
|
switchoverDelay: 3600
|