unkinben
e6f2cbc363
Authentik's intermittent API 500s (which failed two terraform-authentik CI runs today) were traced to its CNPG postgres replicas being OOMKilled: 512Mi limits leave no headroom over shared_buffers 128MB + max_connections 200, both OOM events matched the 500 bursts to the second, and the session-pinned RO pooler turns each replica death into a batch of severed read connections. The primary is at 84% of its limit and is next. - raise the authentik CNPG memory limit from 512Mi to 1Gi and request from 256Mi to 512Mi Follow-up candidates (not in this PR): RO pooler poolMode session→transaction to shrink the blast radius of a replica loss; revisit max_connections/shared_buffers sizing. Reviewed-on: #300 Co-authored-by: Ben Vincent <ben@unkin.net> Co-committed-by: Ben Vincent <ben@unkin.net>
123 lines
3.4 KiB
YAML
123 lines
3.4 KiB
YAML
---
|
|
apiVersion: postgresql.cnpg.io/v1
|
|
kind: Cluster
|
|
metadata:
|
|
name: postgres
|
|
namespace: authentik
|
|
spec:
|
|
affinity:
|
|
podAntiAffinityType: preferred
|
|
backup:
|
|
# 30-day retention (DEFAULT — adjust per cluster if needed). Enforced by CNPG
|
|
# against the object store on each successful base backup.
|
|
retentionPolicy: 30d
|
|
barmanObjectStore:
|
|
# Dedicated per-cluster Ceph RGW bucket (cephrgw-operator provisions it).
|
|
destinationPath: s3://cnpg-authentik
|
|
endpointURL: https://s3.ceph.unkin.net
|
|
# radosgw serves a Vault-PKI cert; trust the internal CA (reflected into
|
|
# every namespace as the vault-ca-cert Secret).
|
|
endpointCA:
|
|
name: vault-ca-cert
|
|
key: ca.crt
|
|
# Keys minted by the ObjectStoreUser in cnpg_backup.yaml; never hardcoded.
|
|
s3Credentials:
|
|
accessKeyId:
|
|
name: cnpg-authentik-backup-s3
|
|
key: AWS_ACCESS_KEY_ID
|
|
secretAccessKey:
|
|
name: cnpg-authentik-backup-s3
|
|
key: AWS_SECRET_ACCESS_KEY
|
|
# Path prefix within the bucket; keep stable across restores (see docs).
|
|
serverName: authentik
|
|
data:
|
|
compression: bzip2
|
|
jobs: 2
|
|
wal:
|
|
compression: zstd
|
|
maxParallel: 2
|
|
bootstrap:
|
|
initdb:
|
|
database: authentik
|
|
encoding: UTF8
|
|
localeCType: C
|
|
localeCollate: C
|
|
owner: authentik
|
|
secret:
|
|
name: postgres-credentials
|
|
enablePDB: true
|
|
enableSuperuserAccess: false
|
|
failoverDelay: 0
|
|
imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie
|
|
instances: 3
|
|
logLevel: info
|
|
maxSyncReplicas: 0
|
|
minSyncReplicas: 0
|
|
monitoring:
|
|
customQueriesConfigMap:
|
|
- key: queries
|
|
name: cnpg-default-monitoring
|
|
disableDefaultQueries: false
|
|
enablePodMonitor: false
|
|
postgresql:
|
|
parameters:
|
|
archive_mode: "on"
|
|
archive_timeout: 5min
|
|
dynamic_shared_memory_type: posix
|
|
effective_cache_size: 256MB
|
|
full_page_writes: "on"
|
|
log_destination: csvlog
|
|
log_directory: /controller/log
|
|
log_filename: postgres
|
|
log_rotation_age: "0"
|
|
log_rotation_size: "0"
|
|
log_truncate_on_rotation: "false"
|
|
logging_collector: "on"
|
|
max_connections: "200"
|
|
max_parallel_workers: "16"
|
|
max_replication_slots: "16"
|
|
max_worker_processes: "16"
|
|
shared_buffers: 128MB
|
|
shared_memory_type: mmap
|
|
ssl_max_protocol_version: TLSv1.3
|
|
ssl_min_protocol_version: TLSv1.3
|
|
wal_keep_size: 256MB
|
|
wal_level: logical
|
|
wal_log_hints: "on"
|
|
wal_receiver_timeout: 5s
|
|
wal_sender_timeout: 5s
|
|
syncReplicaElectionConstraint:
|
|
enabled: false
|
|
primaryUpdateMethod: restart
|
|
primaryUpdateStrategy: unsupervised
|
|
probes:
|
|
liveness:
|
|
isolationCheck:
|
|
connectionTimeout: 1000
|
|
enabled: true
|
|
requestTimeout: 1000
|
|
replicationSlots:
|
|
highAvailability:
|
|
enabled: true
|
|
slotPrefix: _cnpg_
|
|
synchronizeReplicas:
|
|
enabled: true
|
|
updateInterval: 30
|
|
resources:
|
|
limits:
|
|
cpu: 500m
|
|
# 512Mi OOMKilled replicas under load (shared_buffers 128MB +
|
|
# max_connections 200 leave no headroom) — see incident 2026-07-28.
|
|
memory: 1Gi
|
|
requests:
|
|
cpu: 50m
|
|
memory: 512Mi
|
|
smartShutdownTimeout: 180
|
|
startDelay: 3600
|
|
stopDelay: 1800
|
|
storage:
|
|
resizeInUseVolumes: true
|
|
size: 20Gi
|
|
storageClass: cephrbd-fast-delete
|
|
switchoverDelay: 3600
|