From 7d186a9b36e9c923ccee40f8aec1d2dae531b72b Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 19 Sep 2026 13:56:00 +1000 Subject: [PATCH 1/2] Enable hot_standby_feedback on the authentik postgres cluster Slow authentik reads served by a hot standby exhaust max_standby_streaming_delay and are cancelled with "conflict with recovery", surfacing as HTTP 500s. - set hot_standby_feedback on (SIGHUP reload, no restart) --- apps/base/authentik/cnpg_cluster.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/base/authentik/cnpg_cluster.yaml b/apps/base/authentik/cnpg_cluster.yaml index a7490f7..c1dd6f9 100644 --- a/apps/base/authentik/cnpg_cluster.yaml +++ b/apps/base/authentik/cnpg_cluster.yaml @@ -66,6 +66,10 @@ spec: dynamic_shared_memory_type: posix effective_cache_size: 256MB full_page_writes: "on" + # Replicas report their oldest xmin to the primary, so multi-second reads on + # a hot standby stop exhausting max_standby_streaming_delay and being + # cancelled. Retained-dead-tuple cost is negligible on a ~155MB database. + hot_standby_feedback: "on" log_destination: csvlog log_directory: /controller/log log_filename: postgres -- 2.47.3 From e4856d24a4c6d471f6558cd8d1fa7c0c7ec9a5cb Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 19 Sep 2026 13:57:06 +1000 Subject: [PATCH 2/2] Raise authentik postgres CPU, memory and buffer sizing The 500m CPU limit is a 50ms CFS quota per 100ms period, so the pods throttle on every burst even at idle and each query pays the latency. 128MB shared_buffers and a 256MB cache estimate also under-serve the planner on the polymorphic joins authentik issues. - raise resources to requests 500m/1Gi, limits 2/2Gi - raise shared_buffers to 512MB and effective_cache_size to 1536MB - keep the post-incident memory headroom multiple over shared_buffers --- apps/base/authentik/cnpg_cluster.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/apps/base/authentik/cnpg_cluster.yaml b/apps/base/authentik/cnpg_cluster.yaml index c1dd6f9..5dd813d 100644 --- a/apps/base/authentik/cnpg_cluster.yaml +++ b/apps/base/authentik/cnpg_cluster.yaml @@ -64,7 +64,7 @@ spec: archive_mode: "on" archive_timeout: 5min dynamic_shared_memory_type: posix - effective_cache_size: 256MB + effective_cache_size: 1536MB full_page_writes: "on" # Replicas report their oldest xmin to the primary, so multi-second reads on # a hot standby stop exhausting max_standby_streaming_delay and being @@ -81,7 +81,7 @@ spec: max_parallel_workers: "16" max_replication_slots: "16" max_worker_processes: "16" - shared_buffers: 128MB + shared_buffers: 512MB shared_memory_type: mmap ssl_max_protocol_version: TLSv1.3 ssl_min_protocol_version: TLSv1.3 @@ -109,13 +109,16 @@ spec: updateInterval: 30 resources: limits: - cpu: 500m + # 500m is a 50ms CFS quota per 100ms period, exhausted by bursts even at + # ~0.01 cores average, so every query pays throttle latency. + cpu: "2" # 512Mi OOMKilled replicas under load (shared_buffers 128MB + # max_connections 200 leave no headroom) — see incident 2026-07-28. - memory: 1Gi + # shared_buffers 512MB needs the same headroom multiple, hence 2Gi. + memory: 2Gi requests: - cpu: 50m - memory: 512Mi + cpu: 500m + memory: 1Gi smartShutdownTimeout: 180 startDelay: 3600 stopDelay: 1800 -- 2.47.3