From 5ae14b3a38963e176a1518a7b9ed1125b66ad23b Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Tue, 28 Jul 2026 18:16:40 +1000 Subject: [PATCH] Reduce retention to 3 days (NATS + ClickHouse); shrink PVCs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ben: 1.2 TiB is too much. Cut both stores to 3d and resize honestly. - NATS LOGS stream max_age 168h -> 72h in the nats-stream-limits ConfigMap. At the stated ~33 GiB/day compressed (S2), 3d ~= 100 GiB, so max_bytes -> 130 GiB and the file-store PVC -> 180Gi/node. 3 replicas = ~0.5 TiB total (down from 1.2 TiB). - ClickHouse logs.raw TTL 30d -> 3d in the bootstrap DDL. At ~130 GiB/day raw, LZ4/ZSTD (~6x) stores ~20-25 GiB/day => ~60-75 GiB/3d; with merge headroom the CHI PVC -> 150Gi (from 200Gi). logs.raw is the only table. - Long-term retention now lives EXCLUSIVELY in the S3 archive (logarchiver) for the configured subjects; everything else is gone after 3 days — accepted design, documented in the runbook + PR body. - Update all 7d/30d comments and the replay window (now 3d). PVC shrink is a plan-time change: this stack is not deployed yet, so it is clean. Once deployed, PVCs cannot shrink in place (would need recreate). Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv --- apps/base/logging/clickhouseinstallation.yaml | 6 +++++- apps/base/logging/job_clickhouse-schema.yaml | 2 +- apps/base/logging/kustomization.yaml | 8 ++++---- apps/base/logging/nats-bootstrap-job.yaml | 6 +++--- apps/base/logging/vector/aggregator.yaml | 2 +- apps/base/logging/vector/archiver.yaml | 2 +- apps/overlays/au-syd1/logging/values-nats.yaml | 8 ++++---- 7 files changed, 19 insertions(+), 15 deletions(-) diff --git a/apps/base/logging/clickhouseinstallation.yaml b/apps/base/logging/clickhouseinstallation.yaml index 6c82e59..f6b63f0 100644 --- a/apps/base/logging/clickhouseinstallation.yaml +++ b/apps/base/logging/clickhouseinstallation.yaml @@ -46,7 +46,11 @@ spec: - ReadWriteOnce resources: requests: - storage: 200Gi + # 3d TTL on logs.raw. At ~130 GiB/day raw, ClickHouse LZ4/ZSTD + # (~6x on log text) stores ~20-25 GiB/day => ~60-75 GiB/3d, plus + # merge headroom (~2x peak). logs.raw is the only table. 150Gi + # gives comfortable headroom; long-term data lives in S3, not here. + storage: 150Gi serviceTemplates: - name: chi-service generateName: "clickhouse-{chi}" diff --git a/apps/base/logging/job_clickhouse-schema.yaml b/apps/base/logging/job_clickhouse-schema.yaml index 1e0f801..ff36675 100644 --- a/apps/base/logging/job_clickhouse-schema.yaml +++ b/apps/base/logging/job_clickhouse-schema.yaml @@ -86,7 +86,7 @@ spec: ENGINE = MergeTree PARTITION BY toDate(timestamp) ORDER BY (source, namespace, host, timestamp) - TTL toDateTime(timestamp) + INTERVAL 30 DAY + TTL toDateTime(timestamp) + INTERVAL 3 DAY SETTINGS index_granularity = 8192; EOSQL echo "Schema applied." diff --git a/apps/base/logging/kustomization.yaml b/apps/base/logging/kustomization.yaml index 0896d77..4fd721f 100644 --- a/apps/base/logging/kustomization.yaml +++ b/apps/base/logging/kustomization.yaml @@ -21,13 +21,13 @@ configMapGenerator: # renames the ConfigMap, which rewrites the Job's env reference, which changes # the PostSync hook Job's spec and forces Argo to re-run it -> new limits apply. # Sizing assumes ~1500 events/s avg @ ~1 KiB/event with S2 compression (~4x): - # ~33 GiB/day compressed -> ~230 GiB/7d per replica. max_bytes 300 GiB sits - # under the 400Gi/node PVC (see values-nats.yaml). Raising retention beyond the + # ~33 GiB/day compressed -> ~100 GiB/3d per replica. max_bytes 130 GiB sits + # under the 180Gi/node PVC (see values-nats.yaml). Raising retention beyond the # PVC requires bumping BOTH max_bytes here and fileStore PVC size in values. - name: nats-stream-limits literals: - - max_age=168h - - max_bytes=322122547200 + - max_age=72h + - max_bytes=139586437120 - dupe_window=2m - name: vector-agent-config files: diff --git a/apps/base/logging/nats-bootstrap-job.yaml b/apps/base/logging/nats-bootstrap-job.yaml index 9ace885..5e2fc68 100644 --- a/apps/base/logging/nats-bootstrap-job.yaml +++ b/apps/base/logging/nats-bootstrap-job.yaml @@ -5,8 +5,8 @@ # Stream LOGS: file storage, 3 replicas, retention=limits (NOT workqueue) so the # transform tier AND the archiver each independently see every message — reading # never deletes; only max-age/max-bytes do. S2 compression is on (logs compress -# well). Replay window = max-age (7d default). Beyond that, the S3 archive is the -# long-term replay source. +# well). Replay window = max-age (3d default). Beyond that, the S3 archive is the +# ONLY long-term source — everything else is gone after 3 days (accepted design). # # TUNABLE LIMITS LIVE IN A CONFIGMAP (nats-stream-limits): max_age, max_bytes, # dupe_window. Change the ConfigMap and re-sync — this Job re-runs and applies @@ -24,7 +24,7 @@ # --filter). Exact default set is an open decision for Ben. # # Runbook (replay): -# (a) reprocess from JetStream (within max-age, 7d): scale the transform tier +# (a) reprocess from JetStream (within max-age, 3d): scale the transform tier # to 0, then `nats consumer rm LOGS transform` and re-run this Job # (recreates at DeliverAll), or `nats consumer edit`/`--replay` from a # start seq/time. diff --git a/apps/base/logging/vector/aggregator.yaml b/apps/base/logging/vector/aggregator.yaml index 4944ef9..f22b849 100644 --- a/apps/base/logging/vector/aggregator.yaml +++ b/apps/base/logging/vector/aggregator.yaml @@ -7,7 +7,7 @@ # insert a transform and append its id to the clickhouse sink `inputs` — no edge # or VM rollout required. # -# Durability model: JetStream (7d / 300 GiB, S2-compressed) is the SOLE +# Durability model: JetStream (3d / 130 GiB, S2-compressed) is the SOLE # durability layer and the replay window. This # tier is stateless (no PVC, memory buffer). If ClickHouse is down the sink # blocks (when_full=block); back-pressure stops the source pulling, so unpulled diff --git a/apps/base/logging/vector/archiver.yaml b/apps/base/logging/vector/archiver.yaml index 075e34d..60b5102 100644 --- a/apps/base/logging/vector/archiver.yaml +++ b/apps/base/logging/vector/archiver.yaml @@ -4,7 +4,7 @@ # from the ClickHouse transform path (archive lag can never stall ingest — true # fan-out). Writes RAW, pre-transform events (as they sit in JetStream) as # gzipped NDJSON, partitioned by subject + date. This is the long-horizon replay -# source beyond JetStream's 7d retention window. +# source beyond JetStream's 3d retention window. data_dir: /vector-data-dir api: diff --git a/apps/overlays/au-syd1/logging/values-nats.yaml b/apps/overlays/au-syd1/logging/values-nats.yaml index c5443cd..956ebd2 100644 --- a/apps/overlays/au-syd1/logging/values-nats.yaml +++ b/apps/overlays/au-syd1/logging/values-nats.yaml @@ -13,13 +13,13 @@ config: enabled: true fileStore: pvc: - # Sized for 7d retention: ~230 GiB/7d compressed (see + # Sized for 3d retention: ~100 GiB/3d compressed (see # nats-stream-limits ConfigMap) + file-store WAL/index/overhead, kept - # safely above the 300 GiB max_bytes cap. 3 replicas => 1.2 TiB total + # safely above the 130 GiB max_bytes cap. 3 replicas => ~0.5 TiB total # provisioned on cephrbd-fast-delete. NB: this is the honest number for # the assumed ~1500 events/s; higher real volume needs a bigger PVC + - # max_bytes together, else discard=old truncates retention below 7d. - size: 400Gi + # max_bytes together, else discard=old truncates retention below 3d. + size: 180Gi storageClassName: cephrbd-fast-delete # Per-user auth with publish/subscribe separation. Passwords are injected as # env vars from the Vault-synced nats-auth Secret (NATS expands $VAR in config).