diff --git a/apps/base/logging/kustomization.yaml b/apps/base/logging/kustomization.yaml index cf94855..0896d77 100644 --- a/apps/base/logging/kustomization.yaml +++ b/apps/base/logging/kustomization.yaml @@ -16,6 +16,19 @@ resources: # Vector pipelines are the single source of truth (also validated by # `vector test` in CI). Mounted into each tier via `existingConfigMaps`. configMapGenerator: + # Tunable JetStream stream limits (the nats-bootstrap Job reads these and does + # create-or-update). Hash suffix is INTENTIONALLY left on: editing a value + # renames the ConfigMap, which rewrites the Job's env reference, which changes + # the PostSync hook Job's spec and forces Argo to re-run it -> new limits apply. + # Sizing assumes ~1500 events/s avg @ ~1 KiB/event with S2 compression (~4x): + # ~33 GiB/day compressed -> ~230 GiB/7d per replica. max_bytes 300 GiB sits + # under the 400Gi/node PVC (see values-nats.yaml). Raising retention beyond the + # PVC requires bumping BOTH max_bytes here and fileStore PVC size in values. + - name: nats-stream-limits + literals: + - max_age=168h + - max_bytes=322122547200 + - dupe_window=2m - name: vector-agent-config files: - agent.yaml=vector/agent.yaml diff --git a/apps/base/logging/nats-bootstrap-job.yaml b/apps/base/logging/nats-bootstrap-job.yaml index a21b5d2..9ace885 100644 --- a/apps/base/logging/nats-bootstrap-job.yaml +++ b/apps/base/logging/nats-bootstrap-job.yaml @@ -1,13 +1,21 @@ --- # Declarative JetStream provisioning: the LOGS stream + durable consumers. -# ArgoCD PostSync hook, idempotent (add-or-converge), re-runs each sync. +# ArgoCD PostSync hook, idempotent create-or-UPDATE, re-runs each sync. # -# Stream LOGS: file storage, 3 replicas, retention=limits (NOT workqueue) so -# multiple durable consumers fan out and can independently replay within the -# window. Sized as the ClickHouse-outage buffer: 40 GiB / 72h (per-replica PVC -# is 50Gi, see values-nats.yaml). Beyond that window the S3 archive is the +# Stream LOGS: file storage, 3 replicas, retention=limits (NOT workqueue) so the +# transform tier AND the archiver each independently see every message — reading +# never deletes; only max-age/max-bytes do. S2 compression is on (logs compress +# well). Replay window = max-age (7d default). Beyond that, the S3 archive is the # long-term replay source. # +# TUNABLE LIMITS LIVE IN A CONFIGMAP (nats-stream-limits): max_age, max_bytes, +# dupe_window. Change the ConfigMap and re-sync — this Job re-runs and applies +# the new limits via `nats stream edit` (no manual surgery). The ConfigMap is +# generated with a content-hash suffix (kustomize), so editing it changes both +# the ConfigMap name AND this Job's env reference → the PostSync hook Job's spec +# changes and Argo re-runs it (belt-and-suspenders on top of hooks running each +# sync; hook-delete-policy=BeforeHookCreation recreates it every time). +# # Consumers (independent offsets = true fan-out): # transform -> whole log stream, feeds the ClickHouse transform tier # archiver -> configurable security-relevant subset, feeds the S3 archiver. @@ -16,9 +24,10 @@ # --filter). Exact default set is an open decision for Ben. # # Runbook (replay): -# (a) reprocess from JetStream (within 72h): scale the transform tier to 0, -# then `nats consumer rm LOGS transform` and re-run this Job (recreates at -# DeliverAll), or `nats consumer edit`/`--replay` from a start seq/time. +# (a) reprocess from JetStream (within max-age, 7d): scale the transform tier +# to 0, then `nats consumer rm LOGS transform` and re-run this Job +# (recreates at DeliverAll), or `nats consumer edit`/`--replay` from a +# start seq/time. # (b) long-horizon (beyond JetStream): re-ingest S3 archive objects back # through the transform tier (vector aws_s3 source or a one-shot Job). apiVersion: batch/v1 @@ -66,6 +75,22 @@ spec: secretKeyRef: name: nats-auth key: admin_password + # Tunable stream limits — sourced from the ConfigMap. + - name: MAX_AGE + valueFrom: + configMapKeyRef: + name: nats-stream-limits + key: max_age + - name: MAX_BYTES + valueFrom: + configMapKeyRef: + name: nats-stream-limits + key: max_bytes + - name: DUPE_WINDOW + valueFrom: + configMapKeyRef: + name: nats-stream-limits + key: dupe_window # Space-separated subject filters for the archiver consumer. - name: ARCHIVE_SUBJECTS value: "logs.k8s.vault.>" @@ -79,16 +104,22 @@ spec: echo " not ready, retry in 5s"; sleep 5 done - echo "Ensuring stream LOGS ..." + echo "Ensuring stream LOGS (max_age=$MAX_AGE max_bytes=$MAX_BYTES dupe=$DUPE_WINDOW) ..." + # Create if absent; otherwise converge the mutable limits from the + # ConfigMap. (storage/retention/replicas are immutable, set only on + # create.) nats stream add LOGS \ --subjects='logs.>' --storage=file --replicas=3 \ - --retention=limits --discard=old \ - --max-age=72h --max-bytes=42949672960 \ + --retention=limits --discard=old --compression=s2 \ + --max-age="$MAX_AGE" --max-bytes="$MAX_BYTES" \ --max-msgs=-1 --max-msgs-per-subject=-1 --max-msg-size=-1 \ - --max-consumers=-1 --dupe-window=2m --defaults 2>/dev/null \ + --max-consumers=-1 --dupe-window="$DUPE_WINDOW" --defaults 2>/dev/null \ + && echo " created" \ || nats stream edit -f LOGS \ - --subjects='logs.>' --discard=old \ - --max-age=72h --max-bytes=42949672960 --dupe-window=2m + --subjects='logs.>' --discard=old --compression=s2 \ + --max-age="$MAX_AGE" --max-bytes="$MAX_BYTES" \ + --max-msgs=-1 --max-msgs-per-subject=-1 --max-msg-size=-1 \ + --max-consumers=-1 --dupe-window="$DUPE_WINDOW" echo "Ensuring consumer transform ..." nats consumer add LOGS transform \ diff --git a/apps/base/logging/vector/aggregator.yaml b/apps/base/logging/vector/aggregator.yaml index 03bb18a..4944ef9 100644 --- a/apps/base/logging/vector/aggregator.yaml +++ b/apps/base/logging/vector/aggregator.yaml @@ -7,7 +7,8 @@ # insert a transform and append its id to the clickhouse sink `inputs` — no edge # or VM rollout required. # -# Durability model: JetStream (72h / 40GiB) is the SOLE durability layer. This +# Durability model: JetStream (7d / 300 GiB, S2-compressed) is the SOLE +# durability layer and the replay window. This # tier is stateless (no PVC, memory buffer). If ClickHouse is down the sink # blocks (when_full=block); back-pressure stops the source pulling, so unpulled # messages stay in JetStream and are redelivered. NB: Vector's NATS source has diff --git a/apps/base/logging/vector/archiver.yaml b/apps/base/logging/vector/archiver.yaml index c7347b2..075e34d 100644 --- a/apps/base/logging/vector/archiver.yaml +++ b/apps/base/logging/vector/archiver.yaml @@ -4,7 +4,7 @@ # from the ClickHouse transform path (archive lag can never stall ingest — true # fan-out). Writes RAW, pre-transform events (as they sit in JetStream) as # gzipped NDJSON, partitioned by subject + date. This is the long-horizon replay -# source beyond JetStream's 72h retention window. +# source beyond JetStream's 7d retention window. data_dir: /vector-data-dir api: diff --git a/apps/overlays/au-syd1/logging/values-nats.yaml b/apps/overlays/au-syd1/logging/values-nats.yaml index 8e70444..c5443cd 100644 --- a/apps/overlays/au-syd1/logging/values-nats.yaml +++ b/apps/overlays/au-syd1/logging/values-nats.yaml @@ -13,7 +13,13 @@ config: enabled: true fileStore: pvc: - size: 50Gi + # Sized for 7d retention: ~230 GiB/7d compressed (see + # nats-stream-limits ConfigMap) + file-store WAL/index/overhead, kept + # safely above the 300 GiB max_bytes cap. 3 replicas => 1.2 TiB total + # provisioned on cephrbd-fast-delete. NB: this is the honest number for + # the assumed ~1500 events/s; higher real volume needs a bigger PVC + + # max_bytes together, else discard=old truncates retention below 7d. + size: 400Gi storageClassName: cephrbd-fast-delete # Per-user auth with publish/subscribe separation. Passwords are injected as # env vars from the Vault-synced nats-auth Secret (NATS expands $VAR in config).