From 10020033d9a4728126539b3f57cc8698e0821d76 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Mon, 27 Jul 2026 19:49:17 +1000 Subject: [PATCH 1/6] Add ClickHouse + Vector centralized logging stack Stand up a centralized logging estate that captures ALL logs from k8s pods and (via a reachable ingestion endpoint) puppet-managed VMs, storing them in ClickHouse for query/retention. Metrics already live in VictoriaMetrics; this adds the logs pillar under a dedicated `logging` ArgoCD project. Deploy the Altinity clickhouse-operator (clickhouse-system) and a single-shard ClickHouseInstallation (logging) on cephrbd-fast-delete with a MergeTree logs.raw table (30d TTL) bootstrapped by an idempotent PostSync Job. Deploy Vector as an explicit two tiers: - Edge (thin): a DaemonSet tails every node's pod logs and forwards over the Vector-native protocol to the aggregator; no parsing at the edge. Future VM agents follow the same thin pattern. - Aggregator (brain): HA StatefulSet that is the sole ClickHouse writer, holds the only ClickHouse credentials, owns all transforms, batches into few fat inserts (avoid too-many-parts), and buffers to disk (PVC) to ride out a ClickHouse outage. Its pipeline is a single source-of-truth config validated by `vector test` in CI; per-app pipelines become aggregator-only changes. Expose the VM ingestion endpoint at logs-ingest.k8s.syd1.au.unkin.net via the internal Traefik gateway (cert-manager + external-dns), routing to the aggregator's HTTP source so puppet VMs can reach it over TLS. Source ClickHouse credentials from Vault via the existing VaultStaticSecret pattern (templated k8s auth policy already grants the logging namespace); password hash never lands in git. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv --- .woodpecker/vector-test.yaml | 22 ++++ .../base/clickhouse-system/kustomization.yaml | 6 + apps/base/clickhouse-system/namespace.yaml | 7 + apps/base/logging/clickhouseinstallation.yaml | 76 +++++++++++ apps/base/logging/gateway.yaml | 44 +++++++ apps/base/logging/httproute.yaml | 55 ++++++++ apps/base/logging/job_clickhouse-schema.yaml | 105 +++++++++++++++ apps/base/logging/kustomization.yaml | 21 +++ apps/base/logging/namespace.yaml | 7 + apps/base/logging/vaultauth.yaml | 18 +++ apps/base/logging/vaultstaticsecret.yaml | 28 ++++ .../base/logging/vector/aggregator-tests.yaml | 49 +++++++ apps/base/logging/vector/aggregator.yaml | 122 ++++++++++++++++++ .../clickhouse-system/kustomization.yaml | 16 +++ .../au-syd1/clickhouse-system/values.yaml | 29 +++++ .../au-syd1/logging/kustomization.yaml | 25 ++++ .../au-syd1/logging/values-vector-agent.yaml | 44 +++++++ .../logging/values-vector-aggregator.yaml | 82 ++++++++++++ argocd/applicationsets/kustomization.yaml | 1 + argocd/applicationsets/logging.yaml | 33 +++++ argocd/projects/kustomization.yaml | 1 + argocd/projects/logging.yaml | 29 +++++ 22 files changed, 820 insertions(+) create mode 100644 .woodpecker/vector-test.yaml create mode 100644 apps/base/clickhouse-system/kustomization.yaml create mode 100644 apps/base/clickhouse-system/namespace.yaml create mode 100644 apps/base/logging/clickhouseinstallation.yaml create mode 100644 apps/base/logging/gateway.yaml create mode 100644 apps/base/logging/httproute.yaml create mode 100644 apps/base/logging/job_clickhouse-schema.yaml create mode 100644 apps/base/logging/kustomization.yaml create mode 100644 apps/base/logging/namespace.yaml create mode 100644 apps/base/logging/vaultauth.yaml create mode 100644 apps/base/logging/vaultstaticsecret.yaml create mode 100644 apps/base/logging/vector/aggregator-tests.yaml create mode 100644 apps/base/logging/vector/aggregator.yaml create mode 100644 apps/overlays/au-syd1/clickhouse-system/kustomization.yaml create mode 100644 apps/overlays/au-syd1/clickhouse-system/values.yaml create mode 100644 apps/overlays/au-syd1/logging/kustomization.yaml create mode 100644 apps/overlays/au-syd1/logging/values-vector-agent.yaml create mode 100644 apps/overlays/au-syd1/logging/values-vector-aggregator.yaml create mode 100644 argocd/applicationsets/logging.yaml create mode 100644 argocd/projects/logging.yaml diff --git a/.woodpecker/vector-test.yaml b/.woodpecker/vector-test.yaml new file mode 100644 index 0000000..9cea466 --- /dev/null +++ b/.woodpecker/vector-test.yaml @@ -0,0 +1,22 @@ +when: + - event: pull_request + +steps: + - name: vector-test + image: timberio/vector:0.57.0-debian + commands: + # Dummy creds + a writable data_dir so the full topology (incl. the + # ClickHouse sink disk buffer) builds; tests only exercise the transforms. + - export CLICKHOUSE_USER=ci CLICKHOUSE_PASSWORD=ci + - mkdir -p /vector-data-dir + - vector test apps/base/logging/vector/aggregator.yaml apps/base/logging/vector/aggregator-tests.yaml + backend_options: + kubernetes: + serviceAccountName: default + resources: + requests: + memory: 256Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 1 diff --git a/apps/base/clickhouse-system/kustomization.yaml b/apps/base/clickhouse-system/kustomization.yaml new file mode 100644 index 0000000..5122f9e --- /dev/null +++ b/apps/base/clickhouse-system/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml diff --git a/apps/base/clickhouse-system/namespace.yaml b/apps/base/clickhouse-system/namespace.yaml new file mode 100644 index 0000000..519dfb3 --- /dev/null +++ b/apps/base/clickhouse-system/namespace.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/name: clickhouse-system + name: clickhouse-system diff --git a/apps/base/logging/clickhouseinstallation.yaml b/apps/base/logging/clickhouseinstallation.yaml new file mode 100644 index 0000000..ea6bb44 --- /dev/null +++ b/apps/base/logging/clickhouseinstallation.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: clickhouse.altinity.com/v1 +kind: ClickHouseInstallation +metadata: + name: logs + namespace: logging +spec: + defaults: + templates: + dataVolumeClaimTemplate: data-volume + serviceTemplate: chi-service + podTemplate: clickhouse + configuration: + users: + # Password hash is sourced from the Vault-synced clickhouse-credentials + # Secret; the plaintext never lands in git or the ClickHouse config. + vector/password_sha256_hex: + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: password_sha256_hex + vector/networks/ip: + - "::/0" + vector/profile: default + vector/quota: default + # Allow the vector user to create the logs database/table (bootstrap Job) + # and to INSERT. Restrict the built-in default user to loopback only. + vector/access_management: "1" + default/networks/ip: + - "127.0.0.1" + - "::1" + profiles: + default/max_memory_usage: "10000000000" + default/max_execution_time: "120" + clusters: + - name: logs + layout: + shardsCount: 1 + replicasCount: 1 + templates: + volumeClaimTemplates: + - name: data-volume + spec: + storageClassName: cephrbd-fast-delete + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 200Gi + serviceTemplates: + - name: chi-service + generateName: "clickhouse-{chi}" + spec: + type: ClusterIP + ports: + - name: http + port: 8123 + - name: tcp + port: 9000 + podTemplates: + - name: clickhouse + spec: + securityContext: + fsGroup: 101 + runAsUser: 101 + runAsGroup: 101 + containers: + - name: clickhouse + image: clickhouse/clickhouse-server:24.8 + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: "2" + memory: 8Gi diff --git a/apps/base/logging/gateway.yaml b/apps/base/logging/gateway.yaml new file mode 100644 index 0000000..bb072dc --- /dev/null +++ b/apps/base/logging/gateway.yaml @@ -0,0 +1,44 @@ +--- +# Log ingestion endpoint for puppet-managed VMs (and any non-k8s client). +# Reuses the internal Traefik gateway + cert-manager + external-dns pattern so +# VMs reach the Vector aggregator's HTTP source over TLS at a DNS name they can +# resolve. The puppet-side Vector rollout ships NDJSON to +# https://logs-ingest.k8s.syd1.au.unkin.net/ (a later task). +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: logs-ingest + namespace: logging + labels: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/component: ingest + traefik.io/instance: internal + annotations: + cert-manager.io/cluster-issuer: vault-issuer + cert-manager.io/common-name: logs-ingest.k8s.syd1.au.unkin.net + cert-manager.io/private-key-size: "4096" + external-dns.alpha.kubernetes.io/hostname: logs-ingest.k8s.syd1.au.unkin.net + external-dns.alpha.kubernetes.io/target: 198.18.200.4 +spec: + gatewayClassName: traefik-internal + listeners: + - name: http + port: 80 + protocol: HTTP + hostname: logs-ingest.k8s.syd1.au.unkin.net + allowedRoutes: + namespaces: + from: Same + - name: https + port: 443 + protocol: HTTPS + hostname: logs-ingest.k8s.syd1.au.unkin.net + allowedRoutes: + namespaces: + from: Same + tls: + mode: Terminate + certificateRefs: + - group: "" + kind: Secret + name: logs-ingest-tls diff --git a/apps/base/logging/httproute.yaml b/apps/base/logging/httproute.yaml new file mode 100644 index 0000000..ba8f241 --- /dev/null +++ b/apps/base/logging/httproute.yaml @@ -0,0 +1,55 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: logs-ingest-http-redirect + namespace: logging + labels: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/component: ingest +spec: + hostnames: + - logs-ingest.k8s.syd1.au.unkin.net + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: logs-ingest + sectionName: http + rules: + - filters: + - type: RequestRedirect + requestRedirect: + scheme: https + statusCode: 301 + matches: + - path: + type: PathPrefix + value: / +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: logs-ingest + namespace: logging + labels: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/component: ingest +spec: + hostnames: + - logs-ingest.k8s.syd1.au.unkin.net + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: logs-ingest + sectionName: https + rules: + - backendRefs: + - group: "" + kind: Service + name: vector-aggregator + port: 8080 + weight: 1 + matches: + - path: + type: PathPrefix + value: / diff --git a/apps/base/logging/job_clickhouse-schema.yaml b/apps/base/logging/job_clickhouse-schema.yaml new file mode 100644 index 0000000..b1e3dd4 --- /dev/null +++ b/apps/base/logging/job_clickhouse-schema.yaml @@ -0,0 +1,105 @@ +--- +# Declarative ClickHouse schema bootstrap. Runs as an ArgoCD PostSync hook so it +# executes after the ClickHouseInstallation is reconciled, and re-runs on every +# sync (idempotent CREATE ... IF NOT EXISTS). Edit the DDL here to evolve the +# schema; the Vector aggregator writes to logs.raw with skip_unknown_fields, so +# adding columns is backward-compatible. +apiVersion: batch/v1 +kind: Job +metadata: + name: clickhouse-schema + namespace: logging + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation + labels: + app.kubernetes.io/name: clickhouse-schema + app.kubernetes.io/component: bootstrap +spec: + backoffLimit: 20 + activeDeadlineSeconds: 1800 + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + app.kubernetes.io/name: clickhouse-schema + vector.dev/exclude: "true" + spec: + restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + containers: + - name: clickhouse-schema + image: clickhouse/clickhouse-server:24.8 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + env: + - name: HOME + value: /tmp + - name: CLICKHOUSE_USER + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: username + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: password + command: + - /bin/bash + - -ec + - | + host=clickhouse-logs.logging.svc.cluster.local + echo "Waiting for ClickHouse at ${host}:9000 ..." + until clickhouse-client --host "$host" --port 9000 \ + --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \ + --query "SELECT 1" >/dev/null 2>&1; do + echo " not ready, retrying in 5s"; sleep 5 + done + echo "Applying schema ..." + clickhouse-client --host "$host" --port 9000 \ + --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \ + --multiquery <<'EOSQL' + CREATE DATABASE IF NOT EXISTS logs; + + CREATE TABLE IF NOT EXISTS logs.raw + ( + timestamp DateTime64(3) DEFAULT now64(3), + host LowCardinality(String) DEFAULT '', + source LowCardinality(String) DEFAULT '', + namespace LowCardinality(String) DEFAULT '', + pod String DEFAULT '', + container LowCardinality(String) DEFAULT '', + stream LowCardinality(String) DEFAULT '', + severity LowCardinality(String) DEFAULT '', + message String DEFAULT '', + labels Map(LowCardinality(String), String), + fields Map(LowCardinality(String), String) + ) + ENGINE = MergeTree + PARTITION BY toDate(timestamp) + ORDER BY (source, namespace, host, timestamp) + TTL toDateTime(timestamp) + INTERVAL 30 DAY + SETTINGS index_granularity = 8192; + EOSQL + echo "Schema applied." + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/apps/base/logging/kustomization.yaml b/apps/base/logging/kustomization.yaml new file mode 100644 index 0000000..934f466 --- /dev/null +++ b/apps/base/logging/kustomization.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - vaultauth.yaml + - vaultstaticsecret.yaml + - clickhouseinstallation.yaml + - job_clickhouse-schema.yaml + - gateway.yaml + - httproute.yaml + +# Aggregator pipeline is the single source of truth (also validated by +# `vector test` in CI). Mounted into the aggregator via `existingConfigMaps`. +configMapGenerator: + - name: vector-aggregator-config + files: + - aggregator.yaml=vector/aggregator.yaml + options: + disableNameSuffixHash: true diff --git a/apps/base/logging/namespace.yaml b/apps/base/logging/namespace.yaml new file mode 100644 index 0000000..1ccca01 --- /dev/null +++ b/apps/base/logging/namespace.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/name: logging + name: logging diff --git a/apps/base/logging/vaultauth.yaml b/apps/base/logging/vaultauth.yaml new file mode 100644 index 0000000..bf3b375 --- /dev/null +++ b/apps/base/logging/vaultauth.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: default + namespace: logging +spec: + allowedNamespaces: + - logging + kubernetes: + audiences: + - vault + role: default + serviceAccount: default + tokenExpirationSeconds: 600 + method: kubernetes + mount: k8s/au/syd1 + vaultConnectionRef: vso-system/default diff --git a/apps/base/logging/vaultstaticsecret.yaml b/apps/base/logging/vaultstaticsecret.yaml new file mode 100644 index 0000000..d37129c --- /dev/null +++ b/apps/base/logging/vaultstaticsecret.yaml @@ -0,0 +1,28 @@ +--- +# ClickHouse credentials for the `vector` user. +# +# Seed the Vault KV entry once (values are NOT stored in git), e.g.: +# PW=$(openssl rand -base64 24) +# HASH=$(printf '%s' "$PW" | sha256sum | cut -d' ' -f1) +# vault kv put kv/kubernetes/namespace/logging/default/clickhouse-credentials \ +# username=vector password="$PW" password_sha256_hex="$HASH" +# +# The `logging/default` ServiceAccount reads this path via the templated +# `policies/kv/kubernetes/default.yaml` policy (k8s auth role `default`), so no +# terraform-vault change is required — only the value above must be written. +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: clickhouse-credentials + namespace: logging +spec: + destination: + create: true + name: clickhouse-credentials + overwrite: true + hmacSecretData: true + mount: kv + path: kubernetes/namespace/logging/default/clickhouse-credentials + refreshAfter: 5m + type: kv-v2 + vaultAuthRef: default diff --git a/apps/base/logging/vector/aggregator-tests.yaml b/apps/base/logging/vector/aggregator-tests.yaml new file mode 100644 index 0000000..94a8c60 --- /dev/null +++ b/apps/base/logging/vector/aggregator-tests.yaml @@ -0,0 +1,49 @@ +--- +# `vector test` unit tests for the aggregator transforms. Merged with +# aggregator.yaml in CI (.woodpecker/vector-test.yaml). This is the pattern the +# per-app parsing follow-ups extend: add a test per new transform here. +tests: + - name: k8s_log_is_normalised + inputs: + - insert_at: k8s_shape + type: log + log_fields: + message: "hello from pod" + stream: "stdout" + timestamp: "2026-07-27T00:00:00Z" + kubernetes.pod_name: "web-abc" + kubernetes.pod_namespace: "shop" + kubernetes.container_name: "web" + kubernetes.pod_node_name: "node-1" + outputs: + - extract_from: k8s_shape + conditions: + - type: vrl + source: | + assert_eq!(.source, "k8s") + assert_eq!(.namespace, "shop") + assert_eq!(.pod, "web-abc") + assert_eq!(.container, "web") + assert_eq!(.host, "node-1") + assert_eq!(.stream, "stdout") + assert_eq!(.message, "hello from pod") + + - name: vm_log_is_normalised + inputs: + - insert_at: vm_shape + type: log + log_fields: + message: "sshd started" + host: "vm-db-1" + severity: "info" + role: "database" + outputs: + - extract_from: vm_shape + conditions: + - type: vrl + source: | + assert_eq!(.source, "vm") + assert_eq!(.host, "vm-db-1") + assert_eq!(.severity, "info") + assert_eq!(.message, "sshd started") + assert_eq!(.labels.role, "database") diff --git a/apps/base/logging/vector/aggregator.yaml b/apps/base/logging/vector/aggregator.yaml new file mode 100644 index 0000000..78bd816 --- /dev/null +++ b/apps/base/logging/vector/aggregator.yaml @@ -0,0 +1,122 @@ +--- +# Vector AGGREGATOR pipeline (the "brain") — single source of truth. +# +# This file is the ConfigMap the aggregator StatefulSet runs AND the config +# `vector test` validates in CI (see .woodpecker/vector-test.yaml, which merges +# this with aggregator-tests.yaml). The edge tier (k8s DaemonSet + future VM +# agents) stays thin: it only collects and attaches source metadata, then +# forwards over the Vector-native protocol. All shaping, routing, enrichment, +# batching, buffering and the ONLY ClickHouse credentials live here. +# +# Per-app pipelines arrive as follow-up tasks and are aggregator-only changes: +# add a `remap`/`route`/enrichment transform below and append its id to the +# clickhouse sink `inputs` — no fleet/DaemonSet rollout required. +data_dir: /vector-data-dir + +api: + enabled: true + address: 0.0.0.0:8686 + +sources: + # In-cluster pod logs from the Vector agent DaemonSet (Vector-native proto). + from_agents: + type: vector + address: 0.0.0.0:6000 + + # Puppet-managed VM logs over HTTP (NDJSON), exposed via the logs-ingest + # Gateway at logs-ingest.k8s.syd1.au.unkin.net. + vm_http: + type: http_server + address: 0.0.0.0:8080 + path: / + method: POST + decoding: + codec: json + framing: + method: newline_delimited + +transforms: + # ---- Default normalisation into the logs.raw columns. This is intentionally + # the ONLY shaping today; per-app parsing is added here as follow-ups. ---- + k8s_shape: + type: remap + inputs: + - from_agents + source: | + ts = .timestamp || now() + node = to_string(.kubernetes.pod_node_name || "") ?? "" + ns = to_string(.kubernetes.pod_namespace || "") ?? "" + pod = to_string(.kubernetes.pod_name || "") ?? "" + container = to_string(.kubernetes.container_name || "") ?? "" + strm = to_string(.stream || "") ?? "" + msg = to_string(.message || "") ?? "" + lbls = object(.kubernetes.pod_labels) ?? {} + . = { + "timestamp": ts, + "host": node, + "source": "k8s", + "namespace": ns, + "pod": pod, + "container": container, + "stream": strm, + "severity": "", + "message": msg, + "labels": lbls, + "fields": {} + } + vm_shape: + type: remap + inputs: + - vm_http + source: | + ts = .timestamp || .ts || now() + host = to_string(.host || .hostname || "") ?? "" + msg = to_string(.message || .msg || "") ?? "" + sev = to_string(.severity || .level || "") ?? "" + role = to_string(.role || "") ?? "" + lbls = {} + if role != "" { + lbls = {"role": role} + } + . = { + "timestamp": ts, + "host": host, + "source": "vm", + "namespace": "", + "pod": "", + "container": "", + "stream": "", + "severity": sev, + "message": msg, + "labels": lbls, + "fields": {} + } + +sinks: + clickhouse: + type: clickhouse + inputs: + - k8s_shape + - vm_shape + endpoint: http://clickhouse-logs.logging.svc.cluster.local:8123 + database: logs + table: raw + skip_unknown_fields: true + date_time_best_effort: true + auth: + strategy: basic + user: "${CLICKHOUSE_USER}" + password: "${CLICKHOUSE_PASSWORD}" + # Fat, infrequent inserts keep ClickHouse part-count low (avoid too-many-parts). + batch: + max_events: 500000 + max_bytes: 134217728 + timeout_secs: 10 + # Ride out a ClickHouse outage without dropping logs: on-disk buffer on the + # aggregator PVC; block upstream (back-pressure to the edge) when full. + buffer: + type: disk + max_size: 8589934592 + when_full: block + healthcheck: + enabled: true diff --git a/apps/overlays/au-syd1/clickhouse-system/kustomization.yaml b/apps/overlays/au-syd1/clickhouse-system/kustomization.yaml new file mode 100644 index 0000000..498f573 --- /dev/null +++ b/apps/overlays/au-syd1/clickhouse-system/kustomization.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: clickhouse-system + +resources: + - ../../../base/clickhouse-system + +helmCharts: + - name: altinity-clickhouse-operator + repo: https://helm.altinity.com + version: "0.27.2" + releaseName: clickhouse-operator + namespace: clickhouse-system + valuesFile: values.yaml diff --git a/apps/overlays/au-syd1/clickhouse-system/values.yaml b/apps/overlays/au-syd1/clickhouse-system/values.yaml new file mode 100644 index 0000000..8eb31f0 --- /dev/null +++ b/apps/overlays/au-syd1/clickhouse-system/values.yaml @@ -0,0 +1,29 @@ +# Altinity ClickHouse operator. Cluster-scoped: watches ClickHouseInstallation +# resources in all namespaces (the logs cluster lives in the `logging` namespace). +# CRDs are installed at runtime by the chart's crdHook Job. +crdHook: + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 250m + memory: 128Mi + +operator: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +metrics: + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi diff --git a/apps/overlays/au-syd1/logging/kustomization.yaml b/apps/overlays/au-syd1/logging/kustomization.yaml new file mode 100644 index 0000000..0ae2aa6 --- /dev/null +++ b/apps/overlays/au-syd1/logging/kustomization.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: logging + +resources: + - ../../../base/logging + +helmCharts: + # Node-level agent: tails every pod's logs and forwards to the aggregator. + - name: vector + repo: https://helm.vector.dev + version: "0.57.0" + releaseName: vector-agent + namespace: logging + valuesFile: values-vector-agent.yaml + # Aggregator: receives from agents (vector proto) and from VMs (HTTP), + # shapes events, and writes to ClickHouse. + - name: vector + repo: https://helm.vector.dev + version: "0.57.0" + releaseName: vector-aggregator + namespace: logging + valuesFile: values-vector-aggregator.yaml diff --git a/apps/overlays/au-syd1/logging/values-vector-agent.yaml b/apps/overlays/au-syd1/logging/values-vector-agent.yaml new file mode 100644 index 0000000..3e8e064 --- /dev/null +++ b/apps/overlays/au-syd1/logging/values-vector-agent.yaml @@ -0,0 +1,44 @@ +# Vector Agent (DaemonSet) — captures ALL pod logs on every node (including +# control-plane, via the blanket toleration) and forwards them to the +# aggregator over the Vector native protocol. No per-app parsing here: shaping +# and the ClickHouse write live on the aggregator. +role: Agent +fullnameOverride: vector-agent + +rbac: + create: true +serviceAccount: + create: true + +podLabels: + vector.dev/exclude: "true" + +# Run on every node so no host's pod logs are missed. +tolerations: + - operator: Exists + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + +# Disable the chart's auto-generated service for the agent (it ships, not serves). +service: + enabled: false + +customConfig: + data_dir: /vector-data-dir + api: + enabled: false + sources: + kubernetes_logs: + type: kubernetes_logs + sinks: + to_aggregator: + type: vector + inputs: + - kubernetes_logs + address: vector-aggregator.logging.svc.cluster.local:6000 diff --git a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml new file mode 100644 index 0000000..681d21f --- /dev/null +++ b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml @@ -0,0 +1,82 @@ +# Vector AGGREGATOR (StatefulSet) — the "brain" tier: sole ClickHouse writer, +# owns all transforms, holds the only ClickHouse credentials, HA replicas with +# on-disk buffers to ride out a ClickHouse outage. +# +# The pipeline itself is NOT inlined here: it lives in +# apps/base/logging/vector/aggregator.yaml (single source of truth, unit-tested +# by `vector test` in CI) and is mounted via existingConfigMaps. This file only +# owns the k8s deployment shape (ports, creds, storage, HA). +role: Aggregator +fullnameOverride: vector-aggregator +replicas: 2 + +# Reload the StatefulSet when the aggregator ConfigMap or the creds Secret change. +workloadResourceAnnotations: + reloader.stakater.com/auto: "true" + +podLabels: + vector.dev/exclude: "true" + +# Mount the aggregator pipeline ConfigMap (rendered from base by kustomize). +# dataDir must be set when using existingConfigMaps; it is also where the +# clickhouse sink's disk buffer lives (backed by the PVC below). +dataDir: /vector-data-dir +existingConfigMaps: + - vector-aggregator-config + +# On-disk buffer storage so a ClickHouse outage does not drop logs. +persistence: + enabled: true + storageClassName: cephrbd-fast-delete + size: 20Gi + accessModes: + - ReadWriteOnce + +# ClickHouse basic-auth creds — the ONLY place these are consumed. +env: + - name: CLICKHOUSE_USER + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: username + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: password + +containerPorts: + - name: vector + containerPort: 6000 + protocol: TCP + - name: http-ingest + containerPort: 8080 + protocol: TCP + - name: api + containerPort: 8686 + protocol: TCP + +service: + enabled: true + type: ClusterIP + ports: + - name: vector + port: 6000 + targetPort: 6000 + protocol: TCP + - name: http-ingest + port: 8080 + targetPort: 8080 + protocol: TCP + - name: api + port: 8686 + targetPort: 8686 + protocol: TCP + +resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi diff --git a/argocd/applicationsets/kustomization.yaml b/argocd/applicationsets/kustomization.yaml index 87d07c2..b54cc35 100644 --- a/argocd/applicationsets/kustomization.yaml +++ b/argocd/applicationsets/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - aitooling.yaml + - logging.yaml - observability.yaml - platform.yaml - storage.yaml diff --git a/argocd/applicationsets/logging.yaml b/argocd/applicationsets/logging.yaml new file mode 100644 index 0000000..ea542d4 --- /dev/null +++ b/argocd/applicationsets/logging.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: logging-apps + namespace: argocd +spec: + generators: + - git: + repoURL: https://git.unkin.net/unkin/argocd-apps + revision: HEAD + directories: + - path: apps/overlays/*/clickhouse-system + - path: apps/overlays/*/logging + template: + metadata: + name: 'logging-{{path[3]}}' + spec: + project: logging + source: + repoURL: https://git.unkin.net/unkin/argocd-apps + targetRevision: HEAD + path: '{{path}}' + destination: + server: https://kubernetes.default.svc + namespace: '{{path[3]}}' + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true + - CreateNamespace=false diff --git a/argocd/projects/kustomization.yaml b/argocd/projects/kustomization.yaml index 87d07c2..b54cc35 100644 --- a/argocd/projects/kustomization.yaml +++ b/argocd/projects/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - aitooling.yaml + - logging.yaml - observability.yaml - platform.yaml - storage.yaml diff --git a/argocd/projects/logging.yaml b/argocd/projects/logging.yaml new file mode 100644 index 0000000..591153a --- /dev/null +++ b/argocd/projects/logging.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: logging + namespace: argocd +spec: + description: Centralized logging stack (ClickHouse + Vector) + sourceRepos: + - https://git.unkin.net/unkin/argocd-apps + - https://helm.altinity.com + - https://helm.vector.dev + destinations: + - namespace: 'logging' + server: https://kubernetes.default.svc + - namespace: 'clickhouse-system' + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: '' + kind: Namespace + - group: 'rbac.authorization.k8s.io' + kind: ClusterRole + - group: 'rbac.authorization.k8s.io' + kind: ClusterRoleBinding + - group: 'apiextensions.k8s.io' + kind: CustomResourceDefinition + namespaceResourceWhitelist: + - group: '*' + kind: '*' -- 2.47.3 From c39af2f9c3e56cc36b939cd858447709d9134646 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Mon, 27 Jul 2026 20:23:55 +1000 Subject: [PATCH 2/6] Insert NATS JetStream log bus + S3 raw archive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rework the logging pipeline around a durable message bus so logs survive a ClickHouse outage, can be replayed after a bad transform, and fan out to multiple independent consumers. Add long-term raw-log backup to S3. Topology becomes edge -> JetStream -> consumers -> sinks: - Dedicated JetStream NATS cluster (3 replicas, file storage) in the logging namespace. Deliberately separate from app messaging (streamstack) for blast-radius isolation. Stream LOGS (subjects logs.>, retention=limits, 40GiB / 72h) is the outage buffer; durable consumers give independent offsets. - Edge publishers (thin): the k8s DaemonSet and a new VM-ingest Deployment (HTTP NDJSON front door behind the logs-ingest Gateway) publish into JetStream (logs.k8s.. / logs.vm.). No parsing on the edge. - Transform tier (StatefulSet): pulls the whole stream via the durable `transform` consumer, routes by subject, shapes, and remains the sole ClickHouse writer. Its disk buffer shrinks (JetStream is the outage buffer). - Archiver (Deployment): its OWN durable `archiver` consumer (independent offsets — archive lag never affects the ClickHouse path) writes RAW, pre-transform events to a Ceph RGW S3 bucket (cephrgw-operator ObjectStoreUser + Bucket + BucketAccess) as gzipped NDJSON keyed by raw//YYYY/MM/DD/. Default subject filter is Vault audit (logs.k8s.vault.>), configurable. Auth: distinct NATS users (producer publish-only, consumer pull+ack, admin for the stream/consumer bootstrap Job) with passwords from Vault (nats-auth Secret); S3 creds from the BucketAccess Secret. Streams/consumers are provisioned by an idempotent PostSync bootstrap Job. Add local kubeconform schemas for the ceph.unkin.net CRDs (datreeio lacks them) and extend the vector-test CI to cover the agent, VM-ingest and archiver configs. Verified end-to-end locally: NATS ACLs, vector JetStream publish, and durable-consumer pull+ack (at-least-once) all work. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv --- .woodpecker/vector-test.yaml | 13 +- apps/base/logging/cephrgw.yaml | 43 ++++ apps/base/logging/httproute.yaml | 2 +- apps/base/logging/kustomization.yaml | 21 +- apps/base/logging/nats-bootstrap-job.yaml | 122 +++++++++++ apps/base/logging/vaultstaticsecret.yaml | 22 ++ apps/base/logging/vector/agent.yaml | 43 ++++ .../base/logging/vector/aggregator-tests.yaml | 14 ++ apps/base/logging/vector/aggregator.yaml | 70 ++++--- apps/base/logging/vector/archiver.yaml | 62 ++++++ apps/base/logging/vector/vm-ingest-tests.yaml | 29 +++ apps/base/logging/vector/vm-ingest.yaml | 50 +++++ .../au-syd1/logging/kustomization.yaml | 26 ++- .../overlays/au-syd1/logging/values-nats.yaml | 83 ++++++++ .../au-syd1/logging/values-vector-agent.yaml | 38 ++-- .../logging/values-vector-aggregator.yaml | 43 ++-- .../logging/values-vector-archiver.yaml | 52 +++++ .../logging/values-vector-vm-ingest.yaml | 52 +++++ schemas/ceph.unkin.net/bucket_v1alpha1.json | 176 ++++++++++++++++ .../ceph.unkin.net/bucketaccess_v1alpha1.json | 191 ++++++++++++++++++ .../objectstoreuser_v1alpha1.json | 132 ++++++++++++ 21 files changed, 1195 insertions(+), 89 deletions(-) create mode 100644 apps/base/logging/cephrgw.yaml create mode 100644 apps/base/logging/nats-bootstrap-job.yaml create mode 100644 apps/base/logging/vector/agent.yaml create mode 100644 apps/base/logging/vector/archiver.yaml create mode 100644 apps/base/logging/vector/vm-ingest-tests.yaml create mode 100644 apps/base/logging/vector/vm-ingest.yaml create mode 100644 apps/overlays/au-syd1/logging/values-nats.yaml create mode 100644 apps/overlays/au-syd1/logging/values-vector-archiver.yaml create mode 100644 apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml create mode 100644 schemas/ceph.unkin.net/bucket_v1alpha1.json create mode 100644 schemas/ceph.unkin.net/bucketaccess_v1alpha1.json create mode 100644 schemas/ceph.unkin.net/objectstoreuser_v1alpha1.json diff --git a/.woodpecker/vector-test.yaml b/.woodpecker/vector-test.yaml index 9cea466..8949ce3 100644 --- a/.woodpecker/vector-test.yaml +++ b/.woodpecker/vector-test.yaml @@ -5,11 +5,18 @@ steps: - name: vector-test image: timberio/vector:0.57.0-debian commands: - # Dummy creds + a writable data_dir so the full topology (incl. the - # ClickHouse sink disk buffer) builds; tests only exercise the transforms. + # Dummy creds + writable dirs so the full topologies build; the unit tests + # only exercise the transforms (sources are not started). - export CLICKHOUSE_USER=ci CLICKHOUSE_PASSWORD=ci - - mkdir -p /vector-data-dir + - export NATS_PRODUCER_PASSWORD=ci NATS_CONSUMER_PASSWORD=ci + - mkdir -p /vector-data-dir /etc/vault-ca + - cp /etc/ssl/certs/ca-certificates.crt /etc/vault-ca/ca.crt + # Transform tier + VM ingest: unit-tested transforms. - vector test apps/base/logging/vector/aggregator.yaml apps/base/logging/vector/aggregator-tests.yaml + - vector test apps/base/logging/vector/vm-ingest.yaml apps/base/logging/vector/vm-ingest-tests.yaml + # Agent + archiver have no transforms to unit-test; validate they build. + - vector validate --no-environment apps/base/logging/vector/agent.yaml + - vector validate --no-environment apps/base/logging/vector/archiver.yaml backend_options: kubernetes: serviceAccountName: default diff --git a/apps/base/logging/cephrgw.yaml b/apps/base/logging/cephrgw.yaml new file mode 100644 index 0000000..ee1bd8e --- /dev/null +++ b/apps/base/logging/cephrgw.yaml @@ -0,0 +1,43 @@ +--- +# S3 bucket (Ceph RGW) for the long-term raw-log archive, provisioned by the +# in-estate cephrgw-operator. The archiver Vector deployment writes here. +apiVersion: ceph.unkin.net/v1alpha1 +kind: ObjectStoreUser +metadata: + name: logs-archive-owner + namespace: logging +spec: + displayName: "Logging raw-archive bucket owner" + maxBuckets: 5 + quota: + enabled: true + # 5 TiB soft cap; real retention is enforced RGW-side by a bucket lifecycle + # policy (see PR notes) — the operator does not manage lifecycle. + maxSizeBytes: 5497558138880 +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: Bucket +metadata: + name: logs-archive + namespace: logging +spec: + bucketName: logs-archive + ownerRef: logs-archive-owner + versioning: false + tags: + app: logging + purpose: raw-log-archive + # Keep the bucket (and its objects) if this CR is ever deleted. + retainOnDelete: true +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: BucketAccess +metadata: + name: logs-archive-writer + namespace: logging +spec: + bucketRef: logs-archive + level: read-write + # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID, + # S3_ENDPOINT, BUCKET_NAME) into this Secret; the archiver consumes it. + secretName: logs-archive-s3 diff --git a/apps/base/logging/httproute.yaml b/apps/base/logging/httproute.yaml index ba8f241..2400863 100644 --- a/apps/base/logging/httproute.yaml +++ b/apps/base/logging/httproute.yaml @@ -46,7 +46,7 @@ spec: - backendRefs: - group: "" kind: Service - name: vector-aggregator + name: vector-vm-ingest port: 8080 weight: 1 matches: diff --git a/apps/base/logging/kustomization.yaml b/apps/base/logging/kustomization.yaml index 934f466..cf94855 100644 --- a/apps/base/logging/kustomization.yaml +++ b/apps/base/logging/kustomization.yaml @@ -8,14 +8,31 @@ resources: - vaultstaticsecret.yaml - clickhouseinstallation.yaml - job_clickhouse-schema.yaml + - nats-bootstrap-job.yaml + - cephrgw.yaml - gateway.yaml - httproute.yaml -# Aggregator pipeline is the single source of truth (also validated by -# `vector test` in CI). Mounted into the aggregator via `existingConfigMaps`. +# Vector pipelines are the single source of truth (also validated by +# `vector test` in CI). Mounted into each tier via `existingConfigMaps`. configMapGenerator: + - name: vector-agent-config + files: + - agent.yaml=vector/agent.yaml + options: + disableNameSuffixHash: true - name: vector-aggregator-config files: - aggregator.yaml=vector/aggregator.yaml options: disableNameSuffixHash: true + - name: vector-vm-ingest-config + files: + - vm-ingest.yaml=vector/vm-ingest.yaml + options: + disableNameSuffixHash: true + - name: vector-archiver-config + files: + - archiver.yaml=vector/archiver.yaml + options: + disableNameSuffixHash: true diff --git a/apps/base/logging/nats-bootstrap-job.yaml b/apps/base/logging/nats-bootstrap-job.yaml new file mode 100644 index 0000000..72e5ad9 --- /dev/null +++ b/apps/base/logging/nats-bootstrap-job.yaml @@ -0,0 +1,122 @@ +--- +# Declarative JetStream provisioning: the LOGS stream + durable consumers. +# ArgoCD PostSync hook, idempotent (add-or-converge), re-runs each sync. +# +# Stream LOGS: file storage, 3 replicas, retention=limits (NOT workqueue) so +# multiple durable consumers fan out and can independently replay within the +# window. Sized as the ClickHouse-outage buffer: 40 GiB / 72h (per-replica PVC +# is 50Gi, see values-nats.yaml). Beyond that window the S3 archive is the +# long-term replay source. +# +# Consumers (independent offsets = true fan-out): +# transform -> whole log stream, feeds the ClickHouse transform tier +# archiver -> configurable security-relevant subset, feeds the S3 archiver. +# Default filter is Vault audit (logs.k8s.vault.>); ADD subjects +# by editing ARCHIVE_SUBJECTS (space-separated -> repeated +# --filter). Exact default set is an open decision for Ben. +# +# Runbook (replay): +# (a) reprocess from JetStream (within 72h): scale the transform tier to 0, +# then `nats consumer rm LOGS transform` and re-run this Job (recreates at +# DeliverAll), or `nats consumer edit`/`--replay` from a start seq/time. +# (b) long-horizon (beyond JetStream): re-ingest S3 archive objects back +# through the transform tier (vector aws_s3 source or a one-shot Job). +apiVersion: batch/v1 +kind: Job +metadata: + name: nats-bootstrap + namespace: logging + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation + labels: + app.kubernetes.io/name: nats-bootstrap + app.kubernetes.io/component: bootstrap +spec: + backoffLimit: 20 + activeDeadlineSeconds: 1800 + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + app.kubernetes.io/name: nats-bootstrap + vector.dev/exclude: "true" + spec: + restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + containers: + - name: nats-bootstrap + image: natsio/nats-box:0.18.0 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + env: + - name: HOME + value: /tmp + - name: NATS_URL + value: "nats://nats.logging.svc.cluster.local:4222" + - name: NATS_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: nats-auth + key: admin_password + # Space-separated subject filters for the archiver consumer. + - name: ARCHIVE_SUBJECTS + value: "logs.k8s.vault.>" + command: + - /bin/sh + - -ec + - | + export NATS_USER=log-admin NATS_PASSWORD="$NATS_ADMIN_PASSWORD" + echo "Waiting for NATS + JetStream ..." + until nats --server "$NATS_URL" account info >/dev/null 2>&1; do + echo " not ready, retry in 5s"; sleep 5 + done + + echo "Ensuring stream LOGS ..." + nats stream add LOGS \ + --subjects='logs.>' --storage=file --replicas=3 \ + --retention=limits --discard=old \ + --max-age=72h --max-bytes=42949672960 \ + --max-msgs=-1 --max-msgs-per-subject=-1 --max-msg-size=-1 \ + --max-consumers=-1 --dupe-window=2m --defaults 2>/dev/null \ + || nats stream edit -f LOGS \ + --subjects='logs.>' --discard=old \ + --max-age=72h --max-bytes=42949672960 --dupe-window=2m + + echo "Ensuring consumer transform ..." + nats consumer add LOGS transform \ + --pull --filter='logs.>' --deliver=all --ack=explicit \ + --max-deliver=-1 --replay=instant --defaults 2>/dev/null \ + || echo " transform already exists" + + echo "Ensuring consumer archiver (filters: $ARCHIVE_SUBJECTS) ..." + filter_args="" + for s in $ARCHIVE_SUBJECTS; do filter_args="$filter_args --filter=$s"; done + # shellcheck disable=SC2086 + nats consumer add LOGS archiver \ + --pull $filter_args --deliver=all --ack=explicit \ + --max-deliver=-1 --replay=instant --defaults 2>/dev/null \ + || echo " archiver already exists" + + echo "Done." + nats stream info LOGS + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/apps/base/logging/vaultstaticsecret.yaml b/apps/base/logging/vaultstaticsecret.yaml index d37129c..45b56ab 100644 --- a/apps/base/logging/vaultstaticsecret.yaml +++ b/apps/base/logging/vaultstaticsecret.yaml @@ -26,3 +26,25 @@ spec: refreshAfter: 5m type: kv-v2 vaultAuthRef: default +--- +# NATS JetStream auth. Distinct passwords for the producer (edge), consumer +# (transform tier + archiver) and admin (bootstrap Job) users. Seed once: +# for k in admin producer consumer; do declare P_$k=$(openssl rand -base64 24); done +# vault kv put kv/kubernetes/namespace/logging/default/nats-auth \ +# admin_password="$P_admin" producer_password="$P_producer" consumer_password="$P_consumer" +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: nats-auth + namespace: logging +spec: + destination: + create: true + name: nats-auth + overwrite: true + hmacSecretData: true + mount: kv + path: kubernetes/namespace/logging/default/nats-auth + refreshAfter: 5m + type: kv-v2 + vaultAuthRef: default diff --git a/apps/base/logging/vector/agent.yaml b/apps/base/logging/vector/agent.yaml new file mode 100644 index 0000000..efbc35f --- /dev/null +++ b/apps/base/logging/vector/agent.yaml @@ -0,0 +1,43 @@ +--- +# Vector EDGE agent pipeline (DaemonSet) — thin publisher, single source of +# truth. Mounted via existingConfigMaps (NOT the chart's customConfig, whose +# Helm `tpl` pass collides with Vector's own {{ }} / ${ } syntax). Tails all pod +# logs, attaches only routing tokens, publishes to JetStream. No parsing. +data_dir: /vector-data-dir + +api: + enabled: false + +sources: + kubernetes_logs: + type: kubernetes_logs + +transforms: + # Routing metadata only: NATS-subject-safe namespace + container tokens. + keymeta: + type: remap + inputs: + - kubernetes_logs + source: | + ns = to_string(.kubernetes.pod_namespace || "unknown") ?? "unknown" + .ns_token = replace(ns, r'[^a-zA-Z0-9_-]', "_") + cont = to_string(.kubernetes.container_name || "unknown") ?? "unknown" + .cont_token = replace(cont, r'[^a-zA-Z0-9_-]', "_") + +sinks: + to_jetstream: + type: nats + inputs: + - keymeta + url: nats://nats.logging.svc.cluster.local:4222 + connection_name: vector-agent + subject: "logs.k8s.{{ ns_token }}.{{ cont_token }}" + jetstream: + enabled: true + auth: + strategy: user_password + user_password: + user: log-producer + password: ${NATS_PRODUCER_PASSWORD} + encoding: + codec: json diff --git a/apps/base/logging/vector/aggregator-tests.yaml b/apps/base/logging/vector/aggregator-tests.yaml index 94a8c60..8a194fa 100644 --- a/apps/base/logging/vector/aggregator-tests.yaml +++ b/apps/base/logging/vector/aggregator-tests.yaml @@ -3,6 +3,20 @@ # aggregator.yaml in CI (.woodpecker/vector-test.yaml). This is the pattern the # per-app parsing follow-ups extend: add a test per new transform here. tests: + - name: subject_routes_k8s_vs_vm + inputs: + - insert_at: route + type: log + log_fields: + subject: "logs.k8s.shop.web" + message: "routed" + outputs: + - extract_from: route.k8s + conditions: + - type: vrl + source: | + assert_eq!(.message, "routed") + - name: k8s_log_is_normalised inputs: - insert_at: k8s_shape diff --git a/apps/base/logging/vector/aggregator.yaml b/apps/base/logging/vector/aggregator.yaml index 78bd816..ad35505 100644 --- a/apps/base/logging/vector/aggregator.yaml +++ b/apps/base/logging/vector/aggregator.yaml @@ -1,16 +1,16 @@ --- -# Vector AGGREGATOR pipeline (the "brain") — single source of truth. +# Vector TRANSFORM tier (the "brain") — single source of truth, also validated +# by `vector test` in CI. Consumes the whole log stream from JetStream via the +# durable `transform` consumer (at-least-once; durable offsets tracked by +# JetStream), routes by subject, normalises into the logs.raw columns, and is +# the ONLY ClickHouse writer. Per-app parsing is added here as follow-ups: +# insert a transform and append its id to the clickhouse sink `inputs` — no edge +# or VM rollout required. # -# This file is the ConfigMap the aggregator StatefulSet runs AND the config -# `vector test` validates in CI (see .woodpecker/vector-test.yaml, which merges -# this with aggregator-tests.yaml). The edge tier (k8s DaemonSet + future VM -# agents) stays thin: it only collects and attaches source metadata, then -# forwards over the Vector-native protocol. All shaping, routing, enrichment, -# batching, buffering and the ONLY ClickHouse credentials live here. -# -# Per-app pipelines arrive as follow-up tasks and are aggregator-only changes: -# add a `remap`/`route`/enrichment transform below and append its id to the -# clickhouse sink `inputs` — no fleet/DaemonSet rollout required. +# Durability model: JetStream (72h / 40GiB) is the outage buffer. If ClickHouse +# is down the sink blocks, back-pressure stops acking, and JetStream retains +# messages for replay. The local disk buffer is small (survives pod restarts of +# in-flight events only). data_dir: /vector-data-dir api: @@ -18,30 +18,35 @@ api: address: 0.0.0.0:8686 sources: - # In-cluster pod logs from the Vector agent DaemonSet (Vector-native proto). - from_agents: - type: vector - address: 0.0.0.0:6000 - - # Puppet-managed VM logs over HTTP (NDJSON), exposed via the logs-ingest - # Gateway at logs-ingest.k8s.syd1.au.unkin.net. - vm_http: - type: http_server - address: 0.0.0.0:8080 - path: / - method: POST + js_in: + type: nats + url: nats://nats.logging.svc.cluster.local:4222 + connection_name: vector-transform + subject: "logs.>" + jetstream: + stream: LOGS + consumer: transform + auth: + strategy: user_password + user_password: + user: log-consumer + password: ${NATS_CONSUMER_PASSWORD} decoding: codec: json - framing: - method: newline_delimited transforms: - # ---- Default normalisation into the logs.raw columns. This is intentionally - # the ONLY shaping today; per-app parsing is added here as follow-ups. ---- + route: + type: route + inputs: + - js_in + route: + k8s: 'starts_with(to_string(.subject) ?? "", "logs.k8s.")' + vm: 'starts_with(to_string(.subject) ?? "", "logs.vm.")' + k8s_shape: type: remap inputs: - - from_agents + - route.k8s source: | ts = .timestamp || now() node = to_string(.kubernetes.pod_node_name || "") ?? "" @@ -64,10 +69,11 @@ transforms: "labels": lbls, "fields": {} } + vm_shape: type: remap inputs: - - vm_http + - route.vm source: | ts = .timestamp || .ts || now() host = to_string(.host || .hostname || "") ?? "" @@ -107,16 +113,14 @@ sinks: strategy: basic user: "${CLICKHOUSE_USER}" password: "${CLICKHOUSE_PASSWORD}" - # Fat, infrequent inserts keep ClickHouse part-count low (avoid too-many-parts). batch: max_events: 500000 max_bytes: 134217728 timeout_secs: 10 - # Ride out a ClickHouse outage without dropping logs: on-disk buffer on the - # aggregator PVC; block upstream (back-pressure to the edge) when full. + # Small local buffer — JetStream is the real outage buffer now. buffer: type: disk - max_size: 8589934592 + max_size: 2147483648 when_full: block healthcheck: enabled: true diff --git a/apps/base/logging/vector/archiver.yaml b/apps/base/logging/vector/archiver.yaml new file mode 100644 index 0000000..c7347b2 --- /dev/null +++ b/apps/base/logging/vector/archiver.yaml @@ -0,0 +1,62 @@ +--- +# Vector ARCHIVER tier — long-term raw-log backup to S3 (Ceph RGW). Independent +# durable JetStream consumer (`archiver`) so its offsets/lag are fully isolated +# from the ClickHouse transform path (archive lag can never stall ingest — true +# fan-out). Writes RAW, pre-transform events (as they sit in JetStream) as +# gzipped NDJSON, partitioned by subject + date. This is the long-horizon replay +# source beyond JetStream's 72h retention window. +data_dir: /vector-data-dir + +api: + enabled: true + address: 0.0.0.0:8686 + +sources: + js_archive: + type: nats + url: nats://nats.logging.svc.cluster.local:4222 + connection_name: vector-archiver + subject: "logs.>" + jetstream: + stream: LOGS + consumer: archiver + auth: + strategy: user_password + user_password: + user: log-consumer + password: ${NATS_CONSUMER_PASSWORD} + decoding: + codec: json + +sinks: + s3: + type: aws_s3 + inputs: + - js_archive + bucket: logs-archive + endpoint: https://s3.ceph.unkin.net + region: us-east-1 + force_path_style: true + tls: + ca_file: /etc/vault-ca/ca.crt + # AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY come from the logs-archive-s3 + # Secret (cephrgw-operator) via envFrom on the deployment. + key_prefix: "raw/{{ subject }}/%Y/%m/%d/" + compression: gzip + encoding: + codec: json + framing: + method: newline_delimited + filename_time_format: "%Y%m%dT%H%M%SZ" + filename_append_uuid: true + batch: + max_bytes: 134217728 + timeout_secs: 300 + buffer: + type: memory + max_events: 5000 + when_full: block + # Disabled so slow BucketAccess credential propagation doesn't crash-loop + # the pod; RGW reachability is proven by the operator's own health. + healthcheck: + enabled: false diff --git a/apps/base/logging/vector/vm-ingest-tests.yaml b/apps/base/logging/vector/vm-ingest-tests.yaml new file mode 100644 index 0000000..fed2928 --- /dev/null +++ b/apps/base/logging/vector/vm-ingest-tests.yaml @@ -0,0 +1,29 @@ +--- +# `vector test` unit tests for the VM-ingest routing transform. +tests: + - name: host_token_is_subject_safe + inputs: + - insert_at: tag + type: log + log_fields: + host: "db1.syd1.example.net" + message: "sshd accepted" + outputs: + - extract_from: tag + conditions: + - type: vrl + source: | + assert_eq!(.host_token, "db1_syd1_example_net") + + - name: missing_host_defaults_to_unknown + inputs: + - insert_at: tag + type: log + log_fields: + message: "no host field" + outputs: + - extract_from: tag + conditions: + - type: vrl + source: | + assert_eq!(.host_token, "unknown") diff --git a/apps/base/logging/vector/vm-ingest.yaml b/apps/base/logging/vector/vm-ingest.yaml new file mode 100644 index 0000000..8e2b069 --- /dev/null +++ b/apps/base/logging/vector/vm-ingest.yaml @@ -0,0 +1,50 @@ +--- +# Vector VM-INGEST tier — the VM front door. Thin: accepts NDJSON over HTTPS +# (behind the logs-ingest Gateway) from puppet-managed VMs, attaches only a +# routing token, and publishes into JetStream (subject logs.vm.). No +# parsing here — shaping happens in the transform tier after JetStream, so VM +# logs get the same durability/replay/fan-out as k8s logs. +data_dir: /vector-data-dir + +api: + enabled: true + address: 0.0.0.0:8686 + +sources: + vm_http: + type: http_server + address: 0.0.0.0:8080 + path: / + method: POST + decoding: + codec: json + framing: + method: newline_delimited + +transforms: + # Routing metadata only: derive a NATS-subject-safe host token. + tag: + type: remap + inputs: + - vm_http + source: | + host = to_string(.host || .hostname || "unknown") ?? "unknown" + .host_token = replace(host, r'[^a-zA-Z0-9_-]', "_") + +sinks: + to_jetstream: + type: nats + inputs: + - tag + url: nats://nats.logging.svc.cluster.local:4222 + connection_name: vector-vm-ingest + subject: "logs.vm.{{ host_token }}" + jetstream: + enabled: true + auth: + strategy: user_password + user_password: + user: log-producer + password: ${NATS_PRODUCER_PASSWORD} + encoding: + codec: json diff --git a/apps/overlays/au-syd1/logging/kustomization.yaml b/apps/overlays/au-syd1/logging/kustomization.yaml index 0ae2aa6..a81c658 100644 --- a/apps/overlays/au-syd1/logging/kustomization.yaml +++ b/apps/overlays/au-syd1/logging/kustomization.yaml @@ -8,18 +8,38 @@ resources: - ../../../base/logging helmCharts: - # Node-level agent: tails every pod's logs and forwards to the aggregator. + # Dedicated JetStream NATS cluster — the durable log bus. + - name: nats + repo: https://nats-io.github.io/k8s/helm/charts + version: "2.14.2" + releaseName: nats + namespace: logging + valuesFile: values-nats.yaml + # Edge agent (DaemonSet): tails every pod's logs, publishes to JetStream. - name: vector repo: https://helm.vector.dev version: "0.57.0" releaseName: vector-agent namespace: logging valuesFile: values-vector-agent.yaml - # Aggregator: receives from agents (vector proto) and from VMs (HTTP), - # shapes events, and writes to ClickHouse. + # VM ingest (Deployment): HTTP NDJSON front door -> JetStream. + - name: vector + repo: https://helm.vector.dev + version: "0.57.0" + releaseName: vector-vm-ingest + namespace: logging + valuesFile: values-vector-vm-ingest.yaml + # Transform tier (StatefulSet): JetStream consumer -> shape -> ClickHouse. - name: vector repo: https://helm.vector.dev version: "0.57.0" releaseName: vector-aggregator namespace: logging valuesFile: values-vector-aggregator.yaml + # Archiver (Deployment): independent JetStream consumer -> raw logs to S3. + - name: vector + repo: https://helm.vector.dev + version: "0.57.0" + releaseName: vector-archiver + namespace: logging + valuesFile: values-vector-archiver.yaml diff --git a/apps/overlays/au-syd1/logging/values-nats.yaml b/apps/overlays/au-syd1/logging/values-nats.yaml new file mode 100644 index 0000000..f6f14e9 --- /dev/null +++ b/apps/overlays/au-syd1/logging/values-nats.yaml @@ -0,0 +1,83 @@ +# Dedicated JetStream-enabled NATS cluster for the log bus. Deliberately NOT +# shared with app messaging (streamstack et al. run their own NATS in their own +# repo) — a separate cluster isolates logging blast-radius from app messaging +# and lets us size retention/storage purely for the log outage-buffer + replay +# use-case. +fullnameOverride: nats + +config: + cluster: + enabled: true + replicas: 3 + jetstream: + enabled: true + fileStore: + pvc: + size: 50Gi + storageClassName: cephrbd-fast-delete + # Per-user auth with publish/subscribe separation. Passwords are injected as + # env vars from the Vault-synced nats-auth Secret (NATS expands $VAR in config). + merge: + authorization: + users: + # Bootstrap Job (stream/consumer management) — full JetStream API. + - user: log-admin + password: $NATS_ADMIN_PASSWORD + # Edge publishers (k8s DaemonSet + VM ingest) — publish only. + - user: log-producer + password: $NATS_PRODUCER_PASSWORD + permissions: + publish: + allow: + - "logs.>" + subscribe: + allow: + - "_INBOX.>" + # Consumers (transform tier + archiver) — pull + ack only, no publish + # to log subjects. + - user: log-consumer + password: $NATS_CONSUMER_PASSWORD + permissions: + publish: + allow: + - "$JS.API.CONSUMER.>" + - "$JS.API.STREAM.INFO.>" + - "$JS.ACK.LOGS.>" + subscribe: + allow: + - "_INBOX.>" + +container: + env: + NATS_ADMIN_PASSWORD: + valueFrom: + secretKeyRef: + name: nats-auth + key: admin_password + NATS_PRODUCER_PASSWORD: + valueFrom: + secretKeyRef: + name: nats-auth + key: producer_password + NATS_CONSUMER_PASSWORD: + valueFrom: + secretKeyRef: + name: nats-auth + key: consumer_password + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 4Gi + +# Roll the StatefulSet when nats-auth changes. +podTemplate: + merge: + metadata: + annotations: + reloader.stakater.com/auto: "true" + +natsBox: + enabled: false diff --git a/apps/overlays/au-syd1/logging/values-vector-agent.yaml b/apps/overlays/au-syd1/logging/values-vector-agent.yaml index 3e8e064..e9551e7 100644 --- a/apps/overlays/au-syd1/logging/values-vector-agent.yaml +++ b/apps/overlays/au-syd1/logging/values-vector-agent.yaml @@ -1,7 +1,7 @@ -# Vector Agent (DaemonSet) — captures ALL pod logs on every node (including -# control-plane, via the blanket toleration) and forwards them to the -# aggregator over the Vector native protocol. No per-app parsing here: shaping -# and the ClickHouse write live on the aggregator. +# Vector EDGE agent (DaemonSet) — thin publisher. Tails every node's pod logs +# (incl. control-plane via the blanket toleration) and publishes them into +# JetStream over the Vector NATS sink. No parsing; only a routing subject token +# is attached. Shaping happens in the transform tier after JetStream. role: Agent fullnameOverride: vector-agent @@ -13,10 +13,16 @@ serviceAccount: podLabels: vector.dev/exclude: "true" -# Run on every node so no host's pod logs are missed. tolerations: - operator: Exists +env: + - name: NATS_PRODUCER_PASSWORD + valueFrom: + secretKeyRef: + name: nats-auth + key: producer_password + resources: requests: cpu: 100m @@ -25,20 +31,14 @@ resources: cpu: "1" memory: 1Gi -# Disable the chart's auto-generated service for the agent (it ships, not serves). service: enabled: false -customConfig: - data_dir: /vector-data-dir - api: - enabled: false - sources: - kubernetes_logs: - type: kubernetes_logs - sinks: - to_aggregator: - type: vector - inputs: - - kubernetes_logs - address: vector-aggregator.logging.svc.cluster.local:6000 +# Pipeline is the single source of truth in apps/base/logging/vector/agent.yaml, +# mounted via existingConfigMaps (avoids the chart's customConfig Helm-tpl pass). +dataDir: /vector-data-dir +existingConfigMaps: + - vector-agent-config + +workloadResourceAnnotations: + reloader.stakater.com/auto: "true" diff --git a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml index 681d21f..6f6667b 100644 --- a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml +++ b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml @@ -1,38 +1,33 @@ -# Vector AGGREGATOR (StatefulSet) — the "brain" tier: sole ClickHouse writer, -# owns all transforms, holds the only ClickHouse credentials, HA replicas with -# on-disk buffers to ride out a ClickHouse outage. +# Vector TRANSFORM tier (StatefulSet) — the "brain": sole ClickHouse writer, +# owns all transforms, holds the only ClickHouse + NATS-consumer credentials. +# It is a pure JetStream pull consumer (no inbound ports) — durability lives in +# JetStream, so the local disk buffer is small (5Gi PVC / 2GiB buffer). # -# The pipeline itself is NOT inlined here: it lives in -# apps/base/logging/vector/aggregator.yaml (single source of truth, unit-tested -# by `vector test` in CI) and is mounted via existingConfigMaps. This file only -# owns the k8s deployment shape (ports, creds, storage, HA). +# Pipeline is the single source of truth in apps/base/logging/vector/ +# aggregator.yaml (unit-tested by `vector test` in CI), mounted via +# existingConfigMaps. role: Aggregator fullnameOverride: vector-aggregator replicas: 2 -# Reload the StatefulSet when the aggregator ConfigMap or the creds Secret change. workloadResourceAnnotations: reloader.stakater.com/auto: "true" podLabels: vector.dev/exclude: "true" -# Mount the aggregator pipeline ConfigMap (rendered from base by kustomize). -# dataDir must be set when using existingConfigMaps; it is also where the -# clickhouse sink's disk buffer lives (backed by the PVC below). dataDir: /vector-data-dir existingConfigMaps: - vector-aggregator-config -# On-disk buffer storage so a ClickHouse outage does not drop logs. persistence: enabled: true storageClassName: cephrbd-fast-delete - size: 20Gi + size: 5Gi accessModes: - ReadWriteOnce -# ClickHouse basic-auth creds — the ONLY place these are consumed. +# The ONLY place ClickHouse + NATS-consumer creds are consumed. env: - name: CLICKHOUSE_USER valueFrom: @@ -44,14 +39,14 @@ env: secretKeyRef: name: clickhouse-credentials key: password + - name: NATS_CONSUMER_PASSWORD + valueFrom: + secretKeyRef: + name: nats-auth + key: consumer_password +# Pure consumer: expose only the Vector API for debugging. containerPorts: - - name: vector - containerPort: 6000 - protocol: TCP - - name: http-ingest - containerPort: 8080 - protocol: TCP - name: api containerPort: 8686 protocol: TCP @@ -60,14 +55,6 @@ service: enabled: true type: ClusterIP ports: - - name: vector - port: 6000 - targetPort: 6000 - protocol: TCP - - name: http-ingest - port: 8080 - targetPort: 8080 - protocol: TCP - name: api port: 8686 targetPort: 8686 diff --git a/apps/overlays/au-syd1/logging/values-vector-archiver.yaml b/apps/overlays/au-syd1/logging/values-vector-archiver.yaml new file mode 100644 index 0000000..c7c13c8 --- /dev/null +++ b/apps/overlays/au-syd1/logging/values-vector-archiver.yaml @@ -0,0 +1,52 @@ +# Vector ARCHIVER tier (Deployment) — independent JetStream consumer writing raw +# logs to S3 (Ceph RGW). Isolated from the ClickHouse path (own durable +# consumer). Pipeline: apps/base/logging/vector/archiver.yaml. +role: Stateless-Aggregator +fullnameOverride: vector-archiver +replicas: 1 + +workloadResourceAnnotations: + reloader.stakater.com/auto: "true" + +podLabels: + vector.dev/exclude: "true" + +dataDir: /vector-data-dir +existingConfigMaps: + - vector-archiver-config + +env: + - name: NATS_CONSUMER_PASSWORD + valueFrom: + secretKeyRef: + name: nats-auth + key: consumer_password + +# S3 creds (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY) from the cephrgw-operator +# BucketAccess Secret. +envFrom: + - secretRef: + name: logs-archive-s3 + +# Trust the internal unkin.net Vault-PKI CA to verify s3.ceph.unkin.net. +# vault-ca-cert is reflected into every namespace from the certificates ns. +extraVolumes: + - name: vault-ca-cert + secret: + secretName: vault-ca-cert +extraVolumeMounts: + - name: vault-ca-cert + mountPath: /etc/vault-ca/ca.crt + subPath: ca.crt + readOnly: true + +service: + enabled: false + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi diff --git a/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml b/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml new file mode 100644 index 0000000..39b5df5 --- /dev/null +++ b/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml @@ -0,0 +1,52 @@ +# Vector VM-INGEST tier (Deployment) — VM front door. HTTP NDJSON in (behind the +# logs-ingest Gateway), publishes into JetStream. Stateless publisher. +# Pipeline: apps/base/logging/vector/vm-ingest.yaml (unit-tested in CI). +role: Stateless-Aggregator +fullnameOverride: vector-vm-ingest +replicas: 2 + +workloadResourceAnnotations: + reloader.stakater.com/auto: "true" + +podLabels: + vector.dev/exclude: "true" + +dataDir: /vector-data-dir +existingConfigMaps: + - vector-vm-ingest-config + +env: + - name: NATS_PRODUCER_PASSWORD + valueFrom: + secretKeyRef: + name: nats-auth + key: producer_password + +containerPorts: + - name: http-ingest + containerPort: 8080 + protocol: TCP + - name: api + containerPort: 8686 + protocol: TCP + +service: + enabled: true + type: ClusterIP + ports: + - name: http-ingest + port: 8080 + targetPort: 8080 + protocol: TCP + - name: api + port: 8686 + targetPort: 8686 + protocol: TCP + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi diff --git a/schemas/ceph.unkin.net/bucket_v1alpha1.json b/schemas/ceph.unkin.net/bucket_v1alpha1.json new file mode 100644 index 0000000..37cdf6f --- /dev/null +++ b/schemas/ceph.unkin.net/bucket_v1alpha1.json @@ -0,0 +1,176 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "apiVersion": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "metadata": { + "type": "object" + }, + "spec": { + "properties": { + "bucketName": { + "type": "string" + }, + "managePolicy": { + "default": true, + "type": "boolean" + }, + "objectLock": { + "properties": { + "days": { + "format": "int32", + "type": "integer" + }, + "enabled": { + "type": "boolean" + }, + "mode": { + "enum": [ + "GOVERNANCE", + "COMPLIANCE" + ], + "type": "string" + }, + "years": { + "format": "int32", + "type": "integer" + } + }, + "required": [ + "enabled" + ], + "type": "object" + }, + "ownerRef": { + "type": "string" + }, + "placementTarget": { + "type": "string" + }, + "purgeOnDelete": { + "type": "boolean" + }, + "quota": { + "properties": { + "enabled": { + "default": true, + "type": "boolean" + }, + "maxObjects": { + "format": "int64", + "type": "integer" + }, + "maxSizeBytes": { + "format": "int64", + "type": "integer" + } + }, + "type": "object" + }, + "retainOnDelete": { + "type": "boolean" + }, + "tags": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "versioning": { + "type": "boolean" + }, + "zonegroup": { + "type": "string" + } + }, + "required": [ + "ownerRef" + ], + "type": "object" + }, + "status": { + "properties": { + "adopted": { + "type": "boolean" + }, + "bucketID": { + "type": "string" + }, + "bucketName": { + "type": "string" + }, + "conditions": { + "items": { + "properties": { + "lastTransitionTime": { + "format": "date-time", + "type": "string" + }, + "message": { + "maxLength": 32768, + "type": "string" + }, + "observedGeneration": { + "format": "int64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "maxLength": 1024, + "minLength": 1, + "pattern": "^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$", + "type": "string" + }, + "status": { + "enum": [ + "True", + "False", + "Unknown" + ], + "type": "string" + }, + "type": { + "maxLength": 316, + "pattern": "^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$", + "type": "string" + } + }, + "required": [ + "lastTransitionTime", + "message", + "reason", + "status", + "type" + ], + "type": "object" + }, + "type": "array", + "x-kubernetes-list-map-keys": [ + "type" + ], + "x-kubernetes-list-type": "map" + }, + "observedGeneration": { + "format": "int64", + "type": "integer" + }, + "owner": { + "type": "string" + }, + "phase": { + "type": "string" + }, + "policyPrincipals": { + "format": "int32", + "type": "integer" + } + }, + "type": "object" + } + }, + "type": "object" +} diff --git a/schemas/ceph.unkin.net/bucketaccess_v1alpha1.json b/schemas/ceph.unkin.net/bucketaccess_v1alpha1.json new file mode 100644 index 0000000..9e783e2 --- /dev/null +++ b/schemas/ceph.unkin.net/bucketaccess_v1alpha1.json @@ -0,0 +1,191 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "apiVersion": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "metadata": { + "type": "object" + }, + "spec": { + "properties": { + "actions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "bucketRef": { + "type": "string" + }, + "conditions": { + "properties": { + "secureTransportOnly": { + "type": "boolean" + }, + "sourceIPs": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "level": { + "enum": [ + "read-only", + "read-write", + "full" + ], + "type": "string" + }, + "paths": { + "items": { + "type": "string" + }, + "type": "array" + }, + "rawStatements": { + "items": { + "properties": { + "actions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "conditions": { + "additionalProperties": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": "object" + }, + "type": "object" + }, + "effect": { + "default": "Allow", + "enum": [ + "Allow", + "Deny" + ], + "type": "string" + }, + "resources": { + "items": { + "type": "string" + }, + "type": "array" + }, + "sid": { + "type": "string" + } + }, + "required": [ + "actions" + ], + "type": "object" + }, + "type": "array" + }, + "retainOnDelete": { + "type": "boolean" + }, + "secretName": { + "type": "string" + }, + "uid": { + "type": "string" + }, + "userRef": { + "type": "string" + } + }, + "required": [ + "bucketRef", + "level" + ], + "type": "object" + }, + "status": { + "properties": { + "bound": { + "type": "boolean" + }, + "conditions": { + "items": { + "properties": { + "lastTransitionTime": { + "format": "date-time", + "type": "string" + }, + "message": { + "maxLength": 32768, + "type": "string" + }, + "observedGeneration": { + "format": "int64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "maxLength": 1024, + "minLength": 1, + "pattern": "^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$", + "type": "string" + }, + "status": { + "enum": [ + "True", + "False", + "Unknown" + ], + "type": "string" + }, + "type": { + "maxLength": 316, + "pattern": "^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$", + "type": "string" + } + }, + "required": [ + "lastTransitionTime", + "message", + "reason", + "status", + "type" + ], + "type": "object" + }, + "type": "array", + "x-kubernetes-list-map-keys": [ + "type" + ], + "x-kubernetes-list-type": "map" + }, + "observedGeneration": { + "format": "int64", + "type": "integer" + }, + "phase": { + "type": "string" + }, + "secretName": { + "type": "string" + }, + "uid": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" +} diff --git a/schemas/ceph.unkin.net/objectstoreuser_v1alpha1.json b/schemas/ceph.unkin.net/objectstoreuser_v1alpha1.json new file mode 100644 index 0000000..3648da2 --- /dev/null +++ b/schemas/ceph.unkin.net/objectstoreuser_v1alpha1.json @@ -0,0 +1,132 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "apiVersion": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "metadata": { + "type": "object" + }, + "spec": { + "properties": { + "displayName": { + "type": "string" + }, + "email": { + "type": "string" + }, + "maxBuckets": { + "default": 1000, + "format": "int32", + "type": "integer" + }, + "quota": { + "properties": { + "enabled": { + "default": true, + "type": "boolean" + }, + "maxObjects": { + "format": "int64", + "type": "integer" + }, + "maxSizeBytes": { + "format": "int64", + "type": "integer" + } + }, + "type": "object" + }, + "retainOnDelete": { + "type": "boolean" + }, + "secretName": { + "type": "string" + }, + "suspended": { + "type": "boolean" + }, + "uid": { + "type": "string" + } + }, + "type": "object" + }, + "status": { + "properties": { + "adopted": { + "type": "boolean" + }, + "conditions": { + "items": { + "properties": { + "lastTransitionTime": { + "format": "date-time", + "type": "string" + }, + "message": { + "maxLength": 32768, + "type": "string" + }, + "observedGeneration": { + "format": "int64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "maxLength": 1024, + "minLength": 1, + "pattern": "^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$", + "type": "string" + }, + "status": { + "enum": [ + "True", + "False", + "Unknown" + ], + "type": "string" + }, + "type": { + "maxLength": 316, + "pattern": "^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$", + "type": "string" + } + }, + "required": [ + "lastTransitionTime", + "message", + "reason", + "status", + "type" + ], + "type": "object" + }, + "type": "array", + "x-kubernetes-list-map-keys": [ + "type" + ], + "x-kubernetes-list-type": "map" + }, + "observedGeneration": { + "format": "int64", + "type": "integer" + }, + "phase": { + "type": "string" + }, + "secretName": { + "type": "string" + }, + "uid": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" +} -- 2.47.3 From 1202aae06fc19e2d1eba75a4c76ab1e09fdefee2 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Mon, 27 Jul 2026 21:19:44 +1000 Subject: [PATCH 3/6] Pull images via artifactapi; make transform tier stateless MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes from review: 1. Pull every container image through the artifactapi dockerhub remote instead of direct upstream: clickhouse-server, altinity operator + metrics-exporter, bitnami/kubectl (crdHook), nats + nats-server-config-reloader, nats-box (bootstrap Job), and vector (all tiers + the CI image). Requires terraform-artifactapi#16 (dockerhub allowlist patterns) merged first. 2. Keep upstream official images (no Docker Hardened Images). DHI exists for clickhouse-server and vector but is subscription-gated and served from a private org namespace not reachable via the anonymous artifactapi dockerhub proxy; its shell-less images would also break the bash bootstrap Jobs and the shell-based vector-test CI step. Use vector's distroless-libc for runtime pods (near-hardened) and the debian variant only for CI. 3. Make the transform tier a stateless Deployment (was a StatefulSet): no PVC, no disk buffer — JetStream is the sole durability layer. The ClickHouse sink uses an in-memory block buffer so a ClickHouse outage back-pressures the JetStream pull source (unpulled messages are retained/redelivered). Add a CPU HPA (2-8) — safe because JetStream pull consumers distribute work across N replicas on the one durable consumer. Caveat documented: vector's NATS source has no end-to-end acks (acks on receipt), so a pod killed mid-outage can lose its in-memory buffer window; accepted trade for a stateless autoscaling tier. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv --- .woodpecker/vector-test.yaml | 2 +- apps/base/logging/clickhouseinstallation.yaml | 2 +- apps/base/logging/job_clickhouse-schema.yaml | 2 +- apps/base/logging/nats-bootstrap-job.yaml | 2 +- apps/base/logging/vector/aggregator.yaml | 18 +++++--- .../au-syd1/clickhouse-system/values.yaml | 11 +++++ .../overlays/au-syd1/logging/values-nats.yaml | 11 +++++ .../au-syd1/logging/values-vector-agent.yaml | 6 +++ .../logging/values-vector-aggregator.yaml | 43 ++++++++++++------- .../logging/values-vector-archiver.yaml | 4 ++ .../logging/values-vector-vm-ingest.yaml | 4 ++ 11 files changed, 78 insertions(+), 27 deletions(-) diff --git a/.woodpecker/vector-test.yaml b/.woodpecker/vector-test.yaml index 8949ce3..aace88e 100644 --- a/.woodpecker/vector-test.yaml +++ b/.woodpecker/vector-test.yaml @@ -3,7 +3,7 @@ when: steps: - name: vector-test - image: timberio/vector:0.57.0-debian + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector:0.57.0-debian commands: # Dummy creds + writable dirs so the full topologies build; the unit tests # only exercise the transforms (sources are not started). diff --git a/apps/base/logging/clickhouseinstallation.yaml b/apps/base/logging/clickhouseinstallation.yaml index ea6bb44..6c82e59 100644 --- a/apps/base/logging/clickhouseinstallation.yaml +++ b/apps/base/logging/clickhouseinstallation.yaml @@ -66,7 +66,7 @@ spec: runAsGroup: 101 containers: - name: clickhouse - image: clickhouse/clickhouse-server:24.8 + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/clickhouse/clickhouse-server:24.8 resources: requests: cpu: 500m diff --git a/apps/base/logging/job_clickhouse-schema.yaml b/apps/base/logging/job_clickhouse-schema.yaml index b1e3dd4..1e0f801 100644 --- a/apps/base/logging/job_clickhouse-schema.yaml +++ b/apps/base/logging/job_clickhouse-schema.yaml @@ -32,7 +32,7 @@ spec: runAsGroup: 101 containers: - name: clickhouse-schema - image: clickhouse/clickhouse-server:24.8 + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/clickhouse/clickhouse-server:24.8 securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/apps/base/logging/nats-bootstrap-job.yaml b/apps/base/logging/nats-bootstrap-job.yaml index 72e5ad9..a21b5d2 100644 --- a/apps/base/logging/nats-bootstrap-job.yaml +++ b/apps/base/logging/nats-bootstrap-job.yaml @@ -49,7 +49,7 @@ spec: runAsGroup: 1000 containers: - name: nats-bootstrap - image: natsio/nats-box:0.18.0 + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/natsio/nats-box:0.18.0 securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/apps/base/logging/vector/aggregator.yaml b/apps/base/logging/vector/aggregator.yaml index ad35505..03bb18a 100644 --- a/apps/base/logging/vector/aggregator.yaml +++ b/apps/base/logging/vector/aggregator.yaml @@ -7,10 +7,13 @@ # insert a transform and append its id to the clickhouse sink `inputs` — no edge # or VM rollout required. # -# Durability model: JetStream (72h / 40GiB) is the outage buffer. If ClickHouse -# is down the sink blocks, back-pressure stops acking, and JetStream retains -# messages for replay. The local disk buffer is small (survives pod restarts of -# in-flight events only). +# Durability model: JetStream (72h / 40GiB) is the SOLE durability layer. This +# tier is stateless (no PVC, memory buffer). If ClickHouse is down the sink +# blocks (when_full=block); back-pressure stops the source pulling, so unpulled +# messages stay in JetStream and are redelivered. NB: Vector's NATS source has +# no end-to-end acks (acks on receipt), so a pod killed mid-outage can lose the +# in-memory buffer's worth of already-pulled events — accepted for a stateless, +# autoscalable tier. data_dir: /vector-data-dir api: @@ -117,10 +120,11 @@ sinks: max_events: 500000 max_bytes: 134217728 timeout_secs: 10 - # Small local buffer — JetStream is the real outage buffer now. + # Stateless: in-memory buffer, block on full so back-pressure reaches the + # JetStream pull source (which then stops acking). JetStream is durability. buffer: - type: disk - max_size: 2147483648 + type: memory + max_events: 2000 when_full: block healthcheck: enabled: true diff --git a/apps/overlays/au-syd1/clickhouse-system/values.yaml b/apps/overlays/au-syd1/clickhouse-system/values.yaml index 8eb31f0..ee144fc 100644 --- a/apps/overlays/au-syd1/clickhouse-system/values.yaml +++ b/apps/overlays/au-syd1/clickhouse-system/values.yaml @@ -1,7 +1,14 @@ # Altinity ClickHouse operator. Cluster-scoped: watches ClickHouseInstallation # resources in all namespaces (the logs cluster lives in the `logging` namespace). # CRDs are installed at runtime by the chart's crdHook Job. +# +# All images are pulled through the artifactapi dockerhub remote (no direct +# upstream). Upstream official images are used; no Docker Hardened Image variant +# is adopted (DHI is subscription-gated and served from a private org namespace +# not reachable via the anonymous artifactapi dockerhub proxy). crdHook: + image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/bitnami/kubectl resources: requests: cpu: 50m @@ -11,6 +18,8 @@ crdHook: memory: 128Mi operator: + image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/altinity/clickhouse-operator resources: requests: cpu: 100m @@ -20,6 +29,8 @@ operator: memory: 512Mi metrics: + image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/altinity/metrics-exporter resources: requests: cpu: 50m diff --git a/apps/overlays/au-syd1/logging/values-nats.yaml b/apps/overlays/au-syd1/logging/values-nats.yaml index f6f14e9..8e70444 100644 --- a/apps/overlays/au-syd1/logging/values-nats.yaml +++ b/apps/overlays/au-syd1/logging/values-nats.yaml @@ -48,6 +48,11 @@ config: - "_INBOX.>" container: + # Pulled through the artifactapi dockerhub remote (upstream official nats; + # no DHI variant available for nats). + image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/nats + tag: 2.14.2-alpine env: NATS_ADMIN_PASSWORD: valueFrom: @@ -79,5 +84,11 @@ podTemplate: annotations: reloader.stakater.com/auto: "true" +# Config-reloader sidecar image, also through artifactapi. +reloader: + image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/natsio/nats-server-config-reloader + tag: "0.23.0" + natsBox: enabled: false diff --git a/apps/overlays/au-syd1/logging/values-vector-agent.yaml b/apps/overlays/au-syd1/logging/values-vector-agent.yaml index e9551e7..bff67a5 100644 --- a/apps/overlays/au-syd1/logging/values-vector-agent.yaml +++ b/apps/overlays/au-syd1/logging/values-vector-agent.yaml @@ -5,6 +5,12 @@ role: Agent fullnameOverride: vector-agent +# Pulled through the artifactapi dockerhub remote; distroless-libc (no DHI — +# subscription-gated/private-namespace, not reachable via the anon proxy). +image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector + tag: 0.57.0-distroless-libc + rbac: create: true serviceAccount: diff --git a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml index 6f6667b..4ee0d14 100644 --- a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml +++ b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml @@ -1,14 +1,29 @@ -# Vector TRANSFORM tier (StatefulSet) — the "brain": sole ClickHouse writer, -# owns all transforms, holds the only ClickHouse + NATS-consumer credentials. -# It is a pure JetStream pull consumer (no inbound ports) — durability lives in -# JetStream, so the local disk buffer is small (5Gi PVC / 2GiB buffer). +# Vector TRANSFORM tier (STATELESS Deployment) — the "brain": sole ClickHouse +# writer, owns all transforms, holds the only ClickHouse + NATS-consumer creds. # -# Pipeline is the single source of truth in apps/base/logging/vector/ -# aggregator.yaml (unit-tested by `vector test` in CI), mounted via -# existingConfigMaps. -role: Aggregator +# Stateless by design: a JetStream pull consumer with NO PVC and NO disk buffer. +# JetStream is the sole durability layer. On a ClickHouse outage the clickhouse +# sink blocks (buffer when_full=block), back-pressure stops the source pulling, +# and unpulled messages stay in JetStream for redelivery. Because Vector's NATS +# source does NOT support end-to-end acknowledgements (it acks on receipt, not +# after the sink), the only at-risk window is the in-memory buffer's worth of +# already-pulled events if a pod is killed mid-outage — the accepted trade for a +# horizontally-autoscalable stateless tier. Multiple replicas share the one +# durable consumer `transform` (JetStream pull consumers distribute work), so +# HPA is safe. +role: Stateless-Aggregator fullnameOverride: vector-aggregator -replicas: 2 + +image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector + tag: 0.57.0-distroless-libc + +# Horizontal autoscaling on CPU — safe with N replicas on one durable consumer. +autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 8 + targetCPUUtilizationPercentage: 70 workloadResourceAnnotations: reloader.stakater.com/auto: "true" @@ -16,17 +31,13 @@ workloadResourceAnnotations: podLabels: vector.dev/exclude: "true" +# Pipeline is the single source of truth in apps/base/logging/vector/ +# aggregator.yaml (unit-tested by `vector test` in CI), mounted via +# existingConfigMaps. No persistence — stateless. dataDir: /vector-data-dir existingConfigMaps: - vector-aggregator-config -persistence: - enabled: true - storageClassName: cephrbd-fast-delete - size: 5Gi - accessModes: - - ReadWriteOnce - # The ONLY place ClickHouse + NATS-consumer creds are consumed. env: - name: CLICKHOUSE_USER diff --git a/apps/overlays/au-syd1/logging/values-vector-archiver.yaml b/apps/overlays/au-syd1/logging/values-vector-archiver.yaml index c7c13c8..671844a 100644 --- a/apps/overlays/au-syd1/logging/values-vector-archiver.yaml +++ b/apps/overlays/au-syd1/logging/values-vector-archiver.yaml @@ -5,6 +5,10 @@ role: Stateless-Aggregator fullnameOverride: vector-archiver replicas: 1 +image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector + tag: 0.57.0-distroless-libc + workloadResourceAnnotations: reloader.stakater.com/auto: "true" diff --git a/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml b/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml index 39b5df5..12fe8b7 100644 --- a/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml +++ b/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml @@ -5,6 +5,10 @@ role: Stateless-Aggregator fullnameOverride: vector-vm-ingest replicas: 2 +image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector + tag: 0.57.0-distroless-libc + workloadResourceAnnotations: reloader.stakater.com/auto: "true" -- 2.47.3 From b18e9a669f49226a9c6c7e6d3438decda710b550 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Mon, 27 Jul 2026 21:40:10 +1000 Subject: [PATCH 4/6] Move JetStream limits to a ConfigMap; 7d retention, honest sizing - Retention -> 7 days (max_age=168h), still retention=limits/discard=old so the transform tier and the archiver each independently see every message; reading never deletes. - Put the tunable stream limits (max_age, max_bytes, dupe_window) in the nats-stream-limits ConfigMap. The bootstrap Job reads them and does an idempotent create-or-UPDATE (nats stream add || nats stream edit), so changing the ConfigMap + re-sync applies new limits with no manual surgery. The ConfigMap keeps its kustomize content-hash suffix, so an edit renames it and rewrites the Job's env refs -> the PostSync hook Job's spec changes and Argo re-runs it (on top of hooks running each sync). Verified end-to-end against a real nats-server: create, idempotent re-run, and a max_age change all apply. - Honest 7d sizing: assume ~1500 events/s avg @ ~1 KiB/event; with S2 stream compression (~4x) that's ~33 GiB/day -> ~230 GiB/7d per replica. Enable S2 compression on the stream, set max_bytes=300 GiB, and raise the file-store PVC to 400Gi/node (3 replicas = 1.2 TiB) so the byte cap can't silently truncate retention below 7d. Numbers + assumptions flagged in the PR body. - Update runbook/comments: replay window is now 7d. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv --- apps/base/logging/kustomization.yaml | 13 ++++ apps/base/logging/nats-bootstrap-job.yaml | 59 ++++++++++++++----- apps/base/logging/vector/aggregator.yaml | 3 +- apps/base/logging/vector/archiver.yaml | 2 +- .../overlays/au-syd1/logging/values-nats.yaml | 8 ++- 5 files changed, 68 insertions(+), 17 deletions(-) diff --git a/apps/base/logging/kustomization.yaml b/apps/base/logging/kustomization.yaml index cf94855..0896d77 100644 --- a/apps/base/logging/kustomization.yaml +++ b/apps/base/logging/kustomization.yaml @@ -16,6 +16,19 @@ resources: # Vector pipelines are the single source of truth (also validated by # `vector test` in CI). Mounted into each tier via `existingConfigMaps`. configMapGenerator: + # Tunable JetStream stream limits (the nats-bootstrap Job reads these and does + # create-or-update). Hash suffix is INTENTIONALLY left on: editing a value + # renames the ConfigMap, which rewrites the Job's env reference, which changes + # the PostSync hook Job's spec and forces Argo to re-run it -> new limits apply. + # Sizing assumes ~1500 events/s avg @ ~1 KiB/event with S2 compression (~4x): + # ~33 GiB/day compressed -> ~230 GiB/7d per replica. max_bytes 300 GiB sits + # under the 400Gi/node PVC (see values-nats.yaml). Raising retention beyond the + # PVC requires bumping BOTH max_bytes here and fileStore PVC size in values. + - name: nats-stream-limits + literals: + - max_age=168h + - max_bytes=322122547200 + - dupe_window=2m - name: vector-agent-config files: - agent.yaml=vector/agent.yaml diff --git a/apps/base/logging/nats-bootstrap-job.yaml b/apps/base/logging/nats-bootstrap-job.yaml index a21b5d2..9ace885 100644 --- a/apps/base/logging/nats-bootstrap-job.yaml +++ b/apps/base/logging/nats-bootstrap-job.yaml @@ -1,13 +1,21 @@ --- # Declarative JetStream provisioning: the LOGS stream + durable consumers. -# ArgoCD PostSync hook, idempotent (add-or-converge), re-runs each sync. +# ArgoCD PostSync hook, idempotent create-or-UPDATE, re-runs each sync. # -# Stream LOGS: file storage, 3 replicas, retention=limits (NOT workqueue) so -# multiple durable consumers fan out and can independently replay within the -# window. Sized as the ClickHouse-outage buffer: 40 GiB / 72h (per-replica PVC -# is 50Gi, see values-nats.yaml). Beyond that window the S3 archive is the +# Stream LOGS: file storage, 3 replicas, retention=limits (NOT workqueue) so the +# transform tier AND the archiver each independently see every message — reading +# never deletes; only max-age/max-bytes do. S2 compression is on (logs compress +# well). Replay window = max-age (7d default). Beyond that, the S3 archive is the # long-term replay source. # +# TUNABLE LIMITS LIVE IN A CONFIGMAP (nats-stream-limits): max_age, max_bytes, +# dupe_window. Change the ConfigMap and re-sync — this Job re-runs and applies +# the new limits via `nats stream edit` (no manual surgery). The ConfigMap is +# generated with a content-hash suffix (kustomize), so editing it changes both +# the ConfigMap name AND this Job's env reference → the PostSync hook Job's spec +# changes and Argo re-runs it (belt-and-suspenders on top of hooks running each +# sync; hook-delete-policy=BeforeHookCreation recreates it every time). +# # Consumers (independent offsets = true fan-out): # transform -> whole log stream, feeds the ClickHouse transform tier # archiver -> configurable security-relevant subset, feeds the S3 archiver. @@ -16,9 +24,10 @@ # --filter). Exact default set is an open decision for Ben. # # Runbook (replay): -# (a) reprocess from JetStream (within 72h): scale the transform tier to 0, -# then `nats consumer rm LOGS transform` and re-run this Job (recreates at -# DeliverAll), or `nats consumer edit`/`--replay` from a start seq/time. +# (a) reprocess from JetStream (within max-age, 7d): scale the transform tier +# to 0, then `nats consumer rm LOGS transform` and re-run this Job +# (recreates at DeliverAll), or `nats consumer edit`/`--replay` from a +# start seq/time. # (b) long-horizon (beyond JetStream): re-ingest S3 archive objects back # through the transform tier (vector aws_s3 source or a one-shot Job). apiVersion: batch/v1 @@ -66,6 +75,22 @@ spec: secretKeyRef: name: nats-auth key: admin_password + # Tunable stream limits — sourced from the ConfigMap. + - name: MAX_AGE + valueFrom: + configMapKeyRef: + name: nats-stream-limits + key: max_age + - name: MAX_BYTES + valueFrom: + configMapKeyRef: + name: nats-stream-limits + key: max_bytes + - name: DUPE_WINDOW + valueFrom: + configMapKeyRef: + name: nats-stream-limits + key: dupe_window # Space-separated subject filters for the archiver consumer. - name: ARCHIVE_SUBJECTS value: "logs.k8s.vault.>" @@ -79,16 +104,22 @@ spec: echo " not ready, retry in 5s"; sleep 5 done - echo "Ensuring stream LOGS ..." + echo "Ensuring stream LOGS (max_age=$MAX_AGE max_bytes=$MAX_BYTES dupe=$DUPE_WINDOW) ..." + # Create if absent; otherwise converge the mutable limits from the + # ConfigMap. (storage/retention/replicas are immutable, set only on + # create.) nats stream add LOGS \ --subjects='logs.>' --storage=file --replicas=3 \ - --retention=limits --discard=old \ - --max-age=72h --max-bytes=42949672960 \ + --retention=limits --discard=old --compression=s2 \ + --max-age="$MAX_AGE" --max-bytes="$MAX_BYTES" \ --max-msgs=-1 --max-msgs-per-subject=-1 --max-msg-size=-1 \ - --max-consumers=-1 --dupe-window=2m --defaults 2>/dev/null \ + --max-consumers=-1 --dupe-window="$DUPE_WINDOW" --defaults 2>/dev/null \ + && echo " created" \ || nats stream edit -f LOGS \ - --subjects='logs.>' --discard=old \ - --max-age=72h --max-bytes=42949672960 --dupe-window=2m + --subjects='logs.>' --discard=old --compression=s2 \ + --max-age="$MAX_AGE" --max-bytes="$MAX_BYTES" \ + --max-msgs=-1 --max-msgs-per-subject=-1 --max-msg-size=-1 \ + --max-consumers=-1 --dupe-window="$DUPE_WINDOW" echo "Ensuring consumer transform ..." nats consumer add LOGS transform \ diff --git a/apps/base/logging/vector/aggregator.yaml b/apps/base/logging/vector/aggregator.yaml index 03bb18a..4944ef9 100644 --- a/apps/base/logging/vector/aggregator.yaml +++ b/apps/base/logging/vector/aggregator.yaml @@ -7,7 +7,8 @@ # insert a transform and append its id to the clickhouse sink `inputs` — no edge # or VM rollout required. # -# Durability model: JetStream (72h / 40GiB) is the SOLE durability layer. This +# Durability model: JetStream (7d / 300 GiB, S2-compressed) is the SOLE +# durability layer and the replay window. This # tier is stateless (no PVC, memory buffer). If ClickHouse is down the sink # blocks (when_full=block); back-pressure stops the source pulling, so unpulled # messages stay in JetStream and are redelivered. NB: Vector's NATS source has diff --git a/apps/base/logging/vector/archiver.yaml b/apps/base/logging/vector/archiver.yaml index c7347b2..075e34d 100644 --- a/apps/base/logging/vector/archiver.yaml +++ b/apps/base/logging/vector/archiver.yaml @@ -4,7 +4,7 @@ # from the ClickHouse transform path (archive lag can never stall ingest — true # fan-out). Writes RAW, pre-transform events (as they sit in JetStream) as # gzipped NDJSON, partitioned by subject + date. This is the long-horizon replay -# source beyond JetStream's 72h retention window. +# source beyond JetStream's 7d retention window. data_dir: /vector-data-dir api: diff --git a/apps/overlays/au-syd1/logging/values-nats.yaml b/apps/overlays/au-syd1/logging/values-nats.yaml index 8e70444..c5443cd 100644 --- a/apps/overlays/au-syd1/logging/values-nats.yaml +++ b/apps/overlays/au-syd1/logging/values-nats.yaml @@ -13,7 +13,13 @@ config: enabled: true fileStore: pvc: - size: 50Gi + # Sized for 7d retention: ~230 GiB/7d compressed (see + # nats-stream-limits ConfigMap) + file-store WAL/index/overhead, kept + # safely above the 300 GiB max_bytes cap. 3 replicas => 1.2 TiB total + # provisioned on cephrbd-fast-delete. NB: this is the honest number for + # the assumed ~1500 events/s; higher real volume needs a bigger PVC + + # max_bytes together, else discard=old truncates retention below 7d. + size: 400Gi storageClassName: cephrbd-fast-delete # Per-user auth with publish/subscribe separation. Passwords are injected as # env vars from the Vault-synced nats-auth Secret (NATS expands $VAR in config). -- 2.47.3 From c60a639563dc002b6d577bc9a2d4c74afabc0323 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Mon, 27 Jul 2026 22:13:20 +1000 Subject: [PATCH 5/6] ci: re-run vector-test after artifactapi allowlist applied terraform-artifactapi #16 is merged and the dockerhub allowlist now serves timberio/vector (verified 200); this empty commit re-triggers CI. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv -- 2.47.3 From 5ae14b3a38963e176a1518a7b9ed1125b66ad23b Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Tue, 28 Jul 2026 18:16:40 +1000 Subject: [PATCH 6/6] Reduce retention to 3 days (NATS + ClickHouse); shrink PVCs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ben: 1.2 TiB is too much. Cut both stores to 3d and resize honestly. - NATS LOGS stream max_age 168h -> 72h in the nats-stream-limits ConfigMap. At the stated ~33 GiB/day compressed (S2), 3d ~= 100 GiB, so max_bytes -> 130 GiB and the file-store PVC -> 180Gi/node. 3 replicas = ~0.5 TiB total (down from 1.2 TiB). - ClickHouse logs.raw TTL 30d -> 3d in the bootstrap DDL. At ~130 GiB/day raw, LZ4/ZSTD (~6x) stores ~20-25 GiB/day => ~60-75 GiB/3d; with merge headroom the CHI PVC -> 150Gi (from 200Gi). logs.raw is the only table. - Long-term retention now lives EXCLUSIVELY in the S3 archive (logarchiver) for the configured subjects; everything else is gone after 3 days — accepted design, documented in the runbook + PR body. - Update all 7d/30d comments and the replay window (now 3d). PVC shrink is a plan-time change: this stack is not deployed yet, so it is clean. Once deployed, PVCs cannot shrink in place (would need recreate). Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv --- apps/base/logging/clickhouseinstallation.yaml | 6 +++++- apps/base/logging/job_clickhouse-schema.yaml | 2 +- apps/base/logging/kustomization.yaml | 8 ++++---- apps/base/logging/nats-bootstrap-job.yaml | 6 +++--- apps/base/logging/vector/aggregator.yaml | 2 +- apps/base/logging/vector/archiver.yaml | 2 +- apps/overlays/au-syd1/logging/values-nats.yaml | 8 ++++---- 7 files changed, 19 insertions(+), 15 deletions(-) diff --git a/apps/base/logging/clickhouseinstallation.yaml b/apps/base/logging/clickhouseinstallation.yaml index 6c82e59..f6b63f0 100644 --- a/apps/base/logging/clickhouseinstallation.yaml +++ b/apps/base/logging/clickhouseinstallation.yaml @@ -46,7 +46,11 @@ spec: - ReadWriteOnce resources: requests: - storage: 200Gi + # 3d TTL on logs.raw. At ~130 GiB/day raw, ClickHouse LZ4/ZSTD + # (~6x on log text) stores ~20-25 GiB/day => ~60-75 GiB/3d, plus + # merge headroom (~2x peak). logs.raw is the only table. 150Gi + # gives comfortable headroom; long-term data lives in S3, not here. + storage: 150Gi serviceTemplates: - name: chi-service generateName: "clickhouse-{chi}" diff --git a/apps/base/logging/job_clickhouse-schema.yaml b/apps/base/logging/job_clickhouse-schema.yaml index 1e0f801..ff36675 100644 --- a/apps/base/logging/job_clickhouse-schema.yaml +++ b/apps/base/logging/job_clickhouse-schema.yaml @@ -86,7 +86,7 @@ spec: ENGINE = MergeTree PARTITION BY toDate(timestamp) ORDER BY (source, namespace, host, timestamp) - TTL toDateTime(timestamp) + INTERVAL 30 DAY + TTL toDateTime(timestamp) + INTERVAL 3 DAY SETTINGS index_granularity = 8192; EOSQL echo "Schema applied." diff --git a/apps/base/logging/kustomization.yaml b/apps/base/logging/kustomization.yaml index 0896d77..4fd721f 100644 --- a/apps/base/logging/kustomization.yaml +++ b/apps/base/logging/kustomization.yaml @@ -21,13 +21,13 @@ configMapGenerator: # renames the ConfigMap, which rewrites the Job's env reference, which changes # the PostSync hook Job's spec and forces Argo to re-run it -> new limits apply. # Sizing assumes ~1500 events/s avg @ ~1 KiB/event with S2 compression (~4x): - # ~33 GiB/day compressed -> ~230 GiB/7d per replica. max_bytes 300 GiB sits - # under the 400Gi/node PVC (see values-nats.yaml). Raising retention beyond the + # ~33 GiB/day compressed -> ~100 GiB/3d per replica. max_bytes 130 GiB sits + # under the 180Gi/node PVC (see values-nats.yaml). Raising retention beyond the # PVC requires bumping BOTH max_bytes here and fileStore PVC size in values. - name: nats-stream-limits literals: - - max_age=168h - - max_bytes=322122547200 + - max_age=72h + - max_bytes=139586437120 - dupe_window=2m - name: vector-agent-config files: diff --git a/apps/base/logging/nats-bootstrap-job.yaml b/apps/base/logging/nats-bootstrap-job.yaml index 9ace885..5e2fc68 100644 --- a/apps/base/logging/nats-bootstrap-job.yaml +++ b/apps/base/logging/nats-bootstrap-job.yaml @@ -5,8 +5,8 @@ # Stream LOGS: file storage, 3 replicas, retention=limits (NOT workqueue) so the # transform tier AND the archiver each independently see every message — reading # never deletes; only max-age/max-bytes do. S2 compression is on (logs compress -# well). Replay window = max-age (7d default). Beyond that, the S3 archive is the -# long-term replay source. +# well). Replay window = max-age (3d default). Beyond that, the S3 archive is the +# ONLY long-term source — everything else is gone after 3 days (accepted design). # # TUNABLE LIMITS LIVE IN A CONFIGMAP (nats-stream-limits): max_age, max_bytes, # dupe_window. Change the ConfigMap and re-sync — this Job re-runs and applies @@ -24,7 +24,7 @@ # --filter). Exact default set is an open decision for Ben. # # Runbook (replay): -# (a) reprocess from JetStream (within max-age, 7d): scale the transform tier +# (a) reprocess from JetStream (within max-age, 3d): scale the transform tier # to 0, then `nats consumer rm LOGS transform` and re-run this Job # (recreates at DeliverAll), or `nats consumer edit`/`--replay` from a # start seq/time. diff --git a/apps/base/logging/vector/aggregator.yaml b/apps/base/logging/vector/aggregator.yaml index 4944ef9..f22b849 100644 --- a/apps/base/logging/vector/aggregator.yaml +++ b/apps/base/logging/vector/aggregator.yaml @@ -7,7 +7,7 @@ # insert a transform and append its id to the clickhouse sink `inputs` — no edge # or VM rollout required. # -# Durability model: JetStream (7d / 300 GiB, S2-compressed) is the SOLE +# Durability model: JetStream (3d / 130 GiB, S2-compressed) is the SOLE # durability layer and the replay window. This # tier is stateless (no PVC, memory buffer). If ClickHouse is down the sink # blocks (when_full=block); back-pressure stops the source pulling, so unpulled diff --git a/apps/base/logging/vector/archiver.yaml b/apps/base/logging/vector/archiver.yaml index 075e34d..60b5102 100644 --- a/apps/base/logging/vector/archiver.yaml +++ b/apps/base/logging/vector/archiver.yaml @@ -4,7 +4,7 @@ # from the ClickHouse transform path (archive lag can never stall ingest — true # fan-out). Writes RAW, pre-transform events (as they sit in JetStream) as # gzipped NDJSON, partitioned by subject + date. This is the long-horizon replay -# source beyond JetStream's 7d retention window. +# source beyond JetStream's 3d retention window. data_dir: /vector-data-dir api: diff --git a/apps/overlays/au-syd1/logging/values-nats.yaml b/apps/overlays/au-syd1/logging/values-nats.yaml index c5443cd..956ebd2 100644 --- a/apps/overlays/au-syd1/logging/values-nats.yaml +++ b/apps/overlays/au-syd1/logging/values-nats.yaml @@ -13,13 +13,13 @@ config: enabled: true fileStore: pvc: - # Sized for 7d retention: ~230 GiB/7d compressed (see + # Sized for 3d retention: ~100 GiB/3d compressed (see # nats-stream-limits ConfigMap) + file-store WAL/index/overhead, kept - # safely above the 300 GiB max_bytes cap. 3 replicas => 1.2 TiB total + # safely above the 130 GiB max_bytes cap. 3 replicas => ~0.5 TiB total # provisioned on cephrbd-fast-delete. NB: this is the honest number for # the assumed ~1500 events/s; higher real volume needs a bigger PVC + - # max_bytes together, else discard=old truncates retention below 7d. - size: 400Gi + # max_bytes together, else discard=old truncates retention below 3d. + size: 180Gi storageClassName: cephrbd-fast-delete # Per-user auth with publish/subscribe separation. Passwords are injected as # env vars from the Vault-synced nats-auth Secret (NATS expands $VAR in config). -- 2.47.3