From 9c10b9096a463c2c30df20e3336884376196330e Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Wed, 29 Jul 2026 21:07:20 +1000 Subject: [PATCH] Replace vector-archiver with logarchiver (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why The Vector archiver leg wrote gzip NDJSON to S3 with no index or encryption. logarchiver replaces it with a Go service that seals raw logs to S3 as zstd + OpenPGP objects and indexes each object in ClickHouse (`logs.archive_index`), acking JetStream only after the object is stored and indexed. ## Changes - Add logarchiver Deployment (`git.unkin.net/unkin/logarchiver:v0.1.0`), ConfigMap, and dedicated ServiceAccount, reusing the archiver's NATS (`log-consumer` / durable `archiver` / `ARCHIVE_SUBJECTS=logs.k8s.vault.>`), S3 (`logs-archive-s3`), ClickHouse (`clickhouse-credentials`) and `vault-ca` wiring. - Encrypts to the `logarchive` gpg public key, fetched from the gpg engine via k8s auth (role `logging_logarchiver`, projected vault-audience token). `ack_wait` (5m) > batch `max_age` (2m) so messages aren't redelivered mid-batch. - Add `logs.archive_index` DDL to the clickhouse-schema bootstrap Job (no TTL — outlives `logs.raw`). - Remove the vector-archiver Helm release, values and pipeline ConfigMap. Cross-repo: apply **terraform-vault #106** (gpg key + role/policy) before this syncs, or the pod can't fetch the public key. Sequencing: apply after #306 (already merged). https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv --------- Co-authored-by: benvin Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/308 Co-authored-by: Ben Vincent Co-committed-by: Ben Vincent --- .woodpecker/vector-test.yaml | 4 +- apps/base/logging/configmap_logarchiver.yaml | 40 ++++++ apps/base/logging/deployment_logarchiver.yaml | 133 ++++++++++++++++++ apps/base/logging/job_clickhouse-schema.yaml | 26 ++++ apps/base/logging/kustomization.yaml | 8 +- .../logging/serviceaccount_logarchiver.yaml | 9 ++ apps/base/logging/vector/archiver.yaml | 62 -------- .../au-syd1/logging/kustomization.yaml | 8 +- .../logging/values-vector-archiver.yaml | 59 -------- 9 files changed, 214 insertions(+), 135 deletions(-) create mode 100644 apps/base/logging/configmap_logarchiver.yaml create mode 100644 apps/base/logging/deployment_logarchiver.yaml create mode 100644 apps/base/logging/serviceaccount_logarchiver.yaml delete mode 100644 apps/base/logging/vector/archiver.yaml delete mode 100644 apps/overlays/au-syd1/logging/values-vector-archiver.yaml diff --git a/.woodpecker/vector-test.yaml b/.woodpecker/vector-test.yaml index aace88e..5d9180d 100644 --- a/.woodpecker/vector-test.yaml +++ b/.woodpecker/vector-test.yaml @@ -14,9 +14,9 @@ steps: # Transform tier + VM ingest: unit-tested transforms. - vector test apps/base/logging/vector/aggregator.yaml apps/base/logging/vector/aggregator-tests.yaml - vector test apps/base/logging/vector/vm-ingest.yaml apps/base/logging/vector/vm-ingest-tests.yaml - # Agent + archiver have no transforms to unit-test; validate they build. + # Agent has no transforms to unit-test; validate it builds. (The archiver + # leg is now the logarchiver service, not a Vector pipeline.) - vector validate --no-environment apps/base/logging/vector/agent.yaml - - vector validate --no-environment apps/base/logging/vector/archiver.yaml backend_options: kubernetes: serviceAccountName: default diff --git a/apps/base/logging/configmap_logarchiver.yaml b/apps/base/logging/configmap_logarchiver.yaml new file mode 100644 index 0000000..b6d4996 --- /dev/null +++ b/apps/base/logging/configmap_logarchiver.yaml @@ -0,0 +1,40 @@ +--- +# logarchiver non-secret config. Secrets (NATS/S3/ClickHouse creds) and the +# subject filter come from env; everything else uses the binary's built-in +# defaults, which already target this stack. ack_wait MUST exceed batch.max_age +# so unacked messages in an open batch are not redelivered mid-batch. +apiVersion: v1 +kind: ConfigMap +metadata: + name: logarchiver-config + namespace: logging +data: + config.yaml: | + nats: + ack_wait: 5m + fetch_batch: 512 + batch: + max_bytes: 67108864 + max_events: 200000 + max_age: 2m + # Pin the proven RGW endpoint/bucket; ignore the secret's S3_ENDPOINT/BUCKET_NAME + # (AWS creds still come from the secret env). endpoint_env/bucket_env off. + s3: + endpoint: "https://s3.ceph.unkin.net" + bucket: "logs-archive" + region: "us-east-1" + path_style: true + ca_file: /etc/vault-ca/ca.crt + endpoint_env: "" + bucket_env: "" + crypto: + key_name: logarchive + pubkey_source: vault + vault: + address: "https://vault.service.consul:8200" + mount: gpg + auth_method: kubernetes + k8s_mount: k8s/au/syd1 + k8s_role: logging_logarchiver + k8s_jwt_path: /var/run/secrets/vault/token + ca_file: /etc/vault-ca/ca.crt diff --git a/apps/base/logging/deployment_logarchiver.yaml b/apps/base/logging/deployment_logarchiver.yaml new file mode 100644 index 0000000..73f70a2 --- /dev/null +++ b/apps/base/logging/deployment_logarchiver.yaml @@ -0,0 +1,133 @@ +--- +# logarchiver — replaces the vector-archiver leg. Independent JetStream durable +# consumer (archiver) that seals raw logs to S3 as zstd + OpenPGP objects and +# indexes each object in ClickHouse (logs.archive_index). Acks only after the +# object is in S3 AND indexed. Reuses the same NATS/S3/CA wiring the Vector +# archiver used; the OpenPGP public key is delivered as a mounted file. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: logarchiver + namespace: logging + annotations: + reloader.stakater.com/auto: "true" + labels: + app.kubernetes.io/name: logarchiver + app.kubernetes.io/component: archiver +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: logarchiver + template: + metadata: + labels: + app.kubernetes.io/name: logarchiver + vector.dev/exclude: "true" + spec: + # Dedicated SA whose projected vault-audience token authenticates the + # k8s-auth login used to fetch the logarchive public key from the gpg engine. + serviceAccountName: logarchiver + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: logarchiver + image: git.unkin.net/unkin/logarchiver:v0.1.0 + imagePullPolicy: IfNotPresent + args: ["run"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + ports: + - containerPort: 9090 + name: metrics + protocol: TCP + env: + - name: LOGARCHIVER_CONFIG + value: /etc/logarchiver/config.yaml + # Server-side subject filter; must match the archiver consumer's filter. + - name: ARCHIVE_SUBJECTS + value: "logs.k8s.vault.>" + - name: NATS_CONSUMER_PASSWORD + valueFrom: + secretKeyRef: + name: nats-auth + key: consumer_password + - name: CLICKHOUSE_USER + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: username + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: password + # S3 creds + S3_ENDPOINT + BUCKET_NAME from the cephrgw BucketAccess Secret. + envFrom: + - secretRef: + name: logs-archive-s3 + livenessProbe: + httpGet: + path: /healthz + port: metrics + initialDelaySeconds: 15 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthz + port: metrics + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: config + mountPath: /etc/logarchiver/config.yaml + subPath: config.yaml + readOnly: true + - name: vault-token + mountPath: /var/run/secrets/vault + readOnly: true + - name: vault-ca-cert + mountPath: /etc/vault-ca/ca.crt + subPath: ca.crt + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: config + configMap: + name: logarchiver-config + # Projected SA token with audience "vault" for the gpg-engine k8s login. + - name: vault-token + projected: + sources: + - serviceAccountToken: + path: token + audience: vault + expirationSeconds: 600 + - name: vault-ca-cert + secret: + secretName: vault-ca-cert + - name: tmp + emptyDir: {} diff --git a/apps/base/logging/job_clickhouse-schema.yaml b/apps/base/logging/job_clickhouse-schema.yaml index ff36675..2df42b8 100644 --- a/apps/base/logging/job_clickhouse-schema.yaml +++ b/apps/base/logging/job_clickhouse-schema.yaml @@ -88,6 +88,32 @@ spec: ORDER BY (source, namespace, host, timestamp) TTL toDateTime(timestamp) + INTERVAL 3 DAY SETTINGS index_granularity = 8192; + + -- One row per archived S3 object (written by logarchiver). No TTL: + -- the index must outlive logs.raw so the long-term S3 archive stays + -- searchable. Keep in sync with logarchiver internal/index/ddl.go. + CREATE TABLE IF NOT EXISTS logs.archive_index + ( + object_key String, + bucket LowCardinality(String), + subject LowCardinality(String), + hosts Array(LowCardinality(String)), + min_ts DateTime64(3), + max_ts DateTime64(3), + event_count UInt64, + raw_bytes UInt64, + stored_bytes UInt64, + compression LowCardinality(String), + cipher LowCardinality(String), + container_format LowCardinality(String), + key_name LowCardinality(String), + key_fingerprint String, + created_at DateTime64(3) DEFAULT now64(3), + INDEX idx_hosts hosts TYPE bloom_filter GRANULARITY 1 + ) + ENGINE = MergeTree + PARTITION BY toYYYYMM(min_ts) + ORDER BY (subject, min_ts, object_key); EOSQL echo "Schema applied." resources: diff --git a/apps/base/logging/kustomization.yaml b/apps/base/logging/kustomization.yaml index 4fd721f..a51e19a 100644 --- a/apps/base/logging/kustomization.yaml +++ b/apps/base/logging/kustomization.yaml @@ -12,6 +12,9 @@ resources: - cephrgw.yaml - gateway.yaml - httproute.yaml + - serviceaccount_logarchiver.yaml + - configmap_logarchiver.yaml + - deployment_logarchiver.yaml # Vector pipelines are the single source of truth (also validated by # `vector test` in CI). Mounted into each tier via `existingConfigMaps`. @@ -44,8 +47,3 @@ configMapGenerator: - vm-ingest.yaml=vector/vm-ingest.yaml options: disableNameSuffixHash: true - - name: vector-archiver-config - files: - - archiver.yaml=vector/archiver.yaml - options: - disableNameSuffixHash: true diff --git a/apps/base/logging/serviceaccount_logarchiver.yaml b/apps/base/logging/serviceaccount_logarchiver.yaml new file mode 100644 index 0000000..fb9823f --- /dev/null +++ b/apps/base/logging/serviceaccount_logarchiver.yaml @@ -0,0 +1,9 @@ +--- +# Dedicated SA for logarchiver's Vault k8s-auth login (role logging_logarchiver, +# terraform-vault). Only used to fetch the logarchive public key. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: logarchiver + namespace: logging +automountServiceAccountToken: false diff --git a/apps/base/logging/vector/archiver.yaml b/apps/base/logging/vector/archiver.yaml deleted file mode 100644 index 60b5102..0000000 --- a/apps/base/logging/vector/archiver.yaml +++ /dev/null @@ -1,62 +0,0 @@ ---- -# Vector ARCHIVER tier — long-term raw-log backup to S3 (Ceph RGW). Independent -# durable JetStream consumer (`archiver`) so its offsets/lag are fully isolated -# from the ClickHouse transform path (archive lag can never stall ingest — true -# fan-out). Writes RAW, pre-transform events (as they sit in JetStream) as -# gzipped NDJSON, partitioned by subject + date. This is the long-horizon replay -# source beyond JetStream's 3d retention window. -data_dir: /vector-data-dir - -api: - enabled: true - address: 0.0.0.0:8686 - -sources: - js_archive: - type: nats - url: nats://nats.logging.svc.cluster.local:4222 - connection_name: vector-archiver - subject: "logs.>" - jetstream: - stream: LOGS - consumer: archiver - auth: - strategy: user_password - user_password: - user: log-consumer - password: ${NATS_CONSUMER_PASSWORD} - decoding: - codec: json - -sinks: - s3: - type: aws_s3 - inputs: - - js_archive - bucket: logs-archive - endpoint: https://s3.ceph.unkin.net - region: us-east-1 - force_path_style: true - tls: - ca_file: /etc/vault-ca/ca.crt - # AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY come from the logs-archive-s3 - # Secret (cephrgw-operator) via envFrom on the deployment. - key_prefix: "raw/{{ subject }}/%Y/%m/%d/" - compression: gzip - encoding: - codec: json - framing: - method: newline_delimited - filename_time_format: "%Y%m%dT%H%M%SZ" - filename_append_uuid: true - batch: - max_bytes: 134217728 - timeout_secs: 300 - buffer: - type: memory - max_events: 5000 - when_full: block - # Disabled so slow BucketAccess credential propagation doesn't crash-loop - # the pod; RGW reachability is proven by the operator's own health. - healthcheck: - enabled: false diff --git a/apps/overlays/au-syd1/logging/kustomization.yaml b/apps/overlays/au-syd1/logging/kustomization.yaml index a81c658..9939c00 100644 --- a/apps/overlays/au-syd1/logging/kustomization.yaml +++ b/apps/overlays/au-syd1/logging/kustomization.yaml @@ -36,10 +36,4 @@ helmCharts: releaseName: vector-aggregator namespace: logging valuesFile: values-vector-aggregator.yaml - # Archiver (Deployment): independent JetStream consumer -> raw logs to S3. - - name: vector - repo: https://helm.vector.dev - version: "0.57.0" - releaseName: vector-archiver - namespace: logging - valuesFile: values-vector-archiver.yaml + # Archiver: replaced by the logarchiver Deployment (apps/base/logging). diff --git a/apps/overlays/au-syd1/logging/values-vector-archiver.yaml b/apps/overlays/au-syd1/logging/values-vector-archiver.yaml deleted file mode 100644 index 534f23f..0000000 --- a/apps/overlays/au-syd1/logging/values-vector-archiver.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# Vector ARCHIVER tier (Deployment) — independent JetStream consumer writing raw -# logs to S3 (Ceph RGW). Isolated from the ClickHouse path (own durable -# consumer). Pipeline: apps/base/logging/vector/archiver.yaml. -role: Stateless-Aggregator -fullnameOverride: vector-archiver -replicas: 1 - -image: - repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector - tag: 0.57.0-distroless-libc - -workloadResourceAnnotations: - reloader.stakater.com/auto: "true" - -podLabels: - vector.dev/exclude: "true" - -dataDir: /vector-data-dir -existingConfigMaps: - - vector-archiver-config - -env: - # Vector 0.57 disables ${VAR} config interpolation by default; auth needs it. - - name: VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION - value: "true" - - name: NATS_CONSUMER_PASSWORD - valueFrom: - secretKeyRef: - name: nats-auth - key: consumer_password - -# S3 creds (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY) from the cephrgw-operator -# BucketAccess Secret. -envFrom: - - secretRef: - name: logs-archive-s3 - -# Trust the internal unkin.net Vault-PKI CA to verify s3.ceph.unkin.net. -# vault-ca-cert is reflected into every namespace from the certificates ns. -extraVolumes: - - name: vault-ca-cert - secret: - secretName: vault-ca-cert -extraVolumeMounts: - - name: vault-ca-cert - mountPath: /etc/vault-ca/ca.crt - subPath: ca.crt - readOnly: true - -service: - enabled: false - -resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: "1" - memory: 1Gi