unkinben
c39af2f9c3
Rework the logging pipeline around a durable message bus so logs survive a ClickHouse outage, can be replayed after a bad transform, and fan out to multiple independent consumers. Add long-term raw-log backup to S3. Topology becomes edge -> JetStream -> consumers -> sinks: - Dedicated JetStream NATS cluster (3 replicas, file storage) in the logging namespace. Deliberately separate from app messaging (streamstack) for blast-radius isolation. Stream LOGS (subjects logs.>, retention=limits, 40GiB / 72h) is the outage buffer; durable consumers give independent offsets. - Edge publishers (thin): the k8s DaemonSet and a new VM-ingest Deployment (HTTP NDJSON front door behind the logs-ingest Gateway) publish into JetStream (logs.k8s.<ns>.<container> / logs.vm.<host>). No parsing on the edge. - Transform tier (StatefulSet): pulls the whole stream via the durable `transform` consumer, routes by subject, shapes, and remains the sole ClickHouse writer. Its disk buffer shrinks (JetStream is the outage buffer). - Archiver (Deployment): its OWN durable `archiver` consumer (independent offsets — archive lag never affects the ClickHouse path) writes RAW, pre-transform events to a Ceph RGW S3 bucket (cephrgw-operator ObjectStoreUser + Bucket + BucketAccess) as gzipped NDJSON keyed by raw/<subject>/YYYY/MM/DD/. Default subject filter is Vault audit (logs.k8s.vault.>), configurable. Auth: distinct NATS users (producer publish-only, consumer pull+ack, admin for the stream/consumer bootstrap Job) with passwords from Vault (nats-auth Secret); S3 creds from the BucketAccess Secret. Streams/consumers are provisioned by an idempotent PostSync bootstrap Job. Add local kubeconform schemas for the ceph.unkin.net CRDs (datreeio lacks them) and extend the vector-test CI to cover the agent, VM-ingest and archiver configs. Verified end-to-end locally: NATS ACLs, vector JetStream publish, and durable-consumer pull+ack (at-least-once) all work. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
127 lines
3.4 KiB
YAML
127 lines
3.4 KiB
YAML
---
|
|
# Vector TRANSFORM tier (the "brain") — single source of truth, also validated
|
|
# by `vector test` in CI. Consumes the whole log stream from JetStream via the
|
|
# durable `transform` consumer (at-least-once; durable offsets tracked by
|
|
# JetStream), routes by subject, normalises into the logs.raw columns, and is
|
|
# the ONLY ClickHouse writer. Per-app parsing is added here as follow-ups:
|
|
# insert a transform and append its id to the clickhouse sink `inputs` — no edge
|
|
# or VM rollout required.
|
|
#
|
|
# Durability model: JetStream (72h / 40GiB) is the outage buffer. If ClickHouse
|
|
# is down the sink blocks, back-pressure stops acking, and JetStream retains
|
|
# messages for replay. The local disk buffer is small (survives pod restarts of
|
|
# in-flight events only).
|
|
data_dir: /vector-data-dir
|
|
|
|
api:
|
|
enabled: true
|
|
address: 0.0.0.0:8686
|
|
|
|
sources:
|
|
js_in:
|
|
type: nats
|
|
url: nats://nats.logging.svc.cluster.local:4222
|
|
connection_name: vector-transform
|
|
subject: "logs.>"
|
|
jetstream:
|
|
stream: LOGS
|
|
consumer: transform
|
|
auth:
|
|
strategy: user_password
|
|
user_password:
|
|
user: log-consumer
|
|
password: ${NATS_CONSUMER_PASSWORD}
|
|
decoding:
|
|
codec: json
|
|
|
|
transforms:
|
|
route:
|
|
type: route
|
|
inputs:
|
|
- js_in
|
|
route:
|
|
k8s: 'starts_with(to_string(.subject) ?? "", "logs.k8s.")'
|
|
vm: 'starts_with(to_string(.subject) ?? "", "logs.vm.")'
|
|
|
|
k8s_shape:
|
|
type: remap
|
|
inputs:
|
|
- route.k8s
|
|
source: |
|
|
ts = .timestamp || now()
|
|
node = to_string(.kubernetes.pod_node_name || "") ?? ""
|
|
ns = to_string(.kubernetes.pod_namespace || "") ?? ""
|
|
pod = to_string(.kubernetes.pod_name || "") ?? ""
|
|
container = to_string(.kubernetes.container_name || "") ?? ""
|
|
strm = to_string(.stream || "") ?? ""
|
|
msg = to_string(.message || "") ?? ""
|
|
lbls = object(.kubernetes.pod_labels) ?? {}
|
|
. = {
|
|
"timestamp": ts,
|
|
"host": node,
|
|
"source": "k8s",
|
|
"namespace": ns,
|
|
"pod": pod,
|
|
"container": container,
|
|
"stream": strm,
|
|
"severity": "",
|
|
"message": msg,
|
|
"labels": lbls,
|
|
"fields": {}
|
|
}
|
|
|
|
vm_shape:
|
|
type: remap
|
|
inputs:
|
|
- route.vm
|
|
source: |
|
|
ts = .timestamp || .ts || now()
|
|
host = to_string(.host || .hostname || "") ?? ""
|
|
msg = to_string(.message || .msg || "") ?? ""
|
|
sev = to_string(.severity || .level || "") ?? ""
|
|
role = to_string(.role || "") ?? ""
|
|
lbls = {}
|
|
if role != "" {
|
|
lbls = {"role": role}
|
|
}
|
|
. = {
|
|
"timestamp": ts,
|
|
"host": host,
|
|
"source": "vm",
|
|
"namespace": "",
|
|
"pod": "",
|
|
"container": "",
|
|
"stream": "",
|
|
"severity": sev,
|
|
"message": msg,
|
|
"labels": lbls,
|
|
"fields": {}
|
|
}
|
|
|
|
sinks:
|
|
clickhouse:
|
|
type: clickhouse
|
|
inputs:
|
|
- k8s_shape
|
|
- vm_shape
|
|
endpoint: http://clickhouse-logs.logging.svc.cluster.local:8123
|
|
database: logs
|
|
table: raw
|
|
skip_unknown_fields: true
|
|
date_time_best_effort: true
|
|
auth:
|
|
strategy: basic
|
|
user: "${CLICKHOUSE_USER}"
|
|
password: "${CLICKHOUSE_PASSWORD}"
|
|
batch:
|
|
max_events: 500000
|
|
max_bytes: 134217728
|
|
timeout_secs: 10
|
|
# Small local buffer — JetStream is the real outage buffer now.
|
|
buffer:
|
|
type: disk
|
|
max_size: 2147483648
|
|
when_full: block
|
|
healthcheck:
|
|
enabled: true
|