Add ClickHouse + Vector centralized logging stack
Stand up a centralized logging estate that captures ALL logs from k8s pods and (via a reachable ingestion endpoint) puppet-managed VMs, storing them in ClickHouse for query/retention. Metrics already live in VictoriaMetrics; this adds the logs pillar under a dedicated `logging` ArgoCD project. Deploy the Altinity clickhouse-operator (clickhouse-system) and a single-shard ClickHouseInstallation (logging) on cephrbd-fast-delete with a MergeTree logs.raw table (30d TTL) bootstrapped by an idempotent PostSync Job. Deploy Vector as an explicit two tiers: - Edge (thin): a DaemonSet tails every node's pod logs and forwards over the Vector-native protocol to the aggregator; no parsing at the edge. Future VM agents follow the same thin pattern. - Aggregator (brain): HA StatefulSet that is the sole ClickHouse writer, holds the only ClickHouse credentials, owns all transforms, batches into few fat inserts (avoid too-many-parts), and buffers to disk (PVC) to ride out a ClickHouse outage. Its pipeline is a single source-of-truth config validated by `vector test` in CI; per-app pipelines become aggregator-only changes. Expose the VM ingestion endpoint at logs-ingest.k8s.syd1.au.unkin.net via the internal Traefik gateway (cert-manager + external-dns), routing to the aggregator's HTTP source so puppet VMs can reach it over TLS. Source ClickHouse credentials from Vault via the existing VaultStaticSecret pattern (templated k8s auth policy already grants the logging namespace); password hash never lands in git. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
---
|
||||
# `vector test` unit tests for the aggregator transforms. Merged with
|
||||
# aggregator.yaml in CI (.woodpecker/vector-test.yaml). This is the pattern the
|
||||
# per-app parsing follow-ups extend: add a test per new transform here.
|
||||
tests:
|
||||
- name: k8s_log_is_normalised
|
||||
inputs:
|
||||
- insert_at: k8s_shape
|
||||
type: log
|
||||
log_fields:
|
||||
message: "hello from pod"
|
||||
stream: "stdout"
|
||||
timestamp: "2026-07-27T00:00:00Z"
|
||||
kubernetes.pod_name: "web-abc"
|
||||
kubernetes.pod_namespace: "shop"
|
||||
kubernetes.container_name: "web"
|
||||
kubernetes.pod_node_name: "node-1"
|
||||
outputs:
|
||||
- extract_from: k8s_shape
|
||||
conditions:
|
||||
- type: vrl
|
||||
source: |
|
||||
assert_eq!(.source, "k8s")
|
||||
assert_eq!(.namespace, "shop")
|
||||
assert_eq!(.pod, "web-abc")
|
||||
assert_eq!(.container, "web")
|
||||
assert_eq!(.host, "node-1")
|
||||
assert_eq!(.stream, "stdout")
|
||||
assert_eq!(.message, "hello from pod")
|
||||
|
||||
- name: vm_log_is_normalised
|
||||
inputs:
|
||||
- insert_at: vm_shape
|
||||
type: log
|
||||
log_fields:
|
||||
message: "sshd started"
|
||||
host: "vm-db-1"
|
||||
severity: "info"
|
||||
role: "database"
|
||||
outputs:
|
||||
- extract_from: vm_shape
|
||||
conditions:
|
||||
- type: vrl
|
||||
source: |
|
||||
assert_eq!(.source, "vm")
|
||||
assert_eq!(.host, "vm-db-1")
|
||||
assert_eq!(.severity, "info")
|
||||
assert_eq!(.message, "sshd started")
|
||||
assert_eq!(.labels.role, "database")
|
||||
@@ -0,0 +1,122 @@
|
||||
---
|
||||
# Vector AGGREGATOR pipeline (the "brain") — single source of truth.
|
||||
#
|
||||
# This file is the ConfigMap the aggregator StatefulSet runs AND the config
|
||||
# `vector test` validates in CI (see .woodpecker/vector-test.yaml, which merges
|
||||
# this with aggregator-tests.yaml). The edge tier (k8s DaemonSet + future VM
|
||||
# agents) stays thin: it only collects and attaches source metadata, then
|
||||
# forwards over the Vector-native protocol. All shaping, routing, enrichment,
|
||||
# batching, buffering and the ONLY ClickHouse credentials live here.
|
||||
#
|
||||
# Per-app pipelines arrive as follow-up tasks and are aggregator-only changes:
|
||||
# add a `remap`/`route`/enrichment transform below and append its id to the
|
||||
# clickhouse sink `inputs` — no fleet/DaemonSet rollout required.
|
||||
data_dir: /vector-data-dir
|
||||
|
||||
api:
|
||||
enabled: true
|
||||
address: 0.0.0.0:8686
|
||||
|
||||
sources:
|
||||
# In-cluster pod logs from the Vector agent DaemonSet (Vector-native proto).
|
||||
from_agents:
|
||||
type: vector
|
||||
address: 0.0.0.0:6000
|
||||
|
||||
# Puppet-managed VM logs over HTTP (NDJSON), exposed via the logs-ingest
|
||||
# Gateway at logs-ingest.k8s.syd1.au.unkin.net.
|
||||
vm_http:
|
||||
type: http_server
|
||||
address: 0.0.0.0:8080
|
||||
path: /
|
||||
method: POST
|
||||
decoding:
|
||||
codec: json
|
||||
framing:
|
||||
method: newline_delimited
|
||||
|
||||
transforms:
|
||||
# ---- Default normalisation into the logs.raw columns. This is intentionally
|
||||
# the ONLY shaping today; per-app parsing is added here as follow-ups. ----
|
||||
k8s_shape:
|
||||
type: remap
|
||||
inputs:
|
||||
- from_agents
|
||||
source: |
|
||||
ts = .timestamp || now()
|
||||
node = to_string(.kubernetes.pod_node_name || "") ?? ""
|
||||
ns = to_string(.kubernetes.pod_namespace || "") ?? ""
|
||||
pod = to_string(.kubernetes.pod_name || "") ?? ""
|
||||
container = to_string(.kubernetes.container_name || "") ?? ""
|
||||
strm = to_string(.stream || "") ?? ""
|
||||
msg = to_string(.message || "") ?? ""
|
||||
lbls = object(.kubernetes.pod_labels) ?? {}
|
||||
. = {
|
||||
"timestamp": ts,
|
||||
"host": node,
|
||||
"source": "k8s",
|
||||
"namespace": ns,
|
||||
"pod": pod,
|
||||
"container": container,
|
||||
"stream": strm,
|
||||
"severity": "",
|
||||
"message": msg,
|
||||
"labels": lbls,
|
||||
"fields": {}
|
||||
}
|
||||
vm_shape:
|
||||
type: remap
|
||||
inputs:
|
||||
- vm_http
|
||||
source: |
|
||||
ts = .timestamp || .ts || now()
|
||||
host = to_string(.host || .hostname || "") ?? ""
|
||||
msg = to_string(.message || .msg || "") ?? ""
|
||||
sev = to_string(.severity || .level || "") ?? ""
|
||||
role = to_string(.role || "") ?? ""
|
||||
lbls = {}
|
||||
if role != "" {
|
||||
lbls = {"role": role}
|
||||
}
|
||||
. = {
|
||||
"timestamp": ts,
|
||||
"host": host,
|
||||
"source": "vm",
|
||||
"namespace": "",
|
||||
"pod": "",
|
||||
"container": "",
|
||||
"stream": "",
|
||||
"severity": sev,
|
||||
"message": msg,
|
||||
"labels": lbls,
|
||||
"fields": {}
|
||||
}
|
||||
|
||||
sinks:
|
||||
clickhouse:
|
||||
type: clickhouse
|
||||
inputs:
|
||||
- k8s_shape
|
||||
- vm_shape
|
||||
endpoint: http://clickhouse-logs.logging.svc.cluster.local:8123
|
||||
database: logs
|
||||
table: raw
|
||||
skip_unknown_fields: true
|
||||
date_time_best_effort: true
|
||||
auth:
|
||||
strategy: basic
|
||||
user: "${CLICKHOUSE_USER}"
|
||||
password: "${CLICKHOUSE_PASSWORD}"
|
||||
# Fat, infrequent inserts keep ClickHouse part-count low (avoid too-many-parts).
|
||||
batch:
|
||||
max_events: 500000
|
||||
max_bytes: 134217728
|
||||
timeout_secs: 10
|
||||
# Ride out a ClickHouse outage without dropping logs: on-disk buffer on the
|
||||
# aggregator PVC; block upstream (back-pressure to the edge) when full.
|
||||
buffer:
|
||||
type: disk
|
||||
max_size: 8589934592
|
||||
when_full: block
|
||||
healthcheck:
|
||||
enabled: true
|
||||
Reference in New Issue
Block a user