Add ClickHouse + Vector centralized logging stack
Stand up a centralized logging estate that captures ALL logs from k8s pods and (via a reachable ingestion endpoint) puppet-managed VMs, storing them in ClickHouse for query/retention. Metrics already live in VictoriaMetrics; this adds the logs pillar under a dedicated `logging` ArgoCD project. Deploy the Altinity clickhouse-operator (clickhouse-system) and a single-shard ClickHouseInstallation (logging) on cephrbd-fast-delete with a MergeTree logs.raw table (30d TTL) bootstrapped by an idempotent PostSync Job. Deploy Vector as an explicit two tiers: - Edge (thin): a DaemonSet tails every node's pod logs and forwards over the Vector-native protocol to the aggregator; no parsing at the edge. Future VM agents follow the same thin pattern. - Aggregator (brain): HA StatefulSet that is the sole ClickHouse writer, holds the only ClickHouse credentials, owns all transforms, batches into few fat inserts (avoid too-many-parts), and buffers to disk (PVC) to ride out a ClickHouse outage. Its pipeline is a single source-of-truth config validated by `vector test` in CI; per-app pipelines become aggregator-only changes. Expose the VM ingestion endpoint at logs-ingest.k8s.syd1.au.unkin.net via the internal Traefik gateway (cert-manager + external-dns), routing to the aggregator's HTTP source so puppet VMs can reach it over TLS. Source ClickHouse credentials from Vault via the existing VaultStaticSecret pattern (templated k8s auth policy already grants the logging namespace); password hash never lands in git. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
This commit is contained in:
@@ -0,0 +1,22 @@
|
||||
when:
|
||||
- event: pull_request
|
||||
|
||||
steps:
|
||||
- name: vector-test
|
||||
image: timberio/vector:0.57.0-debian
|
||||
commands:
|
||||
# Dummy creds + a writable data_dir so the full topology (incl. the
|
||||
# ClickHouse sink disk buffer) builds; tests only exercise the transforms.
|
||||
- export CLICKHOUSE_USER=ci CLICKHOUSE_PASSWORD=ci
|
||||
- mkdir -p /vector-data-dir
|
||||
- vector test apps/base/logging/vector/aggregator.yaml apps/base/logging/vector/aggregator-tests.yaml
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 256Mi
|
||||
cpu: 250m
|
||||
limits:
|
||||
memory: 1Gi
|
||||
cpu: 1
|
||||
@@ -0,0 +1,6 @@
|
||||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- namespace.yaml
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: clickhouse-system
|
||||
name: clickhouse-system
|
||||
@@ -0,0 +1,76 @@
|
||||
---
|
||||
apiVersion: clickhouse.altinity.com/v1
|
||||
kind: ClickHouseInstallation
|
||||
metadata:
|
||||
name: logs
|
||||
namespace: logging
|
||||
spec:
|
||||
defaults:
|
||||
templates:
|
||||
dataVolumeClaimTemplate: data-volume
|
||||
serviceTemplate: chi-service
|
||||
podTemplate: clickhouse
|
||||
configuration:
|
||||
users:
|
||||
# Password hash is sourced from the Vault-synced clickhouse-credentials
|
||||
# Secret; the plaintext never lands in git or the ClickHouse config.
|
||||
vector/password_sha256_hex:
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: clickhouse-credentials
|
||||
key: password_sha256_hex
|
||||
vector/networks/ip:
|
||||
- "::/0"
|
||||
vector/profile: default
|
||||
vector/quota: default
|
||||
# Allow the vector user to create the logs database/table (bootstrap Job)
|
||||
# and to INSERT. Restrict the built-in default user to loopback only.
|
||||
vector/access_management: "1"
|
||||
default/networks/ip:
|
||||
- "127.0.0.1"
|
||||
- "::1"
|
||||
profiles:
|
||||
default/max_memory_usage: "10000000000"
|
||||
default/max_execution_time: "120"
|
||||
clusters:
|
||||
- name: logs
|
||||
layout:
|
||||
shardsCount: 1
|
||||
replicasCount: 1
|
||||
templates:
|
||||
volumeClaimTemplates:
|
||||
- name: data-volume
|
||||
spec:
|
||||
storageClassName: cephrbd-fast-delete
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
serviceTemplates:
|
||||
- name: chi-service
|
||||
generateName: "clickhouse-{chi}"
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- name: http
|
||||
port: 8123
|
||||
- name: tcp
|
||||
port: 9000
|
||||
podTemplates:
|
||||
- name: clickhouse
|
||||
spec:
|
||||
securityContext:
|
||||
fsGroup: 101
|
||||
runAsUser: 101
|
||||
runAsGroup: 101
|
||||
containers:
|
||||
- name: clickhouse
|
||||
image: clickhouse/clickhouse-server:24.8
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 2Gi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 8Gi
|
||||
@@ -0,0 +1,44 @@
|
||||
---
|
||||
# Log ingestion endpoint for puppet-managed VMs (and any non-k8s client).
|
||||
# Reuses the internal Traefik gateway + cert-manager + external-dns pattern so
|
||||
# VMs reach the Vector aggregator's HTTP source over TLS at a DNS name they can
|
||||
# resolve. The puppet-side Vector rollout ships NDJSON to
|
||||
# https://logs-ingest.k8s.syd1.au.unkin.net/ (a later task).
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: Gateway
|
||||
metadata:
|
||||
name: logs-ingest
|
||||
namespace: logging
|
||||
labels:
|
||||
app.kubernetes.io/name: vector-aggregator
|
||||
app.kubernetes.io/component: ingest
|
||||
traefik.io/instance: internal
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: vault-issuer
|
||||
cert-manager.io/common-name: logs-ingest.k8s.syd1.au.unkin.net
|
||||
cert-manager.io/private-key-size: "4096"
|
||||
external-dns.alpha.kubernetes.io/hostname: logs-ingest.k8s.syd1.au.unkin.net
|
||||
external-dns.alpha.kubernetes.io/target: 198.18.200.4
|
||||
spec:
|
||||
gatewayClassName: traefik-internal
|
||||
listeners:
|
||||
- name: http
|
||||
port: 80
|
||||
protocol: HTTP
|
||||
hostname: logs-ingest.k8s.syd1.au.unkin.net
|
||||
allowedRoutes:
|
||||
namespaces:
|
||||
from: Same
|
||||
- name: https
|
||||
port: 443
|
||||
protocol: HTTPS
|
||||
hostname: logs-ingest.k8s.syd1.au.unkin.net
|
||||
allowedRoutes:
|
||||
namespaces:
|
||||
from: Same
|
||||
tls:
|
||||
mode: Terminate
|
||||
certificateRefs:
|
||||
- group: ""
|
||||
kind: Secret
|
||||
name: logs-ingest-tls
|
||||
@@ -0,0 +1,55 @@
|
||||
---
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: logs-ingest-http-redirect
|
||||
namespace: logging
|
||||
labels:
|
||||
app.kubernetes.io/name: vector-aggregator
|
||||
app.kubernetes.io/component: ingest
|
||||
spec:
|
||||
hostnames:
|
||||
- logs-ingest.k8s.syd1.au.unkin.net
|
||||
parentRefs:
|
||||
- group: gateway.networking.k8s.io
|
||||
kind: Gateway
|
||||
name: logs-ingest
|
||||
sectionName: http
|
||||
rules:
|
||||
- filters:
|
||||
- type: RequestRedirect
|
||||
requestRedirect:
|
||||
scheme: https
|
||||
statusCode: 301
|
||||
matches:
|
||||
- path:
|
||||
type: PathPrefix
|
||||
value: /
|
||||
---
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: logs-ingest
|
||||
namespace: logging
|
||||
labels:
|
||||
app.kubernetes.io/name: vector-aggregator
|
||||
app.kubernetes.io/component: ingest
|
||||
spec:
|
||||
hostnames:
|
||||
- logs-ingest.k8s.syd1.au.unkin.net
|
||||
parentRefs:
|
||||
- group: gateway.networking.k8s.io
|
||||
kind: Gateway
|
||||
name: logs-ingest
|
||||
sectionName: https
|
||||
rules:
|
||||
- backendRefs:
|
||||
- group: ""
|
||||
kind: Service
|
||||
name: vector-aggregator
|
||||
port: 8080
|
||||
weight: 1
|
||||
matches:
|
||||
- path:
|
||||
type: PathPrefix
|
||||
value: /
|
||||
@@ -0,0 +1,105 @@
|
||||
---
|
||||
# Declarative ClickHouse schema bootstrap. Runs as an ArgoCD PostSync hook so it
|
||||
# executes after the ClickHouseInstallation is reconciled, and re-runs on every
|
||||
# sync (idempotent CREATE ... IF NOT EXISTS). Edit the DDL here to evolve the
|
||||
# schema; the Vector aggregator writes to logs.raw with skip_unknown_fields, so
|
||||
# adding columns is backward-compatible.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: clickhouse-schema
|
||||
namespace: logging
|
||||
annotations:
|
||||
argocd.argoproj.io/hook: PostSync
|
||||
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
|
||||
labels:
|
||||
app.kubernetes.io/name: clickhouse-schema
|
||||
app.kubernetes.io/component: bootstrap
|
||||
spec:
|
||||
backoffLimit: 20
|
||||
activeDeadlineSeconds: 1800
|
||||
ttlSecondsAfterFinished: 3600
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: clickhouse-schema
|
||||
vector.dev/exclude: "true"
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 101
|
||||
runAsGroup: 101
|
||||
containers:
|
||||
- name: clickhouse-schema
|
||||
image: clickhouse/clickhouse-server:24.8
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
env:
|
||||
- name: HOME
|
||||
value: /tmp
|
||||
- name: CLICKHOUSE_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: clickhouse-credentials
|
||||
key: username
|
||||
- name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: clickhouse-credentials
|
||||
key: password
|
||||
command:
|
||||
- /bin/bash
|
||||
- -ec
|
||||
- |
|
||||
host=clickhouse-logs.logging.svc.cluster.local
|
||||
echo "Waiting for ClickHouse at ${host}:9000 ..."
|
||||
until clickhouse-client --host "$host" --port 9000 \
|
||||
--user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \
|
||||
--query "SELECT 1" >/dev/null 2>&1; do
|
||||
echo " not ready, retrying in 5s"; sleep 5
|
||||
done
|
||||
echo "Applying schema ..."
|
||||
clickhouse-client --host "$host" --port 9000 \
|
||||
--user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \
|
||||
--multiquery <<'EOSQL'
|
||||
CREATE DATABASE IF NOT EXISTS logs;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS logs.raw
|
||||
(
|
||||
timestamp DateTime64(3) DEFAULT now64(3),
|
||||
host LowCardinality(String) DEFAULT '',
|
||||
source LowCardinality(String) DEFAULT '',
|
||||
namespace LowCardinality(String) DEFAULT '',
|
||||
pod String DEFAULT '',
|
||||
container LowCardinality(String) DEFAULT '',
|
||||
stream LowCardinality(String) DEFAULT '',
|
||||
severity LowCardinality(String) DEFAULT '',
|
||||
message String DEFAULT '',
|
||||
labels Map(LowCardinality(String), String),
|
||||
fields Map(LowCardinality(String), String)
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(timestamp)
|
||||
ORDER BY (source, namespace, host, timestamp)
|
||||
TTL toDateTime(timestamp) + INTERVAL 30 DAY
|
||||
SETTINGS index_granularity = 8192;
|
||||
EOSQL
|
||||
echo "Schema applied."
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
@@ -0,0 +1,21 @@
|
||||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- vaultauth.yaml
|
||||
- vaultstaticsecret.yaml
|
||||
- clickhouseinstallation.yaml
|
||||
- job_clickhouse-schema.yaml
|
||||
- gateway.yaml
|
||||
- httproute.yaml
|
||||
|
||||
# Aggregator pipeline is the single source of truth (also validated by
|
||||
# `vector test` in CI). Mounted into the aggregator via `existingConfigMaps`.
|
||||
configMapGenerator:
|
||||
- name: vector-aggregator-config
|
||||
files:
|
||||
- aggregator.yaml=vector/aggregator.yaml
|
||||
options:
|
||||
disableNameSuffixHash: true
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: logging
|
||||
name: logging
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultAuth
|
||||
metadata:
|
||||
name: default
|
||||
namespace: logging
|
||||
spec:
|
||||
allowedNamespaces:
|
||||
- logging
|
||||
kubernetes:
|
||||
audiences:
|
||||
- vault
|
||||
role: default
|
||||
serviceAccount: default
|
||||
tokenExpirationSeconds: 600
|
||||
method: kubernetes
|
||||
mount: k8s/au/syd1
|
||||
vaultConnectionRef: vso-system/default
|
||||
@@ -0,0 +1,28 @@
|
||||
---
|
||||
# ClickHouse credentials for the `vector` user.
|
||||
#
|
||||
# Seed the Vault KV entry once (values are NOT stored in git), e.g.:
|
||||
# PW=$(openssl rand -base64 24)
|
||||
# HASH=$(printf '%s' "$PW" | sha256sum | cut -d' ' -f1)
|
||||
# vault kv put kv/kubernetes/namespace/logging/default/clickhouse-credentials \
|
||||
# username=vector password="$PW" password_sha256_hex="$HASH"
|
||||
#
|
||||
# The `logging/default` ServiceAccount reads this path via the templated
|
||||
# `policies/kv/kubernetes/default.yaml` policy (k8s auth role `default`), so no
|
||||
# terraform-vault change is required — only the value above must be written.
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultStaticSecret
|
||||
metadata:
|
||||
name: clickhouse-credentials
|
||||
namespace: logging
|
||||
spec:
|
||||
destination:
|
||||
create: true
|
||||
name: clickhouse-credentials
|
||||
overwrite: true
|
||||
hmacSecretData: true
|
||||
mount: kv
|
||||
path: kubernetes/namespace/logging/default/clickhouse-credentials
|
||||
refreshAfter: 5m
|
||||
type: kv-v2
|
||||
vaultAuthRef: default
|
||||
@@ -0,0 +1,49 @@
|
||||
---
|
||||
# `vector test` unit tests for the aggregator transforms. Merged with
|
||||
# aggregator.yaml in CI (.woodpecker/vector-test.yaml). This is the pattern the
|
||||
# per-app parsing follow-ups extend: add a test per new transform here.
|
||||
tests:
|
||||
- name: k8s_log_is_normalised
|
||||
inputs:
|
||||
- insert_at: k8s_shape
|
||||
type: log
|
||||
log_fields:
|
||||
message: "hello from pod"
|
||||
stream: "stdout"
|
||||
timestamp: "2026-07-27T00:00:00Z"
|
||||
kubernetes.pod_name: "web-abc"
|
||||
kubernetes.pod_namespace: "shop"
|
||||
kubernetes.container_name: "web"
|
||||
kubernetes.pod_node_name: "node-1"
|
||||
outputs:
|
||||
- extract_from: k8s_shape
|
||||
conditions:
|
||||
- type: vrl
|
||||
source: |
|
||||
assert_eq!(.source, "k8s")
|
||||
assert_eq!(.namespace, "shop")
|
||||
assert_eq!(.pod, "web-abc")
|
||||
assert_eq!(.container, "web")
|
||||
assert_eq!(.host, "node-1")
|
||||
assert_eq!(.stream, "stdout")
|
||||
assert_eq!(.message, "hello from pod")
|
||||
|
||||
- name: vm_log_is_normalised
|
||||
inputs:
|
||||
- insert_at: vm_shape
|
||||
type: log
|
||||
log_fields:
|
||||
message: "sshd started"
|
||||
host: "vm-db-1"
|
||||
severity: "info"
|
||||
role: "database"
|
||||
outputs:
|
||||
- extract_from: vm_shape
|
||||
conditions:
|
||||
- type: vrl
|
||||
source: |
|
||||
assert_eq!(.source, "vm")
|
||||
assert_eq!(.host, "vm-db-1")
|
||||
assert_eq!(.severity, "info")
|
||||
assert_eq!(.message, "sshd started")
|
||||
assert_eq!(.labels.role, "database")
|
||||
@@ -0,0 +1,122 @@
|
||||
---
|
||||
# Vector AGGREGATOR pipeline (the "brain") — single source of truth.
|
||||
#
|
||||
# This file is the ConfigMap the aggregator StatefulSet runs AND the config
|
||||
# `vector test` validates in CI (see .woodpecker/vector-test.yaml, which merges
|
||||
# this with aggregator-tests.yaml). The edge tier (k8s DaemonSet + future VM
|
||||
# agents) stays thin: it only collects and attaches source metadata, then
|
||||
# forwards over the Vector-native protocol. All shaping, routing, enrichment,
|
||||
# batching, buffering and the ONLY ClickHouse credentials live here.
|
||||
#
|
||||
# Per-app pipelines arrive as follow-up tasks and are aggregator-only changes:
|
||||
# add a `remap`/`route`/enrichment transform below and append its id to the
|
||||
# clickhouse sink `inputs` — no fleet/DaemonSet rollout required.
|
||||
data_dir: /vector-data-dir
|
||||
|
||||
api:
|
||||
enabled: true
|
||||
address: 0.0.0.0:8686
|
||||
|
||||
sources:
|
||||
# In-cluster pod logs from the Vector agent DaemonSet (Vector-native proto).
|
||||
from_agents:
|
||||
type: vector
|
||||
address: 0.0.0.0:6000
|
||||
|
||||
# Puppet-managed VM logs over HTTP (NDJSON), exposed via the logs-ingest
|
||||
# Gateway at logs-ingest.k8s.syd1.au.unkin.net.
|
||||
vm_http:
|
||||
type: http_server
|
||||
address: 0.0.0.0:8080
|
||||
path: /
|
||||
method: POST
|
||||
decoding:
|
||||
codec: json
|
||||
framing:
|
||||
method: newline_delimited
|
||||
|
||||
transforms:
|
||||
# ---- Default normalisation into the logs.raw columns. This is intentionally
|
||||
# the ONLY shaping today; per-app parsing is added here as follow-ups. ----
|
||||
k8s_shape:
|
||||
type: remap
|
||||
inputs:
|
||||
- from_agents
|
||||
source: |
|
||||
ts = .timestamp || now()
|
||||
node = to_string(.kubernetes.pod_node_name || "") ?? ""
|
||||
ns = to_string(.kubernetes.pod_namespace || "") ?? ""
|
||||
pod = to_string(.kubernetes.pod_name || "") ?? ""
|
||||
container = to_string(.kubernetes.container_name || "") ?? ""
|
||||
strm = to_string(.stream || "") ?? ""
|
||||
msg = to_string(.message || "") ?? ""
|
||||
lbls = object(.kubernetes.pod_labels) ?? {}
|
||||
. = {
|
||||
"timestamp": ts,
|
||||
"host": node,
|
||||
"source": "k8s",
|
||||
"namespace": ns,
|
||||
"pod": pod,
|
||||
"container": container,
|
||||
"stream": strm,
|
||||
"severity": "",
|
||||
"message": msg,
|
||||
"labels": lbls,
|
||||
"fields": {}
|
||||
}
|
||||
vm_shape:
|
||||
type: remap
|
||||
inputs:
|
||||
- vm_http
|
||||
source: |
|
||||
ts = .timestamp || .ts || now()
|
||||
host = to_string(.host || .hostname || "") ?? ""
|
||||
msg = to_string(.message || .msg || "") ?? ""
|
||||
sev = to_string(.severity || .level || "") ?? ""
|
||||
role = to_string(.role || "") ?? ""
|
||||
lbls = {}
|
||||
if role != "" {
|
||||
lbls = {"role": role}
|
||||
}
|
||||
. = {
|
||||
"timestamp": ts,
|
||||
"host": host,
|
||||
"source": "vm",
|
||||
"namespace": "",
|
||||
"pod": "",
|
||||
"container": "",
|
||||
"stream": "",
|
||||
"severity": sev,
|
||||
"message": msg,
|
||||
"labels": lbls,
|
||||
"fields": {}
|
||||
}
|
||||
|
||||
sinks:
|
||||
clickhouse:
|
||||
type: clickhouse
|
||||
inputs:
|
||||
- k8s_shape
|
||||
- vm_shape
|
||||
endpoint: http://clickhouse-logs.logging.svc.cluster.local:8123
|
||||
database: logs
|
||||
table: raw
|
||||
skip_unknown_fields: true
|
||||
date_time_best_effort: true
|
||||
auth:
|
||||
strategy: basic
|
||||
user: "${CLICKHOUSE_USER}"
|
||||
password: "${CLICKHOUSE_PASSWORD}"
|
||||
# Fat, infrequent inserts keep ClickHouse part-count low (avoid too-many-parts).
|
||||
batch:
|
||||
max_events: 500000
|
||||
max_bytes: 134217728
|
||||
timeout_secs: 10
|
||||
# Ride out a ClickHouse outage without dropping logs: on-disk buffer on the
|
||||
# aggregator PVC; block upstream (back-pressure to the edge) when full.
|
||||
buffer:
|
||||
type: disk
|
||||
max_size: 8589934592
|
||||
when_full: block
|
||||
healthcheck:
|
||||
enabled: true
|
||||
@@ -0,0 +1,16 @@
|
||||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
namespace: clickhouse-system
|
||||
|
||||
resources:
|
||||
- ../../../base/clickhouse-system
|
||||
|
||||
helmCharts:
|
||||
- name: altinity-clickhouse-operator
|
||||
repo: https://helm.altinity.com
|
||||
version: "0.27.2"
|
||||
releaseName: clickhouse-operator
|
||||
namespace: clickhouse-system
|
||||
valuesFile: values.yaml
|
||||
@@ -0,0 +1,29 @@
|
||||
# Altinity ClickHouse operator. Cluster-scoped: watches ClickHouseInstallation
|
||||
# resources in all namespaces (the logs cluster lives in the `logging` namespace).
|
||||
# CRDs are installed at runtime by the chart's crdHook Job.
|
||||
crdHook:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 128Mi
|
||||
|
||||
operator:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
metrics:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
@@ -0,0 +1,25 @@
|
||||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
namespace: logging
|
||||
|
||||
resources:
|
||||
- ../../../base/logging
|
||||
|
||||
helmCharts:
|
||||
# Node-level agent: tails every pod's logs and forwards to the aggregator.
|
||||
- name: vector
|
||||
repo: https://helm.vector.dev
|
||||
version: "0.57.0"
|
||||
releaseName: vector-agent
|
||||
namespace: logging
|
||||
valuesFile: values-vector-agent.yaml
|
||||
# Aggregator: receives from agents (vector proto) and from VMs (HTTP),
|
||||
# shapes events, and writes to ClickHouse.
|
||||
- name: vector
|
||||
repo: https://helm.vector.dev
|
||||
version: "0.57.0"
|
||||
releaseName: vector-aggregator
|
||||
namespace: logging
|
||||
valuesFile: values-vector-aggregator.yaml
|
||||
@@ -0,0 +1,44 @@
|
||||
# Vector Agent (DaemonSet) — captures ALL pod logs on every node (including
|
||||
# control-plane, via the blanket toleration) and forwards them to the
|
||||
# aggregator over the Vector native protocol. No per-app parsing here: shaping
|
||||
# and the ClickHouse write live on the aggregator.
|
||||
role: Agent
|
||||
fullnameOverride: vector-agent
|
||||
|
||||
rbac:
|
||||
create: true
|
||||
serviceAccount:
|
||||
create: true
|
||||
|
||||
podLabels:
|
||||
vector.dev/exclude: "true"
|
||||
|
||||
# Run on every node so no host's pod logs are missed.
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
|
||||
# Disable the chart's auto-generated service for the agent (it ships, not serves).
|
||||
service:
|
||||
enabled: false
|
||||
|
||||
customConfig:
|
||||
data_dir: /vector-data-dir
|
||||
api:
|
||||
enabled: false
|
||||
sources:
|
||||
kubernetes_logs:
|
||||
type: kubernetes_logs
|
||||
sinks:
|
||||
to_aggregator:
|
||||
type: vector
|
||||
inputs:
|
||||
- kubernetes_logs
|
||||
address: vector-aggregator.logging.svc.cluster.local:6000
|
||||
@@ -0,0 +1,82 @@
|
||||
# Vector AGGREGATOR (StatefulSet) — the "brain" tier: sole ClickHouse writer,
|
||||
# owns all transforms, holds the only ClickHouse credentials, HA replicas with
|
||||
# on-disk buffers to ride out a ClickHouse outage.
|
||||
#
|
||||
# The pipeline itself is NOT inlined here: it lives in
|
||||
# apps/base/logging/vector/aggregator.yaml (single source of truth, unit-tested
|
||||
# by `vector test` in CI) and is mounted via existingConfigMaps. This file only
|
||||
# owns the k8s deployment shape (ports, creds, storage, HA).
|
||||
role: Aggregator
|
||||
fullnameOverride: vector-aggregator
|
||||
replicas: 2
|
||||
|
||||
# Reload the StatefulSet when the aggregator ConfigMap or the creds Secret change.
|
||||
workloadResourceAnnotations:
|
||||
reloader.stakater.com/auto: "true"
|
||||
|
||||
podLabels:
|
||||
vector.dev/exclude: "true"
|
||||
|
||||
# Mount the aggregator pipeline ConfigMap (rendered from base by kustomize).
|
||||
# dataDir must be set when using existingConfigMaps; it is also where the
|
||||
# clickhouse sink's disk buffer lives (backed by the PVC below).
|
||||
dataDir: /vector-data-dir
|
||||
existingConfigMaps:
|
||||
- vector-aggregator-config
|
||||
|
||||
# On-disk buffer storage so a ClickHouse outage does not drop logs.
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClassName: cephrbd-fast-delete
|
||||
size: 20Gi
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
|
||||
# ClickHouse basic-auth creds — the ONLY place these are consumed.
|
||||
env:
|
||||
- name: CLICKHOUSE_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: clickhouse-credentials
|
||||
key: username
|
||||
- name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: clickhouse-credentials
|
||||
key: password
|
||||
|
||||
containerPorts:
|
||||
- name: vector
|
||||
containerPort: 6000
|
||||
protocol: TCP
|
||||
- name: http-ingest
|
||||
containerPort: 8080
|
||||
protocol: TCP
|
||||
- name: api
|
||||
containerPort: 8686
|
||||
protocol: TCP
|
||||
|
||||
service:
|
||||
enabled: true
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- name: vector
|
||||
port: 6000
|
||||
targetPort: 6000
|
||||
protocol: TCP
|
||||
- name: http-ingest
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
protocol: TCP
|
||||
- name: api
|
||||
port: 8686
|
||||
targetPort: 8686
|
||||
protocol: TCP
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 2Gi
|
||||
@@ -4,6 +4,7 @@ kind: Kustomization
|
||||
|
||||
resources:
|
||||
- aitooling.yaml
|
||||
- logging.yaml
|
||||
- observability.yaml
|
||||
- platform.yaml
|
||||
- storage.yaml
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
---
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: ApplicationSet
|
||||
metadata:
|
||||
name: logging-apps
|
||||
namespace: argocd
|
||||
spec:
|
||||
generators:
|
||||
- git:
|
||||
repoURL: https://git.unkin.net/unkin/argocd-apps
|
||||
revision: HEAD
|
||||
directories:
|
||||
- path: apps/overlays/*/clickhouse-system
|
||||
- path: apps/overlays/*/logging
|
||||
template:
|
||||
metadata:
|
||||
name: 'logging-{{path[3]}}'
|
||||
spec:
|
||||
project: logging
|
||||
source:
|
||||
repoURL: https://git.unkin.net/unkin/argocd-apps
|
||||
targetRevision: HEAD
|
||||
path: '{{path}}'
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: '{{path[3]}}'
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- ServerSideApply=true
|
||||
- CreateNamespace=false
|
||||
@@ -4,6 +4,7 @@ kind: Kustomization
|
||||
|
||||
resources:
|
||||
- aitooling.yaml
|
||||
- logging.yaml
|
||||
- observability.yaml
|
||||
- platform.yaml
|
||||
- storage.yaml
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
---
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: AppProject
|
||||
metadata:
|
||||
name: logging
|
||||
namespace: argocd
|
||||
spec:
|
||||
description: Centralized logging stack (ClickHouse + Vector)
|
||||
sourceRepos:
|
||||
- https://git.unkin.net/unkin/argocd-apps
|
||||
- https://helm.altinity.com
|
||||
- https://helm.vector.dev
|
||||
destinations:
|
||||
- namespace: 'logging'
|
||||
server: https://kubernetes.default.svc
|
||||
- namespace: 'clickhouse-system'
|
||||
server: https://kubernetes.default.svc
|
||||
clusterResourceWhitelist:
|
||||
- group: ''
|
||||
kind: Namespace
|
||||
- group: 'rbac.authorization.k8s.io'
|
||||
kind: ClusterRole
|
||||
- group: 'rbac.authorization.k8s.io'
|
||||
kind: ClusterRoleBinding
|
||||
- group: 'apiextensions.k8s.io'
|
||||
kind: CustomResourceDefinition
|
||||
namespaceResourceWhitelist:
|
||||
- group: '*'
|
||||
kind: '*'
|
||||
Reference in New Issue
Block a user