Add ClickHouse + Vector + NATS JetStream centralized logging (with S3 raw archive) #296
@@ -0,0 +1,22 @@
|
||||
when:
|
||||
- event: pull_request
|
||||
|
||||
steps:
|
||||
- name: vector-test
|
||||
image: timberio/vector:0.57.0-debian
|
||||
commands:
|
||||
# Dummy creds + a writable data_dir so the full topology (incl. the
|
||||
# ClickHouse sink disk buffer) builds; tests only exercise the transforms.
|
||||
- export CLICKHOUSE_USER=ci CLICKHOUSE_PASSWORD=ci
|
||||
- mkdir -p /vector-data-dir
|
||||
- vector test apps/base/logging/vector/aggregator.yaml apps/base/logging/vector/aggregator-tests.yaml
|
||||
backend_options:
|
||||
kubernetes:
|
||||
serviceAccountName: default
|
||||
resources:
|
||||
requests:
|
||||
memory: 256Mi
|
||||
cpu: 250m
|
||||
limits:
|
||||
memory: 1Gi
|
||||
cpu: 1
|
||||
@@ -0,0 +1,6 @@
|
||||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- namespace.yaml
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: clickhouse-system
|
||||
name: clickhouse-system
|
||||
@@ -0,0 +1,76 @@
|
||||
---
|
||||
apiVersion: clickhouse.altinity.com/v1
|
||||
kind: ClickHouseInstallation
|
||||
metadata:
|
||||
name: logs
|
||||
namespace: logging
|
||||
spec:
|
||||
defaults:
|
||||
templates:
|
||||
dataVolumeClaimTemplate: data-volume
|
||||
serviceTemplate: chi-service
|
||||
podTemplate: clickhouse
|
||||
configuration:
|
||||
users:
|
||||
# Password hash is sourced from the Vault-synced clickhouse-credentials
|
||||
# Secret; the plaintext never lands in git or the ClickHouse config.
|
||||
vector/password_sha256_hex:
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: clickhouse-credentials
|
||||
key: password_sha256_hex
|
||||
vector/networks/ip:
|
||||
- "::/0"
|
||||
vector/profile: default
|
||||
vector/quota: default
|
||||
# Allow the vector user to create the logs database/table (bootstrap Job)
|
||||
# and to INSERT. Restrict the built-in default user to loopback only.
|
||||
vector/access_management: "1"
|
||||
default/networks/ip:
|
||||
- "127.0.0.1"
|
||||
- "::1"
|
||||
profiles:
|
||||
default/max_memory_usage: "10000000000"
|
||||
default/max_execution_time: "120"
|
||||
clusters:
|
||||
- name: logs
|
||||
layout:
|
||||
shardsCount: 1
|
||||
replicasCount: 1
|
||||
templates:
|
||||
volumeClaimTemplates:
|
||||
- name: data-volume
|
||||
spec:
|
||||
storageClassName: cephrbd-fast-delete
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
serviceTemplates:
|
||||
- name: chi-service
|
||||
generateName: "clickhouse-{chi}"
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- name: http
|
||||
port: 8123
|
||||
- name: tcp
|
||||
port: 9000
|
||||
podTemplates:
|
||||
- name: clickhouse
|
||||
spec:
|
||||
securityContext:
|
||||
fsGroup: 101
|
||||
runAsUser: 101
|
||||
runAsGroup: 101
|
||||
containers:
|
||||
- name: clickhouse
|
||||
image: clickhouse/clickhouse-server:24.8
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 2Gi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 8Gi
|
||||
@@ -0,0 +1,44 @@
|
||||
---
|
||||
# Log ingestion endpoint for puppet-managed VMs (and any non-k8s client).
|
||||
# Reuses the internal Traefik gateway + cert-manager + external-dns pattern so
|
||||
# VMs reach the Vector aggregator's HTTP source over TLS at a DNS name they can
|
||||
# resolve. The puppet-side Vector rollout ships NDJSON to
|
||||
# https://logs-ingest.k8s.syd1.au.unkin.net/ (a later task).
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: Gateway
|
||||
metadata:
|
||||
name: logs-ingest
|
||||
namespace: logging
|
||||
labels:
|
||||
app.kubernetes.io/name: vector-aggregator
|
||||
app.kubernetes.io/component: ingest
|
||||
traefik.io/instance: internal
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: vault-issuer
|
||||
cert-manager.io/common-name: logs-ingest.k8s.syd1.au.unkin.net
|
||||
cert-manager.io/private-key-size: "4096"
|
||||
external-dns.alpha.kubernetes.io/hostname: logs-ingest.k8s.syd1.au.unkin.net
|
||||
external-dns.alpha.kubernetes.io/target: 198.18.200.4
|
||||
spec:
|
||||
gatewayClassName: traefik-internal
|
||||
listeners:
|
||||
- name: http
|
||||
port: 80
|
||||
protocol: HTTP
|
||||
hostname: logs-ingest.k8s.syd1.au.unkin.net
|
||||
allowedRoutes:
|
||||
namespaces:
|
||||
from: Same
|
||||
- name: https
|
||||
port: 443
|
||||
protocol: HTTPS
|
||||
hostname: logs-ingest.k8s.syd1.au.unkin.net
|
||||
allowedRoutes:
|
||||
namespaces:
|
||||
from: Same
|
||||
tls:
|
||||
mode: Terminate
|
||||
certificateRefs:
|
||||
- group: ""
|
||||
kind: Secret
|
||||
name: logs-ingest-tls
|
||||
@@ -0,0 +1,55 @@
|
||||
---
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: logs-ingest-http-redirect
|
||||
namespace: logging
|
||||
labels:
|
||||
app.kubernetes.io/name: vector-aggregator
|
||||
app.kubernetes.io/component: ingest
|
||||
spec:
|
||||
hostnames:
|
||||
- logs-ingest.k8s.syd1.au.unkin.net
|
||||
parentRefs:
|
||||
- group: gateway.networking.k8s.io
|
||||
kind: Gateway
|
||||
name: logs-ingest
|
||||
sectionName: http
|
||||
rules:
|
||||
- filters:
|
||||
- type: RequestRedirect
|
||||
requestRedirect:
|
||||
scheme: https
|
||||
statusCode: 301
|
||||
matches:
|
||||
- path:
|
||||
type: PathPrefix
|
||||
value: /
|
||||
---
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: logs-ingest
|
||||
namespace: logging
|
||||
labels:
|
||||
app.kubernetes.io/name: vector-aggregator
|
||||
app.kubernetes.io/component: ingest
|
||||
spec:
|
||||
hostnames:
|
||||
- logs-ingest.k8s.syd1.au.unkin.net
|
||||
parentRefs:
|
||||
- group: gateway.networking.k8s.io
|
||||
kind: Gateway
|
||||
name: logs-ingest
|
||||
sectionName: https
|
||||
rules:
|
||||
- backendRefs:
|
||||
- group: ""
|
||||
kind: Service
|
||||
name: vector-aggregator
|
||||
port: 8080
|
||||
weight: 1
|
||||
matches:
|
||||
- path:
|
||||
type: PathPrefix
|
||||
value: /
|
||||
@@ -0,0 +1,105 @@
|
||||
---
|
||||
# Declarative ClickHouse schema bootstrap. Runs as an ArgoCD PostSync hook so it
|
||||
# executes after the ClickHouseInstallation is reconciled, and re-runs on every
|
||||
# sync (idempotent CREATE ... IF NOT EXISTS). Edit the DDL here to evolve the
|
||||
# schema; the Vector aggregator writes to logs.raw with skip_unknown_fields, so
|
||||
# adding columns is backward-compatible.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: clickhouse-schema
|
||||
namespace: logging
|
||||
annotations:
|
||||
argocd.argoproj.io/hook: PostSync
|
||||
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
|
||||
labels:
|
||||
app.kubernetes.io/name: clickhouse-schema
|
||||
app.kubernetes.io/component: bootstrap
|
||||
spec:
|
||||
backoffLimit: 20
|
||||
activeDeadlineSeconds: 1800
|
||||
ttlSecondsAfterFinished: 3600
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: clickhouse-schema
|
||||
vector.dev/exclude: "true"
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 101
|
||||
runAsGroup: 101
|
||||
containers:
|
||||
- name: clickhouse-schema
|
||||
image: clickhouse/clickhouse-server:24.8
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
env:
|
||||
- name: HOME
|
||||
value: /tmp
|
||||
- name: CLICKHOUSE_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: clickhouse-credentials
|
||||
key: username
|
||||
- name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: clickhouse-credentials
|
||||
key: password
|
||||
command:
|
||||
- /bin/bash
|
||||
- -ec
|
||||
- |
|
||||
host=clickhouse-logs.logging.svc.cluster.local
|
||||
echo "Waiting for ClickHouse at ${host}:9000 ..."
|
||||
until clickhouse-client --host "$host" --port 9000 \
|
||||
--user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \
|
||||
--query "SELECT 1" >/dev/null 2>&1; do
|
||||
echo " not ready, retrying in 5s"; sleep 5
|
||||
done
|
||||
echo "Applying schema ..."
|
||||
clickhouse-client --host "$host" --port 9000 \
|
||||
--user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \
|
||||
--multiquery <<'EOSQL'
|
||||
CREATE DATABASE IF NOT EXISTS logs;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS logs.raw
|
||||
(
|
||||
timestamp DateTime64(3) DEFAULT now64(3),
|
||||
host LowCardinality(String) DEFAULT '',
|
||||
source LowCardinality(String) DEFAULT '',
|
||||
namespace LowCardinality(String) DEFAULT '',
|
||||
pod String DEFAULT '',
|
||||
container LowCardinality(String) DEFAULT '',
|
||||
stream LowCardinality(String) DEFAULT '',
|
||||
severity LowCardinality(String) DEFAULT '',
|
||||
message String DEFAULT '',
|
||||
labels Map(LowCardinality(String), String),
|
||||
fields Map(LowCardinality(String), String)
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(timestamp)
|
||||
ORDER BY (source, namespace, host, timestamp)
|
||||
TTL toDateTime(timestamp) + INTERVAL 30 DAY
|
||||
SETTINGS index_granularity = 8192;
|
||||
EOSQL
|
||||
echo "Schema applied."
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
@@ -0,0 +1,21 @@
|
||||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- vaultauth.yaml
|
||||
- vaultstaticsecret.yaml
|
||||
- clickhouseinstallation.yaml
|
||||
- job_clickhouse-schema.yaml
|
||||
- gateway.yaml
|
||||
- httproute.yaml
|
||||
|
||||
# Aggregator pipeline is the single source of truth (also validated by
|
||||
# `vector test` in CI). Mounted into the aggregator via `existingConfigMaps`.
|
||||
configMapGenerator:
|
||||
- name: vector-aggregator-config
|
||||
files:
|
||||
- aggregator.yaml=vector/aggregator.yaml
|
||||
options:
|
||||
disableNameSuffixHash: true
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: logging
|
||||
name: logging
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultAuth
|
||||
metadata:
|
||||
name: default
|
||||
namespace: logging
|
||||
spec:
|
||||
allowedNamespaces:
|
||||
- logging
|
||||
kubernetes:
|
||||
audiences:
|
||||
- vault
|
||||
role: default
|
||||
serviceAccount: default
|
||||
tokenExpirationSeconds: 600
|
||||
method: kubernetes
|
||||
mount: k8s/au/syd1
|
||||
vaultConnectionRef: vso-system/default
|
||||
@@ -0,0 +1,28 @@
|
||||
---
|
||||
# ClickHouse credentials for the `vector` user.
|
||||
#
|
||||
# Seed the Vault KV entry once (values are NOT stored in git), e.g.:
|
||||
# PW=$(openssl rand -base64 24)
|
||||
# HASH=$(printf '%s' "$PW" | sha256sum | cut -d' ' -f1)
|
||||
# vault kv put kv/kubernetes/namespace/logging/default/clickhouse-credentials \
|
||||
# username=vector password="$PW" password_sha256_hex="$HASH"
|
||||
#
|
||||
# The `logging/default` ServiceAccount reads this path via the templated
|
||||
# `policies/kv/kubernetes/default.yaml` policy (k8s auth role `default`), so no
|
||||
# terraform-vault change is required — only the value above must be written.
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultStaticSecret
|
||||
metadata:
|
||||
name: clickhouse-credentials
|
||||
namespace: logging
|
||||
spec:
|
||||
destination:
|
||||
create: true
|
||||
name: clickhouse-credentials
|
||||
overwrite: true
|
||||
hmacSecretData: true
|
||||
mount: kv
|
||||
path: kubernetes/namespace/logging/default/clickhouse-credentials
|
||||
refreshAfter: 5m
|
||||
type: kv-v2
|
||||
vaultAuthRef: default
|
||||
@@ -0,0 +1,49 @@
|
||||
---
|
||||
# `vector test` unit tests for the aggregator transforms. Merged with
|
||||
# aggregator.yaml in CI (.woodpecker/vector-test.yaml). This is the pattern the
|
||||
# per-app parsing follow-ups extend: add a test per new transform here.
|
||||
tests:
|
||||
- name: k8s_log_is_normalised
|
||||
inputs:
|
||||
- insert_at: k8s_shape
|
||||
type: log
|
||||
log_fields:
|
||||
message: "hello from pod"
|
||||
stream: "stdout"
|
||||
timestamp: "2026-07-27T00:00:00Z"
|
||||
kubernetes.pod_name: "web-abc"
|
||||
kubernetes.pod_namespace: "shop"
|
||||
kubernetes.container_name: "web"
|
||||
kubernetes.pod_node_name: "node-1"
|
||||
outputs:
|
||||
- extract_from: k8s_shape
|
||||
conditions:
|
||||
- type: vrl
|
||||
source: |
|
||||
assert_eq!(.source, "k8s")
|
||||
assert_eq!(.namespace, "shop")
|
||||
assert_eq!(.pod, "web-abc")
|
||||
assert_eq!(.container, "web")
|
||||
assert_eq!(.host, "node-1")
|
||||
assert_eq!(.stream, "stdout")
|
||||
assert_eq!(.message, "hello from pod")
|
||||
|
||||
- name: vm_log_is_normalised
|
||||
inputs:
|
||||
- insert_at: vm_shape
|
||||
type: log
|
||||
log_fields:
|
||||
message: "sshd started"
|
||||
host: "vm-db-1"
|
||||
severity: "info"
|
||||
role: "database"
|
||||
outputs:
|
||||
- extract_from: vm_shape
|
||||
conditions:
|
||||
- type: vrl
|
||||
source: |
|
||||
assert_eq!(.source, "vm")
|
||||
assert_eq!(.host, "vm-db-1")
|
||||
assert_eq!(.severity, "info")
|
||||
assert_eq!(.message, "sshd started")
|
||||
assert_eq!(.labels.role, "database")
|
||||
@@ -0,0 +1,122 @@
|
||||
---
|
||||
# Vector AGGREGATOR pipeline (the "brain") — single source of truth.
|
||||
#
|
||||
# This file is the ConfigMap the aggregator StatefulSet runs AND the config
|
||||
# `vector test` validates in CI (see .woodpecker/vector-test.yaml, which merges
|
||||
# this with aggregator-tests.yaml). The edge tier (k8s DaemonSet + future VM
|
||||
# agents) stays thin: it only collects and attaches source metadata, then
|
||||
# forwards over the Vector-native protocol. All shaping, routing, enrichment,
|
||||
# batching, buffering and the ONLY ClickHouse credentials live here.
|
||||
#
|
||||
# Per-app pipelines arrive as follow-up tasks and are aggregator-only changes:
|
||||
# add a `remap`/`route`/enrichment transform below and append its id to the
|
||||
# clickhouse sink `inputs` — no fleet/DaemonSet rollout required.
|
||||
data_dir: /vector-data-dir
|
||||
|
||||
api:
|
||||
enabled: true
|
||||
address: 0.0.0.0:8686
|
||||
|
||||
sources:
|
||||
# In-cluster pod logs from the Vector agent DaemonSet (Vector-native proto).
|
||||
from_agents:
|
||||
type: vector
|
||||
address: 0.0.0.0:6000
|
||||
|
||||
# Puppet-managed VM logs over HTTP (NDJSON), exposed via the logs-ingest
|
||||
# Gateway at logs-ingest.k8s.syd1.au.unkin.net.
|
||||
vm_http:
|
||||
type: http_server
|
||||
address: 0.0.0.0:8080
|
||||
path: /
|
||||
method: POST
|
||||
decoding:
|
||||
codec: json
|
||||
framing:
|
||||
method: newline_delimited
|
||||
|
||||
transforms:
|
||||
# ---- Default normalisation into the logs.raw columns. This is intentionally
|
||||
# the ONLY shaping today; per-app parsing is added here as follow-ups. ----
|
||||
k8s_shape:
|
||||
type: remap
|
||||
inputs:
|
||||
- from_agents
|
||||
source: |
|
||||
ts = .timestamp || now()
|
||||
node = to_string(.kubernetes.pod_node_name || "") ?? ""
|
||||
ns = to_string(.kubernetes.pod_namespace || "") ?? ""
|
||||
pod = to_string(.kubernetes.pod_name || "") ?? ""
|
||||
container = to_string(.kubernetes.container_name || "") ?? ""
|
||||
strm = to_string(.stream || "") ?? ""
|
||||
msg = to_string(.message || "") ?? ""
|
||||
lbls = object(.kubernetes.pod_labels) ?? {}
|
||||
. = {
|
||||
"timestamp": ts,
|
||||
"host": node,
|
||||
"source": "k8s",
|
||||
"namespace": ns,
|
||||
"pod": pod,
|
||||
"container": container,
|
||||
"stream": strm,
|
||||
"severity": "",
|
||||
"message": msg,
|
||||
"labels": lbls,
|
||||
"fields": {}
|
||||
}
|
||||
vm_shape:
|
||||
type: remap
|
||||
inputs:
|
||||
- vm_http
|
||||
source: |
|
||||
ts = .timestamp || .ts || now()
|
||||
host = to_string(.host || .hostname || "") ?? ""
|
||||
msg = to_string(.message || .msg || "") ?? ""
|
||||
sev = to_string(.severity || .level || "") ?? ""
|
||||
role = to_string(.role || "") ?? ""
|
||||
lbls = {}
|
||||
if role != "" {
|
||||
lbls = {"role": role}
|
||||
}
|
||||
. = {
|
||||
"timestamp": ts,
|
||||
"host": host,
|
||||
"source": "vm",
|
||||
"namespace": "",
|
||||
"pod": "",
|
||||
"container": "",
|
||||
"stream": "",
|
||||
"severity": sev,
|
||||
"message": msg,
|
||||
"labels": lbls,
|
||||
"fields": {}
|
||||
}
|
||||
|
||||
sinks:
|
||||
clickhouse:
|
||||
type: clickhouse
|
||||
inputs:
|
||||
- k8s_shape
|
||||
- vm_shape
|
||||
endpoint: http://clickhouse-logs.logging.svc.cluster.local:8123
|
||||
database: logs
|
||||
table: raw
|
||||
skip_unknown_fields: true
|
||||
date_time_best_effort: true
|
||||
auth:
|
||||
strategy: basic
|
||||
user: "${CLICKHOUSE_USER}"
|
||||
password: "${CLICKHOUSE_PASSWORD}"
|
||||
# Fat, infrequent inserts keep ClickHouse part-count low (avoid too-many-parts).
|
||||
batch:
|
||||
max_events: 500000
|
||||
max_bytes: 134217728
|
||||
timeout_secs: 10
|
||||
# Ride out a ClickHouse outage without dropping logs: on-disk buffer on the
|
||||
# aggregator PVC; block upstream (back-pressure to the edge) when full.
|
||||
buffer:
|
||||
type: disk
|
||||
max_size: 8589934592
|
||||
when_full: block
|
||||
healthcheck:
|
||||
enabled: true
|
||||
@@ -0,0 +1,16 @@
|
||||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
namespace: clickhouse-system
|
||||
|
||||
resources:
|
||||
- ../../../base/clickhouse-system
|
||||
|
||||
helmCharts:
|
||||
- name: altinity-clickhouse-operator
|
||||
repo: https://helm.altinity.com
|
||||
version: "0.27.2"
|
||||
releaseName: clickhouse-operator
|
||||
namespace: clickhouse-system
|
||||
valuesFile: values.yaml
|
||||
@@ -0,0 +1,29 @@
|
||||
# Altinity ClickHouse operator. Cluster-scoped: watches ClickHouseInstallation
|
||||
# resources in all namespaces (the logs cluster lives in the `logging` namespace).
|
||||
# CRDs are installed at runtime by the chart's crdHook Job.
|
||||
crdHook:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 128Mi
|
||||
|
||||
operator:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
metrics:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
@@ -0,0 +1,25 @@
|
||||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
namespace: logging
|
||||
|
||||
resources:
|
||||
- ../../../base/logging
|
||||
|
||||
helmCharts:
|
||||
# Node-level agent: tails every pod's logs and forwards to the aggregator.
|
||||
- name: vector
|
||||
repo: https://helm.vector.dev
|
||||
version: "0.57.0"
|
||||
releaseName: vector-agent
|
||||
namespace: logging
|
||||
valuesFile: values-vector-agent.yaml
|
||||
# Aggregator: receives from agents (vector proto) and from VMs (HTTP),
|
||||
# shapes events, and writes to ClickHouse.
|
||||
- name: vector
|
||||
repo: https://helm.vector.dev
|
||||
version: "0.57.0"
|
||||
releaseName: vector-aggregator
|
||||
namespace: logging
|
||||
valuesFile: values-vector-aggregator.yaml
|
||||
@@ -0,0 +1,44 @@
|
||||
# Vector Agent (DaemonSet) — captures ALL pod logs on every node (including
|
||||
# control-plane, via the blanket toleration) and forwards them to the
|
||||
# aggregator over the Vector native protocol. No per-app parsing here: shaping
|
||||
# and the ClickHouse write live on the aggregator.
|
||||
role: Agent
|
||||
fullnameOverride: vector-agent
|
||||
|
||||
rbac:
|
||||
create: true
|
||||
serviceAccount:
|
||||
create: true
|
||||
|
||||
podLabels:
|
||||
vector.dev/exclude: "true"
|
||||
|
||||
# Run on every node so no host's pod logs are missed.
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
|
||||
# Disable the chart's auto-generated service for the agent (it ships, not serves).
|
||||
service:
|
||||
enabled: false
|
||||
|
||||
customConfig:
|
||||
data_dir: /vector-data-dir
|
||||
api:
|
||||
enabled: false
|
||||
sources:
|
||||
kubernetes_logs:
|
||||
type: kubernetes_logs
|
||||
sinks:
|
||||
to_aggregator:
|
||||
type: vector
|
||||
inputs:
|
||||
- kubernetes_logs
|
||||
address: vector-aggregator.logging.svc.cluster.local:6000
|
||||
@@ -0,0 +1,82 @@
|
||||
# Vector AGGREGATOR (StatefulSet) — the "brain" tier: sole ClickHouse writer,
|
||||
# owns all transforms, holds the only ClickHouse credentials, HA replicas with
|
||||
# on-disk buffers to ride out a ClickHouse outage.
|
||||
#
|
||||
# The pipeline itself is NOT inlined here: it lives in
|
||||
# apps/base/logging/vector/aggregator.yaml (single source of truth, unit-tested
|
||||
# by `vector test` in CI) and is mounted via existingConfigMaps. This file only
|
||||
# owns the k8s deployment shape (ports, creds, storage, HA).
|
||||
role: Aggregator
|
||||
fullnameOverride: vector-aggregator
|
||||
replicas: 2
|
||||
|
||||
# Reload the StatefulSet when the aggregator ConfigMap or the creds Secret change.
|
||||
workloadResourceAnnotations:
|
||||
reloader.stakater.com/auto: "true"
|
||||
|
||||
podLabels:
|
||||
vector.dev/exclude: "true"
|
||||
|
||||
# Mount the aggregator pipeline ConfigMap (rendered from base by kustomize).
|
||||
# dataDir must be set when using existingConfigMaps; it is also where the
|
||||
# clickhouse sink's disk buffer lives (backed by the PVC below).
|
||||
dataDir: /vector-data-dir
|
||||
existingConfigMaps:
|
||||
- vector-aggregator-config
|
||||
|
||||
# On-disk buffer storage so a ClickHouse outage does not drop logs.
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClassName: cephrbd-fast-delete
|
||||
size: 20Gi
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
|
||||
# ClickHouse basic-auth creds — the ONLY place these are consumed.
|
||||
env:
|
||||
- name: CLICKHOUSE_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: clickhouse-credentials
|
||||
key: username
|
||||
- name: CLICKHOUSE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: clickhouse-credentials
|
||||
key: password
|
||||
|
||||
containerPorts:
|
||||
- name: vector
|
||||
containerPort: 6000
|
||||
protocol: TCP
|
||||
- name: http-ingest
|
||||
containerPort: 8080
|
||||
protocol: TCP
|
||||
- name: api
|
||||
containerPort: 8686
|
||||
protocol: TCP
|
||||
|
||||
service:
|
||||
enabled: true
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- name: vector
|
||||
port: 6000
|
||||
targetPort: 6000
|
||||
protocol: TCP
|
||||
- name: http-ingest
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
protocol: TCP
|
||||
- name: api
|
||||
port: 8686
|
||||
targetPort: 8686
|
||||
protocol: TCP
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 2Gi
|
||||
@@ -4,6 +4,7 @@ kind: Kustomization
|
||||
|
||||
resources:
|
||||
- aitooling.yaml
|
||||
- logging.yaml
|
||||
- observability.yaml
|
||||
- platform.yaml
|
||||
- storage.yaml
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
---
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: ApplicationSet
|
||||
metadata:
|
||||
name: logging-apps
|
||||
namespace: argocd
|
||||
spec:
|
||||
generators:
|
||||
- git:
|
||||
repoURL: https://git.unkin.net/unkin/argocd-apps
|
||||
revision: HEAD
|
||||
directories:
|
||||
- path: apps/overlays/*/clickhouse-system
|
||||
- path: apps/overlays/*/logging
|
||||
template:
|
||||
metadata:
|
||||
name: 'logging-{{path[3]}}'
|
||||
spec:
|
||||
project: logging
|
||||
source:
|
||||
repoURL: https://git.unkin.net/unkin/argocd-apps
|
||||
targetRevision: HEAD
|
||||
path: '{{path}}'
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: '{{path[3]}}'
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- ServerSideApply=true
|
||||
- CreateNamespace=false
|
||||
@@ -4,6 +4,7 @@ kind: Kustomization
|
||||
|
||||
resources:
|
||||
- aitooling.yaml
|
||||
- logging.yaml
|
||||
- observability.yaml
|
||||
- platform.yaml
|
||||
- storage.yaml
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
---
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: AppProject
|
||||
metadata:
|
||||
name: logging
|
||||
namespace: argocd
|
||||
spec:
|
||||
description: Centralized logging stack (ClickHouse + Vector)
|
||||
sourceRepos:
|
||||
- https://git.unkin.net/unkin/argocd-apps
|
||||
- https://helm.altinity.com
|
||||
- https://helm.vector.dev
|
||||
destinations:
|
||||
- namespace: 'logging'
|
||||
server: https://kubernetes.default.svc
|
||||
- namespace: 'clickhouse-system'
|
||||
server: https://kubernetes.default.svc
|
||||
clusterResourceWhitelist:
|
||||
- group: ''
|
||||
kind: Namespace
|
||||
- group: 'rbac.authorization.k8s.io'
|
||||
kind: ClusterRole
|
||||
- group: 'rbac.authorization.k8s.io'
|
||||
kind: ClusterRoleBinding
|
||||
- group: 'apiextensions.k8s.io'
|
||||
kind: CustomResourceDefinition
|
||||
namespaceResourceWhitelist:
|
||||
- group: '*'
|
||||
kind: '*'
|
||||
Reference in New Issue
Block a user