diff --git a/.woodpecker/vector-test.yaml b/.woodpecker/vector-test.yaml new file mode 100644 index 0000000..9cea466 --- /dev/null +++ b/.woodpecker/vector-test.yaml @@ -0,0 +1,22 @@ +when: + - event: pull_request + +steps: + - name: vector-test + image: timberio/vector:0.57.0-debian + commands: + # Dummy creds + a writable data_dir so the full topology (incl. the + # ClickHouse sink disk buffer) builds; tests only exercise the transforms. + - export CLICKHOUSE_USER=ci CLICKHOUSE_PASSWORD=ci + - mkdir -p /vector-data-dir + - vector test apps/base/logging/vector/aggregator.yaml apps/base/logging/vector/aggregator-tests.yaml + backend_options: + kubernetes: + serviceAccountName: default + resources: + requests: + memory: 256Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 1 diff --git a/apps/base/clickhouse-system/kustomization.yaml b/apps/base/clickhouse-system/kustomization.yaml new file mode 100644 index 0000000..5122f9e --- /dev/null +++ b/apps/base/clickhouse-system/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml diff --git a/apps/base/clickhouse-system/namespace.yaml b/apps/base/clickhouse-system/namespace.yaml new file mode 100644 index 0000000..519dfb3 --- /dev/null +++ b/apps/base/clickhouse-system/namespace.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/name: clickhouse-system + name: clickhouse-system diff --git a/apps/base/logging/clickhouseinstallation.yaml b/apps/base/logging/clickhouseinstallation.yaml new file mode 100644 index 0000000..ea6bb44 --- /dev/null +++ b/apps/base/logging/clickhouseinstallation.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: clickhouse.altinity.com/v1 +kind: ClickHouseInstallation +metadata: + name: logs + namespace: logging +spec: + defaults: + templates: + dataVolumeClaimTemplate: data-volume + serviceTemplate: chi-service + podTemplate: clickhouse + configuration: + users: + # Password hash is sourced from the Vault-synced clickhouse-credentials + # Secret; the plaintext never lands in git or the ClickHouse config. + vector/password_sha256_hex: + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: password_sha256_hex + vector/networks/ip: + - "::/0" + vector/profile: default + vector/quota: default + # Allow the vector user to create the logs database/table (bootstrap Job) + # and to INSERT. Restrict the built-in default user to loopback only. + vector/access_management: "1" + default/networks/ip: + - "127.0.0.1" + - "::1" + profiles: + default/max_memory_usage: "10000000000" + default/max_execution_time: "120" + clusters: + - name: logs + layout: + shardsCount: 1 + replicasCount: 1 + templates: + volumeClaimTemplates: + - name: data-volume + spec: + storageClassName: cephrbd-fast-delete + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 200Gi + serviceTemplates: + - name: chi-service + generateName: "clickhouse-{chi}" + spec: + type: ClusterIP + ports: + - name: http + port: 8123 + - name: tcp + port: 9000 + podTemplates: + - name: clickhouse + spec: + securityContext: + fsGroup: 101 + runAsUser: 101 + runAsGroup: 101 + containers: + - name: clickhouse + image: clickhouse/clickhouse-server:24.8 + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: "2" + memory: 8Gi diff --git a/apps/base/logging/gateway.yaml b/apps/base/logging/gateway.yaml new file mode 100644 index 0000000..bb072dc --- /dev/null +++ b/apps/base/logging/gateway.yaml @@ -0,0 +1,44 @@ +--- +# Log ingestion endpoint for puppet-managed VMs (and any non-k8s client). +# Reuses the internal Traefik gateway + cert-manager + external-dns pattern so +# VMs reach the Vector aggregator's HTTP source over TLS at a DNS name they can +# resolve. The puppet-side Vector rollout ships NDJSON to +# https://logs-ingest.k8s.syd1.au.unkin.net/ (a later task). +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: logs-ingest + namespace: logging + labels: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/component: ingest + traefik.io/instance: internal + annotations: + cert-manager.io/cluster-issuer: vault-issuer + cert-manager.io/common-name: logs-ingest.k8s.syd1.au.unkin.net + cert-manager.io/private-key-size: "4096" + external-dns.alpha.kubernetes.io/hostname: logs-ingest.k8s.syd1.au.unkin.net + external-dns.alpha.kubernetes.io/target: 198.18.200.4 +spec: + gatewayClassName: traefik-internal + listeners: + - name: http + port: 80 + protocol: HTTP + hostname: logs-ingest.k8s.syd1.au.unkin.net + allowedRoutes: + namespaces: + from: Same + - name: https + port: 443 + protocol: HTTPS + hostname: logs-ingest.k8s.syd1.au.unkin.net + allowedRoutes: + namespaces: + from: Same + tls: + mode: Terminate + certificateRefs: + - group: "" + kind: Secret + name: logs-ingest-tls diff --git a/apps/base/logging/httproute.yaml b/apps/base/logging/httproute.yaml new file mode 100644 index 0000000..ba8f241 --- /dev/null +++ b/apps/base/logging/httproute.yaml @@ -0,0 +1,55 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: logs-ingest-http-redirect + namespace: logging + labels: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/component: ingest +spec: + hostnames: + - logs-ingest.k8s.syd1.au.unkin.net + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: logs-ingest + sectionName: http + rules: + - filters: + - type: RequestRedirect + requestRedirect: + scheme: https + statusCode: 301 + matches: + - path: + type: PathPrefix + value: / +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: logs-ingest + namespace: logging + labels: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/component: ingest +spec: + hostnames: + - logs-ingest.k8s.syd1.au.unkin.net + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: logs-ingest + sectionName: https + rules: + - backendRefs: + - group: "" + kind: Service + name: vector-aggregator + port: 8080 + weight: 1 + matches: + - path: + type: PathPrefix + value: / diff --git a/apps/base/logging/job_clickhouse-schema.yaml b/apps/base/logging/job_clickhouse-schema.yaml new file mode 100644 index 0000000..b1e3dd4 --- /dev/null +++ b/apps/base/logging/job_clickhouse-schema.yaml @@ -0,0 +1,105 @@ +--- +# Declarative ClickHouse schema bootstrap. Runs as an ArgoCD PostSync hook so it +# executes after the ClickHouseInstallation is reconciled, and re-runs on every +# sync (idempotent CREATE ... IF NOT EXISTS). Edit the DDL here to evolve the +# schema; the Vector aggregator writes to logs.raw with skip_unknown_fields, so +# adding columns is backward-compatible. +apiVersion: batch/v1 +kind: Job +metadata: + name: clickhouse-schema + namespace: logging + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation + labels: + app.kubernetes.io/name: clickhouse-schema + app.kubernetes.io/component: bootstrap +spec: + backoffLimit: 20 + activeDeadlineSeconds: 1800 + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + app.kubernetes.io/name: clickhouse-schema + vector.dev/exclude: "true" + spec: + restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + containers: + - name: clickhouse-schema + image: clickhouse/clickhouse-server:24.8 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + env: + - name: HOME + value: /tmp + - name: CLICKHOUSE_USER + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: username + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: password + command: + - /bin/bash + - -ec + - | + host=clickhouse-logs.logging.svc.cluster.local + echo "Waiting for ClickHouse at ${host}:9000 ..." + until clickhouse-client --host "$host" --port 9000 \ + --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \ + --query "SELECT 1" >/dev/null 2>&1; do + echo " not ready, retrying in 5s"; sleep 5 + done + echo "Applying schema ..." + clickhouse-client --host "$host" --port 9000 \ + --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \ + --multiquery <<'EOSQL' + CREATE DATABASE IF NOT EXISTS logs; + + CREATE TABLE IF NOT EXISTS logs.raw + ( + timestamp DateTime64(3) DEFAULT now64(3), + host LowCardinality(String) DEFAULT '', + source LowCardinality(String) DEFAULT '', + namespace LowCardinality(String) DEFAULT '', + pod String DEFAULT '', + container LowCardinality(String) DEFAULT '', + stream LowCardinality(String) DEFAULT '', + severity LowCardinality(String) DEFAULT '', + message String DEFAULT '', + labels Map(LowCardinality(String), String), + fields Map(LowCardinality(String), String) + ) + ENGINE = MergeTree + PARTITION BY toDate(timestamp) + ORDER BY (source, namespace, host, timestamp) + TTL toDateTime(timestamp) + INTERVAL 30 DAY + SETTINGS index_granularity = 8192; + EOSQL + echo "Schema applied." + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/apps/base/logging/kustomization.yaml b/apps/base/logging/kustomization.yaml new file mode 100644 index 0000000..934f466 --- /dev/null +++ b/apps/base/logging/kustomization.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - vaultauth.yaml + - vaultstaticsecret.yaml + - clickhouseinstallation.yaml + - job_clickhouse-schema.yaml + - gateway.yaml + - httproute.yaml + +# Aggregator pipeline is the single source of truth (also validated by +# `vector test` in CI). Mounted into the aggregator via `existingConfigMaps`. +configMapGenerator: + - name: vector-aggregator-config + files: + - aggregator.yaml=vector/aggregator.yaml + options: + disableNameSuffixHash: true diff --git a/apps/base/logging/namespace.yaml b/apps/base/logging/namespace.yaml new file mode 100644 index 0000000..1ccca01 --- /dev/null +++ b/apps/base/logging/namespace.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/name: logging + name: logging diff --git a/apps/base/logging/vaultauth.yaml b/apps/base/logging/vaultauth.yaml new file mode 100644 index 0000000..bf3b375 --- /dev/null +++ b/apps/base/logging/vaultauth.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: default + namespace: logging +spec: + allowedNamespaces: + - logging + kubernetes: + audiences: + - vault + role: default + serviceAccount: default + tokenExpirationSeconds: 600 + method: kubernetes + mount: k8s/au/syd1 + vaultConnectionRef: vso-system/default diff --git a/apps/base/logging/vaultstaticsecret.yaml b/apps/base/logging/vaultstaticsecret.yaml new file mode 100644 index 0000000..d37129c --- /dev/null +++ b/apps/base/logging/vaultstaticsecret.yaml @@ -0,0 +1,28 @@ +--- +# ClickHouse credentials for the `vector` user. +# +# Seed the Vault KV entry once (values are NOT stored in git), e.g.: +# PW=$(openssl rand -base64 24) +# HASH=$(printf '%s' "$PW" | sha256sum | cut -d' ' -f1) +# vault kv put kv/kubernetes/namespace/logging/default/clickhouse-credentials \ +# username=vector password="$PW" password_sha256_hex="$HASH" +# +# The `logging/default` ServiceAccount reads this path via the templated +# `policies/kv/kubernetes/default.yaml` policy (k8s auth role `default`), so no +# terraform-vault change is required — only the value above must be written. +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: clickhouse-credentials + namespace: logging +spec: + destination: + create: true + name: clickhouse-credentials + overwrite: true + hmacSecretData: true + mount: kv + path: kubernetes/namespace/logging/default/clickhouse-credentials + refreshAfter: 5m + type: kv-v2 + vaultAuthRef: default diff --git a/apps/base/logging/vector/aggregator-tests.yaml b/apps/base/logging/vector/aggregator-tests.yaml new file mode 100644 index 0000000..94a8c60 --- /dev/null +++ b/apps/base/logging/vector/aggregator-tests.yaml @@ -0,0 +1,49 @@ +--- +# `vector test` unit tests for the aggregator transforms. Merged with +# aggregator.yaml in CI (.woodpecker/vector-test.yaml). This is the pattern the +# per-app parsing follow-ups extend: add a test per new transform here. +tests: + - name: k8s_log_is_normalised + inputs: + - insert_at: k8s_shape + type: log + log_fields: + message: "hello from pod" + stream: "stdout" + timestamp: "2026-07-27T00:00:00Z" + kubernetes.pod_name: "web-abc" + kubernetes.pod_namespace: "shop" + kubernetes.container_name: "web" + kubernetes.pod_node_name: "node-1" + outputs: + - extract_from: k8s_shape + conditions: + - type: vrl + source: | + assert_eq!(.source, "k8s") + assert_eq!(.namespace, "shop") + assert_eq!(.pod, "web-abc") + assert_eq!(.container, "web") + assert_eq!(.host, "node-1") + assert_eq!(.stream, "stdout") + assert_eq!(.message, "hello from pod") + + - name: vm_log_is_normalised + inputs: + - insert_at: vm_shape + type: log + log_fields: + message: "sshd started" + host: "vm-db-1" + severity: "info" + role: "database" + outputs: + - extract_from: vm_shape + conditions: + - type: vrl + source: | + assert_eq!(.source, "vm") + assert_eq!(.host, "vm-db-1") + assert_eq!(.severity, "info") + assert_eq!(.message, "sshd started") + assert_eq!(.labels.role, "database") diff --git a/apps/base/logging/vector/aggregator.yaml b/apps/base/logging/vector/aggregator.yaml new file mode 100644 index 0000000..78bd816 --- /dev/null +++ b/apps/base/logging/vector/aggregator.yaml @@ -0,0 +1,122 @@ +--- +# Vector AGGREGATOR pipeline (the "brain") — single source of truth. +# +# This file is the ConfigMap the aggregator StatefulSet runs AND the config +# `vector test` validates in CI (see .woodpecker/vector-test.yaml, which merges +# this with aggregator-tests.yaml). The edge tier (k8s DaemonSet + future VM +# agents) stays thin: it only collects and attaches source metadata, then +# forwards over the Vector-native protocol. All shaping, routing, enrichment, +# batching, buffering and the ONLY ClickHouse credentials live here. +# +# Per-app pipelines arrive as follow-up tasks and are aggregator-only changes: +# add a `remap`/`route`/enrichment transform below and append its id to the +# clickhouse sink `inputs` — no fleet/DaemonSet rollout required. +data_dir: /vector-data-dir + +api: + enabled: true + address: 0.0.0.0:8686 + +sources: + # In-cluster pod logs from the Vector agent DaemonSet (Vector-native proto). + from_agents: + type: vector + address: 0.0.0.0:6000 + + # Puppet-managed VM logs over HTTP (NDJSON), exposed via the logs-ingest + # Gateway at logs-ingest.k8s.syd1.au.unkin.net. + vm_http: + type: http_server + address: 0.0.0.0:8080 + path: / + method: POST + decoding: + codec: json + framing: + method: newline_delimited + +transforms: + # ---- Default normalisation into the logs.raw columns. This is intentionally + # the ONLY shaping today; per-app parsing is added here as follow-ups. ---- + k8s_shape: + type: remap + inputs: + - from_agents + source: | + ts = .timestamp || now() + node = to_string(.kubernetes.pod_node_name || "") ?? "" + ns = to_string(.kubernetes.pod_namespace || "") ?? "" + pod = to_string(.kubernetes.pod_name || "") ?? "" + container = to_string(.kubernetes.container_name || "") ?? "" + strm = to_string(.stream || "") ?? "" + msg = to_string(.message || "") ?? "" + lbls = object(.kubernetes.pod_labels) ?? {} + . = { + "timestamp": ts, + "host": node, + "source": "k8s", + "namespace": ns, + "pod": pod, + "container": container, + "stream": strm, + "severity": "", + "message": msg, + "labels": lbls, + "fields": {} + } + vm_shape: + type: remap + inputs: + - vm_http + source: | + ts = .timestamp || .ts || now() + host = to_string(.host || .hostname || "") ?? "" + msg = to_string(.message || .msg || "") ?? "" + sev = to_string(.severity || .level || "") ?? "" + role = to_string(.role || "") ?? "" + lbls = {} + if role != "" { + lbls = {"role": role} + } + . = { + "timestamp": ts, + "host": host, + "source": "vm", + "namespace": "", + "pod": "", + "container": "", + "stream": "", + "severity": sev, + "message": msg, + "labels": lbls, + "fields": {} + } + +sinks: + clickhouse: + type: clickhouse + inputs: + - k8s_shape + - vm_shape + endpoint: http://clickhouse-logs.logging.svc.cluster.local:8123 + database: logs + table: raw + skip_unknown_fields: true + date_time_best_effort: true + auth: + strategy: basic + user: "${CLICKHOUSE_USER}" + password: "${CLICKHOUSE_PASSWORD}" + # Fat, infrequent inserts keep ClickHouse part-count low (avoid too-many-parts). + batch: + max_events: 500000 + max_bytes: 134217728 + timeout_secs: 10 + # Ride out a ClickHouse outage without dropping logs: on-disk buffer on the + # aggregator PVC; block upstream (back-pressure to the edge) when full. + buffer: + type: disk + max_size: 8589934592 + when_full: block + healthcheck: + enabled: true diff --git a/apps/overlays/au-syd1/clickhouse-system/kustomization.yaml b/apps/overlays/au-syd1/clickhouse-system/kustomization.yaml new file mode 100644 index 0000000..498f573 --- /dev/null +++ b/apps/overlays/au-syd1/clickhouse-system/kustomization.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: clickhouse-system + +resources: + - ../../../base/clickhouse-system + +helmCharts: + - name: altinity-clickhouse-operator + repo: https://helm.altinity.com + version: "0.27.2" + releaseName: clickhouse-operator + namespace: clickhouse-system + valuesFile: values.yaml diff --git a/apps/overlays/au-syd1/clickhouse-system/values.yaml b/apps/overlays/au-syd1/clickhouse-system/values.yaml new file mode 100644 index 0000000..8eb31f0 --- /dev/null +++ b/apps/overlays/au-syd1/clickhouse-system/values.yaml @@ -0,0 +1,29 @@ +# Altinity ClickHouse operator. Cluster-scoped: watches ClickHouseInstallation +# resources in all namespaces (the logs cluster lives in the `logging` namespace). +# CRDs are installed at runtime by the chart's crdHook Job. +crdHook: + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 250m + memory: 128Mi + +operator: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +metrics: + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi diff --git a/apps/overlays/au-syd1/logging/kustomization.yaml b/apps/overlays/au-syd1/logging/kustomization.yaml new file mode 100644 index 0000000..0ae2aa6 --- /dev/null +++ b/apps/overlays/au-syd1/logging/kustomization.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: logging + +resources: + - ../../../base/logging + +helmCharts: + # Node-level agent: tails every pod's logs and forwards to the aggregator. + - name: vector + repo: https://helm.vector.dev + version: "0.57.0" + releaseName: vector-agent + namespace: logging + valuesFile: values-vector-agent.yaml + # Aggregator: receives from agents (vector proto) and from VMs (HTTP), + # shapes events, and writes to ClickHouse. + - name: vector + repo: https://helm.vector.dev + version: "0.57.0" + releaseName: vector-aggregator + namespace: logging + valuesFile: values-vector-aggregator.yaml diff --git a/apps/overlays/au-syd1/logging/values-vector-agent.yaml b/apps/overlays/au-syd1/logging/values-vector-agent.yaml new file mode 100644 index 0000000..3e8e064 --- /dev/null +++ b/apps/overlays/au-syd1/logging/values-vector-agent.yaml @@ -0,0 +1,44 @@ +# Vector Agent (DaemonSet) — captures ALL pod logs on every node (including +# control-plane, via the blanket toleration) and forwards them to the +# aggregator over the Vector native protocol. No per-app parsing here: shaping +# and the ClickHouse write live on the aggregator. +role: Agent +fullnameOverride: vector-agent + +rbac: + create: true +serviceAccount: + create: true + +podLabels: + vector.dev/exclude: "true" + +# Run on every node so no host's pod logs are missed. +tolerations: + - operator: Exists + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + +# Disable the chart's auto-generated service for the agent (it ships, not serves). +service: + enabled: false + +customConfig: + data_dir: /vector-data-dir + api: + enabled: false + sources: + kubernetes_logs: + type: kubernetes_logs + sinks: + to_aggregator: + type: vector + inputs: + - kubernetes_logs + address: vector-aggregator.logging.svc.cluster.local:6000 diff --git a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml new file mode 100644 index 0000000..681d21f --- /dev/null +++ b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml @@ -0,0 +1,82 @@ +# Vector AGGREGATOR (StatefulSet) — the "brain" tier: sole ClickHouse writer, +# owns all transforms, holds the only ClickHouse credentials, HA replicas with +# on-disk buffers to ride out a ClickHouse outage. +# +# The pipeline itself is NOT inlined here: it lives in +# apps/base/logging/vector/aggregator.yaml (single source of truth, unit-tested +# by `vector test` in CI) and is mounted via existingConfigMaps. This file only +# owns the k8s deployment shape (ports, creds, storage, HA). +role: Aggregator +fullnameOverride: vector-aggregator +replicas: 2 + +# Reload the StatefulSet when the aggregator ConfigMap or the creds Secret change. +workloadResourceAnnotations: + reloader.stakater.com/auto: "true" + +podLabels: + vector.dev/exclude: "true" + +# Mount the aggregator pipeline ConfigMap (rendered from base by kustomize). +# dataDir must be set when using existingConfigMaps; it is also where the +# clickhouse sink's disk buffer lives (backed by the PVC below). +dataDir: /vector-data-dir +existingConfigMaps: + - vector-aggregator-config + +# On-disk buffer storage so a ClickHouse outage does not drop logs. +persistence: + enabled: true + storageClassName: cephrbd-fast-delete + size: 20Gi + accessModes: + - ReadWriteOnce + +# ClickHouse basic-auth creds — the ONLY place these are consumed. +env: + - name: CLICKHOUSE_USER + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: username + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: password + +containerPorts: + - name: vector + containerPort: 6000 + protocol: TCP + - name: http-ingest + containerPort: 8080 + protocol: TCP + - name: api + containerPort: 8686 + protocol: TCP + +service: + enabled: true + type: ClusterIP + ports: + - name: vector + port: 6000 + targetPort: 6000 + protocol: TCP + - name: http-ingest + port: 8080 + targetPort: 8080 + protocol: TCP + - name: api + port: 8686 + targetPort: 8686 + protocol: TCP + +resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi diff --git a/argocd/applicationsets/kustomization.yaml b/argocd/applicationsets/kustomization.yaml index 87d07c2..b54cc35 100644 --- a/argocd/applicationsets/kustomization.yaml +++ b/argocd/applicationsets/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - aitooling.yaml + - logging.yaml - observability.yaml - platform.yaml - storage.yaml diff --git a/argocd/applicationsets/logging.yaml b/argocd/applicationsets/logging.yaml new file mode 100644 index 0000000..ea542d4 --- /dev/null +++ b/argocd/applicationsets/logging.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: logging-apps + namespace: argocd +spec: + generators: + - git: + repoURL: https://git.unkin.net/unkin/argocd-apps + revision: HEAD + directories: + - path: apps/overlays/*/clickhouse-system + - path: apps/overlays/*/logging + template: + metadata: + name: 'logging-{{path[3]}}' + spec: + project: logging + source: + repoURL: https://git.unkin.net/unkin/argocd-apps + targetRevision: HEAD + path: '{{path}}' + destination: + server: https://kubernetes.default.svc + namespace: '{{path[3]}}' + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true + - CreateNamespace=false diff --git a/argocd/projects/kustomization.yaml b/argocd/projects/kustomization.yaml index 87d07c2..b54cc35 100644 --- a/argocd/projects/kustomization.yaml +++ b/argocd/projects/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - aitooling.yaml + - logging.yaml - observability.yaml - platform.yaml - storage.yaml diff --git a/argocd/projects/logging.yaml b/argocd/projects/logging.yaml new file mode 100644 index 0000000..591153a --- /dev/null +++ b/argocd/projects/logging.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: logging + namespace: argocd +spec: + description: Centralized logging stack (ClickHouse + Vector) + sourceRepos: + - https://git.unkin.net/unkin/argocd-apps + - https://helm.altinity.com + - https://helm.vector.dev + destinations: + - namespace: 'logging' + server: https://kubernetes.default.svc + - namespace: 'clickhouse-system' + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: '' + kind: Namespace + - group: 'rbac.authorization.k8s.io' + kind: ClusterRole + - group: 'rbac.authorization.k8s.io' + kind: ClusterRoleBinding + - group: 'apiextensions.k8s.io' + kind: CustomResourceDefinition + namespaceResourceWhitelist: + - group: '*' + kind: '*'