diff --git a/.woodpecker/vector-test.yaml b/.woodpecker/vector-test.yaml new file mode 100644 index 0000000..aace88e --- /dev/null +++ b/.woodpecker/vector-test.yaml @@ -0,0 +1,29 @@ +when: + - event: pull_request + +steps: + - name: vector-test + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector:0.57.0-debian + commands: + # Dummy creds + writable dirs so the full topologies build; the unit tests + # only exercise the transforms (sources are not started). + - export CLICKHOUSE_USER=ci CLICKHOUSE_PASSWORD=ci + - export NATS_PRODUCER_PASSWORD=ci NATS_CONSUMER_PASSWORD=ci + - mkdir -p /vector-data-dir /etc/vault-ca + - cp /etc/ssl/certs/ca-certificates.crt /etc/vault-ca/ca.crt + # Transform tier + VM ingest: unit-tested transforms. + - vector test apps/base/logging/vector/aggregator.yaml apps/base/logging/vector/aggregator-tests.yaml + - vector test apps/base/logging/vector/vm-ingest.yaml apps/base/logging/vector/vm-ingest-tests.yaml + # Agent + archiver have no transforms to unit-test; validate they build. + - vector validate --no-environment apps/base/logging/vector/agent.yaml + - vector validate --no-environment apps/base/logging/vector/archiver.yaml + backend_options: + kubernetes: + serviceAccountName: default + resources: + requests: + memory: 256Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 1 diff --git a/apps/base/clickhouse-system/kustomization.yaml b/apps/base/clickhouse-system/kustomization.yaml new file mode 100644 index 0000000..5122f9e --- /dev/null +++ b/apps/base/clickhouse-system/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml diff --git a/apps/base/clickhouse-system/namespace.yaml b/apps/base/clickhouse-system/namespace.yaml new file mode 100644 index 0000000..519dfb3 --- /dev/null +++ b/apps/base/clickhouse-system/namespace.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/name: clickhouse-system + name: clickhouse-system diff --git a/apps/base/logging/cephrgw.yaml b/apps/base/logging/cephrgw.yaml new file mode 100644 index 0000000..ee1bd8e --- /dev/null +++ b/apps/base/logging/cephrgw.yaml @@ -0,0 +1,43 @@ +--- +# S3 bucket (Ceph RGW) for the long-term raw-log archive, provisioned by the +# in-estate cephrgw-operator. The archiver Vector deployment writes here. +apiVersion: ceph.unkin.net/v1alpha1 +kind: ObjectStoreUser +metadata: + name: logs-archive-owner + namespace: logging +spec: + displayName: "Logging raw-archive bucket owner" + maxBuckets: 5 + quota: + enabled: true + # 5 TiB soft cap; real retention is enforced RGW-side by a bucket lifecycle + # policy (see PR notes) — the operator does not manage lifecycle. + maxSizeBytes: 5497558138880 +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: Bucket +metadata: + name: logs-archive + namespace: logging +spec: + bucketName: logs-archive + ownerRef: logs-archive-owner + versioning: false + tags: + app: logging + purpose: raw-log-archive + # Keep the bucket (and its objects) if this CR is ever deleted. + retainOnDelete: true +--- +apiVersion: ceph.unkin.net/v1alpha1 +kind: BucketAccess +metadata: + name: logs-archive-writer + namespace: logging +spec: + bucketRef: logs-archive + level: read-write + # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID, + # S3_ENDPOINT, BUCKET_NAME) into this Secret; the archiver consumes it. + secretName: logs-archive-s3 diff --git a/apps/base/logging/clickhouseinstallation.yaml b/apps/base/logging/clickhouseinstallation.yaml new file mode 100644 index 0000000..f6b63f0 --- /dev/null +++ b/apps/base/logging/clickhouseinstallation.yaml @@ -0,0 +1,80 @@ +--- +apiVersion: clickhouse.altinity.com/v1 +kind: ClickHouseInstallation +metadata: + name: logs + namespace: logging +spec: + defaults: + templates: + dataVolumeClaimTemplate: data-volume + serviceTemplate: chi-service + podTemplate: clickhouse + configuration: + users: + # Password hash is sourced from the Vault-synced clickhouse-credentials + # Secret; the plaintext never lands in git or the ClickHouse config. + vector/password_sha256_hex: + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: password_sha256_hex + vector/networks/ip: + - "::/0" + vector/profile: default + vector/quota: default + # Allow the vector user to create the logs database/table (bootstrap Job) + # and to INSERT. Restrict the built-in default user to loopback only. + vector/access_management: "1" + default/networks/ip: + - "127.0.0.1" + - "::1" + profiles: + default/max_memory_usage: "10000000000" + default/max_execution_time: "120" + clusters: + - name: logs + layout: + shardsCount: 1 + replicasCount: 1 + templates: + volumeClaimTemplates: + - name: data-volume + spec: + storageClassName: cephrbd-fast-delete + accessModes: + - ReadWriteOnce + resources: + requests: + # 3d TTL on logs.raw. At ~130 GiB/day raw, ClickHouse LZ4/ZSTD + # (~6x on log text) stores ~20-25 GiB/day => ~60-75 GiB/3d, plus + # merge headroom (~2x peak). logs.raw is the only table. 150Gi + # gives comfortable headroom; long-term data lives in S3, not here. + storage: 150Gi + serviceTemplates: + - name: chi-service + generateName: "clickhouse-{chi}" + spec: + type: ClusterIP + ports: + - name: http + port: 8123 + - name: tcp + port: 9000 + podTemplates: + - name: clickhouse + spec: + securityContext: + fsGroup: 101 + runAsUser: 101 + runAsGroup: 101 + containers: + - name: clickhouse + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/clickhouse/clickhouse-server:24.8 + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: "2" + memory: 8Gi diff --git a/apps/base/logging/gateway.yaml b/apps/base/logging/gateway.yaml new file mode 100644 index 0000000..bb072dc --- /dev/null +++ b/apps/base/logging/gateway.yaml @@ -0,0 +1,44 @@ +--- +# Log ingestion endpoint for puppet-managed VMs (and any non-k8s client). +# Reuses the internal Traefik gateway + cert-manager + external-dns pattern so +# VMs reach the Vector aggregator's HTTP source over TLS at a DNS name they can +# resolve. The puppet-side Vector rollout ships NDJSON to +# https://logs-ingest.k8s.syd1.au.unkin.net/ (a later task). +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: logs-ingest + namespace: logging + labels: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/component: ingest + traefik.io/instance: internal + annotations: + cert-manager.io/cluster-issuer: vault-issuer + cert-manager.io/common-name: logs-ingest.k8s.syd1.au.unkin.net + cert-manager.io/private-key-size: "4096" + external-dns.alpha.kubernetes.io/hostname: logs-ingest.k8s.syd1.au.unkin.net + external-dns.alpha.kubernetes.io/target: 198.18.200.4 +spec: + gatewayClassName: traefik-internal + listeners: + - name: http + port: 80 + protocol: HTTP + hostname: logs-ingest.k8s.syd1.au.unkin.net + allowedRoutes: + namespaces: + from: Same + - name: https + port: 443 + protocol: HTTPS + hostname: logs-ingest.k8s.syd1.au.unkin.net + allowedRoutes: + namespaces: + from: Same + tls: + mode: Terminate + certificateRefs: + - group: "" + kind: Secret + name: logs-ingest-tls diff --git a/apps/base/logging/httproute.yaml b/apps/base/logging/httproute.yaml new file mode 100644 index 0000000..2400863 --- /dev/null +++ b/apps/base/logging/httproute.yaml @@ -0,0 +1,55 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: logs-ingest-http-redirect + namespace: logging + labels: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/component: ingest +spec: + hostnames: + - logs-ingest.k8s.syd1.au.unkin.net + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: logs-ingest + sectionName: http + rules: + - filters: + - type: RequestRedirect + requestRedirect: + scheme: https + statusCode: 301 + matches: + - path: + type: PathPrefix + value: / +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: logs-ingest + namespace: logging + labels: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/component: ingest +spec: + hostnames: + - logs-ingest.k8s.syd1.au.unkin.net + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: logs-ingest + sectionName: https + rules: + - backendRefs: + - group: "" + kind: Service + name: vector-vm-ingest + port: 8080 + weight: 1 + matches: + - path: + type: PathPrefix + value: / diff --git a/apps/base/logging/job_clickhouse-schema.yaml b/apps/base/logging/job_clickhouse-schema.yaml new file mode 100644 index 0000000..ff36675 --- /dev/null +++ b/apps/base/logging/job_clickhouse-schema.yaml @@ -0,0 +1,105 @@ +--- +# Declarative ClickHouse schema bootstrap. Runs as an ArgoCD PostSync hook so it +# executes after the ClickHouseInstallation is reconciled, and re-runs on every +# sync (idempotent CREATE ... IF NOT EXISTS). Edit the DDL here to evolve the +# schema; the Vector aggregator writes to logs.raw with skip_unknown_fields, so +# adding columns is backward-compatible. +apiVersion: batch/v1 +kind: Job +metadata: + name: clickhouse-schema + namespace: logging + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation + labels: + app.kubernetes.io/name: clickhouse-schema + app.kubernetes.io/component: bootstrap +spec: + backoffLimit: 20 + activeDeadlineSeconds: 1800 + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + app.kubernetes.io/name: clickhouse-schema + vector.dev/exclude: "true" + spec: + restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + containers: + - name: clickhouse-schema + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/clickhouse/clickhouse-server:24.8 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + env: + - name: HOME + value: /tmp + - name: CLICKHOUSE_USER + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: username + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: password + command: + - /bin/bash + - -ec + - | + host=clickhouse-logs.logging.svc.cluster.local + echo "Waiting for ClickHouse at ${host}:9000 ..." + until clickhouse-client --host "$host" --port 9000 \ + --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \ + --query "SELECT 1" >/dev/null 2>&1; do + echo " not ready, retrying in 5s"; sleep 5 + done + echo "Applying schema ..." + clickhouse-client --host "$host" --port 9000 \ + --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \ + --multiquery <<'EOSQL' + CREATE DATABASE IF NOT EXISTS logs; + + CREATE TABLE IF NOT EXISTS logs.raw + ( + timestamp DateTime64(3) DEFAULT now64(3), + host LowCardinality(String) DEFAULT '', + source LowCardinality(String) DEFAULT '', + namespace LowCardinality(String) DEFAULT '', + pod String DEFAULT '', + container LowCardinality(String) DEFAULT '', + stream LowCardinality(String) DEFAULT '', + severity LowCardinality(String) DEFAULT '', + message String DEFAULT '', + labels Map(LowCardinality(String), String), + fields Map(LowCardinality(String), String) + ) + ENGINE = MergeTree + PARTITION BY toDate(timestamp) + ORDER BY (source, namespace, host, timestamp) + TTL toDateTime(timestamp) + INTERVAL 3 DAY + SETTINGS index_granularity = 8192; + EOSQL + echo "Schema applied." + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/apps/base/logging/kustomization.yaml b/apps/base/logging/kustomization.yaml new file mode 100644 index 0000000..4fd721f --- /dev/null +++ b/apps/base/logging/kustomization.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - vaultauth.yaml + - vaultstaticsecret.yaml + - clickhouseinstallation.yaml + - job_clickhouse-schema.yaml + - nats-bootstrap-job.yaml + - cephrgw.yaml + - gateway.yaml + - httproute.yaml + +# Vector pipelines are the single source of truth (also validated by +# `vector test` in CI). Mounted into each tier via `existingConfigMaps`. +configMapGenerator: + # Tunable JetStream stream limits (the nats-bootstrap Job reads these and does + # create-or-update). Hash suffix is INTENTIONALLY left on: editing a value + # renames the ConfigMap, which rewrites the Job's env reference, which changes + # the PostSync hook Job's spec and forces Argo to re-run it -> new limits apply. + # Sizing assumes ~1500 events/s avg @ ~1 KiB/event with S2 compression (~4x): + # ~33 GiB/day compressed -> ~100 GiB/3d per replica. max_bytes 130 GiB sits + # under the 180Gi/node PVC (see values-nats.yaml). Raising retention beyond the + # PVC requires bumping BOTH max_bytes here and fileStore PVC size in values. + - name: nats-stream-limits + literals: + - max_age=72h + - max_bytes=139586437120 + - dupe_window=2m + - name: vector-agent-config + files: + - agent.yaml=vector/agent.yaml + options: + disableNameSuffixHash: true + - name: vector-aggregator-config + files: + - aggregator.yaml=vector/aggregator.yaml + options: + disableNameSuffixHash: true + - name: vector-vm-ingest-config + files: + - vm-ingest.yaml=vector/vm-ingest.yaml + options: + disableNameSuffixHash: true + - name: vector-archiver-config + files: + - archiver.yaml=vector/archiver.yaml + options: + disableNameSuffixHash: true diff --git a/apps/base/logging/namespace.yaml b/apps/base/logging/namespace.yaml new file mode 100644 index 0000000..1ccca01 --- /dev/null +++ b/apps/base/logging/namespace.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/name: logging + name: logging diff --git a/apps/base/logging/nats-bootstrap-job.yaml b/apps/base/logging/nats-bootstrap-job.yaml new file mode 100644 index 0000000..5e2fc68 --- /dev/null +++ b/apps/base/logging/nats-bootstrap-job.yaml @@ -0,0 +1,153 @@ +--- +# Declarative JetStream provisioning: the LOGS stream + durable consumers. +# ArgoCD PostSync hook, idempotent create-or-UPDATE, re-runs each sync. +# +# Stream LOGS: file storage, 3 replicas, retention=limits (NOT workqueue) so the +# transform tier AND the archiver each independently see every message — reading +# never deletes; only max-age/max-bytes do. S2 compression is on (logs compress +# well). Replay window = max-age (3d default). Beyond that, the S3 archive is the +# ONLY long-term source — everything else is gone after 3 days (accepted design). +# +# TUNABLE LIMITS LIVE IN A CONFIGMAP (nats-stream-limits): max_age, max_bytes, +# dupe_window. Change the ConfigMap and re-sync — this Job re-runs and applies +# the new limits via `nats stream edit` (no manual surgery). The ConfigMap is +# generated with a content-hash suffix (kustomize), so editing it changes both +# the ConfigMap name AND this Job's env reference → the PostSync hook Job's spec +# changes and Argo re-runs it (belt-and-suspenders on top of hooks running each +# sync; hook-delete-policy=BeforeHookCreation recreates it every time). +# +# Consumers (independent offsets = true fan-out): +# transform -> whole log stream, feeds the ClickHouse transform tier +# archiver -> configurable security-relevant subset, feeds the S3 archiver. +# Default filter is Vault audit (logs.k8s.vault.>); ADD subjects +# by editing ARCHIVE_SUBJECTS (space-separated -> repeated +# --filter). Exact default set is an open decision for Ben. +# +# Runbook (replay): +# (a) reprocess from JetStream (within max-age, 3d): scale the transform tier +# to 0, then `nats consumer rm LOGS transform` and re-run this Job +# (recreates at DeliverAll), or `nats consumer edit`/`--replay` from a +# start seq/time. +# (b) long-horizon (beyond JetStream): re-ingest S3 archive objects back +# through the transform tier (vector aws_s3 source or a one-shot Job). +apiVersion: batch/v1 +kind: Job +metadata: + name: nats-bootstrap + namespace: logging + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation + labels: + app.kubernetes.io/name: nats-bootstrap + app.kubernetes.io/component: bootstrap +spec: + backoffLimit: 20 + activeDeadlineSeconds: 1800 + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + app.kubernetes.io/name: nats-bootstrap + vector.dev/exclude: "true" + spec: + restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + containers: + - name: nats-bootstrap + image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/natsio/nats-box:0.18.0 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + env: + - name: HOME + value: /tmp + - name: NATS_URL + value: "nats://nats.logging.svc.cluster.local:4222" + - name: NATS_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: nats-auth + key: admin_password + # Tunable stream limits — sourced from the ConfigMap. + - name: MAX_AGE + valueFrom: + configMapKeyRef: + name: nats-stream-limits + key: max_age + - name: MAX_BYTES + valueFrom: + configMapKeyRef: + name: nats-stream-limits + key: max_bytes + - name: DUPE_WINDOW + valueFrom: + configMapKeyRef: + name: nats-stream-limits + key: dupe_window + # Space-separated subject filters for the archiver consumer. + - name: ARCHIVE_SUBJECTS + value: "logs.k8s.vault.>" + command: + - /bin/sh + - -ec + - | + export NATS_USER=log-admin NATS_PASSWORD="$NATS_ADMIN_PASSWORD" + echo "Waiting for NATS + JetStream ..." + until nats --server "$NATS_URL" account info >/dev/null 2>&1; do + echo " not ready, retry in 5s"; sleep 5 + done + + echo "Ensuring stream LOGS (max_age=$MAX_AGE max_bytes=$MAX_BYTES dupe=$DUPE_WINDOW) ..." + # Create if absent; otherwise converge the mutable limits from the + # ConfigMap. (storage/retention/replicas are immutable, set only on + # create.) + nats stream add LOGS \ + --subjects='logs.>' --storage=file --replicas=3 \ + --retention=limits --discard=old --compression=s2 \ + --max-age="$MAX_AGE" --max-bytes="$MAX_BYTES" \ + --max-msgs=-1 --max-msgs-per-subject=-1 --max-msg-size=-1 \ + --max-consumers=-1 --dupe-window="$DUPE_WINDOW" --defaults 2>/dev/null \ + && echo " created" \ + || nats stream edit -f LOGS \ + --subjects='logs.>' --discard=old --compression=s2 \ + --max-age="$MAX_AGE" --max-bytes="$MAX_BYTES" \ + --max-msgs=-1 --max-msgs-per-subject=-1 --max-msg-size=-1 \ + --max-consumers=-1 --dupe-window="$DUPE_WINDOW" + + echo "Ensuring consumer transform ..." + nats consumer add LOGS transform \ + --pull --filter='logs.>' --deliver=all --ack=explicit \ + --max-deliver=-1 --replay=instant --defaults 2>/dev/null \ + || echo " transform already exists" + + echo "Ensuring consumer archiver (filters: $ARCHIVE_SUBJECTS) ..." + filter_args="" + for s in $ARCHIVE_SUBJECTS; do filter_args="$filter_args --filter=$s"; done + # shellcheck disable=SC2086 + nats consumer add LOGS archiver \ + --pull $filter_args --deliver=all --ack=explicit \ + --max-deliver=-1 --replay=instant --defaults 2>/dev/null \ + || echo " archiver already exists" + + echo "Done." + nats stream info LOGS + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/apps/base/logging/vaultauth.yaml b/apps/base/logging/vaultauth.yaml new file mode 100644 index 0000000..bf3b375 --- /dev/null +++ b/apps/base/logging/vaultauth.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: default + namespace: logging +spec: + allowedNamespaces: + - logging + kubernetes: + audiences: + - vault + role: default + serviceAccount: default + tokenExpirationSeconds: 600 + method: kubernetes + mount: k8s/au/syd1 + vaultConnectionRef: vso-system/default diff --git a/apps/base/logging/vaultstaticsecret.yaml b/apps/base/logging/vaultstaticsecret.yaml new file mode 100644 index 0000000..45b56ab --- /dev/null +++ b/apps/base/logging/vaultstaticsecret.yaml @@ -0,0 +1,50 @@ +--- +# ClickHouse credentials for the `vector` user. +# +# Seed the Vault KV entry once (values are NOT stored in git), e.g.: +# PW=$(openssl rand -base64 24) +# HASH=$(printf '%s' "$PW" | sha256sum | cut -d' ' -f1) +# vault kv put kv/kubernetes/namespace/logging/default/clickhouse-credentials \ +# username=vector password="$PW" password_sha256_hex="$HASH" +# +# The `logging/default` ServiceAccount reads this path via the templated +# `policies/kv/kubernetes/default.yaml` policy (k8s auth role `default`), so no +# terraform-vault change is required — only the value above must be written. +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: clickhouse-credentials + namespace: logging +spec: + destination: + create: true + name: clickhouse-credentials + overwrite: true + hmacSecretData: true + mount: kv + path: kubernetes/namespace/logging/default/clickhouse-credentials + refreshAfter: 5m + type: kv-v2 + vaultAuthRef: default +--- +# NATS JetStream auth. Distinct passwords for the producer (edge), consumer +# (transform tier + archiver) and admin (bootstrap Job) users. Seed once: +# for k in admin producer consumer; do declare P_$k=$(openssl rand -base64 24); done +# vault kv put kv/kubernetes/namespace/logging/default/nats-auth \ +# admin_password="$P_admin" producer_password="$P_producer" consumer_password="$P_consumer" +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: nats-auth + namespace: logging +spec: + destination: + create: true + name: nats-auth + overwrite: true + hmacSecretData: true + mount: kv + path: kubernetes/namespace/logging/default/nats-auth + refreshAfter: 5m + type: kv-v2 + vaultAuthRef: default diff --git a/apps/base/logging/vector/agent.yaml b/apps/base/logging/vector/agent.yaml new file mode 100644 index 0000000..efbc35f --- /dev/null +++ b/apps/base/logging/vector/agent.yaml @@ -0,0 +1,43 @@ +--- +# Vector EDGE agent pipeline (DaemonSet) — thin publisher, single source of +# truth. Mounted via existingConfigMaps (NOT the chart's customConfig, whose +# Helm `tpl` pass collides with Vector's own {{ }} / ${ } syntax). Tails all pod +# logs, attaches only routing tokens, publishes to JetStream. No parsing. +data_dir: /vector-data-dir + +api: + enabled: false + +sources: + kubernetes_logs: + type: kubernetes_logs + +transforms: + # Routing metadata only: NATS-subject-safe namespace + container tokens. + keymeta: + type: remap + inputs: + - kubernetes_logs + source: | + ns = to_string(.kubernetes.pod_namespace || "unknown") ?? "unknown" + .ns_token = replace(ns, r'[^a-zA-Z0-9_-]', "_") + cont = to_string(.kubernetes.container_name || "unknown") ?? "unknown" + .cont_token = replace(cont, r'[^a-zA-Z0-9_-]', "_") + +sinks: + to_jetstream: + type: nats + inputs: + - keymeta + url: nats://nats.logging.svc.cluster.local:4222 + connection_name: vector-agent + subject: "logs.k8s.{{ ns_token }}.{{ cont_token }}" + jetstream: + enabled: true + auth: + strategy: user_password + user_password: + user: log-producer + password: ${NATS_PRODUCER_PASSWORD} + encoding: + codec: json diff --git a/apps/base/logging/vector/aggregator-tests.yaml b/apps/base/logging/vector/aggregator-tests.yaml new file mode 100644 index 0000000..8a194fa --- /dev/null +++ b/apps/base/logging/vector/aggregator-tests.yaml @@ -0,0 +1,63 @@ +--- +# `vector test` unit tests for the aggregator transforms. Merged with +# aggregator.yaml in CI (.woodpecker/vector-test.yaml). This is the pattern the +# per-app parsing follow-ups extend: add a test per new transform here. +tests: + - name: subject_routes_k8s_vs_vm + inputs: + - insert_at: route + type: log + log_fields: + subject: "logs.k8s.shop.web" + message: "routed" + outputs: + - extract_from: route.k8s + conditions: + - type: vrl + source: | + assert_eq!(.message, "routed") + + - name: k8s_log_is_normalised + inputs: + - insert_at: k8s_shape + type: log + log_fields: + message: "hello from pod" + stream: "stdout" + timestamp: "2026-07-27T00:00:00Z" + kubernetes.pod_name: "web-abc" + kubernetes.pod_namespace: "shop" + kubernetes.container_name: "web" + kubernetes.pod_node_name: "node-1" + outputs: + - extract_from: k8s_shape + conditions: + - type: vrl + source: | + assert_eq!(.source, "k8s") + assert_eq!(.namespace, "shop") + assert_eq!(.pod, "web-abc") + assert_eq!(.container, "web") + assert_eq!(.host, "node-1") + assert_eq!(.stream, "stdout") + assert_eq!(.message, "hello from pod") + + - name: vm_log_is_normalised + inputs: + - insert_at: vm_shape + type: log + log_fields: + message: "sshd started" + host: "vm-db-1" + severity: "info" + role: "database" + outputs: + - extract_from: vm_shape + conditions: + - type: vrl + source: | + assert_eq!(.source, "vm") + assert_eq!(.host, "vm-db-1") + assert_eq!(.severity, "info") + assert_eq!(.message, "sshd started") + assert_eq!(.labels.role, "database") diff --git a/apps/base/logging/vector/aggregator.yaml b/apps/base/logging/vector/aggregator.yaml new file mode 100644 index 0000000..f22b849 --- /dev/null +++ b/apps/base/logging/vector/aggregator.yaml @@ -0,0 +1,131 @@ +--- +# Vector TRANSFORM tier (the "brain") — single source of truth, also validated +# by `vector test` in CI. Consumes the whole log stream from JetStream via the +# durable `transform` consumer (at-least-once; durable offsets tracked by +# JetStream), routes by subject, normalises into the logs.raw columns, and is +# the ONLY ClickHouse writer. Per-app parsing is added here as follow-ups: +# insert a transform and append its id to the clickhouse sink `inputs` — no edge +# or VM rollout required. +# +# Durability model: JetStream (3d / 130 GiB, S2-compressed) is the SOLE +# durability layer and the replay window. This +# tier is stateless (no PVC, memory buffer). If ClickHouse is down the sink +# blocks (when_full=block); back-pressure stops the source pulling, so unpulled +# messages stay in JetStream and are redelivered. NB: Vector's NATS source has +# no end-to-end acks (acks on receipt), so a pod killed mid-outage can lose the +# in-memory buffer's worth of already-pulled events — accepted for a stateless, +# autoscalable tier. +data_dir: /vector-data-dir + +api: + enabled: true + address: 0.0.0.0:8686 + +sources: + js_in: + type: nats + url: nats://nats.logging.svc.cluster.local:4222 + connection_name: vector-transform + subject: "logs.>" + jetstream: + stream: LOGS + consumer: transform + auth: + strategy: user_password + user_password: + user: log-consumer + password: ${NATS_CONSUMER_PASSWORD} + decoding: + codec: json + +transforms: + route: + type: route + inputs: + - js_in + route: + k8s: 'starts_with(to_string(.subject) ?? "", "logs.k8s.")' + vm: 'starts_with(to_string(.subject) ?? "", "logs.vm.")' + + k8s_shape: + type: remap + inputs: + - route.k8s + source: | + ts = .timestamp || now() + node = to_string(.kubernetes.pod_node_name || "") ?? "" + ns = to_string(.kubernetes.pod_namespace || "") ?? "" + pod = to_string(.kubernetes.pod_name || "") ?? "" + container = to_string(.kubernetes.container_name || "") ?? "" + strm = to_string(.stream || "") ?? "" + msg = to_string(.message || "") ?? "" + lbls = object(.kubernetes.pod_labels) ?? {} + . = { + "timestamp": ts, + "host": node, + "source": "k8s", + "namespace": ns, + "pod": pod, + "container": container, + "stream": strm, + "severity": "", + "message": msg, + "labels": lbls, + "fields": {} + } + + vm_shape: + type: remap + inputs: + - route.vm + source: | + ts = .timestamp || .ts || now() + host = to_string(.host || .hostname || "") ?? "" + msg = to_string(.message || .msg || "") ?? "" + sev = to_string(.severity || .level || "") ?? "" + role = to_string(.role || "") ?? "" + lbls = {} + if role != "" { + lbls = {"role": role} + } + . = { + "timestamp": ts, + "host": host, + "source": "vm", + "namespace": "", + "pod": "", + "container": "", + "stream": "", + "severity": sev, + "message": msg, + "labels": lbls, + "fields": {} + } + +sinks: + clickhouse: + type: clickhouse + inputs: + - k8s_shape + - vm_shape + endpoint: http://clickhouse-logs.logging.svc.cluster.local:8123 + database: logs + table: raw + skip_unknown_fields: true + date_time_best_effort: true + auth: + strategy: basic + user: "${CLICKHOUSE_USER}" + password: "${CLICKHOUSE_PASSWORD}" + batch: + max_events: 500000 + max_bytes: 134217728 + timeout_secs: 10 + # Stateless: in-memory buffer, block on full so back-pressure reaches the + # JetStream pull source (which then stops acking). JetStream is durability. + buffer: + type: memory + max_events: 2000 + when_full: block + healthcheck: + enabled: true diff --git a/apps/base/logging/vector/archiver.yaml b/apps/base/logging/vector/archiver.yaml new file mode 100644 index 0000000..60b5102 --- /dev/null +++ b/apps/base/logging/vector/archiver.yaml @@ -0,0 +1,62 @@ +--- +# Vector ARCHIVER tier — long-term raw-log backup to S3 (Ceph RGW). Independent +# durable JetStream consumer (`archiver`) so its offsets/lag are fully isolated +# from the ClickHouse transform path (archive lag can never stall ingest — true +# fan-out). Writes RAW, pre-transform events (as they sit in JetStream) as +# gzipped NDJSON, partitioned by subject + date. This is the long-horizon replay +# source beyond JetStream's 3d retention window. +data_dir: /vector-data-dir + +api: + enabled: true + address: 0.0.0.0:8686 + +sources: + js_archive: + type: nats + url: nats://nats.logging.svc.cluster.local:4222 + connection_name: vector-archiver + subject: "logs.>" + jetstream: + stream: LOGS + consumer: archiver + auth: + strategy: user_password + user_password: + user: log-consumer + password: ${NATS_CONSUMER_PASSWORD} + decoding: + codec: json + +sinks: + s3: + type: aws_s3 + inputs: + - js_archive + bucket: logs-archive + endpoint: https://s3.ceph.unkin.net + region: us-east-1 + force_path_style: true + tls: + ca_file: /etc/vault-ca/ca.crt + # AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY come from the logs-archive-s3 + # Secret (cephrgw-operator) via envFrom on the deployment. + key_prefix: "raw/{{ subject }}/%Y/%m/%d/" + compression: gzip + encoding: + codec: json + framing: + method: newline_delimited + filename_time_format: "%Y%m%dT%H%M%SZ" + filename_append_uuid: true + batch: + max_bytes: 134217728 + timeout_secs: 300 + buffer: + type: memory + max_events: 5000 + when_full: block + # Disabled so slow BucketAccess credential propagation doesn't crash-loop + # the pod; RGW reachability is proven by the operator's own health. + healthcheck: + enabled: false diff --git a/apps/base/logging/vector/vm-ingest-tests.yaml b/apps/base/logging/vector/vm-ingest-tests.yaml new file mode 100644 index 0000000..fed2928 --- /dev/null +++ b/apps/base/logging/vector/vm-ingest-tests.yaml @@ -0,0 +1,29 @@ +--- +# `vector test` unit tests for the VM-ingest routing transform. +tests: + - name: host_token_is_subject_safe + inputs: + - insert_at: tag + type: log + log_fields: + host: "db1.syd1.example.net" + message: "sshd accepted" + outputs: + - extract_from: tag + conditions: + - type: vrl + source: | + assert_eq!(.host_token, "db1_syd1_example_net") + + - name: missing_host_defaults_to_unknown + inputs: + - insert_at: tag + type: log + log_fields: + message: "no host field" + outputs: + - extract_from: tag + conditions: + - type: vrl + source: | + assert_eq!(.host_token, "unknown") diff --git a/apps/base/logging/vector/vm-ingest.yaml b/apps/base/logging/vector/vm-ingest.yaml new file mode 100644 index 0000000..8e2b069 --- /dev/null +++ b/apps/base/logging/vector/vm-ingest.yaml @@ -0,0 +1,50 @@ +--- +# Vector VM-INGEST tier — the VM front door. Thin: accepts NDJSON over HTTPS +# (behind the logs-ingest Gateway) from puppet-managed VMs, attaches only a +# routing token, and publishes into JetStream (subject logs.vm.). No +# parsing here — shaping happens in the transform tier after JetStream, so VM +# logs get the same durability/replay/fan-out as k8s logs. +data_dir: /vector-data-dir + +api: + enabled: true + address: 0.0.0.0:8686 + +sources: + vm_http: + type: http_server + address: 0.0.0.0:8080 + path: / + method: POST + decoding: + codec: json + framing: + method: newline_delimited + +transforms: + # Routing metadata only: derive a NATS-subject-safe host token. + tag: + type: remap + inputs: + - vm_http + source: | + host = to_string(.host || .hostname || "unknown") ?? "unknown" + .host_token = replace(host, r'[^a-zA-Z0-9_-]', "_") + +sinks: + to_jetstream: + type: nats + inputs: + - tag + url: nats://nats.logging.svc.cluster.local:4222 + connection_name: vector-vm-ingest + subject: "logs.vm.{{ host_token }}" + jetstream: + enabled: true + auth: + strategy: user_password + user_password: + user: log-producer + password: ${NATS_PRODUCER_PASSWORD} + encoding: + codec: json diff --git a/apps/overlays/au-syd1/clickhouse-system/kustomization.yaml b/apps/overlays/au-syd1/clickhouse-system/kustomization.yaml new file mode 100644 index 0000000..498f573 --- /dev/null +++ b/apps/overlays/au-syd1/clickhouse-system/kustomization.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: clickhouse-system + +resources: + - ../../../base/clickhouse-system + +helmCharts: + - name: altinity-clickhouse-operator + repo: https://helm.altinity.com + version: "0.27.2" + releaseName: clickhouse-operator + namespace: clickhouse-system + valuesFile: values.yaml diff --git a/apps/overlays/au-syd1/clickhouse-system/values.yaml b/apps/overlays/au-syd1/clickhouse-system/values.yaml new file mode 100644 index 0000000..ee144fc --- /dev/null +++ b/apps/overlays/au-syd1/clickhouse-system/values.yaml @@ -0,0 +1,40 @@ +# Altinity ClickHouse operator. Cluster-scoped: watches ClickHouseInstallation +# resources in all namespaces (the logs cluster lives in the `logging` namespace). +# CRDs are installed at runtime by the chart's crdHook Job. +# +# All images are pulled through the artifactapi dockerhub remote (no direct +# upstream). Upstream official images are used; no Docker Hardened Image variant +# is adopted (DHI is subscription-gated and served from a private org namespace +# not reachable via the anonymous artifactapi dockerhub proxy). +crdHook: + image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/bitnami/kubectl + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 250m + memory: 128Mi + +operator: + image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/altinity/clickhouse-operator + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +metrics: + image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/altinity/metrics-exporter + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi diff --git a/apps/overlays/au-syd1/logging/kustomization.yaml b/apps/overlays/au-syd1/logging/kustomization.yaml new file mode 100644 index 0000000..a81c658 --- /dev/null +++ b/apps/overlays/au-syd1/logging/kustomization.yaml @@ -0,0 +1,45 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: logging + +resources: + - ../../../base/logging + +helmCharts: + # Dedicated JetStream NATS cluster — the durable log bus. + - name: nats + repo: https://nats-io.github.io/k8s/helm/charts + version: "2.14.2" + releaseName: nats + namespace: logging + valuesFile: values-nats.yaml + # Edge agent (DaemonSet): tails every pod's logs, publishes to JetStream. + - name: vector + repo: https://helm.vector.dev + version: "0.57.0" + releaseName: vector-agent + namespace: logging + valuesFile: values-vector-agent.yaml + # VM ingest (Deployment): HTTP NDJSON front door -> JetStream. + - name: vector + repo: https://helm.vector.dev + version: "0.57.0" + releaseName: vector-vm-ingest + namespace: logging + valuesFile: values-vector-vm-ingest.yaml + # Transform tier (StatefulSet): JetStream consumer -> shape -> ClickHouse. + - name: vector + repo: https://helm.vector.dev + version: "0.57.0" + releaseName: vector-aggregator + namespace: logging + valuesFile: values-vector-aggregator.yaml + # Archiver (Deployment): independent JetStream consumer -> raw logs to S3. + - name: vector + repo: https://helm.vector.dev + version: "0.57.0" + releaseName: vector-archiver + namespace: logging + valuesFile: values-vector-archiver.yaml diff --git a/apps/overlays/au-syd1/logging/values-nats.yaml b/apps/overlays/au-syd1/logging/values-nats.yaml new file mode 100644 index 0000000..956ebd2 --- /dev/null +++ b/apps/overlays/au-syd1/logging/values-nats.yaml @@ -0,0 +1,100 @@ +# Dedicated JetStream-enabled NATS cluster for the log bus. Deliberately NOT +# shared with app messaging (streamstack et al. run their own NATS in their own +# repo) — a separate cluster isolates logging blast-radius from app messaging +# and lets us size retention/storage purely for the log outage-buffer + replay +# use-case. +fullnameOverride: nats + +config: + cluster: + enabled: true + replicas: 3 + jetstream: + enabled: true + fileStore: + pvc: + # Sized for 3d retention: ~100 GiB/3d compressed (see + # nats-stream-limits ConfigMap) + file-store WAL/index/overhead, kept + # safely above the 130 GiB max_bytes cap. 3 replicas => ~0.5 TiB total + # provisioned on cephrbd-fast-delete. NB: this is the honest number for + # the assumed ~1500 events/s; higher real volume needs a bigger PVC + + # max_bytes together, else discard=old truncates retention below 3d. + size: 180Gi + storageClassName: cephrbd-fast-delete + # Per-user auth with publish/subscribe separation. Passwords are injected as + # env vars from the Vault-synced nats-auth Secret (NATS expands $VAR in config). + merge: + authorization: + users: + # Bootstrap Job (stream/consumer management) — full JetStream API. + - user: log-admin + password: $NATS_ADMIN_PASSWORD + # Edge publishers (k8s DaemonSet + VM ingest) — publish only. + - user: log-producer + password: $NATS_PRODUCER_PASSWORD + permissions: + publish: + allow: + - "logs.>" + subscribe: + allow: + - "_INBOX.>" + # Consumers (transform tier + archiver) — pull + ack only, no publish + # to log subjects. + - user: log-consumer + password: $NATS_CONSUMER_PASSWORD + permissions: + publish: + allow: + - "$JS.API.CONSUMER.>" + - "$JS.API.STREAM.INFO.>" + - "$JS.ACK.LOGS.>" + subscribe: + allow: + - "_INBOX.>" + +container: + # Pulled through the artifactapi dockerhub remote (upstream official nats; + # no DHI variant available for nats). + image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/nats + tag: 2.14.2-alpine + env: + NATS_ADMIN_PASSWORD: + valueFrom: + secretKeyRef: + name: nats-auth + key: admin_password + NATS_PRODUCER_PASSWORD: + valueFrom: + secretKeyRef: + name: nats-auth + key: producer_password + NATS_CONSUMER_PASSWORD: + valueFrom: + secretKeyRef: + name: nats-auth + key: consumer_password + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 4Gi + +# Roll the StatefulSet when nats-auth changes. +podTemplate: + merge: + metadata: + annotations: + reloader.stakater.com/auto: "true" + +# Config-reloader sidecar image, also through artifactapi. +reloader: + image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/natsio/nats-server-config-reloader + tag: "0.23.0" + +natsBox: + enabled: false diff --git a/apps/overlays/au-syd1/logging/values-vector-agent.yaml b/apps/overlays/au-syd1/logging/values-vector-agent.yaml new file mode 100644 index 0000000..bff67a5 --- /dev/null +++ b/apps/overlays/au-syd1/logging/values-vector-agent.yaml @@ -0,0 +1,50 @@ +# Vector EDGE agent (DaemonSet) — thin publisher. Tails every node's pod logs +# (incl. control-plane via the blanket toleration) and publishes them into +# JetStream over the Vector NATS sink. No parsing; only a routing subject token +# is attached. Shaping happens in the transform tier after JetStream. +role: Agent +fullnameOverride: vector-agent + +# Pulled through the artifactapi dockerhub remote; distroless-libc (no DHI — +# subscription-gated/private-namespace, not reachable via the anon proxy). +image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector + tag: 0.57.0-distroless-libc + +rbac: + create: true +serviceAccount: + create: true + +podLabels: + vector.dev/exclude: "true" + +tolerations: + - operator: Exists + +env: + - name: NATS_PRODUCER_PASSWORD + valueFrom: + secretKeyRef: + name: nats-auth + key: producer_password + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + +service: + enabled: false + +# Pipeline is the single source of truth in apps/base/logging/vector/agent.yaml, +# mounted via existingConfigMaps (avoids the chart's customConfig Helm-tpl pass). +dataDir: /vector-data-dir +existingConfigMaps: + - vector-agent-config + +workloadResourceAnnotations: + reloader.stakater.com/auto: "true" diff --git a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml new file mode 100644 index 0000000..4ee0d14 --- /dev/null +++ b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml @@ -0,0 +1,80 @@ +# Vector TRANSFORM tier (STATELESS Deployment) — the "brain": sole ClickHouse +# writer, owns all transforms, holds the only ClickHouse + NATS-consumer creds. +# +# Stateless by design: a JetStream pull consumer with NO PVC and NO disk buffer. +# JetStream is the sole durability layer. On a ClickHouse outage the clickhouse +# sink blocks (buffer when_full=block), back-pressure stops the source pulling, +# and unpulled messages stay in JetStream for redelivery. Because Vector's NATS +# source does NOT support end-to-end acknowledgements (it acks on receipt, not +# after the sink), the only at-risk window is the in-memory buffer's worth of +# already-pulled events if a pod is killed mid-outage — the accepted trade for a +# horizontally-autoscalable stateless tier. Multiple replicas share the one +# durable consumer `transform` (JetStream pull consumers distribute work), so +# HPA is safe. +role: Stateless-Aggregator +fullnameOverride: vector-aggregator + +image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector + tag: 0.57.0-distroless-libc + +# Horizontal autoscaling on CPU — safe with N replicas on one durable consumer. +autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 8 + targetCPUUtilizationPercentage: 70 + +workloadResourceAnnotations: + reloader.stakater.com/auto: "true" + +podLabels: + vector.dev/exclude: "true" + +# Pipeline is the single source of truth in apps/base/logging/vector/ +# aggregator.yaml (unit-tested by `vector test` in CI), mounted via +# existingConfigMaps. No persistence — stateless. +dataDir: /vector-data-dir +existingConfigMaps: + - vector-aggregator-config + +# The ONLY place ClickHouse + NATS-consumer creds are consumed. +env: + - name: CLICKHOUSE_USER + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: username + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: clickhouse-credentials + key: password + - name: NATS_CONSUMER_PASSWORD + valueFrom: + secretKeyRef: + name: nats-auth + key: consumer_password + +# Pure consumer: expose only the Vector API for debugging. +containerPorts: + - name: api + containerPort: 8686 + protocol: TCP + +service: + enabled: true + type: ClusterIP + ports: + - name: api + port: 8686 + targetPort: 8686 + protocol: TCP + +resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi diff --git a/apps/overlays/au-syd1/logging/values-vector-archiver.yaml b/apps/overlays/au-syd1/logging/values-vector-archiver.yaml new file mode 100644 index 0000000..671844a --- /dev/null +++ b/apps/overlays/au-syd1/logging/values-vector-archiver.yaml @@ -0,0 +1,56 @@ +# Vector ARCHIVER tier (Deployment) — independent JetStream consumer writing raw +# logs to S3 (Ceph RGW). Isolated from the ClickHouse path (own durable +# consumer). Pipeline: apps/base/logging/vector/archiver.yaml. +role: Stateless-Aggregator +fullnameOverride: vector-archiver +replicas: 1 + +image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector + tag: 0.57.0-distroless-libc + +workloadResourceAnnotations: + reloader.stakater.com/auto: "true" + +podLabels: + vector.dev/exclude: "true" + +dataDir: /vector-data-dir +existingConfigMaps: + - vector-archiver-config + +env: + - name: NATS_CONSUMER_PASSWORD + valueFrom: + secretKeyRef: + name: nats-auth + key: consumer_password + +# S3 creds (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY) from the cephrgw-operator +# BucketAccess Secret. +envFrom: + - secretRef: + name: logs-archive-s3 + +# Trust the internal unkin.net Vault-PKI CA to verify s3.ceph.unkin.net. +# vault-ca-cert is reflected into every namespace from the certificates ns. +extraVolumes: + - name: vault-ca-cert + secret: + secretName: vault-ca-cert +extraVolumeMounts: + - name: vault-ca-cert + mountPath: /etc/vault-ca/ca.crt + subPath: ca.crt + readOnly: true + +service: + enabled: false + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi diff --git a/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml b/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml new file mode 100644 index 0000000..12fe8b7 --- /dev/null +++ b/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml @@ -0,0 +1,56 @@ +# Vector VM-INGEST tier (Deployment) — VM front door. HTTP NDJSON in (behind the +# logs-ingest Gateway), publishes into JetStream. Stateless publisher. +# Pipeline: apps/base/logging/vector/vm-ingest.yaml (unit-tested in CI). +role: Stateless-Aggregator +fullnameOverride: vector-vm-ingest +replicas: 2 + +image: + repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector + tag: 0.57.0-distroless-libc + +workloadResourceAnnotations: + reloader.stakater.com/auto: "true" + +podLabels: + vector.dev/exclude: "true" + +dataDir: /vector-data-dir +existingConfigMaps: + - vector-vm-ingest-config + +env: + - name: NATS_PRODUCER_PASSWORD + valueFrom: + secretKeyRef: + name: nats-auth + key: producer_password + +containerPorts: + - name: http-ingest + containerPort: 8080 + protocol: TCP + - name: api + containerPort: 8686 + protocol: TCP + +service: + enabled: true + type: ClusterIP + ports: + - name: http-ingest + port: 8080 + targetPort: 8080 + protocol: TCP + - name: api + port: 8686 + targetPort: 8686 + protocol: TCP + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi diff --git a/argocd/applicationsets/kustomization.yaml b/argocd/applicationsets/kustomization.yaml index 87d07c2..b54cc35 100644 --- a/argocd/applicationsets/kustomization.yaml +++ b/argocd/applicationsets/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - aitooling.yaml + - logging.yaml - observability.yaml - platform.yaml - storage.yaml diff --git a/argocd/applicationsets/logging.yaml b/argocd/applicationsets/logging.yaml new file mode 100644 index 0000000..ea542d4 --- /dev/null +++ b/argocd/applicationsets/logging.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: logging-apps + namespace: argocd +spec: + generators: + - git: + repoURL: https://git.unkin.net/unkin/argocd-apps + revision: HEAD + directories: + - path: apps/overlays/*/clickhouse-system + - path: apps/overlays/*/logging + template: + metadata: + name: 'logging-{{path[3]}}' + spec: + project: logging + source: + repoURL: https://git.unkin.net/unkin/argocd-apps + targetRevision: HEAD + path: '{{path}}' + destination: + server: https://kubernetes.default.svc + namespace: '{{path[3]}}' + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true + - CreateNamespace=false diff --git a/argocd/projects/kustomization.yaml b/argocd/projects/kustomization.yaml index 87d07c2..b54cc35 100644 --- a/argocd/projects/kustomization.yaml +++ b/argocd/projects/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - aitooling.yaml + - logging.yaml - observability.yaml - platform.yaml - storage.yaml diff --git a/argocd/projects/logging.yaml b/argocd/projects/logging.yaml new file mode 100644 index 0000000..591153a --- /dev/null +++ b/argocd/projects/logging.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: logging + namespace: argocd +spec: + description: Centralized logging stack (ClickHouse + Vector) + sourceRepos: + - https://git.unkin.net/unkin/argocd-apps + - https://helm.altinity.com + - https://helm.vector.dev + destinations: + - namespace: 'logging' + server: https://kubernetes.default.svc + - namespace: 'clickhouse-system' + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: '' + kind: Namespace + - group: 'rbac.authorization.k8s.io' + kind: ClusterRole + - group: 'rbac.authorization.k8s.io' + kind: ClusterRoleBinding + - group: 'apiextensions.k8s.io' + kind: CustomResourceDefinition + namespaceResourceWhitelist: + - group: '*' + kind: '*'