From 7dddf8c5aa5787d7ce177dedfa9ee22ebd872a76 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Wed, 29 Jul 2026 00:11:33 +1000 Subject: [PATCH] Fix logging deploy: enable Vector env interpolation; operator watches logging ns (#301) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why PR #296 merged and deployed, but the stack came up partially broken. Diagnosed live (cluster-admin) — two independent root causes, both fixed here. ## Root cause 1 — Vector env-var interpolation is off by default in 0.57 `vector-aggregator` and `vector-archiver` crash-looped with `async_nats::connector: authorization violation`; `vector-agent` / `vector-vm-ingest` were "Running" but silently failing to publish. Diagnosis (evidence): - The NATS server config **does** expand `$NATS_*_PASSWORD` (a `nats` CLI login with the real secret value authenticated fine), and the ACL was **not** the problem (a local repro with the narrow ACL + no stream connects cleanly and returns "stream not found", not an auth violation). - The failure reproduces locally: a hardcoded password connects; the **same value via `${NATS_CONSUMER_PASSWORD}` fails**. Configuring the server to expect the literal string `${NATS_CONSUMER_PASSWORD}` makes Vector connect — proving **Vector sends the literal, un-interpolated string**. - `vector --help` shows `--dangerously-allow-env-var-interpolation` — in 0.57 `${VAR}` interpolation is **opt-in**. An unset-var test confirms interpolation is off (no "unknown env var" error). - Verified fix: with `VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION=true` → **connects and authenticates**. Every tier uses `${...}` for auth (`${NATS_*_PASSWORD}`, `${CLICKHOUSE_*}`), so the env var is added to **all four** vector deployments. (This slipped past CI because `vector test` never opens the NATS connection.) ## Root cause 2 — operator watches only its own namespace `kubectl get chi -n logging` showed the `logs` CHI existed but with **empty status / no finalizer** — the operator never touched it, so the `logging-logging` Argo sync was stuck `Progressing` on *"waiting for healthy state of ClickHouseInstallation/logs"*, and the PostSync hooks (nats-bootstrap stream+consumers, clickhouse-schema) never ran (no stream → the consumers had nothing to bind even once auth is fixed). Diagnosis: forcing an update event on the CHI produced zero operator reaction; a full operator restart didn't help. The Altinity chart README states `watchNamespaces: []` (our value) makes the operator **watch only its own namespace** (`clickhouse-system`). The CHI is in `logging`. Fix: `watchNamespaces: ["logging"]` → operator config `watch.namespaces.include: [logging]`. ## Changes - `apps/overlays/au-syd1/logging/values-vector-{agent,vm-ingest,aggregator,archiver}.yaml`: add `VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION=true`. - `apps/overlays/au-syd1/clickhouse-system/values.yaml`: `watchNamespaces: ["logging"]`. No NATS ACL change (the original narrow ACL is correct). No secret/base changes. ## Expected recovery after merge + sync 1. clickhouse-system syncs → operator config gains `logging` → operator restarts → reconciles the `logs` CHI → CHI pod comes up healthy. 2. `logging-logging` sync unblocks → PostSync hooks run → JetStream `LOGS` stream + `transform`/`archiver` consumers created; `logs.raw` table created. 3. Vector pods roll with interpolation enabled → agents/vm-ingest authenticate and publish; aggregator/archiver authenticate, bind their durable consumers, and write to ClickHouse / S3. 4. Verify: `nats stream info LOGS` shows messages; `SELECT count() FROM logs.raw` increases. ## Validation kustomize build + kubeconform clean (clickhouse-system 22, logging 40); operator config renders `watch.namespaces.include: [logging]`; all 4 vector deployments carry the interpolation env; pre-commit clean. The interpolation fix was verified end-to-end against a real nats-server (fails without the flag, connects with it). https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv Reviewed-on: https://git.unkin.net/unkin/argocd-apps/pulls/301 Co-authored-by: Ben Vincent Co-committed-by: Ben Vincent --- apps/overlays/au-syd1/clickhouse-system/values.yaml | 6 ++++++ apps/overlays/au-syd1/logging/values-vector-agent.yaml | 3 +++ apps/overlays/au-syd1/logging/values-vector-aggregator.yaml | 4 ++++ apps/overlays/au-syd1/logging/values-vector-archiver.yaml | 3 +++ apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml | 3 +++ 5 files changed, 19 insertions(+) diff --git a/apps/overlays/au-syd1/clickhouse-system/values.yaml b/apps/overlays/au-syd1/clickhouse-system/values.yaml index ee144fc..52b09b9 100644 --- a/apps/overlays/au-syd1/clickhouse-system/values.yaml +++ b/apps/overlays/au-syd1/clickhouse-system/values.yaml @@ -6,6 +6,12 @@ # upstream). Upstream official images are used; no Docker Hardened Image variant # is adopted (DHI is subscription-gated and served from a private org namespace # not reachable via the anonymous artifactapi dockerhub proxy). +# +# Watch the logging namespace where the ClickHouseInstallation lives. The chart +# default (watchNamespaces: []) makes the operator watch ONLY its own namespace +# (clickhouse-system), so the logs CHI was never reconciled — set it explicitly. +watchNamespaces: + - logging crdHook: image: repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/bitnami/kubectl diff --git a/apps/overlays/au-syd1/logging/values-vector-agent.yaml b/apps/overlays/au-syd1/logging/values-vector-agent.yaml index bff67a5..1be370d 100644 --- a/apps/overlays/au-syd1/logging/values-vector-agent.yaml +++ b/apps/overlays/au-syd1/logging/values-vector-agent.yaml @@ -23,6 +23,9 @@ tolerations: - operator: Exists env: + # Vector 0.57 disables ${VAR} config interpolation by default; auth needs it. + - name: VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION + value: "true" - name: NATS_PRODUCER_PASSWORD valueFrom: secretKeyRef: diff --git a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml index 4ee0d14..b4d48f9 100644 --- a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml +++ b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml @@ -40,6 +40,10 @@ existingConfigMaps: # The ONLY place ClickHouse + NATS-consumer creds are consumed. env: + # Vector 0.57 disables ${VAR} config interpolation by default; the pipeline's + # auth (${CLICKHOUSE_*}, ${NATS_CONSUMER_PASSWORD}) needs it enabled. + - name: VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION + value: "true" - name: CLICKHOUSE_USER valueFrom: secretKeyRef: diff --git a/apps/overlays/au-syd1/logging/values-vector-archiver.yaml b/apps/overlays/au-syd1/logging/values-vector-archiver.yaml index 671844a..534f23f 100644 --- a/apps/overlays/au-syd1/logging/values-vector-archiver.yaml +++ b/apps/overlays/au-syd1/logging/values-vector-archiver.yaml @@ -20,6 +20,9 @@ existingConfigMaps: - vector-archiver-config env: + # Vector 0.57 disables ${VAR} config interpolation by default; auth needs it. + - name: VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION + value: "true" - name: NATS_CONSUMER_PASSWORD valueFrom: secretKeyRef: diff --git a/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml b/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml index 12fe8b7..dcb4942 100644 --- a/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml +++ b/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml @@ -20,6 +20,9 @@ existingConfigMaps: - vector-vm-ingest-config env: + # Vector 0.57 disables ${VAR} config interpolation by default; auth needs it. + - name: VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION + value: "true" - name: NATS_PRODUCER_PASSWORD valueFrom: secretKeyRef: