From ab76fb842044d66f375b5ef6d69254f568d1f9b1 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Tue, 28 Jul 2026 21:34:09 +1000
Subject: [PATCH] Fix logging deploy: enable Vector env interpolation; operator
 watches logging ns

Two root causes found by diagnosing the live cluster after #296 deployed.

1. Vector 0.57 disables ${VAR} config interpolation by default (it now requires
   --dangerously-allow-env-var-interpolation). So every ${...PASSWORD} in the
   pipelines was sent as a literal string, breaking NATS auth for ALL vector
   tiers: the aggregator/archiver crash-looped with "authorization violation",
   and the agent/vm-ingest producers silently failed to publish (ClickHouse
   creds would have failed the same way). Enable interpolation via
   VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION=true on all four vector
   deployments. Verified end-to-end: without the flag -> authorization
   violation; with it -> connects and authenticates (narrow ACL unchanged, so
   the earlier ACL theory was wrong and nothing there needed changing).

2. The Altinity operator's watchNamespaces defaulted to [] which, per the chart,
   makes it watch ONLY its own namespace (clickhouse-system). The
   ClickHouseInstallation lives in `logging`, so it was never reconciled (no
   finalizer, empty status) -> the logging-logging Argo sync blocked forever on
   "waiting for healthy CHI" -> the PostSync hooks (nats-bootstrap stream/
   consumers, clickhouse-schema) never ran. Set watchNamespaces: [logging] so
   the operator reconciles the CHI, the sync completes, and the hooks run.

Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
---
 apps/overlays/au-syd1/clickhouse-system/values.yaml         | 6 ++++++
 apps/overlays/au-syd1/logging/values-vector-agent.yaml      | 3 +++
 apps/overlays/au-syd1/logging/values-vector-aggregator.yaml | 4 ++++
 apps/overlays/au-syd1/logging/values-vector-archiver.yaml   | 3 +++
 apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml  | 3 +++
 5 files changed, 19 insertions(+)

diff --git a/apps/overlays/au-syd1/clickhouse-system/values.yaml b/apps/overlays/au-syd1/clickhouse-system/values.yaml
index ee144fc..52b09b9 100644
--- a/apps/overlays/au-syd1/clickhouse-system/values.yaml
+++ b/apps/overlays/au-syd1/clickhouse-system/values.yaml
@@ -6,6 +6,12 @@
 # upstream). Upstream official images are used; no Docker Hardened Image variant
 # is adopted (DHI is subscription-gated and served from a private org namespace
 # not reachable via the anonymous artifactapi dockerhub proxy).
+#
+# Watch the logging namespace where the ClickHouseInstallation lives. The chart
+# default (watchNamespaces: []) makes the operator watch ONLY its own namespace
+# (clickhouse-system), so the logs CHI was never reconciled — set it explicitly.
+watchNamespaces:
+  - logging
 crdHook:
   image:
     repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/bitnami/kubectl
diff --git a/apps/overlays/au-syd1/logging/values-vector-agent.yaml b/apps/overlays/au-syd1/logging/values-vector-agent.yaml
index bff67a5..1be370d 100644
--- a/apps/overlays/au-syd1/logging/values-vector-agent.yaml
+++ b/apps/overlays/au-syd1/logging/values-vector-agent.yaml
@@ -23,6 +23,9 @@ tolerations:
   - operator: Exists
 
 env:
+  # Vector 0.57 disables ${VAR} config interpolation by default; auth needs it.
+  - name: VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION
+    value: "true"
   - name: NATS_PRODUCER_PASSWORD
     valueFrom:
       secretKeyRef:
diff --git a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml
index 4ee0d14..b4d48f9 100644
--- a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml
+++ b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml
@@ -40,6 +40,10 @@ existingConfigMaps:
 
 # The ONLY place ClickHouse + NATS-consumer creds are consumed.
 env:
+  # Vector 0.57 disables ${VAR} config interpolation by default; the pipeline's
+  # auth (${CLICKHOUSE_*}, ${NATS_CONSUMER_PASSWORD}) needs it enabled.
+  - name: VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION
+    value: "true"
   - name: CLICKHOUSE_USER
     valueFrom:
       secretKeyRef:
diff --git a/apps/overlays/au-syd1/logging/values-vector-archiver.yaml b/apps/overlays/au-syd1/logging/values-vector-archiver.yaml
index 671844a..534f23f 100644
--- a/apps/overlays/au-syd1/logging/values-vector-archiver.yaml
+++ b/apps/overlays/au-syd1/logging/values-vector-archiver.yaml
@@ -20,6 +20,9 @@ existingConfigMaps:
   - vector-archiver-config
 
 env:
+  # Vector 0.57 disables ${VAR} config interpolation by default; auth needs it.
+  - name: VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION
+    value: "true"
   - name: NATS_CONSUMER_PASSWORD
     valueFrom:
       secretKeyRef:
diff --git a/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml b/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml
index 12fe8b7..dcb4942 100644
--- a/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml
+++ b/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml
@@ -20,6 +20,9 @@ existingConfigMaps:
   - vector-vm-ingest-config
 
 env:
+  # Vector 0.57 disables ${VAR} config interpolation by default; auth needs it.
+  - name: VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION
+    value: "true"
   - name: NATS_PRODUCER_PASSWORD
     valueFrom:
       secretKeyRef: