From 1202aae06fc19e2d1eba75a4c76ab1e09fdefee2 Mon Sep 17 00:00:00 2001
From: Ben Vincent <ben@unkin.net>
Date: Mon, 27 Jul 2026 21:19:44 +1000
Subject: [PATCH] Pull images via artifactapi; make transform tier stateless
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Three changes from review:

1. Pull every container image through the artifactapi dockerhub remote instead
   of direct upstream: clickhouse-server, altinity operator + metrics-exporter,
   bitnami/kubectl (crdHook), nats + nats-server-config-reloader, nats-box
   (bootstrap Job), and vector (all tiers + the CI image). Requires
   terraform-artifactapi#16 (dockerhub allowlist patterns) merged first.

2. Keep upstream official images (no Docker Hardened Images). DHI exists for
   clickhouse-server and vector but is subscription-gated and served from a
   private org namespace not reachable via the anonymous artifactapi dockerhub
   proxy; its shell-less images would also break the bash bootstrap Jobs and the
   shell-based vector-test CI step. Use vector's distroless-libc for runtime
   pods (near-hardened) and the debian variant only for CI.

3. Make the transform tier a stateless Deployment (was a StatefulSet): no PVC,
   no disk buffer — JetStream is the sole durability layer. The ClickHouse sink
   uses an in-memory block buffer so a ClickHouse outage back-pressures the
   JetStream pull source (unpulled messages are retained/redelivered). Add a CPU
   HPA (2-8) — safe because JetStream pull consumers distribute work across N
   replicas on the one durable consumer. Caveat documented: vector's NATS source
   has no end-to-end acks (acks on receipt), so a pod killed mid-outage can lose
   its in-memory buffer window; accepted trade for a stateless autoscaling tier.

Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
---
 .woodpecker/vector-test.yaml                  |  2 +-
 apps/base/logging/clickhouseinstallation.yaml |  2 +-
 apps/base/logging/job_clickhouse-schema.yaml  |  2 +-
 apps/base/logging/nats-bootstrap-job.yaml     |  2 +-
 apps/base/logging/vector/aggregator.yaml      | 18 +++++---
 .../au-syd1/clickhouse-system/values.yaml     | 11 +++++
 .../overlays/au-syd1/logging/values-nats.yaml | 11 +++++
 .../au-syd1/logging/values-vector-agent.yaml  |  6 +++
 .../logging/values-vector-aggregator.yaml     | 43 ++++++++++++-------
 .../logging/values-vector-archiver.yaml       |  4 ++
 .../logging/values-vector-vm-ingest.yaml      |  4 ++
 11 files changed, 78 insertions(+), 27 deletions(-)

diff --git a/.woodpecker/vector-test.yaml b/.woodpecker/vector-test.yaml
index 8949ce3..aace88e 100644
--- a/.woodpecker/vector-test.yaml
+++ b/.woodpecker/vector-test.yaml
@@ -3,7 +3,7 @@ when:
 
 steps:
   - name: vector-test
-    image: timberio/vector:0.57.0-debian
+    image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector:0.57.0-debian
     commands:
       # Dummy creds + writable dirs so the full topologies build; the unit tests
       # only exercise the transforms (sources are not started).
diff --git a/apps/base/logging/clickhouseinstallation.yaml b/apps/base/logging/clickhouseinstallation.yaml
index ea6bb44..6c82e59 100644
--- a/apps/base/logging/clickhouseinstallation.yaml
+++ b/apps/base/logging/clickhouseinstallation.yaml
@@ -66,7 +66,7 @@ spec:
             runAsGroup: 101
           containers:
             - name: clickhouse
-              image: clickhouse/clickhouse-server:24.8
+              image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/clickhouse/clickhouse-server:24.8
               resources:
                 requests:
                   cpu: 500m
diff --git a/apps/base/logging/job_clickhouse-schema.yaml b/apps/base/logging/job_clickhouse-schema.yaml
index b1e3dd4..1e0f801 100644
--- a/apps/base/logging/job_clickhouse-schema.yaml
+++ b/apps/base/logging/job_clickhouse-schema.yaml
@@ -32,7 +32,7 @@ spec:
         runAsGroup: 101
       containers:
         - name: clickhouse-schema
-          image: clickhouse/clickhouse-server:24.8
+          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/clickhouse/clickhouse-server:24.8
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
diff --git a/apps/base/logging/nats-bootstrap-job.yaml b/apps/base/logging/nats-bootstrap-job.yaml
index 72e5ad9..a21b5d2 100644
--- a/apps/base/logging/nats-bootstrap-job.yaml
+++ b/apps/base/logging/nats-bootstrap-job.yaml
@@ -49,7 +49,7 @@ spec:
         runAsGroup: 1000
       containers:
         - name: nats-bootstrap
-          image: natsio/nats-box:0.18.0
+          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/natsio/nats-box:0.18.0
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
diff --git a/apps/base/logging/vector/aggregator.yaml b/apps/base/logging/vector/aggregator.yaml
index ad35505..03bb18a 100644
--- a/apps/base/logging/vector/aggregator.yaml
+++ b/apps/base/logging/vector/aggregator.yaml
@@ -7,10 +7,13 @@
 # insert a transform and append its id to the clickhouse sink `inputs` — no edge
 # or VM rollout required.
 #
-# Durability model: JetStream (72h / 40GiB) is the outage buffer. If ClickHouse
-# is down the sink blocks, back-pressure stops acking, and JetStream retains
-# messages for replay. The local disk buffer is small (survives pod restarts of
-# in-flight events only).
+# Durability model: JetStream (72h / 40GiB) is the SOLE durability layer. This
+# tier is stateless (no PVC, memory buffer). If ClickHouse is down the sink
+# blocks (when_full=block); back-pressure stops the source pulling, so unpulled
+# messages stay in JetStream and are redelivered. NB: Vector's NATS source has
+# no end-to-end acks (acks on receipt), so a pod killed mid-outage can lose the
+# in-memory buffer's worth of already-pulled events — accepted for a stateless,
+# autoscalable tier.
 data_dir: /vector-data-dir
 
 api:
@@ -117,10 +120,11 @@ sinks:
       max_events: 500000
       max_bytes: 134217728
       timeout_secs: 10
-    # Small local buffer — JetStream is the real outage buffer now.
+    # Stateless: in-memory buffer, block on full so back-pressure reaches the
+    # JetStream pull source (which then stops acking). JetStream is durability.
     buffer:
-      type: disk
-      max_size: 2147483648
+      type: memory
+      max_events: 2000
       when_full: block
     healthcheck:
       enabled: true
diff --git a/apps/overlays/au-syd1/clickhouse-system/values.yaml b/apps/overlays/au-syd1/clickhouse-system/values.yaml
index 8eb31f0..ee144fc 100644
--- a/apps/overlays/au-syd1/clickhouse-system/values.yaml
+++ b/apps/overlays/au-syd1/clickhouse-system/values.yaml
@@ -1,7 +1,14 @@
 # Altinity ClickHouse operator. Cluster-scoped: watches ClickHouseInstallation
 # resources in all namespaces (the logs cluster lives in the `logging` namespace).
 # CRDs are installed at runtime by the chart's crdHook Job.
+#
+# All images are pulled through the artifactapi dockerhub remote (no direct
+# upstream). Upstream official images are used; no Docker Hardened Image variant
+# is adopted (DHI is subscription-gated and served from a private org namespace
+# not reachable via the anonymous artifactapi dockerhub proxy).
 crdHook:
+  image:
+    repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/bitnami/kubectl
   resources:
     requests:
       cpu: 50m
@@ -11,6 +18,8 @@ crdHook:
       memory: 128Mi
 
 operator:
+  image:
+    repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/altinity/clickhouse-operator
   resources:
     requests:
       cpu: 100m
@@ -20,6 +29,8 @@ operator:
       memory: 512Mi
 
 metrics:
+  image:
+    repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/altinity/metrics-exporter
   resources:
     requests:
       cpu: 50m
diff --git a/apps/overlays/au-syd1/logging/values-nats.yaml b/apps/overlays/au-syd1/logging/values-nats.yaml
index f6f14e9..8e70444 100644
--- a/apps/overlays/au-syd1/logging/values-nats.yaml
+++ b/apps/overlays/au-syd1/logging/values-nats.yaml
@@ -48,6 +48,11 @@ config:
                 - "_INBOX.>"
 
 container:
+  # Pulled through the artifactapi dockerhub remote (upstream official nats;
+  # no DHI variant available for nats).
+  image:
+    repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/library/nats
+    tag: 2.14.2-alpine
   env:
     NATS_ADMIN_PASSWORD:
       valueFrom:
@@ -79,5 +84,11 @@ podTemplate:
       annotations:
         reloader.stakater.com/auto: "true"
 
+# Config-reloader sidecar image, also through artifactapi.
+reloader:
+  image:
+    repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/natsio/nats-server-config-reloader
+    tag: "0.23.0"
+
 natsBox:
   enabled: false
diff --git a/apps/overlays/au-syd1/logging/values-vector-agent.yaml b/apps/overlays/au-syd1/logging/values-vector-agent.yaml
index e9551e7..bff67a5 100644
--- a/apps/overlays/au-syd1/logging/values-vector-agent.yaml
+++ b/apps/overlays/au-syd1/logging/values-vector-agent.yaml
@@ -5,6 +5,12 @@
 role: Agent
 fullnameOverride: vector-agent
 
+# Pulled through the artifactapi dockerhub remote; distroless-libc (no DHI —
+# subscription-gated/private-namespace, not reachable via the anon proxy).
+image:
+  repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector
+  tag: 0.57.0-distroless-libc
+
 rbac:
   create: true
 serviceAccount:
diff --git a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml
index 6f6667b..4ee0d14 100644
--- a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml
+++ b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml
@@ -1,14 +1,29 @@
-# Vector TRANSFORM tier (StatefulSet) — the "brain": sole ClickHouse writer,
-# owns all transforms, holds the only ClickHouse + NATS-consumer credentials.
-# It is a pure JetStream pull consumer (no inbound ports) — durability lives in
-# JetStream, so the local disk buffer is small (5Gi PVC / 2GiB buffer).
+# Vector TRANSFORM tier (STATELESS Deployment) — the "brain": sole ClickHouse
+# writer, owns all transforms, holds the only ClickHouse + NATS-consumer creds.
 #
-# Pipeline is the single source of truth in apps/base/logging/vector/
-# aggregator.yaml (unit-tested by `vector test` in CI), mounted via
-# existingConfigMaps.
-role: Aggregator
+# Stateless by design: a JetStream pull consumer with NO PVC and NO disk buffer.
+# JetStream is the sole durability layer. On a ClickHouse outage the clickhouse
+# sink blocks (buffer when_full=block), back-pressure stops the source pulling,
+# and unpulled messages stay in JetStream for redelivery. Because Vector's NATS
+# source does NOT support end-to-end acknowledgements (it acks on receipt, not
+# after the sink), the only at-risk window is the in-memory buffer's worth of
+# already-pulled events if a pod is killed mid-outage — the accepted trade for a
+# horizontally-autoscalable stateless tier. Multiple replicas share the one
+# durable consumer `transform` (JetStream pull consumers distribute work), so
+# HPA is safe.
+role: Stateless-Aggregator
 fullnameOverride: vector-aggregator
-replicas: 2
+
+image:
+  repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector
+  tag: 0.57.0-distroless-libc
+
+# Horizontal autoscaling on CPU — safe with N replicas on one durable consumer.
+autoscaling:
+  enabled: true
+  minReplicas: 2
+  maxReplicas: 8
+  targetCPUUtilizationPercentage: 70
 
 workloadResourceAnnotations:
   reloader.stakater.com/auto: "true"
@@ -16,17 +31,13 @@ workloadResourceAnnotations:
 podLabels:
   vector.dev/exclude: "true"
 
+# Pipeline is the single source of truth in apps/base/logging/vector/
+# aggregator.yaml (unit-tested by `vector test` in CI), mounted via
+# existingConfigMaps. No persistence — stateless.
 dataDir: /vector-data-dir
 existingConfigMaps:
   - vector-aggregator-config
 
-persistence:
-  enabled: true
-  storageClassName: cephrbd-fast-delete
-  size: 5Gi
-  accessModes:
-    - ReadWriteOnce
-
 # The ONLY place ClickHouse + NATS-consumer creds are consumed.
 env:
   - name: CLICKHOUSE_USER
diff --git a/apps/overlays/au-syd1/logging/values-vector-archiver.yaml b/apps/overlays/au-syd1/logging/values-vector-archiver.yaml
index c7c13c8..671844a 100644
--- a/apps/overlays/au-syd1/logging/values-vector-archiver.yaml
+++ b/apps/overlays/au-syd1/logging/values-vector-archiver.yaml
@@ -5,6 +5,10 @@ role: Stateless-Aggregator
 fullnameOverride: vector-archiver
 replicas: 1
 
+image:
+  repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector
+  tag: 0.57.0-distroless-libc
+
 workloadResourceAnnotations:
   reloader.stakater.com/auto: "true"
 
diff --git a/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml b/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml
index 39b5df5..12fe8b7 100644
--- a/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml
+++ b/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml
@@ -5,6 +5,10 @@ role: Stateless-Aggregator
 fullnameOverride: vector-vm-ingest
 replicas: 2
 
+image:
+  repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector
+  tag: 0.57.0-distroless-libc
+
 workloadResourceAnnotations:
   reloader.stakater.com/auto: "true"