Replace vector-archiver with logarchiver

The Vector archiver leg wrote gzip NDJSON to S3 with no index or encryption.
logarchiver replaces it with a Go service that seals raw logs to S3 as zstd +
OpenPGP objects and indexes each object in ClickHouse (logs.archive_index),
acking JetStream only after the object is stored and indexed.

- Add logarchiver Deployment (image git.unkin.net/unkin/logarchiver:v0.1.0),
  ConfigMap, and dedicated ServiceAccount. Reuses the archiver's NATS
  (log-consumer / durable archiver / ARCHIVE_SUBJECTS=logs.k8s.vault.>), S3
  (logs-archive-s3 BucketAccess), ClickHouse (clickhouse-credentials) and
  vault-ca wiring. Encrypts to the logarchive gpg public key, fetched from the
  gpg engine via k8s auth (role logging_logarchiver, projected vault-audience
  token). ack_wait > batch max_age so messages are not redelivered mid-batch.
- Add logs.archive_index DDL to the clickhouse-schema bootstrap Job (no TTL).
- Remove the vector-archiver Helm release, values and pipeline ConfigMap.

Cross-repo: apply terraform-vault #106 (gpg key + role/policy) before this syncs,
or the pod cannot fetch the public key. Sequencing: apply after argocd-apps #306.

Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv

apps/base/logging/configmap_logarchiver.yaml

@@ -0,0 +1,30 @@
+---
+# logarchiver non-secret config. Secrets (NATS/S3/ClickHouse creds) and the
+# subject filter come from env; everything else uses the binary's built-in
+# defaults, which already target this stack. ack_wait MUST exceed batch.max_age
+# so unacked messages in an open batch are not redelivered mid-batch.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logarchiver-config
+  namespace: logging
+data:
+  config.yaml: |
+    nats:
+      ack_wait: 5m
+      fetch_batch: 512
+    batch:
+      max_bytes: 67108864
+      max_events: 200000
+      max_age: 2m
+    crypto:
+      key_name: logarchive
+      pubkey_source: vault
+      vault:
+        address: "https://vault.service.consul:8200"
+        mount: gpg
+        auth_method: kubernetes
+        k8s_mount: k8s/au/syd1
+        k8s_role: logging_logarchiver
+        k8s_jwt_path: /var/run/secrets/vault/token
+        ca_file: /etc/vault-ca/ca.crt

apps/base/logging/deployment_logarchiver.yaml

@@ -0,0 +1,133 @@
+---
+# logarchiver — replaces the vector-archiver leg. Independent JetStream durable
+# consumer (archiver) that seals raw logs to S3 as zstd + OpenPGP objects and
+# indexes each object in ClickHouse (logs.archive_index). Acks only after the
+# object is in S3 AND indexed. Reuses the same NATS/S3/CA wiring the Vector
+# archiver used; the OpenPGP public key is delivered as a mounted file.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: logarchiver
+  namespace: logging
+  annotations:
+    reloader.stakater.com/auto: "true"
+  labels:
+    app.kubernetes.io/name: logarchiver
+    app.kubernetes.io/component: archiver
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: logarchiver
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: logarchiver
+        vector.dev/exclude: "true"
+    spec:
+      # Dedicated SA whose projected vault-audience token authenticates the
+      # k8s-auth login used to fetch the logarchive public key from the gpg engine.
+      serviceAccountName: logarchiver
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: logarchiver
+          image: git.unkin.net/unkin/logarchiver:v0.1.0
+          imagePullPolicy: IfNotPresent
+          args: ["run"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          ports:
+            - containerPort: 9090
+              name: metrics
+              protocol: TCP
+          env:
+            - name: LOGARCHIVER_CONFIG
+              value: /etc/logarchiver/config.yaml
+            # Server-side subject filter; must match the archiver consumer's filter.
+            - name: ARCHIVE_SUBJECTS
+              value: "logs.k8s.vault.>"
+            - name: NATS_CONSUMER_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: nats-auth
+                  key: consumer_password
+            - name: CLICKHOUSE_USER
+              valueFrom:
+                secretKeyRef:
+                  name: clickhouse-credentials
+                  key: username
+            - name: CLICKHOUSE_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: clickhouse-credentials
+                  key: password
+          # S3 creds + S3_ENDPOINT + BUCKET_NAME from the cephrgw BucketAccess Secret.
+          envFrom:
+            - secretRef:
+                name: logs-archive-s3
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 15
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: "1"
+              memory: 1Gi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/logarchiver/config.yaml
+              subPath: config.yaml
+              readOnly: true
+            - name: vault-token
+              mountPath: /var/run/secrets/vault
+              readOnly: true
+            - name: vault-ca-cert
+              mountPath: /etc/vault-ca/ca.crt
+              subPath: ca.crt
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: config
+          configMap:
+            name: logarchiver-config
+        # Projected SA token with audience "vault" for the gpg-engine k8s login.
+        - name: vault-token
+          projected:
+            sources:
+              - serviceAccountToken:
+                  path: token
+                  audience: vault
+                  expirationSeconds: 600
+        - name: vault-ca-cert
+          secret:
+            secretName: vault-ca-cert
+        - name: tmp
+          emptyDir: {}

apps/base/logging/job_clickhouse-schema.yaml

@@ -88,6 +88,32 @@ spec:
               ORDER BY (source, namespace, host, timestamp)
               TTL toDateTime(timestamp) + INTERVAL 3 DAY
               SETTINGS index_granularity = 8192;
+
+              -- One row per archived S3 object (written by logarchiver). No TTL:
+              -- the index must outlive logs.raw so the long-term S3 archive stays
+              -- searchable. Keep in sync with logarchiver internal/index/ddl.go.
+              CREATE TABLE IF NOT EXISTS logs.archive_index
+              (
+                  object_key       String,
+                  bucket           LowCardinality(String),
+                  subject          LowCardinality(String),
+                  hosts            Array(LowCardinality(String)),
+                  min_ts           DateTime64(3),
+                  max_ts           DateTime64(3),
+                  event_count      UInt64,
+                  raw_bytes        UInt64,
+                  stored_bytes     UInt64,
+                  compression      LowCardinality(String),
+                  cipher           LowCardinality(String),
+                  container_format LowCardinality(String),
+                  key_name         LowCardinality(String),
+                  key_fingerprint  String,
+                  created_at       DateTime64(3) DEFAULT now64(3),
+                  INDEX idx_hosts hosts TYPE bloom_filter GRANULARITY 1
+              )
+              ENGINE = MergeTree
+              PARTITION BY toYYYYMM(min_ts)
+              ORDER BY (subject, min_ts, object_key);
               EOSQL
               echo "Schema applied."
           resources:

apps/base/logging/kustomization.yaml

@@ -12,6 +12,9 @@ resources:
   - cephrgw.yaml
   - gateway.yaml
   - httproute.yaml
+  - serviceaccount_logarchiver.yaml
+  - configmap_logarchiver.yaml
+  - deployment_logarchiver.yaml
 
 # Vector pipelines are the single source of truth (also validated by
 # `vector test` in CI). Mounted into each tier via `existingConfigMaps`.
@@ -44,8 +47,3 @@ configMapGenerator:
       - vm-ingest.yaml=vector/vm-ingest.yaml
     options:
       disableNameSuffixHash: true
-  - name: vector-archiver-config
-    files:
-      - archiver.yaml=vector/archiver.yaml
-    options:
-      disableNameSuffixHash: true

apps/base/logging/serviceaccount_logarchiver.yaml

@@ -0,0 +1,9 @@
+---
+# Dedicated SA for logarchiver's Vault k8s-auth login (role logging_logarchiver,
+# terraform-vault). Only used to fetch the logarchive public key.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: logarchiver
+  namespace: logging
+automountServiceAccountToken: false

apps/base/logging/vector/archiver.yaml

@@ -1,62 +0,0 @@
----
-# Vector ARCHIVER tier — long-term raw-log backup to S3 (Ceph RGW). Independent
-# durable JetStream consumer (`archiver`) so its offsets/lag are fully isolated
-# from the ClickHouse transform path (archive lag can never stall ingest — true
-# fan-out). Writes RAW, pre-transform events (as they sit in JetStream) as
-# gzipped NDJSON, partitioned by subject + date. This is the long-horizon replay
-# source beyond JetStream's 3d retention window.
-data_dir: /vector-data-dir
-
-api:
-  enabled: true
-  address: 0.0.0.0:8686
-
-sources:
-  js_archive:
-    type: nats
-    url: nats://nats.logging.svc.cluster.local:4222
-    connection_name: vector-archiver
-    subject: "logs.>"
-    jetstream:
-      stream: LOGS
-      consumer: archiver
-    auth:
-      strategy: user_password
-      user_password:
-        user: log-consumer
-        password: ${NATS_CONSUMER_PASSWORD}
-    decoding:
-      codec: json
-
-sinks:
-  s3:
-    type: aws_s3
-    inputs:
-      - js_archive
-    bucket: logs-archive
-    endpoint: https://s3.ceph.unkin.net
-    region: us-east-1
-    force_path_style: true
-    tls:
-      ca_file: /etc/vault-ca/ca.crt
-    # AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY come from the logs-archive-s3
-    # Secret (cephrgw-operator) via envFrom on the deployment.
-    key_prefix: "raw/{{ subject }}/%Y/%m/%d/"
-    compression: gzip
-    encoding:
-      codec: json
-    framing:
-      method: newline_delimited
-    filename_time_format: "%Y%m%dT%H%M%SZ"
-    filename_append_uuid: true
-    batch:
-      max_bytes: 134217728
-      timeout_secs: 300
-    buffer:
-      type: memory
-      max_events: 5000
-      when_full: block
-    # Disabled so slow BucketAccess credential propagation doesn't crash-loop
-    # the pod; RGW reachability is proven by the operator's own health.
-    healthcheck:
-      enabled: false

apps/overlays/au-syd1/logging/kustomization.yaml

@@ -36,10 +36,4 @@ helmCharts:
     releaseName: vector-aggregator
     namespace: logging
     valuesFile: values-vector-aggregator.yaml
-  # Archiver (Deployment): independent JetStream consumer -> raw logs to S3.
+  # Archiver: replaced by the logarchiver Deployment (apps/base/logging).
-  - name: vector
-    repo: https://helm.vector.dev
-    version: "0.57.0"
-    releaseName: vector-archiver
-    namespace: logging
-    valuesFile: values-vector-archiver.yaml

apps/overlays/au-syd1/logging/values-vector-archiver.yaml

@@ -1,59 +0,0 @@
-# Vector ARCHIVER tier (Deployment) — independent JetStream consumer writing raw
-# logs to S3 (Ceph RGW). Isolated from the ClickHouse path (own durable
-# consumer). Pipeline: apps/base/logging/vector/archiver.yaml.
-role: Stateless-Aggregator
-fullnameOverride: vector-archiver
-replicas: 1
-
-image:
-  repository: artifactapi.k8s.syd1.au.unkin.net/dockerhub/timberio/vector
-  tag: 0.57.0-distroless-libc
-
-workloadResourceAnnotations:
-  reloader.stakater.com/auto: "true"
-
-podLabels:
-  vector.dev/exclude: "true"
-
-dataDir: /vector-data-dir
-existingConfigMaps:
-  - vector-archiver-config
-
-env:
-  # Vector 0.57 disables ${VAR} config interpolation by default; auth needs it.
-  - name: VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION
-    value: "true"
-  - name: NATS_CONSUMER_PASSWORD
-    valueFrom:
-      secretKeyRef:
-        name: nats-auth
-        key: consumer_password
-
-# S3 creds (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY) from the cephrgw-operator
-# BucketAccess Secret.
-envFrom:
-  - secretRef:
-      name: logs-archive-s3
-
-# Trust the internal unkin.net Vault-PKI CA to verify s3.ceph.unkin.net.
-# vault-ca-cert is reflected into every namespace from the certificates ns.
-extraVolumes:
-  - name: vault-ca-cert
-    secret:
-      secretName: vault-ca-cert
-extraVolumeMounts:
-  - name: vault-ca-cert
-    mountPath: /etc/vault-ca/ca.crt
-    subPath: ca.crt
-    readOnly: true
-
-service:
-  enabled: false
-
-resources:
-  requests:
-    cpu: 100m
-    memory: 256Mi
-  limits:
-    cpu: "1"
-    memory: 1Gi