argocd-apps/apps/base/logging/job_clickhouse-schema.yaml

---
# Declarative ClickHouse schema bootstrap. Runs as an ArgoCD PostSync hook so it
# executes after the ClickHouseInstallation is reconciled, and re-runs on every
# sync (idempotent CREATE ... IF NOT EXISTS). Edit the DDL here to evolve the
# schema; the Vector aggregator writes to logs.raw with skip_unknown_fields, so
# adding columns is backward-compatible.
apiVersion: batch/v1
kind: Job
metadata:
  name: clickhouse-schema
  namespace: logging
  annotations:
    argocd.argoproj.io/hook: PostSync
    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
  labels:
    app.kubernetes.io/name: clickhouse-schema
    app.kubernetes.io/component: bootstrap
spec:
  backoffLimit: 20
  activeDeadlineSeconds: 1800
  ttlSecondsAfterFinished: 3600
  template:
    metadata:
      labels:
        app.kubernetes.io/name: clickhouse-schema
        vector.dev/exclude: "true"
    spec:
      restartPolicy: OnFailure
      securityContext:
        runAsNonRoot: true
        runAsUser: 101
        runAsGroup: 101
      containers:
        - name: clickhouse-schema
          image: artifactapi.k8s.syd1.au.unkin.net/dockerhub/clickhouse/clickhouse-server:24.8
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop:
                - ALL
          env:
            - name: HOME
              value: /tmp
            - name: CLICKHOUSE_USER
              valueFrom:
                secretKeyRef:
                  name: clickhouse-credentials
                  key: username
            - name: CLICKHOUSE_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: clickhouse-credentials
                  key: password
          command:
            - /bin/bash
            - -ec
            - |
              host=clickhouse-logs.logging.svc.cluster.local
              echo "Waiting for ClickHouse at ${host}:9000 ..."
              until clickhouse-client --host "$host" --port 9000 \
                  --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \
                  --query "SELECT 1" >/dev/null 2>&1; do
                echo "  not ready, retrying in 5s"; sleep 5
              done
              echo "Applying schema ..."
              clickhouse-client --host "$host" --port 9000 \
                --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \
                --multiquery <<'EOSQL'
              CREATE DATABASE IF NOT EXISTS logs;

              CREATE TABLE IF NOT EXISTS logs.raw
              (
                  timestamp  DateTime64(3)           DEFAULT now64(3),
                  host       LowCardinality(String)  DEFAULT '',
                  source     LowCardinality(String)  DEFAULT '',
                  namespace  LowCardinality(String)  DEFAULT '',
                  pod        String                  DEFAULT '',
                  container  LowCardinality(String)  DEFAULT '',
                  stream     LowCardinality(String)  DEFAULT '',
                  severity   LowCardinality(String)  DEFAULT '',
                  message    String                  DEFAULT '',
                  labels     Map(LowCardinality(String), String),
                  fields     Map(LowCardinality(String), String)
              )
              ENGINE = MergeTree
              PARTITION BY toDate(timestamp)
              ORDER BY (source, namespace, host, timestamp)
              TTL toDateTime(timestamp) + INTERVAL 3 DAY
              SETTINGS index_granularity = 8192;

              -- One row per archived S3 object (written by logarchiver). No TTL:
              -- the index must outlive logs.raw so the long-term S3 archive stays
              -- searchable. Keep in sync with logarchiver internal/index/ddl.go.
              CREATE TABLE IF NOT EXISTS logs.archive_index
              (
                  object_key       String,
                  bucket           LowCardinality(String),
                  subject          LowCardinality(String),
                  hosts            Array(LowCardinality(String)),
                  min_ts           DateTime64(3),
                  max_ts           DateTime64(3),
                  event_count      UInt64,
                  raw_bytes        UInt64,
                  stored_bytes     UInt64,
                  compression      LowCardinality(String),
                  cipher           LowCardinality(String),
                  container_format LowCardinality(String),
                  key_name         LowCardinality(String),
                  key_fingerprint  String,
                  created_at       DateTime64(3) DEFAULT now64(3),
                  INDEX idx_hosts hosts TYPE bloom_filter GRANULARITY 1
              )
              ENGINE = MergeTree
              PARTITION BY toYYYYMM(min_ts)
              ORDER BY (subject, min_ts, object_key);
              EOSQL
              echo "Schema applied."
          resources:
            requests:
              cpu: 50m
              memory: 128Mi
            limits:
              cpu: 500m
              memory: 512Mi
          volumeMounts:
            - name: tmp
              mountPath: /tmp
      volumes:
        - name: tmp
          emptyDir: {}