diff --git a/apps/base/clickhouse-system/kustomization.yaml b/apps/base/clickhouse-system/kustomization.yaml deleted file mode 100644 index 5122f9e..0000000 --- a/apps/base/clickhouse-system/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - namespace.yaml diff --git a/apps/base/clickhouse-system/namespace.yaml b/apps/base/clickhouse-system/namespace.yaml deleted file mode 100644 index 519dfb3..0000000 --- a/apps/base/clickhouse-system/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - labels: - app.kubernetes.io/name: clickhouse-system - name: clickhouse-system diff --git a/apps/base/logging/cephrgw.yaml b/apps/base/logging/cephrgw.yaml deleted file mode 100644 index ee1bd8e..0000000 --- a/apps/base/logging/cephrgw.yaml +++ /dev/null @@ -1,43 +0,0 @@ ---- -# S3 bucket (Ceph RGW) for the long-term raw-log archive, provisioned by the -# in-estate cephrgw-operator. The archiver Vector deployment writes here. -apiVersion: ceph.unkin.net/v1alpha1 -kind: ObjectStoreUser -metadata: - name: logs-archive-owner - namespace: logging -spec: - displayName: "Logging raw-archive bucket owner" - maxBuckets: 5 - quota: - enabled: true - # 5 TiB soft cap; real retention is enforced RGW-side by a bucket lifecycle - # policy (see PR notes) — the operator does not manage lifecycle. - maxSizeBytes: 5497558138880 ---- -apiVersion: ceph.unkin.net/v1alpha1 -kind: Bucket -metadata: - name: logs-archive - namespace: logging -spec: - bucketName: logs-archive - ownerRef: logs-archive-owner - versioning: false - tags: - app: logging - purpose: raw-log-archive - # Keep the bucket (and its objects) if this CR is ever deleted. - retainOnDelete: true ---- -apiVersion: ceph.unkin.net/v1alpha1 -kind: BucketAccess -metadata: - name: logs-archive-writer - namespace: logging -spec: - bucketRef: logs-archive - level: read-write - # Operator writes AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (+ RGW_UID, - # S3_ENDPOINT, BUCKET_NAME) into this Secret; the archiver consumes it. - secretName: logs-archive-s3 diff --git a/apps/base/logging/clickhouseinstallation.yaml b/apps/base/logging/clickhouseinstallation.yaml deleted file mode 100644 index 886bbcf..0000000 --- a/apps/base/logging/clickhouseinstallation.yaml +++ /dev/null @@ -1,97 +0,0 @@ ---- -apiVersion: clickhouse.altinity.com/v1 -kind: ClickHouseInstallation -metadata: - name: logs - namespace: logging -spec: - defaults: - templates: - dataVolumeClaimTemplate: data-volume - serviceTemplate: chi-service - podTemplate: clickhouse - configuration: - users: - # Password hash is sourced from the Vault-synced clickhouse-credentials - # Secret; the plaintext never lands in git or the ClickHouse config. - vector/password_sha256_hex: - valueFrom: - secretKeyRef: - name: clickhouse-credentials - key: password_sha256_hex - vector/networks/ip: - - "::/0" - vector/profile: default - vector/quota: default - # Allow the vector user to create the logs database/table (bootstrap Job) - # and to INSERT. Restrict the built-in default user to loopback only. - vector/access_management: "1" - default/networks/ip: - - "127.0.0.1" - - "::1" - # Read-only user for CLI tools + the logviewer UI. Hash sourced from the - # Vault-synced clickhouse-logreader Secret, same mechanism as vector. - # Scoped to the logs database only (unlike vector, which bootstraps it). - logreader/password_sha256_hex: - valueFrom: - secretKeyRef: - name: clickhouse-logreader - key: password_sha256_hex - logreader/networks/ip: - - "::/0" - logreader/profile: readonly - logreader/quota: default - logreader/allow_databases/database: - - "logs" - profiles: - default/max_memory_usage: "10000000000" - default/max_execution_time: "120" - readonly/readonly: "2" - readonly/max_memory_usage: "10000000000" - readonly/max_execution_time: "120" - clusters: - - name: logs - layout: - shardsCount: 1 - replicasCount: 1 - templates: - volumeClaimTemplates: - - name: data-volume - spec: - storageClassName: cephrbd-fast-delete - accessModes: - - ReadWriteOnce - resources: - requests: - # 3d TTL on logs.raw. At ~130 GiB/day raw, ClickHouse LZ4/ZSTD - # (~6x on log text) stores ~20-25 GiB/day => ~60-75 GiB/3d, plus - # merge headroom (~2x peak). logs.raw is the only table. 150Gi - # gives comfortable headroom; long-term data lives in S3, not here. - storage: 150Gi - serviceTemplates: - - name: chi-service - generateName: "clickhouse-{chi}" - spec: - type: ClusterIP - ports: - - name: http - port: 8123 - - name: tcp - port: 9000 - podTemplates: - - name: clickhouse - spec: - securityContext: - fsGroup: 101 - runAsUser: 101 - runAsGroup: 101 - containers: - - name: clickhouse - image: docker.io/clickhouse/clickhouse-server:24.8 - resources: - requests: - cpu: 500m - memory: 2Gi - limits: - cpu: "2" - memory: 8Gi diff --git a/apps/base/logging/configmap_logarchiver.yaml b/apps/base/logging/configmap_logarchiver.yaml deleted file mode 100644 index b6d4996..0000000 --- a/apps/base/logging/configmap_logarchiver.yaml +++ /dev/null @@ -1,40 +0,0 @@ ---- -# logarchiver non-secret config. Secrets (NATS/S3/ClickHouse creds) and the -# subject filter come from env; everything else uses the binary's built-in -# defaults, which already target this stack. ack_wait MUST exceed batch.max_age -# so unacked messages in an open batch are not redelivered mid-batch. -apiVersion: v1 -kind: ConfigMap -metadata: - name: logarchiver-config - namespace: logging -data: - config.yaml: | - nats: - ack_wait: 5m - fetch_batch: 512 - batch: - max_bytes: 67108864 - max_events: 200000 - max_age: 2m - # Pin the proven RGW endpoint/bucket; ignore the secret's S3_ENDPOINT/BUCKET_NAME - # (AWS creds still come from the secret env). endpoint_env/bucket_env off. - s3: - endpoint: "https://s3.ceph.unkin.net" - bucket: "logs-archive" - region: "us-east-1" - path_style: true - ca_file: /etc/vault-ca/ca.crt - endpoint_env: "" - bucket_env: "" - crypto: - key_name: logarchive - pubkey_source: vault - vault: - address: "https://vault.service.consul:8200" - mount: gpg - auth_method: kubernetes - k8s_mount: k8s/au/syd1 - k8s_role: logging_logarchiver - k8s_jwt_path: /var/run/secrets/vault/token - ca_file: /etc/vault-ca/ca.crt diff --git a/apps/base/logging/deployment_logarchiver.yaml b/apps/base/logging/deployment_logarchiver.yaml deleted file mode 100644 index 2a12f78..0000000 --- a/apps/base/logging/deployment_logarchiver.yaml +++ /dev/null @@ -1,134 +0,0 @@ ---- -# logarchiver — replaces the vector-archiver leg. Independent JetStream durable -# consumer (archiver) that seals raw logs to S3 as zstd + OpenPGP objects and -# indexes each object in ClickHouse (logs.archive_index). Acks only after the -# object is in S3 AND indexed. Reuses the same NATS/S3/CA wiring the Vector -# archiver used; the OpenPGP public key is delivered as a mounted file. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: logarchiver - namespace: logging - annotations: - configmap.reloader.stakater.com/auto: "true" - secret.reloader.stakater.com/reload: "vault-ca-cert" - labels: - app.kubernetes.io/name: logarchiver - app.kubernetes.io/component: archiver -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: logarchiver - template: - metadata: - labels: - app.kubernetes.io/name: logarchiver - vector.dev/exclude: "true" - spec: - # Dedicated SA whose projected vault-audience token authenticates the - # k8s-auth login used to fetch the logarchive public key from the gpg engine. - serviceAccountName: logarchiver - automountServiceAccountToken: false - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - seccompProfile: - type: RuntimeDefault - containers: - - name: logarchiver - image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/logarchiver:v0.1.0 - imagePullPolicy: IfNotPresent - args: ["run"] - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - ports: - - containerPort: 9090 - name: metrics - protocol: TCP - env: - - name: LOGARCHIVER_CONFIG - value: /etc/logarchiver/config.yaml - # Server-side subject filter; must match the archiver consumer's filter. - - name: ARCHIVE_SUBJECTS - value: "logs.k8s.vault.>" - - name: NATS_CONSUMER_PASSWORD - valueFrom: - secretKeyRef: - name: nats-auth - key: consumer_password - - name: CLICKHOUSE_USER - valueFrom: - secretKeyRef: - name: clickhouse-credentials - key: username - - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: clickhouse-credentials - key: password - # S3 creds + S3_ENDPOINT + BUCKET_NAME from the cephrgw BucketAccess Secret. - envFrom: - - secretRef: - name: logs-archive-s3 - livenessProbe: - httpGet: - path: /healthz - port: metrics - initialDelaySeconds: 15 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /healthz - port: metrics - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: "1" - memory: 1Gi - volumeMounts: - - name: config - mountPath: /etc/logarchiver/config.yaml - subPath: config.yaml - readOnly: true - - name: vault-token - mountPath: /var/run/secrets/vault - readOnly: true - - name: vault-ca-cert - mountPath: /etc/vault-ca/ca.crt - subPath: ca.crt - readOnly: true - - name: tmp - mountPath: /tmp - volumes: - - name: config - configMap: - name: logarchiver-config - # Projected SA token with audience "vault" for the gpg-engine k8s login. - - name: vault-token - projected: - sources: - - serviceAccountToken: - path: token - audience: vault - expirationSeconds: 600 - - name: vault-ca-cert - secret: - secretName: vault-ca-cert - - name: tmp - emptyDir: {} diff --git a/apps/base/logging/job_clickhouse-schema.yaml b/apps/base/logging/job_clickhouse-schema.yaml deleted file mode 100644 index ecc3143..0000000 --- a/apps/base/logging/job_clickhouse-schema.yaml +++ /dev/null @@ -1,131 +0,0 @@ ---- -# Declarative ClickHouse schema bootstrap. Runs as an ArgoCD PostSync hook so it -# executes after the ClickHouseInstallation is reconciled, and re-runs on every -# sync (idempotent CREATE ... IF NOT EXISTS). Edit the DDL here to evolve the -# schema; the Vector aggregator writes to logs.raw with skip_unknown_fields, so -# adding columns is backward-compatible. -apiVersion: batch/v1 -kind: Job -metadata: - name: clickhouse-schema - namespace: logging - annotations: - argocd.argoproj.io/hook: PostSync - argocd.argoproj.io/hook-delete-policy: BeforeHookCreation - labels: - app.kubernetes.io/name: clickhouse-schema - app.kubernetes.io/component: bootstrap -spec: - backoffLimit: 20 - activeDeadlineSeconds: 1800 - ttlSecondsAfterFinished: 3600 - template: - metadata: - labels: - app.kubernetes.io/name: clickhouse-schema - vector.dev/exclude: "true" - spec: - restartPolicy: OnFailure - securityContext: - runAsNonRoot: true - runAsUser: 101 - runAsGroup: 101 - containers: - - name: clickhouse-schema - image: docker.io/clickhouse/clickhouse-server:24.8 - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - env: - - name: HOME - value: /tmp - - name: CLICKHOUSE_USER - valueFrom: - secretKeyRef: - name: clickhouse-credentials - key: username - - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: clickhouse-credentials - key: password - command: - - /bin/bash - - -ec - - | - host=clickhouse-logs.logging.svc.cluster.local - echo "Waiting for ClickHouse at ${host}:9000 ..." - until clickhouse-client --host "$host" --port 9000 \ - --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \ - --query "SELECT 1" >/dev/null 2>&1; do - echo " not ready, retrying in 5s"; sleep 5 - done - echo "Applying schema ..." - clickhouse-client --host "$host" --port 9000 \ - --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" \ - --multiquery <<'EOSQL' - CREATE DATABASE IF NOT EXISTS logs; - - CREATE TABLE IF NOT EXISTS logs.raw - ( - timestamp DateTime64(3) DEFAULT now64(3), - host LowCardinality(String) DEFAULT '', - source LowCardinality(String) DEFAULT '', - namespace LowCardinality(String) DEFAULT '', - pod String DEFAULT '', - container LowCardinality(String) DEFAULT '', - stream LowCardinality(String) DEFAULT '', - severity LowCardinality(String) DEFAULT '', - message String DEFAULT '', - labels Map(LowCardinality(String), String), - fields Map(LowCardinality(String), String) - ) - ENGINE = MergeTree - PARTITION BY toDate(timestamp) - ORDER BY (source, namespace, host, timestamp) - TTL toDateTime(timestamp) + INTERVAL 3 DAY - SETTINGS index_granularity = 8192; - - -- One row per archived S3 object (written by logarchiver). No TTL: - -- the index must outlive logs.raw so the long-term S3 archive stays - -- searchable. Keep in sync with logarchiver internal/index/ddl.go. - CREATE TABLE IF NOT EXISTS logs.archive_index - ( - object_key String, - bucket LowCardinality(String), - subject LowCardinality(String), - hosts Array(LowCardinality(String)), - min_ts DateTime64(3), - max_ts DateTime64(3), - event_count UInt64, - raw_bytes UInt64, - stored_bytes UInt64, - compression LowCardinality(String), - cipher LowCardinality(String), - container_format LowCardinality(String), - key_name LowCardinality(String), - key_fingerprint String, - created_at DateTime64(3) DEFAULT now64(3), - INDEX idx_hosts hosts TYPE bloom_filter GRANULARITY 1 - ) - ENGINE = MergeTree - PARTITION BY toYYYYMM(min_ts) - ORDER BY (subject, min_ts, object_key); - EOSQL - echo "Schema applied." - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 500m - memory: 512Mi - volumeMounts: - - name: tmp - mountPath: /tmp - volumes: - - name: tmp - emptyDir: {} diff --git a/apps/base/logging/kustomization.yaml b/apps/base/logging/kustomization.yaml index d954254..7fdff26 100644 --- a/apps/base/logging/kustomization.yaml +++ b/apps/base/logging/kustomization.yaml @@ -5,48 +5,8 @@ kind: Kustomization resources: - namespace.yaml - vaultauth.yaml - - vaultstaticsecret.yaml - - clickhouseinstallation.yaml - - job_clickhouse-schema.yaml - - nats-bootstrap-job.yaml - - cephrgw.yaml - vlcluster.yaml + - vlagent.yaml - gateway.yaml - httproute.yaml - - serviceaccount_logarchiver.yaml - - configmap_logarchiver.yaml - - deployment_logarchiver.yaml - - logviewer - vlogs - -# Vector pipelines are the single source of truth (also validated by -# `vector test` in CI). Mounted into each tier via `existingConfigMaps`. -configMapGenerator: - # Tunable JetStream stream limits (the nats-bootstrap Job reads these and does - # create-or-update). Hash suffix is INTENTIONALLY left on: editing a value - # renames the ConfigMap, which rewrites the Job's env reference, which changes - # the PostSync hook Job's spec and forces Argo to re-run it -> new limits apply. - # Sizing assumes ~1500 events/s avg @ ~1 KiB/event with S2 compression (~4x): - # ~33 GiB/day compressed -> ~100 GiB/3d per replica. max_bytes 130 GiB sits - # under the 180Gi/node PVC (see values-nats.yaml). Raising retention beyond the - # PVC requires bumping BOTH max_bytes here and fileStore PVC size in values. - - name: nats-stream-limits - literals: - - max_age=72h - - max_bytes=139586437120 - - dupe_window=2m - - name: vector-agent-config - files: - - agent.yaml=vector/agent.yaml - options: - disableNameSuffixHash: true - - name: vector-aggregator-config - files: - - aggregator.yaml=vector/aggregator.yaml - options: - disableNameSuffixHash: true - - name: vector-vm-ingest-config - files: - - vm-ingest.yaml=vector/vm-ingest.yaml - options: - disableNameSuffixHash: true diff --git a/apps/base/logging/logviewer/deployment.yaml b/apps/base/logging/logviewer/deployment.yaml deleted file mode 100644 index c0394d4..0000000 --- a/apps/base/logging/logviewer/deployment.yaml +++ /dev/null @@ -1,80 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: logviewer - namespace: logging - annotations: - secret.reloader.stakater.com/reload: "clickhouse-logreader" -spec: - replicas: 1 - selector: - matchLabels: - app: logviewer - strategy: - type: Recreate - template: - metadata: - labels: - app: logviewer - spec: - serviceAccountName: default - automountServiceAccountToken: false - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - fsGroup: 65532 - seccompProfile: - type: RuntimeDefault - containers: - - name: logviewer - image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/logviewer:v0.1.0 - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8080 - name: http - protocol: TCP - env: - - name: CH_URL - value: http://clickhouse-logs.logging.svc.cluster.local:8123 - - name: CH_USER - valueFrom: - secretKeyRef: - name: clickhouse-logreader - key: username - - name: CH_PASSWORD - valueFrom: - secretKeyRef: - name: clickhouse-logreader - key: password - livenessProbe: - httpGet: - path: /healthz - port: http - initialDelaySeconds: 10 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /healthz - port: http - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 512Mi - restartPolicy: Always diff --git a/apps/base/logging/logviewer/gateway.yaml b/apps/base/logging/logviewer/gateway.yaml deleted file mode 100644 index fbac5bc..0000000 --- a/apps/base/logging/logviewer/gateway.yaml +++ /dev/null @@ -1,38 +0,0 @@ ---- -# Internal front for the logviewer UI (cf. mediamover/pdbmux). -apiVersion: gateway.networking.k8s.io/v1 -kind: Gateway -metadata: - labels: - traefik.io/instance: internal - annotations: - cert-manager.io/cluster-issuer: vault-issuer - cert-manager.io/common-name: logviewer.unkin.net - cert-manager.io/private-key-size: "4096" - external-dns.alpha.kubernetes.io/hostname: logviewer.unkin.net - external-dns.alpha.kubernetes.io/target: 198.18.200.4 - name: logviewer - namespace: logging -spec: - gatewayClassName: traefik-internal - listeners: - - allowedRoutes: - namespaces: - from: Same - hostname: logviewer.unkin.net - name: http - port: 80 - protocol: HTTP - - allowedRoutes: - namespaces: - from: Same - hostname: logviewer.unkin.net - name: https - port: 443 - protocol: HTTPS - tls: - certificateRefs: - - group: "" - kind: Secret - name: logviewer-tls - mode: Terminate diff --git a/apps/base/logging/logviewer/httproute.yaml b/apps/base/logging/logviewer/httproute.yaml deleted file mode 100644 index efe4464..0000000 --- a/apps/base/logging/logviewer/httproute.yaml +++ /dev/null @@ -1,49 +0,0 @@ ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: logviewer-http-redirect - namespace: logging -spec: - hostnames: - - logviewer.unkin.net - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: logviewer - sectionName: http - rules: - - filters: - - type: RequestRedirect - requestRedirect: - scheme: https - statusCode: 301 - matches: - - path: - type: PathPrefix - value: / ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: logviewer - namespace: logging -spec: - hostnames: - - logviewer.unkin.net - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: logviewer - sectionName: https - rules: - - backendRefs: - - group: "" - kind: Service - name: logviewer-oauth2 - port: 80 - weight: 1 - matches: - - path: - type: PathPrefix - value: / diff --git a/apps/base/logging/logviewer/kustomization.yaml b/apps/base/logging/logviewer/kustomization.yaml deleted file mode 100644 index 2189b2e..0000000 --- a/apps/base/logging/logviewer/kustomization.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - vaultstaticsecret.yaml - - deployment.yaml - - oauth2-proxy-configmap.yaml - - oauth2-proxy-deployment.yaml - - service.yaml - - gateway.yaml - - httproute.yaml diff --git a/apps/base/logging/logviewer/oauth2-proxy-configmap.yaml b/apps/base/logging/logviewer/oauth2-proxy-configmap.yaml deleted file mode 100644 index 9f5631e..0000000 --- a/apps/base/logging/logviewer/oauth2-proxy-configmap.yaml +++ /dev/null @@ -1,32 +0,0 @@ ---- -# Non-secret oauth2-proxy configuration (client_id/secret/cookie_secret come -# from the logviewer-oauth-credentials Secret). Single auth front for the -# logviewer UI: everything requires an Authentik session in akP-logviewer-admin. -apiVersion: v1 -kind: ConfigMap -metadata: - name: logviewer-oauth2-env - namespace: logging -data: - OAUTH2_PROXY_HTTP_ADDRESS: "0.0.0.0:4180" - OAUTH2_PROXY_PROVIDER: "oidc" - # Authentik logviewer app discovery issuer (terraform-authentik PR #21). - OAUTH2_PROXY_OIDC_ISSUER_URL: "https://identity.unkin.net/application/o/logviewer/" - OAUTH2_PROXY_REDIRECT_URL: "https://logviewer.unkin.net/oauth2/callback" - OAUTH2_PROXY_UPSTREAMS: "http://logviewer.logging.svc.cluster.local:8080/" - OAUTH2_PROXY_SCOPE: "openid email profile ak_groups" - # Populate session.Groups from the Authentik ak_groups claim. - OAUTH2_PROXY_OIDC_GROUPS_CLAIM: "ak_groups" - OAUTH2_PROXY_ALLOWED_GROUPS: "akP-logviewer-admin" - OAUTH2_PROXY_PASS_USER_HEADERS: "true" - OAUTH2_PROXY_EMAIL_DOMAINS: "*" - # Authentik hardcodes email_verified=false in the id_token; authorization is - # enforced via ak_groups, so accepting the unverified email is safe. - OAUTH2_PROXY_INSECURE_OIDC_ALLOW_UNVERIFIED_EMAIL: "true" - OAUTH2_PROXY_COOKIE_SECURE: "true" - OAUTH2_PROXY_COOKIE_DOMAINS: "logviewer.unkin.net" - OAUTH2_PROXY_WHITELIST_DOMAINS: "logviewer.unkin.net" - OAUTH2_PROXY_REVERSE_PROXY: "true" - OAUTH2_PROXY_PROVIDER_CA_FILES: "/etc/ssl/combined/ca-certificates.crt" - OAUTH2_PROXY_CODE_CHALLENGE_METHOD: "S256" - OAUTH2_PROXY_SKIP_PROVIDER_BUTTON: "true" diff --git a/apps/base/logging/logviewer/oauth2-proxy-deployment.yaml b/apps/base/logging/logviewer/oauth2-proxy-deployment.yaml deleted file mode 100644 index 57b5e9a..0000000 --- a/apps/base/logging/logviewer/oauth2-proxy-deployment.yaml +++ /dev/null @@ -1,132 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: logviewer-oauth2 - namespace: logging - annotations: - configmap.reloader.stakater.com/auto: "true" - secret.reloader.stakater.com/reload: "logviewer-oauth-credentials,vault-ca-cert" -spec: - replicas: 1 - selector: - matchLabels: - app: logviewer-oauth2 - strategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate - template: - metadata: - labels: - app: logviewer-oauth2 - spec: - serviceAccountName: default - automountServiceAccountToken: false - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - fsGroup: 65532 - seccompProfile: - type: RuntimeDefault - initContainers: - # identity.unkin.net serves a Vault-PKI cert; combine the system roots - # with the internal CA so oauth2-proxy's OIDC HTTP client trusts it. - - name: combine-certs - image: docker.io/library/alpine:3 - imagePullPolicy: IfNotPresent - command: - - sh - - -c - - cat /etc/ssl/certs/ca-certificates.crt /custom-ca/ca.crt > /combined-certs/ca-certificates.crt - volumeMounts: - - name: vault-ca-cert - mountPath: /custom-ca - readOnly: true - - name: combined-certs - mountPath: /combined-certs - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - resources: - requests: - cpu: 50m - memory: 32Mi - limits: - cpu: 200m - memory: 64Mi - containers: - - name: oauth2-proxy - image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.3 - imagePullPolicy: IfNotPresent - ports: - - containerPort: 4180 - name: http - protocol: TCP - envFrom: - - configMapRef: - name: logviewer-oauth2-env - optional: false - env: - - name: OAUTH2_PROXY_CLIENT_ID - valueFrom: - secretKeyRef: - name: logviewer-oauth-credentials - key: client_id - - name: OAUTH2_PROXY_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: logviewer-oauth-credentials - key: client_secret - - name: OAUTH2_PROXY_COOKIE_SECRET - valueFrom: - secretKeyRef: - name: logviewer-oauth-credentials - key: cookie_secret - volumeMounts: - - name: combined-certs - mountPath: /etc/ssl/combined - readOnly: true - livenessProbe: - httpGet: - path: /ping - port: http - initialDelaySeconds: 10 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /ready - port: http - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 500m - memory: 256Mi - volumes: - - name: vault-ca-cert - secret: - secretName: vault-ca-cert - items: - - key: ca.crt - path: ca.crt - - name: combined-certs - emptyDir: {} - restartPolicy: Always diff --git a/apps/base/logging/logviewer/service.yaml b/apps/base/logging/logviewer/service.yaml deleted file mode 100644 index dfc72a4..0000000 --- a/apps/base/logging/logviewer/service.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: logviewer - namespace: logging -spec: - internalTrafficPolicy: Cluster - ports: - - name: http - port: 8080 - protocol: TCP - targetPort: http - selector: - app: logviewer - sessionAffinity: None - type: ClusterIP ---- -# Front-door entry Service: the HTTPRoute for logviewer.unkin.net targets this; -# all traffic enters via oauth2-proxy. -apiVersion: v1 -kind: Service -metadata: - name: logviewer-oauth2 - namespace: logging -spec: - internalTrafficPolicy: Cluster - ports: - - name: http - port: 80 - protocol: TCP - targetPort: http - selector: - app: logviewer-oauth2 - sessionAffinity: None - type: ClusterIP diff --git a/apps/base/logging/logviewer/vaultstaticsecret.yaml b/apps/base/logging/logviewer/vaultstaticsecret.yaml deleted file mode 100644 index 47ad5bb..0000000 --- a/apps/base/logging/logviewer/vaultstaticsecret.yaml +++ /dev/null @@ -1,21 +0,0 @@ ---- -# Authentik OIDC client for logviewer (client_id, client_secret, cookie_secret) -# seeded at kv/kubernetes/namespace/logging/default/oauth-credentials; the -# logging/default templated policy already grants read, so no terraform-vault -# change is needed. -apiVersion: secrets.hashicorp.com/v1beta1 -kind: VaultStaticSecret -metadata: - name: logviewer-oauth-credentials - namespace: logging -spec: - destination: - create: true - name: logviewer-oauth-credentials - overwrite: true - hmacSecretData: true - mount: kv - path: kubernetes/namespace/logging/default/oauth-credentials - refreshAfter: 5m - type: kv-v2 - vaultAuthRef: default diff --git a/apps/base/logging/nats-bootstrap-job.yaml b/apps/base/logging/nats-bootstrap-job.yaml deleted file mode 100644 index 296b1a8..0000000 --- a/apps/base/logging/nats-bootstrap-job.yaml +++ /dev/null @@ -1,158 +0,0 @@ ---- -# Declarative JetStream provisioning: the LOGS stream + durable consumers. -# ArgoCD PostSync hook, idempotent create-or-UPDATE, re-runs each sync. -# -# Stream LOGS: file storage, 3 replicas, retention=limits (NOT workqueue) so the -# transform tier AND the archiver each independently see every message — reading -# never deletes; only max-age/max-bytes do. S2 compression is on (logs compress -# well). Replay window = max-age (3d default). Beyond that, the S3 archive is the -# ONLY long-term source — everything else is gone after 3 days (accepted design). -# -# TUNABLE LIMITS LIVE IN A CONFIGMAP (nats-stream-limits): max_age, max_bytes, -# dupe_window. Change the ConfigMap and re-sync — this Job re-runs and applies -# the new limits via `nats stream edit` (no manual surgery). The ConfigMap is -# generated with a content-hash suffix (kustomize), so editing it changes both -# the ConfigMap name AND this Job's env reference → the PostSync hook Job's spec -# changes and Argo re-runs it (belt-and-suspenders on top of hooks running each -# sync; hook-delete-policy=BeforeHookCreation recreates it every time). -# -# Consumers (independent offsets = true fan-out): -# transform -> whole log stream, feeds the ClickHouse transform tier -# archiver -> configurable security-relevant subset, feeds the S3 archiver. -# Default filter is Vault audit (logs.k8s.vault.>); ADD subjects -# by editing ARCHIVE_SUBJECTS (space-separated -> repeated -# --filter). Exact default set is an open decision for Ben. -# -# Runbook (replay): -# (a) reprocess from JetStream (within max-age, 3d): scale the transform tier -# to 0, then `nats consumer rm LOGS transform` and re-run this Job -# (recreates at DeliverAll), or `nats consumer edit`/`--replay` from a -# start seq/time. -# (b) long-horizon (beyond JetStream): re-ingest S3 archive objects back -# through the transform tier (vector aws_s3 source or a one-shot Job). -apiVersion: batch/v1 -kind: Job -metadata: - name: nats-bootstrap - namespace: logging - annotations: - argocd.argoproj.io/hook: PostSync - argocd.argoproj.io/hook-delete-policy: BeforeHookCreation - labels: - app.kubernetes.io/name: nats-bootstrap - app.kubernetes.io/component: bootstrap -spec: - backoffLimit: 20 - activeDeadlineSeconds: 1800 - ttlSecondsAfterFinished: 3600 - template: - metadata: - labels: - app.kubernetes.io/name: nats-bootstrap - vector.dev/exclude: "true" - spec: - restartPolicy: OnFailure - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - containers: - - name: nats-bootstrap - image: docker.io/natsio/nats-box:0.18.0 - # nats CLI stats the working directory when loading its response - # schemas; under readOnlyRootFilesystem + runAsUser 1000 the image's - # default WORKDIR is not accessible ("stat .: permission denied"), so - # run from the writable /tmp emptyDir. - workingDir: /tmp - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - env: - - name: HOME - value: /tmp - - name: NATS_URL - value: "nats://nats.logging.svc.cluster.local:4222" - - name: NATS_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: nats-auth - key: admin_password - # Tunable stream limits — sourced from the ConfigMap. - - name: MAX_AGE - valueFrom: - configMapKeyRef: - name: nats-stream-limits - key: max_age - - name: MAX_BYTES - valueFrom: - configMapKeyRef: - name: nats-stream-limits - key: max_bytes - - name: DUPE_WINDOW - valueFrom: - configMapKeyRef: - name: nats-stream-limits - key: dupe_window - # Space-separated subject filters for the archiver consumer. - - name: ARCHIVE_SUBJECTS - value: "logs.k8s.vault.>" - command: - - /bin/sh - - -ec - - | - export NATS_USER=log-admin NATS_PASSWORD="$NATS_ADMIN_PASSWORD" - echo "Waiting for NATS + JetStream ..." - until nats --server "$NATS_URL" account info >/dev/null 2>&1; do - echo " not ready, retry in 5s"; sleep 5 - done - - echo "Ensuring stream LOGS (max_age=$MAX_AGE max_bytes=$MAX_BYTES dupe=$DUPE_WINDOW) ..." - # Create if absent; otherwise converge the mutable limits from the - # ConfigMap. (storage/retention/replicas are immutable, set only on - # create.) - nats stream add LOGS \ - --subjects='logs.>' --storage=file --replicas=3 \ - --retention=limits --discard=old --compression=s2 \ - --max-age="$MAX_AGE" --max-bytes="$MAX_BYTES" \ - --max-msgs=-1 --max-msgs-per-subject=-1 --max-msg-size=-1 \ - --max-consumers=-1 --dupe-window="$DUPE_WINDOW" --defaults 2>/dev/null \ - && echo " created" \ - || nats stream edit -f LOGS \ - --subjects='logs.>' --discard=old --compression=s2 \ - --max-age="$MAX_AGE" --max-bytes="$MAX_BYTES" \ - --max-msgs=-1 --max-msgs-per-subject=-1 --max-msg-size=-1 \ - --max-consumers=-1 --dupe-window="$DUPE_WINDOW" - - echo "Ensuring consumer transform ..." - nats consumer add LOGS transform \ - --pull --filter='logs.>' --deliver=all --ack=explicit \ - --max-deliver=-1 --replay=instant --defaults 2>/dev/null \ - || echo " transform already exists" - - echo "Ensuring consumer archiver (filters: $ARCHIVE_SUBJECTS) ..." - filter_args="" - for s in $ARCHIVE_SUBJECTS; do filter_args="$filter_args --filter=$s"; done - # shellcheck disable=SC2086 - nats consumer add LOGS archiver \ - --pull $filter_args --deliver=all --ack=explicit \ - --max-deliver=-1 --replay=instant --defaults 2>/dev/null \ - || echo " archiver already exists" - - echo "Done." - nats stream info LOGS - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 500m - memory: 256Mi - volumeMounts: - - name: tmp - mountPath: /tmp - volumes: - - name: tmp - emptyDir: {} diff --git a/apps/base/logging/serviceaccount_logarchiver.yaml b/apps/base/logging/serviceaccount_logarchiver.yaml deleted file mode 100644 index fb9823f..0000000 --- a/apps/base/logging/serviceaccount_logarchiver.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -# Dedicated SA for logarchiver's Vault k8s-auth login (role logging_logarchiver, -# terraform-vault). Only used to fetch the logarchive public key. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: logarchiver - namespace: logging -automountServiceAccountToken: false diff --git a/apps/base/logging/vaultstaticsecret.yaml b/apps/base/logging/vaultstaticsecret.yaml deleted file mode 100644 index 9113945..0000000 --- a/apps/base/logging/vaultstaticsecret.yaml +++ /dev/null @@ -1,73 +0,0 @@ ---- -# ClickHouse credentials for the `vector` user. -# -# Seed the Vault KV entry once (values are NOT stored in git), e.g.: -# PW=$(openssl rand -base64 24) -# HASH=$(printf '%s' "$PW" | sha256sum | cut -d' ' -f1) -# vault kv put kv/kubernetes/namespace/logging/default/clickhouse-credentials \ -# username=vector password="$PW" password_sha256_hex="$HASH" -# -# The `logging/default` ServiceAccount reads this path via the templated -# `policies/kv/kubernetes/default.yaml` policy (k8s auth role `default`), so no -# terraform-vault change is required — only the value above must be written. -apiVersion: secrets.hashicorp.com/v1beta1 -kind: VaultStaticSecret -metadata: - name: clickhouse-credentials - namespace: logging -spec: - destination: - create: true - name: clickhouse-credentials - overwrite: true - hmacSecretData: true - mount: kv - path: kubernetes/namespace/logging/default/clickhouse-credentials - refreshAfter: 5m - type: kv-v2 - vaultAuthRef: default ---- -# ClickHouse credentials for the read-only `logreader` user (CLI tools + -# logviewer UI). Seeded the same way as clickhouse-credentials above: -# PW=$(openssl rand -hex 24) -# HASH=$(printf '%s' "$PW" | sha256sum | cut -d' ' -f1) -# vault kv put kv/kubernetes/namespace/logging/default/clickhouse-logreader \ -# username=logreader password="$PW" password_sha256_hex="$HASH" -apiVersion: secrets.hashicorp.com/v1beta1 -kind: VaultStaticSecret -metadata: - name: clickhouse-logreader - namespace: logging -spec: - destination: - create: true - name: clickhouse-logreader - overwrite: true - hmacSecretData: true - mount: kv - path: kubernetes/namespace/logging/default/clickhouse-logreader - refreshAfter: 5m - type: kv-v2 - vaultAuthRef: default ---- -# NATS JetStream auth. Distinct passwords for the producer (edge), consumer -# (transform tier + archiver) and admin (bootstrap Job) users. Seed once: -# for k in admin producer consumer; do declare P_$k=$(openssl rand -base64 24); done -# vault kv put kv/kubernetes/namespace/logging/default/nats-auth \ -# admin_password="$P_admin" producer_password="$P_producer" consumer_password="$P_consumer" -apiVersion: secrets.hashicorp.com/v1beta1 -kind: VaultStaticSecret -metadata: - name: nats-auth - namespace: logging -spec: - destination: - create: true - name: nats-auth - overwrite: true - hmacSecretData: true - mount: kv - path: kubernetes/namespace/logging/default/nats-auth - refreshAfter: 5m - type: kv-v2 - vaultAuthRef: default diff --git a/apps/base/logging/vector/agent.yaml b/apps/base/logging/vector/agent.yaml deleted file mode 100644 index efbc35f..0000000 --- a/apps/base/logging/vector/agent.yaml +++ /dev/null @@ -1,43 +0,0 @@ ---- -# Vector EDGE agent pipeline (DaemonSet) — thin publisher, single source of -# truth. Mounted via existingConfigMaps (NOT the chart's customConfig, whose -# Helm `tpl` pass collides with Vector's own {{ }} / ${ } syntax). Tails all pod -# logs, attaches only routing tokens, publishes to JetStream. No parsing. -data_dir: /vector-data-dir - -api: - enabled: false - -sources: - kubernetes_logs: - type: kubernetes_logs - -transforms: - # Routing metadata only: NATS-subject-safe namespace + container tokens. - keymeta: - type: remap - inputs: - - kubernetes_logs - source: | - ns = to_string(.kubernetes.pod_namespace || "unknown") ?? "unknown" - .ns_token = replace(ns, r'[^a-zA-Z0-9_-]', "_") - cont = to_string(.kubernetes.container_name || "unknown") ?? "unknown" - .cont_token = replace(cont, r'[^a-zA-Z0-9_-]', "_") - -sinks: - to_jetstream: - type: nats - inputs: - - keymeta - url: nats://nats.logging.svc.cluster.local:4222 - connection_name: vector-agent - subject: "logs.k8s.{{ ns_token }}.{{ cont_token }}" - jetstream: - enabled: true - auth: - strategy: user_password - user_password: - user: log-producer - password: ${NATS_PRODUCER_PASSWORD} - encoding: - codec: json diff --git a/apps/base/logging/vector/aggregator-tests.yaml b/apps/base/logging/vector/aggregator-tests.yaml deleted file mode 100644 index deb2e9b..0000000 --- a/apps/base/logging/vector/aggregator-tests.yaml +++ /dev/null @@ -1,668 +0,0 @@ ---- -# `vector test` unit tests for the aggregator transforms. Merged with -# aggregator.yaml in CI (.woodpecker/vector-test.yaml). This is the pattern the -# per-app parsing follow-ups extend: add a test per new transform here. -tests: - - name: subject_routes_k8s_vs_vm - inputs: - - insert_at: route - type: log - log_fields: - subject: "logs.k8s.shop.web" - message: "routed" - outputs: - - extract_from: route.k8s - conditions: - - type: vrl - source: | - assert_eq!(.message, "routed") - - - name: k8s_log_is_normalised - inputs: - - insert_at: k8s_shape - type: log - log_fields: - message: "hello from pod" - stream: "stdout" - timestamp: "2026-07-27T00:00:00Z" - kubernetes.pod_name: "web-abc" - kubernetes.pod_namespace: "shop" - kubernetes.container_name: "web" - kubernetes.pod_node_name: "node-1" - outputs: - - extract_from: k8s_shape - conditions: - - type: vrl - source: | - assert_eq!(.source, "k8s") - assert_eq!(.namespace, "shop") - assert_eq!(.pod, "web-abc") - assert_eq!(.container, "web") - assert_eq!(.host, "node-1") - assert_eq!(.stream, "stdout") - assert_eq!(.message, "hello from pod") - - - name: vm_log_is_normalised - inputs: - - insert_at: vm_shape - type: log - log_fields: - message: "sshd started" - host: "vm-db-1" - severity: "info" - role: "database" - outputs: - - extract_from: vm_shape - conditions: - - type: vrl - source: | - assert_eq!(.source, "vm") - assert_eq!(.host, "vm-db-1") - assert_eq!(.severity, "info") - assert_eq!(.message, "sshd started") - assert_eq!(.labels.role, "database") - - # --- catch-all preservation: an unclaimed k8s event still flows app_route -> - # generic route -> k8s_shape (proves the two-stage chain keeps the fallback) --- - - name: unclaimed_k8s_falls_through_to_generic - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.k8s.shop.web" - message: "plain app log" - outputs: - - extract_from: route.k8s - conditions: - - type: vrl - source: | - assert_eq!(.message, "plain app log") - - # --- Tier-1: Authentik SSO (k8s, LIVE NOW) --- - - name: authentik_routes_by_subject - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.k8s.authentik.server" - message: "routed" - outputs: - - extract_from: app_route.authentik - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - - name: authentik_parse_extracts_event - inputs: - - insert_at: authentik_parse - type: log - log_fields: - subject: "logs.k8s.authentik.server" - stream: "stdout" - kubernetes.pod_namespace: "authentik" - kubernetes.container_name: "server" - kubernetes.pod_node_name: "node-2" - message: '{"event":"login","action":"login","user":"alice","client_ip":"203.0.113.9","result":"success","level":"info","logger":"authentik.events","timestamp":"2026-07-27T00:00:00Z"}' - outputs: - - extract_from: authentik_parse - conditions: - - type: vrl - source: | - assert_eq!(.source, "k8s") - assert_eq!(.namespace, "authentik") - assert_eq!(.container, "server") - assert_eq!(.severity, "info") - assert_eq!(.message, "login") - assert_eq!(.labels.app, "authentik") - assert_eq!(.fields.event, "login") - assert_eq!(.fields.action, "login") - assert_eq!(.fields.user, "alice") - assert_eq!(.fields.client_ip, "203.0.113.9") - assert_eq!(.fields.result, "success") - - # --- Tier-1: Traefik ingress (k8s, JSON access logs) --- - - name: traefik_routes_by_subject - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.k8s.traefik-system.traefik" - message: "routed" - outputs: - - extract_from: app_route.traefik - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - - name: traefik_parse_extracts_access_fields - inputs: - - insert_at: traefik_parse - type: log - log_fields: - subject: "logs.k8s.traefik-system.traefik" - kubernetes.pod_namespace: "traefik-system" - kubernetes.container_name: "traefik" - kubernetes.pod_node_name: "node-3" - message: '{"RouterName":"web@kubernetes","ServiceName":"shop-svc@kubernetes","RequestMethod":"GET","RequestPath":"/api","RequestHost":"shop.example.net","RequestProtocol":"HTTP/1.1","DownstreamStatus":200,"Duration":5000000,"ClientHost":"203.0.113.5","StartUTC":"2026-07-27T00:00:00Z"}' - outputs: - - extract_from: traefik_parse - conditions: - - type: vrl - source: | - assert_eq!(.source, "k8s") - assert_eq!(.namespace, "traefik-system") - assert_eq!(.labels.app, "traefik") - assert_eq!(.message, "GET /api 200") - assert_eq!(.fields.route, "web@kubernetes") - assert_eq!(.fields.service, "shop-svc@kubernetes") - assert_eq!(.fields.method, "GET") - assert_eq!(.fields.path, "/api") - assert_eq!(.fields.host, "shop.example.net") - assert_eq!(.fields.status, "200") - assert_eq!(.fields.duration_ms, "5") - assert_eq!(.fields.client_ip, "203.0.113.5") - - # --- Tier-1: Vault/OpenBao file audit (VM, awaiting VM vector) --- - - name: vault_routes_by_file - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.vm.vault1_syd1" - file: "/var/log/vault_audit.log" - message: "routed" - outputs: - - extract_from: app_route.vault - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - - name: vault_parse_extracts_request - inputs: - - insert_at: vault_parse - type: log - log_fields: - subject: "logs.vm.vault1_syd1" - host: "vault1" - file: "/var/log/vault_audit.log" - message: '{"time":"2026-07-27T00:00:00Z","type":"response","auth":{"display_name":"token"},"request":{"operation":"read","path":"secret/data/app","remote_address":"10.0.0.9"},"error":""}' - outputs: - - extract_from: vault_parse - conditions: - - type: vrl - source: | - assert_eq!(.source, "vm") - assert_eq!(.host, "vault1") - assert_eq!(.labels.app, "vault") - assert_eq!(.message, "read secret/data/app") - assert_eq!(.fields.type, "response") - assert_eq!(.fields.display_name, "token") - assert_eq!(.fields.operation, "read") - assert_eq!(.fields.path, "secret/data/app") - assert_eq!(.fields.remote_address, "10.0.0.9") - - # --- Tier-1: nginx access (VM, awaiting VM vector) --- - - name: nginx_access_routes_by_file - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.vm.web1_syd1" - file: "/var/log/nginx/shop_access.log" - message: "routed" - outputs: - - extract_from: app_route.nginx_access - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - - name: nginx_access_parse_extracts_combined - inputs: - - insert_at: nginx_access_parse - type: log - log_fields: - subject: "logs.vm.web1_syd1" - host: "web1" - file: "/var/log/nginx/shop_access.log" - message: '192.0.2.10 - - [27/Jul/2026:00:00:00 +0000] "GET /index.html HTTP/1.1" 200 1024 "https://ref.example/" "Mozilla/5.0" 0.012' - outputs: - - extract_from: nginx_access_parse - conditions: - - type: vrl - source: | - assert_eq!(.source, "vm") - assert_eq!(.stream, "access") - assert_eq!(.labels.log_type, "access") - assert_eq!(.fields.client_ip, "192.0.2.10") - assert_eq!(.fields.method, "GET") - assert_eq!(.fields.path, "/index.html") - assert_eq!(.fields.status, "200") - assert_eq!(.fields.bytes, "1024") - assert_eq!(.fields.referer, "https://ref.example/") - assert_eq!(.fields.user_agent, "Mozilla/5.0") - assert_eq!(.fields.request_time, "0.012") - - # --- Tier-1: nginx error (VM, awaiting VM vector) --- - - name: nginx_error_routes_by_file - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.vm.web1_syd1" - file: "/var/log/nginx/shop_error.log" - message: "routed" - outputs: - - extract_from: app_route.nginx_error - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - - name: nginx_error_parse_extracts_fields - inputs: - - insert_at: nginx_error_parse - type: log - log_fields: - subject: "logs.vm.web1_syd1" - host: "web1" - file: "/var/log/nginx/shop_error.log" - message: '2026/07/27 00:00:00 [error] 1234#0: *5 open() "/var/www/x" failed (2: No such file or directory), client: 192.0.2.20, server: shop, request: "GET / HTTP/1.1", host: "shop"' - outputs: - - extract_from: nginx_error_parse - conditions: - - type: vrl - source: | - assert_eq!(.stream, "error") - assert_eq!(.severity, "error") - assert_eq!(.labels.log_type, "error") - assert_eq!(.fields.level, "error") - assert_eq!(.fields.pid, "1234") - assert_eq!(.fields.cid, "5") - assert_eq!(.fields.client_ip, "192.0.2.20") - - # --- Tier-1: HAProxy httplog (VM journald, awaiting VM vector) --- - - name: haproxy_routes_by_identifier - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.vm.halb1_syd1" - SYSLOG_IDENTIFIER: "haproxy" - message: "routed" - outputs: - - extract_from: app_route.haproxy - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - - name: haproxy_parse_extracts_timers - inputs: - - insert_at: haproxy_parse - type: log - log_fields: - subject: "logs.vm.halb1_syd1" - host: "halb1" - SYSLOG_IDENTIFIER: "haproxy" - message: '192.0.2.30:54321 [27/Jul/2026:00:00:00.123] fe_http be_app/app1 10/0/1/2/13 200 512 - - ---- 5/4/3/2/0 0/0 "GET /health HTTP/1.1"' - outputs: - - extract_from: haproxy_parse - conditions: - - type: vrl - source: | - assert_eq!(.source, "vm") - assert_eq!(.labels.app, "haproxy") - assert_eq!(.fields.client_ip, "192.0.2.30") - assert_eq!(.fields.frontend, "fe_http") - assert_eq!(.fields.backend, "be_app") - assert_eq!(.fields.server, "app1") - assert_eq!(.fields.tq, "10") - assert_eq!(.fields.tw, "0") - assert_eq!(.fields.tc, "1") - assert_eq!(.fields.tr, "2") - assert_eq!(.fields.tt, "13") - assert_eq!(.fields.termination_state, "----") - assert_eq!(.fields.retries, "0") - assert_eq!(.fields.status, "200") - assert_eq!(.fields.bytes, "512") - - # --- Tier-1: glauth LDAP (VM, awaiting VM vector) --- - - name: glauth_routes_by_identifier - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.vm.ldap1_syd1" - SYSLOG_IDENTIFIER: "glauth" - message: "routed" - outputs: - - extract_from: app_route.glauth - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - - name: glauth_parse_extracts_bind - inputs: - - insert_at: glauth_parse - type: log - log_fields: - subject: "logs.vm.ldap1_syd1" - host: "ldap1" - SYSLOG_IDENTIFIER: "glauth" - message: '{"level":"info","msg":"Bind success as user","bindDN":"cn=admin,dc=example,dc=com","src":"192.0.2.40:1234","time":"2026-07-27T00:00:00Z"}' - outputs: - - extract_from: glauth_parse - conditions: - - type: vrl - source: | - assert_eq!(.source, "vm") - assert_eq!(.host, "ldap1") - assert_eq!(.severity, "info") - assert_eq!(.labels.app, "glauth") - assert_eq!(.fields.bindDN, "cn=admin,dc=example,dc=com") - assert_eq!(.fields.remote, "192.0.2.40:1234") - assert_eq!(.fields.success, "true") - - # ================= Tier-2 (stacks on #318) ================= - - # --- BIND query logs (k8s bind-* + VM named) --- - - name: bind_routes_k8s_by_subject - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.k8s.bind-internal.named" - message: "routed" - outputs: - - extract_from: app_route.bind_query - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - - name: bind_routes_vm_by_identifier - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.vm.dns1_syd1" - SYSLOG_IDENTIFIER: "named" - message: "routed" - outputs: - - extract_from: app_route.bind_query - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - - name: bind_parse_extracts_query - inputs: - - insert_at: bind_query_parse - type: log - log_fields: - subject: "logs.k8s.bind-internal.named" - kubernetes.pod_namespace: "bind-internal" - kubernetes.container_name: "named" - kubernetes.pod_node_name: "node-4" - message: '02-Aug-2026 00:00:00.123 client @0x7f 192.0.2.1#40426 (www.example.com): view internal: query: www.example.com IN A +E(0)K (198.18.200.7)' - outputs: - - extract_from: bind_query_parse - conditions: - - type: vrl - source: | - assert_eq!(.source, "k8s") - assert_eq!(.namespace, "bind-internal") - assert_eq!(.labels.app, "bind") - assert_eq!(.message, "query www.example.com A") - assert_eq!(.fields.client_ip, "192.0.2.1") - assert_eq!(.fields.qname, "www.example.com") - assert_eq!(.fields.qclass, "IN") - assert_eq!(.fields.qtype, "A") - assert_eq!(.fields.view, "internal") - - # --- Rancher audit (k8s, cattle-system sidecar) --- - - name: rancher_routes_by_subject - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.k8s.cattle-system.rancher-audit-log" - message: "routed" - outputs: - - extract_from: app_route.rancher_audit - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - - name: rancher_parse_extracts_audit - inputs: - - insert_at: rancher_audit_parse - type: log - log_fields: - subject: "logs.k8s.cattle-system.rancher-audit-log" - kubernetes.pod_namespace: "cattle-system" - kubernetes.container_name: "rancher-audit-log" - kubernetes.pod_node_name: "node-5" - message: '{"auditID":"abc-123","requestURI":"/v3/tokens","user":{"name":"u-alice","group":["admins"]},"method":"GET","remoteAddr":"10.42.0.9:1234","responseCode":200,"requestTimestamp":"2026-08-01T00:00:00Z"}' - outputs: - - extract_from: rancher_audit_parse - conditions: - - type: vrl - source: | - assert_eq!(.source, "k8s") - assert_eq!(.namespace, "cattle-system") - assert_eq!(.labels.app, "rancher") - assert_eq!(.labels.log_type, "audit") - assert_eq!(.message, "GET /v3/tokens 200") - assert_eq!(.fields.user, "u-alice") - assert_eq!(.fields.verb, "GET") - assert_eq!(.fields.uri, "/v3/tokens") - assert_eq!(.fields.status, "200") - - # --- CNPG Postgres (ONE transform, all clusters) --- - - name: cnpg_routes_by_postgres_container - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.k8s.litellm.postgres" - message: "routed" - outputs: - - extract_from: app_route.cnpg_pg - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - # mutual exclusivity: an app-namespace CNPG pod (authentik) is claimed by - # cnpg_pg, NOT the authentik app route (which now carves out .postgres). - - name: cnpg_authentik_postgres_routes_to_cnpg - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.k8s.authentik.postgres" - message: "routed" - outputs: - - extract_from: app_route.cnpg_pg - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - - name: cnpg_parse_extracts_record - inputs: - - insert_at: cnpg_pg_parse - type: log - log_fields: - subject: "logs.k8s.litellm.postgres" - kubernetes.pod_namespace: "litellm" - kubernetes.container_name: "postgres" - kubernetes.pod_node_name: "node-6" - kubernetes.pod_labels."cnpg.io/cluster": "litellm-postgres" - message: '{"level":"info","ts":"2026-08-01T00:00:00Z","logger":"postgres","msg":"record","record":{"user_name":"litellm","database_name":"litellm","error_severity":"LOG","message":"duration: 12.345 ms statement: SELECT 1","query":""}}' - outputs: - - extract_from: cnpg_pg_parse - conditions: - - type: vrl - source: | - assert_eq!(.source, "k8s") - assert_eq!(.namespace, "litellm") - assert_eq!(.severity, "LOG") - assert_eq!(.labels.app, "cnpg") - assert_eq!(.labels.cluster, "litellm-postgres") - assert_eq!(.fields.error_severity, "LOG") - assert_eq!(.fields.duration_ms, "12.345") - assert_eq!(.fields.user, "litellm") - assert_eq!(.fields.database, "litellm") - - # --- Gitea router/access (k8s + VM) --- - - name: gitea_routes_k8s_by_subject - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.k8s.gitea.gitea" - message: "routed" - outputs: - - extract_from: app_route.gitea - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - - name: gitea_parse_router_line - inputs: - - insert_at: gitea_parse - type: log - log_fields: - subject: "logs.k8s.gitea.gitea" - kubernetes.pod_namespace: "gitea" - kubernetes.container_name: "gitea" - kubernetes.pod_node_name: "node-7" - message: '2026/08/01 00:00:00 .../router.go:100:func() [I] router: completed GET /user/login for 10.0.0.1:0, 200 OK in 12.3ms @ web/base.go:1' - outputs: - - extract_from: gitea_parse - conditions: - - type: vrl - source: | - assert_eq!(.source, "k8s") - assert_eq!(.namespace, "gitea") - assert_eq!(.labels.app, "gitea") - assert_eq!(.message, "GET /user/login 200") - assert_eq!(.fields.method, "GET") - assert_eq!(.fields.path, "/user/login") - assert_eq!(.fields.status, "200") - assert_eq!(.fields.latency, "12.3ms") - - name: gitea_parse_access_line - inputs: - - insert_at: gitea_parse - type: log - log_fields: - subject: "logs.k8s.gitea.gitea" - kubernetes.pod_namespace: "gitea" - kubernetes.container_name: "gitea" - message: '10.0.0.5 - alice [01/Aug/2026:00:00:00 +0000] "POST /repo/foo HTTP/1.1" 201 512 "-" "git/2.0"' - outputs: - - extract_from: gitea_parse - conditions: - - type: vrl - source: | - assert_eq!(.fields.method, "POST") - assert_eq!(.fields.path, "/repo/foo") - assert_eq!(.fields.status, "201") - assert_eq!(.fields.user, "alice") - assert_eq!(.fields.client_ip, "10.0.0.5") - - # --- PuppetServer / PuppetDB (k8s stdout) --- - - name: puppet_routes_by_subject - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.k8s.puppet.puppetserver" - message: "routed" - outputs: - - extract_from: app_route.puppet - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - - name: puppet_parse_logback_line - inputs: - - insert_at: puppet_parse - type: log - log_fields: - subject: "logs.k8s.puppet.puppetserver" - kubernetes.pod_namespace: "puppet" - kubernetes.container_name: "puppetserver" - kubernetes.pod_node_name: "node-8" - message: '2026-08-01 00:00:00,123 INFO [qtp123-45] [puppetserver] Compiled catalog for web01.unkin.net in environment production in 1.23 seconds' - outputs: - - extract_from: puppet_parse - conditions: - - type: vrl - source: | - assert_eq!(.source, "k8s") - assert_eq!(.namespace, "puppet") - assert_eq!(.severity, "INFO") - assert_eq!(.labels.app, "puppet") - assert_eq!(.fields.level, "INFO") - assert_eq!(.fields.logger, "puppetserver") - assert_eq!(.fields.node, "web01.unkin.net") - - # --- LiteLLM request logs (k8s JSON) --- - - name: litellm_routes_by_subject - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.k8s.litellm.litellm" - message: "routed" - outputs: - - extract_from: app_route.litellm - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - - name: litellm_parse_extracts_request - inputs: - - insert_at: litellm_parse - type: log - log_fields: - subject: "logs.k8s.litellm.litellm" - kubernetes.pod_namespace: "litellm" - kubernetes.container_name: "litellm" - kubernetes.pod_node_name: "node-9" - message: '{"message":"Request completed","level":"info","model":"gpt-4o","total_tokens":1234,"response_time":0.532,"api_key":"sk-abc","status":"success","timestamp":"2026-08-01T00:00:00Z"}' - outputs: - - extract_from: litellm_parse - conditions: - - type: vrl - source: | - assert_eq!(.source, "k8s") - assert_eq!(.namespace, "litellm") - assert_eq!(.severity, "info") - assert_eq!(.message, "Request completed") - assert_eq!(.labels.app, "litellm") - assert_eq!(.fields.model, "gpt-4o") - assert_eq!(.fields.tokens, "1234") - assert_eq!(.fields.latency, "0.532") - assert_eq!(.fields.key, "sk-abc") - assert_eq!(.fields.status, "success") - - # --- Postfix maillog (VM, per-line best-effort) --- - - name: postfix_routes_by_identifier - inputs: - - insert_at: app_route - type: log - log_fields: - subject: "logs.vm.mail1_syd1" - SYSLOG_IDENTIFIER: "postfix/qmgr" - message: "routed" - outputs: - - extract_from: app_route.postfix - conditions: - - type: vrl - source: 'assert_eq!(.message, "routed")' - - name: postfix_parse_extracts_line - inputs: - - insert_at: postfix_parse - type: log - log_fields: - subject: "logs.vm.mail1_syd1" - host: "mail1" - SYSLOG_IDENTIFIER: "postfix/smtp" - message: 'ABC123DEF: to=, relay=mx.example.com[1.2.3.4]:25, delay=1.2, delays=0.1/0/0.5/0.6, dsn=2.0.0, status=sent (250 OK)' - outputs: - - extract_from: postfix_parse - conditions: - - type: vrl - source: | - assert_eq!(.source, "vm") - assert_eq!(.host, "mail1") - assert_eq!(.labels.app, "postfix") - assert_eq!(.fields.qid, "ABC123DEF") - assert_eq!(.fields.to, "rcpt@example.com") - assert_eq!(.fields.relay, "mx.example.com[1.2.3.4]:25") - assert_eq!(.fields.delay, "1.2") - assert_eq!(.fields.status, "sent") - assert_eq!(.fields.program, "postfix/smtp") diff --git a/apps/base/logging/vector/aggregator.yaml b/apps/base/logging/vector/aggregator.yaml deleted file mode 100644 index 4e445a3..0000000 --- a/apps/base/logging/vector/aggregator.yaml +++ /dev/null @@ -1,900 +0,0 @@ ---- -# Vector TRANSFORM tier (the "brain") — single source of truth, also validated -# by `vector test` in CI. Consumes the whole log stream from JetStream via the -# durable `transform` consumer (at-least-once; durable offsets tracked by -# JetStream), routes by subject, normalises into the logs.raw columns, and is -# the ONLY ClickHouse writer. -# -# Routing model (two stages): -# 1. app_route — peels off Tier-1 per-app streams by subject / source tag and -# hands each to a dedicated parse transform that emits the full logs.raw -# shape plus structured .fields. Conditions are MUTUALLY EXCLUSIVE, so an -# event is claimed by at most one app (no double-insert). -# 2. route (generic catch-all) — everything app_route did NOT claim -# (app_route._unmatched) is split k8s/vm and shaped generically. This is the -# fallback for all un-parsed traffic and MUST stay intact. -# Add a new per-app pipeline by appending a mutually-exclusive route to -# app_route, a parse transform, and its id to the clickhouse sink `inputs`. -# -# Structured fields go into the logs.raw `fields Map(String,String)` column — no -# DDL change is needed (values are stringified; empties are compacted away). -# -# VM source-tag convention (the puppet-side vector rollout MUST follow it so -# these transforms light up): file sources set `.file` (absolute log path); -# journald sources set `.SYSLOG_IDENTIFIER` (falls back to `.program`/`.appname`). -# -# Durability model: JetStream (3d / 130 GiB, S2-compressed) is the SOLE -# durability layer and the replay window. This -# tier is stateless (no PVC, memory buffer). If ClickHouse is down the sink -# blocks (when_full=block); back-pressure stops the source pulling, so unpulled -# messages stay in JetStream and are redelivered. NB: Vector's NATS source has -# no end-to-end acks (acks on receipt), so a pod killed mid-outage can lose the -# in-memory buffer's worth of already-pulled events — accepted for a stateless, -# autoscalable tier. -data_dir: /vector-data-dir - -api: - enabled: true - address: 0.0.0.0:8686 - -sources: - js_in: - type: nats - url: nats://nats.logging.svc.cluster.local:4222 - connection_name: vector-transform - subject: "logs.>" - jetstream: - stream: LOGS - consumer: transform - auth: - strategy: user_password - user_password: - user: log-consumer - password: ${NATS_CONSUMER_PASSWORD} - decoding: - codec: json - -transforms: - # Stage 1: peel off Tier-1 per-app streams. Mutually exclusive conditions; - # anything unmatched falls through to the generic `route` below. - app_route: - type: route - inputs: - - js_in - route: - # k8s: authentik SSO — structlog JSON on stdout. The `.postgres` container - # is the authentik-namespace CNPG cluster; carve it out so it is claimed by - # the single `cnpg_pg` route below (keeps app_route mutually exclusive). - authentik: 'starts_with(to_string(.subject) ?? "", "logs.k8s.authentik.") && !ends_with(to_string(.subject) ?? "", ".postgres")' - # k8s: Traefik ingress — JSON access logs (requires logs.access.format=json, - # flipped in the traefik-system overlay values in this same change). - traefik: 'starts_with(to_string(.subject) ?? "", "logs.k8s.traefik-system.")' - # VM: Vault/OpenBao file audit device (/var/log/vault_audit.log), JSON. - vault: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && contains(to_string(.file) ?? "", "vault_audit")' - # VM: nginx combined access log (/var/log/nginx/_access.log). - nginx_access: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && contains(to_string(.file) ?? "", "nginx") && ends_with(to_string(.file) ?? "", "access.log")' - # VM: nginx error log (/var/log/nginx/_error.log). - nginx_error: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && contains(to_string(.file) ?? "", "nginx") && ends_with(to_string(.file) ?? "", "error.log")' - # VM: HAProxy httplog via journald. - haproxy: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && ((to_string(.SYSLOG_IDENTIFIER) ?? "") == "haproxy" || (to_string(.program) ?? "") == "haproxy" || (to_string(.appname) ?? "") == "haproxy")' - # VM: glauth LDAP — structuredlog (logrus) JSON. - glauth: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && (contains(to_string(.file) ?? "", "glauth") || (to_string(.SYSLOG_IDENTIFIER) ?? "") == "glauth" || (to_string(.program) ?? "") == "glauth" || (to_string(.appname) ?? "") == "glauth")' - - # --- Tier-2 (stacks on #318) --- - # BIND query logs, k8s + VM. k8s: any bind-* namespace (bind-internal DNS - # servers, bind-system operator) — query logging enabled via `querylog yes` - # in the BindCluster extraOptions in this change. VM: puppet-managed named - # (file /var/log/named/*.log or journald `named`) — puppet-side enable is a - # required follow-up (profiles/dns/server.pp). - bind_query: 'starts_with(to_string(.subject) ?? "", "logs.k8s.bind") || (starts_with(to_string(.subject) ?? "", "logs.vm.") && (contains(to_string(.file) ?? "", "named") || (to_string(.SYSLOG_IDENTIFIER) ?? "") == "named" || (to_string(.program) ?? "") == "named" || (to_string(.appname) ?? "") == "named"))' - # k8s: Rancher audit log — JSON, emitted by the `rancher-audit-log` sidecar - # (auditLog.enabled level 1, already on in the cattle-system overlay). - rancher_audit: 'starts_with(to_string(.subject) ?? "", "logs.k8s.cattle-system.rancher-audit-log")' - # k8s: CNPG Postgres — ONE route for ALL clusters. The CNPG main container is - # always named `postgres`, so logs.k8s..postgres uniquely identifies every - # cluster across all namespaces (authentik/litellm/artifactapi/woodpecker/ - # puppet/paperclip/grafana/netbox/gitea/encapi). Mutually exclusive because the - # app routes above/below carve out `.postgres`. - cnpg_pg: 'starts_with(to_string(.subject) ?? "", "logs.k8s.") && ends_with(to_string(.subject) ?? "", ".postgres")' - # Gitea router/access logs. k8s: the new k8s gitea (ns gitea) with router + - # access logging enabled in the overlay values in this change — carve out - # `.postgres` (gitea-namespace CNPG). VM: puppet-managed gitea (file or - # journald `gitea`) — puppet-side log-format enable is a follow-up. - gitea: '(starts_with(to_string(.subject) ?? "", "logs.k8s.gitea.") && !ends_with(to_string(.subject) ?? "", ".postgres")) || (starts_with(to_string(.subject) ?? "", "logs.vm.") && (contains(to_string(.file) ?? "", "gitea") || (to_string(.SYSLOG_IDENTIFIER) ?? "") == "gitea" || (to_string(.program) ?? "") == "gitea" || (to_string(.appname) ?? "") == "gitea"))' - # PuppetServer / PuppetDB. k8s: openvoxserver/openvoxdb stdout (ns puppet) — - # carve out `.postgres` (puppet-namespace CNPG). VM file logs (multiline - # logback + puppetserver-access.log) are a puppet-side vector concern (the - # multiline join must happen at the edge) — follow-up. - puppet: 'starts_with(to_string(.subject) ?? "", "logs.k8s.puppet.") && !ends_with(to_string(.subject) ?? "", ".postgres")' - # k8s: LiteLLM request logs — JSON once JSON_LOGS=True (flipped in the litellm - # env in this change). Carve out `.postgres` (litellm-namespace CNPG). - litellm: 'starts_with(to_string(.subject) ?? "", "logs.k8s.litellm.") && !ends_with(to_string(.subject) ?? "", ".postgres")' - # VM: Postfix maillog — journald (SYSLOG_IDENTIFIER postfix/*) or file maillog. - postfix: 'starts_with(to_string(.subject) ?? "", "logs.vm.") && (starts_with(to_string(.SYSLOG_IDENTIFIER) ?? "", "postfix") || starts_with(to_string(.program) ?? "", "postfix") || starts_with(to_string(.appname) ?? "", "postfix") || contains(to_string(.file) ?? "", "maillog"))' - - # Stage 2: generic catch-all for everything app_route did not claim. - route: - type: route - inputs: - - app_route._unmatched - route: - k8s: 'starts_with(to_string(.subject) ?? "", "logs.k8s.")' - vm: 'starts_with(to_string(.subject) ?? "", "logs.vm.")' - - k8s_shape: - type: remap - inputs: - - route.k8s - source: | - ts = .timestamp || now() - node = to_string(.kubernetes.pod_node_name || "") ?? "" - ns = to_string(.kubernetes.pod_namespace || "") ?? "" - pod = to_string(.kubernetes.pod_name || "") ?? "" - container = to_string(.kubernetes.container_name || "") ?? "" - strm = to_string(.stream || "") ?? "" - msg = to_string(.message || "") ?? "" - lbls = object(.kubernetes.pod_labels) ?? {} - . = { - "timestamp": ts, - "host": node, - "source": "k8s", - "namespace": ns, - "pod": pod, - "container": container, - "stream": strm, - "severity": "", - "message": msg, - "labels": lbls, - "fields": {} - } - - vm_shape: - type: remap - inputs: - - route.vm - source: | - ts = .timestamp || .ts || now() - host = to_string(.host || .hostname || "") ?? "" - msg = to_string(.message || .msg || "") ?? "" - sev = to_string(.severity || .level || "") ?? "" - role = to_string(.role || "") ?? "" - lbls = {} - if role != "" { - lbls = {"role": role} - } - . = { - "timestamp": ts, - "host": host, - "source": "vm", - "namespace": "", - "pod": "", - "container": "", - "stream": "", - "severity": sev, - "message": msg, - "labels": lbls, - "fields": {} - } - - # --- Tier-1 per-app parse transforms (each emits the full logs.raw shape) --- - - # Authentik SSO (k8s, ns authentik) — structlog JSON on stdout. - # LIVE NOW: authentik pods already stream to logs.k8s.authentik.*. - authentik_parse: - type: remap - inputs: - - app_route.authentik - source: | - node = to_string(.kubernetes.pod_node_name || "") ?? "" - pod = to_string(.kubernetes.pod_name || "") ?? "" - container = to_string(.kubernetes.container_name || "") ?? "" - strm = to_string(.stream || "") ?? "" - raw = to_string(.message || "") ?? "" - ev = object(parse_json(raw) ?? {}) ?? {} - ts = ev.timestamp || .timestamp || now() - user = "" - if is_string(ev.user) { - user = to_string(ev.user) ?? "" - } else if is_object(ev.user) { - user = to_string(ev.user.username) ?? "" - } - fields = compact({ - "event": to_string(ev.event) ?? "", - "action": to_string(ev.action) ?? "", - "user": user, - "client_ip": to_string(ev.client_ip) ?? "", - "result": to_string(ev.result) ?? "", - "logger": to_string(ev.logger) ?? "" - }, string: true) - sev = to_string(ev.level) ?? "" - msg = raw - if ev.event != null { - msg = to_string(ev.event) ?? raw - } - . = { - "timestamp": ts, - "host": node, - "source": "k8s", - "namespace": "authentik", - "pod": pod, - "container": container, - "stream": strm, - "severity": sev, - "message": msg, - "labels": {"app": "authentik"}, - "fields": fields - } - - # Traefik ingress (k8s, ns traefik-system) — JSON access logs. Non-access - # traefik lines (app logs) simply parse to no access fields and keep .message. - # geoip on client_ip is a PREREQUISITE (no enrichment table yet — see PR note). - traefik_parse: - type: remap - inputs: - - app_route.traefik - source: | - node = to_string(.kubernetes.pod_node_name || "") ?? "" - pod = to_string(.kubernetes.pod_name || "") ?? "" - container = to_string(.kubernetes.container_name || "") ?? "" - strm = to_string(.stream || "") ?? "" - raw = to_string(.message || "") ?? "" - ev = object(parse_json(raw) ?? {}) ?? {} - ts = ev.StartUTC || ev.time || .timestamp || now() - status = "" - if ev.DownstreamStatus != null { - status = to_string(ev.DownstreamStatus) ?? "" - } - dur_ns = to_int(ev.Duration) ?? 0 - dur_ms = "" - if dur_ns > 0 { - dur_ms = to_string(dur_ns / 1000000) - } - method = to_string(ev.RequestMethod) ?? "" - path = to_string(ev.RequestPath) ?? "" - fields = compact({ - "route": to_string(ev.RouterName) ?? "", - "service": to_string(ev.ServiceName) ?? "", - "method": method, - "path": path, - "host": to_string(ev.RequestHost) ?? "", - "status": status, - "duration_ms": dur_ms, - "client_ip": to_string(ev.ClientHost) ?? "", - "protocol": to_string(ev.RequestProtocol) ?? "" - }, string: true) - msg = raw - if method != "" { - msg = method + " " + path + " " + status - } - . = { - "timestamp": ts, - "host": node, - "source": "k8s", - "namespace": "traefik-system", - "pod": pod, - "container": container, - "stream": strm, - "severity": "", - "message": msg, - "labels": {"app": "traefik"}, - "fields": fields - } - - # Vault/OpenBao file audit device (VM, /var/log/vault_audit.log) — JSON, one - # object per request/response. AWAITING VM VECTOR (in-cluster vault is quiet; - # lights up when the puppet vector rollout ships logs.vm.* with .file set). - vault_parse: - type: remap - inputs: - - app_route.vault - source: | - host = to_string(.host || .hostname || "") ?? "" - raw = to_string(.message || .msg || "") ?? "" - ev = object(parse_json(raw) ?? {}) ?? {} - ts = ev.time || .timestamp || .ts || now() - auth = object(ev.auth) ?? {} - req = object(ev.request) ?? {} - fields = compact({ - "type": to_string(ev.type) ?? "", - "display_name": to_string(auth.display_name) ?? "", - "operation": to_string(req.operation) ?? "", - "path": to_string(req.path) ?? "", - "remote_address": to_string(req.remote_address) ?? "", - "error": to_string(ev.error) ?? "" - }, string: true) - op = to_string(req.operation) ?? "" - pth = to_string(req.path) ?? "" - msg = raw - if op != "" || pth != "" { - msg = op + " " + pth - } - . = { - "timestamp": ts, - "host": host, - "source": "vm", - "namespace": "", - "pod": "", - "container": "", - "stream": "", - "severity": "", - "message": msg, - "labels": {"app": "vault"}, - "fields": fields - } - - # nginx access log (VM) — combined/CLF + optional trailing request_time. - # AWAITING VM VECTOR. geoip on client_ip is a PREREQUISITE (see PR note). - nginx_access_parse: - type: remap - inputs: - - app_route.nginx_access - source: | - host = to_string(.host || .hostname || "") ?? "" - raw = to_string(.message || .msg || "") ?? "" - ts = .timestamp || .ts || now() - m = parse_regex(raw, r'^(?P\S+) \S+ (?P\S+) \[(?P[^\]]+)\] "(?P\S+) (?P\S+) (?P[^"]*)" (?P\d{3}) (?P\d+|-) "(?P[^"]*)" "(?P[^"]*)"(?: (?P[\d.]+))?') ?? {} - fields = compact({ - "client_ip": to_string(m.client_ip), - "method": to_string(m.method), - "path": to_string(m.path), - "status": to_string(m.status), - "bytes": to_string(m.bytes), - "referer": to_string(m.referer), - "user_agent": to_string(m.user_agent), - "request_time": to_string(m.request_time) - }, string: true) - . = { - "timestamp": ts, - "host": host, - "source": "vm", - "namespace": "", - "pod": "", - "container": "", - "stream": "access", - "severity": "", - "message": raw, - "labels": {"app": "nginx", "log_type": "access"}, - "fields": fields - } - - # nginx error log (VM). AWAITING VM VECTOR. - nginx_error_parse: - type: remap - inputs: - - app_route.nginx_error - source: | - host = to_string(.host || .hostname || "") ?? "" - raw = to_string(.message || .msg || "") ?? "" - ts = .timestamp || .ts || now() - m = parse_regex(raw, r'^(?P\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?P\w+)\] (?P\d+)#(?P\d+): (?:\*(?P\d+) )?(?P.*)$') ?? {} - c = parse_regex(raw, r'client: (?P[0-9a-fA-F:.]+)') ?? {} - lvl = to_string(m.level) - err = to_string(m.err) - fields = compact({ - "level": lvl, - "pid": to_string(m.pid), - "cid": to_string(m.cid), - "client_ip": to_string(c.client_ip), - "error": err - }, string: true) - msg = raw - if err != "" { - msg = err - } - . = { - "timestamp": ts, - "host": host, - "source": "vm", - "namespace": "", - "pod": "", - "container": "", - "stream": "error", - "severity": lvl, - "message": msg, - "labels": {"app": "nginx", "log_type": "error"}, - "fields": fields - } - - # HAProxy httplog (VM, journald). AWAITING VM VECTOR. - # httplog: %ci:%cp [%tr] %ft %b/%s %Tq/%Tw/%Tc/%Tr/%Tt %ST %B %CC %CS %tsc - # %ac/%fc/%bc/%sc/%rc %sq/%bq {hdrs} "%r" - haproxy_parse: - type: remap - inputs: - - app_route.haproxy - source: | - host = to_string(.host || .hostname || "") ?? "" - raw = to_string(.message || .msg || "") ?? "" - ts = .timestamp || .ts || now() - m = parse_regex(raw, r'(?P\d{1,3}(?:\.\d{1,3}){3}):(?P\d+) \[(?P[^\]]+)\] (?P\S+) (?P[^/ ]+)/(?P\S+) (?P-?\d+)/(?P-?\d+)/(?P-?\d+)/(?P-?\d+)/(?P[+-]?\d+) (?P\d{3}) (?P\d+) \S+ \S+ (?P\S{4}) (?P\d+)/(?P\d+)/(?P\d+)/(?P\d+)/(?P\d+) (?P\d+)/(?P\d+)') ?? {} - fields = compact({ - "client_ip": to_string(m.client_ip), - "frontend": to_string(m.frontend), - "backend": to_string(m.backend), - "server": to_string(m.server), - "tq": to_string(m.tq), - "tw": to_string(m.tw), - "tc": to_string(m.tc), - "tr": to_string(m.tr), - "tt": to_string(m.tt), - "termination_state": to_string(m.termination_state), - "retries": to_string(m.retries), - "status": to_string(m.status), - "bytes": to_string(m.bytes) - }, string: true) - . = { - "timestamp": ts, - "host": host, - "source": "vm", - "namespace": "", - "pod": "", - "container": "", - "stream": "", - "severity": "", - "message": raw, - "labels": {"app": "haproxy"}, - "fields": fields - } - - # glauth LDAP (VM) — structuredlog (logrus) JSON. AWAITING VM VECTOR. - glauth_parse: - type: remap - inputs: - - app_route.glauth - source: | - host = to_string(.host || .hostname || "") ?? "" - raw = to_string(.message || .msg || "") ?? "" - ev = object(parse_json(raw) ?? {}) ?? {} - ts = ev.time || .timestamp || .ts || now() - binddn = to_string(ev.bindDN) ?? "" - if binddn == "" { - binddn = to_string(ev.binddn) ?? "" - } - remote = to_string(ev.src) ?? "" - if remote == "" { - remote = to_string(ev.remoteAddr) ?? "" - } - lvl = to_string(ev.level) ?? "" - gmsg = to_string(ev.msg) ?? "" - success = "false" - if contains(downcase(gmsg), "success") || (lvl == "info" && contains(downcase(gmsg), "bind")) { - success = "true" - } - fields = compact({ - "bindDN": binddn, - "remote": remote, - "success": success, - "level": lvl, - "msg": gmsg - }, string: true) - msg = gmsg - if msg == "" { - msg = raw - } - . = { - "timestamp": ts, - "host": host, - "source": "vm", - "namespace": "", - "pod": "", - "container": "", - "stream": "", - "severity": lvl, - "message": msg, - "labels": {"app": "glauth"}, - "fields": fields - } - - # --- Tier-2 per-app parse transforms (stacks on #318) --- - - # BIND query logs (k8s bind-* namespaces + VM named). LIVE on k8s once the - # `querylog yes` extraOptions (this change) roll out; VM AWAITS the puppet-side - # enable (profiles/dns/server.pp). rcode is NOT present in standard query-log - # lines (that needs response logging / dnstap) — extracted only if a - # response-style `status:` line is seen. Non-query lines keep .message. - bind_query_parse: - type: remap - inputs: - - app_route.bind_query - source: | - subj = to_string(.subject) ?? "" - is_k8s = starts_with(subj, "logs.k8s.") - raw = to_string(.message || .msg || "") ?? "" - ts = .timestamp || .ts || now() - node = "" - ns = "" - pod = "" - container = "" - strm = "" - hostv = "" - src = "vm" - if is_k8s { - src = "k8s" - node = to_string(.kubernetes.pod_node_name || "") ?? "" - ns = to_string(.kubernetes.pod_namespace || "") ?? "" - pod = to_string(.kubernetes.pod_name || "") ?? "" - container = to_string(.kubernetes.container_name || "") ?? "" - strm = to_string(.stream || "") ?? "" - hostv = node - } else { - hostv = to_string(.host || .hostname || "") ?? "" - } - m = parse_regex(raw, r'client\s+(?:@\S+\s+)?(?P[0-9a-fA-F:.]+)#(?P\d+)(?:\s+\([^)]*\))?:\s+(?:view\s+(?P\S+):\s+)?query:\s+(?P\S+)\s+(?P\S+)\s+(?P\S+)(?:\s+(?P\S+))?') ?? {} - rc = parse_regex(raw, r'status:\s+(?P\w+)') ?? {} - fields = compact({ - "client_ip": to_string(m.client_ip), - "qname": to_string(m.qname), - "qtype": to_string(m.qtype), - "qclass": to_string(m.qclass), - "view": to_string(m.view), - "flags": to_string(m.flags), - "rcode": to_string(rc.rcode) - }, string: true) - qn = to_string(m.qname) - msg = raw - if qn != "" { - msg = "query " + qn + " " + to_string(m.qtype) - } - . = { - "timestamp": ts, - "host": hostv, - "source": src, - "namespace": ns, - "pod": pod, - "container": container, - "stream": strm, - "severity": "", - "message": msg, - "labels": {"app": "bind"}, - "fields": fields - } - - # Rancher audit log (k8s, cattle-system rancher-audit-log sidecar) — JSON, - # auditLog level 1 (already enabled in the overlay). LIVE NOW. - rancher_audit_parse: - type: remap - inputs: - - app_route.rancher_audit - source: | - node = to_string(.kubernetes.pod_node_name || "") ?? "" - pod = to_string(.kubernetes.pod_name || "") ?? "" - container = to_string(.kubernetes.container_name || "") ?? "" - strm = to_string(.stream || "") ?? "" - raw = to_string(.message || "") ?? "" - ev = object(parse_json(raw) ?? {}) ?? {} - ts = ev.requestTimestamp || ev.time || .timestamp || now() - user = "" - if is_object(ev.user) { - user = to_string(ev.user.name) ?? "" - } else if is_string(ev.user) { - user = to_string(ev.user) ?? "" - } - verb = to_string(ev.method) ?? "" - if verb == "" { verb = to_string(ev.verb) ?? "" } - uri = to_string(ev.requestURI) ?? "" - if uri == "" { uri = to_string(ev.uri) ?? "" } - status = "" - if ev.responseCode != null { status = to_string(ev.responseCode) ?? "" } - if status == "" && is_object(ev.responseStatus) { status = to_string(ev.responseStatus.code) ?? "" } - fields = compact({ - "user": user, - "verb": verb, - "uri": uri, - "status": status, - "auditID": to_string(ev.auditID) ?? "", - "remote_addr": to_string(ev.remoteAddr) ?? "" - }, string: true) - msg = raw - if verb != "" || uri != "" { - msg = verb + " " + uri + " " + status - } - . = { - "timestamp": ts, - "host": node, - "source": "k8s", - "namespace": "cattle-system", - "pod": pod, - "container": container, - "stream": strm, - "severity": "", - "message": msg, - "labels": {"app": "rancher", "log_type": "audit"}, - "fields": fields - } - - # CNPG Postgres — ONE transform for ALL clusters (10 namespaces). The instance - # manager wraps postgres logs as JSON on stdout; the postgres CSV columns nest - # under `.record` (logger == "postgres"). Non-postgres lines (instance-manager - # operator logs) keep .message and set no PG fields. LIVE NOW. - cnpg_pg_parse: - type: remap - inputs: - - app_route.cnpg_pg - source: | - node = to_string(.kubernetes.pod_node_name || "") ?? "" - ns = to_string(.kubernetes.pod_namespace || "") ?? "" - pod = to_string(.kubernetes.pod_name || "") ?? "" - container = to_string(.kubernetes.container_name || "") ?? "" - strm = to_string(.stream || "") ?? "" - raw = to_string(.message || "") ?? "" - cluster = to_string(.kubernetes.pod_labels."cnpg.io/cluster" || "") ?? "" - ev = object(parse_json(raw) ?? {}) ?? {} - ts = .timestamp || now() - rec = object(ev.record) ?? {} - logger = to_string(ev.logger) ?? "" - sev = "" - pgmsg = "" - fields = {} - if logger == "postgres" { - sev = to_string(rec.error_severity) ?? "" - pgmsg = to_string(rec.message) ?? "" - dm = parse_regex(pgmsg, r'duration:\s+(?P[0-9.]+)\s+ms') ?? {} - fields = compact({ - "error_severity": sev, - "message": pgmsg, - "query": to_string(rec.query) ?? "", - "duration_ms": to_string(dm.ms), - "user": to_string(rec.user_name) ?? "", - "database": to_string(rec.database_name) ?? "" - }, string: true) - } - lbls = {"app": "cnpg"} - if cluster != "" { - lbls = {"app": "cnpg", "cluster": cluster} - } - msg = raw - if pgmsg != "" { msg = pgmsg } - . = { - "timestamp": ts, - "host": node, - "source": "k8s", - "namespace": ns, - "pod": pod, - "container": container, - "stream": strm, - "severity": sev, - "message": msg, - "labels": lbls, - "fields": fields - } - - # Gitea router/access logs (k8s gitea + VM gitea). Router "completed" lines give - # method/path/status/latency; NCSA access lines give method/path/status/user. - # k8s LIVE once the overlay log config (this change) rolls out; VM AWAITS the - # puppet-side log-format enable. - gitea_parse: - type: remap - inputs: - - app_route.gitea - source: | - subj = to_string(.subject) ?? "" - is_k8s = starts_with(subj, "logs.k8s.") - raw = to_string(.message || .msg || "") ?? "" - ts = .timestamp || .ts || now() - node = "" - ns = "" - pod = "" - container = "" - strm = "" - hostv = "" - src = "vm" - if is_k8s { - src = "k8s" - node = to_string(.kubernetes.pod_node_name || "") ?? "" - ns = to_string(.kubernetes.pod_namespace || "") ?? "" - pod = to_string(.kubernetes.pod_name || "") ?? "" - container = to_string(.kubernetes.container_name || "") ?? "" - strm = to_string(.stream || "") ?? "" - hostv = node - } else { - hostv = to_string(.host || .hostname || "") ?? "" - } - r = parse_regex(raw, r'completed (?P\S+) (?P\S+) for (?P\S+), (?P\d{3}) [^ ]+ in (?P[0-9.]+\w+)') ?? {} - a = parse_regex(raw, r'^(?P\S+) \S+ (?P\S+) \[[^\]]+\] "(?P\S+) (?P\S+) [^"]*" (?P\d{3})') ?? {} - method = to_string(r.method) - if method == "" { method = to_string(a.method) } - path = to_string(r.path) - if path == "" { path = to_string(a.path) } - status = to_string(r.status) - if status == "" { status = to_string(a.status) } - user = to_string(a.user) - if user == "-" { user = "" } - fields = compact({ - "method": method, - "path": path, - "status": status, - "latency": to_string(r.latency), - "user": user, - "client_ip": to_string(a.client_ip) - }, string: true) - msg = raw - if method != "" { - msg = method + " " + path + " " + status - } - . = { - "timestamp": ts, - "host": hostv, - "source": src, - "namespace": ns, - "pod": pod, - "container": container, - "stream": strm, - "severity": "", - "message": msg, - "labels": {"app": "gitea"}, - "fields": fields - } - - # PuppetServer / PuppetDB (k8s openvoxserver/openvoxdb stdout, ns puppet). Per - # line logback parse (level/logger/message + node) and an access-log line - # (method/status/node) where present. VM multiline stacktrace join + - # puppetserver-access.log are a puppet-side edge concern (follow-up). LIVE NOW. - puppet_parse: - type: remap - inputs: - - app_route.puppet - source: | - node = to_string(.kubernetes.pod_node_name || "") ?? "" - pod = to_string(.kubernetes.pod_name || "") ?? "" - container = to_string(.kubernetes.container_name || "") ?? "" - strm = to_string(.stream || "") ?? "" - raw = to_string(.message || "") ?? "" - ts = .timestamp || now() - lb = parse_regex(raw, r'^(?P\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}[,.]\d+)\s+(?P[A-Z]+)\s+\[(?P[^\]]*)\]\s+\[(?P[^\]]*)\]\s+(?P.*)$') ?? {} - ac = parse_regex(raw, r'^(?P\S+) \S+ \S+ \[[^\]]+\] "(?P\S+) (?P\S+) [^"]*" (?P\d{3})') ?? {} - nd = parse_regex(raw, r'(?:catalog for|for node)\s+(?P[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+)') ?? {} - lvl = to_string(lb.level) - pmsg = to_string(lb.msg) - err = "" - if lvl == "ERROR" { err = pmsg } - fields = compact({ - "level": lvl, - "logger": to_string(lb.logger), - "node": to_string(nd.node), - "method": to_string(ac.method), - "status": to_string(ac.status), - "path": to_string(ac.path), - "client_ip": to_string(ac.client_ip), - "error": err - }, string: true) - msg = raw - if pmsg != "" { msg = pmsg } - . = { - "timestamp": ts, - "host": node, - "source": "k8s", - "namespace": "puppet", - "pod": pod, - "container": container, - "stream": strm, - "severity": lvl, - "message": msg, - "labels": {"app": "puppet"}, - "fields": fields - } - - # LiteLLM request logs (k8s) — JSON once JSON_LOGS=True (flipped in the litellm - # env this change). parse_json -> model/tokens/latency/key/status (best-effort - # against litellm's JSON schema); non-JSON lines keep .message. Field keys light - # up once the env flip rolls out. - litellm_parse: - type: remap - inputs: - - app_route.litellm - source: | - node = to_string(.kubernetes.pod_node_name || "") ?? "" - pod = to_string(.kubernetes.pod_name || "") ?? "" - container = to_string(.kubernetes.container_name || "") ?? "" - strm = to_string(.stream || "") ?? "" - raw = to_string(.message || "") ?? "" - ev = object(parse_json(raw) ?? {}) ?? {} - ts = ev.timestamp || .timestamp || now() - sev = to_string(ev.level) ?? "" - lmsg = to_string(ev.message) ?? "" - fields = compact({ - "model": to_string(ev.model) ?? "", - "tokens": to_string(ev.total_tokens) ?? "", - "latency": to_string(ev.response_time) ?? "", - "key": to_string(ev.api_key) ?? "", - "status": to_string(ev.status) ?? "", - "user": to_string(ev.user) ?? "" - }, string: true) - msg = raw - if lmsg != "" { msg = lmsg } - . = { - "timestamp": ts, - "host": node, - "source": "k8s", - "namespace": "litellm", - "pod": pod, - "container": container, - "stream": strm, - "severity": sev, - "message": msg, - "labels": {"app": "litellm"}, - "fields": fields - } - - # Postfix maillog (VM) — best-effort PER-LINE parse (qid + from/to/status/relay/ - # delay). Full qid-lifecycle correlation is a query-time GROUP BY qid in - # ClickHouse, NOT a stateless-aggregator job (stitching the multi-line lifecycle - # needs a stateful reduce). AWAITS VM VECTOR. - postfix_parse: - type: remap - inputs: - - app_route.postfix - source: | - host = to_string(.host || .hostname || "") ?? "" - raw = to_string(.message || .msg || "") ?? "" - ts = .timestamp || .ts || now() - prog = to_string(.SYSLOG_IDENTIFIER || .program || .appname || "") ?? "" - q = parse_regex(raw, r'^(?P[0-9A-F]{6,}):') ?? {} - frm = parse_regex(raw, r'from=<(?P[^>]*)>') ?? {} - rcpt = parse_regex(raw, r'to=<(?P[^>]*)>') ?? {} - st = parse_regex(raw, r'status=(?P\w+)') ?? {} - rel = parse_regex(raw, r'relay=(?P[^,]+)') ?? {} - dly = parse_regex(raw, r'delay=(?P[0-9.]+)') ?? {} - fields = compact({ - "qid": to_string(q.qid), - "from": to_string(frm.from), - "to": to_string(rcpt.to), - "status": to_string(st.status), - "relay": to_string(rel.relay), - "delay": to_string(dly.delay), - "program": prog - }, string: true) - . = { - "timestamp": ts, - "host": host, - "source": "vm", - "namespace": "", - "pod": "", - "container": "", - "stream": "", - "severity": "", - "message": raw, - "labels": {"app": "postfix"}, - "fields": fields - } - -sinks: - clickhouse: - type: clickhouse - inputs: - - k8s_shape - - vm_shape - - authentik_parse - - traefik_parse - - vault_parse - - nginx_access_parse - - nginx_error_parse - - haproxy_parse - - glauth_parse - - bind_query_parse - - rancher_audit_parse - - cnpg_pg_parse - - gitea_parse - - puppet_parse - - litellm_parse - - postfix_parse - endpoint: http://clickhouse-logs.logging.svc.cluster.local:8123 - database: logs - table: raw - skip_unknown_fields: true - date_time_best_effort: true - auth: - strategy: basic - user: "${CLICKHOUSE_USER}" - password: "${CLICKHOUSE_PASSWORD}" - batch: - max_events: 500000 - max_bytes: 134217728 - timeout_secs: 10 - # Stateless: in-memory buffer, block on full so back-pressure reaches the - # JetStream pull source (which then stops acking). JetStream is durability. - buffer: - type: memory - max_events: 2000 - when_full: block - healthcheck: - enabled: true diff --git a/apps/base/logging/vector/vm-ingest-tests.yaml b/apps/base/logging/vector/vm-ingest-tests.yaml deleted file mode 100644 index fed2928..0000000 --- a/apps/base/logging/vector/vm-ingest-tests.yaml +++ /dev/null @@ -1,29 +0,0 @@ ---- -# `vector test` unit tests for the VM-ingest routing transform. -tests: - - name: host_token_is_subject_safe - inputs: - - insert_at: tag - type: log - log_fields: - host: "db1.syd1.example.net" - message: "sshd accepted" - outputs: - - extract_from: tag - conditions: - - type: vrl - source: | - assert_eq!(.host_token, "db1_syd1_example_net") - - - name: missing_host_defaults_to_unknown - inputs: - - insert_at: tag - type: log - log_fields: - message: "no host field" - outputs: - - extract_from: tag - conditions: - - type: vrl - source: | - assert_eq!(.host_token, "unknown") diff --git a/apps/base/logging/vector/vm-ingest.yaml b/apps/base/logging/vector/vm-ingest.yaml deleted file mode 100644 index 8e2b069..0000000 --- a/apps/base/logging/vector/vm-ingest.yaml +++ /dev/null @@ -1,50 +0,0 @@ ---- -# Vector VM-INGEST tier — the VM front door. Thin: accepts NDJSON over HTTPS -# (behind the logs-ingest Gateway) from puppet-managed VMs, attaches only a -# routing token, and publishes into JetStream (subject logs.vm.). No -# parsing here — shaping happens in the transform tier after JetStream, so VM -# logs get the same durability/replay/fan-out as k8s logs. -data_dir: /vector-data-dir - -api: - enabled: true - address: 0.0.0.0:8686 - -sources: - vm_http: - type: http_server - address: 0.0.0.0:8080 - path: / - method: POST - decoding: - codec: json - framing: - method: newline_delimited - -transforms: - # Routing metadata only: derive a NATS-subject-safe host token. - tag: - type: remap - inputs: - - vm_http - source: | - host = to_string(.host || .hostname || "unknown") ?? "unknown" - .host_token = replace(host, r'[^a-zA-Z0-9_-]', "_") - -sinks: - to_jetstream: - type: nats - inputs: - - tag - url: nats://nats.logging.svc.cluster.local:4222 - connection_name: vector-vm-ingest - subject: "logs.vm.{{ host_token }}" - jetstream: - enabled: true - auth: - strategy: user_password - user_password: - user: log-producer - password: ${NATS_PRODUCER_PASSWORD} - encoding: - codec: json diff --git a/apps/base/logging/vlagent.yaml b/apps/base/logging/vlagent.yaml new file mode 100644 index 0000000..052f54e --- /dev/null +++ b/apps/base/logging/vlagent.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: operator.victoriametrics.com/v1 +kind: VLAgent +metadata: + name: logs + namespace: logging +spec: + componentVersion: v1.52.0 + k8sCollector: + enabled: true + remoteWrite: + # vlagent appends ?version=v1 itself; the path must be the native endpoint + - url: http://vlinsert-logs.logging.svc.cluster.local:9481/insert/native + remoteWriteSettings: + # collector mode buffers to the node's /var/lib/vlagent-data, so cap it: + # a vlinsert outage must not fill the host disk (500MB chunks) + maxDiskUsagePerURL: 2GiB + tolerations: + - operator: Exists + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi diff --git a/apps/overlays/au-syd1/clickhouse-system/kustomization.yaml b/apps/overlays/au-syd1/clickhouse-system/kustomization.yaml deleted file mode 100644 index 498f573..0000000 --- a/apps/overlays/au-syd1/clickhouse-system/kustomization.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namespace: clickhouse-system - -resources: - - ../../../base/clickhouse-system - -helmCharts: - - name: altinity-clickhouse-operator - repo: https://helm.altinity.com - version: "0.27.2" - releaseName: clickhouse-operator - namespace: clickhouse-system - valuesFile: values.yaml diff --git a/apps/overlays/au-syd1/clickhouse-system/values.yaml b/apps/overlays/au-syd1/clickhouse-system/values.yaml deleted file mode 100644 index 74b21a9..0000000 --- a/apps/overlays/au-syd1/clickhouse-system/values.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# Altinity ClickHouse operator. Cluster-scoped: watches ClickHouseInstallation -# resources in all namespaces (the logs cluster lives in the `logging` namespace). -# CRDs are installed at runtime by the chart's crdHook Job. -# -# Upstream official images; no Docker Hardened Image variant is adopted (DHI is -# subscription-gated and served from a private org namespace not reachable via -# the anonymous artifactapi dockerhub proxy). -# -# Watch the logging namespace where the ClickHouseInstallation lives. The chart -# default (watchNamespaces: []) makes the operator watch ONLY its own namespace -# (clickhouse-system), so the logs CHI was never reconciled — set it explicitly. -watchNamespaces: - - logging -crdHook: - image: - repository: docker.io/bitnami/kubectl - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 250m - memory: 128Mi - -operator: - image: - repository: docker.io/altinity/clickhouse-operator - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - -metrics: - image: - repository: docker.io/altinity/metrics-exporter - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 250m - memory: 256Mi diff --git a/apps/overlays/au-syd1/logging/kustomization.yaml b/apps/overlays/au-syd1/logging/kustomization.yaml index 9939c00..f3dfb7f 100644 --- a/apps/overlays/au-syd1/logging/kustomization.yaml +++ b/apps/overlays/au-syd1/logging/kustomization.yaml @@ -6,34 +6,3 @@ namespace: logging resources: - ../../../base/logging - -helmCharts: - # Dedicated JetStream NATS cluster — the durable log bus. - - name: nats - repo: https://nats-io.github.io/k8s/helm/charts - version: "2.14.2" - releaseName: nats - namespace: logging - valuesFile: values-nats.yaml - # Edge agent (DaemonSet): tails every pod's logs, publishes to JetStream. - - name: vector - repo: https://helm.vector.dev - version: "0.57.0" - releaseName: vector-agent - namespace: logging - valuesFile: values-vector-agent.yaml - # VM ingest (Deployment): HTTP NDJSON front door -> JetStream. - - name: vector - repo: https://helm.vector.dev - version: "0.57.0" - releaseName: vector-vm-ingest - namespace: logging - valuesFile: values-vector-vm-ingest.yaml - # Transform tier (StatefulSet): JetStream consumer -> shape -> ClickHouse. - - name: vector - repo: https://helm.vector.dev - version: "0.57.0" - releaseName: vector-aggregator - namespace: logging - valuesFile: values-vector-aggregator.yaml - # Archiver: replaced by the logarchiver Deployment (apps/base/logging). diff --git a/apps/overlays/au-syd1/logging/values-nats.yaml b/apps/overlays/au-syd1/logging/values-nats.yaml deleted file mode 100644 index 6c0d582..0000000 --- a/apps/overlays/au-syd1/logging/values-nats.yaml +++ /dev/null @@ -1,103 +0,0 @@ -# Dedicated JetStream-enabled NATS cluster for the log bus. Deliberately NOT -# shared with app messaging (streamstack et al. run their own NATS in their own -# repo) — a separate cluster isolates logging blast-radius from app messaging -# and lets us size retention/storage purely for the log outage-buffer + replay -# use-case. -fullnameOverride: nats - -config: - cluster: - enabled: true - replicas: 3 - jetstream: - enabled: true - fileStore: - pvc: - # Sized for 3d retention: ~100 GiB/3d compressed (see - # nats-stream-limits ConfigMap) + file-store WAL/index/overhead, kept - # safely above the 130 GiB max_bytes cap. 3 replicas => ~0.5 TiB total - # provisioned on cephrbd-fast-delete. NB: this is the honest number for - # the assumed ~1500 events/s; higher real volume needs a bigger PVC + - # max_bytes together, else discard=old truncates retention below 3d. - size: 180Gi - storageClassName: cephrbd-fast-delete - # Per-user auth with publish/subscribe separation. Passwords are injected as - # env vars from the Vault-synced nats-auth Secret. The `<< $VAR >>` wrapping is - # REQUIRED by this chart: it renders the value UNQUOTED in nats.conf so the - # NATS server expands the env var. A plain `$VAR` is JSON-quoted ("$VAR") and - # NATS then treats it as a literal string — which broke auth for every client. - merge: - authorization: - users: - # Bootstrap Job (stream/consumer management) — full JetStream API. - - user: log-admin - password: << $NATS_ADMIN_PASSWORD >> - # Edge publishers (k8s DaemonSet + VM ingest) — publish only. - - user: log-producer - password: << $NATS_PRODUCER_PASSWORD >> - permissions: - publish: - allow: - - "logs.>" - subscribe: - allow: - - "_INBOX.>" - # Consumers (transform tier + archiver) — pull + ack only, no publish - # to log subjects. - - user: log-consumer - password: << $NATS_CONSUMER_PASSWORD >> - permissions: - publish: - allow: - - "$JS.API.CONSUMER.>" - - "$JS.API.STREAM.INFO.>" - - "$JS.ACK.LOGS.>" - subscribe: - allow: - - "_INBOX.>" - -container: - # Upstream official nats; no DHI variant available. - image: - repository: docker.io/library/nats - tag: 2.14.2-alpine - env: - NATS_ADMIN_PASSWORD: - valueFrom: - secretKeyRef: - name: nats-auth - key: admin_password - NATS_PRODUCER_PASSWORD: - valueFrom: - secretKeyRef: - name: nats-auth - key: producer_password - NATS_CONSUMER_PASSWORD: - valueFrom: - secretKeyRef: - name: nats-auth - key: consumer_password - resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: "2" - memory: 4Gi - -# Roll the StatefulSet on config changes only; nats-auth is Vault-rotated (VSO) -# so it is deliberately not watched here (no restart on routine key rotation). -podTemplate: - merge: - metadata: - annotations: - configmap.reloader.stakater.com/auto: "true" - -# Config-reloader sidecar image. -reloader: - image: - repository: docker.io/natsio/nats-server-config-reloader - tag: "0.23.0" - -natsBox: - enabled: false diff --git a/apps/overlays/au-syd1/logging/values-vector-agent.yaml b/apps/overlays/au-syd1/logging/values-vector-agent.yaml deleted file mode 100644 index a05a805..0000000 --- a/apps/overlays/au-syd1/logging/values-vector-agent.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Vector EDGE agent (DaemonSet) — thin publisher. Tails every node's pod logs -# (incl. control-plane via the blanket toleration) and publishes them into -# JetStream over the Vector NATS sink. No parsing; only a routing subject token -# is attached. Shaping happens in the transform tier after JetStream. -role: Agent -fullnameOverride: vector-agent - -# distroless-libc (no DHI — subscription-gated/private-namespace, not reachable -# via the anon proxy). -image: - repository: docker.io/timberio/vector - tag: 0.57.0-distroless-libc - -rbac: - create: true -serviceAccount: - create: true - -podLabels: - vector.dev/exclude: "true" - -tolerations: - - operator: Exists - -env: - # Vector 0.57 disables ${VAR} config interpolation by default; auth needs it. - - name: VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION - value: "true" - - name: NATS_PRODUCER_PASSWORD - valueFrom: - secretKeyRef: - name: nats-auth - key: producer_password - -resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: "1" - memory: 1Gi - -service: - enabled: false - -# Pipeline is the single source of truth in apps/base/logging/vector/agent.yaml, -# mounted via existingConfigMaps (avoids the chart's customConfig Helm-tpl pass). -dataDir: /vector-data-dir -existingConfigMaps: - - vector-agent-config - -workloadResourceAnnotations: - configmap.reloader.stakater.com/auto: "true" diff --git a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml b/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml deleted file mode 100644 index 549c19e..0000000 --- a/apps/overlays/au-syd1/logging/values-vector-aggregator.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# Vector TRANSFORM tier (STATELESS Deployment) — the "brain": sole ClickHouse -# writer, owns all transforms, holds the only ClickHouse + NATS-consumer creds. -# -# Stateless by design: a JetStream pull consumer with NO PVC and NO disk buffer. -# JetStream is the sole durability layer. On a ClickHouse outage the clickhouse -# sink blocks (buffer when_full=block), back-pressure stops the source pulling, -# and unpulled messages stay in JetStream for redelivery. Because Vector's NATS -# source does NOT support end-to-end acknowledgements (it acks on receipt, not -# after the sink), the only at-risk window is the in-memory buffer's worth of -# already-pulled events if a pod is killed mid-outage — the accepted trade for a -# horizontally-autoscalable stateless tier. Multiple replicas share the one -# durable consumer `transform` (JetStream pull consumers distribute work), so -# HPA is safe. -role: Stateless-Aggregator -fullnameOverride: vector-aggregator - -image: - repository: docker.io/timberio/vector - tag: 0.57.0-distroless-libc - -# Horizontal autoscaling on CPU — safe with N replicas on one durable consumer. -autoscaling: - enabled: true - minReplicas: 2 - maxReplicas: 8 - targetCPUUtilizationPercentage: 70 - -workloadResourceAnnotations: - configmap.reloader.stakater.com/auto: "true" - -podLabels: - vector.dev/exclude: "true" - -# Pipeline is the single source of truth in apps/base/logging/vector/ -# aggregator.yaml (unit-tested by `vector test` in CI), mounted via -# existingConfigMaps. No persistence — stateless. -dataDir: /vector-data-dir -existingConfigMaps: - - vector-aggregator-config - -# The ONLY place ClickHouse + NATS-consumer creds are consumed. -env: - # Vector 0.57 disables ${VAR} config interpolation by default; the pipeline's - # auth (${CLICKHOUSE_*}, ${NATS_CONSUMER_PASSWORD}) needs it enabled. - - name: VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION - value: "true" - - name: CLICKHOUSE_USER - valueFrom: - secretKeyRef: - name: clickhouse-credentials - key: username - - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: clickhouse-credentials - key: password - - name: NATS_CONSUMER_PASSWORD - valueFrom: - secretKeyRef: - name: nats-auth - key: consumer_password - -# Pure consumer: expose only the Vector API for debugging. -containerPorts: - - name: api - containerPort: 8686 - protocol: TCP - -service: - enabled: true - type: ClusterIP - ports: - - name: api - port: 8686 - targetPort: 8686 - protocol: TCP - -resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: "2" - memory: 2Gi diff --git a/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml b/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml deleted file mode 100644 index 9e66a91..0000000 --- a/apps/overlays/au-syd1/logging/values-vector-vm-ingest.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# Vector VM-INGEST tier (Deployment) — VM front door. HTTP NDJSON in (behind the -# logs-ingest Gateway), publishes into JetStream. Stateless publisher. -# Pipeline: apps/base/logging/vector/vm-ingest.yaml (unit-tested in CI). -role: Stateless-Aggregator -fullnameOverride: vector-vm-ingest -replicas: 2 - -image: - repository: docker.io/timberio/vector - tag: 0.57.0-distroless-libc - -workloadResourceAnnotations: - configmap.reloader.stakater.com/auto: "true" - -podLabels: - vector.dev/exclude: "true" - -dataDir: /vector-data-dir -existingConfigMaps: - - vector-vm-ingest-config - -env: - # Vector 0.57 disables ${VAR} config interpolation by default; auth needs it. - - name: VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION - value: "true" - - name: NATS_PRODUCER_PASSWORD - valueFrom: - secretKeyRef: - name: nats-auth - key: producer_password - -containerPorts: - - name: http-ingest - containerPort: 8080 - protocol: TCP - - name: api - containerPort: 8686 - protocol: TCP - -service: - enabled: true - type: ClusterIP - ports: - - name: http-ingest - port: 8080 - targetPort: 8080 - protocol: TCP - - name: api - port: 8686 - targetPort: 8686 - protocol: TCP - -resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: "1" - memory: 1Gi diff --git a/argocd/applicationsets/logging.yaml b/argocd/applicationsets/logging.yaml index ea542d4..acd02e4 100644 --- a/argocd/applicationsets/logging.yaml +++ b/argocd/applicationsets/logging.yaml @@ -10,7 +10,6 @@ spec: repoURL: https://git.unkin.net/unkin/argocd-apps revision: HEAD directories: - - path: apps/overlays/*/clickhouse-system - path: apps/overlays/*/logging template: metadata: