benvin
c05ccfcb5d
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
85 lines
3.0 KiB
YAML
85 lines
3.0 KiB
YAML
# logarchiver configuration example.
|
|
#
|
|
# Every field below shows its built-in default (the defaults are tuned for the
|
|
# centralized logging stack in argocd-apps #296). In k8s you typically deploy
|
|
# with NO config file and let the defaults + secret env vars drive everything;
|
|
# this file documents the knobs and is handy for local/dev runs.
|
|
#
|
|
# Precedence: built-in defaults < this file < environment variables.
|
|
|
|
nats:
|
|
url: "nats://nats.logging.svc.cluster.local:4222"
|
|
stream: "LOGS"
|
|
# Durable pull-consumer name. Reusing "archiver" takes over the leg the Vector
|
|
# archiver currently owns; use a distinct name (e.g. "archiver-canary") to run
|
|
# alongside it during migration.
|
|
durable: "archiver"
|
|
# Server-side subject filter(s). Overridable via env ARCHIVE_SUBJECTS
|
|
# (space-separated). Default archives Vault audit logs only.
|
|
subjects:
|
|
- "logs.k8s.vault.>"
|
|
user: "log-consumer"
|
|
# Password comes from the nats-auth secret via NATS_CONSUMER_PASSWORD.
|
|
password: ""
|
|
password_env: "NATS_CONSUMER_PASSWORD"
|
|
ca_file: "" # in-cluster NATS is plaintext
|
|
fetch_batch: 512
|
|
ack_wait: "2m"
|
|
|
|
batch:
|
|
# A per-subject batch becomes one object when any bound is hit.
|
|
max_bytes: 67108864 # 64 MiB raw NDJSON
|
|
max_events: 200000
|
|
max_age: "5m"
|
|
|
|
s3:
|
|
endpoint: "https://s3.ceph.unkin.net"
|
|
bucket: "logs-archive"
|
|
region: "us-east-1"
|
|
path_style: true
|
|
# Object key template. Fields: {{.Subject}} {{.Year}} {{.Month}} {{.Day}}.
|
|
key_prefix: "archive/{{.Subject}}/{{.Year}}/{{.Month}}/{{.Day}}/"
|
|
ca_file: "/etc/vault-ca/ca.crt"
|
|
# Credentials come from the cephrgw BucketAccess secret logs-archive-s3 via the
|
|
# standard AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars. Endpoint/bucket
|
|
# can also be sourced from that secret's S3_ENDPOINT / BUCKET_NAME keys:
|
|
endpoint_env: "S3_ENDPOINT"
|
|
bucket_env: "BUCKET_NAME"
|
|
|
|
crypto:
|
|
# Vault GPG engine key used to encrypt objects (public key) and decrypt on
|
|
# retrieval (private key, server-side in Vault).
|
|
key_name: "logarchive"
|
|
# Where the service gets the PUBLIC key: "file" (mounted armored key, no Vault
|
|
# dependency for the service) or "vault" (read gpg/keys/<name>).
|
|
pubkey_source: "file"
|
|
pubkey_file: "/etc/logarchiver/pubkey.asc"
|
|
refresh_interval: "1h"
|
|
frame_size: 1048576 # 1 MiB AES-GCM frames (streaming decrypt granularity)
|
|
vault:
|
|
address: "" # falls back to VAULT_ADDR
|
|
mount: "gpg"
|
|
auth_method: "kubernetes" # service: kubernetes; CLI always uses token
|
|
k8s_role: "default"
|
|
k8s_mount: "k8s/au/syd1"
|
|
k8s_jwt_path: "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
|
ca_file: ""
|
|
|
|
index:
|
|
enabled: true
|
|
address: "clickhouse-logs.logging.svc.cluster.local:9000" # native protocol
|
|
database: "logs"
|
|
table: "archive_index"
|
|
username: "vector"
|
|
password: ""
|
|
password_env: "CLICKHOUSE_PASSWORD"
|
|
tls: false # in-cluster ClickHouse is plaintext
|
|
|
|
metrics:
|
|
enabled: true
|
|
address: ":9090" # /metrics and /healthz
|
|
|
|
log:
|
|
level: "info" # debug|info|warn|error
|
|
format: "json" # json|text
|