benvin
c05ccfcb5d
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
56 lines
1.4 KiB
Go
56 lines
1.4 KiB
Go
package archiver
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestKeyBuilder(t *testing.T) {
|
|
kb, err := NewKeyBuilder("archive/{{.Subject}}/{{.Year}}/{{.Month}}/{{.Day}}/")
|
|
if err != nil {
|
|
t.Fatalf("NewKeyBuilder: %v", err)
|
|
}
|
|
ts := time.Date(2026, 7, 5, 10, 15, 0, 0, time.UTC)
|
|
key, err := kb.Build("logs.k8s.vault.audit", ts)
|
|
if err != nil {
|
|
t.Fatalf("Build: %v", err)
|
|
}
|
|
if !strings.HasPrefix(key, "archive/logs.k8s.vault.audit/2026/07/05/") {
|
|
t.Errorf("unexpected prefix: %s", key)
|
|
}
|
|
if !strings.HasSuffix(key, ObjectExt) {
|
|
t.Errorf("missing container suffix: %s", key)
|
|
}
|
|
if !strings.Contains(key, "20260705T101500Z-") {
|
|
t.Errorf("missing timestamp token: %s", key)
|
|
}
|
|
}
|
|
|
|
func TestKeyBuilderUnique(t *testing.T) {
|
|
kb, _ := NewKeyBuilder("p/{{.Subject}}/")
|
|
ts := time.Date(2026, 7, 5, 10, 15, 0, 0, time.UTC)
|
|
k1, _ := kb.Build("s", ts)
|
|
k2, _ := kb.Build("s", ts)
|
|
if k1 == k2 {
|
|
t.Errorf("keys should be unique: %s", k1)
|
|
}
|
|
}
|
|
|
|
func TestKeyBuilderSanitizesSubject(t *testing.T) {
|
|
kb, _ := NewKeyBuilder("p/{{.Subject}}/")
|
|
key, _ := kb.Build("logs.k8s.a/b", time.Now())
|
|
if strings.Contains(key, "a/b") {
|
|
t.Errorf("subject slash not sanitized: %s", key)
|
|
}
|
|
if !strings.Contains(key, "a_b") {
|
|
t.Errorf("expected sanitized a_b: %s", key)
|
|
}
|
|
}
|
|
|
|
func TestKeyBuilderBadTemplate(t *testing.T) {
|
|
if _, err := NewKeyBuilder("{{.Nope"); err == nil {
|
|
t.Errorf("expected template parse error")
|
|
}
|
|
}
|