benvin
c05ccfcb5d
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
107 lines
3.1 KiB
Go
107 lines
3.1 KiB
Go
package config
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestDefaultIsValid(t *testing.T) {
|
|
cfg := Default()
|
|
if err := cfg.Validate(); err != nil {
|
|
t.Fatalf("default config should validate: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestLoadEnvOverrides(t *testing.T) {
|
|
t.Setenv("ARCHIVE_SUBJECTS", "logs.vm.> logs.k8s.vault.>")
|
|
t.Setenv("LOGARCHIVER_S3_BUCKET", "my-bucket")
|
|
t.Setenv("LOGARCHIVER_NATS_DURABLE", "archiver-canary")
|
|
t.Setenv("NATS_CONSUMER_PASSWORD", "s3cr3t")
|
|
t.Setenv("S3_ENDPOINT", "https://rgw.internal")
|
|
t.Setenv("BUCKET_NAME", "logs-archive-override")
|
|
|
|
cfg, err := Load("")
|
|
if err != nil {
|
|
t.Fatalf("Load: %v", err)
|
|
}
|
|
if got, want := cfg.NATS.Subjects, []string{"logs.vm.>", "logs.k8s.vault.>"}; len(got) != len(want) || got[0] != want[0] || got[1] != want[1] {
|
|
t.Errorf("subjects = %v, want %v", got, want)
|
|
}
|
|
if cfg.NATS.Durable != "archiver-canary" {
|
|
t.Errorf("durable = %q", cfg.NATS.Durable)
|
|
}
|
|
if cfg.NATS.Password != "s3cr3t" {
|
|
t.Errorf("password from env not resolved: %q", cfg.NATS.Password)
|
|
}
|
|
// BUCKET_NAME (secret) should win over LOGARCHIVER_S3_BUCKET default flow.
|
|
if cfg.S3.Bucket != "logs-archive-override" {
|
|
t.Errorf("bucket = %q, want cephrgw secret override", cfg.S3.Bucket)
|
|
}
|
|
if cfg.S3.Endpoint != "https://rgw.internal" {
|
|
t.Errorf("endpoint = %q", cfg.S3.Endpoint)
|
|
}
|
|
}
|
|
|
|
func TestValidateErrors(t *testing.T) {
|
|
cases := map[string]func(*Config){
|
|
"no subjects": func(c *Config) { c.NATS.Subjects = nil },
|
|
"no bucket": func(c *Config) { c.S3.Bucket = "" },
|
|
"no key name": func(c *Config) { c.Crypto.KeyName = "" },
|
|
"bad source": func(c *Config) { c.Crypto.Source = "elsewhere" },
|
|
"file no path": func(c *Config) { c.Crypto.Source = PubkeyFile; c.Crypto.PubkeyFile = "" },
|
|
"vault no addr": func(c *Config) { c.Crypto.Source = PubkeyVault; c.Crypto.Vault.Address = "" },
|
|
"zero frame": func(c *Config) { c.Crypto.FrameSize = 0 },
|
|
"no batch bounds": func(c *Config) { c.Batch = BatchConfig{} },
|
|
}
|
|
for name, mutate := range cases {
|
|
t.Run(name, func(t *testing.T) {
|
|
cfg := Default()
|
|
mutate(&cfg)
|
|
if err := cfg.Validate(); err == nil {
|
|
t.Errorf("expected validation error for %q", name)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestValidateVaultSourceOK(t *testing.T) {
|
|
cfg := Default()
|
|
cfg.Crypto.Source = PubkeyVault
|
|
cfg.Crypto.Vault.Address = "https://vault.example:8200"
|
|
cfg.Crypto.Vault.Mount = "gpg"
|
|
if err := cfg.Validate(); err != nil {
|
|
t.Fatalf("vault source should validate: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestParseSize(t *testing.T) {
|
|
cases := map[string]int64{
|
|
"1024": 1024,
|
|
"64Mi": 64 << 20,
|
|
"2Gi": 2 << 30,
|
|
"1Ki": 1024,
|
|
"5MB": 5_000_000,
|
|
" 10 ": 10,
|
|
}
|
|
for in, want := range cases {
|
|
got, err := ParseSize(in)
|
|
if err != nil {
|
|
t.Errorf("ParseSize(%q): %v", in, err)
|
|
continue
|
|
}
|
|
if got != want {
|
|
t.Errorf("ParseSize(%q) = %d, want %d", in, got, want)
|
|
}
|
|
}
|
|
if _, err := ParseSize("bogus"); err == nil {
|
|
t.Errorf("expected error for bogus size")
|
|
}
|
|
}
|
|
|
|
func TestDefaultBatchDurations(t *testing.T) {
|
|
cfg := Default()
|
|
if cfg.Batch.MaxAge != 5*time.Minute {
|
|
t.Errorf("default MaxAge = %v", cfg.Batch.MaxAge)
|
|
}
|
|
}
|