c05ccfcb5d
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
149 lines
4.2 KiB
Go
149 lines
4.2 KiB
Go
package batcher
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func item(subject, host string, ts time.Time, hasTS bool, raw string) Item {
|
|
return Item{Subject: subject, Host: host, Timestamp: ts, HasTS: hasTS, Raw: []byte(raw)}
|
|
}
|
|
|
|
func TestFullByEvents(t *testing.T) {
|
|
b := New(Limits{MaxEvents: 3})
|
|
if got := b.Add(item("s", "h", time.Now(), true, "a")); got != nil {
|
|
t.Fatalf("should not be full at 1")
|
|
}
|
|
if got := b.Add(item("s", "h", time.Now(), true, "b")); got != nil {
|
|
t.Fatalf("should not be full at 2")
|
|
}
|
|
full := b.Add(item("s", "h", time.Now(), true, "c"))
|
|
if full == nil {
|
|
t.Fatalf("should be full at 3")
|
|
}
|
|
if len(full.Items) != 3 {
|
|
t.Errorf("full batch has %d items", len(full.Items))
|
|
}
|
|
// After a full flush the subject batch is reset.
|
|
if b.Pending() != 0 {
|
|
t.Errorf("pending after flush = %d, want 0", b.Pending())
|
|
}
|
|
}
|
|
|
|
func TestFullByBytes(t *testing.T) {
|
|
b := New(Limits{MaxBytes: 10})
|
|
if b.Add(item("s", "h", time.Now(), true, "12345")) != nil {
|
|
t.Fatalf("5 bytes should not fill")
|
|
}
|
|
full := b.Add(item("s", "h", time.Now(), true, "67890"))
|
|
if full == nil {
|
|
t.Fatalf("10 bytes should fill")
|
|
}
|
|
if full.RawBytes != 10 {
|
|
t.Errorf("RawBytes = %d", full.RawBytes)
|
|
}
|
|
}
|
|
|
|
func TestSeparateSubjects(t *testing.T) {
|
|
b := New(Limits{MaxEvents: 2})
|
|
b.Add(item("a", "h", time.Now(), true, "x"))
|
|
b.Add(item("b", "h", time.Now(), true, "y"))
|
|
if b.Pending() != 2 {
|
|
t.Errorf("pending = %d, want 2 across subjects", b.Pending())
|
|
}
|
|
full := b.Add(item("a", "h", time.Now(), true, "z"))
|
|
if full == nil || full.Subject != "a" {
|
|
t.Fatalf("subject a should flush independently")
|
|
}
|
|
if b.Pending() != 1 {
|
|
t.Errorf("pending after a flush = %d, want 1 (subject b)", b.Pending())
|
|
}
|
|
}
|
|
|
|
func TestDueByAge(t *testing.T) {
|
|
base := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC)
|
|
b := New(Limits{MaxAge: time.Minute})
|
|
b.nowFn = func() time.Time { return base }
|
|
b.Add(item("s", "h", base, true, "x"))
|
|
|
|
if due := b.DueByAge(base.Add(30 * time.Second)); len(due) != 0 {
|
|
t.Fatalf("not due at 30s")
|
|
}
|
|
due := b.DueByAge(base.Add(90 * time.Second))
|
|
if len(due) != 1 {
|
|
t.Fatalf("should be due at 90s, got %d", len(due))
|
|
}
|
|
if b.Pending() != 0 {
|
|
t.Errorf("due batch not removed")
|
|
}
|
|
}
|
|
|
|
func TestDrain(t *testing.T) {
|
|
b := New(Limits{MaxEvents: 100})
|
|
b.Add(item("a", "h", time.Now(), true, "x"))
|
|
b.Add(item("b", "h", time.Now(), true, "y"))
|
|
all := b.Drain()
|
|
if len(all) != 2 {
|
|
t.Fatalf("drain returned %d, want 2", len(all))
|
|
}
|
|
if b.Pending() != 0 {
|
|
t.Errorf("pending after drain = %d", b.Pending())
|
|
}
|
|
}
|
|
|
|
func TestNDJSON(t *testing.T) {
|
|
b := &Batch{Subject: "s"}
|
|
b.Items = []Item{
|
|
{Raw: []byte(`{"a":1}`)},
|
|
{Raw: []byte(`{"b":2}` + "\n")}, // trailing newline trimmed and re-added
|
|
}
|
|
got := string(b.NDJSON())
|
|
want := "{\"a\":1}\n{\"b\":2}\n"
|
|
if got != want {
|
|
t.Errorf("NDJSON = %q, want %q", got, want)
|
|
}
|
|
if strings.Count(got, "\n") != 2 {
|
|
t.Errorf("expected exactly 2 newlines")
|
|
}
|
|
}
|
|
|
|
func TestSummarize(t *testing.T) {
|
|
t1 := time.Date(2026, 7, 27, 1, 0, 0, 0, time.UTC)
|
|
t2 := time.Date(2026, 7, 27, 3, 0, 0, 0, time.UTC)
|
|
fallback := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
|
|
b := &Batch{Subject: "s"}
|
|
b.Items = []Item{
|
|
item("s", "host-b", t2, true, "x"),
|
|
item("s", "host-a", t1, true, "y"),
|
|
item("s", "", time.Time{}, false, "z"), // no ts -> fallback, no host
|
|
item("s", "host-a", t1, true, "w"), // dup host
|
|
}
|
|
s := b.Summarize(fallback)
|
|
if s.EventCount != 4 {
|
|
t.Errorf("EventCount = %d", s.EventCount)
|
|
}
|
|
if len(s.Hosts) != 2 || s.Hosts[0] != "host-a" || s.Hosts[1] != "host-b" {
|
|
t.Errorf("Hosts = %v, want sorted unique [host-a host-b]", s.Hosts)
|
|
}
|
|
if !s.MinTS.Equal(t1) {
|
|
t.Errorf("MinTS = %v, want %v", s.MinTS, t1)
|
|
}
|
|
// max should be the fallback (9:00) since event z used fallback which is latest
|
|
if !s.MaxTS.Equal(fallback) {
|
|
t.Errorf("MaxTS = %v, want fallback %v", s.MaxTS, fallback)
|
|
}
|
|
}
|
|
|
|
func TestSummarizeAllFallback(t *testing.T) {
|
|
fallback := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
|
|
b := &Batch{Items: []Item{{Raw: []byte("x")}}}
|
|
s := b.Summarize(fallback)
|
|
if !s.MinTS.Equal(fallback) || !s.MaxTS.Equal(fallback) {
|
|
t.Errorf("all-fallback range wrong: %v..%v", s.MinTS, s.MaxTS)
|
|
}
|
|
if s.HasTS {
|
|
t.Errorf("HasTS should be false")
|
|
}
|
|
}
|