Initial implementation: NATS->S3 archiver + search/retrieve CLI

logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
2026-07-27 22:11:54 +10:00
parent d036d31f12
commit c05ccfcb5d
58 changed files with 5946 additions and 1 deletions
@@ -0,0 +1,177 @@
+// Package batcher groups incoming log events into per-subject batches and
+// decides when a batch is ready to become one archived object. It is
+// deliberately not concurrent: the consumer run loop owns a Batcher and drives
+// it from a single goroutine (Add on receive, DueByAge on a ticker, Drain on
+// shutdown), which keeps the ack-after-persist accounting simple and race-free.
+package batcher
+
+import (
+	"bytes"
+	"sort"
+	"time"
+)
+
+// Item is one log event routed into a batch. Ack is an opaque token (a
+// jetstream.Msg in production) that the caller acknowledges only after the batch
+// has been durably persisted.
+type Item struct {
+	Subject   string
+	Raw       []byte
+	Host      string
+	Timestamp time.Time
+	HasTS     bool
+	Ack       any
+}
+
+// Limits bound a single batch/object.
+type Limits struct {
+	MaxBytes  int64
+	MaxEvents int
+	MaxAge    time.Duration
+}
+
+// Batch is a ready (or in-progress) group of events for one subject.
+type Batch struct {
+	Subject  string
+	Items    []Item
+	RawBytes int64
+	OpenedAt time.Time
+}
+
+// Batcher accumulates open batches keyed by subject.
+type Batcher struct {
+	limits Limits
+	open   map[string]*Batch
+	nowFn  func() time.Time
+}
+
+// New returns a Batcher enforcing limits.
+func New(limits Limits) *Batcher {
+	return &Batcher{limits: limits, open: map[string]*Batch{}, nowFn: time.Now}
+}
+
+// Add appends it to its subject's open batch. If that batch is now full (by
+// bytes or event count), it is removed from the open set and returned so the
+// caller can flush it; otherwise Add returns nil.
+func (b *Batcher) Add(it Item) *Batch {
+	batch := b.open[it.Subject]
+	if batch == nil {
+		batch = &Batch{Subject: it.Subject, OpenedAt: b.nowFn()}
+		b.open[it.Subject] = batch
+	}
+	batch.Items = append(batch.Items, it)
+	batch.RawBytes += int64(len(it.Raw))
+
+	if b.full(batch) {
+		delete(b.open, it.Subject)
+		return batch
+	}
+	return nil
+}
+
+func (b *Batcher) full(batch *Batch) bool {
+	if b.limits.MaxBytes > 0 && batch.RawBytes >= b.limits.MaxBytes {
+		return true
+	}
+	if b.limits.MaxEvents > 0 && len(batch.Items) >= b.limits.MaxEvents {
+		return true
+	}
+	return false
+}
+
+// DueByAge removes and returns every open batch older than MaxAge as of now.
+func (b *Batcher) DueByAge(now time.Time) []*Batch {
+	if b.limits.MaxAge <= 0 {
+		return nil
+	}
+	var due []*Batch
+	for subj, batch := range b.open {
+		if now.Sub(batch.OpenedAt) >= b.limits.MaxAge {
+			due = append(due, batch)
+			delete(b.open, subj)
+		}
+	}
+	sortBatches(due)
+	return due
+}
+
+// Drain removes and returns all open batches (used on graceful shutdown so
+// in-flight events are persisted and acked before exit).
+func (b *Batcher) Drain() []*Batch {
+	var all []*Batch
+	for subj, batch := range b.open {
+		all = append(all, batch)
+		delete(b.open, subj)
+	}
+	sortBatches(all)
+	return all
+}
+
+// Pending reports how many events sit in open batches.
+func (b *Batcher) Pending() int {
+	n := 0
+	for _, batch := range b.open {
+		n += len(batch.Items)
+	}
+	return n
+}
+
+func sortBatches(bs []*Batch) {
+	sort.Slice(bs, func(i, j int) bool { return bs[i].Subject < bs[j].Subject })
+}
+
+// NDJSON renders the batch as newline-delimited JSON (one raw event per line),
+// matching the raw archive format the logging stack expects.
+func (b *Batch) NDJSON() []byte {
+	var buf bytes.Buffer
+	buf.Grow(int(b.RawBytes) + len(b.Items))
+	for _, it := range b.Items {
+		buf.Write(bytes.TrimRight(it.Raw, "\n"))
+		buf.WriteByte('\n')
+	}
+	return buf.Bytes()
+}
+
+// Summary is the index-relevant projection of a batch.
+type Summary struct {
+	Hosts      []string
+	MinTS      time.Time
+	MaxTS      time.Time
+	EventCount int
+	HasTS      bool
+}
+
+// Summarize computes hosts (unique, sorted) and the timestamp range. fallback is
+// used for events whose payload lacked a parseable timestamp (ingest time).
+func (b *Batch) Summarize(fallback time.Time) Summary {
+	s := Summary{EventCount: len(b.Items)}
+	hostSet := map[string]struct{}{}
+	for _, it := range b.Items {
+		if it.Host != "" {
+			hostSet[it.Host] = struct{}{}
+		}
+		ts := it.Timestamp
+		if !it.HasTS {
+			ts = fallback
+		} else {
+			s.HasTS = true
+		}
+		if s.MinTS.IsZero() || ts.Before(s.MinTS) {
+			s.MinTS = ts
+		}
+		if s.MaxTS.IsZero() || ts.After(s.MaxTS) {
+			s.MaxTS = ts
+		}
+	}
+	if s.MinTS.IsZero() {
+		s.MinTS = fallback
+	}
+	if s.MaxTS.IsZero() {
+		s.MaxTS = fallback
+	}
+	for h := range hostSet {
+		s.Hosts = append(s.Hosts, h)
+	}
+	sort.Strings(s.Hosts)
+	return s
+}
@@ -0,0 +1,148 @@
+package batcher
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+func item(subject, host string, ts time.Time, hasTS bool, raw string) Item {
+	return Item{Subject: subject, Host: host, Timestamp: ts, HasTS: hasTS, Raw: []byte(raw)}
+}
+
+func TestFullByEvents(t *testing.T) {
+	b := New(Limits{MaxEvents: 3})
+	if got := b.Add(item("s", "h", time.Now(), true, "a")); got != nil {
+		t.Fatalf("should not be full at 1")
+	}
+	if got := b.Add(item("s", "h", time.Now(), true, "b")); got != nil {
+		t.Fatalf("should not be full at 2")
+	}
+	full := b.Add(item("s", "h", time.Now(), true, "c"))
+	if full == nil {
+		t.Fatalf("should be full at 3")
+	}
+	if len(full.Items) != 3 {
+		t.Errorf("full batch has %d items", len(full.Items))
+	}
+	// After a full flush the subject batch is reset.
+	if b.Pending() != 0 {
+		t.Errorf("pending after flush = %d, want 0", b.Pending())
+	}
+}
+
+func TestFullByBytes(t *testing.T) {
+	b := New(Limits{MaxBytes: 10})
+	if b.Add(item("s", "h", time.Now(), true, "12345")) != nil {
+		t.Fatalf("5 bytes should not fill")
+	}
+	full := b.Add(item("s", "h", time.Now(), true, "67890"))
+	if full == nil {
+		t.Fatalf("10 bytes should fill")
+	}
+	if full.RawBytes != 10 {
+		t.Errorf("RawBytes = %d", full.RawBytes)
+	}
+}
+
+func TestSeparateSubjects(t *testing.T) {
+	b := New(Limits{MaxEvents: 2})
+	b.Add(item("a", "h", time.Now(), true, "x"))
+	b.Add(item("b", "h", time.Now(), true, "y"))
+	if b.Pending() != 2 {
+		t.Errorf("pending = %d, want 2 across subjects", b.Pending())
+	}
+	full := b.Add(item("a", "h", time.Now(), true, "z"))
+	if full == nil || full.Subject != "a" {
+		t.Fatalf("subject a should flush independently")
+	}
+	if b.Pending() != 1 {
+		t.Errorf("pending after a flush = %d, want 1 (subject b)", b.Pending())
+	}
+}
+
+func TestDueByAge(t *testing.T) {
+	base := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC)
+	b := New(Limits{MaxAge: time.Minute})
+	b.nowFn = func() time.Time { return base }
+	b.Add(item("s", "h", base, true, "x"))
+
+	if due := b.DueByAge(base.Add(30 * time.Second)); len(due) != 0 {
+		t.Fatalf("not due at 30s")
+	}
+	due := b.DueByAge(base.Add(90 * time.Second))
+	if len(due) != 1 {
+		t.Fatalf("should be due at 90s, got %d", len(due))
+	}
+	if b.Pending() != 0 {
+		t.Errorf("due batch not removed")
+	}
+}
+
+func TestDrain(t *testing.T) {
+	b := New(Limits{MaxEvents: 100})
+	b.Add(item("a", "h", time.Now(), true, "x"))
+	b.Add(item("b", "h", time.Now(), true, "y"))
+	all := b.Drain()
+	if len(all) != 2 {
+		t.Fatalf("drain returned %d, want 2", len(all))
+	}
+	if b.Pending() != 0 {
+		t.Errorf("pending after drain = %d", b.Pending())
+	}
+}
+
+func TestNDJSON(t *testing.T) {
+	b := &Batch{Subject: "s"}
+	b.Items = []Item{
+		{Raw: []byte(`{"a":1}`)},
+		{Raw: []byte(`{"b":2}` + "\n")}, // trailing newline trimmed and re-added
+	}
+	got := string(b.NDJSON())
+	want := "{\"a\":1}\n{\"b\":2}\n"
+	if got != want {
+		t.Errorf("NDJSON = %q, want %q", got, want)
+	}
+	if strings.Count(got, "\n") != 2 {
+		t.Errorf("expected exactly 2 newlines")
+	}
+}
+
+func TestSummarize(t *testing.T) {
+	t1 := time.Date(2026, 7, 27, 1, 0, 0, 0, time.UTC)
+	t2 := time.Date(2026, 7, 27, 3, 0, 0, 0, time.UTC)
+	fallback := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
+	b := &Batch{Subject: "s"}
+	b.Items = []Item{
+		item("s", "host-b", t2, true, "x"),
+		item("s", "host-a", t1, true, "y"),
+		item("s", "", time.Time{}, false, "z"), // no ts -> fallback, no host
+		item("s", "host-a", t1, true, "w"),     // dup host
+	}
+	s := b.Summarize(fallback)
+	if s.EventCount != 4 {
+		t.Errorf("EventCount = %d", s.EventCount)
+	}
+	if len(s.Hosts) != 2 || s.Hosts[0] != "host-a" || s.Hosts[1] != "host-b" {
+		t.Errorf("Hosts = %v, want sorted unique [host-a host-b]", s.Hosts)
+	}
+	if !s.MinTS.Equal(t1) {
+		t.Errorf("MinTS = %v, want %v", s.MinTS, t1)
+	}
+	// max should be the fallback (9:00) since event z used fallback which is latest
+	if !s.MaxTS.Equal(fallback) {
+		t.Errorf("MaxTS = %v, want fallback %v", s.MaxTS, fallback)
+	}
+}
+
+func TestSummarizeAllFallback(t *testing.T) {
+	fallback := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
+	b := &Batch{Items: []Item{{Raw: []byte("x")}}}
+	s := b.Summarize(fallback)
+	if !s.MinTS.Equal(fallback) || !s.MaxTS.Equal(fallback) {
+		t.Errorf("all-fallback range wrong: %v..%v", s.MinTS, s.MaxTS)
+	}
+	if s.HasTS {
+		t.Errorf("HasTS should be false")
+	}
+}