Initial implementation: NATS->S3 archiver + search/retrieve CLI
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
This commit is contained in:
benvin
@@ -0,0 +1,177 @@
|
||||
// Package batcher groups incoming log events into per-subject batches and
|
||||
// decides when a batch is ready to become one archived object. It is
|
||||
// deliberately not concurrent: the consumer run loop owns a Batcher and drives
|
||||
// it from a single goroutine (Add on receive, DueByAge on a ticker, Drain on
|
||||
// shutdown), which keeps the ack-after-persist accounting simple and race-free.
|
||||
package batcher
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Item is one log event routed into a batch. Ack is an opaque token (a
|
||||
// jetstream.Msg in production) that the caller acknowledges only after the batch
|
||||
// has been durably persisted.
|
||||
type Item struct {
|
||||
Subject string
|
||||
Raw []byte
|
||||
Host string
|
||||
Timestamp time.Time
|
||||
HasTS bool
|
||||
Ack any
|
||||
}
|
||||
|
||||
// Limits bound a single batch/object.
|
||||
type Limits struct {
|
||||
MaxBytes int64
|
||||
MaxEvents int
|
||||
MaxAge time.Duration
|
||||
}
|
||||
|
||||
// Batch is a ready (or in-progress) group of events for one subject.
|
||||
type Batch struct {
|
||||
Subject string
|
||||
Items []Item
|
||||
RawBytes int64
|
||||
OpenedAt time.Time
|
||||
}
|
||||
|
||||
// Batcher accumulates open batches keyed by subject.
|
||||
type Batcher struct {
|
||||
limits Limits
|
||||
open map[string]*Batch
|
||||
nowFn func() time.Time
|
||||
}
|
||||
|
||||
// New returns a Batcher enforcing limits.
|
||||
func New(limits Limits) *Batcher {
|
||||
return &Batcher{limits: limits, open: map[string]*Batch{}, nowFn: time.Now}
|
||||
}
|
||||
|
||||
// Add appends it to its subject's open batch. If that batch is now full (by
|
||||
// bytes or event count), it is removed from the open set and returned so the
|
||||
// caller can flush it; otherwise Add returns nil.
|
||||
func (b *Batcher) Add(it Item) *Batch {
|
||||
batch := b.open[it.Subject]
|
||||
if batch == nil {
|
||||
batch = &Batch{Subject: it.Subject, OpenedAt: b.nowFn()}
|
||||
b.open[it.Subject] = batch
|
||||
}
|
||||
batch.Items = append(batch.Items, it)
|
||||
batch.RawBytes += int64(len(it.Raw))
|
||||
|
||||
if b.full(batch) {
|
||||
delete(b.open, it.Subject)
|
||||
return batch
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Batcher) full(batch *Batch) bool {
|
||||
if b.limits.MaxBytes > 0 && batch.RawBytes >= b.limits.MaxBytes {
|
||||
return true
|
||||
}
|
||||
if b.limits.MaxEvents > 0 && len(batch.Items) >= b.limits.MaxEvents {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// DueByAge removes and returns every open batch older than MaxAge as of now.
|
||||
func (b *Batcher) DueByAge(now time.Time) []*Batch {
|
||||
if b.limits.MaxAge <= 0 {
|
||||
return nil
|
||||
}
|
||||
var due []*Batch
|
||||
for subj, batch := range b.open {
|
||||
if now.Sub(batch.OpenedAt) >= b.limits.MaxAge {
|
||||
due = append(due, batch)
|
||||
delete(b.open, subj)
|
||||
}
|
||||
}
|
||||
sortBatches(due)
|
||||
return due
|
||||
}
|
||||
|
||||
// Drain removes and returns all open batches (used on graceful shutdown so
|
||||
// in-flight events are persisted and acked before exit).
|
||||
func (b *Batcher) Drain() []*Batch {
|
||||
var all []*Batch
|
||||
for subj, batch := range b.open {
|
||||
all = append(all, batch)
|
||||
delete(b.open, subj)
|
||||
}
|
||||
sortBatches(all)
|
||||
return all
|
||||
}
|
||||
|
||||
// Pending reports how many events sit in open batches.
|
||||
func (b *Batcher) Pending() int {
|
||||
n := 0
|
||||
for _, batch := range b.open {
|
||||
n += len(batch.Items)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func sortBatches(bs []*Batch) {
|
||||
sort.Slice(bs, func(i, j int) bool { return bs[i].Subject < bs[j].Subject })
|
||||
}
|
||||
|
||||
// NDJSON renders the batch as newline-delimited JSON (one raw event per line),
|
||||
// matching the raw archive format the logging stack expects.
|
||||
func (b *Batch) NDJSON() []byte {
|
||||
var buf bytes.Buffer
|
||||
buf.Grow(int(b.RawBytes) + len(b.Items))
|
||||
for _, it := range b.Items {
|
||||
buf.Write(bytes.TrimRight(it.Raw, "\n"))
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// Summary is the index-relevant projection of a batch.
|
||||
type Summary struct {
|
||||
Hosts []string
|
||||
MinTS time.Time
|
||||
MaxTS time.Time
|
||||
EventCount int
|
||||
HasTS bool
|
||||
}
|
||||
|
||||
// Summarize computes hosts (unique, sorted) and the timestamp range. fallback is
|
||||
// used for events whose payload lacked a parseable timestamp (ingest time).
|
||||
func (b *Batch) Summarize(fallback time.Time) Summary {
|
||||
s := Summary{EventCount: len(b.Items)}
|
||||
hostSet := map[string]struct{}{}
|
||||
for _, it := range b.Items {
|
||||
if it.Host != "" {
|
||||
hostSet[it.Host] = struct{}{}
|
||||
}
|
||||
ts := it.Timestamp
|
||||
if !it.HasTS {
|
||||
ts = fallback
|
||||
} else {
|
||||
s.HasTS = true
|
||||
}
|
||||
if s.MinTS.IsZero() || ts.Before(s.MinTS) {
|
||||
s.MinTS = ts
|
||||
}
|
||||
if s.MaxTS.IsZero() || ts.After(s.MaxTS) {
|
||||
s.MaxTS = ts
|
||||
}
|
||||
}
|
||||
if s.MinTS.IsZero() {
|
||||
s.MinTS = fallback
|
||||
}
|
||||
if s.MaxTS.IsZero() {
|
||||
s.MaxTS = fallback
|
||||
}
|
||||
for h := range hostSet {
|
||||
s.Hosts = append(s.Hosts, h)
|
||||
}
|
||||
sort.Strings(s.Hosts)
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package batcher
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func item(subject, host string, ts time.Time, hasTS bool, raw string) Item {
|
||||
return Item{Subject: subject, Host: host, Timestamp: ts, HasTS: hasTS, Raw: []byte(raw)}
|
||||
}
|
||||
|
||||
func TestFullByEvents(t *testing.T) {
|
||||
b := New(Limits{MaxEvents: 3})
|
||||
if got := b.Add(item("s", "h", time.Now(), true, "a")); got != nil {
|
||||
t.Fatalf("should not be full at 1")
|
||||
}
|
||||
if got := b.Add(item("s", "h", time.Now(), true, "b")); got != nil {
|
||||
t.Fatalf("should not be full at 2")
|
||||
}
|
||||
full := b.Add(item("s", "h", time.Now(), true, "c"))
|
||||
if full == nil {
|
||||
t.Fatalf("should be full at 3")
|
||||
}
|
||||
if len(full.Items) != 3 {
|
||||
t.Errorf("full batch has %d items", len(full.Items))
|
||||
}
|
||||
// After a full flush the subject batch is reset.
|
||||
if b.Pending() != 0 {
|
||||
t.Errorf("pending after flush = %d, want 0", b.Pending())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFullByBytes(t *testing.T) {
|
||||
b := New(Limits{MaxBytes: 10})
|
||||
if b.Add(item("s", "h", time.Now(), true, "12345")) != nil {
|
||||
t.Fatalf("5 bytes should not fill")
|
||||
}
|
||||
full := b.Add(item("s", "h", time.Now(), true, "67890"))
|
||||
if full == nil {
|
||||
t.Fatalf("10 bytes should fill")
|
||||
}
|
||||
if full.RawBytes != 10 {
|
||||
t.Errorf("RawBytes = %d", full.RawBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeparateSubjects(t *testing.T) {
|
||||
b := New(Limits{MaxEvents: 2})
|
||||
b.Add(item("a", "h", time.Now(), true, "x"))
|
||||
b.Add(item("b", "h", time.Now(), true, "y"))
|
||||
if b.Pending() != 2 {
|
||||
t.Errorf("pending = %d, want 2 across subjects", b.Pending())
|
||||
}
|
||||
full := b.Add(item("a", "h", time.Now(), true, "z"))
|
||||
if full == nil || full.Subject != "a" {
|
||||
t.Fatalf("subject a should flush independently")
|
||||
}
|
||||
if b.Pending() != 1 {
|
||||
t.Errorf("pending after a flush = %d, want 1 (subject b)", b.Pending())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDueByAge(t *testing.T) {
|
||||
base := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC)
|
||||
b := New(Limits{MaxAge: time.Minute})
|
||||
b.nowFn = func() time.Time { return base }
|
||||
b.Add(item("s", "h", base, true, "x"))
|
||||
|
||||
if due := b.DueByAge(base.Add(30 * time.Second)); len(due) != 0 {
|
||||
t.Fatalf("not due at 30s")
|
||||
}
|
||||
due := b.DueByAge(base.Add(90 * time.Second))
|
||||
if len(due) != 1 {
|
||||
t.Fatalf("should be due at 90s, got %d", len(due))
|
||||
}
|
||||
if b.Pending() != 0 {
|
||||
t.Errorf("due batch not removed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDrain(t *testing.T) {
|
||||
b := New(Limits{MaxEvents: 100})
|
||||
b.Add(item("a", "h", time.Now(), true, "x"))
|
||||
b.Add(item("b", "h", time.Now(), true, "y"))
|
||||
all := b.Drain()
|
||||
if len(all) != 2 {
|
||||
t.Fatalf("drain returned %d, want 2", len(all))
|
||||
}
|
||||
if b.Pending() != 0 {
|
||||
t.Errorf("pending after drain = %d", b.Pending())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNDJSON(t *testing.T) {
|
||||
b := &Batch{Subject: "s"}
|
||||
b.Items = []Item{
|
||||
{Raw: []byte(`{"a":1}`)},
|
||||
{Raw: []byte(`{"b":2}` + "\n")}, // trailing newline trimmed and re-added
|
||||
}
|
||||
got := string(b.NDJSON())
|
||||
want := "{\"a\":1}\n{\"b\":2}\n"
|
||||
if got != want {
|
||||
t.Errorf("NDJSON = %q, want %q", got, want)
|
||||
}
|
||||
if strings.Count(got, "\n") != 2 {
|
||||
t.Errorf("expected exactly 2 newlines")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarize(t *testing.T) {
|
||||
t1 := time.Date(2026, 7, 27, 1, 0, 0, 0, time.UTC)
|
||||
t2 := time.Date(2026, 7, 27, 3, 0, 0, 0, time.UTC)
|
||||
fallback := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
|
||||
b := &Batch{Subject: "s"}
|
||||
b.Items = []Item{
|
||||
item("s", "host-b", t2, true, "x"),
|
||||
item("s", "host-a", t1, true, "y"),
|
||||
item("s", "", time.Time{}, false, "z"), // no ts -> fallback, no host
|
||||
item("s", "host-a", t1, true, "w"), // dup host
|
||||
}
|
||||
s := b.Summarize(fallback)
|
||||
if s.EventCount != 4 {
|
||||
t.Errorf("EventCount = %d", s.EventCount)
|
||||
}
|
||||
if len(s.Hosts) != 2 || s.Hosts[0] != "host-a" || s.Hosts[1] != "host-b" {
|
||||
t.Errorf("Hosts = %v, want sorted unique [host-a host-b]", s.Hosts)
|
||||
}
|
||||
if !s.MinTS.Equal(t1) {
|
||||
t.Errorf("MinTS = %v, want %v", s.MinTS, t1)
|
||||
}
|
||||
// max should be the fallback (9:00) since event z used fallback which is latest
|
||||
if !s.MaxTS.Equal(fallback) {
|
||||
t.Errorf("MaxTS = %v, want fallback %v", s.MaxTS, fallback)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarizeAllFallback(t *testing.T) {
|
||||
fallback := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
|
||||
b := &Batch{Items: []Item{{Raw: []byte("x")}}}
|
||||
s := b.Summarize(fallback)
|
||||
if !s.MinTS.Equal(fallback) || !s.MaxTS.Equal(fallback) {
|
||||
t.Errorf("all-fallback range wrong: %v..%v", s.MinTS, s.MaxTS)
|
||||
}
|
||||
if s.HasTS {
|
||||
t.Errorf("HasTS should be false")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user