Files
logarchiver/internal/batcher/batcher_test.go
T
benvin c05ccfcb5d
ci/woodpecker/pr/build Pipeline was successful
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
Initial implementation: NATS->S3 archiver + search/retrieve CLI
logarchiver replaces the plain Vector archiver leg of the centralized
logging stack (argocd-apps #296) with a Go service that archives raw logs
from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed
objects, plus an operator CLI to search the index and retrieve/decrypt
archived logs. It adds the things that outgrew Vector: zstd compression,
encryption keyed from Ben's Vault GPG secrets engine, a searchable
ClickHouse index, and sink-conditional acks (a batch is acknowledged to
JetStream only after the object is durably in S3 AND indexed).

Service (`logarchiver run`):
- Durable JetStream pull consumer (stream LOGS, durable archiver, subject
  filter default logs.k8s.vault.>), explicit acks, independent offsets.
- Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3
  PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and
  redelivered, so nothing is lost on a sink outage.
- Encryption is a wrapped-DEK envelope (container LARC1): the bulk is
  AES-256-GCM framed under a random data key, and only that 32-byte key is
  OpenPGP-encrypted to the engine's public key. This is because the Vault
  GPG engine does whole-payload decrypt only; retrieval round-trips just the
  tiny wrapped key regardless of object size. Public key fetched from the
  engine or a mounted file (configurable); key fingerprint recorded per
  object; periodic pubkey refresh for rotation.
- Prometheus metrics, structured slog, graceful drain on shutdown.

CLI:
- `search` queries the index (subject/host/time) and lists matching objects.
- `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits
  NDJSON (optionally re-filtered by host/time).
- `init-schema` creates/prints the ClickHouse archive_index DDL.
- cobra `completion` subcommands.

Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/
ClickHouse/Vault) behind interfaces with unit tests (config, batching,
host/subject extraction, crypto roundtrip with a test key, ack-after-persist
with fakes, search query building). go build/vet/test -race clean;
golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v*
tag a container image plus a Gitea binary release + rpm-internal RPM. Docs
per subcommand + architecture + retrieval runbook + deployment drop-in.

Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
2026-07-27 23:22:40 +10:00

149 lines
4.2 KiB
Go

package batcher
import (
"strings"
"testing"
"time"
)
func item(subject, host string, ts time.Time, hasTS bool, raw string) Item {
return Item{Subject: subject, Host: host, Timestamp: ts, HasTS: hasTS, Raw: []byte(raw)}
}
func TestFullByEvents(t *testing.T) {
b := New(Limits{MaxEvents: 3})
if got := b.Add(item("s", "h", time.Now(), true, "a")); got != nil {
t.Fatalf("should not be full at 1")
}
if got := b.Add(item("s", "h", time.Now(), true, "b")); got != nil {
t.Fatalf("should not be full at 2")
}
full := b.Add(item("s", "h", time.Now(), true, "c"))
if full == nil {
t.Fatalf("should be full at 3")
}
if len(full.Items) != 3 {
t.Errorf("full batch has %d items", len(full.Items))
}
// After a full flush the subject batch is reset.
if b.Pending() != 0 {
t.Errorf("pending after flush = %d, want 0", b.Pending())
}
}
func TestFullByBytes(t *testing.T) {
b := New(Limits{MaxBytes: 10})
if b.Add(item("s", "h", time.Now(), true, "12345")) != nil {
t.Fatalf("5 bytes should not fill")
}
full := b.Add(item("s", "h", time.Now(), true, "67890"))
if full == nil {
t.Fatalf("10 bytes should fill")
}
if full.RawBytes != 10 {
t.Errorf("RawBytes = %d", full.RawBytes)
}
}
func TestSeparateSubjects(t *testing.T) {
b := New(Limits{MaxEvents: 2})
b.Add(item("a", "h", time.Now(), true, "x"))
b.Add(item("b", "h", time.Now(), true, "y"))
if b.Pending() != 2 {
t.Errorf("pending = %d, want 2 across subjects", b.Pending())
}
full := b.Add(item("a", "h", time.Now(), true, "z"))
if full == nil || full.Subject != "a" {
t.Fatalf("subject a should flush independently")
}
if b.Pending() != 1 {
t.Errorf("pending after a flush = %d, want 1 (subject b)", b.Pending())
}
}
func TestDueByAge(t *testing.T) {
base := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC)
b := New(Limits{MaxAge: time.Minute})
b.nowFn = func() time.Time { return base }
b.Add(item("s", "h", base, true, "x"))
if due := b.DueByAge(base.Add(30 * time.Second)); len(due) != 0 {
t.Fatalf("not due at 30s")
}
due := b.DueByAge(base.Add(90 * time.Second))
if len(due) != 1 {
t.Fatalf("should be due at 90s, got %d", len(due))
}
if b.Pending() != 0 {
t.Errorf("due batch not removed")
}
}
func TestDrain(t *testing.T) {
b := New(Limits{MaxEvents: 100})
b.Add(item("a", "h", time.Now(), true, "x"))
b.Add(item("b", "h", time.Now(), true, "y"))
all := b.Drain()
if len(all) != 2 {
t.Fatalf("drain returned %d, want 2", len(all))
}
if b.Pending() != 0 {
t.Errorf("pending after drain = %d", b.Pending())
}
}
func TestNDJSON(t *testing.T) {
b := &Batch{Subject: "s"}
b.Items = []Item{
{Raw: []byte(`{"a":1}`)},
{Raw: []byte(`{"b":2}` + "\n")}, // trailing newline trimmed and re-added
}
got := string(b.NDJSON())
want := "{\"a\":1}\n{\"b\":2}\n"
if got != want {
t.Errorf("NDJSON = %q, want %q", got, want)
}
if strings.Count(got, "\n") != 2 {
t.Errorf("expected exactly 2 newlines")
}
}
func TestSummarize(t *testing.T) {
t1 := time.Date(2026, 7, 27, 1, 0, 0, 0, time.UTC)
t2 := time.Date(2026, 7, 27, 3, 0, 0, 0, time.UTC)
fallback := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
b := &Batch{Subject: "s"}
b.Items = []Item{
item("s", "host-b", t2, true, "x"),
item("s", "host-a", t1, true, "y"),
item("s", "", time.Time{}, false, "z"), // no ts -> fallback, no host
item("s", "host-a", t1, true, "w"), // dup host
}
s := b.Summarize(fallback)
if s.EventCount != 4 {
t.Errorf("EventCount = %d", s.EventCount)
}
if len(s.Hosts) != 2 || s.Hosts[0] != "host-a" || s.Hosts[1] != "host-b" {
t.Errorf("Hosts = %v, want sorted unique [host-a host-b]", s.Hosts)
}
if !s.MinTS.Equal(t1) {
t.Errorf("MinTS = %v, want %v", s.MinTS, t1)
}
// max should be the fallback (9:00) since event z used fallback which is latest
if !s.MaxTS.Equal(fallback) {
t.Errorf("MaxTS = %v, want fallback %v", s.MaxTS, fallback)
}
}
func TestSummarizeAllFallback(t *testing.T) {
fallback := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC)
b := &Batch{Items: []Item{{Raw: []byte("x")}}}
s := b.Summarize(fallback)
if !s.MinTS.Equal(fallback) || !s.MaxTS.Equal(fallback) {
t.Errorf("all-fallback range wrong: %v..%v", s.MinTS, s.MaxTS)
}
if s.HasTS {
t.Errorf("HasTS should be false")
}
}