5ec89b0028
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
152 lines
4.2 KiB
Go
152 lines
4.2 KiB
Go
package consumer
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.unkin.net/unkin/logarchiver/internal/batcher"
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
)
|
|
|
|
// fakeMsg is a minimal jetstream.Msg recording ack/nak calls.
|
|
type fakeMsg struct {
|
|
subject string
|
|
data []byte
|
|
mu sync.Mutex
|
|
acked bool
|
|
naked bool
|
|
}
|
|
|
|
func (m *fakeMsg) Metadata() (*jetstream.MsgMetadata, error) { return &jetstream.MsgMetadata{}, nil }
|
|
func (m *fakeMsg) Data() []byte { return m.data }
|
|
func (m *fakeMsg) Headers() nats.Header { return nil }
|
|
func (m *fakeMsg) Subject() string { return m.subject }
|
|
func (m *fakeMsg) Reply() string { return "" }
|
|
func (m *fakeMsg) Ack() error {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
m.acked = true
|
|
return nil
|
|
}
|
|
func (m *fakeMsg) DoubleAck(context.Context) error { return nil }
|
|
func (m *fakeMsg) Nak() error {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
m.naked = true
|
|
return nil
|
|
}
|
|
func (m *fakeMsg) NakWithDelay(time.Duration) error {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
m.naked = true
|
|
return nil
|
|
}
|
|
func (m *fakeMsg) InProgress() error { return nil }
|
|
func (m *fakeMsg) Term() error { return nil }
|
|
func (m *fakeMsg) TermWithReason(string) error { return nil }
|
|
|
|
func (m *fakeMsg) isAcked() bool {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
return m.acked
|
|
}
|
|
func (m *fakeMsg) isNaked() bool {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
return m.naked
|
|
}
|
|
|
|
// fakePersister records calls and can be made to fail.
|
|
type fakePersister struct {
|
|
fail bool
|
|
called int
|
|
}
|
|
|
|
func (p *fakePersister) Store(_ context.Context, b *batcher.Batch) (StoreResult, error) {
|
|
p.called++
|
|
if p.fail {
|
|
return StoreResult{}, errors.New("boom")
|
|
}
|
|
return StoreResult{ObjectKey: "k", Events: len(b.Items)}, nil
|
|
}
|
|
|
|
func batchWith(msgs ...*fakeMsg) *batcher.Batch {
|
|
b := &batcher.Batch{Subject: "s"}
|
|
for _, m := range msgs {
|
|
b.Items = append(b.Items, batcher.Item{Subject: "s", Raw: m.data, Ack: jetstream.Msg(m)})
|
|
}
|
|
return b
|
|
}
|
|
|
|
// TestFlushAcksOnlyAfterPersist is the core correctness test: messages are acked
|
|
// exactly when Store succeeds, and Nak'd (never acked) when it fails.
|
|
func TestFlushAcksOnSuccess(t *testing.T) {
|
|
p := &fakePersister{}
|
|
r := NewRunner(Options{Persister: p})
|
|
m1 := &fakeMsg{subject: "s", data: []byte(`{"host":"h"}`)}
|
|
m2 := &fakeMsg{subject: "s", data: []byte(`{"host":"h2"}`)}
|
|
|
|
r.flush(context.Background(), batchWith(m1, m2), "test")
|
|
|
|
if p.called != 1 {
|
|
t.Fatalf("Store called %d times, want 1", p.called)
|
|
}
|
|
if !m1.isAcked() || !m2.isAcked() {
|
|
t.Errorf("messages should be acked after successful persist")
|
|
}
|
|
if m1.isNaked() || m2.isNaked() {
|
|
t.Errorf("messages must not be naked on success")
|
|
}
|
|
}
|
|
|
|
func TestFlushNaksOnFailure(t *testing.T) {
|
|
p := &fakePersister{fail: true}
|
|
r := NewRunner(Options{Persister: p})
|
|
m1 := &fakeMsg{subject: "s", data: []byte(`{"host":"h"}`)}
|
|
|
|
r.flush(context.Background(), batchWith(m1), "test")
|
|
|
|
if m1.isAcked() {
|
|
t.Errorf("message must NOT be acked when persist fails")
|
|
}
|
|
if !m1.isNaked() {
|
|
t.Errorf("message should be naked so JetStream redelivers")
|
|
}
|
|
}
|
|
|
|
func TestFlushEmptyBatchNoop(t *testing.T) {
|
|
p := &fakePersister{}
|
|
r := NewRunner(Options{Persister: p})
|
|
r.flush(context.Background(), &batcher.Batch{Subject: "s"}, "test")
|
|
if p.called != 0 {
|
|
t.Errorf("empty batch should not call Store")
|
|
}
|
|
}
|
|
|
|
// TestRouteAndFlushIntegration wires a real batcher: adding enough messages to
|
|
// fill the batch triggers a full flush that persists and acks exactly those.
|
|
func TestRouteFlushViaBatcher(t *testing.T) {
|
|
p := &fakePersister{}
|
|
bat := batcher.New(batcher.Limits{MaxEvents: 2})
|
|
r := NewRunner(Options{Persister: p, Batcher: bat})
|
|
|
|
m1 := &fakeMsg{subject: "s", data: []byte(`{"host":"a"}`)}
|
|
m2 := &fakeMsg{subject: "s", data: []byte(`{"host":"b"}`)}
|
|
r.route(context.Background(), m1)
|
|
if m1.isAcked() {
|
|
t.Errorf("first message should not be acked before batch fills")
|
|
}
|
|
r.route(context.Background(), m2) // fills batch -> flush
|
|
|
|
if p.called != 1 {
|
|
t.Fatalf("Store called %d times, want 1 after fill", p.called)
|
|
}
|
|
if !m1.isAcked() || !m2.isAcked() {
|
|
t.Errorf("both messages should be acked after the full-batch flush")
|
|
}
|
|
}
|