benvin
c05ccfcb5d
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
89 lines
2.6 KiB
Go
89 lines
2.6 KiB
Go
package event
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestExtractK8s(t *testing.T) {
|
|
raw := []byte(`{"message":"hello from pod","stream":"stdout","timestamp":"2026-07-27T00:00:00Z","kubernetes":{"pod_name":"web-abc","pod_namespace":"shop","container_name":"web","pod_node_name":"node-1"},"ns_token":"shop","cont_token":"web"}`)
|
|
m := Extract(raw)
|
|
if m.Host != "node-1" {
|
|
t.Errorf("host = %q, want node-1 (pod_node_name)", m.Host)
|
|
}
|
|
if !m.Ok {
|
|
t.Fatalf("timestamp not parsed")
|
|
}
|
|
if !m.Timestamp.Equal(time.Date(2026, 7, 27, 0, 0, 0, 0, time.UTC)) {
|
|
t.Errorf("timestamp = %v", m.Timestamp)
|
|
}
|
|
}
|
|
|
|
func TestExtractVMHostAndFallbacks(t *testing.T) {
|
|
raw := []byte(`{"message":"sshd started","host":"vm-db-1","severity":"info","role":"database","host_token":"vm-db-1"}`)
|
|
m := Extract(raw)
|
|
if m.Host != "vm-db-1" {
|
|
t.Errorf("host = %q, want vm-db-1", m.Host)
|
|
}
|
|
if m.Ok {
|
|
t.Errorf("no timestamp field present; Ok should be false")
|
|
}
|
|
}
|
|
|
|
func TestExtractHostnameFallback(t *testing.T) {
|
|
m := Extract([]byte(`{"hostname":"legacy-box","ts":"2026-01-02T03:04:05Z"}`))
|
|
if m.Host != "legacy-box" {
|
|
t.Errorf("host = %q, want legacy-box (hostname fallback)", m.Host)
|
|
}
|
|
if !m.Ok || !m.Timestamp.Equal(time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) {
|
|
t.Errorf("ts fallback failed: ok=%v ts=%v", m.Ok, m.Timestamp)
|
|
}
|
|
}
|
|
|
|
func TestExtractHostPrecedence(t *testing.T) {
|
|
// .host wins over .kubernetes.pod_node_name when both present.
|
|
m := Extract([]byte(`{"host":"explicit","kubernetes":{"pod_node_name":"node-x"}}`))
|
|
if m.Host != "explicit" {
|
|
t.Errorf("host precedence wrong: %q", m.Host)
|
|
}
|
|
}
|
|
|
|
func TestExtractNumericTimestamp(t *testing.T) {
|
|
// unix millis
|
|
m := Extract([]byte(`{"host":"h","timestamp":1769472000000}`))
|
|
if !m.Ok {
|
|
t.Fatalf("numeric millis not parsed")
|
|
}
|
|
if !m.Timestamp.Equal(time.Date(2026, 1, 27, 0, 0, 0, 0, time.UTC)) {
|
|
t.Errorf("numeric ts = %v", m.Timestamp.UTC())
|
|
}
|
|
}
|
|
|
|
func TestExtractMalformed(t *testing.T) {
|
|
m := Extract([]byte(`not json`))
|
|
if m.Host != "" || m.Ok {
|
|
t.Errorf("malformed event should yield zero Meta, got %+v", m)
|
|
}
|
|
}
|
|
|
|
func TestExtractEmptyHost(t *testing.T) {
|
|
m := Extract([]byte(`{"host":"","hostname":"backup"}`))
|
|
if m.Host != "backup" {
|
|
t.Errorf("empty host should fall through to hostname, got %q", m.Host)
|
|
}
|
|
}
|
|
|
|
func TestSubjectToken(t *testing.T) {
|
|
cases := map[string]string{
|
|
"logs.k8s.vault.audit": "logs.k8s.vault.audit",
|
|
"logs.vm.vm-db-1": "logs.vm.vm-db-1",
|
|
"logs.k8s.a/b.c": "logs.k8s.a_b.c",
|
|
"": "_",
|
|
}
|
|
for in, want := range cases {
|
|
if got := SubjectToken(in); got != want {
|
|
t.Errorf("SubjectToken(%q) = %q, want %q", in, got, want)
|
|
}
|
|
}
|
|
}
|