Initial implementation: NATS->S3 archiver + search/retrieve CLI

logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
2026-07-27 22:11:54 +10:00
parent d036d31f12
commit c05ccfcb5d
58 changed files with 5946 additions and 1 deletions
@@ -0,0 +1,182 @@
+// Package event extracts the fields logarchiver needs (host, timestamp) from a
+// raw log event as it flows through the centralized logging JetStream stream
+// (argocd-apps #296). Events are one JSON object per NATS message and are NOT
+// normalized — logarchiver persists them raw, so extraction must be tolerant of
+// the two shapes that share the logs.> subject space:
+//
+//   - k8s pod logs (Vector kubernetes_logs): host lives at .kubernetes.pod_node_name,
+//     timestamp at .timestamp (RFC3339).
+//   - VM logs (vm-ingest): host at .host (fallback .hostname), timestamp at
+//     .timestamp (fallback .ts).
+//
+// The NATS subject itself is authoritative for partitioning and is supplied by
+// the consumer, not read from the payload.
+package event
+
+import (
+	"encoding/json"
+	"strings"
+	"time"
+)
+
+// Meta is the minimal, index-relevant projection of a raw log event.
+type Meta struct {
+	// Host is the best-effort source host/node for the event, or "" if none
+	// could be determined.
+	Host string
+	// Timestamp is the event time. Ok reports whether a timestamp field was
+	// found and parsed; when false callers should fall back to ingest time.
+	Timestamp time.Time
+	Ok        bool
+}
+
+// hostPaths and tsPaths are tried in order. Dotted paths descend into nested
+// objects (only .kubernetes.pod_node_name is nested today).
+var (
+	hostPaths = [][]string{
+		{"host"},
+		{"hostname"},
+		{"kubernetes", "pod_node_name"},
+	}
+	tsPaths = [][]string{
+		{"timestamp"},
+		{"ts"},
+		{"@timestamp"},
+	}
+)
+
+// Extract parses raw (a single JSON log event) and returns its host/timestamp
+// projection. It never errors: malformed or field-less events yield a zero-value
+// Meta (Host=="", Ok==false) so the archiver still stores the raw bytes and the
+// caller can fall back to ingest time. Only the fields of interest are decoded.
+func Extract(raw []byte) Meta {
+	var doc map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &doc); err != nil {
+		return Meta{}
+	}
+	m := Meta{}
+	m.Host = firstString(doc, hostPaths)
+	if ts, ok := firstTime(doc, tsPaths); ok {
+		m.Timestamp = ts
+		m.Ok = true
+	}
+	return m
+}
+
+// firstString walks each path and returns the first value that decodes to a
+// non-empty string.
+func firstString(doc map[string]json.RawMessage, paths [][]string) string {
+	for _, p := range paths {
+		if v, ok := lookup(doc, p); ok {
+			var s string
+			if json.Unmarshal(v, &s) == nil && s != "" {
+				return s
+			}
+		}
+	}
+	return ""
+}
+
+// firstTime walks each path and returns the first value that parses as a
+// timestamp (RFC3339/RFC3339Nano string, or a numeric unix seconds/millis).
+func firstTime(doc map[string]json.RawMessage, paths [][]string) (time.Time, bool) {
+	for _, p := range paths {
+		v, ok := lookup(doc, p)
+		if !ok {
+			continue
+		}
+		var s string
+		if json.Unmarshal(v, &s) == nil && s != "" {
+			if t, err := parseTimeString(s); err == nil {
+				return t, true
+			}
+		}
+		var n json.Number
+		if json.Unmarshal(v, &n) == nil {
+			if t, ok := parseNumericTime(n); ok {
+				return t, true
+			}
+		}
+	}
+	return time.Time{}, false
+}
+
+// lookup descends doc following path. Intermediate elements must be JSON objects.
+func lookup(doc map[string]json.RawMessage, path []string) (json.RawMessage, bool) {
+	cur := doc
+	for i, key := range path {
+		v, ok := cur[key]
+		if !ok {
+			return nil, false
+		}
+		if i == len(path)-1 {
+			return v, true
+		}
+		var next map[string]json.RawMessage
+		if json.Unmarshal(v, &next) != nil {
+			return nil, false
+		}
+		cur = next
+	}
+	return nil, false
+}
+
+var timeLayouts = []string{
+	time.RFC3339Nano,
+	time.RFC3339,
+	"2006-01-02T15:04:05.999999999Z0700",
+	"2006-01-02 15:04:05.999999999Z07:00",
+	"2006-01-02 15:04:05",
+}
+
+func parseTimeString(s string) (time.Time, error) {
+	var lastErr error
+	for _, l := range timeLayouts {
+		t, err := time.Parse(l, s)
+		if err == nil {
+			return t.UTC(), nil
+		}
+		lastErr = err
+	}
+	return time.Time{}, lastErr
+}
+
+// parseNumericTime interprets n as unix seconds, milliseconds, microseconds, or
+// nanoseconds based on magnitude. Fractional seconds are supported.
+func parseNumericTime(n json.Number) (time.Time, bool) {
+	f, err := n.Float64()
+	if err != nil || f <= 0 {
+		return time.Time{}, false
+	}
+	switch {
+	case f >= 1e18: // nanoseconds
+		return time.Unix(0, int64(f)).UTC(), true
+	case f >= 1e15: // microseconds
+		return time.Unix(0, int64(f*1e3)).UTC(), true
+	case f >= 1e12: // milliseconds
+		return time.Unix(0, int64(f*1e6)).UTC(), true
+	default: // seconds (possibly fractional)
+		sec := int64(f)
+		nsec := int64((f - float64(sec)) * 1e9)
+		return time.Unix(sec, nsec).UTC(), true
+	}
+}
+
+// SubjectToken sanitizes a NATS subject into a filesystem/object-key-safe token,
+// matching the logging stack's convention of replacing [^a-zA-Z0-9_.-] with '_'.
+// Dots are preserved because subjects are dot-delimited.
+func SubjectToken(subject string) string {
+	if subject == "" {
+		return "_"
+	}
+	var b strings.Builder
+	for _, r := range subject {
+		switch {
+		case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '_', r == '.', r == '-':
+			b.WriteRune(r)
+		default:
+			b.WriteRune('_')
+		}
+	}
+	return b.String()
+}
@@ -0,0 +1,88 @@
+package event
+
+import (
+	"testing"
+	"time"
+)
+
+func TestExtractK8s(t *testing.T) {
+	raw := []byte(`{"message":"hello from pod","stream":"stdout","timestamp":"2026-07-27T00:00:00Z","kubernetes":{"pod_name":"web-abc","pod_namespace":"shop","container_name":"web","pod_node_name":"node-1"},"ns_token":"shop","cont_token":"web"}`)
+	m := Extract(raw)
+	if m.Host != "node-1" {
+		t.Errorf("host = %q, want node-1 (pod_node_name)", m.Host)
+	}
+	if !m.Ok {
+		t.Fatalf("timestamp not parsed")
+	}
+	if !m.Timestamp.Equal(time.Date(2026, 7, 27, 0, 0, 0, 0, time.UTC)) {
+		t.Errorf("timestamp = %v", m.Timestamp)
+	}
+}
+
+func TestExtractVMHostAndFallbacks(t *testing.T) {
+	raw := []byte(`{"message":"sshd started","host":"vm-db-1","severity":"info","role":"database","host_token":"vm-db-1"}`)
+	m := Extract(raw)
+	if m.Host != "vm-db-1" {
+		t.Errorf("host = %q, want vm-db-1", m.Host)
+	}
+	if m.Ok {
+		t.Errorf("no timestamp field present; Ok should be false")
+	}
+}
+
+func TestExtractHostnameFallback(t *testing.T) {
+	m := Extract([]byte(`{"hostname":"legacy-box","ts":"2026-01-02T03:04:05Z"}`))
+	if m.Host != "legacy-box" {
+		t.Errorf("host = %q, want legacy-box (hostname fallback)", m.Host)
+	}
+	if !m.Ok || !m.Timestamp.Equal(time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) {
+		t.Errorf("ts fallback failed: ok=%v ts=%v", m.Ok, m.Timestamp)
+	}
+}
+
+func TestExtractHostPrecedence(t *testing.T) {
+	// .host wins over .kubernetes.pod_node_name when both present.
+	m := Extract([]byte(`{"host":"explicit","kubernetes":{"pod_node_name":"node-x"}}`))
+	if m.Host != "explicit" {
+		t.Errorf("host precedence wrong: %q", m.Host)
+	}
+}
+
+func TestExtractNumericTimestamp(t *testing.T) {
+	// unix millis
+	m := Extract([]byte(`{"host":"h","timestamp":1769472000000}`))
+	if !m.Ok {
+		t.Fatalf("numeric millis not parsed")
+	}
+	if !m.Timestamp.Equal(time.Date(2026, 1, 27, 0, 0, 0, 0, time.UTC)) {
+		t.Errorf("numeric ts = %v", m.Timestamp.UTC())
+	}
+}
+
+func TestExtractMalformed(t *testing.T) {
+	m := Extract([]byte(`not json`))
+	if m.Host != "" || m.Ok {
+		t.Errorf("malformed event should yield zero Meta, got %+v", m)
+	}
+}
+
+func TestExtractEmptyHost(t *testing.T) {
+	m := Extract([]byte(`{"host":"","hostname":"backup"}`))
+	if m.Host != "backup" {
+		t.Errorf("empty host should fall through to hostname, got %q", m.Host)
+	}
+}
+
+func TestSubjectToken(t *testing.T) {
+	cases := map[string]string{
+		"logs.k8s.vault.audit": "logs.k8s.vault.audit",
+		"logs.vm.vm-db-1":      "logs.vm.vm-db-1",
+		"logs.k8s.a/b.c":       "logs.k8s.a_b.c",
+		"":                     "_",
+	}
+	for in, want := range cases {
+		if got := SubjectToken(in); got != want {
+			t.Errorf("SubjectToken(%q) = %q, want %q", in, got, want)
+		}
+	}
+}