benvin
c05ccfcb5d
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
74 lines
2.0 KiB
Go
74 lines
2.0 KiB
Go
package archiver
|
|
|
|
import (
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"strings"
|
|
"text/template"
|
|
"time"
|
|
|
|
"git.unkin.net/unkin/logarchiver/internal/event"
|
|
)
|
|
|
|
// ObjectExt is the suffix for logarchiver container objects (zstd + framed
|
|
// AES-GCM + wrapped OpenPGP DEK). It is deliberately NOT .gz/.pgp because the
|
|
// object is a logarchiver-specific container, not a bare gpg file.
|
|
const ObjectExt = ".ndjson.zst.larc"
|
|
|
|
// KeyBuilder renders S3 object keys from a prefix template. Template fields:
|
|
// {{.Subject}} (sanitized), {{.Year}} {{.Month}} {{.Day}} (UTC, zero-padded).
|
|
type KeyBuilder struct {
|
|
tmpl *template.Template
|
|
}
|
|
|
|
// NewKeyBuilder compiles the prefix template.
|
|
func NewKeyBuilder(prefixTemplate string) (*KeyBuilder, error) {
|
|
t, err := template.New("key").Option("missingkey=error").Parse(prefixTemplate)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parse key_prefix template: %w", err)
|
|
}
|
|
return &KeyBuilder{tmpl: t}, nil
|
|
}
|
|
|
|
type keyData struct {
|
|
Subject string
|
|
Year string
|
|
Month string
|
|
Day string
|
|
}
|
|
|
|
// Build returns a unique object key for a batch of subject at ts. The filename
|
|
// is <UTCstamp>-<random>.ndjson.zst.larc so keys are collision-free and sortable.
|
|
func (k *KeyBuilder) Build(subject string, ts time.Time) (string, error) {
|
|
ts = ts.UTC()
|
|
var sb strings.Builder
|
|
err := k.tmpl.Execute(&sb, keyData{
|
|
Subject: event.SubjectToken(subject),
|
|
Year: fmt.Sprintf("%04d", ts.Year()),
|
|
Month: fmt.Sprintf("%02d", int(ts.Month())),
|
|
Day: fmt.Sprintf("%02d", ts.Day()),
|
|
})
|
|
if err != nil {
|
|
return "", fmt.Errorf("render key prefix: %w", err)
|
|
}
|
|
prefix := sb.String()
|
|
if prefix != "" && !strings.HasSuffix(prefix, "/") {
|
|
prefix += "/"
|
|
}
|
|
suffix, err := randHex(8)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
name := ts.Format("20060102T150405Z") + "-" + suffix + ObjectExt
|
|
return prefix + name, nil
|
|
}
|
|
|
|
func randHex(n int) (string, error) {
|
|
b := make([]byte, n)
|
|
if _, err := rand.Read(b); err != nil {
|
|
return "", fmt.Errorf("random: %w", err)
|
|
}
|
|
return hex.EncodeToString(b), nil
|
|
}
|