Initial implementation: NATS->S3 archiver + search/retrieve CLI
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
This commit is contained in:
@@ -0,0 +1,95 @@
|
||||
// Package cli implements the logarchiver command tree (service + operator CLI)
|
||||
// using cobra, which also provides the `completion` subcommand the estate's
|
||||
// nfpm packaging installs.
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/logarchiver/internal/config"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// version is set at build time via -ldflags "-X ...cli.version=...".
|
||||
var version = "dev"
|
||||
|
||||
// SetVersion lets main inject the linker-provided version string.
|
||||
func SetVersion(v string) {
|
||||
if v != "" {
|
||||
version = v
|
||||
}
|
||||
}
|
||||
|
||||
var configPath string
|
||||
|
||||
// NewRootCmd builds the root command.
|
||||
func NewRootCmd() *cobra.Command {
|
||||
root := &cobra.Command{
|
||||
Use: "logarchiver",
|
||||
Short: "Archive NATS JetStream logs to S3 (zstd + OpenPGP) and search/retrieve them",
|
||||
Long: `logarchiver archives raw logs from the centralized logging JetStream stream
|
||||
to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, and provides a
|
||||
CLI to search the index and retrieve/decrypt archived logs.`,
|
||||
SilenceUsage: true,
|
||||
SilenceErrors: true,
|
||||
}
|
||||
root.PersistentFlags().StringVarP(&configPath, "config", "c", os.Getenv("LOGARCHIVER_CONFIG"),
|
||||
"path to config file (env LOGARCHIVER_CONFIG)")
|
||||
|
||||
root.AddCommand(
|
||||
newRunCmd(),
|
||||
newSearchCmd(),
|
||||
newFetchCmd(),
|
||||
newInitSchemaCmd(),
|
||||
newVersionCmd(),
|
||||
)
|
||||
return root
|
||||
}
|
||||
|
||||
// Execute runs the root command.
|
||||
func Execute() int {
|
||||
if err := NewRootCmd().Execute(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func newVersionCmd() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "Print the version",
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
_, _ = fmt.Fprintln(cmd.OutOrStdout(), version)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// loadConfig loads config from the --config path (or defaults+env).
|
||||
func loadConfig() (config.Config, error) {
|
||||
return config.Load(configPath)
|
||||
}
|
||||
|
||||
// newLogger builds a slog logger from config.
|
||||
func newLogger(cfg config.LogConfig) *slog.Logger {
|
||||
level := slog.LevelInfo
|
||||
switch strings.ToLower(cfg.Level) {
|
||||
case "debug":
|
||||
level = slog.LevelDebug
|
||||
case "warn":
|
||||
level = slog.LevelWarn
|
||||
case "error":
|
||||
level = slog.LevelError
|
||||
}
|
||||
opts := &slog.HandlerOptions{Level: level}
|
||||
var h slog.Handler
|
||||
if strings.ToLower(cfg.Format) == "text" {
|
||||
h = slog.NewTextHandler(os.Stderr, opts)
|
||||
} else {
|
||||
h = slog.NewJSONHandler(os.Stderr, opts)
|
||||
}
|
||||
return slog.New(h)
|
||||
}
|
||||
Reference in New Issue
Block a user