benvin
5ec89b0028
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
96 lines
2.3 KiB
Go
96 lines
2.3 KiB
Go
// Package cli implements the logarchiver command tree (service + operator CLI)
|
|
// using cobra, which also provides the `completion` subcommand the estate's
|
|
// nfpm packaging installs.
|
|
package cli
|
|
|
|
import (
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"strings"
|
|
|
|
"git.unkin.net/unkin/logarchiver/internal/config"
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
// version is set at build time via -ldflags "-X ...cli.version=...".
|
|
var version = "dev"
|
|
|
|
// SetVersion lets main inject the linker-provided version string.
|
|
func SetVersion(v string) {
|
|
if v != "" {
|
|
version = v
|
|
}
|
|
}
|
|
|
|
var configPath string
|
|
|
|
// NewRootCmd builds the root command.
|
|
func NewRootCmd() *cobra.Command {
|
|
root := &cobra.Command{
|
|
Use: "logarchiver",
|
|
Short: "Archive NATS JetStream logs to S3 (zstd + OpenPGP) and search/retrieve them",
|
|
Long: `logarchiver archives raw logs from the centralized logging JetStream stream
|
|
to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, and provides a
|
|
CLI to search the index and retrieve/decrypt archived logs.`,
|
|
SilenceUsage: true,
|
|
SilenceErrors: true,
|
|
}
|
|
root.PersistentFlags().StringVarP(&configPath, "config", "c", os.Getenv("LOGARCHIVER_CONFIG"),
|
|
"path to config file (env LOGARCHIVER_CONFIG)")
|
|
|
|
root.AddCommand(
|
|
newRunCmd(),
|
|
newSearchCmd(),
|
|
newFetchCmd(),
|
|
newInitSchemaCmd(),
|
|
newVersionCmd(),
|
|
)
|
|
return root
|
|
}
|
|
|
|
// Execute runs the root command.
|
|
func Execute() int {
|
|
if err := NewRootCmd().Execute(); err != nil {
|
|
fmt.Fprintln(os.Stderr, "error:", err)
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func newVersionCmd() *cobra.Command {
|
|
return &cobra.Command{
|
|
Use: "version",
|
|
Short: "Print the version",
|
|
Run: func(cmd *cobra.Command, _ []string) {
|
|
_, _ = fmt.Fprintln(cmd.OutOrStdout(), version)
|
|
},
|
|
}
|
|
}
|
|
|
|
// loadConfig loads config from the --config path (or defaults+env).
|
|
func loadConfig() (config.Config, error) {
|
|
return config.Load(configPath)
|
|
}
|
|
|
|
// newLogger builds a slog logger from config.
|
|
func newLogger(cfg config.LogConfig) *slog.Logger {
|
|
level := slog.LevelInfo
|
|
switch strings.ToLower(cfg.Level) {
|
|
case "debug":
|
|
level = slog.LevelDebug
|
|
case "warn":
|
|
level = slog.LevelWarn
|
|
case "error":
|
|
level = slog.LevelError
|
|
}
|
|
opts := &slog.HandlerOptions{Level: level}
|
|
var h slog.Handler
|
|
if strings.ToLower(cfg.Format) == "text" {
|
|
h = slog.NewTextHandler(os.Stderr, opts)
|
|
} else {
|
|
h = slog.NewJSONHandler(os.Stderr, opts)
|
|
}
|
|
return slog.New(h)
|
|
}
|