benvin
c05ccfcb5d
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
91 lines
2.6 KiB
Go
91 lines
2.6 KiB
Go
package cli
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.unkin.net/unkin/logarchiver/internal/index"
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
// selectFlags are the shared object-selection flags for search and fetch.
|
|
type selectFlags struct {
|
|
subject string
|
|
host string
|
|
from string
|
|
to string
|
|
limit int
|
|
}
|
|
|
|
func (s *selectFlags) bind(cmd *cobra.Command) {
|
|
f := cmd.Flags()
|
|
f.StringVar(&s.subject, "subject", "", "NATS-style subject glob (e.g. 'logs.vm.*' or 'logs.k8s.vault.>')")
|
|
f.StringVar(&s.host, "host", "", "source host to match (exact, or a glob with '*')")
|
|
f.StringVar(&s.from, "from", "", "start of time window (RFC3339, 'YYYY-MM-DD', or relative like '-24h')")
|
|
f.StringVar(&s.to, "to", "", "end of time window (RFC3339, 'YYYY-MM-DD', or relative like '-1h')")
|
|
f.IntVar(&s.limit, "limit", 100, "max objects to return (0 = no limit)")
|
|
}
|
|
|
|
// query builds an index.SearchQuery from the flags.
|
|
func (s *selectFlags) query(now time.Time) (index.SearchQuery, error) {
|
|
q := index.SearchQuery{Subject: s.subject, Host: s.host, Limit: s.limit}
|
|
if s.from != "" {
|
|
t, err := parseTimeArg(s.from, now)
|
|
if err != nil {
|
|
return q, fmt.Errorf("--from: %w", err)
|
|
}
|
|
q.From = t
|
|
}
|
|
if s.to != "" {
|
|
t, err := parseTimeArg(s.to, now)
|
|
if err != nil {
|
|
return q, fmt.Errorf("--to: %w", err)
|
|
}
|
|
q.To = t
|
|
}
|
|
if !q.From.IsZero() && !q.To.IsZero() && q.To.Before(q.From) {
|
|
return q, fmt.Errorf("--to (%s) is before --from (%s)", q.To, q.From)
|
|
}
|
|
return q, nil
|
|
}
|
|
|
|
// parseTimeArg accepts RFC3339[/Nano], "YYYY-MM-DD", "YYYY-MM-DDTHH:MM:SS", or a
|
|
// signed Go duration relative to now (e.g. "-24h", "30m").
|
|
func parseTimeArg(s string, now time.Time) (time.Time, error) {
|
|
s = strings.TrimSpace(s)
|
|
for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05", "2006-01-02"} {
|
|
if t, err := time.Parse(layout, s); err == nil {
|
|
return t.UTC(), nil
|
|
}
|
|
}
|
|
if d, err := time.ParseDuration(s); err == nil {
|
|
return now.Add(d).UTC(), nil
|
|
}
|
|
return time.Time{}, fmt.Errorf("unrecognized time %q (use RFC3339, YYYY-MM-DD, or a duration like -24h)", s)
|
|
}
|
|
|
|
func newIndexClient(cmd *cobra.Command) (index.Index, error) {
|
|
cfg, err := loadConfig()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !cfg.Index.Enabled {
|
|
return nil, fmt.Errorf("index is disabled in config; search/fetch-by-query require the ClickHouse index")
|
|
}
|
|
return index.NewClickHouse(cmd.Context(), indexConfig(cfg.Index))
|
|
}
|
|
|
|
func humanBytes(n uint64) string {
|
|
const unit = 1024
|
|
if n < unit {
|
|
return fmt.Sprintf("%dB", n)
|
|
}
|
|
div, exp := int64(unit), 0
|
|
for x := n / unit; x >= unit; x /= unit {
|
|
div *= unit
|
|
exp++
|
|
}
|
|
return fmt.Sprintf("%.1f%ciB", float64(n)/float64(div), "KMGTPE"[exp])
|
|
}
|