Initial implementation: NATS->S3 archiver + search/retrieve CLI

logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
2026-07-27 22:11:54 +10:00
parent d036d31f12
commit c05ccfcb5d
58 changed files with 5946 additions and 1 deletions
@@ -0,0 +1,379 @@
+// Package config defines logarchiver's configuration and loads it from a YAML
+// file with environment-variable overrides, so the same binary is
+// k8s-friendly (env/secret-driven) and laptop-friendly (a config file).
+//
+// Precedence: built-in defaults < YAML file < environment variables.
+package config
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"gopkg.in/yaml.v3"
+)
+
+// Config is the full logarchiver configuration.
+type Config struct {
+	NATS    NATSConfig    `yaml:"nats"`
+	Batch   BatchConfig   `yaml:"batch"`
+	S3      S3Config      `yaml:"s3"`
+	Crypto  CryptoConfig  `yaml:"crypto"`
+	Index   IndexConfig   `yaml:"index"`
+	Metrics MetricsConfig `yaml:"metrics"`
+	Log     LogConfig     `yaml:"log"`
+}
+
+// NATSConfig configures the JetStream pull consumer that logarchiver binds. It
+// mirrors the logging stack's `LOGS` stream / `archiver` durable / `log-consumer`
+// user conventions (argocd-apps #296).
+type NATSConfig struct {
+	URL      string   `yaml:"url"`
+	Stream   string   `yaml:"stream"`
+	Durable  string   `yaml:"durable"`
+	Subjects []string `yaml:"subjects"`
+	User     string   `yaml:"user"`
+	// Password is the NATS user password. In-cluster it comes from the
+	// nats-auth secret via NATS_CONSUMER_PASSWORD (see PasswordEnv).
+	Password string `yaml:"password"`
+	// PasswordEnv names the env var holding the password when Password is empty.
+	PasswordEnv string `yaml:"password_env"`
+	// CAFile trusts a custom CA for TLS to NATS (usually unset; in-cluster is plaintext).
+	CAFile string `yaml:"ca_file"`
+	// FetchBatch is the max messages pulled per Fetch call.
+	FetchBatch int `yaml:"fetch_batch"`
+	// AckWait is the JetStream redelivery timeout; must exceed a worst-case
+	// batch flush (compress+encrypt+S3 PUT+index write).
+	AckWait time.Duration `yaml:"ack_wait"`
+}
+
+// BatchConfig bounds a single archived object. A per-subject batch is flushed
+// when any bound is hit. Keep MaxBytes well under the crypto/engine ceiling so
+// even a whole-object decrypt path stays viable; the wrapped-DEK envelope means
+// object size is not limited by Vault, but smaller objects retrieve faster.
+type BatchConfig struct {
+	MaxBytes  int64         `yaml:"max_bytes"`
+	MaxEvents int           `yaml:"max_events"`
+	MaxAge    time.Duration `yaml:"max_age"`
+}
+
+// S3Config targets the Ceph RGW bucket. Credentials are read from the standard
+// AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars (cephrgw BucketAccess
+// secret logs-archive-s3), so they are intentionally absent here.
+type S3Config struct {
+	Endpoint  string `yaml:"endpoint"`
+	Bucket    string `yaml:"bucket"`
+	Region    string `yaml:"region"`
+	PathStyle bool   `yaml:"path_style"`
+	// KeyPrefix is a text/template with {{.Subject}} {{.Year}} {{.Month}} {{.Day}}.
+	KeyPrefix string `yaml:"key_prefix"`
+	// CAFile trusts the internal Vault-PKI CA for https://s3.ceph.unkin.net.
+	CAFile string `yaml:"ca_file"`
+	// EndpointEnv / BucketEnv let the cephrgw secret (S3_ENDPOINT / BUCKET_NAME)
+	// override endpoint/bucket without a config edit.
+	EndpointEnv string `yaml:"endpoint_env"`
+	BucketEnv   string `yaml:"bucket_env"`
+}
+
+// PubkeySource selects where the OpenPGP public key is fetched from.
+type PubkeySource string
+
+const (
+	PubkeyVault PubkeySource = "vault" // read gpg/keys/<name> from the Vault GPG engine
+	PubkeyFile  PubkeySource = "file"  // read an armored public key from a mounted file
+)
+
+// CryptoConfig controls encryption. The service only ever needs the PUBLIC key;
+// decryption (CLI fetch) always goes through the Vault GPG engine.
+type CryptoConfig struct {
+	KeyName string       `yaml:"key_name"`
+	Source  PubkeySource `yaml:"pubkey_source"`
+	// PubkeyFile is the armored public key path when Source==file.
+	PubkeyFile string `yaml:"pubkey_file"`
+	// RefreshInterval re-fetches the public key periodically (rotation aware).
+	RefreshInterval time.Duration `yaml:"refresh_interval"`
+	// FrameSize is the AES-GCM frame plaintext size in bytes (streaming decrypt).
+	FrameSize int         `yaml:"frame_size"`
+	Vault     VaultConfig `yaml:"vault"`
+}
+
+// VaultConfig configures access to the Vault GPG secrets engine. For the
+// service (pubkey fetch) k8s auth is used in-cluster; the CLI relies on the
+// operator's ambient VAULT_TOKEN (~/.vault-token), like passv.
+type VaultConfig struct {
+	Address string `yaml:"address"`
+	// Mount is the GPG engine mount path (e.g. "gpg").
+	Mount string `yaml:"mount"`
+	// AuthMethod is "token" or "kubernetes".
+	AuthMethod string `yaml:"auth_method"`
+	// K8sRole / K8sMount / K8sJWTPath configure kubernetes auth.
+	K8sRole    string `yaml:"k8s_role"`
+	K8sMount   string `yaml:"k8s_mount"`
+	K8sJWTPath string `yaml:"k8s_jwt_path"`
+	CAFile     string `yaml:"ca_file"`
+}
+
+// IndexConfig targets the ClickHouse archive index. Credentials come from the
+// clickhouse-credentials secret via env by default.
+type IndexConfig struct {
+	Enabled     bool   `yaml:"enabled"`
+	Address     string `yaml:"address"` // host:port for the native protocol (9000)
+	Database    string `yaml:"database"`
+	Table       string `yaml:"table"`
+	Username    string `yaml:"username"`
+	Password    string `yaml:"password"`
+	PasswordEnv string `yaml:"password_env"`
+	TLS         bool   `yaml:"tls"`
+}
+
+// MetricsConfig configures the Prometheus /metrics listener.
+type MetricsConfig struct {
+	Enabled bool   `yaml:"enabled"`
+	Address string `yaml:"address"`
+}
+
+// LogConfig configures structured logging.
+type LogConfig struct {
+	Level  string `yaml:"level"`  // debug|info|warn|error
+	Format string `yaml:"format"` // json|text
+}
+
+// Default returns a Config pre-populated with the logging-stack conventions so
+// an in-cluster deployment needs only secrets (creds) supplied via env.
+func Default() Config {
+	return Config{
+		NATS: NATSConfig{
+			URL:         "nats://nats.logging.svc.cluster.local:4222",
+			Stream:      "LOGS",
+			Durable:     "archiver",
+			Subjects:    []string{"logs.k8s.vault.>"},
+			User:        "log-consumer",
+			PasswordEnv: "NATS_CONSUMER_PASSWORD",
+			FetchBatch:  512,
+			AckWait:     2 * time.Minute,
+		},
+		Batch: BatchConfig{
+			MaxBytes:  64 * 1024 * 1024, // 64 MiB raw NDJSON per object
+			MaxEvents: 200000,
+			MaxAge:    5 * time.Minute,
+		},
+		S3: S3Config{
+			Endpoint:    "https://s3.ceph.unkin.net",
+			Bucket:      "logs-archive",
+			Region:      "us-east-1",
+			PathStyle:   true,
+			KeyPrefix:   "archive/{{.Subject}}/{{.Year}}/{{.Month}}/{{.Day}}/",
+			CAFile:      "/etc/vault-ca/ca.crt",
+			EndpointEnv: "S3_ENDPOINT",
+			BucketEnv:   "BUCKET_NAME",
+		},
+		Crypto: CryptoConfig{
+			KeyName:         "logarchive",
+			Source:          PubkeyFile,
+			PubkeyFile:      "/etc/logarchiver/pubkey.asc",
+			RefreshInterval: time.Hour,
+			FrameSize:       1 << 20, // 1 MiB frames
+			Vault: VaultConfig{
+				Mount:      "gpg",
+				AuthMethod: "kubernetes",
+				K8sMount:   "k8s/au/syd1",
+				K8sRole:    "default",
+				K8sJWTPath: "/var/run/secrets/kubernetes.io/serviceaccount/token",
+			},
+		},
+		Index: IndexConfig{
+			Enabled:     true,
+			Address:     "clickhouse-logs.logging.svc.cluster.local:9000",
+			Database:    "logs",
+			Table:       "archive_index",
+			Username:    "vector",
+			PasswordEnv: "CLICKHOUSE_PASSWORD",
+			TLS:         false,
+		},
+		Metrics: MetricsConfig{Enabled: true, Address: ":9090"},
+		Log:     LogConfig{Level: "info", Format: "json"},
+	}
+}
+
+// Load reads defaults, overlays the YAML file at path (if non-empty), then
+// applies environment overrides, and validates the result.
+func Load(path string) (Config, error) {
+	cfg := Default()
+	if path != "" {
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return Config{}, fmt.Errorf("read config %s: %w", path, err)
+		}
+		if err := yaml.Unmarshal(data, &cfg); err != nil {
+			return Config{}, fmt.Errorf("parse config %s: %w", path, err)
+		}
+	}
+	cfg.applyEnv()
+	cfg.resolveSecretEnvs()
+	if err := cfg.Validate(); err != nil {
+		return Config{}, err
+	}
+	return cfg, nil
+}
+
+// applyEnv overlays scalar overrides from the environment. Only the knobs an
+// operator commonly flips are wired; secrets are handled by resolveSecretEnvs.
+func (c *Config) applyEnv() {
+	if v := os.Getenv("LOGARCHIVER_NATS_URL"); v != "" {
+		c.NATS.URL = v
+	}
+	if v := os.Getenv("LOGARCHIVER_NATS_DURABLE"); v != "" {
+		c.NATS.Durable = v
+	}
+	if v := os.Getenv("ARCHIVE_SUBJECTS"); v != "" {
+		c.NATS.Subjects = splitFields(v)
+	}
+	if v := os.Getenv("LOGARCHIVER_S3_ENDPOINT"); v != "" {
+		c.S3.Endpoint = v
+	}
+	if v := os.Getenv("LOGARCHIVER_S3_BUCKET"); v != "" {
+		c.S3.Bucket = v
+	}
+	if v := os.Getenv("LOGARCHIVER_KEY_NAME"); v != "" {
+		c.Crypto.KeyName = v
+	}
+	if v := os.Getenv("LOGARCHIVER_PUBKEY_SOURCE"); v != "" {
+		c.Crypto.Source = PubkeySource(v)
+	}
+	if v := os.Getenv("LOGARCHIVER_PUBKEY_FILE"); v != "" {
+		c.Crypto.PubkeyFile = v
+	}
+	if v := os.Getenv("VAULT_ADDR"); v != "" && c.Crypto.Vault.Address == "" {
+		c.Crypto.Vault.Address = v
+	}
+	if v := os.Getenv("LOGARCHIVER_VAULT_MOUNT"); v != "" {
+		c.Crypto.Vault.Mount = v
+	}
+	if v := os.Getenv("LOGARCHIVER_CLICKHOUSE_ADDR"); v != "" {
+		c.Index.Address = v
+	}
+	if v := os.Getenv("CLICKHOUSE_USER"); v != "" {
+		c.Index.Username = v
+	}
+	if v := os.Getenv("LOGARCHIVER_METRICS_ADDR"); v != "" {
+		c.Metrics.Address = v
+	}
+	if v := os.Getenv("LOGARCHIVER_LOG_LEVEL"); v != "" {
+		c.Log.Level = v
+	}
+	if v := os.Getenv("LOGARCHIVER_LOG_FORMAT"); v != "" {
+		c.Log.Format = v
+	}
+	// Endpoint/bucket sourced from the cephrgw secret, if present.
+	if c.S3.EndpointEnv != "" {
+		if v := os.Getenv(c.S3.EndpointEnv); v != "" {
+			c.S3.Endpoint = v
+		}
+	}
+	if c.S3.BucketEnv != "" {
+		if v := os.Getenv(c.S3.BucketEnv); v != "" {
+			c.S3.Bucket = v
+		}
+	}
+}
+
+// resolveSecretEnvs pulls passwords from their named env vars when not set inline.
+func (c *Config) resolveSecretEnvs() {
+	if c.NATS.Password == "" && c.NATS.PasswordEnv != "" {
+		c.NATS.Password = os.Getenv(c.NATS.PasswordEnv)
+	}
+	if c.Index.Password == "" && c.Index.PasswordEnv != "" {
+		c.Index.Password = os.Getenv(c.Index.PasswordEnv)
+	}
+}
+
+// Validate checks required fields and coherence.
+func (c *Config) Validate() error {
+	if c.NATS.URL == "" {
+		return fmt.Errorf("nats.url is required")
+	}
+	if c.NATS.Stream == "" {
+		return fmt.Errorf("nats.stream is required")
+	}
+	if c.NATS.Durable == "" {
+		return fmt.Errorf("nats.durable is required")
+	}
+	if len(c.NATS.Subjects) == 0 {
+		return fmt.Errorf("nats.subjects must list at least one filter subject")
+	}
+	if c.S3.Bucket == "" {
+		return fmt.Errorf("s3.bucket is required")
+	}
+	if c.S3.Endpoint == "" {
+		return fmt.Errorf("s3.endpoint is required")
+	}
+	if c.Crypto.KeyName == "" {
+		return fmt.Errorf("crypto.key_name is required")
+	}
+	switch c.Crypto.Source {
+	case PubkeyVault:
+		if c.Crypto.Vault.Address == "" {
+			return fmt.Errorf("crypto.vault.address is required when pubkey_source=vault")
+		}
+		if c.Crypto.Vault.Mount == "" {
+			return fmt.Errorf("crypto.vault.mount is required when pubkey_source=vault")
+		}
+	case PubkeyFile:
+		if c.Crypto.PubkeyFile == "" {
+			return fmt.Errorf("crypto.pubkey_file is required when pubkey_source=file")
+		}
+	default:
+		return fmt.Errorf("crypto.pubkey_source must be 'vault' or 'file', got %q", c.Crypto.Source)
+	}
+	if c.Crypto.FrameSize <= 0 {
+		return fmt.Errorf("crypto.frame_size must be positive")
+	}
+	if c.Batch.MaxBytes <= 0 && c.Batch.MaxEvents <= 0 && c.Batch.MaxAge <= 0 {
+		return fmt.Errorf("batch must set at least one of max_bytes/max_events/max_age")
+	}
+	if c.Index.Enabled && c.Index.Address == "" {
+		return fmt.Errorf("index.address is required when index.enabled")
+	}
+	return nil
+}
+
+func splitFields(s string) []string {
+	var out []string
+	for _, f := range strings.Fields(s) {
+		if f != "" {
+			out = append(out, f)
+		}
+	}
+	return out
+}
+
+// ParseSize parses a byte size like "64Mi", "128MB", "1024". It is a helper for
+// CLI flags; the YAML fields are plain integers.
+func ParseSize(s string) (int64, error) {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return 0, fmt.Errorf("empty size")
+	}
+	mult := int64(1)
+	switch {
+	case strings.HasSuffix(s, "Gi"):
+		mult, s = 1<<30, strings.TrimSuffix(s, "Gi")
+	case strings.HasSuffix(s, "Mi"):
+		mult, s = 1<<20, strings.TrimSuffix(s, "Mi")
+	case strings.HasSuffix(s, "Ki"):
+		mult, s = 1<<10, strings.TrimSuffix(s, "Ki")
+	case strings.HasSuffix(s, "GB"):
+		mult, s = 1e9, strings.TrimSuffix(s, "GB")
+	case strings.HasSuffix(s, "MB"):
+		mult, s = 1e6, strings.TrimSuffix(s, "MB")
+	case strings.HasSuffix(s, "KB"):
+		mult, s = 1e3, strings.TrimSuffix(s, "KB")
+	}
+	n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("parse size %q: %w", s, err)
+	}
+	return n * mult, nil
+}
@@ -0,0 +1,106 @@
+package config
+
+import (
+	"testing"
+	"time"
+)
+
+func TestDefaultIsValid(t *testing.T) {
+	cfg := Default()
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("default config should validate: %v", err)
+	}
+}
+
+func TestLoadEnvOverrides(t *testing.T) {
+	t.Setenv("ARCHIVE_SUBJECTS", "logs.vm.> logs.k8s.vault.>")
+	t.Setenv("LOGARCHIVER_S3_BUCKET", "my-bucket")
+	t.Setenv("LOGARCHIVER_NATS_DURABLE", "archiver-canary")
+	t.Setenv("NATS_CONSUMER_PASSWORD", "s3cr3t")
+	t.Setenv("S3_ENDPOINT", "https://rgw.internal")
+	t.Setenv("BUCKET_NAME", "logs-archive-override")
+
+	cfg, err := Load("")
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if got, want := cfg.NATS.Subjects, []string{"logs.vm.>", "logs.k8s.vault.>"}; len(got) != len(want) || got[0] != want[0] || got[1] != want[1] {
+		t.Errorf("subjects = %v, want %v", got, want)
+	}
+	if cfg.NATS.Durable != "archiver-canary" {
+		t.Errorf("durable = %q", cfg.NATS.Durable)
+	}
+	if cfg.NATS.Password != "s3cr3t" {
+		t.Errorf("password from env not resolved: %q", cfg.NATS.Password)
+	}
+	// BUCKET_NAME (secret) should win over LOGARCHIVER_S3_BUCKET default flow.
+	if cfg.S3.Bucket != "logs-archive-override" {
+		t.Errorf("bucket = %q, want cephrgw secret override", cfg.S3.Bucket)
+	}
+	if cfg.S3.Endpoint != "https://rgw.internal" {
+		t.Errorf("endpoint = %q", cfg.S3.Endpoint)
+	}
+}
+
+func TestValidateErrors(t *testing.T) {
+	cases := map[string]func(*Config){
+		"no subjects":     func(c *Config) { c.NATS.Subjects = nil },
+		"no bucket":       func(c *Config) { c.S3.Bucket = "" },
+		"no key name":     func(c *Config) { c.Crypto.KeyName = "" },
+		"bad source":      func(c *Config) { c.Crypto.Source = "elsewhere" },
+		"file no path":    func(c *Config) { c.Crypto.Source = PubkeyFile; c.Crypto.PubkeyFile = "" },
+		"vault no addr":   func(c *Config) { c.Crypto.Source = PubkeyVault; c.Crypto.Vault.Address = "" },
+		"zero frame":      func(c *Config) { c.Crypto.FrameSize = 0 },
+		"no batch bounds": func(c *Config) { c.Batch = BatchConfig{} },
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			cfg := Default()
+			mutate(&cfg)
+			if err := cfg.Validate(); err == nil {
+				t.Errorf("expected validation error for %q", name)
+			}
+		})
+	}
+}
+
+func TestValidateVaultSourceOK(t *testing.T) {
+	cfg := Default()
+	cfg.Crypto.Source = PubkeyVault
+	cfg.Crypto.Vault.Address = "https://vault.example:8200"
+	cfg.Crypto.Vault.Mount = "gpg"
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("vault source should validate: %v", err)
+	}
+}
+
+func TestParseSize(t *testing.T) {
+	cases := map[string]int64{
+		"1024": 1024,
+		"64Mi": 64 << 20,
+		"2Gi":  2 << 30,
+		"1Ki":  1024,
+		"5MB":  5_000_000,
+		" 10 ": 10,
+	}
+	for in, want := range cases {
+		got, err := ParseSize(in)
+		if err != nil {
+			t.Errorf("ParseSize(%q): %v", in, err)
+			continue
+		}
+		if got != want {
+			t.Errorf("ParseSize(%q) = %d, want %d", in, got, want)
+		}
+	}
+	if _, err := ParseSize("bogus"); err == nil {
+		t.Errorf("expected error for bogus size")
+	}
+}
+
+func TestDefaultBatchDurations(t *testing.T) {
+	cfg := Default()
+	if cfg.Batch.MaxAge != 5*time.Minute {
+		t.Errorf("default MaxAge = %v", cfg.Batch.MaxAge)
+	}
+}