benvin
c05ccfcb5d
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
90 lines
2.5 KiB
Go
90 lines
2.5 KiB
Go
package archiver
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"sync"
|
|
|
|
"git.unkin.net/unkin/logarchiver/internal/config"
|
|
"git.unkin.net/unkin/logarchiver/internal/crypto"
|
|
"git.unkin.net/unkin/logarchiver/internal/vaultgpg"
|
|
)
|
|
|
|
// PubkeyLoader fetches the current armored public key from the configured source.
|
|
type PubkeyLoader func(ctx context.Context) (*crypto.PublicKey, error)
|
|
|
|
// PubkeyProvider caches the current public key and supports periodic refresh so
|
|
// key rotation in the Vault GPG engine is picked up without a restart.
|
|
type PubkeyProvider struct {
|
|
load PubkeyLoader
|
|
mu sync.RWMutex
|
|
key *crypto.PublicKey
|
|
}
|
|
|
|
// NewPubkeyProvider builds a provider and loads the key once.
|
|
func NewPubkeyProvider(ctx context.Context, load PubkeyLoader) (*PubkeyProvider, error) {
|
|
p := &PubkeyProvider{load: load}
|
|
if err := p.Refresh(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
return p, nil
|
|
}
|
|
|
|
// Refresh reloads the public key.
|
|
func (p *PubkeyProvider) Refresh(ctx context.Context) error {
|
|
key, err := p.load(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
p.mu.Lock()
|
|
p.key = key
|
|
p.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
// Current returns the cached public key.
|
|
func (p *PubkeyProvider) Current() *crypto.PublicKey {
|
|
p.mu.RLock()
|
|
defer p.mu.RUnlock()
|
|
return p.key
|
|
}
|
|
|
|
// PubkeyLoaderFromConfig builds a loader for the configured source. For
|
|
// pubkey_source=vault it also verifies the parsed key's fingerprint matches the
|
|
// fingerprint the engine reports, catching armor corruption.
|
|
func PubkeyLoaderFromConfig(cfg config.CryptoConfig, vc *vaultgpg.Client) (PubkeyLoader, error) {
|
|
switch cfg.Source {
|
|
case config.PubkeyFile:
|
|
path := cfg.PubkeyFile
|
|
return func(_ context.Context) (*crypto.PublicKey, error) {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read pubkey file %s: %w", path, err)
|
|
}
|
|
return crypto.LoadPublicKey(data)
|
|
}, nil
|
|
case config.PubkeyVault:
|
|
if vc == nil {
|
|
return nil, fmt.Errorf("pubkey_source=vault requires a vault client")
|
|
}
|
|
name := cfg.KeyName
|
|
return func(ctx context.Context) (*crypto.PublicKey, error) {
|
|
pk, err := vc.FetchPublicKey(ctx, name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
key, err := crypto.LoadPublicKey([]byte(pk.Armored))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if pk.Fingerprint != "" && key.Fingerprint != pk.Fingerprint {
|
|
return nil, fmt.Errorf("pubkey fingerprint mismatch: engine=%s parsed=%s", pk.Fingerprint, key.Fingerprint)
|
|
}
|
|
return key, nil
|
|
}, nil
|
|
default:
|
|
return nil, fmt.Errorf("unknown pubkey_source %q", cfg.Source)
|
|
}
|
|
}
|