benvin
c05ccfcb5d
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
82 lines
3.0 KiB
Go
82 lines
3.0 KiB
Go
// Package crypto implements logarchiver's object encryption.
|
|
//
|
|
// # Why not plain OpenPGP-encrypt the whole object?
|
|
//
|
|
// The private key lives only in Ben's Vault GPG secrets engine
|
|
// (vault-plugin-secrets-gpg). That engine's decrypt endpoint does WHOLE-payload
|
|
// inline decryption only: you POST the entire OpenPGP message (base64 in a JSON
|
|
// body) and it returns the entire plaintext (base64). There is no session-key /
|
|
// PKESK extraction and no streaming, so a multi-hundred-MiB archive could not be
|
|
// retrieved without blowing Vault's request-size limit and buffering everything
|
|
// twice in the server.
|
|
//
|
|
// # The wrapped-DEK envelope (container "LARC1")
|
|
//
|
|
// logarchiver therefore does hybrid encryption itself:
|
|
//
|
|
// - a fresh random 256-bit Data Encryption Key (DEK) per object;
|
|
// - the bulk (zstd-compressed NDJSON) is encrypted locally with AES-256-GCM in
|
|
// independent frames, so decryption streams frame-by-frame;
|
|
// - only the 32-byte DEK is OpenPGP-encrypted to the engine's PUBLIC key,
|
|
// producing a small (~hundreds of bytes) standard OpenPGP message.
|
|
//
|
|
// On retrieval the CLI sends ONLY that small wrapped-DEK blob to the engine's
|
|
// decrypt endpoint, recovers the DEK, and streams the bulk locally. The Vault
|
|
// round-trip is tiny and constant regardless of object size, and the private key
|
|
// never leaves Vault. The trade-off vs. a single standard OpenPGP object: these
|
|
// objects are a logarchiver-specific container, not decryptable by a bare `gpg`
|
|
// even with the private key. The retrieval runbook documents the format.
|
|
package crypto
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/ProtonMail/go-crypto/openpgp"
|
|
"github.com/ProtonMail/go-crypto/openpgp/armor"
|
|
)
|
|
|
|
// PublicKey is a parsed OpenPGP public key plus its fingerprint (uppercase hex,
|
|
// no spaces — matching the Vault GPG engine's `%X` fingerprint format).
|
|
type PublicKey struct {
|
|
entity *openpgp.Entity
|
|
Fingerprint string
|
|
}
|
|
|
|
// LoadPublicKey parses an ASCII-armored (or binary) OpenPGP public key.
|
|
func LoadPublicKey(data []byte) (*PublicKey, error) {
|
|
var keyring openpgp.EntityList
|
|
var err error
|
|
if strings.Contains(string(data), "BEGIN PGP") {
|
|
block, berr := armor.Decode(strings.NewReader(string(data)))
|
|
if berr != nil {
|
|
return nil, fmt.Errorf("decode armor: %w", berr)
|
|
}
|
|
keyring, err = openpgp.ReadKeyRing(block.Body)
|
|
} else {
|
|
keyring, err = openpgp.ReadKeyRing(strings.NewReader(string(data)))
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read public key: %w", err)
|
|
}
|
|
if len(keyring) == 0 {
|
|
return nil, fmt.Errorf("no public key found")
|
|
}
|
|
ent := keyring[0]
|
|
if ent.PrimaryKey == nil {
|
|
return nil, fmt.Errorf("key has no primary public key")
|
|
}
|
|
return &PublicKey{
|
|
entity: ent,
|
|
Fingerprint: fmt.Sprintf("%X", ent.PrimaryKey.Fingerprint),
|
|
}, nil
|
|
}
|
|
|
|
// digestArmored is used by tests to sanity check key identity independent of
|
|
// go-crypto internals.
|
|
func digestArmored(data []byte) string {
|
|
sum := sha256.Sum256(data)
|
|
return fmt.Sprintf("%x", sum[:8])
|
|
}
|