c05ccfcb5d
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
166 lines
5.0 KiB
Go
166 lines
5.0 KiB
Go
package crypto
|
|
|
|
import (
|
|
"bytes"
|
|
"io"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/ProtonMail/go-crypto/openpgp"
|
|
"github.com/ProtonMail/go-crypto/openpgp/armor"
|
|
)
|
|
|
|
// genTestKey creates an OpenPGP keypair, returns the armored public key (as the
|
|
// service would export from Vault) and an unwrap func that decrypts the wrapped
|
|
// DEK with the private key — simulating the Vault GPG engine's decrypt endpoint
|
|
// (which returns the plaintext of a whole OpenPGP message).
|
|
func genTestKey(t *testing.T) (armoredPub []byte, unwrap UnwrapFunc) {
|
|
t.Helper()
|
|
ent, err := openpgp.NewEntity("logarchiver-test", "unit test", "test@unkin.net", nil)
|
|
if err != nil {
|
|
t.Fatalf("NewEntity: %v", err)
|
|
}
|
|
var buf bytes.Buffer
|
|
w, err := armor.Encode(&buf, openpgp.PublicKeyType, nil)
|
|
if err != nil {
|
|
t.Fatalf("armor encode: %v", err)
|
|
}
|
|
if err := ent.Serialize(w); err != nil {
|
|
t.Fatalf("serialize public: %v", err)
|
|
}
|
|
_ = w.Close()
|
|
|
|
unwrap = func(wrapped []byte) ([]byte, error) {
|
|
md, err := openpgp.ReadMessage(bytes.NewReader(wrapped), openpgp.EntityList{ent}, nil, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return io.ReadAll(md.UnverifiedBody)
|
|
}
|
|
return buf.Bytes(), unwrap
|
|
}
|
|
|
|
func TestRoundTrip(t *testing.T) {
|
|
armoredPub, unwrap := genTestKey(t)
|
|
pub, err := LoadPublicKey(armoredPub)
|
|
if err != nil {
|
|
t.Fatalf("LoadPublicKey: %v", err)
|
|
}
|
|
if len(pub.Fingerprint) != 40 {
|
|
t.Errorf("fingerprint = %q, want 40 hex chars", pub.Fingerprint)
|
|
}
|
|
if pub.Fingerprint != strings.ToUpper(pub.Fingerprint) {
|
|
t.Errorf("fingerprint should be uppercase: %q", pub.Fingerprint)
|
|
}
|
|
|
|
// A multi-line NDJSON payload larger than the frame size (forces >1 frame).
|
|
var payload bytes.Buffer
|
|
for i := 0; i < 5000; i++ {
|
|
payload.WriteString(`{"host":"node-1","message":"line `)
|
|
payload.WriteString(strings.Repeat("x", 50))
|
|
payload.WriteString(`"}` + "\n")
|
|
}
|
|
plaintext := payload.Bytes()
|
|
|
|
var sealed bytes.Buffer
|
|
res, err := Seal(&sealed, plaintext, pub, "logarchive", 4096)
|
|
if err != nil {
|
|
t.Fatalf("Seal: %v", err)
|
|
}
|
|
if res.RawBytes != int64(len(plaintext)) {
|
|
t.Errorf("RawBytes = %d, want %d", res.RawBytes, len(plaintext))
|
|
}
|
|
if int64(sealed.Len()) != res.StoredBytes {
|
|
t.Errorf("StoredBytes = %d, buffer = %d", res.StoredBytes, sealed.Len())
|
|
}
|
|
// Compression should shrink this highly repetitive payload.
|
|
if res.StoredBytes >= res.RawBytes {
|
|
t.Errorf("stored (%d) not smaller than raw (%d)", res.StoredBytes, res.RawBytes)
|
|
}
|
|
// Design property: only a tiny wrapped DEK goes to Vault, regardless of size.
|
|
if res.Header.WrappedDEKLen > 4096 {
|
|
t.Errorf("wrapped DEK unexpectedly large: %d bytes", res.Header.WrappedDEKLen)
|
|
}
|
|
if res.Header.KeyFingerprint != pub.Fingerprint {
|
|
t.Errorf("header fingerprint mismatch")
|
|
}
|
|
|
|
var out bytes.Buffer
|
|
if err := Open(bytes.NewReader(sealed.Bytes()), &out, unwrap); err != nil {
|
|
t.Fatalf("Open: %v", err)
|
|
}
|
|
if !bytes.Equal(out.Bytes(), plaintext) {
|
|
t.Fatalf("round-trip mismatch: got %d bytes, want %d", out.Len(), len(plaintext))
|
|
}
|
|
}
|
|
|
|
func TestRoundTripEmpty(t *testing.T) {
|
|
armoredPub, unwrap := genTestKey(t)
|
|
pub, _ := LoadPublicKey(armoredPub)
|
|
var sealed bytes.Buffer
|
|
if _, err := Seal(&sealed, []byte{}, pub, "k", 4096); err != nil {
|
|
t.Fatalf("Seal empty: %v", err)
|
|
}
|
|
var out bytes.Buffer
|
|
if err := Open(bytes.NewReader(sealed.Bytes()), &out, unwrap); err != nil {
|
|
t.Fatalf("Open empty: %v", err)
|
|
}
|
|
if out.Len() != 0 {
|
|
t.Errorf("empty round-trip produced %d bytes", out.Len())
|
|
}
|
|
}
|
|
|
|
func TestTamperDetected(t *testing.T) {
|
|
armoredPub, unwrap := genTestKey(t)
|
|
pub, _ := LoadPublicKey(armoredPub)
|
|
var sealed bytes.Buffer
|
|
if _, err := Seal(&sealed, []byte("hello world\n"), pub, "k", 4096); err != nil {
|
|
t.Fatalf("Seal: %v", err)
|
|
}
|
|
data := sealed.Bytes()
|
|
// Flip a byte near the end (inside a frame's ciphertext/tag).
|
|
data[len(data)-3] ^= 0xff
|
|
var out bytes.Buffer
|
|
if err := Open(bytes.NewReader(data), &out, unwrap); err == nil {
|
|
t.Fatalf("expected GCM authentication failure on tampered ciphertext")
|
|
}
|
|
}
|
|
|
|
func TestBadMagic(t *testing.T) {
|
|
_, _, err := ReadHeader(bytes.NewReader([]byte("NOTLARC.....")))
|
|
if err == nil {
|
|
t.Fatalf("expected bad-magic error")
|
|
}
|
|
}
|
|
|
|
func TestReadHeaderFields(t *testing.T) {
|
|
armoredPub, _ := genTestKey(t)
|
|
pub, _ := LoadPublicKey(armoredPub)
|
|
var sealed bytes.Buffer
|
|
if _, err := Seal(&sealed, []byte("x\n"), pub, "logarchive", 4096); err != nil {
|
|
t.Fatalf("Seal: %v", err)
|
|
}
|
|
hdr, wrapped, err := ReadHeader(bytes.NewReader(sealed.Bytes()))
|
|
if err != nil {
|
|
t.Fatalf("ReadHeader: %v", err)
|
|
}
|
|
if hdr.KeyName != "logarchive" {
|
|
t.Errorf("KeyName = %q", hdr.KeyName)
|
|
}
|
|
if hdr.Compression != "zstd" || hdr.Cipher != "AES-256-GCM" {
|
|
t.Errorf("algo metadata wrong: %+v", hdr)
|
|
}
|
|
if len(wrapped) != hdr.WrappedDEKLen {
|
|
t.Errorf("wrapped len %d != header %d", len(wrapped), hdr.WrappedDEKLen)
|
|
}
|
|
}
|
|
|
|
func TestDigestArmoredStable(t *testing.T) {
|
|
// Guards the test helper used elsewhere; identical input -> identical digest.
|
|
a := digestArmored([]byte("abc"))
|
|
b := digestArmored([]byte("abc"))
|
|
if a != b || a == "" {
|
|
t.Errorf("digestArmored not stable: %q %q", a, b)
|
|
}
|
|
}
|