Initial implementation: NATS->S3 archiver + search/retrieve CLI
ci/woodpecker/pr/build Pipeline was successful
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful

logarchiver replaces the plain Vector archiver leg of the centralized
logging stack (argocd-apps #296) with a Go service that archives raw logs
from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed
objects, plus an operator CLI to search the index and retrieve/decrypt
archived logs. It adds the things that outgrew Vector: zstd compression,
encryption keyed from Ben's Vault GPG secrets engine, a searchable
ClickHouse index, and sink-conditional acks (a batch is acknowledged to
JetStream only after the object is durably in S3 AND indexed).

Service (`logarchiver run`):
- Durable JetStream pull consumer (stream LOGS, durable archiver, subject
  filter default logs.k8s.vault.>), explicit acks, independent offsets.
- Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3
  PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and
  redelivered, so nothing is lost on a sink outage.
- Encryption is a wrapped-DEK envelope (container LARC1): the bulk is
  AES-256-GCM framed under a random data key, and only that 32-byte key is
  OpenPGP-encrypted to the engine's public key. This is because the Vault
  GPG engine does whole-payload decrypt only; retrieval round-trips just the
  tiny wrapped key regardless of object size. Public key fetched from the
  engine or a mounted file (configurable); key fingerprint recorded per
  object; periodic pubkey refresh for rotation.
- Prometheus metrics, structured slog, graceful drain on shutdown.

CLI:
- `search` queries the index (subject/host/time) and lists matching objects.
- `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits
  NDJSON (optionally re-filtered by host/time).
- `init-schema` creates/prints the ClickHouse archive_index DDL.
- cobra `completion` subcommands.

Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/
ClickHouse/Vault) behind interfaces with unit tests (config, batching,
host/subject extraction, crypto roundtrip with a test key, ack-after-persist
with fakes, search query building). go build/vet/test -race clean;
golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v*
tag a container image plus a Gitea binary release + rpm-internal RPM. Docs
per subcommand + architecture + retrieval runbook + deployment drop-in.

Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
This commit is contained in:
benvin
2026-07-27 22:11:54 +10:00
committed by Ben Vincent
parent d036d31f12
commit c05ccfcb5d
58 changed files with 5946 additions and 1 deletions
+165
View File
@@ -0,0 +1,165 @@
package crypto
import (
"bytes"
"io"
"strings"
"testing"
"github.com/ProtonMail/go-crypto/openpgp"
"github.com/ProtonMail/go-crypto/openpgp/armor"
)
// genTestKey creates an OpenPGP keypair, returns the armored public key (as the
// service would export from Vault) and an unwrap func that decrypts the wrapped
// DEK with the private key — simulating the Vault GPG engine's decrypt endpoint
// (which returns the plaintext of a whole OpenPGP message).
func genTestKey(t *testing.T) (armoredPub []byte, unwrap UnwrapFunc) {
t.Helper()
ent, err := openpgp.NewEntity("logarchiver-test", "unit test", "test@unkin.net", nil)
if err != nil {
t.Fatalf("NewEntity: %v", err)
}
var buf bytes.Buffer
w, err := armor.Encode(&buf, openpgp.PublicKeyType, nil)
if err != nil {
t.Fatalf("armor encode: %v", err)
}
if err := ent.Serialize(w); err != nil {
t.Fatalf("serialize public: %v", err)
}
_ = w.Close()
unwrap = func(wrapped []byte) ([]byte, error) {
md, err := openpgp.ReadMessage(bytes.NewReader(wrapped), openpgp.EntityList{ent}, nil, nil)
if err != nil {
return nil, err
}
return io.ReadAll(md.UnverifiedBody)
}
return buf.Bytes(), unwrap
}
func TestRoundTrip(t *testing.T) {
armoredPub, unwrap := genTestKey(t)
pub, err := LoadPublicKey(armoredPub)
if err != nil {
t.Fatalf("LoadPublicKey: %v", err)
}
if len(pub.Fingerprint) != 40 {
t.Errorf("fingerprint = %q, want 40 hex chars", pub.Fingerprint)
}
if pub.Fingerprint != strings.ToUpper(pub.Fingerprint) {
t.Errorf("fingerprint should be uppercase: %q", pub.Fingerprint)
}
// A multi-line NDJSON payload larger than the frame size (forces >1 frame).
var payload bytes.Buffer
for i := 0; i < 5000; i++ {
payload.WriteString(`{"host":"node-1","message":"line `)
payload.WriteString(strings.Repeat("x", 50))
payload.WriteString(`"}` + "\n")
}
plaintext := payload.Bytes()
var sealed bytes.Buffer
res, err := Seal(&sealed, plaintext, pub, "logarchive", 4096)
if err != nil {
t.Fatalf("Seal: %v", err)
}
if res.RawBytes != int64(len(plaintext)) {
t.Errorf("RawBytes = %d, want %d", res.RawBytes, len(plaintext))
}
if int64(sealed.Len()) != res.StoredBytes {
t.Errorf("StoredBytes = %d, buffer = %d", res.StoredBytes, sealed.Len())
}
// Compression should shrink this highly repetitive payload.
if res.StoredBytes >= res.RawBytes {
t.Errorf("stored (%d) not smaller than raw (%d)", res.StoredBytes, res.RawBytes)
}
// Design property: only a tiny wrapped DEK goes to Vault, regardless of size.
if res.Header.WrappedDEKLen > 4096 {
t.Errorf("wrapped DEK unexpectedly large: %d bytes", res.Header.WrappedDEKLen)
}
if res.Header.KeyFingerprint != pub.Fingerprint {
t.Errorf("header fingerprint mismatch")
}
var out bytes.Buffer
if err := Open(bytes.NewReader(sealed.Bytes()), &out, unwrap); err != nil {
t.Fatalf("Open: %v", err)
}
if !bytes.Equal(out.Bytes(), plaintext) {
t.Fatalf("round-trip mismatch: got %d bytes, want %d", out.Len(), len(plaintext))
}
}
func TestRoundTripEmpty(t *testing.T) {
armoredPub, unwrap := genTestKey(t)
pub, _ := LoadPublicKey(armoredPub)
var sealed bytes.Buffer
if _, err := Seal(&sealed, []byte{}, pub, "k", 4096); err != nil {
t.Fatalf("Seal empty: %v", err)
}
var out bytes.Buffer
if err := Open(bytes.NewReader(sealed.Bytes()), &out, unwrap); err != nil {
t.Fatalf("Open empty: %v", err)
}
if out.Len() != 0 {
t.Errorf("empty round-trip produced %d bytes", out.Len())
}
}
func TestTamperDetected(t *testing.T) {
armoredPub, unwrap := genTestKey(t)
pub, _ := LoadPublicKey(armoredPub)
var sealed bytes.Buffer
if _, err := Seal(&sealed, []byte("hello world\n"), pub, "k", 4096); err != nil {
t.Fatalf("Seal: %v", err)
}
data := sealed.Bytes()
// Flip a byte near the end (inside a frame's ciphertext/tag).
data[len(data)-3] ^= 0xff
var out bytes.Buffer
if err := Open(bytes.NewReader(data), &out, unwrap); err == nil {
t.Fatalf("expected GCM authentication failure on tampered ciphertext")
}
}
func TestBadMagic(t *testing.T) {
_, _, err := ReadHeader(bytes.NewReader([]byte("NOTLARC.....")))
if err == nil {
t.Fatalf("expected bad-magic error")
}
}
func TestReadHeaderFields(t *testing.T) {
armoredPub, _ := genTestKey(t)
pub, _ := LoadPublicKey(armoredPub)
var sealed bytes.Buffer
if _, err := Seal(&sealed, []byte("x\n"), pub, "logarchive", 4096); err != nil {
t.Fatalf("Seal: %v", err)
}
hdr, wrapped, err := ReadHeader(bytes.NewReader(sealed.Bytes()))
if err != nil {
t.Fatalf("ReadHeader: %v", err)
}
if hdr.KeyName != "logarchive" {
t.Errorf("KeyName = %q", hdr.KeyName)
}
if hdr.Compression != "zstd" || hdr.Cipher != "AES-256-GCM" {
t.Errorf("algo metadata wrong: %+v", hdr)
}
if len(wrapped) != hdr.WrappedDEKLen {
t.Errorf("wrapped len %d != header %d", len(wrapped), hdr.WrappedDEKLen)
}
}
func TestDigestArmoredStable(t *testing.T) {
// Guards the test helper used elsewhere; identical input -> identical digest.
a := digestArmored([]byte("abc"))
b := digestArmored([]byte("abc"))
if a != b || a == "" {
t.Errorf("digestArmored not stable: %q %q", a, b)
}
}
+338
View File
@@ -0,0 +1,338 @@
package crypto
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/binary"
"encoding/json"
"fmt"
"io"
"github.com/ProtonMail/go-crypto/openpgp"
"github.com/ProtonMail/go-crypto/openpgp/packet"
"github.com/klauspost/compress/zstd"
)
// ContainerMagic identifies a logarchiver object and its container version.
var ContainerMagic = []byte("LARC1\n")
const (
dekSize = 32 // AES-256
noncePrefixSize = 4
counterSize = 8
// maxFrameCiphertext bounds a single frame read to avoid unbounded allocation
// from a corrupt/hostile length prefix.
maxFrameCiphertext = 128 << 20
)
// Header is the LARC1 object header (JSON), written after the magic and a
// uint32 big-endian length prefix.
type Header struct {
Version int `json:"v"`
KeyName string `json:"key_name"`
KeyFingerprint string `json:"key_fingerprint"`
WrappedDEKLen int `json:"wrapped_dek_len"`
NoncePrefix []byte `json:"nonce_prefix"` // base64 in JSON
FrameSize int `json:"frame_size"` // plaintext (compressed) bytes per frame
Compression string `json:"compression"` // "zstd"
Cipher string `json:"cipher"` // "AES-256-GCM"
}
// SealResult reports what Seal produced (for the index row).
type SealResult struct {
Header Header
RawBytes int64 // input NDJSON length
StoredBytes int64 // full container length
}
// Seal compresses plaintext with zstd, encrypts it under a fresh random DEK
// using framed AES-256-GCM, wraps the DEK to pub with OpenPGP, and writes the
// LARC1 container to w. keyName is recorded in the header for operator context.
func Seal(w io.Writer, plaintext []byte, pub *PublicKey, keyName string, frameSize int) (SealResult, error) {
if pub == nil {
return SealResult{}, fmt.Errorf("nil public key")
}
if frameSize <= 0 {
frameSize = 1 << 20
}
// 1. Compress.
enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBetterCompression))
if err != nil {
return SealResult{}, fmt.Errorf("zstd writer: %w", err)
}
compressed := enc.EncodeAll(plaintext, nil)
_ = enc.Close()
// 2. DEK + nonce prefix.
dek := make([]byte, dekSize)
if _, err := rand.Read(dek); err != nil {
return SealResult{}, fmt.Errorf("gen dek: %w", err)
}
noncePrefix := make([]byte, noncePrefixSize)
if _, err := rand.Read(noncePrefix); err != nil {
return SealResult{}, fmt.Errorf("gen nonce prefix: %w", err)
}
// 3. Wrap the DEK to the public key (small standard OpenPGP message).
wrapped, err := wrapDEK(dek, pub)
if err != nil {
return SealResult{}, err
}
block, err := aes.NewCipher(dek)
if err != nil {
return SealResult{}, fmt.Errorf("aes cipher: %w", err)
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return SealResult{}, fmt.Errorf("gcm: %w", err)
}
hdr := Header{
Version: 1,
KeyName: keyName,
KeyFingerprint: pub.Fingerprint,
WrappedDEKLen: len(wrapped),
NoncePrefix: noncePrefix,
FrameSize: frameSize,
Compression: "zstd",
Cipher: "AES-256-GCM",
}
hdrJSON, err := json.Marshal(hdr)
if err != nil {
return SealResult{}, fmt.Errorf("marshal header: %w", err)
}
cw := &countingWriter{w: w}
// magic
if _, err := cw.Write(ContainerMagic); err != nil {
return SealResult{}, err
}
// header length + header
if err := writeUint32(cw, uint32(len(hdrJSON))); err != nil {
return SealResult{}, err
}
if _, err := cw.Write(hdrJSON); err != nil {
return SealResult{}, err
}
// wrapped DEK
if _, err := cw.Write(wrapped); err != nil {
return SealResult{}, err
}
// frames
for i, off := 0, 0; off < len(compressed); i++ {
end := off + frameSize
if end > len(compressed) {
end = len(compressed)
}
nonce := frameNonce(noncePrefix, uint64(i))
aad := aadFor(uint64(i))
ct := gcm.Seal(nil, nonce, compressed[off:end], aad)
if err := writeUint32(cw, uint32(len(ct))); err != nil {
return SealResult{}, err
}
if _, err := cw.Write(ct); err != nil {
return SealResult{}, err
}
off = end
}
// terminating zero-length frame
if err := writeUint32(cw, 0); err != nil {
return SealResult{}, err
}
return SealResult{Header: hdr, RawBytes: int64(len(plaintext)), StoredBytes: cw.n}, nil
}
// wrapDEK OpenPGP-encrypts the DEK to pub, producing a compact binary message.
func wrapDEK(dek []byte, pub *PublicKey) ([]byte, error) {
var buf bytes.Buffer
cfg := &packet.Config{
DefaultCipher: packet.CipherAES256,
}
wc, err := openpgp.Encrypt(&buf, []*openpgp.Entity{pub.entity}, nil, nil, cfg)
if err != nil {
return nil, fmt.Errorf("openpgp encrypt dek: %w", err)
}
if _, err := wc.Write(dek); err != nil {
return nil, fmt.Errorf("write dek: %w", err)
}
if err := wc.Close(); err != nil {
return nil, fmt.Errorf("close openpgp: %w", err)
}
return buf.Bytes(), nil
}
// UnwrapFunc recovers the DEK from the wrapped OpenPGP blob. In production this
// calls the Vault GPG engine decrypt endpoint; tests supply a local one.
type UnwrapFunc func(wrappedDEK []byte) (dek []byte, err error)
// ReadHeader reads and validates the LARC1 magic + header and the wrapped DEK,
// leaving r positioned at the first frame. It does not require decryption keys,
// so it is cheap for `search`/metadata inspection.
func ReadHeader(r io.Reader) (Header, []byte, error) {
magic := make([]byte, len(ContainerMagic))
if _, err := io.ReadFull(r, magic); err != nil {
return Header{}, nil, fmt.Errorf("read magic: %w", err)
}
if !bytes.Equal(magic, ContainerMagic) {
return Header{}, nil, fmt.Errorf("bad magic: not a logarchiver (LARC1) object")
}
hlen, err := readUint32(r)
if err != nil {
return Header{}, nil, fmt.Errorf("read header len: %w", err)
}
if hlen == 0 || hlen > 1<<20 {
return Header{}, nil, fmt.Errorf("implausible header length %d", hlen)
}
hdrJSON := make([]byte, hlen)
if _, err := io.ReadFull(r, hdrJSON); err != nil {
return Header{}, nil, fmt.Errorf("read header: %w", err)
}
var hdr Header
if err := json.Unmarshal(hdrJSON, &hdr); err != nil {
return Header{}, nil, fmt.Errorf("parse header: %w", err)
}
if hdr.Version != 1 {
return Header{}, nil, fmt.Errorf("unsupported container version %d", hdr.Version)
}
if hdr.WrappedDEKLen <= 0 || hdr.WrappedDEKLen > 1<<20 {
return Header{}, nil, fmt.Errorf("implausible wrapped dek length %d", hdr.WrappedDEKLen)
}
wrapped := make([]byte, hdr.WrappedDEKLen)
if _, err := io.ReadFull(r, wrapped); err != nil {
return Header{}, nil, fmt.Errorf("read wrapped dek: %w", err)
}
return hdr, wrapped, nil
}
// Open reads a LARC1 container from r, recovers the DEK via unwrap, and streams
// the decrypted, decompressed NDJSON to w.
func Open(r io.Reader, w io.Writer, unwrap UnwrapFunc) error {
hdr, wrapped, err := ReadHeader(r)
if err != nil {
return err
}
dek, err := unwrap(wrapped)
if err != nil {
return fmt.Errorf("unwrap dek: %w", err)
}
if len(dek) != dekSize {
return fmt.Errorf("unwrapped dek has wrong length %d", len(dek))
}
block, err := aes.NewCipher(dek)
if err != nil {
return fmt.Errorf("aes cipher: %w", err)
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return fmt.Errorf("gcm: %w", err)
}
fr := &frameReader{r: r, gcm: gcm, noncePrefix: hdr.NoncePrefix}
zr, err := zstd.NewReader(fr)
if err != nil {
return fmt.Errorf("zstd reader: %w", err)
}
defer zr.Close()
if _, err := io.Copy(w, zr); err != nil {
return fmt.Errorf("decompress: %w", err)
}
return nil
}
// frameReader decrypts LARC1 frames on demand, presenting the decrypted
// (compressed) bytes as an io.Reader for the zstd decoder.
type frameReader struct {
r io.Reader
gcm cipher.AEAD
noncePrefix []byte
counter uint64
buf []byte // leftover decrypted plaintext not yet consumed
done bool
}
func (f *frameReader) Read(p []byte) (int, error) {
if len(f.buf) == 0 && !f.done {
if err := f.next(); err != nil {
return 0, err
}
}
if len(f.buf) == 0 {
return 0, io.EOF
}
n := copy(p, f.buf)
f.buf = f.buf[n:]
return n, nil
}
func (f *frameReader) next() error {
ln, err := readUint32(f.r)
if err != nil {
return fmt.Errorf("read frame len: %w", err)
}
if ln == 0 { // terminator
f.done = true
return nil
}
if ln > maxFrameCiphertext {
return fmt.Errorf("frame length %d exceeds max", ln)
}
ct := make([]byte, ln)
if _, err := io.ReadFull(f.r, ct); err != nil {
return fmt.Errorf("read frame: %w", err)
}
nonce := frameNonce(f.noncePrefix, f.counter)
aad := aadFor(f.counter)
pt, err := f.gcm.Open(nil, nonce, ct, aad)
if err != nil {
return fmt.Errorf("decrypt frame %d: %w", f.counter, err)
}
f.counter++
f.buf = pt
return nil
}
func frameNonce(prefix []byte, counter uint64) []byte {
nonce := make([]byte, noncePrefixSize+counterSize)
copy(nonce, prefix)
binary.BigEndian.PutUint64(nonce[noncePrefixSize:], counter)
return nonce
}
func aadFor(counter uint64) []byte {
aad := make([]byte, counterSize)
binary.BigEndian.PutUint64(aad, counter)
return aad
}
func writeUint32(w io.Writer, v uint32) error {
var b [4]byte
binary.BigEndian.PutUint32(b[:], v)
_, err := w.Write(b[:])
return err
}
func readUint32(r io.Reader) (uint32, error) {
var b [4]byte
if _, err := io.ReadFull(r, b[:]); err != nil {
return 0, err
}
return binary.BigEndian.Uint32(b[:]), nil
}
type countingWriter struct {
w io.Writer
n int64
}
func (c *countingWriter) Write(p []byte) (int, error) {
n, err := c.w.Write(p)
c.n += int64(n)
return n, err
}
+81
View File
@@ -0,0 +1,81 @@
// Package crypto implements logarchiver's object encryption.
//
// # Why not plain OpenPGP-encrypt the whole object?
//
// The private key lives only in Ben's Vault GPG secrets engine
// (vault-plugin-secrets-gpg). That engine's decrypt endpoint does WHOLE-payload
// inline decryption only: you POST the entire OpenPGP message (base64 in a JSON
// body) and it returns the entire plaintext (base64). There is no session-key /
// PKESK extraction and no streaming, so a multi-hundred-MiB archive could not be
// retrieved without blowing Vault's request-size limit and buffering everything
// twice in the server.
//
// # The wrapped-DEK envelope (container "LARC1")
//
// logarchiver therefore does hybrid encryption itself:
//
// - a fresh random 256-bit Data Encryption Key (DEK) per object;
// - the bulk (zstd-compressed NDJSON) is encrypted locally with AES-256-GCM in
// independent frames, so decryption streams frame-by-frame;
// - only the 32-byte DEK is OpenPGP-encrypted to the engine's PUBLIC key,
// producing a small (~hundreds of bytes) standard OpenPGP message.
//
// On retrieval the CLI sends ONLY that small wrapped-DEK blob to the engine's
// decrypt endpoint, recovers the DEK, and streams the bulk locally. The Vault
// round-trip is tiny and constant regardless of object size, and the private key
// never leaves Vault. The trade-off vs. a single standard OpenPGP object: these
// objects are a logarchiver-specific container, not decryptable by a bare `gpg`
// even with the private key. The retrieval runbook documents the format.
package crypto
import (
"crypto/sha256"
"fmt"
"strings"
"github.com/ProtonMail/go-crypto/openpgp"
"github.com/ProtonMail/go-crypto/openpgp/armor"
)
// PublicKey is a parsed OpenPGP public key plus its fingerprint (uppercase hex,
// no spaces — matching the Vault GPG engine's `%X` fingerprint format).
type PublicKey struct {
entity *openpgp.Entity
Fingerprint string
}
// LoadPublicKey parses an ASCII-armored (or binary) OpenPGP public key.
func LoadPublicKey(data []byte) (*PublicKey, error) {
var keyring openpgp.EntityList
var err error
if strings.Contains(string(data), "BEGIN PGP") {
block, berr := armor.Decode(strings.NewReader(string(data)))
if berr != nil {
return nil, fmt.Errorf("decode armor: %w", berr)
}
keyring, err = openpgp.ReadKeyRing(block.Body)
} else {
keyring, err = openpgp.ReadKeyRing(strings.NewReader(string(data)))
}
if err != nil {
return nil, fmt.Errorf("read public key: %w", err)
}
if len(keyring) == 0 {
return nil, fmt.Errorf("no public key found")
}
ent := keyring[0]
if ent.PrimaryKey == nil {
return nil, fmt.Errorf("key has no primary public key")
}
return &PublicKey{
entity: ent,
Fingerprint: fmt.Sprintf("%X", ent.PrimaryKey.Fingerprint),
}, nil
}
// digestArmored is used by tests to sanity check key identity independent of
// go-crypto internals.
func digestArmored(data []byte) string {
sum := sha256.Sum256(data)
return fmt.Sprintf("%x", sum[:8])
}