Initial implementation: NATS->S3 archiver + search/retrieve CLI
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
This commit is contained in:
@@ -0,0 +1,165 @@
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ProtonMail/go-crypto/openpgp"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/armor"
|
||||
)
|
||||
|
||||
// genTestKey creates an OpenPGP keypair, returns the armored public key (as the
|
||||
// service would export from Vault) and an unwrap func that decrypts the wrapped
|
||||
// DEK with the private key — simulating the Vault GPG engine's decrypt endpoint
|
||||
// (which returns the plaintext of a whole OpenPGP message).
|
||||
func genTestKey(t *testing.T) (armoredPub []byte, unwrap UnwrapFunc) {
|
||||
t.Helper()
|
||||
ent, err := openpgp.NewEntity("logarchiver-test", "unit test", "test@unkin.net", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEntity: %v", err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
w, err := armor.Encode(&buf, openpgp.PublicKeyType, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("armor encode: %v", err)
|
||||
}
|
||||
if err := ent.Serialize(w); err != nil {
|
||||
t.Fatalf("serialize public: %v", err)
|
||||
}
|
||||
_ = w.Close()
|
||||
|
||||
unwrap = func(wrapped []byte) ([]byte, error) {
|
||||
md, err := openpgp.ReadMessage(bytes.NewReader(wrapped), openpgp.EntityList{ent}, nil, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.ReadAll(md.UnverifiedBody)
|
||||
}
|
||||
return buf.Bytes(), unwrap
|
||||
}
|
||||
|
||||
func TestRoundTrip(t *testing.T) {
|
||||
armoredPub, unwrap := genTestKey(t)
|
||||
pub, err := LoadPublicKey(armoredPub)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadPublicKey: %v", err)
|
||||
}
|
||||
if len(pub.Fingerprint) != 40 {
|
||||
t.Errorf("fingerprint = %q, want 40 hex chars", pub.Fingerprint)
|
||||
}
|
||||
if pub.Fingerprint != strings.ToUpper(pub.Fingerprint) {
|
||||
t.Errorf("fingerprint should be uppercase: %q", pub.Fingerprint)
|
||||
}
|
||||
|
||||
// A multi-line NDJSON payload larger than the frame size (forces >1 frame).
|
||||
var payload bytes.Buffer
|
||||
for i := 0; i < 5000; i++ {
|
||||
payload.WriteString(`{"host":"node-1","message":"line `)
|
||||
payload.WriteString(strings.Repeat("x", 50))
|
||||
payload.WriteString(`"}` + "\n")
|
||||
}
|
||||
plaintext := payload.Bytes()
|
||||
|
||||
var sealed bytes.Buffer
|
||||
res, err := Seal(&sealed, plaintext, pub, "logarchive", 4096)
|
||||
if err != nil {
|
||||
t.Fatalf("Seal: %v", err)
|
||||
}
|
||||
if res.RawBytes != int64(len(plaintext)) {
|
||||
t.Errorf("RawBytes = %d, want %d", res.RawBytes, len(plaintext))
|
||||
}
|
||||
if int64(sealed.Len()) != res.StoredBytes {
|
||||
t.Errorf("StoredBytes = %d, buffer = %d", res.StoredBytes, sealed.Len())
|
||||
}
|
||||
// Compression should shrink this highly repetitive payload.
|
||||
if res.StoredBytes >= res.RawBytes {
|
||||
t.Errorf("stored (%d) not smaller than raw (%d)", res.StoredBytes, res.RawBytes)
|
||||
}
|
||||
// Design property: only a tiny wrapped DEK goes to Vault, regardless of size.
|
||||
if res.Header.WrappedDEKLen > 4096 {
|
||||
t.Errorf("wrapped DEK unexpectedly large: %d bytes", res.Header.WrappedDEKLen)
|
||||
}
|
||||
if res.Header.KeyFingerprint != pub.Fingerprint {
|
||||
t.Errorf("header fingerprint mismatch")
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := Open(bytes.NewReader(sealed.Bytes()), &out, unwrap); err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
if !bytes.Equal(out.Bytes(), plaintext) {
|
||||
t.Fatalf("round-trip mismatch: got %d bytes, want %d", out.Len(), len(plaintext))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoundTripEmpty(t *testing.T) {
|
||||
armoredPub, unwrap := genTestKey(t)
|
||||
pub, _ := LoadPublicKey(armoredPub)
|
||||
var sealed bytes.Buffer
|
||||
if _, err := Seal(&sealed, []byte{}, pub, "k", 4096); err != nil {
|
||||
t.Fatalf("Seal empty: %v", err)
|
||||
}
|
||||
var out bytes.Buffer
|
||||
if err := Open(bytes.NewReader(sealed.Bytes()), &out, unwrap); err != nil {
|
||||
t.Fatalf("Open empty: %v", err)
|
||||
}
|
||||
if out.Len() != 0 {
|
||||
t.Errorf("empty round-trip produced %d bytes", out.Len())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTamperDetected(t *testing.T) {
|
||||
armoredPub, unwrap := genTestKey(t)
|
||||
pub, _ := LoadPublicKey(armoredPub)
|
||||
var sealed bytes.Buffer
|
||||
if _, err := Seal(&sealed, []byte("hello world\n"), pub, "k", 4096); err != nil {
|
||||
t.Fatalf("Seal: %v", err)
|
||||
}
|
||||
data := sealed.Bytes()
|
||||
// Flip a byte near the end (inside a frame's ciphertext/tag).
|
||||
data[len(data)-3] ^= 0xff
|
||||
var out bytes.Buffer
|
||||
if err := Open(bytes.NewReader(data), &out, unwrap); err == nil {
|
||||
t.Fatalf("expected GCM authentication failure on tampered ciphertext")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBadMagic(t *testing.T) {
|
||||
_, _, err := ReadHeader(bytes.NewReader([]byte("NOTLARC.....")))
|
||||
if err == nil {
|
||||
t.Fatalf("expected bad-magic error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadHeaderFields(t *testing.T) {
|
||||
armoredPub, _ := genTestKey(t)
|
||||
pub, _ := LoadPublicKey(armoredPub)
|
||||
var sealed bytes.Buffer
|
||||
if _, err := Seal(&sealed, []byte("x\n"), pub, "logarchive", 4096); err != nil {
|
||||
t.Fatalf("Seal: %v", err)
|
||||
}
|
||||
hdr, wrapped, err := ReadHeader(bytes.NewReader(sealed.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadHeader: %v", err)
|
||||
}
|
||||
if hdr.KeyName != "logarchive" {
|
||||
t.Errorf("KeyName = %q", hdr.KeyName)
|
||||
}
|
||||
if hdr.Compression != "zstd" || hdr.Cipher != "AES-256-GCM" {
|
||||
t.Errorf("algo metadata wrong: %+v", hdr)
|
||||
}
|
||||
if len(wrapped) != hdr.WrappedDEKLen {
|
||||
t.Errorf("wrapped len %d != header %d", len(wrapped), hdr.WrappedDEKLen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDigestArmoredStable(t *testing.T) {
|
||||
// Guards the test helper used elsewhere; identical input -> identical digest.
|
||||
a := digestArmored([]byte("abc"))
|
||||
b := digestArmored([]byte("abc"))
|
||||
if a != b || a == "" {
|
||||
t.Errorf("digestArmored not stable: %q %q", a, b)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/ProtonMail/go-crypto/openpgp"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/packet"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// ContainerMagic identifies a logarchiver object and its container version.
|
||||
var ContainerMagic = []byte("LARC1\n")
|
||||
|
||||
const (
|
||||
dekSize = 32 // AES-256
|
||||
noncePrefixSize = 4
|
||||
counterSize = 8
|
||||
// maxFrameCiphertext bounds a single frame read to avoid unbounded allocation
|
||||
// from a corrupt/hostile length prefix.
|
||||
maxFrameCiphertext = 128 << 20
|
||||
)
|
||||
|
||||
// Header is the LARC1 object header (JSON), written after the magic and a
|
||||
// uint32 big-endian length prefix.
|
||||
type Header struct {
|
||||
Version int `json:"v"`
|
||||
KeyName string `json:"key_name"`
|
||||
KeyFingerprint string `json:"key_fingerprint"`
|
||||
WrappedDEKLen int `json:"wrapped_dek_len"`
|
||||
NoncePrefix []byte `json:"nonce_prefix"` // base64 in JSON
|
||||
FrameSize int `json:"frame_size"` // plaintext (compressed) bytes per frame
|
||||
Compression string `json:"compression"` // "zstd"
|
||||
Cipher string `json:"cipher"` // "AES-256-GCM"
|
||||
}
|
||||
|
||||
// SealResult reports what Seal produced (for the index row).
|
||||
type SealResult struct {
|
||||
Header Header
|
||||
RawBytes int64 // input NDJSON length
|
||||
StoredBytes int64 // full container length
|
||||
}
|
||||
|
||||
// Seal compresses plaintext with zstd, encrypts it under a fresh random DEK
|
||||
// using framed AES-256-GCM, wraps the DEK to pub with OpenPGP, and writes the
|
||||
// LARC1 container to w. keyName is recorded in the header for operator context.
|
||||
func Seal(w io.Writer, plaintext []byte, pub *PublicKey, keyName string, frameSize int) (SealResult, error) {
|
||||
if pub == nil {
|
||||
return SealResult{}, fmt.Errorf("nil public key")
|
||||
}
|
||||
if frameSize <= 0 {
|
||||
frameSize = 1 << 20
|
||||
}
|
||||
|
||||
// 1. Compress.
|
||||
enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBetterCompression))
|
||||
if err != nil {
|
||||
return SealResult{}, fmt.Errorf("zstd writer: %w", err)
|
||||
}
|
||||
compressed := enc.EncodeAll(plaintext, nil)
|
||||
_ = enc.Close()
|
||||
|
||||
// 2. DEK + nonce prefix.
|
||||
dek := make([]byte, dekSize)
|
||||
if _, err := rand.Read(dek); err != nil {
|
||||
return SealResult{}, fmt.Errorf("gen dek: %w", err)
|
||||
}
|
||||
noncePrefix := make([]byte, noncePrefixSize)
|
||||
if _, err := rand.Read(noncePrefix); err != nil {
|
||||
return SealResult{}, fmt.Errorf("gen nonce prefix: %w", err)
|
||||
}
|
||||
|
||||
// 3. Wrap the DEK to the public key (small standard OpenPGP message).
|
||||
wrapped, err := wrapDEK(dek, pub)
|
||||
if err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
|
||||
block, err := aes.NewCipher(dek)
|
||||
if err != nil {
|
||||
return SealResult{}, fmt.Errorf("aes cipher: %w", err)
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return SealResult{}, fmt.Errorf("gcm: %w", err)
|
||||
}
|
||||
|
||||
hdr := Header{
|
||||
Version: 1,
|
||||
KeyName: keyName,
|
||||
KeyFingerprint: pub.Fingerprint,
|
||||
WrappedDEKLen: len(wrapped),
|
||||
NoncePrefix: noncePrefix,
|
||||
FrameSize: frameSize,
|
||||
Compression: "zstd",
|
||||
Cipher: "AES-256-GCM",
|
||||
}
|
||||
hdrJSON, err := json.Marshal(hdr)
|
||||
if err != nil {
|
||||
return SealResult{}, fmt.Errorf("marshal header: %w", err)
|
||||
}
|
||||
|
||||
cw := &countingWriter{w: w}
|
||||
|
||||
// magic
|
||||
if _, err := cw.Write(ContainerMagic); err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
// header length + header
|
||||
if err := writeUint32(cw, uint32(len(hdrJSON))); err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
if _, err := cw.Write(hdrJSON); err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
// wrapped DEK
|
||||
if _, err := cw.Write(wrapped); err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
|
||||
// frames
|
||||
for i, off := 0, 0; off < len(compressed); i++ {
|
||||
end := off + frameSize
|
||||
if end > len(compressed) {
|
||||
end = len(compressed)
|
||||
}
|
||||
nonce := frameNonce(noncePrefix, uint64(i))
|
||||
aad := aadFor(uint64(i))
|
||||
ct := gcm.Seal(nil, nonce, compressed[off:end], aad)
|
||||
if err := writeUint32(cw, uint32(len(ct))); err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
if _, err := cw.Write(ct); err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
off = end
|
||||
}
|
||||
// terminating zero-length frame
|
||||
if err := writeUint32(cw, 0); err != nil {
|
||||
return SealResult{}, err
|
||||
}
|
||||
|
||||
return SealResult{Header: hdr, RawBytes: int64(len(plaintext)), StoredBytes: cw.n}, nil
|
||||
}
|
||||
|
||||
// wrapDEK OpenPGP-encrypts the DEK to pub, producing a compact binary message.
|
||||
func wrapDEK(dek []byte, pub *PublicKey) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
cfg := &packet.Config{
|
||||
DefaultCipher: packet.CipherAES256,
|
||||
}
|
||||
wc, err := openpgp.Encrypt(&buf, []*openpgp.Entity{pub.entity}, nil, nil, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("openpgp encrypt dek: %w", err)
|
||||
}
|
||||
if _, err := wc.Write(dek); err != nil {
|
||||
return nil, fmt.Errorf("write dek: %w", err)
|
||||
}
|
||||
if err := wc.Close(); err != nil {
|
||||
return nil, fmt.Errorf("close openpgp: %w", err)
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// UnwrapFunc recovers the DEK from the wrapped OpenPGP blob. In production this
|
||||
// calls the Vault GPG engine decrypt endpoint; tests supply a local one.
|
||||
type UnwrapFunc func(wrappedDEK []byte) (dek []byte, err error)
|
||||
|
||||
// ReadHeader reads and validates the LARC1 magic + header and the wrapped DEK,
|
||||
// leaving r positioned at the first frame. It does not require decryption keys,
|
||||
// so it is cheap for `search`/metadata inspection.
|
||||
func ReadHeader(r io.Reader) (Header, []byte, error) {
|
||||
magic := make([]byte, len(ContainerMagic))
|
||||
if _, err := io.ReadFull(r, magic); err != nil {
|
||||
return Header{}, nil, fmt.Errorf("read magic: %w", err)
|
||||
}
|
||||
if !bytes.Equal(magic, ContainerMagic) {
|
||||
return Header{}, nil, fmt.Errorf("bad magic: not a logarchiver (LARC1) object")
|
||||
}
|
||||
hlen, err := readUint32(r)
|
||||
if err != nil {
|
||||
return Header{}, nil, fmt.Errorf("read header len: %w", err)
|
||||
}
|
||||
if hlen == 0 || hlen > 1<<20 {
|
||||
return Header{}, nil, fmt.Errorf("implausible header length %d", hlen)
|
||||
}
|
||||
hdrJSON := make([]byte, hlen)
|
||||
if _, err := io.ReadFull(r, hdrJSON); err != nil {
|
||||
return Header{}, nil, fmt.Errorf("read header: %w", err)
|
||||
}
|
||||
var hdr Header
|
||||
if err := json.Unmarshal(hdrJSON, &hdr); err != nil {
|
||||
return Header{}, nil, fmt.Errorf("parse header: %w", err)
|
||||
}
|
||||
if hdr.Version != 1 {
|
||||
return Header{}, nil, fmt.Errorf("unsupported container version %d", hdr.Version)
|
||||
}
|
||||
if hdr.WrappedDEKLen <= 0 || hdr.WrappedDEKLen > 1<<20 {
|
||||
return Header{}, nil, fmt.Errorf("implausible wrapped dek length %d", hdr.WrappedDEKLen)
|
||||
}
|
||||
wrapped := make([]byte, hdr.WrappedDEKLen)
|
||||
if _, err := io.ReadFull(r, wrapped); err != nil {
|
||||
return Header{}, nil, fmt.Errorf("read wrapped dek: %w", err)
|
||||
}
|
||||
return hdr, wrapped, nil
|
||||
}
|
||||
|
||||
// Open reads a LARC1 container from r, recovers the DEK via unwrap, and streams
|
||||
// the decrypted, decompressed NDJSON to w.
|
||||
func Open(r io.Reader, w io.Writer, unwrap UnwrapFunc) error {
|
||||
hdr, wrapped, err := ReadHeader(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dek, err := unwrap(wrapped)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unwrap dek: %w", err)
|
||||
}
|
||||
if len(dek) != dekSize {
|
||||
return fmt.Errorf("unwrapped dek has wrong length %d", len(dek))
|
||||
}
|
||||
block, err := aes.NewCipher(dek)
|
||||
if err != nil {
|
||||
return fmt.Errorf("aes cipher: %w", err)
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return fmt.Errorf("gcm: %w", err)
|
||||
}
|
||||
|
||||
fr := &frameReader{r: r, gcm: gcm, noncePrefix: hdr.NoncePrefix}
|
||||
zr, err := zstd.NewReader(fr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("zstd reader: %w", err)
|
||||
}
|
||||
defer zr.Close()
|
||||
if _, err := io.Copy(w, zr); err != nil {
|
||||
return fmt.Errorf("decompress: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// frameReader decrypts LARC1 frames on demand, presenting the decrypted
|
||||
// (compressed) bytes as an io.Reader for the zstd decoder.
|
||||
type frameReader struct {
|
||||
r io.Reader
|
||||
gcm cipher.AEAD
|
||||
noncePrefix []byte
|
||||
counter uint64
|
||||
buf []byte // leftover decrypted plaintext not yet consumed
|
||||
done bool
|
||||
}
|
||||
|
||||
func (f *frameReader) Read(p []byte) (int, error) {
|
||||
if len(f.buf) == 0 && !f.done {
|
||||
if err := f.next(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if len(f.buf) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n := copy(p, f.buf)
|
||||
f.buf = f.buf[n:]
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (f *frameReader) next() error {
|
||||
ln, err := readUint32(f.r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read frame len: %w", err)
|
||||
}
|
||||
if ln == 0 { // terminator
|
||||
f.done = true
|
||||
return nil
|
||||
}
|
||||
if ln > maxFrameCiphertext {
|
||||
return fmt.Errorf("frame length %d exceeds max", ln)
|
||||
}
|
||||
ct := make([]byte, ln)
|
||||
if _, err := io.ReadFull(f.r, ct); err != nil {
|
||||
return fmt.Errorf("read frame: %w", err)
|
||||
}
|
||||
nonce := frameNonce(f.noncePrefix, f.counter)
|
||||
aad := aadFor(f.counter)
|
||||
pt, err := f.gcm.Open(nil, nonce, ct, aad)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decrypt frame %d: %w", f.counter, err)
|
||||
}
|
||||
f.counter++
|
||||
f.buf = pt
|
||||
return nil
|
||||
}
|
||||
|
||||
func frameNonce(prefix []byte, counter uint64) []byte {
|
||||
nonce := make([]byte, noncePrefixSize+counterSize)
|
||||
copy(nonce, prefix)
|
||||
binary.BigEndian.PutUint64(nonce[noncePrefixSize:], counter)
|
||||
return nonce
|
||||
}
|
||||
|
||||
func aadFor(counter uint64) []byte {
|
||||
aad := make([]byte, counterSize)
|
||||
binary.BigEndian.PutUint64(aad, counter)
|
||||
return aad
|
||||
}
|
||||
|
||||
func writeUint32(w io.Writer, v uint32) error {
|
||||
var b [4]byte
|
||||
binary.BigEndian.PutUint32(b[:], v)
|
||||
_, err := w.Write(b[:])
|
||||
return err
|
||||
}
|
||||
|
||||
func readUint32(r io.Reader) (uint32, error) {
|
||||
var b [4]byte
|
||||
if _, err := io.ReadFull(r, b[:]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return binary.BigEndian.Uint32(b[:]), nil
|
||||
}
|
||||
|
||||
type countingWriter struct {
|
||||
w io.Writer
|
||||
n int64
|
||||
}
|
||||
|
||||
func (c *countingWriter) Write(p []byte) (int, error) {
|
||||
n, err := c.w.Write(p)
|
||||
c.n += int64(n)
|
||||
return n, err
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
// Package crypto implements logarchiver's object encryption.
|
||||
//
|
||||
// # Why not plain OpenPGP-encrypt the whole object?
|
||||
//
|
||||
// The private key lives only in Ben's Vault GPG secrets engine
|
||||
// (vault-plugin-secrets-gpg). That engine's decrypt endpoint does WHOLE-payload
|
||||
// inline decryption only: you POST the entire OpenPGP message (base64 in a JSON
|
||||
// body) and it returns the entire plaintext (base64). There is no session-key /
|
||||
// PKESK extraction and no streaming, so a multi-hundred-MiB archive could not be
|
||||
// retrieved without blowing Vault's request-size limit and buffering everything
|
||||
// twice in the server.
|
||||
//
|
||||
// # The wrapped-DEK envelope (container "LARC1")
|
||||
//
|
||||
// logarchiver therefore does hybrid encryption itself:
|
||||
//
|
||||
// - a fresh random 256-bit Data Encryption Key (DEK) per object;
|
||||
// - the bulk (zstd-compressed NDJSON) is encrypted locally with AES-256-GCM in
|
||||
// independent frames, so decryption streams frame-by-frame;
|
||||
// - only the 32-byte DEK is OpenPGP-encrypted to the engine's PUBLIC key,
|
||||
// producing a small (~hundreds of bytes) standard OpenPGP message.
|
||||
//
|
||||
// On retrieval the CLI sends ONLY that small wrapped-DEK blob to the engine's
|
||||
// decrypt endpoint, recovers the DEK, and streams the bulk locally. The Vault
|
||||
// round-trip is tiny and constant regardless of object size, and the private key
|
||||
// never leaves Vault. The trade-off vs. a single standard OpenPGP object: these
|
||||
// objects are a logarchiver-specific container, not decryptable by a bare `gpg`
|
||||
// even with the private key. The retrieval runbook documents the format.
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/ProtonMail/go-crypto/openpgp"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/armor"
|
||||
)
|
||||
|
||||
// PublicKey is a parsed OpenPGP public key plus its fingerprint (uppercase hex,
|
||||
// no spaces — matching the Vault GPG engine's `%X` fingerprint format).
|
||||
type PublicKey struct {
|
||||
entity *openpgp.Entity
|
||||
Fingerprint string
|
||||
}
|
||||
|
||||
// LoadPublicKey parses an ASCII-armored (or binary) OpenPGP public key.
|
||||
func LoadPublicKey(data []byte) (*PublicKey, error) {
|
||||
var keyring openpgp.EntityList
|
||||
var err error
|
||||
if strings.Contains(string(data), "BEGIN PGP") {
|
||||
block, berr := armor.Decode(strings.NewReader(string(data)))
|
||||
if berr != nil {
|
||||
return nil, fmt.Errorf("decode armor: %w", berr)
|
||||
}
|
||||
keyring, err = openpgp.ReadKeyRing(block.Body)
|
||||
} else {
|
||||
keyring, err = openpgp.ReadKeyRing(strings.NewReader(string(data)))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read public key: %w", err)
|
||||
}
|
||||
if len(keyring) == 0 {
|
||||
return nil, fmt.Errorf("no public key found")
|
||||
}
|
||||
ent := keyring[0]
|
||||
if ent.PrimaryKey == nil {
|
||||
return nil, fmt.Errorf("key has no primary public key")
|
||||
}
|
||||
return &PublicKey{
|
||||
entity: ent,
|
||||
Fingerprint: fmt.Sprintf("%X", ent.PrimaryKey.Fingerprint),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// digestArmored is used by tests to sanity check key identity independent of
|
||||
// go-crypto internals.
|
||||
func digestArmored(data []byte) string {
|
||||
sum := sha256.Sum256(data)
|
||||
return fmt.Sprintf("%x", sum[:8])
|
||||
}
|
||||
Reference in New Issue
Block a user