Initial implementation: NATS->S3 archiver + search/retrieve CLI

logarchiver replaces the plain Vector archiver leg of the centralized
logging stack (argocd-apps #296) with a Go service that archives raw logs
from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed
objects, plus an operator CLI to search the index and retrieve/decrypt
archived logs. It adds the things that outgrew Vector: zstd compression,
encryption keyed from Ben's Vault GPG secrets engine, a searchable
ClickHouse index, and sink-conditional acks (a batch is acknowledged to
JetStream only after the object is durably in S3 AND indexed).

Service (`logarchiver run`):
- Durable JetStream pull consumer (stream LOGS, durable archiver, subject
  filter default logs.k8s.vault.>), explicit acks, independent offsets.
- Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3
  PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and
  redelivered, so nothing is lost on a sink outage.
- Encryption is a wrapped-DEK envelope (container LARC1): the bulk is
  AES-256-GCM framed under a random data key, and only that 32-byte key is
  OpenPGP-encrypted to the engine's public key. This is because the Vault
  GPG engine does whole-payload decrypt only; retrieval round-trips just the
  tiny wrapped key regardless of object size. Public key fetched from the
  engine or a mounted file (configurable); key fingerprint recorded per
  object; periodic pubkey refresh for rotation.
- Prometheus metrics, structured slog, graceful drain on shutdown.

CLI:
- `search` queries the index (subject/host/time) and lists matching objects.
- `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits
  NDJSON (optionally re-filtered by host/time).
- `init-schema` creates/prints the ClickHouse archive_index DDL.
- cobra `completion` subcommands.

Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/
ClickHouse/Vault) behind interfaces with unit tests (config, batching,
host/subject extraction, crypto roundtrip with a test key, ack-after-persist
with fakes, search query building). go build/vet/test -race clean;
golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v*
tag a container image plus a Gitea binary release + rpm-internal RPM. Docs
per subcommand + architecture + retrieval runbook + deployment drop-in.

Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv

internal/archiver/archiver.go

@@ -0,0 +1,161 @@
+// Package archiver turns a ready batch into a stored, indexed object: build
+// NDJSON, seal it into a LARC1 container, PUT it to S3, then write the index
+// row. Store returns an error if ANY step fails; the caller must not ack the
+// batch's messages until Store succeeds (at-least-once, sink-conditional acks).
+package archiver
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"time"
+
+	"git.unkin.net/unkin/logarchiver/internal/batcher"
+	"git.unkin.net/unkin/logarchiver/internal/crypto"
+	"git.unkin.net/unkin/logarchiver/internal/index"
+	"git.unkin.net/unkin/logarchiver/internal/s3store"
+)
+
+// Metrics is the optional metrics sink (implemented by internal/metrics). A nil
+// Metrics is fine (no-op).
+type Metrics interface {
+	ObjectStored(subject string, events int, rawBytes, storedBytes int64)
+	StoreFailed(subject string)
+	IndexFailed(subject string)
+}
+
+// Archiver persists batches.
+type Archiver struct {
+	keys      *KeyBuilder
+	pubkeys   *PubkeyProvider
+	store     s3store.ObjectStore
+	idx       index.Index // may be nil when indexing is disabled
+	keyName   string
+	frameSize int
+	metrics   Metrics
+	nowFn     func() time.Time
+}
+
+// Options configures an Archiver.
+type Options struct {
+	Keys      *KeyBuilder
+	Pubkeys   *PubkeyProvider
+	Store     s3store.ObjectStore
+	Index     index.Index
+	KeyName   string
+	FrameSize int
+	Metrics   Metrics
+}
+
+// New builds an Archiver.
+func New(o Options) (*Archiver, error) {
+	if o.Keys == nil || o.Pubkeys == nil || o.Store == nil {
+		return nil, fmt.Errorf("archiver requires keys, pubkeys and store")
+	}
+	fs := o.FrameSize
+	if fs <= 0 {
+		fs = 1 << 20
+	}
+	return &Archiver{
+		keys:      o.Keys,
+		pubkeys:   o.Pubkeys,
+		store:     o.Store,
+		idx:       o.Index,
+		keyName:   o.KeyName,
+		frameSize: fs,
+		metrics:   o.Metrics,
+		nowFn:     time.Now,
+	}, nil
+}
+
+// StoreResult reports what Store persisted.
+type StoreResult struct {
+	ObjectKey   string
+	Events      int
+	RawBytes    int64
+	StoredBytes int64
+}
+
+// Store seals, uploads and indexes a batch. On success the caller may ack.
+func (a *Archiver) Store(ctx context.Context, batch *batcher.Batch) (StoreResult, error) {
+	if len(batch.Items) == 0 {
+		return StoreResult{}, nil
+	}
+	now := a.nowFn().UTC()
+	summary := batch.Summarize(now)
+
+	pub := a.pubkeys.Current()
+	if pub == nil {
+		a.metricStoreFailed(batch.Subject)
+		return StoreResult{}, fmt.Errorf("no public key available")
+	}
+
+	// Choose the object key from the batch's max timestamp so it lands in the
+	// date partition of the newest event.
+	key, err := a.keys.Build(batch.Subject, summary.MaxTS)
+	if err != nil {
+		a.metricStoreFailed(batch.Subject)
+		return StoreResult{}, err
+	}
+
+	ndjson := batch.NDJSON()
+	var buf bytes.Buffer
+	sealed, err := crypto.Seal(&buf, ndjson, pub, a.keyName, a.frameSize)
+	if err != nil {
+		a.metricStoreFailed(batch.Subject)
+		return StoreResult{}, fmt.Errorf("seal object %s: %w", key, err)
+	}
+
+	if err := a.store.Put(ctx, key, bytes.NewReader(buf.Bytes()), int64(buf.Len())); err != nil {
+		a.metricStoreFailed(batch.Subject)
+		return StoreResult{}, err
+	}
+
+	if a.idx != nil {
+		row := index.Row{
+			ObjectKey:       key,
+			Bucket:          a.store.Bucket(),
+			Subject:         batch.Subject,
+			Hosts:           summary.Hosts,
+			MinTS:           summary.MinTS,
+			MaxTS:           summary.MaxTS,
+			EventCount:      uint64(summary.EventCount),
+			RawBytes:        uint64(sealed.RawBytes),
+			StoredBytes:     uint64(sealed.StoredBytes),
+			Compression:     sealed.Header.Compression,
+			Cipher:          sealed.Header.Cipher,
+			ContainerFormat: "LARC1",
+			KeyName:         a.keyName,
+			KeyFingerprint:  sealed.Header.KeyFingerprint,
+		}
+		if err := a.idx.Insert(ctx, row); err != nil {
+			// The object is in S3 but unindexed. Do NOT ack: on redelivery the
+			// batch is re-stored (a new object key) and re-indexed. The orphan
+			// object is harmless (retrievable by prefix) and reaped by lifecycle.
+			a.metricIndexFailed(batch.Subject)
+			return StoreResult{}, fmt.Errorf("index object %s: %w", key, err)
+		}
+	}
+
+	if a.metrics != nil {
+		a.metrics.ObjectStored(batch.Subject, summary.EventCount, sealed.RawBytes, sealed.StoredBytes)
+	}
+	return StoreResult{
+		ObjectKey:   key,
+		Events:      summary.EventCount,
+		RawBytes:    sealed.RawBytes,
+		StoredBytes: sealed.StoredBytes,
+	}, nil
+}
+
+func (a *Archiver) metricStoreFailed(subject string) {
+	if a.metrics != nil {
+		a.metrics.StoreFailed(subject)
+	}
+}
+
+func (a *Archiver) metricIndexFailed(subject string) {
+	if a.metrics != nil {
+		a.metrics.IndexFailed(subject)
+	}
+}

internal/archiver/archiver_test.go

@@ -0,0 +1,222 @@
+package archiver
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"testing"
+	"time"
+
+	"git.unkin.net/unkin/logarchiver/internal/batcher"
+	"git.unkin.net/unkin/logarchiver/internal/crypto"
+	"git.unkin.net/unkin/logarchiver/internal/index"
+	"github.com/ProtonMail/go-crypto/openpgp"
+	"github.com/ProtonMail/go-crypto/openpgp/armor"
+)
+
+// --- fakes ---
+
+type fakeStore struct {
+	bucket  string
+	objects map[string][]byte
+	failPut bool
+}
+
+func newFakeStore() *fakeStore {
+	return &fakeStore{bucket: "test-bucket", objects: map[string][]byte{}}
+}
+
+func (f *fakeStore) Put(_ context.Context, key string, body io.Reader, _ int64) error {
+	if f.failPut {
+		return errors.New("simulated s3 failure")
+	}
+	data, err := io.ReadAll(body)
+	if err != nil {
+		return err
+	}
+	f.objects[key] = data
+	return nil
+}
+func (f *fakeStore) Get(_ context.Context, key string) (io.ReadCloser, error) {
+	data, ok := f.objects[key]
+	if !ok {
+		return nil, errors.New("not found")
+	}
+	return io.NopCloser(bytes.NewReader(data)), nil
+}
+func (f *fakeStore) List(_ context.Context, _ string) ([]string, error) { return nil, nil }
+func (f *fakeStore) Bucket() string                                     { return f.bucket }
+
+type fakeIndex struct {
+	rows []index.Row
+	fail bool
+}
+
+func (f *fakeIndex) Insert(_ context.Context, row index.Row) error {
+	if f.fail {
+		return errors.New("simulated index failure")
+	}
+	f.rows = append(f.rows, row)
+	return nil
+}
+func (f *fakeIndex) Search(context.Context, index.SearchQuery) ([]index.Result, error) {
+	return nil, nil
+}
+func (f *fakeIndex) InitSchema(context.Context) error { return nil }
+func (f *fakeIndex) Ping(context.Context) error       { return nil }
+func (f *fakeIndex) Close() error                     { return nil }
+
+func testPubkey(t *testing.T) *crypto.PublicKey {
+	t.Helper()
+	ent, err := openpgp.NewEntity("t", "", "t@unkin.net", nil)
+	if err != nil {
+		t.Fatalf("NewEntity: %v", err)
+	}
+	var buf bytes.Buffer
+	w, _ := armor.Encode(&buf, openpgp.PublicKeyType, nil)
+	_ = ent.Serialize(w)
+	_ = w.Close()
+	pk, err := crypto.LoadPublicKey(buf.Bytes())
+	if err != nil {
+		t.Fatalf("LoadPublicKey: %v", err)
+	}
+	return pk
+}
+
+func newTestArchiver(t *testing.T, store *fakeStore, idx index.Index) *Archiver {
+	t.Helper()
+	pk := testPubkey(t)
+	kb, _ := NewKeyBuilder("archive/{{.Subject}}/{{.Year}}/{{.Month}}/{{.Day}}/")
+	prov, err := NewPubkeyProvider(context.Background(), func(context.Context) (*crypto.PublicKey, error) {
+		return pk, nil
+	})
+	if err != nil {
+		t.Fatalf("provider: %v", err)
+	}
+	a, err := New(Options{
+		Keys:      kb,
+		Pubkeys:   prov,
+		Store:     store,
+		Index:     idx,
+		KeyName:   "logarchive",
+		FrameSize: 4096,
+	})
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	return a
+}
+
+func sampleBatch() *batcher.Batch {
+	ts := time.Date(2026, 7, 27, 1, 0, 0, 0, time.UTC)
+	return &batcher.Batch{
+		Subject: "logs.k8s.vault.audit",
+		Items: []batcher.Item{
+			{Subject: "logs.k8s.vault.audit", Host: "node-1", Timestamp: ts, HasTS: true, Raw: []byte(`{"host":"node-1","message":"a"}`)},
+			{Subject: "logs.k8s.vault.audit", Host: "node-2", Timestamp: ts.Add(time.Hour), HasTS: true, Raw: []byte(`{"host":"node-2","message":"b"}`)},
+		},
+		RawBytes: 62,
+	}
+}
+
+func TestStoreSuccessWritesObjectAndIndex(t *testing.T) {
+	store := newFakeStore()
+	idx := &fakeIndex{}
+	a := newTestArchiver(t, store, idx)
+
+	res, err := a.Store(context.Background(), sampleBatch())
+	if err != nil {
+		t.Fatalf("Store: %v", err)
+	}
+	if res.Events != 2 {
+		t.Errorf("events = %d", res.Events)
+	}
+	if len(store.objects) != 1 {
+		t.Fatalf("expected 1 stored object, got %d", len(store.objects))
+	}
+	// Stored bytes must be a valid LARC1 container.
+	obj := store.objects[res.ObjectKey]
+	if _, _, err := crypto.ReadHeader(bytes.NewReader(obj)); err != nil {
+		t.Errorf("stored object is not a valid container: %v", err)
+	}
+	if len(idx.rows) != 1 {
+		t.Fatalf("expected 1 index row, got %d", len(idx.rows))
+	}
+	row := idx.rows[0]
+	if row.Subject != "logs.k8s.vault.audit" {
+		t.Errorf("row subject = %q", row.Subject)
+	}
+	if row.EventCount != 2 {
+		t.Errorf("row event_count = %d", row.EventCount)
+	}
+	if len(row.Hosts) != 2 || row.Hosts[0] != "node-1" || row.Hosts[1] != "node-2" {
+		t.Errorf("row hosts = %v", row.Hosts)
+	}
+	if row.Bucket != "test-bucket" {
+		t.Errorf("row bucket = %q", row.Bucket)
+	}
+	if row.ContainerFormat != "LARC1" || row.Compression != "zstd" {
+		t.Errorf("row metadata wrong: %+v", row)
+	}
+	if row.KeyFingerprint == "" {
+		t.Errorf("row missing key fingerprint")
+	}
+}
+
+// The central at-least-once property: if S3 fails, Store errors and NOTHING is
+// written to the index, so the caller will not ack.
+func TestStoreS3FailureNoIndexNoAck(t *testing.T) {
+	store := newFakeStore()
+	store.failPut = true
+	idx := &fakeIndex{}
+	a := newTestArchiver(t, store, idx)
+
+	if _, err := a.Store(context.Background(), sampleBatch()); err == nil {
+		t.Fatalf("expected error when S3 put fails")
+	}
+	if len(idx.rows) != 0 {
+		t.Errorf("index written despite S3 failure: %d rows", len(idx.rows))
+	}
+	if len(store.objects) != 0 {
+		t.Errorf("object recorded despite S3 failure")
+	}
+}
+
+// If indexing fails after the S3 put, Store still errors (so no ack); the object
+// exists but is unindexed — acceptable, it will be re-stored on redelivery.
+func TestStoreIndexFailureErrors(t *testing.T) {
+	store := newFakeStore()
+	idx := &fakeIndex{fail: true}
+	a := newTestArchiver(t, store, idx)
+
+	if _, err := a.Store(context.Background(), sampleBatch()); err == nil {
+		t.Fatalf("expected error when index insert fails")
+	}
+	if len(store.objects) != 1 {
+		t.Errorf("object should still be in S3 (orphan), got %d", len(store.objects))
+	}
+}
+
+func TestStoreNilIndexOK(t *testing.T) {
+	store := newFakeStore()
+	a := newTestArchiver(t, store, nil)
+	if _, err := a.Store(context.Background(), sampleBatch()); err != nil {
+		t.Fatalf("Store with nil index: %v", err)
+	}
+	if len(store.objects) != 1 {
+		t.Errorf("expected object stored")
+	}
+}
+
+func TestStoreEmptyBatchNoop(t *testing.T) {
+	store := newFakeStore()
+	a := newTestArchiver(t, store, &fakeIndex{})
+	res, err := a.Store(context.Background(), &batcher.Batch{Subject: "s"})
+	if err != nil {
+		t.Fatalf("empty batch: %v", err)
+	}
+	if res.ObjectKey != "" || len(store.objects) != 0 {
+		t.Errorf("empty batch should store nothing")
+	}
+}

internal/archiver/keys.go

@@ -0,0 +1,73 @@
+package archiver
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"strings"
+	"text/template"
+	"time"
+
+	"git.unkin.net/unkin/logarchiver/internal/event"
+)
+
+// ObjectExt is the suffix for logarchiver container objects (zstd + framed
+// AES-GCM + wrapped OpenPGP DEK). It is deliberately NOT .gz/.pgp because the
+// object is a logarchiver-specific container, not a bare gpg file.
+const ObjectExt = ".ndjson.zst.larc"
+
+// KeyBuilder renders S3 object keys from a prefix template. Template fields:
+// {{.Subject}} (sanitized), {{.Year}} {{.Month}} {{.Day}} (UTC, zero-padded).
+type KeyBuilder struct {
+	tmpl *template.Template
+}
+
+// NewKeyBuilder compiles the prefix template.
+func NewKeyBuilder(prefixTemplate string) (*KeyBuilder, error) {
+	t, err := template.New("key").Option("missingkey=error").Parse(prefixTemplate)
+	if err != nil {
+		return nil, fmt.Errorf("parse key_prefix template: %w", err)
+	}
+	return &KeyBuilder{tmpl: t}, nil
+}
+
+type keyData struct {
+	Subject string
+	Year    string
+	Month   string
+	Day     string
+}
+
+// Build returns a unique object key for a batch of subject at ts. The filename
+// is <UTCstamp>-<random>.ndjson.zst.larc so keys are collision-free and sortable.
+func (k *KeyBuilder) Build(subject string, ts time.Time) (string, error) {
+	ts = ts.UTC()
+	var sb strings.Builder
+	err := k.tmpl.Execute(&sb, keyData{
+		Subject: event.SubjectToken(subject),
+		Year:    fmt.Sprintf("%04d", ts.Year()),
+		Month:   fmt.Sprintf("%02d", int(ts.Month())),
+		Day:     fmt.Sprintf("%02d", ts.Day()),
+	})
+	if err != nil {
+		return "", fmt.Errorf("render key prefix: %w", err)
+	}
+	prefix := sb.String()
+	if prefix != "" && !strings.HasSuffix(prefix, "/") {
+		prefix += "/"
+	}
+	suffix, err := randHex(8)
+	if err != nil {
+		return "", err
+	}
+	name := ts.Format("20060102T150405Z") + "-" + suffix + ObjectExt
+	return prefix + name, nil
+}
+
+func randHex(n int) (string, error) {
+	b := make([]byte, n)
+	if _, err := rand.Read(b); err != nil {
+		return "", fmt.Errorf("random: %w", err)
+	}
+	return hex.EncodeToString(b), nil
+}

internal/archiver/keys_test.go

@@ -0,0 +1,55 @@
+package archiver
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestKeyBuilder(t *testing.T) {
+	kb, err := NewKeyBuilder("archive/{{.Subject}}/{{.Year}}/{{.Month}}/{{.Day}}/")
+	if err != nil {
+		t.Fatalf("NewKeyBuilder: %v", err)
+	}
+	ts := time.Date(2026, 7, 5, 10, 15, 0, 0, time.UTC)
+	key, err := kb.Build("logs.k8s.vault.audit", ts)
+	if err != nil {
+		t.Fatalf("Build: %v", err)
+	}
+	if !strings.HasPrefix(key, "archive/logs.k8s.vault.audit/2026/07/05/") {
+		t.Errorf("unexpected prefix: %s", key)
+	}
+	if !strings.HasSuffix(key, ObjectExt) {
+		t.Errorf("missing container suffix: %s", key)
+	}
+	if !strings.Contains(key, "20260705T101500Z-") {
+		t.Errorf("missing timestamp token: %s", key)
+	}
+}
+
+func TestKeyBuilderUnique(t *testing.T) {
+	kb, _ := NewKeyBuilder("p/{{.Subject}}/")
+	ts := time.Date(2026, 7, 5, 10, 15, 0, 0, time.UTC)
+	k1, _ := kb.Build("s", ts)
+	k2, _ := kb.Build("s", ts)
+	if k1 == k2 {
+		t.Errorf("keys should be unique: %s", k1)
+	}
+}
+
+func TestKeyBuilderSanitizesSubject(t *testing.T) {
+	kb, _ := NewKeyBuilder("p/{{.Subject}}/")
+	key, _ := kb.Build("logs.k8s.a/b", time.Now())
+	if strings.Contains(key, "a/b") {
+		t.Errorf("subject slash not sanitized: %s", key)
+	}
+	if !strings.Contains(key, "a_b") {
+		t.Errorf("expected sanitized a_b: %s", key)
+	}
+}
+
+func TestKeyBuilderBadTemplate(t *testing.T) {
+	if _, err := NewKeyBuilder("{{.Nope"); err == nil {
+		t.Errorf("expected template parse error")
+	}
+}

internal/archiver/pubkey.go

@@ -0,0 +1,89 @@
+package archiver
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+
+	"git.unkin.net/unkin/logarchiver/internal/config"
+	"git.unkin.net/unkin/logarchiver/internal/crypto"
+	"git.unkin.net/unkin/logarchiver/internal/vaultgpg"
+)
+
+// PubkeyLoader fetches the current armored public key from the configured source.
+type PubkeyLoader func(ctx context.Context) (*crypto.PublicKey, error)
+
+// PubkeyProvider caches the current public key and supports periodic refresh so
+// key rotation in the Vault GPG engine is picked up without a restart.
+type PubkeyProvider struct {
+	load PubkeyLoader
+	mu   sync.RWMutex
+	key  *crypto.PublicKey
+}
+
+// NewPubkeyProvider builds a provider and loads the key once.
+func NewPubkeyProvider(ctx context.Context, load PubkeyLoader) (*PubkeyProvider, error) {
+	p := &PubkeyProvider{load: load}
+	if err := p.Refresh(ctx); err != nil {
+		return nil, err
+	}
+	return p, nil
+}
+
+// Refresh reloads the public key.
+func (p *PubkeyProvider) Refresh(ctx context.Context) error {
+	key, err := p.load(ctx)
+	if err != nil {
+		return err
+	}
+	p.mu.Lock()
+	p.key = key
+	p.mu.Unlock()
+	return nil
+}
+
+// Current returns the cached public key.
+func (p *PubkeyProvider) Current() *crypto.PublicKey {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	return p.key
+}
+
+// PubkeyLoaderFromConfig builds a loader for the configured source. For
+// pubkey_source=vault it also verifies the parsed key's fingerprint matches the
+// fingerprint the engine reports, catching armor corruption.
+func PubkeyLoaderFromConfig(cfg config.CryptoConfig, vc *vaultgpg.Client) (PubkeyLoader, error) {
+	switch cfg.Source {
+	case config.PubkeyFile:
+		path := cfg.PubkeyFile
+		return func(_ context.Context) (*crypto.PublicKey, error) {
+			data, err := os.ReadFile(path)
+			if err != nil {
+				return nil, fmt.Errorf("read pubkey file %s: %w", path, err)
+			}
+			return crypto.LoadPublicKey(data)
+		}, nil
+	case config.PubkeyVault:
+		if vc == nil {
+			return nil, fmt.Errorf("pubkey_source=vault requires a vault client")
+		}
+		name := cfg.KeyName
+		return func(ctx context.Context) (*crypto.PublicKey, error) {
+			pk, err := vc.FetchPublicKey(ctx, name)
+			if err != nil {
+				return nil, err
+			}
+			key, err := crypto.LoadPublicKey([]byte(pk.Armored))
+			if err != nil {
+				return nil, err
+			}
+			if pk.Fingerprint != "" && key.Fingerprint != pk.Fingerprint {
+				return nil, fmt.Errorf("pubkey fingerprint mismatch: engine=%s parsed=%s", pk.Fingerprint, key.Fingerprint)
+			}
+			return key, nil
+		}, nil
+	default:
+		return nil, fmt.Errorf("unknown pubkey_source %q", cfg.Source)
+	}
+}