Initial implementation: NATS->S3 archiver + search/retrieve CLI

logarchiver replaces the plain Vector archiver leg of the centralized
logging stack (argocd-apps #296) with a Go service that archives raw logs
from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed
objects, plus an operator CLI to search the index and retrieve/decrypt
archived logs. It adds the things that outgrew Vector: zstd compression,
encryption keyed from Ben's Vault GPG secrets engine, a searchable
ClickHouse index, and sink-conditional acks (a batch is acknowledged to
JetStream only after the object is durably in S3 AND indexed).

Service (`logarchiver run`):
- Durable JetStream pull consumer (stream LOGS, durable archiver, subject
  filter default logs.k8s.vault.>), explicit acks, independent offsets.
- Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3
  PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and
  redelivered, so nothing is lost on a sink outage.
- Encryption is a wrapped-DEK envelope (container LARC1): the bulk is
  AES-256-GCM framed under a random data key, and only that 32-byte key is
  OpenPGP-encrypted to the engine's public key. This is because the Vault
  GPG engine does whole-payload decrypt only; retrieval round-trips just the
  tiny wrapped key regardless of object size. Public key fetched from the
  engine or a mounted file (configurable); key fingerprint recorded per
  object; periodic pubkey refresh for rotation.
- Prometheus metrics, structured slog, graceful drain on shutdown.

CLI:
- `search` queries the index (subject/host/time) and lists matching objects.
- `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits
  NDJSON (optionally re-filtered by host/time).
- `init-schema` creates/prints the ClickHouse archive_index DDL.
- cobra `completion` subcommands.

Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/
ClickHouse/Vault) behind interfaces with unit tests (config, batching,
host/subject extraction, crypto roundtrip with a test key, ack-after-persist
with fakes, search query building). go build/vet/test -race clean;
golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v*
tag a container image plus a Gitea binary release + rpm-internal RPM. Docs
per subcommand + architecture + retrieval runbook + deployment drop-in.

Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv

internal/s3store/s3store.go

@@ -0,0 +1,140 @@
+// Package s3store wraps object storage (Ceph RGW via the S3 API) behind an
+// interface so the archiver and CLI can be tested with a fake. Credentials come
+// from the standard AWS_* environment (the cephrgw BucketAccess secret
+// logs-archive-s3); this package only wires the custom endpoint, path-style
+// addressing, and the internal Vault-PKI CA needed for s3.ceph.unkin.net.
+package s3store
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+// ObjectStore is the minimal object-storage surface logarchiver needs.
+type ObjectStore interface {
+	Put(ctx context.Context, key string, body io.Reader, size int64) error
+	Get(ctx context.Context, key string) (io.ReadCloser, error)
+	List(ctx context.Context, prefix string) ([]string, error)
+	Bucket() string
+}
+
+// Config configures the S3 client.
+type Config struct {
+	Endpoint  string
+	Bucket    string
+	Region    string
+	PathStyle bool
+	CAFile    string
+}
+
+// Store is the S3-backed ObjectStore.
+type Store struct {
+	client *s3.Client
+	bucket string
+}
+
+// New builds a Store, trusting CAFile (in addition to the system roots) when set.
+func New(ctx context.Context, cfg Config) (*Store, error) {
+	httpClient, err := httpClientWithCA(cfg.CAFile)
+	if err != nil {
+		return nil, err
+	}
+	region := cfg.Region
+	if region == "" {
+		region = "us-east-1"
+	}
+	awsCfg, err := awsconfig.LoadDefaultConfig(ctx,
+		awsconfig.WithRegion(region),
+		awsconfig.WithHTTPClient(httpClient),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("load aws config: %w", err)
+	}
+	client := s3.NewFromConfig(awsCfg, func(o *s3.Options) {
+		if cfg.Endpoint != "" {
+			o.BaseEndpoint = &cfg.Endpoint
+		}
+		o.UsePathStyle = cfg.PathStyle
+	})
+	return &Store{client: client, bucket: cfg.Bucket}, nil
+}
+
+// Bucket returns the configured bucket name.
+func (s *Store) Bucket() string { return s.bucket }
+
+// Put uploads body of the given size under key.
+func (s *Store) Put(ctx context.Context, key string, body io.Reader, size int64) error {
+	_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket:        &s.bucket,
+		Key:           &key,
+		Body:          body,
+		ContentLength: &size,
+	})
+	if err != nil {
+		return fmt.Errorf("put s3://%s/%s: %w", s.bucket, key, err)
+	}
+	return nil
+}
+
+// Get streams the object at key.
+func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) {
+	out, err := s.client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: &s.bucket,
+		Key:    &key,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("get s3://%s/%s: %w", s.bucket, key, err)
+	}
+	return out.Body, nil
+}
+
+// List returns object keys under prefix (paginated).
+func (s *Store) List(ctx context.Context, prefix string) ([]string, error) {
+	var keys []string
+	p := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{
+		Bucket: &s.bucket,
+		Prefix: &prefix,
+	})
+	for p.HasMorePages() {
+		page, err := p.NextPage(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("list s3://%s/%s: %w", s.bucket, prefix, err)
+		}
+		for _, obj := range page.Contents {
+			if obj.Key != nil {
+				keys = append(keys, *obj.Key)
+			}
+		}
+	}
+	return keys, nil
+}
+
+func httpClientWithCA(caFile string) (*http.Client, error) {
+	if caFile == "" {
+		return http.DefaultClient, nil
+	}
+	pem, err := os.ReadFile(caFile)
+	if err != nil {
+		return nil, fmt.Errorf("read s3 ca file %s: %w", caFile, err)
+	}
+	pool, err := x509.SystemCertPool()
+	if err != nil || pool == nil {
+		pool = x509.NewCertPool()
+	}
+	if !pool.AppendCertsFromPEM(pem) {
+		return nil, fmt.Errorf("no certificates parsed from %s", caFile)
+	}
+	return &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12},
+		},
+	}, nil
+}