logarchiver/internal/archiver/archiver.go

// Package archiver turns a ready batch into a stored, indexed object: build
// NDJSON, seal it into a LARC1 container, PUT it to S3, then write the index
// row. Store returns an error if ANY step fails; the caller must not ack the
// batch's messages until Store succeeds (at-least-once, sink-conditional acks).
package archiver

import (
	"bytes"
	"context"
	"fmt"
	"time"

	"git.unkin.net/unkin/logarchiver/internal/batcher"
	"git.unkin.net/unkin/logarchiver/internal/crypto"
	"git.unkin.net/unkin/logarchiver/internal/index"
	"git.unkin.net/unkin/logarchiver/internal/s3store"
)

// Metrics is the optional metrics sink (implemented by internal/metrics). A nil
// Metrics is fine (no-op).
type Metrics interface {
	ObjectStored(subject string, events int, rawBytes, storedBytes int64)
	StoreFailed(subject string)
	IndexFailed(subject string)
}

// Archiver persists batches.
type Archiver struct {
	keys      *KeyBuilder
	pubkeys   *PubkeyProvider
	store     s3store.ObjectStore
	idx       index.Index // may be nil when indexing is disabled
	keyName   string
	frameSize int
	metrics   Metrics
	nowFn     func() time.Time
}

// Options configures an Archiver.
type Options struct {
	Keys      *KeyBuilder
	Pubkeys   *PubkeyProvider
	Store     s3store.ObjectStore
	Index     index.Index
	KeyName   string
	FrameSize int
	Metrics   Metrics
}

// New builds an Archiver.
func New(o Options) (*Archiver, error) {
	if o.Keys == nil || o.Pubkeys == nil || o.Store == nil {
		return nil, fmt.Errorf("archiver requires keys, pubkeys and store")
	}
	fs := o.FrameSize
	if fs <= 0 {
		fs = 1 << 20
	}
	return &Archiver{
		keys:      o.Keys,
		pubkeys:   o.Pubkeys,
		store:     o.Store,
		idx:       o.Index,
		keyName:   o.KeyName,
		frameSize: fs,
		metrics:   o.Metrics,
		nowFn:     time.Now,
	}, nil
}

// StoreResult reports what Store persisted.
type StoreResult struct {
	ObjectKey   string
	Events      int
	RawBytes    int64
	StoredBytes int64
}

// Store seals, uploads and indexes a batch. On success the caller may ack.
func (a *Archiver) Store(ctx context.Context, batch *batcher.Batch) (StoreResult, error) {
	if len(batch.Items) == 0 {
		return StoreResult{}, nil
	}
	now := a.nowFn().UTC()
	summary := batch.Summarize(now)

	pub := a.pubkeys.Current()
	if pub == nil {
		a.metricStoreFailed(batch.Subject)
		return StoreResult{}, fmt.Errorf("no public key available")
	}

	// Choose the object key from the batch's max timestamp so it lands in the
	// date partition of the newest event.
	key, err := a.keys.Build(batch.Subject, summary.MaxTS)
	if err != nil {
		a.metricStoreFailed(batch.Subject)
		return StoreResult{}, err
	}

	ndjson := batch.NDJSON()
	var buf bytes.Buffer
	sealed, err := crypto.Seal(&buf, ndjson, pub, a.keyName, a.frameSize)
	if err != nil {
		a.metricStoreFailed(batch.Subject)
		return StoreResult{}, fmt.Errorf("seal object %s: %w", key, err)
	}

	if err := a.store.Put(ctx, key, bytes.NewReader(buf.Bytes()), int64(buf.Len())); err != nil {
		a.metricStoreFailed(batch.Subject)
		return StoreResult{}, err
	}

	if a.idx != nil {
		row := index.Row{
			ObjectKey:       key,
			Bucket:          a.store.Bucket(),
			Subject:         batch.Subject,
			Hosts:           summary.Hosts,
			MinTS:           summary.MinTS,
			MaxTS:           summary.MaxTS,
			EventCount:      uint64(summary.EventCount),
			RawBytes:        uint64(sealed.RawBytes),
			StoredBytes:     uint64(sealed.StoredBytes),
			Compression:     sealed.Header.Compression,
			Cipher:          sealed.Header.Cipher,
			ContainerFormat: "LARC1",
			KeyName:         a.keyName,
			KeyFingerprint:  sealed.Header.KeyFingerprint,
		}
		if err := a.idx.Insert(ctx, row); err != nil {
			// The object is in S3 but unindexed. Do NOT ack: on redelivery the
			// batch is re-stored (a new object key) and re-indexed. The orphan
			// object is harmless (retrievable by prefix) and reaped by lifecycle.
			a.metricIndexFailed(batch.Subject)
			return StoreResult{}, fmt.Errorf("index object %s: %w", key, err)
		}
	}

	if a.metrics != nil {
		a.metrics.ObjectStored(batch.Subject, summary.EventCount, sealed.RawBytes, sealed.StoredBytes)
	}
	return StoreResult{
		ObjectKey:   key,
		Events:      summary.EventCount,
		RawBytes:    sealed.RawBytes,
		StoredBytes: sealed.StoredBytes,
	}, nil
}

func (a *Archiver) metricStoreFailed(subject string) {
	if a.metrics != nil {
		a.metrics.StoreFailed(subject)
	}
}

func (a *Archiver) metricIndexFailed(subject string) {
	if a.metrics != nil {
		a.metrics.IndexFailed(subject)
	}
}