// Package archiver turns a ready batch into a stored, indexed object: build // NDJSON, seal it into a LARC1 container, PUT it to S3, then write the index // row. Store returns an error if ANY step fails; the caller must not ack the // batch's messages until Store succeeds (at-least-once, sink-conditional acks). package archiver import ( "bytes" "context" "fmt" "time" "git.unkin.net/unkin/logarchiver/internal/batcher" "git.unkin.net/unkin/logarchiver/internal/crypto" "git.unkin.net/unkin/logarchiver/internal/index" "git.unkin.net/unkin/logarchiver/internal/s3store" ) // Metrics is the optional metrics sink (implemented by internal/metrics). A nil // Metrics is fine (no-op). type Metrics interface { ObjectStored(subject string, events int, rawBytes, storedBytes int64) StoreFailed(subject string) IndexFailed(subject string) } // Archiver persists batches. type Archiver struct { keys *KeyBuilder pubkeys *PubkeyProvider store s3store.ObjectStore idx index.Index // may be nil when indexing is disabled keyName string frameSize int metrics Metrics nowFn func() time.Time } // Options configures an Archiver. type Options struct { Keys *KeyBuilder Pubkeys *PubkeyProvider Store s3store.ObjectStore Index index.Index KeyName string FrameSize int Metrics Metrics } // New builds an Archiver. func New(o Options) (*Archiver, error) { if o.Keys == nil || o.Pubkeys == nil || o.Store == nil { return nil, fmt.Errorf("archiver requires keys, pubkeys and store") } fs := o.FrameSize if fs <= 0 { fs = 1 << 20 } return &Archiver{ keys: o.Keys, pubkeys: o.Pubkeys, store: o.Store, idx: o.Index, keyName: o.KeyName, frameSize: fs, metrics: o.Metrics, nowFn: time.Now, }, nil } // StoreResult reports what Store persisted. type StoreResult struct { ObjectKey string Events int RawBytes int64 StoredBytes int64 } // Store seals, uploads and indexes a batch. On success the caller may ack. func (a *Archiver) Store(ctx context.Context, batch *batcher.Batch) (StoreResult, error) { if len(batch.Items) == 0 { return StoreResult{}, nil } now := a.nowFn().UTC() summary := batch.Summarize(now) pub := a.pubkeys.Current() if pub == nil { a.metricStoreFailed(batch.Subject) return StoreResult{}, fmt.Errorf("no public key available") } // Choose the object key from the batch's max timestamp so it lands in the // date partition of the newest event. key, err := a.keys.Build(batch.Subject, summary.MaxTS) if err != nil { a.metricStoreFailed(batch.Subject) return StoreResult{}, err } ndjson := batch.NDJSON() var buf bytes.Buffer sealed, err := crypto.Seal(&buf, ndjson, pub, a.keyName, a.frameSize) if err != nil { a.metricStoreFailed(batch.Subject) return StoreResult{}, fmt.Errorf("seal object %s: %w", key, err) } if err := a.store.Put(ctx, key, bytes.NewReader(buf.Bytes()), int64(buf.Len())); err != nil { a.metricStoreFailed(batch.Subject) return StoreResult{}, err } if a.idx != nil { row := index.Row{ ObjectKey: key, Bucket: a.store.Bucket(), Subject: batch.Subject, Hosts: summary.Hosts, MinTS: summary.MinTS, MaxTS: summary.MaxTS, EventCount: uint64(summary.EventCount), RawBytes: uint64(sealed.RawBytes), StoredBytes: uint64(sealed.StoredBytes), Compression: sealed.Header.Compression, Cipher: sealed.Header.Cipher, ContainerFormat: "LARC1", KeyName: a.keyName, KeyFingerprint: sealed.Header.KeyFingerprint, } if err := a.idx.Insert(ctx, row); err != nil { // The object is in S3 but unindexed. Do NOT ack: on redelivery the // batch is re-stored (a new object key) and re-indexed. The orphan // object is harmless (retrievable by prefix) and reaped by lifecycle. a.metricIndexFailed(batch.Subject) return StoreResult{}, fmt.Errorf("index object %s: %w", key, err) } } if a.metrics != nil { a.metrics.ObjectStored(batch.Subject, summary.EventCount, sealed.RawBytes, sealed.StoredBytes) } return StoreResult{ ObjectKey: key, Events: summary.EventCount, RawBytes: sealed.RawBytes, StoredBytes: sealed.StoredBytes, }, nil } func (a *Archiver) metricStoreFailed(subject string) { if a.metrics != nil { a.metrics.StoreFailed(subject) } } func (a *Archiver) metricIndexFailed(subject string) { if a.metrics != nil { a.metrics.IndexFailed(subject) } }