logarchiver/internal/cli/fetch.go

package cli

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"

	"git.unkin.net/unkin/logarchiver/internal/crypto"
	"git.unkin.net/unkin/logarchiver/internal/s3store"
	"git.unkin.net/unkin/logarchiver/internal/vaultgpg"
	"github.com/spf13/cobra"
)

func newFetchCmd() *cobra.Command {
	var sel selectFlags
	var output string
	cmd := &cobra.Command{
		Use:   "fetch [object-key ...]",
		Short: "Download, decrypt and decompress archived objects to NDJSON",
		Long: `fetch retrieves archived objects, decrypts them via the Vault GPG engine
(the engine decrypts only the tiny wrapped data key; the bulk is streamed and
decrypted locally), decompresses the zstd bulk, and emits the original NDJSON.

Objects are selected either by object key arguments, or by the same
--subject/--host/--from/--to query used by 'search'. When --host/--from/--to are
given they ALSO re-filter the emitted events to just the matching lines.`,
		Example: `  logarchiver search --subject 'logs.k8s.vault.>' --from -1h
  logarchiver fetch archive/logs.k8s.vault._/2026/07/27/20260727T101500Z-ab12cd34.ndjson.zst.larc -o -
  logarchiver fetch --subject 'logs.vm.*' --host db-1 --from -24h -o ./out`,
		RunE: func(cmd *cobra.Command, args []string) error {
			cfg, err := loadConfig()
			if err != nil {
				return err
			}
			now := time.Now()
			q, err := sel.query(now)
			if err != nil {
				return err
			}

			// Resolve object keys: explicit args, else via index search.
			keys := args
			if len(keys) == 0 {
				if !cfg.Index.Enabled {
					return fmt.Errorf("no object keys given and index is disabled")
				}
				idx, err := newIndexClient(cmd)
				if err != nil {
					return err
				}
				defer func() { _ = idx.Close() }()
				results, err := idx.Search(cmd.Context(), q)
				if err != nil {
					return err
				}
				for _, r := range results {
					keys = append(keys, r.ObjectKey)
				}
			}
			if len(keys) == 0 {
				_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "no matching objects")
				return nil
			}

			store, err := s3store.New(cmd.Context(), s3store.Config{
				Endpoint:  cfg.S3.Endpoint,
				Bucket:    cfg.S3.Bucket,
				Region:    cfg.S3.Region,
				PathStyle: cfg.S3.PathStyle,
				CAFile:    cfg.S3.CAFile,
			})
			if err != nil {
				return fmt.Errorf("s3 init: %w", err)
			}
			// Operator decrypt path: force token auth (ambient VAULT_TOKEN /
			// ~/.vault-token), like passv, regardless of the service auth_method.
			vcfg := vaultConfig(cfg.Crypto.Vault)
			vcfg.AuthMethod = "token"
			vc, err := vaultgpg.New(cmd.Context(), vcfg)
			if err != nil {
				return fmt.Errorf("vault init: %w", err)
			}

			filter := newLineFilter(sel.host, q.From, q.To)

			var failures int
			for _, key := range keys {
				if err := fetchOne(cmd.Context(), store, vc, key, output, filter); err != nil {
					_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "fetch %s: %v\n", key, err)
					failures++
				}
			}
			if failures > 0 {
				return fmt.Errorf("%d of %d objects failed", failures, len(keys))
			}
			return nil
		},
	}
	sel.bind(cmd)
	cmd.Flags().StringVarP(&output, "output", "o", "-",
		"output: '-' for stdout, or a directory to write one NDJSON file per object")
	return cmd
}

// fetchOne downloads, decrypts and decompresses a single object, applying the
// optional line filter, to stdout or a per-object file under a directory.
func fetchOne(ctx context.Context, store s3store.ObjectStore, vc *vaultgpg.Client, key, output string, filter lineFilter) error {
	body, err := store.Get(ctx, key)
	if err != nil {
		return err
	}
	defer func() { _ = body.Close() }()

	// Buffer the (bounded) object so we can read the header for its key name
	// before decrypting.
	data, err := io.ReadAll(body)
	if err != nil {
		return fmt.Errorf("read object: %w", err)
	}
	hdr, _, err := crypto.ReadHeader(bytes.NewReader(data))
	if err != nil {
		return err
	}
	keyName := hdr.KeyName
	if keyName == "" {
		return fmt.Errorf("object header has no key_name")
	}
	unwrap := func(wrapped []byte) ([]byte, error) {
		return vc.Decrypt(ctx, keyName, wrapped)
	}

	var dst io.Writer
	var closer io.Closer
	if output == "-" || output == "" {
		dst = os.Stdout
	} else {
		if err := os.MkdirAll(output, 0o755); err != nil {
			return fmt.Errorf("create output dir: %w", err)
		}
		outPath := filepath.Join(output, sanitizeKey(key))
		if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil {
			return fmt.Errorf("create output subdir: %w", err)
		}
		f, err := os.Create(outPath)
		if err != nil {
			return fmt.Errorf("create output file: %w", err)
		}
		dst = f
		closer = f
	}

	fw := newFilterWriter(dst, filter)
	if err := crypto.Open(bytes.NewReader(data), fw, unwrap); err != nil {
		if closer != nil {
			_ = closer.Close()
		}
		return err
	}
	if err := fw.Flush(); err != nil {
		if closer != nil {
			_ = closer.Close()
		}
		return err
	}
	if closer != nil {
		return closer.Close()
	}
	return nil
}

// sanitizeKey turns an object key into a safe relative output filename, dropping
// the .larc container suffix in favor of a plain .ndjson.
func sanitizeKey(key string) string {
	name := strings.TrimSuffix(key, ".larc")
	if !strings.HasSuffix(name, ".ndjson") && !strings.HasSuffix(name, ".ndjson.zst") {
		name += ".ndjson"
	}
	name = strings.TrimSuffix(name, ".zst")
	return filepath.Clean("/" + name)[1:]
}