package cli import ( "bytes" "context" "fmt" "io" "os" "path/filepath" "strings" "time" "git.unkin.net/unkin/logarchiver/internal/crypto" "git.unkin.net/unkin/logarchiver/internal/s3store" "git.unkin.net/unkin/logarchiver/internal/vaultgpg" "github.com/spf13/cobra" ) func newFetchCmd() *cobra.Command { var sel selectFlags var output string cmd := &cobra.Command{ Use: "fetch [object-key ...]", Short: "Download, decrypt and decompress archived objects to NDJSON", Long: `fetch retrieves archived objects, decrypts them via the Vault GPG engine (the engine decrypts only the tiny wrapped data key; the bulk is streamed and decrypted locally), decompresses the zstd bulk, and emits the original NDJSON. Objects are selected either by object key arguments, or by the same --subject/--host/--from/--to query used by 'search'. When --host/--from/--to are given they ALSO re-filter the emitted events to just the matching lines.`, Example: ` logarchiver search --subject 'logs.k8s.vault.>' --from -1h logarchiver fetch archive/logs.k8s.vault._/2026/07/27/20260727T101500Z-ab12cd34.ndjson.zst.larc -o - logarchiver fetch --subject 'logs.vm.*' --host db-1 --from -24h -o ./out`, RunE: func(cmd *cobra.Command, args []string) error { cfg, err := loadConfig() if err != nil { return err } now := time.Now() q, err := sel.query(now) if err != nil { return err } // Resolve object keys: explicit args, else via index search. keys := args if len(keys) == 0 { if !cfg.Index.Enabled { return fmt.Errorf("no object keys given and index is disabled") } idx, err := newIndexClient(cmd) if err != nil { return err } defer func() { _ = idx.Close() }() results, err := idx.Search(cmd.Context(), q) if err != nil { return err } for _, r := range results { keys = append(keys, r.ObjectKey) } } if len(keys) == 0 { _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "no matching objects") return nil } store, err := s3store.New(cmd.Context(), s3store.Config{ Endpoint: cfg.S3.Endpoint, Bucket: cfg.S3.Bucket, Region: cfg.S3.Region, PathStyle: cfg.S3.PathStyle, CAFile: cfg.S3.CAFile, }) if err != nil { return fmt.Errorf("s3 init: %w", err) } // Operator decrypt path: force token auth (ambient VAULT_TOKEN / // ~/.vault-token), like passv, regardless of the service auth_method. vcfg := vaultConfig(cfg.Crypto.Vault) vcfg.AuthMethod = "token" vc, err := vaultgpg.New(cmd.Context(), vcfg) if err != nil { return fmt.Errorf("vault init: %w", err) } filter := newLineFilter(sel.host, q.From, q.To) var failures int for _, key := range keys { if err := fetchOne(cmd.Context(), store, vc, key, output, filter); err != nil { _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "fetch %s: %v\n", key, err) failures++ } } if failures > 0 { return fmt.Errorf("%d of %d objects failed", failures, len(keys)) } return nil }, } sel.bind(cmd) cmd.Flags().StringVarP(&output, "output", "o", "-", "output: '-' for stdout, or a directory to write one NDJSON file per object") return cmd } // fetchOne downloads, decrypts and decompresses a single object, applying the // optional line filter, to stdout or a per-object file under a directory. func fetchOne(ctx context.Context, store s3store.ObjectStore, vc *vaultgpg.Client, key, output string, filter lineFilter) error { body, err := store.Get(ctx, key) if err != nil { return err } defer func() { _ = body.Close() }() // Buffer the (bounded) object so we can read the header for its key name // before decrypting. data, err := io.ReadAll(body) if err != nil { return fmt.Errorf("read object: %w", err) } hdr, _, err := crypto.ReadHeader(bytes.NewReader(data)) if err != nil { return err } keyName := hdr.KeyName if keyName == "" { return fmt.Errorf("object header has no key_name") } unwrap := func(wrapped []byte) ([]byte, error) { return vc.Decrypt(ctx, keyName, wrapped) } var dst io.Writer var closer io.Closer if output == "-" || output == "" { dst = os.Stdout } else { if err := os.MkdirAll(output, 0o755); err != nil { return fmt.Errorf("create output dir: %w", err) } outPath := filepath.Join(output, sanitizeKey(key)) if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil { return fmt.Errorf("create output subdir: %w", err) } f, err := os.Create(outPath) if err != nil { return fmt.Errorf("create output file: %w", err) } dst = f closer = f } fw := newFilterWriter(dst, filter) if err := crypto.Open(bytes.NewReader(data), fw, unwrap); err != nil { if closer != nil { _ = closer.Close() } return err } if err := fw.Flush(); err != nil { if closer != nil { _ = closer.Close() } return err } if closer != nil { return closer.Close() } return nil } // sanitizeKey turns an object key into a safe relative output filename, dropping // the .larc container suffix in favor of a plain .ndjson. func sanitizeKey(key string) string { name := strings.TrimSuffix(key, ".larc") if !strings.HasSuffix(name, ".ndjson") && !strings.HasSuffix(name, ".ndjson.zst") { name += ".ndjson" } name = strings.TrimSuffix(name, ".zst") return filepath.Clean("/" + name)[1:] }