benvin
c05ccfcb5d
logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv
68 lines
1.9 KiB
Go
68 lines
1.9 KiB
Go
package cli
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"text/tabwriter"
|
|
"time"
|
|
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
func newSearchCmd() *cobra.Command {
|
|
var sel selectFlags
|
|
var asJSON bool
|
|
cmd := &cobra.Command{
|
|
Use: "search",
|
|
Short: "Search the archive index for matching objects",
|
|
Long: `search queries the ClickHouse archive index and lists the S3 objects whose
|
|
subject/host/time-range match, with event counts and sizes. Use the object keys
|
|
with 'logarchiver fetch' to retrieve and decrypt their contents.`,
|
|
Example: ` logarchiver search --subject 'logs.k8s.vault.>' --host node-1 --from -24h`,
|
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
|
q, err := sel.query(time.Now())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
idx, err := newIndexClient(cmd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = idx.Close() }()
|
|
|
|
results, err := idx.Search(cmd.Context(), q)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
out := cmd.OutOrStdout()
|
|
if asJSON {
|
|
enc := json.NewEncoder(out)
|
|
enc.SetIndent("", " ")
|
|
return enc.Encode(results)
|
|
}
|
|
if len(results) == 0 {
|
|
_, _ = fmt.Fprintln(out, "no matching objects")
|
|
return nil
|
|
}
|
|
tw := tabwriter.NewWriter(out, 0, 2, 2, ' ', 0)
|
|
_, _ = fmt.Fprintln(tw, "OBJECT_KEY\tSUBJECT\tHOSTS\tMIN_TS\tMAX_TS\tEVENTS\tSTORED")
|
|
var totalEvents, totalStored uint64
|
|
for _, r := range results {
|
|
_, _ = fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%d\t%s\n",
|
|
r.ObjectKey, r.Subject, strings.Join(r.Hosts, ","),
|
|
r.MinTS.UTC().Format(time.RFC3339), r.MaxTS.UTC().Format(time.RFC3339),
|
|
r.EventCount, humanBytes(r.StoredBytes))
|
|
totalEvents += r.EventCount
|
|
totalStored += r.StoredBytes
|
|
}
|
|
_ = tw.Flush()
|
|
_, _ = fmt.Fprintf(out, "\n%d objects, %d events, %s stored\n", len(results), totalEvents, humanBytes(totalStored))
|
|
return nil
|
|
},
|
|
}
|
|
sel.bind(cmd)
|
|
cmd.Flags().BoolVar(&asJSON, "json", false, "output results as JSON")
|
|
return cmd
|
|
}
|