From 5ec89b0028c2b64e454992e6f30868505798dfbc Mon Sep 17 00:00:00 2001 From: benvin Date: Mon, 27 Jul 2026 22:11:54 +1000 Subject: [PATCH] Initial implementation: NATS->S3 archiver + search/retrieve CLI logarchiver replaces the plain Vector archiver leg of the centralized logging stack (argocd-apps #296) with a Go service that archives raw logs from NATS JetStream to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, plus an operator CLI to search the index and retrieve/decrypt archived logs. It adds the things that outgrew Vector: zstd compression, encryption keyed from Ben's Vault GPG secrets engine, a searchable ClickHouse index, and sink-conditional acks (a batch is acknowledged to JetStream only after the object is durably in S3 AND indexed). Service (`logarchiver run`): - Durable JetStream pull consumer (stream LOGS, durable archiver, subject filter default logs.k8s.vault.>), explicit acks, independent offsets. - Batch per subject by size/count/time -> NDJSON -> zstd -> encrypt -> S3 PUT -> ClickHouse index row -> ack. On any failure the batch is Nak'd and redelivered, so nothing is lost on a sink outage. - Encryption is a wrapped-DEK envelope (container LARC1): the bulk is AES-256-GCM framed under a random data key, and only that 32-byte key is OpenPGP-encrypted to the engine's public key. This is because the Vault GPG engine does whole-payload decrypt only; retrieval round-trips just the tiny wrapped key regardless of object size. Public key fetched from the engine or a mounted file (configurable); key fingerprint recorded per object; periodic pubkey refresh for rotation. - Prometheus metrics, structured slog, graceful drain on shutdown. CLI: - `search` queries the index (subject/host/time) and lists matching objects. - `fetch` downloads, decrypts via the Vault GPG engine, unzstds and emits NDJSON (optionally re-filtered by host/time). - `init-schema` creates/prints the ClickHouse archive_index DDL. - cobra `completion` subcommands. Config via file+env (k8s-friendly, secrets from env), boundaries (NATS/S3/ ClickHouse/Vault) behind interfaces with unit tests (config, batching, host/subject extraction, crypto roundtrip with a test key, ack-after-persist with fakes, search query building). go build/vet/test -race clean; golangci-lint v2 clean. Woodpecker CI: build/test/pre-commit on PR; on v* tag a container image plus a Gitea binary release + rpm-internal RPM. Docs per subcommand + architecture + retrieval runbook + deployment drop-in. Claude-Session: https://claude.ai/code/session_015ur3i7D2azsMAWTSVABApv --- .gitignore | 8 + .pre-commit-config.yaml | 27 ++ .woodpecker/build.yaml | 18 ++ .woodpecker/docker.yaml | 18 ++ .woodpecker/pre-commit.yaml | 18 ++ .woodpecker/release.yaml | 142 +++++++++++ .woodpecker/test.yaml | 33 +++ Dockerfile | 23 ++ LICENSE | 21 ++ Makefile | 81 ++++++ README.md | 79 +++++- cmd/logarchiver/main.go | 17 ++ config.example.yaml | 84 +++++++ docs/architecture.md | 122 ++++++++++ docs/deployment.md | 55 +++++ docs/fetch.md | 41 ++++ docs/init-schema.md | 18 ++ docs/retrieval-runbook.md | 118 +++++++++ docs/run.md | 40 +++ docs/search.md | 39 +++ go.mod | 78 ++++++ go.sum | 169 +++++++++++++ internal/archiver/archiver.go | 161 ++++++++++++ internal/archiver/archiver_test.go | 222 +++++++++++++++++ internal/archiver/keys.go | 73 ++++++ internal/archiver/keys_test.go | 55 +++++ internal/archiver/pubkey.go | 89 +++++++ internal/batcher/batcher.go | 177 ++++++++++++++ internal/batcher/batcher_test.go | 148 +++++++++++ internal/cli/common.go | 90 +++++++ internal/cli/fetch.go | 185 ++++++++++++++ internal/cli/filter.go | 123 ++++++++++ internal/cli/filter_test.go | 116 +++++++++ internal/cli/initschema.go | 45 ++++ internal/cli/root.go | 95 ++++++++ internal/cli/run.go | 252 +++++++++++++++++++ internal/cli/search.go | 67 +++++ internal/config/config.go | 379 +++++++++++++++++++++++++++++ internal/config/config_test.go | 106 ++++++++ internal/consumer/connect.go | 80 ++++++ internal/consumer/meta.go | 8 + internal/consumer/runner.go | 237 ++++++++++++++++++ internal/consumer/runner_test.go | 151 ++++++++++++ internal/crypto/crypto_test.go | 165 +++++++++++++ internal/crypto/envelope.go | 338 +++++++++++++++++++++++++ internal/crypto/pubkey.go | 81 ++++++ internal/event/event.go | 182 ++++++++++++++ internal/event/event_test.go | 88 +++++++ internal/index/ddl.go | 42 ++++ internal/index/index.go | 159 ++++++++++++ internal/index/query.go | 111 +++++++++ internal/index/query_test.go | 110 +++++++++ internal/metrics/metrics.go | 97 ++++++++ internal/s3store/s3store.go | 140 +++++++++++ internal/vaultgpg/client.go | 195 +++++++++++++++ packaging/nfpm.yaml | 52 ++++ schema/archive_index.sql | 35 +++ scripts/build-rpm.sh | 44 ++++ 58 files changed, 5946 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 .woodpecker/build.yaml create mode 100644 .woodpecker/docker.yaml create mode 100644 .woodpecker/pre-commit.yaml create mode 100644 .woodpecker/release.yaml create mode 100644 .woodpecker/test.yaml create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 cmd/logarchiver/main.go create mode 100644 config.example.yaml create mode 100644 docs/architecture.md create mode 100644 docs/deployment.md create mode 100644 docs/fetch.md create mode 100644 docs/init-schema.md create mode 100644 docs/retrieval-runbook.md create mode 100644 docs/run.md create mode 100644 docs/search.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/archiver/archiver.go create mode 100644 internal/archiver/archiver_test.go create mode 100644 internal/archiver/keys.go create mode 100644 internal/archiver/keys_test.go create mode 100644 internal/archiver/pubkey.go create mode 100644 internal/batcher/batcher.go create mode 100644 internal/batcher/batcher_test.go create mode 100644 internal/cli/common.go create mode 100644 internal/cli/fetch.go create mode 100644 internal/cli/filter.go create mode 100644 internal/cli/filter_test.go create mode 100644 internal/cli/initschema.go create mode 100644 internal/cli/root.go create mode 100644 internal/cli/run.go create mode 100644 internal/cli/search.go create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 internal/consumer/connect.go create mode 100644 internal/consumer/meta.go create mode 100644 internal/consumer/runner.go create mode 100644 internal/consumer/runner_test.go create mode 100644 internal/crypto/crypto_test.go create mode 100644 internal/crypto/envelope.go create mode 100644 internal/crypto/pubkey.go create mode 100644 internal/event/event.go create mode 100644 internal/event/event_test.go create mode 100644 internal/index/ddl.go create mode 100644 internal/index/index.go create mode 100644 internal/index/query.go create mode 100644 internal/index/query_test.go create mode 100644 internal/metrics/metrics.go create mode 100644 internal/s3store/s3store.go create mode 100644 internal/vaultgpg/client.go create mode 100644 packaging/nfpm.yaml create mode 100644 schema/archive_index.sql create mode 100755 scripts/build-rpm.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..febaee8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +/dist/ +*.rpm +*.zip +# Root-level dev binaries only — anchored so cmd/logarchiver (the source +# package) is NOT ignored. +/logarchiver +# cross-compiled release artifacts (e.g. logarchiver-linux-amd64) +/logarchiver-* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..6a26241 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - id: check-merge-conflict + + - repo: https://github.com/dnephin/pre-commit-golang + rev: v0.5.1 + hooks: + - id: go-fmt + - id: go-mod-tidy + + # logarchiver has no root-level Go files (all under cmd/, internal/), so the + # dnephin go-vet hook (which runs `go vet` at the repo root) fails with + # "no Go files". Vet the whole module instead, mirroring encapi/artifactapi. + - repo: local + hooks: + - id: go-vet + name: go vet + entry: go vet ./... + language: system + types: [go] + pass_filenames: false diff --git a/.woodpecker/build.yaml b/.woodpecker/build.yaml new file mode 100644 index 0000000..74123b5 --- /dev/null +++ b/.woodpecker/build.yaml @@ -0,0 +1,18 @@ +when: + - event: pull_request + +steps: + - name: build + image: golang:1.25 + commands: + - make build + backend_options: + kubernetes: + serviceAccountName: default + resources: + requests: + memory: 512Mi + cpu: 1 + limits: + memory: 2Gi + cpu: 2 diff --git a/.woodpecker/docker.yaml b/.woodpecker/docker.yaml new file mode 100644 index 0000000..7e7f5db --- /dev/null +++ b/.woodpecker/docker.yaml @@ -0,0 +1,18 @@ +when: + - event: tag + ref: refs/tags/v* + +steps: + - name: docker-logarchiver + image: woodpeckerci/plugin-docker-buildx + settings: + registry: git.unkin.net + repo: git.unkin.net/unkin/logarchiver + build_args: + VERSION: ${CI_COMMIT_TAG} + username: droneci + password: + from_secret: DRONECI_PASSWORD + tags: + - ${CI_COMMIT_TAG} + - latest diff --git a/.woodpecker/pre-commit.yaml b/.woodpecker/pre-commit.yaml new file mode 100644 index 0000000..d57b508 --- /dev/null +++ b/.woodpecker/pre-commit.yaml @@ -0,0 +1,18 @@ +when: + - event: pull_request + +steps: + - name: pre-commit + image: git.unkin.net/unkin/almalinux9-gobuilder:20260606 + commands: + - uvx pre-commit run --all-files + backend_options: + kubernetes: + serviceAccountName: default + resources: + requests: + memory: 512Mi + cpu: 1 + limits: + memory: 2Gi + cpu: 2 diff --git a/.woodpecker/release.yaml b/.woodpecker/release.yaml new file mode 100644 index 0000000..af932bf --- /dev/null +++ b/.woodpecker/release.yaml @@ -0,0 +1,142 @@ +when: + - event: tag + +steps: + - name: test + image: golang:1.25 + commands: + - go test -race ./... + backend_options: + kubernetes: + serviceAccountName: default + resources: + requests: + memory: 512Mi + cpu: 1 + limits: + memory: 2Gi + cpu: 2 + + # Build the dist/ binary (consumed by the RPM step) plus the cross-platform + # CLI binaries attached to the Gitea release. + - name: build + image: git.unkin.net/unkin/almalinux9-gobuilder:20260606 + commands: + - make build VERSION=${CI_COMMIT_TAG} + # $$ escapes shell vars so Woodpecker leaves them for the shell instead of + # substituting pipeline vars at parse time; ${CI_COMMIT_TAG} is a real var. + - | + for osarch in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64; do + os="$${osarch%/*}"; arch="$${osarch#*/}" + GOOS="$$os" GOARCH="$$arch" \ + go build -ldflags="-s -w -X main.version=${CI_COMMIT_TAG}" \ + -o "logarchiver-$${os}-$${arch}" ./cmd/logarchiver + done + depends_on: [test] + backend_options: + kubernetes: + serviceAccountName: default + resources: + requests: + memory: 512Mi + cpu: 1 + limits: + memory: 2Gi + cpu: 2 + + # Package the built binary + generated shell completions into an RPM. + - name: package + image: git.unkin.net/unkin/almalinux9-rpmbuilder:latest + commands: + - ./scripts/build-rpm.sh ${CI_COMMIT_TAG} + depends_on: [build] + backend_options: + kubernetes: + serviceAccountName: default + resources: + requests: + memory: 512Mi + cpu: 1 + limits: + memory: 2Gi + cpu: 2 + + # Publish the RPM to the artifactapi local rpm repo (a real yum repo; + # repodata regenerates automatically). + - name: upload-rpm + image: git.unkin.net/unkin/almalinux9-base:20260606 + commands: + - | + HOST="https://artifactapi.k8s.syd1.au.unkin.net" + REPO="rpm-internal" + for rpm in dist/*.rpm; do + FILE=$$(basename "$$rpm") + # artifactapi has no HEAD route (405); probe with GET against the + # served path (RPMs are stored under Packages/) to avoid re-upload. + code=$$(curl -s -o /dev/null -w '%{http_code}' "$$HOST/api/v2/remotes/$$REPO/files/Packages/$$FILE" || true) + if [ "$$code" = "200" ]; then + echo "$$FILE already exists in $$REPO (HTTP $$code); skipping upload" + continue + fi + echo "Uploading $$FILE to $$REPO (existence probe returned $$code)" + curl -f -X PUT \ + "$$HOST/api/v2/remotes/$$REPO/files/$$FILE" \ + -H "Content-Type: application/x-rpm" \ + --data-binary @"$$rpm" + done + depends_on: [package] + backend_options: + kubernetes: + serviceAccountName: default + resources: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 512Mi + cpu: 500m + + # Cut a Gitea release with the cross-platform CLI binaries attached. + - name: release + image: git.unkin.net/unkin/almalinux9-base:20260606 + environment: + RELEASER_TOKEN: + from_secret: RELEASER_TOKEN + commands: + - | + curl --output /usr/local/bin/tea https://artifactapi.k8s.syd1.au.unkin.net/api/v1/remote/gitea-dl/tea/0.12.0/tea-0.12.0-linux-amd64 && chmod +x /usr/local/bin/tea + tea logins add --name gitea --url https://git.unkin.net --token "$${RELEASER_TOKEN}" --no-version-check + # Find the previous release tag for the changelog range; skip tags on the + # current commit and pick the newest semver ancestor. + CUR_SHA=$$(git rev-list -n1 "${CI_COMMIT_TAG}") + PREV_TAG="" + for t in $$(git tag --sort=-v:refname); do + [ "$$t" = "${CI_COMMIT_TAG}" ] && continue + [ "$$(git rev-list -n1 "$$t")" = "$$CUR_SHA" ] && continue + if git merge-base --is-ancestor "$$t" "${CI_COMMIT_TAG}" 2>/dev/null; then + PREV_TAG="$$t"; break + fi + done + if [ -n "$$PREV_TAG" ]; then + NOTES=$$(git log "$${PREV_TAG}..${CI_COMMIT_TAG}" --pretty=format:"- %s") + else + NOTES=$$(git log --pretty=format:"- %s") + fi + tea releases create --tag "${CI_COMMIT_TAG}" --title "${CI_COMMIT_TAG}" --note "$${NOTES}" --login gitea --repo "${CI_REPO}" + RPM=$$(ls dist/*.rpm 2>/dev/null | head -1) + ASSETS="logarchiver-linux-amd64 logarchiver-linux-arm64 logarchiver-darwin-amd64 logarchiver-darwin-arm64" + [ -n "$$RPM" ] && ASSETS="$$ASSETS $$RPM" + sha256sum $$ASSETS > sha256sums.txt + tea releases assets create "${CI_COMMIT_TAG}" $$ASSETS sha256sums.txt \ + --login gitea --repo "${CI_REPO}" + depends_on: [upload-rpm] + backend_options: + kubernetes: + serviceAccountName: default + resources: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 512Mi + cpu: 500m diff --git a/.woodpecker/test.yaml b/.woodpecker/test.yaml new file mode 100644 index 0000000..5e179a7 --- /dev/null +++ b/.woodpecker/test.yaml @@ -0,0 +1,33 @@ +when: + - event: pull_request + +steps: + - name: lint + image: golangci/golangci-lint:latest + commands: + - golangci-lint run ./... + backend_options: + kubernetes: + serviceAccountName: default + resources: + requests: + memory: 512Mi + cpu: 1 + limits: + memory: 2Gi + cpu: 2 + + - name: test + image: golang:1.25 + commands: + - go test -v -race ./... + backend_options: + kubernetes: + serviceAccountName: default + resources: + requests: + memory: 512Mi + cpu: 1 + limits: + memory: 2Gi + cpu: 2 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f2032af --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM golang:1.25-alpine AS builder + +RUN apk add --no-cache git + +WORKDIR /build + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +ARG VERSION=dev +RUN CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=${VERSION}" -o logarchiver ./cmd/logarchiver + +FROM gcr.io/distroless/static-debian12:nonroot + +COPY --from=builder /build/logarchiver /usr/local/bin/logarchiver + +# Prometheus metrics / healthz listener. +EXPOSE 9090 + +ENTRYPOINT ["logarchiver"] +CMD ["run"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..81294ea --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Ben Vincent + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..883cb53 --- /dev/null +++ b/Makefile @@ -0,0 +1,81 @@ +BINARY := logarchiver +PKG := ./cmd/logarchiver +DIST := dist +MODULE := git.unkin.net/unkin/logarchiver +VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +GOFLAGS := -ldflags="-s -w -X main.version=$(VERSION)" +OS ?= $(shell go env GOOS) +ARCH ?= $(shell go env GOARCH) + +GO_VERSION_REQUIRED := 1.23 +GO_VERSION_ACTUAL := $(shell go version | sed 's/go version go\([0-9]*\.[0-9]*\).*/\1/') + +.PHONY: all build test test-race lint fmt vet tidy clean install completions rpm rpm-package check-go patch minor major _tag + +all: build + +check-go: + @if [ "$$(printf '%s\n%s' "$(GO_VERSION_REQUIRED)" "$(GO_VERSION_ACTUAL)" | sort -V | head -1)" != "$(GO_VERSION_REQUIRED)" ]; then \ + echo "ERROR: Go >= $(GO_VERSION_REQUIRED) required, found $(GO_VERSION_ACTUAL)"; exit 1; \ + fi + +# Build the single logarchiver binary (service + CLI) into dist/. +build: check-go tidy + CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build $(GOFLAGS) -o $(DIST)/$(BINARY) $(PKG) + +test: check-go + go test -race -count=1 ./... + +test-race: test + +lint: + golangci-lint run ./... + +vet: + go vet ./... + +fmt: + gofmt -w . + +tidy: + go mod tidy + +install: + go install $(GOFLAGS) $(PKG) + +clean: + rm -rf $(DIST) $(BINARY) $(BINARY)-* + +# Generate bash/zsh/fish completions from the freshly built binary. +completions: build + @mkdir -p $(DIST)/completions + $(DIST)/$(BINARY) completion bash > $(DIST)/completions/$(BINARY).bash + $(DIST)/$(BINARY) completion zsh > $(DIST)/completions/_$(BINARY) + $(DIST)/$(BINARY) completion fish > $(DIST)/completions/$(BINARY).fish + +# Build then package the CLI into an RPM (with completions) via nfpm. +rpm: build rpm-package +rpm-package: + ./scripts/build-rpm.sh $(VERSION) + +# Bump helpers — read the latest semver tag and create/push the next one. +_LATEST := $(shell git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' | head -1) +_BASE := $(if $(_LATEST),$(_LATEST),v0.0.0) +_MAJ := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f1) +_MIN := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f2) +_PAT := $(shell echo $(_BASE) | sed 's/^v//' | cut -d. -f3) + +patch: + @NEW=v$(_MAJ).$(_MIN).$(shell expr $(_PAT) + 1); \ + git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW + +minor: + @NEW=v$(_MAJ).$(shell expr $(_MIN) + 1).0; \ + git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW + +major: + @NEW=v$(shell expr $(_MAJ) + 1).0.0; \ + git tag $$NEW && echo "Tagged $$NEW" && $(MAKE) _tag TAG=$$NEW + +_tag: + git push origin $(TAG) diff --git a/README.md b/README.md index dff686b..7a3dfc4 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,80 @@ # logarchiver -Archives raw logs from NATS JetStream to S3 as zstd-compressed, GPG-encrypted, indexed objects; plus a CLI to search and retrieve them. Go service + CLI. \ No newline at end of file +Archive raw logs from the centralized logging NATS JetStream stream to S3 as +**zstd-compressed, OpenPGP-encrypted, indexed** objects — and a CLI to **search** +the index and **retrieve/decrypt** archived logs. + +logarchiver replaces the plain Vector archiver leg of the logging stack +(argocd-apps #296). Where Vector wrote gzipped NDJSON to Ceph RGW with no index +and no encryption, logarchiver adds: + +- **zstd** compression (`github.com/klauspost/compress/zstd`); +- **OpenPGP encryption** with a key held in Ben's Vault GPG secrets engine + (`vault-plugin-secrets-gpg`) — the private key never leaves Vault; +- a **searchable ClickHouse index** so you can answer *"vault logs from host X + between dates Y and Z"* without scanning S3; +- **sink-conditional acks**: a batch's JetStream messages are acknowledged only + after the object is durably in S3 **and** indexed — something a stock Vector + NATS consumer cannot do. + +It is a single Go binary that is both the service (`logarchiver run`) and the +operator CLI (`search` / `fetch` / `init-schema`). + +## Quick start + +```sh +# Service (in-cluster): drains the JetStream 'archiver' consumer to S3 + index. +logarchiver run # defaults suit the logging stack; env supplies secrets + +# Operator CLI (laptop, with VAULT_TOKEN / ~/.vault-token and S3/ClickHouse creds): +logarchiver search --subject 'logs.k8s.vault.>' --host node-1 --from -24h +logarchiver fetch --subject 'logs.vm.*' --host db-1 --from -24h -o ./out +logarchiver fetch archive/logs.k8s.vault._/2026/07/27/…​.ndjson.zst.larc -o - + +# Schema (local/dev; in-cluster the argocd bootstrap Job owns this): +logarchiver init-schema # or: init-schema --print to emit the DDL +``` + +## How it works (short version) + +``` +JetStream LOGS / durable "archiver" ──▶ batch by subject (size/count/time) + │ │ + │ NDJSON ─▶ zstd ─▶ AES-256-GCM frames + │ │ (random per-object data key) + │ data key ─▶ OpenPGP-encrypt to Vault pubkey + │ ▼ + └── ack ONLY after ───────────────── S3 PUT (LARC1 object) + ClickHouse index row +``` + +Retrieval sends only the tiny wrapped data key to the Vault GPG engine (which +does whole-payload decrypt only), recovers it, and streams the bulk locally. +See [docs/architecture.md](docs/architecture.md) and the +[retrieval runbook](docs/retrieval-runbook.md) for the full design and the +honest account of what the GPG engine supports. + +## Documentation + +- [docs/architecture.md](docs/architecture.md) — components, data flow, the + LARC1 container format, index schema, delivery guarantees. +- [docs/retrieval-runbook.md](docs/retrieval-runbook.md) — crypto/decrypt design + and step-by-step retrieval, key setup, failure modes. +- [docs/deployment.md](docs/deployment.md) — drop-in fit with the logging stack + (NATS/S3/ClickHouse/Vault wiring, secrets, k8s). +- Subcommands: [run](docs/run.md) · [search](docs/search.md) · + [fetch](docs/fetch.md) · [init-schema](docs/init-schema.md) +- [config.example.yaml](config.example.yaml) — every config knob with defaults. +- [schema/archive_index.sql](schema/archive_index.sql) — the index DDL. + +## Build / test / lint + +```sh +make build # -> dist/logarchiver +make test # go test -race ./... +make lint # golangci-lint run ./... +make rpm # build + package the CLI RPM (nfpm) +make patch|minor|major# tag + push a release (triggers the Woodpecker pipeline) +``` + +Releases (on a `v*` tag): a container image `git.unkin.net/unkin/logarchiver` +for the service, and a Gitea binary release + `rpm-internal` RPM for the CLI. diff --git a/cmd/logarchiver/main.go b/cmd/logarchiver/main.go new file mode 100644 index 0000000..37d6621 --- /dev/null +++ b/cmd/logarchiver/main.go @@ -0,0 +1,17 @@ +// Command logarchiver archives NATS JetStream logs to S3 (zstd + OpenPGP) and +// provides a CLI to search the index and retrieve/decrypt archived logs. +package main + +import ( + "os" + + "git.unkin.net/unkin/logarchiver/internal/cli" +) + +// version is injected at build time via -ldflags "-X main.version=...". +var version = "dev" + +func main() { + cli.SetVersion(version) + os.Exit(cli.Execute()) +} diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..69f7e7a --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,84 @@ +# logarchiver configuration example. +# +# Every field below shows its built-in default (the defaults are tuned for the +# centralized logging stack in argocd-apps #296). In k8s you typically deploy +# with NO config file and let the defaults + secret env vars drive everything; +# this file documents the knobs and is handy for local/dev runs. +# +# Precedence: built-in defaults < this file < environment variables. + +nats: + url: "nats://nats.logging.svc.cluster.local:4222" + stream: "LOGS" + # Durable pull-consumer name. Reusing "archiver" takes over the leg the Vector + # archiver currently owns; use a distinct name (e.g. "archiver-canary") to run + # alongside it during migration. + durable: "archiver" + # Server-side subject filter(s). Overridable via env ARCHIVE_SUBJECTS + # (space-separated). Default archives Vault audit logs only. + subjects: + - "logs.k8s.vault.>" + user: "log-consumer" + # Password comes from the nats-auth secret via NATS_CONSUMER_PASSWORD. + password: "" + password_env: "NATS_CONSUMER_PASSWORD" + ca_file: "" # in-cluster NATS is plaintext + fetch_batch: 512 + ack_wait: "2m" + +batch: + # A per-subject batch becomes one object when any bound is hit. + max_bytes: 67108864 # 64 MiB raw NDJSON + max_events: 200000 + max_age: "5m" + +s3: + endpoint: "https://s3.ceph.unkin.net" + bucket: "logs-archive" + region: "us-east-1" + path_style: true + # Object key template. Fields: {{.Subject}} {{.Year}} {{.Month}} {{.Day}}. + key_prefix: "archive/{{.Subject}}/{{.Year}}/{{.Month}}/{{.Day}}/" + ca_file: "/etc/vault-ca/ca.crt" + # Credentials come from the cephrgw BucketAccess secret logs-archive-s3 via the + # standard AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars. Endpoint/bucket + # can also be sourced from that secret's S3_ENDPOINT / BUCKET_NAME keys: + endpoint_env: "S3_ENDPOINT" + bucket_env: "BUCKET_NAME" + +crypto: + # Vault GPG engine key used to encrypt objects (public key) and decrypt on + # retrieval (private key, server-side in Vault). + key_name: "logarchive" + # Where the service gets the PUBLIC key: "file" (mounted armored key, no Vault + # dependency for the service) or "vault" (read gpg/keys/). + pubkey_source: "file" + pubkey_file: "/etc/logarchiver/pubkey.asc" + refresh_interval: "1h" + frame_size: 1048576 # 1 MiB AES-GCM frames (streaming decrypt granularity) + vault: + address: "" # falls back to VAULT_ADDR + mount: "gpg" + auth_method: "kubernetes" # service: kubernetes; CLI always uses token + k8s_role: "default" + k8s_mount: "k8s/au/syd1" + k8s_jwt_path: "/var/run/secrets/kubernetes.io/serviceaccount/token" + ca_file: "" + +index: + enabled: true + address: "clickhouse-logs.logging.svc.cluster.local:9000" # native protocol + database: "logs" + table: "archive_index" + username: "vector" + password: "" + password_env: "CLICKHOUSE_PASSWORD" + tls: false # in-cluster ClickHouse is plaintext + +metrics: + enabled: true + address: ":9090" # /metrics and /healthz + +log: + level: "info" # debug|info|warn|error + format: "json" # json|text diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..a3bac66 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,122 @@ +# Architecture + +logarchiver is one Go binary with a service mode (`run`) and an operator CLI +(`search`/`fetch`/`init-schema`). Boundaries to external systems (NATS, S3, +ClickHouse, Vault) sit behind interfaces so they can be faked in tests. + +## Packages + +| Package | Responsibility | +|---|---| +| `internal/config` | Config struct, YAML+env loading, validation, defaults tuned for the logging stack. | +| `internal/event` | Extract host + timestamp from a raw JSON event (k8s and VM shapes), subject sanitization. | +| `internal/batcher` | Group events per subject into batches; decide flush by size/count/age; carry ack tokens. | +| `internal/crypto` | The LARC1 container: zstd + framed AES-256-GCM + OpenPGP-wrapped data key. Seal/Open. | +| `internal/vaultgpg` | Thin client for `vault-plugin-secrets-gpg`: fetch public key, decrypt wrapped data key. | +| `internal/s3store` | S3/Ceph RGW object storage behind an `ObjectStore` interface. | +| `internal/index` | ClickHouse archive index (`Index` interface), DDL, pure search-SQL builder. | +| `internal/archiver` | Orchestrates seal → S3 PUT → index write; object-key builder; public-key provider/refresh. | +| `internal/consumer` | JetStream binding + the fetch→batch→persist→ack loop (sink-conditional acks). | +| `internal/metrics` | Prometheus collectors and `/metrics`. | +| `internal/cli` | cobra command tree; wires everything for each subcommand. | + +## Service data flow + +1. **Consume.** A durable JetStream **pull** consumer (`LOGS` stream, durable + `archiver`, subject filter default `logs.k8s.vault.>`) is created/updated at + startup with explicit acks and independent offsets, so logarchiver's replay + position is decoupled from the ClickHouse transform tier. Messages are pulled + in batches. +2. **Route + batch.** Each message's payload is parsed just enough to extract + the source **host** (`.host` → `.hostname` → `.kubernetes.pod_node_name`) and + **timestamp** (`.timestamp` → `.ts`). Events are grouped by NATS **subject**. + A batch is flushed when it hits `max_bytes` (64 MiB raw), `max_events`, or + `max_age` (5 min) — whichever first — or on shutdown. +3. **Seal.** The batch is rendered to NDJSON, compressed with zstd, and sealed + into a **LARC1** object (see below). +4. **Store.** The object is `PUT` to S3 at + `archive//YYYY/MM/DD/-.ndjson.zst.larc`. +5. **Index.** One row per object is inserted into ClickHouse `logs.archive_index`. +6. **Ack.** Only now are the batch's JetStream messages acknowledged. If seal, + S3, or index fails, the messages are Nak'd (with a backoff) and JetStream + redelivers them — nothing is lost on a sink outage. + +## The LARC1 container format + +The Vault GPG engine can only decrypt a **whole** OpenPGP message inline (no +session-key extraction, no streaming — see the retrieval runbook). To keep +objects any size while making retrieval cheap, logarchiver does its own hybrid +encryption instead of PGP-encrypting the whole object: + +``` ++-----------------------------------------------------------+ +| magic "LARC1\n" (6 bytes) | +| header_len uint32 big-endian | +| header JSON { v, key_name, key_fingerprint, | +| wrapped_dek_len, nonce_prefix, | +| frame_size, compression, cipher } | +| wrapped_dek OpenPGP message encrypting the 32-byte DEK | <- only this goes to Vault +| to the engine's public key (~hundreds of B) | +| frames repeated: uint32 ct_len | ciphertext | <- zstd(NDJSON) in AES-256-GCM +| terminated by a zero-length frame | frames; nonce = prefix||counter ++-----------------------------------------------------------+ +``` + +- A fresh random 256-bit **DEK** and 4-byte nonce prefix are generated per + object. Each frame is AES-256-GCM sealed with nonce `prefix||counter` and AAD + = the frame counter (binds frame order; tampering fails the GCM tag). +- The DEK is OpenPGP-encrypted to the engine's public key. This wrapped blob is + a small, standard OpenPGP message — the only thing ever sent to Vault's + decrypt endpoint, regardless of object size. +- Compression order is NDJSON → zstd → encrypt, so decryption streams frames, + GCM-decrypts, and feeds a streaming zstd decoder to emit NDJSON. + +Because objects are a logarchiver-specific container (not a bare `gpg` file), +retrieval must go through `logarchiver fetch`. This is the deliberate cost of +supporting arbitrary object sizes against an engine that only whole-payload +decrypts. + +## Index schema + +`logs.archive_index` (one row per object), `MergeTree`, +`PARTITION BY toYYYYMM(min_ts)`, `ORDER BY (subject, min_ts, object_key)`, with a +`bloom_filter` skip index on `hosts`: + +| column | type | notes | +|---|---|---| +| object_key | String | S3 key | +| bucket | LowCardinality(String) | e.g. `logs-archive` | +| subject | LowCardinality(String) | NATS subject | +| hosts | Array(LowCardinality(String)) | distinct source hosts | +| min_ts / max_ts | DateTime64(3) | event time range | +| event_count | UInt64 | | +| raw_bytes / stored_bytes | UInt64 | pre/post compression+encryption | +| compression / cipher / container_format | LowCardinality(String) | `zstd` / `AES-256-GCM` / `LARC1` | +| key_name / key_fingerprint | LowCardinality(String) / String | Vault GPG key + 40-hex fingerprint | +| created_at | DateTime64(3) DEFAULT now64(3) | | + +Search overlaps the `[from,to]` window (`max_ts >= from AND min_ts <= to`), +matches the subject glob with an anchored regex (`*` = one token, `>` = rest), +and filters hosts with `has(hosts, …)` (exact) or `arrayExists(… match …)` (glob). + +The DDL is owned in-cluster by the argocd bootstrap Job; `logarchiver +init-schema` applies the same statements and `--print` emits them. Keep +`schema/archive_index.sql` and `internal/index/ddl.go` in sync. + +## Delivery guarantees + +- **At-least-once**, sink-conditional. Acks happen only after S3 + index + success. A failure after S3 but before index leaves an orphan object (still + retrievable by prefix; reaped by bucket lifecycle) and triggers redelivery, + which produces a fresh object — so an event may appear in two objects, never + zero. Downstream consumers of the archive should treat events as at-least-once. +- **Graceful shutdown** drains open batches (bounded timeout) so in-flight + events are persisted and acked before exit. + +## Configuration & secrets + +Config loads defaults < YAML file < env. Secrets are never in the file: +NATS password from `NATS_CONSUMER_PASSWORD`, S3 creds from the standard `AWS_*` +env (cephrgw `logs-archive-s3` secret), ClickHouse password from +`CLICKHOUSE_PASSWORD`, Vault via `VAULT_ADDR` + k8s auth (service) or ambient +token (CLI). See [deployment.md](deployment.md). diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..72eef50 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,55 @@ +# Deployment (drop-in fit with the logging stack) + +This describes how logarchiver slots into the centralized logging stack +(argocd-apps #296). The actual argocd manifest swap is a **separate later task**; +this documents the wiring logarchiver is built for so that swap is mechanical. + +## Where it runs + +- Namespace **`logging`**, ServiceAccount **`default`** (reuses the stack's + `VaultAuth` `default`, k8s auth mount `k8s/au/syd1`, role `default`). +- Single-replica Deployment (independent JetStream offsets; one archiver is + enough — scale by subject-sharding into multiple durables if ever needed). +- Pod label **`vector.dev/exclude: "true"`** so the Vector agent does not scrape + logarchiver's own logs (matches the Vector archiver it replaces). +- Startup healthcheck against RGW should be lenient (RGW cred propagation is + slow), like the Vector archiver. + +## What it binds to (all already provided by #296) + +| Dependency | Wiring | +|---|---| +| **NATS** | `nats://nats.logging.svc.cluster.local:4222`, user `log-consumer`, password from secret **`nats-auth`** key `consumer_password` → env `NATS_CONSUMER_PASSWORD`. Stream `LOGS`, durable `archiver`. | +| **S3 (Ceph RGW)** | Bucket `logs-archive`, endpoint `https://s3.ceph.unkin.net` (path-style, region `us-east-1`), CA `/etc/vault-ca/ca.crt` (mount the `vault-ca-cert` secret). Creds from cephrgw BucketAccess secret **`logs-archive-s3`** via `envFrom` (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`; optional `S3_ENDPOINT`/`BUCKET_NAME`). | +| **ClickHouse** | `clickhouse-logs.logging.svc.cluster.local:9000` (native), database `logs`, table `archive_index`, user `vector`, password from secret **`clickhouse-credentials`** key `password` → env `CLICKHOUSE_PASSWORD`. | +| **Vault GPG engine** | Public key: default `pubkey_source: file` from a mounted armored key (secret/ConfigMap at `/etc/logarchiver/pubkey.asc`) — no Vault dependency on the hot path. Or `pubkey_source: vault` with k8s auth to read `gpg/keys/logarchive`. | + +## Secrets to seed (Vault KV, reusing the `logging/default` path) + +Nothing new is strictly required if you reuse the existing `nats-auth`, +`logs-archive-s3`, and `clickhouse-credentials` secrets. For the file pubkey, +add an armored public key as a mounted secret (e.g. +`kv/kubernetes/namespace/logging/default/logarchiver-pubkey` → VaultStaticSecret +→ file mount). + +## Cross-repo notes + +- **No new ServiceAccount** (reuses `default`), so no argocd-apps SA PR is needed + for CI; the Woodpecker steps already use `serviceAccountName: default`. +- **terraform-vault:** only needed if you (a) use `pubkey_source: vault` and the + `logging/default` policy doesn't already permit `read gpg/keys/logarchive`, or + (b) want a dedicated operator policy for `update gpg/decrypt/logarchive`. + Operators today use their own human Vault tokens for `fetch`, so this is + optional — track as a follow-up, not a blocker. +- **ClickHouse table:** created by the argocd bootstrap Job (embed the output of + `logarchiver init-schema --print`), not by the service at runtime. + +## Migrating off the Vector archiver + +The Vector archiver binds the same `LOGS`/`archiver` durable. To cut over +safely: deploy logarchiver with a **distinct** durable (e.g. +`durable: archiver-canary`) and a narrow `ARCHIVE_SUBJECTS` to validate objects ++ index rows land, then repoint it to the `archiver` durable and scale the +Vector archiver to zero. logarchiver writes a different object prefix +(`archive/…​.ndjson.zst.larc`) than Vector (`raw/…​.log.gz`), so the two never +collide in the bucket. diff --git a/docs/fetch.md b/docs/fetch.md new file mode 100644 index 0000000..7d66b52 --- /dev/null +++ b/docs/fetch.md @@ -0,0 +1,41 @@ +# `logarchiver fetch` + +Download, decrypt, and decompress archived objects to plain NDJSON. + +```sh +logarchiver fetch [object-key ...] [flags] +``` + +Objects are selected either by explicit object-key arguments, or — when no keys +are given — by the same `--subject/--host/--from/--to` query used by +[`search`](search.md). When `--host/--from/--to` are supplied they ALSO +re-filter the emitted events to just the matching lines. + +## Flags + +| Flag | Meaning | +|---|---| +| `-o, --output` | `-` for stdout (default), or a directory to write one NDJSON file per object. | +| `--subject`/`--host`/`--from`/`--to`/`--limit` | Object selection (same as `search`) when no keys are given; host/time also re-filter emitted lines. | + +## How decryption works + +For each object, `fetch` reads the object header to learn which Vault GPG key +wrapped it, sends only the small wrapped data key to the engine's +`gpg/decrypt/` endpoint, recovers the data key, and streams the AES-GCM +frames through local decryption + zstd to emit NDJSON. See the +[retrieval runbook](retrieval-runbook.md) for the full design and required creds +(S3 read, internal CA, ambient `VAULT_TOKEN`/`~/.vault-token`). + +## Examples + +```sh +# Straight to stdout by key: +logarchiver fetch archive/logs.k8s.vault._/2026/07/27/20260727T101500Z-ab12cd34.ndjson.zst.larc -o - + +# From a query, re-filtered to host db-1 in the last 24h, into ./out: +logarchiver fetch --subject 'logs.vm.*' --host db-1 --from -24h -o ./out +``` + +Exit status is non-zero if any selected object fails; per-object errors are +reported on stderr and the remaining objects are still processed. diff --git a/docs/init-schema.md b/docs/init-schema.md new file mode 100644 index 0000000..a1e4554 --- /dev/null +++ b/docs/init-schema.md @@ -0,0 +1,18 @@ +# `logarchiver init-schema` + +Create the ClickHouse archive-index database and table (idempotent). + +```sh +logarchiver init-schema # execute the DDL against index.address +logarchiver init-schema --print # print the DDL instead of executing it +``` + +In-cluster the argocd bootstrap Job owns schema creation (a ClickHouse PostSync +hook, mirroring the logging stack's `clickhouse-schema` job). Use +`init-schema --print` to emit the exact `CREATE DATABASE` / `CREATE TABLE` +statements to embed in that Job, and `init-schema` (executing) for local/dev. + +The statements are the source-of-truth DDL from `internal/index/ddl.go`, also +kept as [`schema/archive_index.sql`](../schema/archive_index.sql). Database and +table names come from `index.database` / `index.table` (defaults `logs` / +`archive_index`). diff --git a/docs/retrieval-runbook.md b/docs/retrieval-runbook.md new file mode 100644 index 0000000..5484f00 --- /dev/null +++ b/docs/retrieval-runbook.md @@ -0,0 +1,118 @@ +# Retrieval runbook & crypto design + +This document is the honest account of how encryption and retrieval work, what +the Vault GPG engine actually supports, and the operational steps to get plain +NDJSON back out of an archived object. + +## What the Vault GPG engine supports (and doesn't) + +logarchiver encrypts to a key in Ben's `vault-plugin-secrets-gpg` engine +(transit-style OpenPGP, mounted at `gpg/`). Relevant endpoints: + +- `GET gpg/keys/` → returns `data.public_key` (armored) and + `data.fingerprint` (40-hex uppercase, no spaces). Used to encrypt locally. +- `POST gpg/decrypt/` with `{ "ciphertext": "" }` → returns `{ "plaintext": "" }`. + +**Crucial limitation:** the decrypt endpoint does **whole-payload inline +decryption only**. It reads the entire OpenPGP message and returns the entire +plaintext. There is **no** session-key (PKESK) extraction, **no** chunking, and +**no** streaming. The whole ciphertext must fit in a base64 JSON request body +(Vault's default `max_request_size` is 32 MiB), and the whole plaintext comes +back base64 in the response. So you cannot feed a large archive object straight +to the engine, and you cannot ask it to decrypt only the session key. + +## The design logarchiver chose + +Rather than bound object sizes to the engine's request limit, logarchiver does +hybrid encryption itself and sends the engine only a tiny key blob: + +1. **Encrypt (service, local):** generate a random 256-bit **DEK**; encrypt the + zstd-compressed NDJSON locally with AES-256-GCM in frames; OpenPGP-encrypt + just the 32-byte DEK to the engine's **public** key. The wrapped DEK is a + small standard OpenPGP message. The private key is never present. +2. **Decrypt (CLI, retrieval):** read the object header, send **only the wrapped + DEK** (a few hundred bytes) to `gpg/decrypt/`, get the DEK back, then + stream-decrypt the AES-GCM frames locally and zstd-decompress to NDJSON. + +The Vault round-trip is tiny and constant regardless of object size, the engine +needs no changes, and the private key stays in Vault. See +[architecture.md](architecture.md#the-larc1-container-format) for the container +layout. + +> **Possible engine enhancement (follow-up, not required):** if the engine ever +> grows a "decrypt session key only / return PKESK plaintext" operation, objects +> could be plain standard OpenPGP messages while retaining tiny Vault +> round-trips. logarchiver's self-managed DEK envelope already achieves the +> operational goal without it, so this is optional. + +## One-time key setup + +Create the archive key in the engine (if it doesn't already exist) and export +the public key for the service: + +```sh +export VAULT_ADDR=https://vault.service.consul:8200 + +# Create the key (idempotent; rsa-3072 default, or ed25519). +vault write gpg/keys/logarchive identity="logarchiver " algorithm=ed25519 + +# Fingerprint the service records per object (sanity check): +vault read -field=fingerprint gpg/keys/logarchive + +# Export the armored public key for the service's file-mounted pubkey source: +vault read -field=public_key gpg/keys/logarchive > pubkey.asc +``` + +In k8s the service can instead read the public key directly from Vault +(`pubkey_source: vault`, k8s auth) — no mounted file needed. The file source is +the default because it removes any Vault dependency from the hot path. + +## Retrieve archived logs + +The CLI needs: S3 read creds (`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` for the +`logs-archive` bucket), the internal CA (`s3.ca_file`), and a Vault token with +decrypt on the key (ambient `VAULT_TOKEN` or `~/.vault-token`, exactly like +`passv`). + +```sh +export VAULT_ADDR=https://vault.service.consul:8200 +vault login ... # or have ~/.vault-token +export AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... + +# 1. Find objects via the index. +logarchiver search --subject 'logs.k8s.vault.>' --host node-1 \ + --from 2026-07-01 --to 2026-07-27 + +# 2a. Retrieve by the object keys search printed, to stdout: +logarchiver fetch [ ...] -o - + +# 2b. Or retrieve straight from a query, re-filtered to just the matching +# host/time events, into a directory (one NDJSON file per object): +logarchiver fetch --subject 'logs.vm.*' --host db-1 --from -24h -o ./out +``` + +`fetch` downloads each object, reads its header to learn which Vault key wrapped +it (`key_name`), decrypts the wrapped DEK via the engine, streams the frames +through local AES-GCM + zstd, and emits NDJSON. When `--host`/`--from`/`--to` +are supplied they also re-filter the emitted lines (events lacking a parseable +timestamp are kept, never silently dropped on time grounds). + +## Failure modes + +| Symptom | Likely cause | Action | +|---|---|---| +| `decrypt via gpg/decrypt/: permission denied` | Token lacks the decrypt policy | Grant `update` on `gpg/decrypt/` (and `read` on `gpg/keys/`). | +| `unwrap dek: … no plaintext` | Wrong key name / key rotated below min_decryption_version | Confirm object `key_name`; the engine tries versions ≥ min_decryption_version. | +| `bad magic: not a logarchiver (LARC1) object` | Fetching a non-logarchiver object (e.g. an old Vector `.log.gz`) | Use the right prefix; Vector objects are plain gzip, decrypt them with gzip. | +| `decrypt frame N: cipher: message authentication failed` | Object corruption/tampering | Object integrity is broken; re-archive from JetStream if still within retention. | +| `pubkey fingerprint mismatch` (service) | Armored pubkey doesn't match the engine's reported fingerprint | Re-export the pubkey; the service refuses to encrypt to a mismatched key. | + +## Key rotation + +Rotating the engine key (`vault write gpg/keys/logarchive/rotate ...`) starts +encrypting new objects to the new version; the service picks up the new public +key within `crypto.refresh_interval` (default 1h) or on restart. Old objects +remain decryptable because the engine tries every key version from +`min_decryption_version` up. Each object records the fingerprint it was sealed +with, so you can always tell which key version applies. diff --git a/docs/run.md b/docs/run.md new file mode 100644 index 0000000..689b4c3 --- /dev/null +++ b/docs/run.md @@ -0,0 +1,40 @@ +# `logarchiver run` + +Runs the archiver service: binds the JetStream pull consumer and drains it to +S3 + the ClickHouse index, acking only after each object is durably persisted. + +```sh +logarchiver run [-c config.yaml] +``` + +## Behaviour + +- Creates/updates the durable consumer (`nats.stream` / `nats.durable`) with the + configured subject filters, explicit acks, and `nats.ack_wait`. +- Batches events per subject and flushes on `batch.max_bytes` / `max_events` / + `max_age`. Each flush seals a LARC1 object, PUTs it to S3, writes one index + row, then acks the batch. On any failure the batch is Nak'd for redelivery. +- Loads the OpenPGP public key from `crypto.pubkey_source` (`file` or `vault`) + and refreshes it every `crypto.refresh_interval`. +- Serves Prometheus metrics and `/healthz` on `metrics.address` (default `:9090`) + when `metrics.enabled`. +- Handles SIGINT/SIGTERM: stops fetching, drains open batches (bounded), exits. + +## Key env vars + +| Env | Purpose | +|---|---| +| `NATS_CONSUMER_PASSWORD` | NATS `log-consumer` password (nats-auth secret). | +| `ARCHIVE_SUBJECTS` | Space-separated subject filters (overrides `nats.subjects`). | +| `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` | S3 creds (cephrgw `logs-archive-s3`). | +| `S3_ENDPOINT` / `BUCKET_NAME` | Optional S3 endpoint/bucket from the cephrgw secret. | +| `CLICKHOUSE_PASSWORD` | ClickHouse `vector` user password. | +| `VAULT_ADDR` | Vault address when `pubkey_source: vault`. | +| `LOGARCHIVER_CONFIG` | Config file path (same as `-c`). | + +## Metrics + +`logarchiver_objects_stored_total`, `_events_archived_total`, +`_raw_bytes_total`, `_stored_bytes_total`, `_store_failures_total`, +`_index_failures_total`, `_messages_fetched_total`, `_acks_total`, +`_batches_flushed_total{trigger}`, `logarchiver_pending_events`. diff --git a/docs/search.md b/docs/search.md new file mode 100644 index 0000000..8cb031d --- /dev/null +++ b/docs/search.md @@ -0,0 +1,39 @@ +# `logarchiver search` + +Query the ClickHouse archive index for objects matching a subject/host/time +window. Prints one row per matching S3 object with event counts and sizes; use +the object keys with [`fetch`](fetch.md). + +```sh +logarchiver search [flags] +``` + +## Flags + +| Flag | Meaning | +|---|---| +| `--subject` | NATS-style subject glob: `*` matches one token, `>` matches the rest. e.g. `logs.vm.*`, `logs.k8s.vault.>`. | +| `--host` | Source host to match: exact, or a glob containing `*`. | +| `--from` | Start of window: RFC3339, `YYYY-MM-DD`, or a relative duration like `-24h`. | +| `--to` | End of window (same formats). | +| `--limit` | Max objects (default 100; `0` = no limit). | +| `--json` | Emit results as JSON instead of a table. | + +An object matches the time window when its `[min_ts, max_ts]` overlaps +`[from, to]`. Objects are ordered by `min_ts`. + +## Examples + +```sh +# Vault audit logs from a node in the last day. +logarchiver search --subject 'logs.k8s.vault.>' --host node-1 --from -24h + +# All VM logs for a host in July, as JSON. +logarchiver search --subject 'logs.vm.*' --host db-1 \ + --from 2026-07-01 --to 2026-08-01 --json +``` + +## Config + +Uses `index.*` (ClickHouse address/database/table/credentials). Requires +`index.enabled: true`. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..e389420 --- /dev/null +++ b/go.mod @@ -0,0 +1,78 @@ +module git.unkin.net/unkin/logarchiver + +go 1.25.9 + +require ( + github.com/ClickHouse/clickhouse-go/v2 v2.47.0 + github.com/ProtonMail/go-crypto v1.4.1 + github.com/aws/aws-sdk-go-v2/config v1.32.31 + github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0 + github.com/hashicorp/vault/api v1.23.0 + github.com/klauspost/compress v1.19.1 + github.com/nats-io/nats.go v1.52.0 + github.com/prometheus/client_golang v1.24.1 + github.com/spf13/cobra v1.10.2 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/ClickHouse/ch-go v0.73.0 // indirect + github.com/andybalholm/brotli v1.2.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.43.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.30 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 // indirect + github.com/aws/smithy-go v1.27.3 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudflare/circl v1.6.2 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect + github.com/go-jose/go-jose/v4 v4.1.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.7 // indirect + github.com/hashicorp/hcl v1.0.1-vault-7 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/paulmach/orb v0.13.0 // indirect + github.com/pierrec/lz4/v4 v4.1.27 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/procfs v0.21.1 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/segmentio/asm v1.2.1 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/spf13/pflag v1.0.9 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/net v0.57.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + golang.org/x/time v0.12.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8c09d37 --- /dev/null +++ b/go.sum @@ -0,0 +1,169 @@ +github.com/ClickHouse/ch-go v0.73.0 h1:jsHiGRbQ3sz+gekvDFJF29LWDo5dzbJm5s1h8TWVP2M= +github.com/ClickHouse/ch-go v0.73.0/go.mod h1:wkFIxrqlXeRJ9cn3r5Fz5Qen9jl5aTMPuGZeuJpANNY= +github.com/ClickHouse/clickhouse-go/v2 v2.47.0 h1:ZDAzrnKSOPTIsm4tdUNfrii2yc8dk4SVRLC77BR7Z5Q= +github.com/ClickHouse/clickhouse-go/v2 v2.47.0/go.mod h1:sPj7C7UYQ2MWHcfX+4eGN6nwnCqwUKfgO6PcwKpd6K8= +github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= +github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= +github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= +github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/aws/aws-sdk-go-v2 v1.43.0 h1:fharf/WhbRAVZ1du0QL7roNFxZ6T/sWr+4Ni617bwSI= +github.com/aws/aws-sdk-go-v2 v1.43.0/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E= +github.com/aws/aws-sdk-go-v2/config v1.32.31 h1:n4nY9O3QKoHIkL85EX+V8RcMFtOhlpTFhGArg915PXk= +github.com/aws/aws-sdk-go-v2/config v1.32.31/go.mod h1:PN0NYDCCoOpGGsZ2+elDUidmHfQBPyYzN2GCgl8HEBs= +github.com/aws/aws-sdk-go-v2/credentials v1.19.30 h1:TTCvvzFU6gXa4iJecNG/0F/B0oYTiazoRECr2XyLHrY= +github.com/aws/aws-sdk-go-v2/credentials v1.19.30/go.mod h1:jKxAp2AEncnliinzpgOSZDFv6+VjvWhjw/AtbfsWT9U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 h1:kfVL5wAunCJycL6MOQ6aNh6PlAYEymflcjuKmrWUA0o= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31/go.mod h1:nWfRNDAppujCQgOUd43lKT4yeLv9z3nJ3bw1G3BgQKo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 h1:Z8F3hfCY33IGpJjFAnv0wvtv1FIKj1GHmRDEYqy64tw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31/go.mod h1:aVyUoytEyOViR6jhq6jula0xkc5NfBE2hgeF6BvOrao= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 h1:hyOxUyXdh3AyjE93gBgsfziJag9ACwcs+ZpDBLzi8mw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31/go.mod h1:OERqI9k0draSLB8O8woxY3q25ZWTELRK4RRoLMuMZFo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 h1:0MrUL35H/Y4kdFfItoR5jCgtDQ4Z/8LudAoIHRfA4hE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32/go.mod h1:2tNZkuWz54arj8mHVf+8Y7cKkcD8Wr/fBpENgEXpjLc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 h1:mdPwDQPqxlw9Sc62Nt15yjEcARaDbPXkjRYtXsUripo= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24/go.mod h1:ls5ytnwLTcQaUu32fMYXFI3MjpKuTwL840PAm9iqyEg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 h1:w2SIhW92DZPFrSL4ksVCr8IYff5OZwIcxg8+95tzvAI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31/go.mod h1:wAhpCQbkov+IcvjozJbd2xRCoZybUEHNkcFunssNACg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 h1:jWXtZdCnhXa9sGFixRaU2AxT4DIVse9HS4E2f+/KwV0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32/go.mod h1:9JS1UpfVvyD/ZPX8GsKb/Pq8scEM+7GP5fqh9SwH7po= +github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0 h1:7QZWVJZWzHivHWIa+5TELLaBBkbuoj0GPwQtMlJ0sqk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0/go.mod h1:fcvq5L7dK+5cQFicEJwpI6e6Wn8NY2i6yT5wRLYVc7s= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 h1:OHH5iTQvVGmfHjX/5Q+vFuA/Rf2x6/95aJ/75QCQSm4= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.0/go.mod h1:mCF3AK9PpL49oOrhniUXWAfhVBVQ/XbytoE5eccZUIs= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 h1:CaJyYhxBE0M/HJX/YvSaSmQlsI91VHB0lKU8LtLxL3A= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.0/go.mod h1:+e6BMRMPjBQoCw/WovYR9GLy2IU0z4Q77smOB1DraSg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 h1:tC323YV77QdafeBr6LUhLDTsboyuyHLNRwAyCP44kGU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0/go.mod h1:SfLK1sgviHmbI+MozR9iDwDjL4cdCVZtahsjoR+z7wg= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 h1:Pd6PNlp4t8PTXxqzstICl52Wsy78vpjFZ7PRUj44mJc= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.0/go.mod h1:rmQ0TnHzuLPmabgjPcsywhsSOmaBDgzR4zvDxSPsGdg= +github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY= +github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudflare/circl v1.6.2 h1:hL7VBpHHKzrV5WTfHCaBsgx/HGbBYlgrwvNXEVDYYsQ= +github.com/cloudflare/circl v1.6.2/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= +github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI= +github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA= +github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= +github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 h1:U+kC2dOhMFQctRfhK0gRctKAPTloZdMU5ZJxaesJ/VM= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= +github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= +github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= +github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= +github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= +github.com/hashicorp/vault/api v1.23.0 h1:gXgluBsSECfRWTSW9niY2jwg2e9mMJc4WoHNv4g3h6A= +github.com/hashicorp/vault/api v1.23.0/go.mod h1:zransKiB9ftp+kgY8ydjnvCU7Wk8i9L0DYWpXeMj9ko= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= +github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= +github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/paulmach/orb v0.13.0 h1:r7n7mQGGF+cj/CbcivEj9J3HGK+XR+yXnvzRdq9saIw= +github.com/paulmach/orb v0.13.0/go.mod h1:6scRWINywA2Jf05dcjOfLfxrUIMECvTSG2MVbRLxu/k= +github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= +github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= +github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go new file mode 100644 index 0000000..afc9d41 --- /dev/null +++ b/internal/archiver/archiver.go @@ -0,0 +1,161 @@ +// Package archiver turns a ready batch into a stored, indexed object: build +// NDJSON, seal it into a LARC1 container, PUT it to S3, then write the index +// row. Store returns an error if ANY step fails; the caller must not ack the +// batch's messages until Store succeeds (at-least-once, sink-conditional acks). +package archiver + +import ( + "bytes" + "context" + "fmt" + "time" + + "git.unkin.net/unkin/logarchiver/internal/batcher" + "git.unkin.net/unkin/logarchiver/internal/crypto" + "git.unkin.net/unkin/logarchiver/internal/index" + "git.unkin.net/unkin/logarchiver/internal/s3store" +) + +// Metrics is the optional metrics sink (implemented by internal/metrics). A nil +// Metrics is fine (no-op). +type Metrics interface { + ObjectStored(subject string, events int, rawBytes, storedBytes int64) + StoreFailed(subject string) + IndexFailed(subject string) +} + +// Archiver persists batches. +type Archiver struct { + keys *KeyBuilder + pubkeys *PubkeyProvider + store s3store.ObjectStore + idx index.Index // may be nil when indexing is disabled + keyName string + frameSize int + metrics Metrics + nowFn func() time.Time +} + +// Options configures an Archiver. +type Options struct { + Keys *KeyBuilder + Pubkeys *PubkeyProvider + Store s3store.ObjectStore + Index index.Index + KeyName string + FrameSize int + Metrics Metrics +} + +// New builds an Archiver. +func New(o Options) (*Archiver, error) { + if o.Keys == nil || o.Pubkeys == nil || o.Store == nil { + return nil, fmt.Errorf("archiver requires keys, pubkeys and store") + } + fs := o.FrameSize + if fs <= 0 { + fs = 1 << 20 + } + return &Archiver{ + keys: o.Keys, + pubkeys: o.Pubkeys, + store: o.Store, + idx: o.Index, + keyName: o.KeyName, + frameSize: fs, + metrics: o.Metrics, + nowFn: time.Now, + }, nil +} + +// StoreResult reports what Store persisted. +type StoreResult struct { + ObjectKey string + Events int + RawBytes int64 + StoredBytes int64 +} + +// Store seals, uploads and indexes a batch. On success the caller may ack. +func (a *Archiver) Store(ctx context.Context, batch *batcher.Batch) (StoreResult, error) { + if len(batch.Items) == 0 { + return StoreResult{}, nil + } + now := a.nowFn().UTC() + summary := batch.Summarize(now) + + pub := a.pubkeys.Current() + if pub == nil { + a.metricStoreFailed(batch.Subject) + return StoreResult{}, fmt.Errorf("no public key available") + } + + // Choose the object key from the batch's max timestamp so it lands in the + // date partition of the newest event. + key, err := a.keys.Build(batch.Subject, summary.MaxTS) + if err != nil { + a.metricStoreFailed(batch.Subject) + return StoreResult{}, err + } + + ndjson := batch.NDJSON() + var buf bytes.Buffer + sealed, err := crypto.Seal(&buf, ndjson, pub, a.keyName, a.frameSize) + if err != nil { + a.metricStoreFailed(batch.Subject) + return StoreResult{}, fmt.Errorf("seal object %s: %w", key, err) + } + + if err := a.store.Put(ctx, key, bytes.NewReader(buf.Bytes()), int64(buf.Len())); err != nil { + a.metricStoreFailed(batch.Subject) + return StoreResult{}, err + } + + if a.idx != nil { + row := index.Row{ + ObjectKey: key, + Bucket: a.store.Bucket(), + Subject: batch.Subject, + Hosts: summary.Hosts, + MinTS: summary.MinTS, + MaxTS: summary.MaxTS, + EventCount: uint64(summary.EventCount), + RawBytes: uint64(sealed.RawBytes), + StoredBytes: uint64(sealed.StoredBytes), + Compression: sealed.Header.Compression, + Cipher: sealed.Header.Cipher, + ContainerFormat: "LARC1", + KeyName: a.keyName, + KeyFingerprint: sealed.Header.KeyFingerprint, + } + if err := a.idx.Insert(ctx, row); err != nil { + // The object is in S3 but unindexed. Do NOT ack: on redelivery the + // batch is re-stored (a new object key) and re-indexed. The orphan + // object is harmless (retrievable by prefix) and reaped by lifecycle. + a.metricIndexFailed(batch.Subject) + return StoreResult{}, fmt.Errorf("index object %s: %w", key, err) + } + } + + if a.metrics != nil { + a.metrics.ObjectStored(batch.Subject, summary.EventCount, sealed.RawBytes, sealed.StoredBytes) + } + return StoreResult{ + ObjectKey: key, + Events: summary.EventCount, + RawBytes: sealed.RawBytes, + StoredBytes: sealed.StoredBytes, + }, nil +} + +func (a *Archiver) metricStoreFailed(subject string) { + if a.metrics != nil { + a.metrics.StoreFailed(subject) + } +} + +func (a *Archiver) metricIndexFailed(subject string) { + if a.metrics != nil { + a.metrics.IndexFailed(subject) + } +} diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go new file mode 100644 index 0000000..50e4fd1 --- /dev/null +++ b/internal/archiver/archiver_test.go @@ -0,0 +1,222 @@ +package archiver + +import ( + "bytes" + "context" + "errors" + "io" + "testing" + "time" + + "git.unkin.net/unkin/logarchiver/internal/batcher" + "git.unkin.net/unkin/logarchiver/internal/crypto" + "git.unkin.net/unkin/logarchiver/internal/index" + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" +) + +// --- fakes --- + +type fakeStore struct { + bucket string + objects map[string][]byte + failPut bool +} + +func newFakeStore() *fakeStore { + return &fakeStore{bucket: "test-bucket", objects: map[string][]byte{}} +} + +func (f *fakeStore) Put(_ context.Context, key string, body io.Reader, _ int64) error { + if f.failPut { + return errors.New("simulated s3 failure") + } + data, err := io.ReadAll(body) + if err != nil { + return err + } + f.objects[key] = data + return nil +} +func (f *fakeStore) Get(_ context.Context, key string) (io.ReadCloser, error) { + data, ok := f.objects[key] + if !ok { + return nil, errors.New("not found") + } + return io.NopCloser(bytes.NewReader(data)), nil +} +func (f *fakeStore) List(_ context.Context, _ string) ([]string, error) { return nil, nil } +func (f *fakeStore) Bucket() string { return f.bucket } + +type fakeIndex struct { + rows []index.Row + fail bool +} + +func (f *fakeIndex) Insert(_ context.Context, row index.Row) error { + if f.fail { + return errors.New("simulated index failure") + } + f.rows = append(f.rows, row) + return nil +} +func (f *fakeIndex) Search(context.Context, index.SearchQuery) ([]index.Result, error) { + return nil, nil +} +func (f *fakeIndex) InitSchema(context.Context) error { return nil } +func (f *fakeIndex) Ping(context.Context) error { return nil } +func (f *fakeIndex) Close() error { return nil } + +func testPubkey(t *testing.T) *crypto.PublicKey { + t.Helper() + ent, err := openpgp.NewEntity("t", "", "t@unkin.net", nil) + if err != nil { + t.Fatalf("NewEntity: %v", err) + } + var buf bytes.Buffer + w, _ := armor.Encode(&buf, openpgp.PublicKeyType, nil) + _ = ent.Serialize(w) + _ = w.Close() + pk, err := crypto.LoadPublicKey(buf.Bytes()) + if err != nil { + t.Fatalf("LoadPublicKey: %v", err) + } + return pk +} + +func newTestArchiver(t *testing.T, store *fakeStore, idx index.Index) *Archiver { + t.Helper() + pk := testPubkey(t) + kb, _ := NewKeyBuilder("archive/{{.Subject}}/{{.Year}}/{{.Month}}/{{.Day}}/") + prov, err := NewPubkeyProvider(context.Background(), func(context.Context) (*crypto.PublicKey, error) { + return pk, nil + }) + if err != nil { + t.Fatalf("provider: %v", err) + } + a, err := New(Options{ + Keys: kb, + Pubkeys: prov, + Store: store, + Index: idx, + KeyName: "logarchive", + FrameSize: 4096, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + return a +} + +func sampleBatch() *batcher.Batch { + ts := time.Date(2026, 7, 27, 1, 0, 0, 0, time.UTC) + return &batcher.Batch{ + Subject: "logs.k8s.vault.audit", + Items: []batcher.Item{ + {Subject: "logs.k8s.vault.audit", Host: "node-1", Timestamp: ts, HasTS: true, Raw: []byte(`{"host":"node-1","message":"a"}`)}, + {Subject: "logs.k8s.vault.audit", Host: "node-2", Timestamp: ts.Add(time.Hour), HasTS: true, Raw: []byte(`{"host":"node-2","message":"b"}`)}, + }, + RawBytes: 62, + } +} + +func TestStoreSuccessWritesObjectAndIndex(t *testing.T) { + store := newFakeStore() + idx := &fakeIndex{} + a := newTestArchiver(t, store, idx) + + res, err := a.Store(context.Background(), sampleBatch()) + if err != nil { + t.Fatalf("Store: %v", err) + } + if res.Events != 2 { + t.Errorf("events = %d", res.Events) + } + if len(store.objects) != 1 { + t.Fatalf("expected 1 stored object, got %d", len(store.objects)) + } + // Stored bytes must be a valid LARC1 container. + obj := store.objects[res.ObjectKey] + if _, _, err := crypto.ReadHeader(bytes.NewReader(obj)); err != nil { + t.Errorf("stored object is not a valid container: %v", err) + } + if len(idx.rows) != 1 { + t.Fatalf("expected 1 index row, got %d", len(idx.rows)) + } + row := idx.rows[0] + if row.Subject != "logs.k8s.vault.audit" { + t.Errorf("row subject = %q", row.Subject) + } + if row.EventCount != 2 { + t.Errorf("row event_count = %d", row.EventCount) + } + if len(row.Hosts) != 2 || row.Hosts[0] != "node-1" || row.Hosts[1] != "node-2" { + t.Errorf("row hosts = %v", row.Hosts) + } + if row.Bucket != "test-bucket" { + t.Errorf("row bucket = %q", row.Bucket) + } + if row.ContainerFormat != "LARC1" || row.Compression != "zstd" { + t.Errorf("row metadata wrong: %+v", row) + } + if row.KeyFingerprint == "" { + t.Errorf("row missing key fingerprint") + } +} + +// The central at-least-once property: if S3 fails, Store errors and NOTHING is +// written to the index, so the caller will not ack. +func TestStoreS3FailureNoIndexNoAck(t *testing.T) { + store := newFakeStore() + store.failPut = true + idx := &fakeIndex{} + a := newTestArchiver(t, store, idx) + + if _, err := a.Store(context.Background(), sampleBatch()); err == nil { + t.Fatalf("expected error when S3 put fails") + } + if len(idx.rows) != 0 { + t.Errorf("index written despite S3 failure: %d rows", len(idx.rows)) + } + if len(store.objects) != 0 { + t.Errorf("object recorded despite S3 failure") + } +} + +// If indexing fails after the S3 put, Store still errors (so no ack); the object +// exists but is unindexed — acceptable, it will be re-stored on redelivery. +func TestStoreIndexFailureErrors(t *testing.T) { + store := newFakeStore() + idx := &fakeIndex{fail: true} + a := newTestArchiver(t, store, idx) + + if _, err := a.Store(context.Background(), sampleBatch()); err == nil { + t.Fatalf("expected error when index insert fails") + } + if len(store.objects) != 1 { + t.Errorf("object should still be in S3 (orphan), got %d", len(store.objects)) + } +} + +func TestStoreNilIndexOK(t *testing.T) { + store := newFakeStore() + a := newTestArchiver(t, store, nil) + if _, err := a.Store(context.Background(), sampleBatch()); err != nil { + t.Fatalf("Store with nil index: %v", err) + } + if len(store.objects) != 1 { + t.Errorf("expected object stored") + } +} + +func TestStoreEmptyBatchNoop(t *testing.T) { + store := newFakeStore() + a := newTestArchiver(t, store, &fakeIndex{}) + res, err := a.Store(context.Background(), &batcher.Batch{Subject: "s"}) + if err != nil { + t.Fatalf("empty batch: %v", err) + } + if res.ObjectKey != "" || len(store.objects) != 0 { + t.Errorf("empty batch should store nothing") + } +} diff --git a/internal/archiver/keys.go b/internal/archiver/keys.go new file mode 100644 index 0000000..8e4144c --- /dev/null +++ b/internal/archiver/keys.go @@ -0,0 +1,73 @@ +package archiver + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "strings" + "text/template" + "time" + + "git.unkin.net/unkin/logarchiver/internal/event" +) + +// ObjectExt is the suffix for logarchiver container objects (zstd + framed +// AES-GCM + wrapped OpenPGP DEK). It is deliberately NOT .gz/.pgp because the +// object is a logarchiver-specific container, not a bare gpg file. +const ObjectExt = ".ndjson.zst.larc" + +// KeyBuilder renders S3 object keys from a prefix template. Template fields: +// {{.Subject}} (sanitized), {{.Year}} {{.Month}} {{.Day}} (UTC, zero-padded). +type KeyBuilder struct { + tmpl *template.Template +} + +// NewKeyBuilder compiles the prefix template. +func NewKeyBuilder(prefixTemplate string) (*KeyBuilder, error) { + t, err := template.New("key").Option("missingkey=error").Parse(prefixTemplate) + if err != nil { + return nil, fmt.Errorf("parse key_prefix template: %w", err) + } + return &KeyBuilder{tmpl: t}, nil +} + +type keyData struct { + Subject string + Year string + Month string + Day string +} + +// Build returns a unique object key for a batch of subject at ts. The filename +// is -.ndjson.zst.larc so keys are collision-free and sortable. +func (k *KeyBuilder) Build(subject string, ts time.Time) (string, error) { + ts = ts.UTC() + var sb strings.Builder + err := k.tmpl.Execute(&sb, keyData{ + Subject: event.SubjectToken(subject), + Year: fmt.Sprintf("%04d", ts.Year()), + Month: fmt.Sprintf("%02d", int(ts.Month())), + Day: fmt.Sprintf("%02d", ts.Day()), + }) + if err != nil { + return "", fmt.Errorf("render key prefix: %w", err) + } + prefix := sb.String() + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + suffix, err := randHex(8) + if err != nil { + return "", err + } + name := ts.Format("20060102T150405Z") + "-" + suffix + ObjectExt + return prefix + name, nil +} + +func randHex(n int) (string, error) { + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + return "", fmt.Errorf("random: %w", err) + } + return hex.EncodeToString(b), nil +} diff --git a/internal/archiver/keys_test.go b/internal/archiver/keys_test.go new file mode 100644 index 0000000..7bab693 --- /dev/null +++ b/internal/archiver/keys_test.go @@ -0,0 +1,55 @@ +package archiver + +import ( + "strings" + "testing" + "time" +) + +func TestKeyBuilder(t *testing.T) { + kb, err := NewKeyBuilder("archive/{{.Subject}}/{{.Year}}/{{.Month}}/{{.Day}}/") + if err != nil { + t.Fatalf("NewKeyBuilder: %v", err) + } + ts := time.Date(2026, 7, 5, 10, 15, 0, 0, time.UTC) + key, err := kb.Build("logs.k8s.vault.audit", ts) + if err != nil { + t.Fatalf("Build: %v", err) + } + if !strings.HasPrefix(key, "archive/logs.k8s.vault.audit/2026/07/05/") { + t.Errorf("unexpected prefix: %s", key) + } + if !strings.HasSuffix(key, ObjectExt) { + t.Errorf("missing container suffix: %s", key) + } + if !strings.Contains(key, "20260705T101500Z-") { + t.Errorf("missing timestamp token: %s", key) + } +} + +func TestKeyBuilderUnique(t *testing.T) { + kb, _ := NewKeyBuilder("p/{{.Subject}}/") + ts := time.Date(2026, 7, 5, 10, 15, 0, 0, time.UTC) + k1, _ := kb.Build("s", ts) + k2, _ := kb.Build("s", ts) + if k1 == k2 { + t.Errorf("keys should be unique: %s", k1) + } +} + +func TestKeyBuilderSanitizesSubject(t *testing.T) { + kb, _ := NewKeyBuilder("p/{{.Subject}}/") + key, _ := kb.Build("logs.k8s.a/b", time.Now()) + if strings.Contains(key, "a/b") { + t.Errorf("subject slash not sanitized: %s", key) + } + if !strings.Contains(key, "a_b") { + t.Errorf("expected sanitized a_b: %s", key) + } +} + +func TestKeyBuilderBadTemplate(t *testing.T) { + if _, err := NewKeyBuilder("{{.Nope"); err == nil { + t.Errorf("expected template parse error") + } +} diff --git a/internal/archiver/pubkey.go b/internal/archiver/pubkey.go new file mode 100644 index 0000000..a914080 --- /dev/null +++ b/internal/archiver/pubkey.go @@ -0,0 +1,89 @@ +package archiver + +import ( + "context" + "fmt" + "os" + "sync" + + "git.unkin.net/unkin/logarchiver/internal/config" + "git.unkin.net/unkin/logarchiver/internal/crypto" + "git.unkin.net/unkin/logarchiver/internal/vaultgpg" +) + +// PubkeyLoader fetches the current armored public key from the configured source. +type PubkeyLoader func(ctx context.Context) (*crypto.PublicKey, error) + +// PubkeyProvider caches the current public key and supports periodic refresh so +// key rotation in the Vault GPG engine is picked up without a restart. +type PubkeyProvider struct { + load PubkeyLoader + mu sync.RWMutex + key *crypto.PublicKey +} + +// NewPubkeyProvider builds a provider and loads the key once. +func NewPubkeyProvider(ctx context.Context, load PubkeyLoader) (*PubkeyProvider, error) { + p := &PubkeyProvider{load: load} + if err := p.Refresh(ctx); err != nil { + return nil, err + } + return p, nil +} + +// Refresh reloads the public key. +func (p *PubkeyProvider) Refresh(ctx context.Context) error { + key, err := p.load(ctx) + if err != nil { + return err + } + p.mu.Lock() + p.key = key + p.mu.Unlock() + return nil +} + +// Current returns the cached public key. +func (p *PubkeyProvider) Current() *crypto.PublicKey { + p.mu.RLock() + defer p.mu.RUnlock() + return p.key +} + +// PubkeyLoaderFromConfig builds a loader for the configured source. For +// pubkey_source=vault it also verifies the parsed key's fingerprint matches the +// fingerprint the engine reports, catching armor corruption. +func PubkeyLoaderFromConfig(cfg config.CryptoConfig, vc *vaultgpg.Client) (PubkeyLoader, error) { + switch cfg.Source { + case config.PubkeyFile: + path := cfg.PubkeyFile + return func(_ context.Context) (*crypto.PublicKey, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read pubkey file %s: %w", path, err) + } + return crypto.LoadPublicKey(data) + }, nil + case config.PubkeyVault: + if vc == nil { + return nil, fmt.Errorf("pubkey_source=vault requires a vault client") + } + name := cfg.KeyName + return func(ctx context.Context) (*crypto.PublicKey, error) { + pk, err := vc.FetchPublicKey(ctx, name) + if err != nil { + return nil, err + } + key, err := crypto.LoadPublicKey([]byte(pk.Armored)) + if err != nil { + return nil, err + } + if pk.Fingerprint != "" && key.Fingerprint != pk.Fingerprint { + return nil, fmt.Errorf("pubkey fingerprint mismatch: engine=%s parsed=%s", pk.Fingerprint, key.Fingerprint) + } + return key, nil + }, nil + default: + return nil, fmt.Errorf("unknown pubkey_source %q", cfg.Source) + } +} diff --git a/internal/batcher/batcher.go b/internal/batcher/batcher.go new file mode 100644 index 0000000..5a99c12 --- /dev/null +++ b/internal/batcher/batcher.go @@ -0,0 +1,177 @@ +// Package batcher groups incoming log events into per-subject batches and +// decides when a batch is ready to become one archived object. It is +// deliberately not concurrent: the consumer run loop owns a Batcher and drives +// it from a single goroutine (Add on receive, DueByAge on a ticker, Drain on +// shutdown), which keeps the ack-after-persist accounting simple and race-free. +package batcher + +import ( + "bytes" + "sort" + "time" +) + +// Item is one log event routed into a batch. Ack is an opaque token (a +// jetstream.Msg in production) that the caller acknowledges only after the batch +// has been durably persisted. +type Item struct { + Subject string + Raw []byte + Host string + Timestamp time.Time + HasTS bool + Ack any +} + +// Limits bound a single batch/object. +type Limits struct { + MaxBytes int64 + MaxEvents int + MaxAge time.Duration +} + +// Batch is a ready (or in-progress) group of events for one subject. +type Batch struct { + Subject string + Items []Item + RawBytes int64 + OpenedAt time.Time +} + +// Batcher accumulates open batches keyed by subject. +type Batcher struct { + limits Limits + open map[string]*Batch + nowFn func() time.Time +} + +// New returns a Batcher enforcing limits. +func New(limits Limits) *Batcher { + return &Batcher{limits: limits, open: map[string]*Batch{}, nowFn: time.Now} +} + +// Add appends it to its subject's open batch. If that batch is now full (by +// bytes or event count), it is removed from the open set and returned so the +// caller can flush it; otherwise Add returns nil. +func (b *Batcher) Add(it Item) *Batch { + batch := b.open[it.Subject] + if batch == nil { + batch = &Batch{Subject: it.Subject, OpenedAt: b.nowFn()} + b.open[it.Subject] = batch + } + batch.Items = append(batch.Items, it) + batch.RawBytes += int64(len(it.Raw)) + + if b.full(batch) { + delete(b.open, it.Subject) + return batch + } + return nil +} + +func (b *Batcher) full(batch *Batch) bool { + if b.limits.MaxBytes > 0 && batch.RawBytes >= b.limits.MaxBytes { + return true + } + if b.limits.MaxEvents > 0 && len(batch.Items) >= b.limits.MaxEvents { + return true + } + return false +} + +// DueByAge removes and returns every open batch older than MaxAge as of now. +func (b *Batcher) DueByAge(now time.Time) []*Batch { + if b.limits.MaxAge <= 0 { + return nil + } + var due []*Batch + for subj, batch := range b.open { + if now.Sub(batch.OpenedAt) >= b.limits.MaxAge { + due = append(due, batch) + delete(b.open, subj) + } + } + sortBatches(due) + return due +} + +// Drain removes and returns all open batches (used on graceful shutdown so +// in-flight events are persisted and acked before exit). +func (b *Batcher) Drain() []*Batch { + var all []*Batch + for subj, batch := range b.open { + all = append(all, batch) + delete(b.open, subj) + } + sortBatches(all) + return all +} + +// Pending reports how many events sit in open batches. +func (b *Batcher) Pending() int { + n := 0 + for _, batch := range b.open { + n += len(batch.Items) + } + return n +} + +func sortBatches(bs []*Batch) { + sort.Slice(bs, func(i, j int) bool { return bs[i].Subject < bs[j].Subject }) +} + +// NDJSON renders the batch as newline-delimited JSON (one raw event per line), +// matching the raw archive format the logging stack expects. +func (b *Batch) NDJSON() []byte { + var buf bytes.Buffer + buf.Grow(int(b.RawBytes) + len(b.Items)) + for _, it := range b.Items { + buf.Write(bytes.TrimRight(it.Raw, "\n")) + buf.WriteByte('\n') + } + return buf.Bytes() +} + +// Summary is the index-relevant projection of a batch. +type Summary struct { + Hosts []string + MinTS time.Time + MaxTS time.Time + EventCount int + HasTS bool +} + +// Summarize computes hosts (unique, sorted) and the timestamp range. fallback is +// used for events whose payload lacked a parseable timestamp (ingest time). +func (b *Batch) Summarize(fallback time.Time) Summary { + s := Summary{EventCount: len(b.Items)} + hostSet := map[string]struct{}{} + for _, it := range b.Items { + if it.Host != "" { + hostSet[it.Host] = struct{}{} + } + ts := it.Timestamp + if !it.HasTS { + ts = fallback + } else { + s.HasTS = true + } + if s.MinTS.IsZero() || ts.Before(s.MinTS) { + s.MinTS = ts + } + if s.MaxTS.IsZero() || ts.After(s.MaxTS) { + s.MaxTS = ts + } + } + if s.MinTS.IsZero() { + s.MinTS = fallback + } + if s.MaxTS.IsZero() { + s.MaxTS = fallback + } + for h := range hostSet { + s.Hosts = append(s.Hosts, h) + } + sort.Strings(s.Hosts) + return s +} diff --git a/internal/batcher/batcher_test.go b/internal/batcher/batcher_test.go new file mode 100644 index 0000000..b29ddca --- /dev/null +++ b/internal/batcher/batcher_test.go @@ -0,0 +1,148 @@ +package batcher + +import ( + "strings" + "testing" + "time" +) + +func item(subject, host string, ts time.Time, hasTS bool, raw string) Item { + return Item{Subject: subject, Host: host, Timestamp: ts, HasTS: hasTS, Raw: []byte(raw)} +} + +func TestFullByEvents(t *testing.T) { + b := New(Limits{MaxEvents: 3}) + if got := b.Add(item("s", "h", time.Now(), true, "a")); got != nil { + t.Fatalf("should not be full at 1") + } + if got := b.Add(item("s", "h", time.Now(), true, "b")); got != nil { + t.Fatalf("should not be full at 2") + } + full := b.Add(item("s", "h", time.Now(), true, "c")) + if full == nil { + t.Fatalf("should be full at 3") + } + if len(full.Items) != 3 { + t.Errorf("full batch has %d items", len(full.Items)) + } + // After a full flush the subject batch is reset. + if b.Pending() != 0 { + t.Errorf("pending after flush = %d, want 0", b.Pending()) + } +} + +func TestFullByBytes(t *testing.T) { + b := New(Limits{MaxBytes: 10}) + if b.Add(item("s", "h", time.Now(), true, "12345")) != nil { + t.Fatalf("5 bytes should not fill") + } + full := b.Add(item("s", "h", time.Now(), true, "67890")) + if full == nil { + t.Fatalf("10 bytes should fill") + } + if full.RawBytes != 10 { + t.Errorf("RawBytes = %d", full.RawBytes) + } +} + +func TestSeparateSubjects(t *testing.T) { + b := New(Limits{MaxEvents: 2}) + b.Add(item("a", "h", time.Now(), true, "x")) + b.Add(item("b", "h", time.Now(), true, "y")) + if b.Pending() != 2 { + t.Errorf("pending = %d, want 2 across subjects", b.Pending()) + } + full := b.Add(item("a", "h", time.Now(), true, "z")) + if full == nil || full.Subject != "a" { + t.Fatalf("subject a should flush independently") + } + if b.Pending() != 1 { + t.Errorf("pending after a flush = %d, want 1 (subject b)", b.Pending()) + } +} + +func TestDueByAge(t *testing.T) { + base := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC) + b := New(Limits{MaxAge: time.Minute}) + b.nowFn = func() time.Time { return base } + b.Add(item("s", "h", base, true, "x")) + + if due := b.DueByAge(base.Add(30 * time.Second)); len(due) != 0 { + t.Fatalf("not due at 30s") + } + due := b.DueByAge(base.Add(90 * time.Second)) + if len(due) != 1 { + t.Fatalf("should be due at 90s, got %d", len(due)) + } + if b.Pending() != 0 { + t.Errorf("due batch not removed") + } +} + +func TestDrain(t *testing.T) { + b := New(Limits{MaxEvents: 100}) + b.Add(item("a", "h", time.Now(), true, "x")) + b.Add(item("b", "h", time.Now(), true, "y")) + all := b.Drain() + if len(all) != 2 { + t.Fatalf("drain returned %d, want 2", len(all)) + } + if b.Pending() != 0 { + t.Errorf("pending after drain = %d", b.Pending()) + } +} + +func TestNDJSON(t *testing.T) { + b := &Batch{Subject: "s"} + b.Items = []Item{ + {Raw: []byte(`{"a":1}`)}, + {Raw: []byte(`{"b":2}` + "\n")}, // trailing newline trimmed and re-added + } + got := string(b.NDJSON()) + want := "{\"a\":1}\n{\"b\":2}\n" + if got != want { + t.Errorf("NDJSON = %q, want %q", got, want) + } + if strings.Count(got, "\n") != 2 { + t.Errorf("expected exactly 2 newlines") + } +} + +func TestSummarize(t *testing.T) { + t1 := time.Date(2026, 7, 27, 1, 0, 0, 0, time.UTC) + t2 := time.Date(2026, 7, 27, 3, 0, 0, 0, time.UTC) + fallback := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC) + b := &Batch{Subject: "s"} + b.Items = []Item{ + item("s", "host-b", t2, true, "x"), + item("s", "host-a", t1, true, "y"), + item("s", "", time.Time{}, false, "z"), // no ts -> fallback, no host + item("s", "host-a", t1, true, "w"), // dup host + } + s := b.Summarize(fallback) + if s.EventCount != 4 { + t.Errorf("EventCount = %d", s.EventCount) + } + if len(s.Hosts) != 2 || s.Hosts[0] != "host-a" || s.Hosts[1] != "host-b" { + t.Errorf("Hosts = %v, want sorted unique [host-a host-b]", s.Hosts) + } + if !s.MinTS.Equal(t1) { + t.Errorf("MinTS = %v, want %v", s.MinTS, t1) + } + // max should be the fallback (9:00) since event z used fallback which is latest + if !s.MaxTS.Equal(fallback) { + t.Errorf("MaxTS = %v, want fallback %v", s.MaxTS, fallback) + } +} + +func TestSummarizeAllFallback(t *testing.T) { + fallback := time.Date(2026, 7, 27, 9, 0, 0, 0, time.UTC) + b := &Batch{Items: []Item{{Raw: []byte("x")}}} + s := b.Summarize(fallback) + if !s.MinTS.Equal(fallback) || !s.MaxTS.Equal(fallback) { + t.Errorf("all-fallback range wrong: %v..%v", s.MinTS, s.MaxTS) + } + if s.HasTS { + t.Errorf("HasTS should be false") + } +} diff --git a/internal/cli/common.go b/internal/cli/common.go new file mode 100644 index 0000000..d839cee --- /dev/null +++ b/internal/cli/common.go @@ -0,0 +1,90 @@ +package cli + +import ( + "fmt" + "strings" + "time" + + "git.unkin.net/unkin/logarchiver/internal/index" + "github.com/spf13/cobra" +) + +// selectFlags are the shared object-selection flags for search and fetch. +type selectFlags struct { + subject string + host string + from string + to string + limit int +} + +func (s *selectFlags) bind(cmd *cobra.Command) { + f := cmd.Flags() + f.StringVar(&s.subject, "subject", "", "NATS-style subject glob (e.g. 'logs.vm.*' or 'logs.k8s.vault.>')") + f.StringVar(&s.host, "host", "", "source host to match (exact, or a glob with '*')") + f.StringVar(&s.from, "from", "", "start of time window (RFC3339, 'YYYY-MM-DD', or relative like '-24h')") + f.StringVar(&s.to, "to", "", "end of time window (RFC3339, 'YYYY-MM-DD', or relative like '-1h')") + f.IntVar(&s.limit, "limit", 100, "max objects to return (0 = no limit)") +} + +// query builds an index.SearchQuery from the flags. +func (s *selectFlags) query(now time.Time) (index.SearchQuery, error) { + q := index.SearchQuery{Subject: s.subject, Host: s.host, Limit: s.limit} + if s.from != "" { + t, err := parseTimeArg(s.from, now) + if err != nil { + return q, fmt.Errorf("--from: %w", err) + } + q.From = t + } + if s.to != "" { + t, err := parseTimeArg(s.to, now) + if err != nil { + return q, fmt.Errorf("--to: %w", err) + } + q.To = t + } + if !q.From.IsZero() && !q.To.IsZero() && q.To.Before(q.From) { + return q, fmt.Errorf("--to (%s) is before --from (%s)", q.To, q.From) + } + return q, nil +} + +// parseTimeArg accepts RFC3339[/Nano], "YYYY-MM-DD", "YYYY-MM-DDTHH:MM:SS", or a +// signed Go duration relative to now (e.g. "-24h", "30m"). +func parseTimeArg(s string, now time.Time) (time.Time, error) { + s = strings.TrimSpace(s) + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05", "2006-01-02"} { + if t, err := time.Parse(layout, s); err == nil { + return t.UTC(), nil + } + } + if d, err := time.ParseDuration(s); err == nil { + return now.Add(d).UTC(), nil + } + return time.Time{}, fmt.Errorf("unrecognized time %q (use RFC3339, YYYY-MM-DD, or a duration like -24h)", s) +} + +func newIndexClient(cmd *cobra.Command) (index.Index, error) { + cfg, err := loadConfig() + if err != nil { + return nil, err + } + if !cfg.Index.Enabled { + return nil, fmt.Errorf("index is disabled in config; search/fetch-by-query require the ClickHouse index") + } + return index.NewClickHouse(cmd.Context(), indexConfig(cfg.Index)) +} + +func humanBytes(n uint64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%dB", n) + } + div, exp := int64(unit), 0 + for x := n / unit; x >= unit; x /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f%ciB", float64(n)/float64(div), "KMGTPE"[exp]) +} diff --git a/internal/cli/fetch.go b/internal/cli/fetch.go new file mode 100644 index 0000000..2ff192c --- /dev/null +++ b/internal/cli/fetch.go @@ -0,0 +1,185 @@ +package cli + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "git.unkin.net/unkin/logarchiver/internal/crypto" + "git.unkin.net/unkin/logarchiver/internal/s3store" + "git.unkin.net/unkin/logarchiver/internal/vaultgpg" + "github.com/spf13/cobra" +) + +func newFetchCmd() *cobra.Command { + var sel selectFlags + var output string + cmd := &cobra.Command{ + Use: "fetch [object-key ...]", + Short: "Download, decrypt and decompress archived objects to NDJSON", + Long: `fetch retrieves archived objects, decrypts them via the Vault GPG engine +(the engine decrypts only the tiny wrapped data key; the bulk is streamed and +decrypted locally), decompresses the zstd bulk, and emits the original NDJSON. + +Objects are selected either by object key arguments, or by the same +--subject/--host/--from/--to query used by 'search'. When --host/--from/--to are +given they ALSO re-filter the emitted events to just the matching lines.`, + Example: ` logarchiver search --subject 'logs.k8s.vault.>' --from -1h + logarchiver fetch archive/logs.k8s.vault._/2026/07/27/20260727T101500Z-ab12cd34.ndjson.zst.larc -o - + logarchiver fetch --subject 'logs.vm.*' --host db-1 --from -24h -o ./out`, + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + now := time.Now() + q, err := sel.query(now) + if err != nil { + return err + } + + // Resolve object keys: explicit args, else via index search. + keys := args + if len(keys) == 0 { + if !cfg.Index.Enabled { + return fmt.Errorf("no object keys given and index is disabled") + } + idx, err := newIndexClient(cmd) + if err != nil { + return err + } + defer func() { _ = idx.Close() }() + results, err := idx.Search(cmd.Context(), q) + if err != nil { + return err + } + for _, r := range results { + keys = append(keys, r.ObjectKey) + } + } + if len(keys) == 0 { + _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "no matching objects") + return nil + } + + store, err := s3store.New(cmd.Context(), s3store.Config{ + Endpoint: cfg.S3.Endpoint, + Bucket: cfg.S3.Bucket, + Region: cfg.S3.Region, + PathStyle: cfg.S3.PathStyle, + CAFile: cfg.S3.CAFile, + }) + if err != nil { + return fmt.Errorf("s3 init: %w", err) + } + // Operator decrypt path: force token auth (ambient VAULT_TOKEN / + // ~/.vault-token), like passv, regardless of the service auth_method. + vcfg := vaultConfig(cfg.Crypto.Vault) + vcfg.AuthMethod = "token" + vc, err := vaultgpg.New(cmd.Context(), vcfg) + if err != nil { + return fmt.Errorf("vault init: %w", err) + } + + filter := newLineFilter(sel.host, q.From, q.To) + + var failures int + for _, key := range keys { + if err := fetchOne(cmd.Context(), store, vc, key, output, filter); err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "fetch %s: %v\n", key, err) + failures++ + } + } + if failures > 0 { + return fmt.Errorf("%d of %d objects failed", failures, len(keys)) + } + return nil + }, + } + sel.bind(cmd) + cmd.Flags().StringVarP(&output, "output", "o", "-", + "output: '-' for stdout, or a directory to write one NDJSON file per object") + return cmd +} + +// fetchOne downloads, decrypts and decompresses a single object, applying the +// optional line filter, to stdout or a per-object file under a directory. +func fetchOne(ctx context.Context, store s3store.ObjectStore, vc *vaultgpg.Client, key, output string, filter lineFilter) error { + body, err := store.Get(ctx, key) + if err != nil { + return err + } + defer func() { _ = body.Close() }() + + // Buffer the (bounded) object so we can read the header for its key name + // before decrypting. + data, err := io.ReadAll(body) + if err != nil { + return fmt.Errorf("read object: %w", err) + } + hdr, _, err := crypto.ReadHeader(bytes.NewReader(data)) + if err != nil { + return err + } + keyName := hdr.KeyName + if keyName == "" { + return fmt.Errorf("object header has no key_name") + } + unwrap := func(wrapped []byte) ([]byte, error) { + return vc.Decrypt(ctx, keyName, wrapped) + } + + var dst io.Writer + var closer io.Closer + if output == "-" || output == "" { + dst = os.Stdout + } else { + if err := os.MkdirAll(output, 0o755); err != nil { + return fmt.Errorf("create output dir: %w", err) + } + outPath := filepath.Join(output, sanitizeKey(key)) + if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil { + return fmt.Errorf("create output subdir: %w", err) + } + f, err := os.Create(outPath) + if err != nil { + return fmt.Errorf("create output file: %w", err) + } + dst = f + closer = f + } + + fw := newFilterWriter(dst, filter) + if err := crypto.Open(bytes.NewReader(data), fw, unwrap); err != nil { + if closer != nil { + _ = closer.Close() + } + return err + } + if err := fw.Flush(); err != nil { + if closer != nil { + _ = closer.Close() + } + return err + } + if closer != nil { + return closer.Close() + } + return nil +} + +// sanitizeKey turns an object key into a safe relative output filename, dropping +// the .larc container suffix in favor of a plain .ndjson. +func sanitizeKey(key string) string { + name := strings.TrimSuffix(key, ".larc") + if !strings.HasSuffix(name, ".ndjson") && !strings.HasSuffix(name, ".ndjson.zst") { + name += ".ndjson" + } + name = strings.TrimSuffix(name, ".zst") + return filepath.Clean("/" + name)[1:] +} diff --git a/internal/cli/filter.go b/internal/cli/filter.go new file mode 100644 index 0000000..e1dccba --- /dev/null +++ b/internal/cli/filter.go @@ -0,0 +1,123 @@ +package cli + +import ( + "bytes" + "io" + "strings" + "time" + + "git.unkin.net/unkin/logarchiver/internal/event" +) + +// lineFilter is a predicate over a single NDJSON event line. +type lineFilter func(raw []byte) bool + +// newLineFilter builds a predicate from optional host/time constraints. A nil +// filter (all constraints empty) means "pass everything". +func newLineFilter(host string, from, to time.Time) lineFilter { + if host == "" && from.IsZero() && to.IsZero() { + return nil + } + return func(raw []byte) bool { + meta := event.Extract(raw) + if host != "" && !globMatch(host, meta.Host) { + return false + } + if !from.IsZero() || !to.IsZero() { + // Events without a parseable timestamp are kept (we cannot exclude + // them on time grounds without dropping data). + if meta.Ok { + if !from.IsZero() && meta.Timestamp.Before(from) { + return false + } + if !to.IsZero() && meta.Timestamp.After(to) { + return false + } + } + } + return true + } +} + +// filterWriter forwards only complete NDJSON lines that satisfy filter. It +// buffers a trailing partial line across Writes so streaming decryption can feed +// it arbitrary chunks. Flush must be called at end to emit any final unterminated +// line. A nil filter forwards bytes verbatim. +type filterWriter struct { + dst io.Writer + filter lineFilter + buf bytes.Buffer +} + +func newFilterWriter(dst io.Writer, filter lineFilter) *filterWriter { + return &filterWriter{dst: dst, filter: filter} +} + +func (w *filterWriter) Write(p []byte) (int, error) { + if w.filter == nil { + return w.dst.Write(p) + } + w.buf.Write(p) + for { + data := w.buf.Bytes() + i := bytes.IndexByte(data, '\n') + if i < 0 { + break + } + line := data[:i] + if len(bytes.TrimSpace(line)) > 0 && w.filter(line) { + if _, err := w.dst.Write(line); err != nil { + return 0, err + } + if _, err := w.dst.Write([]byte{'\n'}); err != nil { + return 0, err + } + } + w.buf.Next(i + 1) + } + return len(p), nil +} + +// Flush emits a trailing line that had no terminating newline. +func (w *filterWriter) Flush() error { + if w.filter == nil { + return nil + } + line := bytes.TrimRight(w.buf.Bytes(), "\n") + w.buf.Reset() + if len(bytes.TrimSpace(line)) > 0 && w.filter(line) { + if _, err := w.dst.Write(line); err != nil { + return err + } + if _, err := w.dst.Write([]byte{'\n'}); err != nil { + return err + } + } + return nil +} + +// globMatch matches pattern against s where '*' matches any run of characters. +// With no '*', it is an exact match. +func globMatch(pattern, s string) bool { + if !strings.Contains(pattern, "*") { + return pattern == s + } + parts := strings.Split(pattern, "*") + // Anchor first part. + if !strings.HasPrefix(s, parts[0]) { + return false + } + s = s[len(parts[0]):] + for _, part := range parts[1 : len(parts)-1] { + if part == "" { + continue + } + idx := strings.Index(s, part) + if idx < 0 { + return false + } + s = s[idx+len(part):] + } + // Anchor last part. + return strings.HasSuffix(s, parts[len(parts)-1]) +} diff --git a/internal/cli/filter_test.go b/internal/cli/filter_test.go new file mode 100644 index 0000000..d36a477 --- /dev/null +++ b/internal/cli/filter_test.go @@ -0,0 +1,116 @@ +package cli + +import ( + "bytes" + "testing" + "time" +) + +func TestGlobMatch(t *testing.T) { + cases := []struct { + pattern, s string + want bool + }{ + {"node-1", "node-1", true}, + {"node-1", "node-2", false}, + {"db-*", "db-1", true}, + {"db-*", "web-1", false}, + {"*-1", "node-1", true}, + {"*vault*", "logs-vault-audit", true}, + {"*vault*", "logs-web", false}, + {"a*b*c", "axxbyyc", true}, + {"a*b*c", "axxc", false}, + } + for _, c := range cases { + if got := globMatch(c.pattern, c.s); got != c.want { + t.Errorf("globMatch(%q,%q) = %v, want %v", c.pattern, c.s, got, c.want) + } + } +} + +func TestFilterWriterPassAll(t *testing.T) { + var out bytes.Buffer + fw := newFilterWriter(&out, nil) // nil filter = passthrough + _, _ = fw.Write([]byte("line1\nline2\n")) + _ = fw.Flush() + if out.String() != "line1\nline2\n" { + t.Errorf("passthrough altered data: %q", out.String()) + } +} + +func TestFilterWriterHostFilterAcrossChunks(t *testing.T) { + var out bytes.Buffer + filter := newLineFilter("node-1", time.Time{}, time.Time{}) + fw := newFilterWriter(&out, filter) + // Feed a line split across two Writes to exercise buffering. + _, _ = fw.Write([]byte(`{"host":"node-1","m":"keep"}` + "\n" + `{"host":"node`)) + _, _ = fw.Write([]byte(`-2","m":"drop"}` + "\n" + `{"host":"node-1","m":"keep2"}` + "\n")) + _ = fw.Flush() + + got := out.String() + if want := `{"host":"node-1","m":"keep"}` + "\n" + `{"host":"node-1","m":"keep2"}` + "\n"; got != want { + t.Errorf("filtered output = %q, want %q", got, want) + } +} + +func TestFilterWriterTimeWindow(t *testing.T) { + from := time.Date(2026, 7, 27, 1, 0, 0, 0, time.UTC) + to := time.Date(2026, 7, 27, 2, 0, 0, 0, time.UTC) + var out bytes.Buffer + fw := newFilterWriter(&out, newLineFilter("", from, to)) + lines := `{"host":"h","timestamp":"2026-07-27T00:30:00Z","m":"before"}` + "\n" + + `{"host":"h","timestamp":"2026-07-27T01:30:00Z","m":"in"}` + "\n" + + `{"host":"h","timestamp":"2026-07-27T03:00:00Z","m":"after"}` + "\n" + + `{"host":"h","m":"no-ts-kept"}` + "\n" + _, _ = fw.Write([]byte(lines)) + _ = fw.Flush() + + got := out.String() + if !bytes.Contains(out.Bytes(), []byte(`"in"`)) { + t.Errorf("in-window line dropped: %q", got) + } + if bytes.Contains(out.Bytes(), []byte(`"before"`)) || bytes.Contains(out.Bytes(), []byte(`"after"`)) { + t.Errorf("out-of-window line kept: %q", got) + } + if !bytes.Contains(out.Bytes(), []byte(`"no-ts-kept"`)) { + t.Errorf("event without timestamp should be kept: %q", got) + } +} + +func TestParseTimeArg(t *testing.T) { + now := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC) + rfc, err := parseTimeArg("2026-07-27T01:00:00Z", now) + if err != nil || !rfc.Equal(time.Date(2026, 7, 27, 1, 0, 0, 0, time.UTC)) { + t.Errorf("RFC3339 parse: %v %v", rfc, err) + } + d, err := parseTimeArg("2026-07-27", now) + if err != nil || !d.Equal(time.Date(2026, 7, 27, 0, 0, 0, 0, time.UTC)) { + t.Errorf("date parse: %v %v", d, err) + } + rel, err := parseTimeArg("-24h", now) + if err != nil || !rel.Equal(now.Add(-24*time.Hour)) { + t.Errorf("relative parse: %v %v", rel, err) + } + if _, err := parseTimeArg("nonsense", now); err == nil { + t.Errorf("expected error for nonsense time") + } +} + +func TestSelectFlagsQueryOrdering(t *testing.T) { + now := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC) + s := &selectFlags{from: "-1h", to: "-2h"} // to before from + if _, err := s.query(now); err == nil { + t.Errorf("expected error when --to before --from") + } +} + +func TestSanitizeKey(t *testing.T) { + got := sanitizeKey("archive/logs.vm.db-1/2026/07/27/20260727T101500Z-abcd.ndjson.zst.larc") + if got != "archive/logs.vm.db-1/2026/07/27/20260727T101500Z-abcd.ndjson" { + t.Errorf("sanitizeKey = %q", got) + } + // Path traversal is neutralized. + if bad := sanitizeKey("../../etc/passwd"); bad != "etc/passwd.ndjson" { + t.Errorf("sanitizeKey traversal = %q", bad) + } +} diff --git a/internal/cli/initschema.go b/internal/cli/initschema.go new file mode 100644 index 0000000..df4d40b --- /dev/null +++ b/internal/cli/initschema.go @@ -0,0 +1,45 @@ +package cli + +import ( + "fmt" + + "git.unkin.net/unkin/logarchiver/internal/index" + "github.com/spf13/cobra" +) + +func newInitSchemaCmd() *cobra.Command { + var printOnly bool + cmd := &cobra.Command{ + Use: "init-schema", + Short: "Create the ClickHouse archive-index database and table (idempotent)", + Long: `init-schema creates the ClickHouse database and archive_index table. + +In-cluster the argocd bootstrap Job owns schema creation (like the logging +stack's clickhouse-schema PostSync hook); this command is for local/dev use and +for emitting the DDL (--print) to embed in that Job.`, + RunE: func(cmd *cobra.Command, _ []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + if printOnly { + out := cmd.OutOrStdout() + _, _ = fmt.Fprintln(out, index.CreateDatabaseSQL(cfg.Index.Database)+";") + _, _ = fmt.Fprintln(out, index.CreateTableSQL(cfg.Index.Database, cfg.Index.Table)+";") + return nil + } + ch, err := index.NewClickHouse(cmd.Context(), indexConfig(cfg.Index)) + if err != nil { + return err + } + defer func() { _ = ch.Close() }() + if err := ch.InitSchema(cmd.Context()); err != nil { + return err + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "schema ready: %s.%s\n", cfg.Index.Database, cfg.Index.Table) + return nil + }, + } + cmd.Flags().BoolVar(&printOnly, "print", false, "print the DDL instead of executing it") + return cmd +} diff --git a/internal/cli/root.go b/internal/cli/root.go new file mode 100644 index 0000000..2e8a1bc --- /dev/null +++ b/internal/cli/root.go @@ -0,0 +1,95 @@ +// Package cli implements the logarchiver command tree (service + operator CLI) +// using cobra, which also provides the `completion` subcommand the estate's +// nfpm packaging installs. +package cli + +import ( + "fmt" + "log/slog" + "os" + "strings" + + "git.unkin.net/unkin/logarchiver/internal/config" + "github.com/spf13/cobra" +) + +// version is set at build time via -ldflags "-X ...cli.version=...". +var version = "dev" + +// SetVersion lets main inject the linker-provided version string. +func SetVersion(v string) { + if v != "" { + version = v + } +} + +var configPath string + +// NewRootCmd builds the root command. +func NewRootCmd() *cobra.Command { + root := &cobra.Command{ + Use: "logarchiver", + Short: "Archive NATS JetStream logs to S3 (zstd + OpenPGP) and search/retrieve them", + Long: `logarchiver archives raw logs from the centralized logging JetStream stream +to S3 as zstd-compressed, OpenPGP-encrypted, indexed objects, and provides a +CLI to search the index and retrieve/decrypt archived logs.`, + SilenceUsage: true, + SilenceErrors: true, + } + root.PersistentFlags().StringVarP(&configPath, "config", "c", os.Getenv("LOGARCHIVER_CONFIG"), + "path to config file (env LOGARCHIVER_CONFIG)") + + root.AddCommand( + newRunCmd(), + newSearchCmd(), + newFetchCmd(), + newInitSchemaCmd(), + newVersionCmd(), + ) + return root +} + +// Execute runs the root command. +func Execute() int { + if err := NewRootCmd().Execute(); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + return 1 + } + return 0 +} + +func newVersionCmd() *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Print the version", + Run: func(cmd *cobra.Command, _ []string) { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), version) + }, + } +} + +// loadConfig loads config from the --config path (or defaults+env). +func loadConfig() (config.Config, error) { + return config.Load(configPath) +} + +// newLogger builds a slog logger from config. +func newLogger(cfg config.LogConfig) *slog.Logger { + level := slog.LevelInfo + switch strings.ToLower(cfg.Level) { + case "debug": + level = slog.LevelDebug + case "warn": + level = slog.LevelWarn + case "error": + level = slog.LevelError + } + opts := &slog.HandlerOptions{Level: level} + var h slog.Handler + if strings.ToLower(cfg.Format) == "text" { + h = slog.NewTextHandler(os.Stderr, opts) + } else { + h = slog.NewJSONHandler(os.Stderr, opts) + } + return slog.New(h) +} diff --git a/internal/cli/run.go b/internal/cli/run.go new file mode 100644 index 0000000..e0fe504 --- /dev/null +++ b/internal/cli/run.go @@ -0,0 +1,252 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "os/signal" + "syscall" + "time" + + "git.unkin.net/unkin/logarchiver/internal/archiver" + "git.unkin.net/unkin/logarchiver/internal/batcher" + "git.unkin.net/unkin/logarchiver/internal/config" + "git.unkin.net/unkin/logarchiver/internal/consumer" + "git.unkin.net/unkin/logarchiver/internal/index" + "git.unkin.net/unkin/logarchiver/internal/metrics" + "git.unkin.net/unkin/logarchiver/internal/s3store" + "git.unkin.net/unkin/logarchiver/internal/vaultgpg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/spf13/cobra" +) + +func newRunCmd() *cobra.Command { + return &cobra.Command{ + Use: "run", + Short: "Run the archiver service (JetStream consumer -> S3 + index)", + RunE: func(cmd *cobra.Command, _ []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + return runService(cmd.Context(), cfg) + }, + } +} + +// persistAdapter bridges *archiver.Archiver to consumer.Persister (different +// StoreResult types across package boundaries). +type persistAdapter struct{ a *archiver.Archiver } + +func (p persistAdapter) Store(ctx context.Context, b *batcher.Batch) (consumer.StoreResult, error) { + res, err := p.a.Store(ctx, b) + return consumer.StoreResult{ + ObjectKey: res.ObjectKey, + Events: res.Events, + RawBytes: res.RawBytes, + StoredBytes: res.StoredBytes, + }, err +} + +func runService(parent context.Context, cfg config.Config) error { + log := newLogger(cfg.Log) + slog.SetDefault(log) + + ctx, stop := signal.NotifyContext(parent, syscall.SIGINT, syscall.SIGTERM) + defer stop() + + // Metrics. + var met *metrics.Metrics + var metricsSrv *http.Server + if cfg.Metrics.Enabled { + reg := prometheus.NewRegistry() + met = metrics.New(reg) + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + metricsSrv = &http.Server{Addr: cfg.Metrics.Address, Handler: mux, ReadHeaderTimeout: 5 * time.Second} + go func() { + log.Info("metrics listening", "addr", cfg.Metrics.Address) + if err := metricsSrv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Error("metrics server failed", "err", err) + } + }() + } + + // Vault client only when we source the public key from Vault; the service + // never needs Vault otherwise (file-mounted pubkey is the default). + var vc *vaultgpg.Client + if cfg.Crypto.Source == config.PubkeyVault { + var err error + vc, err = vaultgpg.New(ctx, vaultConfig(cfg.Crypto.Vault)) + if err != nil { + return fmt.Errorf("vault init: %w", err) + } + } + + // Public key provider. + loader, err := archiver.PubkeyLoaderFromConfig(cfg.Crypto, vc) + if err != nil { + return err + } + pubkeys, err := archiver.NewPubkeyProvider(ctx, loader) + if err != nil { + return fmt.Errorf("load public key: %w", err) + } + log.Info("public key loaded", + "source", cfg.Crypto.Source, "key_name", cfg.Crypto.KeyName, + "fingerprint", pubkeys.Current().Fingerprint) + go refreshPubkey(ctx, log, pubkeys, cfg.Crypto.RefreshInterval) + + // S3. + store, err := s3store.New(ctx, s3store.Config{ + Endpoint: cfg.S3.Endpoint, + Bucket: cfg.S3.Bucket, + Region: cfg.S3.Region, + PathStyle: cfg.S3.PathStyle, + CAFile: cfg.S3.CAFile, + }) + if err != nil { + return fmt.Errorf("s3 init: %w", err) + } + + // Index. + var idx index.Index + if cfg.Index.Enabled { + ch, err := index.NewClickHouse(ctx, indexConfig(cfg.Index)) + if err != nil { + return fmt.Errorf("clickhouse init: %w", err) + } + defer func() { _ = ch.Close() }() + idx = ch + } + + keys, err := archiver.NewKeyBuilder(cfg.S3.KeyPrefix) + if err != nil { + return err + } + + arch, err := archiver.New(archiver.Options{ + Keys: keys, + Pubkeys: pubkeys, + Store: store, + Index: idx, + KeyName: cfg.Crypto.KeyName, + FrameSize: cfg.Crypto.FrameSize, + Metrics: met, + }) + if err != nil { + return err + } + + // NATS + consumer. + nc, js, err := consumer.Connect(cfg.NATS) + if err != nil { + return err + } + defer nc.Close() + cons, err := consumer.EnsureConsumer(ctx, js, cfg.NATS) + if err != nil { + return err + } + log.Info("consumer bound", + "stream", cfg.NATS.Stream, "durable", cfg.NATS.Durable, "subjects", cfg.NATS.Subjects) + + bat := batcher.New(batcher.Limits{ + MaxBytes: cfg.Batch.MaxBytes, + MaxEvents: cfg.Batch.MaxEvents, + MaxAge: cfg.Batch.MaxAge, + }) + + var runnerMetrics consumer.Metrics + if met != nil { + runnerMetrics = met + } + runner := consumer.NewRunner(consumer.Options{ + Consumer: cons, + Batcher: bat, + Persister: persistAdapter{a: arch}, + Logger: log, + Metrics: runnerMetrics, + FetchBatch: cfg.NATS.FetchBatch, + PollWait: pollWait(cfg.Batch.MaxAge), + }) + + runErr := runner.Run(ctx) + + if metricsSrv != nil { + shCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _ = metricsSrv.Shutdown(shCtx) + cancel() + } + if runErr != nil && !errors.Is(runErr, context.Canceled) { + return runErr + } + log.Info("shutdown complete") + return nil +} + +// pollWait picks a fetch/age-check interval that is a fraction of MaxAge so +// aged batches flush promptly, clamped to a sane range. +func pollWait(maxAge time.Duration) time.Duration { + if maxAge <= 0 { + return time.Second + } + w := maxAge / 10 + if w < time.Second { + w = time.Second + } + if w > 10*time.Second { + w = 10 * time.Second + } + return w +} + +func refreshPubkey(ctx context.Context, log *slog.Logger, p *archiver.PubkeyProvider, every time.Duration) { + if every <= 0 { + return + } + t := time.NewTicker(every) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if err := p.Refresh(ctx); err != nil { + log.Warn("pubkey refresh failed; keeping previous key", "err", err) + continue + } + log.Debug("pubkey refreshed", "fingerprint", p.Current().Fingerprint) + } + } +} + +func vaultConfig(v config.VaultConfig) vaultgpg.Config { + return vaultgpg.Config{ + Address: v.Address, + Mount: v.Mount, + AuthMethod: v.AuthMethod, + K8sRole: v.K8sRole, + K8sMount: v.K8sMount, + K8sJWTPath: v.K8sJWTPath, + CAFile: v.CAFile, + } +} + +func indexConfig(i config.IndexConfig) index.Config { + return index.Config{ + Address: i.Address, + Database: i.Database, + Table: i.Table, + Username: i.Username, + Password: i.Password, + TLS: i.TLS, + } +} diff --git a/internal/cli/search.go b/internal/cli/search.go new file mode 100644 index 0000000..79ff393 --- /dev/null +++ b/internal/cli/search.go @@ -0,0 +1,67 @@ +package cli + +import ( + "encoding/json" + "fmt" + "strings" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" +) + +func newSearchCmd() *cobra.Command { + var sel selectFlags + var asJSON bool + cmd := &cobra.Command{ + Use: "search", + Short: "Search the archive index for matching objects", + Long: `search queries the ClickHouse archive index and lists the S3 objects whose +subject/host/time-range match, with event counts and sizes. Use the object keys +with 'logarchiver fetch' to retrieve and decrypt their contents.`, + Example: ` logarchiver search --subject 'logs.k8s.vault.>' --host node-1 --from -24h`, + RunE: func(cmd *cobra.Command, _ []string) error { + q, err := sel.query(time.Now()) + if err != nil { + return err + } + idx, err := newIndexClient(cmd) + if err != nil { + return err + } + defer func() { _ = idx.Close() }() + + results, err := idx.Search(cmd.Context(), q) + if err != nil { + return err + } + out := cmd.OutOrStdout() + if asJSON { + enc := json.NewEncoder(out) + enc.SetIndent("", " ") + return enc.Encode(results) + } + if len(results) == 0 { + _, _ = fmt.Fprintln(out, "no matching objects") + return nil + } + tw := tabwriter.NewWriter(out, 0, 2, 2, ' ', 0) + _, _ = fmt.Fprintln(tw, "OBJECT_KEY\tSUBJECT\tHOSTS\tMIN_TS\tMAX_TS\tEVENTS\tSTORED") + var totalEvents, totalStored uint64 + for _, r := range results { + _, _ = fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%d\t%s\n", + r.ObjectKey, r.Subject, strings.Join(r.Hosts, ","), + r.MinTS.UTC().Format(time.RFC3339), r.MaxTS.UTC().Format(time.RFC3339), + r.EventCount, humanBytes(r.StoredBytes)) + totalEvents += r.EventCount + totalStored += r.StoredBytes + } + _ = tw.Flush() + _, _ = fmt.Fprintf(out, "\n%d objects, %d events, %s stored\n", len(results), totalEvents, humanBytes(totalStored)) + return nil + }, + } + sel.bind(cmd) + cmd.Flags().BoolVar(&asJSON, "json", false, "output results as JSON") + return cmd +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..c98329b --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,379 @@ +// Package config defines logarchiver's configuration and loads it from a YAML +// file with environment-variable overrides, so the same binary is +// k8s-friendly (env/secret-driven) and laptop-friendly (a config file). +// +// Precedence: built-in defaults < YAML file < environment variables. +package config + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +// Config is the full logarchiver configuration. +type Config struct { + NATS NATSConfig `yaml:"nats"` + Batch BatchConfig `yaml:"batch"` + S3 S3Config `yaml:"s3"` + Crypto CryptoConfig `yaml:"crypto"` + Index IndexConfig `yaml:"index"` + Metrics MetricsConfig `yaml:"metrics"` + Log LogConfig `yaml:"log"` +} + +// NATSConfig configures the JetStream pull consumer that logarchiver binds. It +// mirrors the logging stack's `LOGS` stream / `archiver` durable / `log-consumer` +// user conventions (argocd-apps #296). +type NATSConfig struct { + URL string `yaml:"url"` + Stream string `yaml:"stream"` + Durable string `yaml:"durable"` + Subjects []string `yaml:"subjects"` + User string `yaml:"user"` + // Password is the NATS user password. In-cluster it comes from the + // nats-auth secret via NATS_CONSUMER_PASSWORD (see PasswordEnv). + Password string `yaml:"password"` + // PasswordEnv names the env var holding the password when Password is empty. + PasswordEnv string `yaml:"password_env"` + // CAFile trusts a custom CA for TLS to NATS (usually unset; in-cluster is plaintext). + CAFile string `yaml:"ca_file"` + // FetchBatch is the max messages pulled per Fetch call. + FetchBatch int `yaml:"fetch_batch"` + // AckWait is the JetStream redelivery timeout; must exceed a worst-case + // batch flush (compress+encrypt+S3 PUT+index write). + AckWait time.Duration `yaml:"ack_wait"` +} + +// BatchConfig bounds a single archived object. A per-subject batch is flushed +// when any bound is hit. Keep MaxBytes well under the crypto/engine ceiling so +// even a whole-object decrypt path stays viable; the wrapped-DEK envelope means +// object size is not limited by Vault, but smaller objects retrieve faster. +type BatchConfig struct { + MaxBytes int64 `yaml:"max_bytes"` + MaxEvents int `yaml:"max_events"` + MaxAge time.Duration `yaml:"max_age"` +} + +// S3Config targets the Ceph RGW bucket. Credentials are read from the standard +// AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars (cephrgw BucketAccess +// secret logs-archive-s3), so they are intentionally absent here. +type S3Config struct { + Endpoint string `yaml:"endpoint"` + Bucket string `yaml:"bucket"` + Region string `yaml:"region"` + PathStyle bool `yaml:"path_style"` + // KeyPrefix is a text/template with {{.Subject}} {{.Year}} {{.Month}} {{.Day}}. + KeyPrefix string `yaml:"key_prefix"` + // CAFile trusts the internal Vault-PKI CA for https://s3.ceph.unkin.net. + CAFile string `yaml:"ca_file"` + // EndpointEnv / BucketEnv let the cephrgw secret (S3_ENDPOINT / BUCKET_NAME) + // override endpoint/bucket without a config edit. + EndpointEnv string `yaml:"endpoint_env"` + BucketEnv string `yaml:"bucket_env"` +} + +// PubkeySource selects where the OpenPGP public key is fetched from. +type PubkeySource string + +const ( + PubkeyVault PubkeySource = "vault" // read gpg/keys/ from the Vault GPG engine + PubkeyFile PubkeySource = "file" // read an armored public key from a mounted file +) + +// CryptoConfig controls encryption. The service only ever needs the PUBLIC key; +// decryption (CLI fetch) always goes through the Vault GPG engine. +type CryptoConfig struct { + KeyName string `yaml:"key_name"` + Source PubkeySource `yaml:"pubkey_source"` + // PubkeyFile is the armored public key path when Source==file. + PubkeyFile string `yaml:"pubkey_file"` + // RefreshInterval re-fetches the public key periodically (rotation aware). + RefreshInterval time.Duration `yaml:"refresh_interval"` + // FrameSize is the AES-GCM frame plaintext size in bytes (streaming decrypt). + FrameSize int `yaml:"frame_size"` + Vault VaultConfig `yaml:"vault"` +} + +// VaultConfig configures access to the Vault GPG secrets engine. For the +// service (pubkey fetch) k8s auth is used in-cluster; the CLI relies on the +// operator's ambient VAULT_TOKEN (~/.vault-token), like passv. +type VaultConfig struct { + Address string `yaml:"address"` + // Mount is the GPG engine mount path (e.g. "gpg"). + Mount string `yaml:"mount"` + // AuthMethod is "token" or "kubernetes". + AuthMethod string `yaml:"auth_method"` + // K8sRole / K8sMount / K8sJWTPath configure kubernetes auth. + K8sRole string `yaml:"k8s_role"` + K8sMount string `yaml:"k8s_mount"` + K8sJWTPath string `yaml:"k8s_jwt_path"` + CAFile string `yaml:"ca_file"` +} + +// IndexConfig targets the ClickHouse archive index. Credentials come from the +// clickhouse-credentials secret via env by default. +type IndexConfig struct { + Enabled bool `yaml:"enabled"` + Address string `yaml:"address"` // host:port for the native protocol (9000) + Database string `yaml:"database"` + Table string `yaml:"table"` + Username string `yaml:"username"` + Password string `yaml:"password"` + PasswordEnv string `yaml:"password_env"` + TLS bool `yaml:"tls"` +} + +// MetricsConfig configures the Prometheus /metrics listener. +type MetricsConfig struct { + Enabled bool `yaml:"enabled"` + Address string `yaml:"address"` +} + +// LogConfig configures structured logging. +type LogConfig struct { + Level string `yaml:"level"` // debug|info|warn|error + Format string `yaml:"format"` // json|text +} + +// Default returns a Config pre-populated with the logging-stack conventions so +// an in-cluster deployment needs only secrets (creds) supplied via env. +func Default() Config { + return Config{ + NATS: NATSConfig{ + URL: "nats://nats.logging.svc.cluster.local:4222", + Stream: "LOGS", + Durable: "archiver", + Subjects: []string{"logs.k8s.vault.>"}, + User: "log-consumer", + PasswordEnv: "NATS_CONSUMER_PASSWORD", + FetchBatch: 512, + AckWait: 2 * time.Minute, + }, + Batch: BatchConfig{ + MaxBytes: 64 * 1024 * 1024, // 64 MiB raw NDJSON per object + MaxEvents: 200000, + MaxAge: 5 * time.Minute, + }, + S3: S3Config{ + Endpoint: "https://s3.ceph.unkin.net", + Bucket: "logs-archive", + Region: "us-east-1", + PathStyle: true, + KeyPrefix: "archive/{{.Subject}}/{{.Year}}/{{.Month}}/{{.Day}}/", + CAFile: "/etc/vault-ca/ca.crt", + EndpointEnv: "S3_ENDPOINT", + BucketEnv: "BUCKET_NAME", + }, + Crypto: CryptoConfig{ + KeyName: "logarchive", + Source: PubkeyFile, + PubkeyFile: "/etc/logarchiver/pubkey.asc", + RefreshInterval: time.Hour, + FrameSize: 1 << 20, // 1 MiB frames + Vault: VaultConfig{ + Mount: "gpg", + AuthMethod: "kubernetes", + K8sMount: "k8s/au/syd1", + K8sRole: "default", + K8sJWTPath: "/var/run/secrets/kubernetes.io/serviceaccount/token", + }, + }, + Index: IndexConfig{ + Enabled: true, + Address: "clickhouse-logs.logging.svc.cluster.local:9000", + Database: "logs", + Table: "archive_index", + Username: "vector", + PasswordEnv: "CLICKHOUSE_PASSWORD", + TLS: false, + }, + Metrics: MetricsConfig{Enabled: true, Address: ":9090"}, + Log: LogConfig{Level: "info", Format: "json"}, + } +} + +// Load reads defaults, overlays the YAML file at path (if non-empty), then +// applies environment overrides, and validates the result. +func Load(path string) (Config, error) { + cfg := Default() + if path != "" { + data, err := os.ReadFile(path) + if err != nil { + return Config{}, fmt.Errorf("read config %s: %w", path, err) + } + if err := yaml.Unmarshal(data, &cfg); err != nil { + return Config{}, fmt.Errorf("parse config %s: %w", path, err) + } + } + cfg.applyEnv() + cfg.resolveSecretEnvs() + if err := cfg.Validate(); err != nil { + return Config{}, err + } + return cfg, nil +} + +// applyEnv overlays scalar overrides from the environment. Only the knobs an +// operator commonly flips are wired; secrets are handled by resolveSecretEnvs. +func (c *Config) applyEnv() { + if v := os.Getenv("LOGARCHIVER_NATS_URL"); v != "" { + c.NATS.URL = v + } + if v := os.Getenv("LOGARCHIVER_NATS_DURABLE"); v != "" { + c.NATS.Durable = v + } + if v := os.Getenv("ARCHIVE_SUBJECTS"); v != "" { + c.NATS.Subjects = splitFields(v) + } + if v := os.Getenv("LOGARCHIVER_S3_ENDPOINT"); v != "" { + c.S3.Endpoint = v + } + if v := os.Getenv("LOGARCHIVER_S3_BUCKET"); v != "" { + c.S3.Bucket = v + } + if v := os.Getenv("LOGARCHIVER_KEY_NAME"); v != "" { + c.Crypto.KeyName = v + } + if v := os.Getenv("LOGARCHIVER_PUBKEY_SOURCE"); v != "" { + c.Crypto.Source = PubkeySource(v) + } + if v := os.Getenv("LOGARCHIVER_PUBKEY_FILE"); v != "" { + c.Crypto.PubkeyFile = v + } + if v := os.Getenv("VAULT_ADDR"); v != "" && c.Crypto.Vault.Address == "" { + c.Crypto.Vault.Address = v + } + if v := os.Getenv("LOGARCHIVER_VAULT_MOUNT"); v != "" { + c.Crypto.Vault.Mount = v + } + if v := os.Getenv("LOGARCHIVER_CLICKHOUSE_ADDR"); v != "" { + c.Index.Address = v + } + if v := os.Getenv("CLICKHOUSE_USER"); v != "" { + c.Index.Username = v + } + if v := os.Getenv("LOGARCHIVER_METRICS_ADDR"); v != "" { + c.Metrics.Address = v + } + if v := os.Getenv("LOGARCHIVER_LOG_LEVEL"); v != "" { + c.Log.Level = v + } + if v := os.Getenv("LOGARCHIVER_LOG_FORMAT"); v != "" { + c.Log.Format = v + } + // Endpoint/bucket sourced from the cephrgw secret, if present. + if c.S3.EndpointEnv != "" { + if v := os.Getenv(c.S3.EndpointEnv); v != "" { + c.S3.Endpoint = v + } + } + if c.S3.BucketEnv != "" { + if v := os.Getenv(c.S3.BucketEnv); v != "" { + c.S3.Bucket = v + } + } +} + +// resolveSecretEnvs pulls passwords from their named env vars when not set inline. +func (c *Config) resolveSecretEnvs() { + if c.NATS.Password == "" && c.NATS.PasswordEnv != "" { + c.NATS.Password = os.Getenv(c.NATS.PasswordEnv) + } + if c.Index.Password == "" && c.Index.PasswordEnv != "" { + c.Index.Password = os.Getenv(c.Index.PasswordEnv) + } +} + +// Validate checks required fields and coherence. +func (c *Config) Validate() error { + if c.NATS.URL == "" { + return fmt.Errorf("nats.url is required") + } + if c.NATS.Stream == "" { + return fmt.Errorf("nats.stream is required") + } + if c.NATS.Durable == "" { + return fmt.Errorf("nats.durable is required") + } + if len(c.NATS.Subjects) == 0 { + return fmt.Errorf("nats.subjects must list at least one filter subject") + } + if c.S3.Bucket == "" { + return fmt.Errorf("s3.bucket is required") + } + if c.S3.Endpoint == "" { + return fmt.Errorf("s3.endpoint is required") + } + if c.Crypto.KeyName == "" { + return fmt.Errorf("crypto.key_name is required") + } + switch c.Crypto.Source { + case PubkeyVault: + if c.Crypto.Vault.Address == "" { + return fmt.Errorf("crypto.vault.address is required when pubkey_source=vault") + } + if c.Crypto.Vault.Mount == "" { + return fmt.Errorf("crypto.vault.mount is required when pubkey_source=vault") + } + case PubkeyFile: + if c.Crypto.PubkeyFile == "" { + return fmt.Errorf("crypto.pubkey_file is required when pubkey_source=file") + } + default: + return fmt.Errorf("crypto.pubkey_source must be 'vault' or 'file', got %q", c.Crypto.Source) + } + if c.Crypto.FrameSize <= 0 { + return fmt.Errorf("crypto.frame_size must be positive") + } + if c.Batch.MaxBytes <= 0 && c.Batch.MaxEvents <= 0 && c.Batch.MaxAge <= 0 { + return fmt.Errorf("batch must set at least one of max_bytes/max_events/max_age") + } + if c.Index.Enabled && c.Index.Address == "" { + return fmt.Errorf("index.address is required when index.enabled") + } + return nil +} + +func splitFields(s string) []string { + var out []string + for _, f := range strings.Fields(s) { + if f != "" { + out = append(out, f) + } + } + return out +} + +// ParseSize parses a byte size like "64Mi", "128MB", "1024". It is a helper for +// CLI flags; the YAML fields are plain integers. +func ParseSize(s string) (int64, error) { + s = strings.TrimSpace(s) + if s == "" { + return 0, fmt.Errorf("empty size") + } + mult := int64(1) + switch { + case strings.HasSuffix(s, "Gi"): + mult, s = 1<<30, strings.TrimSuffix(s, "Gi") + case strings.HasSuffix(s, "Mi"): + mult, s = 1<<20, strings.TrimSuffix(s, "Mi") + case strings.HasSuffix(s, "Ki"): + mult, s = 1<<10, strings.TrimSuffix(s, "Ki") + case strings.HasSuffix(s, "GB"): + mult, s = 1e9, strings.TrimSuffix(s, "GB") + case strings.HasSuffix(s, "MB"): + mult, s = 1e6, strings.TrimSuffix(s, "MB") + case strings.HasSuffix(s, "KB"): + mult, s = 1e3, strings.TrimSuffix(s, "KB") + } + n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64) + if err != nil { + return 0, fmt.Errorf("parse size %q: %w", s, err) + } + return n * mult, nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..157c39d --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,106 @@ +package config + +import ( + "testing" + "time" +) + +func TestDefaultIsValid(t *testing.T) { + cfg := Default() + if err := cfg.Validate(); err != nil { + t.Fatalf("default config should validate: %v", err) + } +} + +func TestLoadEnvOverrides(t *testing.T) { + t.Setenv("ARCHIVE_SUBJECTS", "logs.vm.> logs.k8s.vault.>") + t.Setenv("LOGARCHIVER_S3_BUCKET", "my-bucket") + t.Setenv("LOGARCHIVER_NATS_DURABLE", "archiver-canary") + t.Setenv("NATS_CONSUMER_PASSWORD", "s3cr3t") + t.Setenv("S3_ENDPOINT", "https://rgw.internal") + t.Setenv("BUCKET_NAME", "logs-archive-override") + + cfg, err := Load("") + if err != nil { + t.Fatalf("Load: %v", err) + } + if got, want := cfg.NATS.Subjects, []string{"logs.vm.>", "logs.k8s.vault.>"}; len(got) != len(want) || got[0] != want[0] || got[1] != want[1] { + t.Errorf("subjects = %v, want %v", got, want) + } + if cfg.NATS.Durable != "archiver-canary" { + t.Errorf("durable = %q", cfg.NATS.Durable) + } + if cfg.NATS.Password != "s3cr3t" { + t.Errorf("password from env not resolved: %q", cfg.NATS.Password) + } + // BUCKET_NAME (secret) should win over LOGARCHIVER_S3_BUCKET default flow. + if cfg.S3.Bucket != "logs-archive-override" { + t.Errorf("bucket = %q, want cephrgw secret override", cfg.S3.Bucket) + } + if cfg.S3.Endpoint != "https://rgw.internal" { + t.Errorf("endpoint = %q", cfg.S3.Endpoint) + } +} + +func TestValidateErrors(t *testing.T) { + cases := map[string]func(*Config){ + "no subjects": func(c *Config) { c.NATS.Subjects = nil }, + "no bucket": func(c *Config) { c.S3.Bucket = "" }, + "no key name": func(c *Config) { c.Crypto.KeyName = "" }, + "bad source": func(c *Config) { c.Crypto.Source = "elsewhere" }, + "file no path": func(c *Config) { c.Crypto.Source = PubkeyFile; c.Crypto.PubkeyFile = "" }, + "vault no addr": func(c *Config) { c.Crypto.Source = PubkeyVault; c.Crypto.Vault.Address = "" }, + "zero frame": func(c *Config) { c.Crypto.FrameSize = 0 }, + "no batch bounds": func(c *Config) { c.Batch = BatchConfig{} }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + cfg := Default() + mutate(&cfg) + if err := cfg.Validate(); err == nil { + t.Errorf("expected validation error for %q", name) + } + }) + } +} + +func TestValidateVaultSourceOK(t *testing.T) { + cfg := Default() + cfg.Crypto.Source = PubkeyVault + cfg.Crypto.Vault.Address = "https://vault.example:8200" + cfg.Crypto.Vault.Mount = "gpg" + if err := cfg.Validate(); err != nil { + t.Fatalf("vault source should validate: %v", err) + } +} + +func TestParseSize(t *testing.T) { + cases := map[string]int64{ + "1024": 1024, + "64Mi": 64 << 20, + "2Gi": 2 << 30, + "1Ki": 1024, + "5MB": 5_000_000, + " 10 ": 10, + } + for in, want := range cases { + got, err := ParseSize(in) + if err != nil { + t.Errorf("ParseSize(%q): %v", in, err) + continue + } + if got != want { + t.Errorf("ParseSize(%q) = %d, want %d", in, got, want) + } + } + if _, err := ParseSize("bogus"); err == nil { + t.Errorf("expected error for bogus size") + } +} + +func TestDefaultBatchDurations(t *testing.T) { + cfg := Default() + if cfg.Batch.MaxAge != 5*time.Minute { + t.Errorf("default MaxAge = %v", cfg.Batch.MaxAge) + } +} diff --git a/internal/consumer/connect.go b/internal/consumer/connect.go new file mode 100644 index 0000000..832ff55 --- /dev/null +++ b/internal/consumer/connect.go @@ -0,0 +1,80 @@ +package consumer + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "os" + "time" + + "git.unkin.net/unkin/logarchiver/internal/config" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// Connect dials NATS as the configured user and returns the connection and a +// JetStream context. Callers must Close the returned *nats.Conn. +func Connect(cfg config.NATSConfig) (*nats.Conn, jetstream.JetStream, error) { + opts := []nats.Option{ + nats.Name("logarchiver"), + nats.MaxReconnects(-1), + nats.ReconnectWait(2 * time.Second), + } + if cfg.User != "" { + opts = append(opts, nats.UserInfo(cfg.User, cfg.Password)) + } + if cfg.CAFile != "" { + pool := x509.NewCertPool() + pem, err := os.ReadFile(cfg.CAFile) + if err != nil { + return nil, nil, fmt.Errorf("read nats ca %s: %w", cfg.CAFile, err) + } + if !pool.AppendCertsFromPEM(pem) { + return nil, nil, fmt.Errorf("no certs parsed from nats ca %s", cfg.CAFile) + } + opts = append(opts, nats.Secure(&tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12})) + } + nc, err := nats.Connect(cfg.URL, opts...) + if err != nil { + return nil, nil, fmt.Errorf("connect nats %s: %w", cfg.URL, err) + } + js, err := jetstream.New(nc) + if err != nil { + nc.Close() + return nil, nil, fmt.Errorf("jetstream context: %w", err) + } + return nc, js, nil +} + +// EnsureConsumer creates or updates the durable pull consumer on the stream with +// the configured subject filters. Independent offsets and explicit acks give +// logarchiver at-least-once delivery decoupled from the transform tier. +func EnsureConsumer(ctx context.Context, js jetstream.JetStream, cfg config.NATSConfig) (jetstream.Consumer, error) { + ackWait := cfg.AckWait + if ackWait <= 0 { + ackWait = 2 * time.Minute + } + consCfg := jetstream.ConsumerConfig{ + Durable: cfg.Durable, + Name: cfg.Durable, + AckPolicy: jetstream.AckExplicitPolicy, + DeliverPolicy: jetstream.DeliverAllPolicy, + AckWait: ackWait, + MaxDeliver: -1, + ReplayPolicy: jetstream.ReplayInstantPolicy, + } + switch len(cfg.Subjects) { + case 0: + return nil, fmt.Errorf("no subject filters configured") + case 1: + consCfg.FilterSubject = cfg.Subjects[0] + default: + consCfg.FilterSubjects = cfg.Subjects + } + cons, err := js.CreateOrUpdateConsumer(ctx, cfg.Stream, consCfg) + if err != nil { + return nil, fmt.Errorf("ensure consumer %s on stream %s: %w", cfg.Durable, cfg.Stream, err) + } + return cons, nil +} diff --git a/internal/consumer/meta.go b/internal/consumer/meta.go new file mode 100644 index 0000000..8759185 --- /dev/null +++ b/internal/consumer/meta.go @@ -0,0 +1,8 @@ +package consumer + +import "git.unkin.net/unkin/logarchiver/internal/event" + +// extractMeta projects the host/timestamp fields from a raw event payload. +func extractMeta(raw []byte) event.Meta { + return event.Extract(raw) +} diff --git a/internal/consumer/runner.go b/internal/consumer/runner.go new file mode 100644 index 0000000..d75c3c6 --- /dev/null +++ b/internal/consumer/runner.go @@ -0,0 +1,237 @@ +// Package consumer binds the JetStream pull consumer and runs the archive loop. +// +// The core correctness property: a batch's messages are acknowledged ONLY after +// the batch has been sealed, uploaded to S3, and indexed in ClickHouse. If any +// of those fails the messages are Nak'd (with a backoff) and JetStream +// redelivers them, so nothing is lost on a sink outage. This sink-conditional +// acking is the main thing logarchiver does that a stock Vector NATS consumer +// cannot. +package consumer + +import ( + "context" + "errors" + "log/slog" + "time" + + "git.unkin.net/unkin/logarchiver/internal/batcher" + "github.com/nats-io/nats.go/jetstream" +) + +// Persister stores a ready batch durably. On success the caller acks. +type Persister interface { + Store(ctx context.Context, batch *batcher.Batch) (StoreResult, error) +} + +// StoreResult mirrors archiver.StoreResult (kept local to avoid an import cycle; +// the archiver's result is adapted at the call site). +type StoreResult struct { + ObjectKey string + Events int + RawBytes int64 + StoredBytes int64 +} + +// Metrics is the optional metrics surface for the loop. +type Metrics interface { + MessagesFetched(n int) + Acked(n int) + BatchFlushed(trigger string) + SetPending(n int) +} + +// Runner drives the fetch → batch → persist → ack loop. +type Runner struct { + cons jetstream.Consumer + batcher *batcher.Batcher + persist Persister + log *slog.Logger + metrics Metrics + fetchBatch int + pollWait time.Duration + nakBackoff time.Duration + drainTO time.Duration + nowFn func() time.Time +} + +// Options configures a Runner. +type Options struct { + Consumer jetstream.Consumer + Batcher *batcher.Batcher + Persister Persister + Logger *slog.Logger + Metrics Metrics + FetchBatch int + // PollWait bounds each Fetch and thus how often age-based flushes are checked. + PollWait time.Duration + // NakBackoff delays redelivery after a persist failure. + NakBackoff time.Duration + // DrainTimeout bounds the shutdown flush. + DrainTimeout time.Duration +} + +// NewRunner builds a Runner. +func NewRunner(o Options) *Runner { + fetch := o.FetchBatch + if fetch <= 0 { + fetch = 512 + } + poll := o.PollWait + if poll <= 0 { + poll = time.Second + } + nak := o.NakBackoff + if nak <= 0 { + nak = 10 * time.Second + } + drain := o.DrainTimeout + if drain <= 0 { + drain = 30 * time.Second + } + log := o.Logger + if log == nil { + log = slog.Default() + } + return &Runner{ + cons: o.Consumer, + batcher: o.Batcher, + persist: o.Persister, + log: log, + metrics: o.Metrics, + fetchBatch: fetch, + pollWait: poll, + nakBackoff: nak, + drainTO: drain, + nowFn: time.Now, + } +} + +// Run loops until ctx is cancelled, then drains open batches before returning. +func (r *Runner) Run(ctx context.Context) error { + r.log.Info("archive loop started", + "fetch_batch", r.fetchBatch, "poll_wait", r.pollWait.String()) + for { + if ctx.Err() != nil { + return r.drain() + } + + // Age-based flush before fetching more. + r.flushBatches(ctx, r.batcher.DueByAge(r.nowFn()), "age") + + msgs, err := r.cons.Fetch(r.fetchBatch, jetstream.FetchMaxWait(r.pollWait)) + if err != nil { + if errors.Is(err, context.Canceled) || ctx.Err() != nil { + return r.drain() + } + r.log.Warn("fetch failed", "err", err) + r.sleep(ctx, r.pollWait) + continue + } + + n := 0 + for msg := range msgs.Messages() { + n++ + r.route(ctx, msg) + } + if ferr := msgs.Error(); ferr != nil && !errors.Is(ferr, context.Canceled) { + r.log.Warn("fetch iteration error", "err", ferr) + } + if r.metrics != nil { + r.metrics.MessagesFetched(n) + r.metrics.SetPending(r.batcher.Pending()) + } + } +} + +// route decodes a message and adds it to the batcher, flushing if the batch +// becomes full. +func (r *Runner) route(ctx context.Context, msg jetstream.Msg) { + meta := extractMeta(msg.Data()) + full := r.batcher.Add(batcher.Item{ + Subject: msg.Subject(), + Raw: msg.Data(), + Host: meta.Host, + Timestamp: meta.Timestamp, + HasTS: meta.Ok, + Ack: msg, + }) + if full != nil { + r.flush(ctx, full, "full") + } +} + +// flushBatches flushes a slice of batches with the given trigger label. +func (r *Runner) flushBatches(ctx context.Context, batches []*batcher.Batch, trigger string) { + for _, b := range batches { + r.flush(ctx, b, trigger) + } +} + +// flush persists a batch and, only on success, acks its messages. On failure it +// Naks with a backoff so JetStream redelivers. +func (r *Runner) flush(ctx context.Context, b *batcher.Batch, trigger string) { + if len(b.Items) == 0 { + return + } + res, err := r.persist.Store(ctx, b) + if err != nil { + r.log.Error("persist failed; batch will be redelivered", + "subject", b.Subject, "events", len(b.Items), "trigger", trigger, "err", err) + r.nakAll(b) + return + } + acked := r.ackAll(b) + if r.metrics != nil { + r.metrics.Acked(acked) + r.metrics.BatchFlushed(trigger) + } + r.log.Info("object archived", + "subject", b.Subject, "object_key", res.ObjectKey, + "events", res.Events, "raw_bytes", res.RawBytes, "stored_bytes", res.StoredBytes, + "trigger", trigger) +} + +func (r *Runner) ackAll(b *batcher.Batch) int { + n := 0 + for _, it := range b.Items { + if msg, ok := it.Ack.(jetstream.Msg); ok { + if err := msg.Ack(); err != nil { + r.log.Warn("ack failed", "subject", b.Subject, "err", err) + continue + } + n++ + } + } + return n +} + +func (r *Runner) nakAll(b *batcher.Batch) { + for _, it := range b.Items { + if msg, ok := it.Ack.(jetstream.Msg); ok { + _ = msg.NakWithDelay(r.nakBackoff) + } + } +} + +// drain flushes all open batches during shutdown with a bounded timeout. +func (r *Runner) drain() error { + batches := r.batcher.Drain() + if len(batches) == 0 { + r.log.Info("archive loop stopped; nothing to drain") + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), r.drainTO) + defer cancel() + r.log.Info("draining open batches", "batches", len(batches)) + r.flushBatches(ctx, batches, "shutdown") + return nil +} + +func (r *Runner) sleep(ctx context.Context, d time.Duration) { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + case <-t.C: + } +} diff --git a/internal/consumer/runner_test.go b/internal/consumer/runner_test.go new file mode 100644 index 0000000..8393f96 --- /dev/null +++ b/internal/consumer/runner_test.go @@ -0,0 +1,151 @@ +package consumer + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "git.unkin.net/unkin/logarchiver/internal/batcher" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// fakeMsg is a minimal jetstream.Msg recording ack/nak calls. +type fakeMsg struct { + subject string + data []byte + mu sync.Mutex + acked bool + naked bool +} + +func (m *fakeMsg) Metadata() (*jetstream.MsgMetadata, error) { return &jetstream.MsgMetadata{}, nil } +func (m *fakeMsg) Data() []byte { return m.data } +func (m *fakeMsg) Headers() nats.Header { return nil } +func (m *fakeMsg) Subject() string { return m.subject } +func (m *fakeMsg) Reply() string { return "" } +func (m *fakeMsg) Ack() error { + m.mu.Lock() + defer m.mu.Unlock() + m.acked = true + return nil +} +func (m *fakeMsg) DoubleAck(context.Context) error { return nil } +func (m *fakeMsg) Nak() error { + m.mu.Lock() + defer m.mu.Unlock() + m.naked = true + return nil +} +func (m *fakeMsg) NakWithDelay(time.Duration) error { + m.mu.Lock() + defer m.mu.Unlock() + m.naked = true + return nil +} +func (m *fakeMsg) InProgress() error { return nil } +func (m *fakeMsg) Term() error { return nil } +func (m *fakeMsg) TermWithReason(string) error { return nil } + +func (m *fakeMsg) isAcked() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.acked +} +func (m *fakeMsg) isNaked() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.naked +} + +// fakePersister records calls and can be made to fail. +type fakePersister struct { + fail bool + called int +} + +func (p *fakePersister) Store(_ context.Context, b *batcher.Batch) (StoreResult, error) { + p.called++ + if p.fail { + return StoreResult{}, errors.New("boom") + } + return StoreResult{ObjectKey: "k", Events: len(b.Items)}, nil +} + +func batchWith(msgs ...*fakeMsg) *batcher.Batch { + b := &batcher.Batch{Subject: "s"} + for _, m := range msgs { + b.Items = append(b.Items, batcher.Item{Subject: "s", Raw: m.data, Ack: jetstream.Msg(m)}) + } + return b +} + +// TestFlushAcksOnlyAfterPersist is the core correctness test: messages are acked +// exactly when Store succeeds, and Nak'd (never acked) when it fails. +func TestFlushAcksOnSuccess(t *testing.T) { + p := &fakePersister{} + r := NewRunner(Options{Persister: p}) + m1 := &fakeMsg{subject: "s", data: []byte(`{"host":"h"}`)} + m2 := &fakeMsg{subject: "s", data: []byte(`{"host":"h2"}`)} + + r.flush(context.Background(), batchWith(m1, m2), "test") + + if p.called != 1 { + t.Fatalf("Store called %d times, want 1", p.called) + } + if !m1.isAcked() || !m2.isAcked() { + t.Errorf("messages should be acked after successful persist") + } + if m1.isNaked() || m2.isNaked() { + t.Errorf("messages must not be naked on success") + } +} + +func TestFlushNaksOnFailure(t *testing.T) { + p := &fakePersister{fail: true} + r := NewRunner(Options{Persister: p}) + m1 := &fakeMsg{subject: "s", data: []byte(`{"host":"h"}`)} + + r.flush(context.Background(), batchWith(m1), "test") + + if m1.isAcked() { + t.Errorf("message must NOT be acked when persist fails") + } + if !m1.isNaked() { + t.Errorf("message should be naked so JetStream redelivers") + } +} + +func TestFlushEmptyBatchNoop(t *testing.T) { + p := &fakePersister{} + r := NewRunner(Options{Persister: p}) + r.flush(context.Background(), &batcher.Batch{Subject: "s"}, "test") + if p.called != 0 { + t.Errorf("empty batch should not call Store") + } +} + +// TestRouteAndFlushIntegration wires a real batcher: adding enough messages to +// fill the batch triggers a full flush that persists and acks exactly those. +func TestRouteFlushViaBatcher(t *testing.T) { + p := &fakePersister{} + bat := batcher.New(batcher.Limits{MaxEvents: 2}) + r := NewRunner(Options{Persister: p, Batcher: bat}) + + m1 := &fakeMsg{subject: "s", data: []byte(`{"host":"a"}`)} + m2 := &fakeMsg{subject: "s", data: []byte(`{"host":"b"}`)} + r.route(context.Background(), m1) + if m1.isAcked() { + t.Errorf("first message should not be acked before batch fills") + } + r.route(context.Background(), m2) // fills batch -> flush + + if p.called != 1 { + t.Fatalf("Store called %d times, want 1 after fill", p.called) + } + if !m1.isAcked() || !m2.isAcked() { + t.Errorf("both messages should be acked after the full-batch flush") + } +} diff --git a/internal/crypto/crypto_test.go b/internal/crypto/crypto_test.go new file mode 100644 index 0000000..c4b80fa --- /dev/null +++ b/internal/crypto/crypto_test.go @@ -0,0 +1,165 @@ +package crypto + +import ( + "bytes" + "io" + "strings" + "testing" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" +) + +// genTestKey creates an OpenPGP keypair, returns the armored public key (as the +// service would export from Vault) and an unwrap func that decrypts the wrapped +// DEK with the private key — simulating the Vault GPG engine's decrypt endpoint +// (which returns the plaintext of a whole OpenPGP message). +func genTestKey(t *testing.T) (armoredPub []byte, unwrap UnwrapFunc) { + t.Helper() + ent, err := openpgp.NewEntity("logarchiver-test", "unit test", "test@unkin.net", nil) + if err != nil { + t.Fatalf("NewEntity: %v", err) + } + var buf bytes.Buffer + w, err := armor.Encode(&buf, openpgp.PublicKeyType, nil) + if err != nil { + t.Fatalf("armor encode: %v", err) + } + if err := ent.Serialize(w); err != nil { + t.Fatalf("serialize public: %v", err) + } + _ = w.Close() + + unwrap = func(wrapped []byte) ([]byte, error) { + md, err := openpgp.ReadMessage(bytes.NewReader(wrapped), openpgp.EntityList{ent}, nil, nil) + if err != nil { + return nil, err + } + return io.ReadAll(md.UnverifiedBody) + } + return buf.Bytes(), unwrap +} + +func TestRoundTrip(t *testing.T) { + armoredPub, unwrap := genTestKey(t) + pub, err := LoadPublicKey(armoredPub) + if err != nil { + t.Fatalf("LoadPublicKey: %v", err) + } + if len(pub.Fingerprint) != 40 { + t.Errorf("fingerprint = %q, want 40 hex chars", pub.Fingerprint) + } + if pub.Fingerprint != strings.ToUpper(pub.Fingerprint) { + t.Errorf("fingerprint should be uppercase: %q", pub.Fingerprint) + } + + // A multi-line NDJSON payload larger than the frame size (forces >1 frame). + var payload bytes.Buffer + for i := 0; i < 5000; i++ { + payload.WriteString(`{"host":"node-1","message":"line `) + payload.WriteString(strings.Repeat("x", 50)) + payload.WriteString(`"}` + "\n") + } + plaintext := payload.Bytes() + + var sealed bytes.Buffer + res, err := Seal(&sealed, plaintext, pub, "logarchive", 4096) + if err != nil { + t.Fatalf("Seal: %v", err) + } + if res.RawBytes != int64(len(plaintext)) { + t.Errorf("RawBytes = %d, want %d", res.RawBytes, len(plaintext)) + } + if int64(sealed.Len()) != res.StoredBytes { + t.Errorf("StoredBytes = %d, buffer = %d", res.StoredBytes, sealed.Len()) + } + // Compression should shrink this highly repetitive payload. + if res.StoredBytes >= res.RawBytes { + t.Errorf("stored (%d) not smaller than raw (%d)", res.StoredBytes, res.RawBytes) + } + // Design property: only a tiny wrapped DEK goes to Vault, regardless of size. + if res.Header.WrappedDEKLen > 4096 { + t.Errorf("wrapped DEK unexpectedly large: %d bytes", res.Header.WrappedDEKLen) + } + if res.Header.KeyFingerprint != pub.Fingerprint { + t.Errorf("header fingerprint mismatch") + } + + var out bytes.Buffer + if err := Open(bytes.NewReader(sealed.Bytes()), &out, unwrap); err != nil { + t.Fatalf("Open: %v", err) + } + if !bytes.Equal(out.Bytes(), plaintext) { + t.Fatalf("round-trip mismatch: got %d bytes, want %d", out.Len(), len(plaintext)) + } +} + +func TestRoundTripEmpty(t *testing.T) { + armoredPub, unwrap := genTestKey(t) + pub, _ := LoadPublicKey(armoredPub) + var sealed bytes.Buffer + if _, err := Seal(&sealed, []byte{}, pub, "k", 4096); err != nil { + t.Fatalf("Seal empty: %v", err) + } + var out bytes.Buffer + if err := Open(bytes.NewReader(sealed.Bytes()), &out, unwrap); err != nil { + t.Fatalf("Open empty: %v", err) + } + if out.Len() != 0 { + t.Errorf("empty round-trip produced %d bytes", out.Len()) + } +} + +func TestTamperDetected(t *testing.T) { + armoredPub, unwrap := genTestKey(t) + pub, _ := LoadPublicKey(armoredPub) + var sealed bytes.Buffer + if _, err := Seal(&sealed, []byte("hello world\n"), pub, "k", 4096); err != nil { + t.Fatalf("Seal: %v", err) + } + data := sealed.Bytes() + // Flip a byte near the end (inside a frame's ciphertext/tag). + data[len(data)-3] ^= 0xff + var out bytes.Buffer + if err := Open(bytes.NewReader(data), &out, unwrap); err == nil { + t.Fatalf("expected GCM authentication failure on tampered ciphertext") + } +} + +func TestBadMagic(t *testing.T) { + _, _, err := ReadHeader(bytes.NewReader([]byte("NOTLARC....."))) + if err == nil { + t.Fatalf("expected bad-magic error") + } +} + +func TestReadHeaderFields(t *testing.T) { + armoredPub, _ := genTestKey(t) + pub, _ := LoadPublicKey(armoredPub) + var sealed bytes.Buffer + if _, err := Seal(&sealed, []byte("x\n"), pub, "logarchive", 4096); err != nil { + t.Fatalf("Seal: %v", err) + } + hdr, wrapped, err := ReadHeader(bytes.NewReader(sealed.Bytes())) + if err != nil { + t.Fatalf("ReadHeader: %v", err) + } + if hdr.KeyName != "logarchive" { + t.Errorf("KeyName = %q", hdr.KeyName) + } + if hdr.Compression != "zstd" || hdr.Cipher != "AES-256-GCM" { + t.Errorf("algo metadata wrong: %+v", hdr) + } + if len(wrapped) != hdr.WrappedDEKLen { + t.Errorf("wrapped len %d != header %d", len(wrapped), hdr.WrappedDEKLen) + } +} + +func TestDigestArmoredStable(t *testing.T) { + // Guards the test helper used elsewhere; identical input -> identical digest. + a := digestArmored([]byte("abc")) + b := digestArmored([]byte("abc")) + if a != b || a == "" { + t.Errorf("digestArmored not stable: %q %q", a, b) + } +} diff --git a/internal/crypto/envelope.go b/internal/crypto/envelope.go new file mode 100644 index 0000000..09a3b94 --- /dev/null +++ b/internal/crypto/envelope.go @@ -0,0 +1,338 @@ +package crypto + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/binary" + "encoding/json" + "fmt" + "io" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/packet" + "github.com/klauspost/compress/zstd" +) + +// ContainerMagic identifies a logarchiver object and its container version. +var ContainerMagic = []byte("LARC1\n") + +const ( + dekSize = 32 // AES-256 + noncePrefixSize = 4 + counterSize = 8 + // maxFrameCiphertext bounds a single frame read to avoid unbounded allocation + // from a corrupt/hostile length prefix. + maxFrameCiphertext = 128 << 20 +) + +// Header is the LARC1 object header (JSON), written after the magic and a +// uint32 big-endian length prefix. +type Header struct { + Version int `json:"v"` + KeyName string `json:"key_name"` + KeyFingerprint string `json:"key_fingerprint"` + WrappedDEKLen int `json:"wrapped_dek_len"` + NoncePrefix []byte `json:"nonce_prefix"` // base64 in JSON + FrameSize int `json:"frame_size"` // plaintext (compressed) bytes per frame + Compression string `json:"compression"` // "zstd" + Cipher string `json:"cipher"` // "AES-256-GCM" +} + +// SealResult reports what Seal produced (for the index row). +type SealResult struct { + Header Header + RawBytes int64 // input NDJSON length + StoredBytes int64 // full container length +} + +// Seal compresses plaintext with zstd, encrypts it under a fresh random DEK +// using framed AES-256-GCM, wraps the DEK to pub with OpenPGP, and writes the +// LARC1 container to w. keyName is recorded in the header for operator context. +func Seal(w io.Writer, plaintext []byte, pub *PublicKey, keyName string, frameSize int) (SealResult, error) { + if pub == nil { + return SealResult{}, fmt.Errorf("nil public key") + } + if frameSize <= 0 { + frameSize = 1 << 20 + } + + // 1. Compress. + enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBetterCompression)) + if err != nil { + return SealResult{}, fmt.Errorf("zstd writer: %w", err) + } + compressed := enc.EncodeAll(plaintext, nil) + _ = enc.Close() + + // 2. DEK + nonce prefix. + dek := make([]byte, dekSize) + if _, err := rand.Read(dek); err != nil { + return SealResult{}, fmt.Errorf("gen dek: %w", err) + } + noncePrefix := make([]byte, noncePrefixSize) + if _, err := rand.Read(noncePrefix); err != nil { + return SealResult{}, fmt.Errorf("gen nonce prefix: %w", err) + } + + // 3. Wrap the DEK to the public key (small standard OpenPGP message). + wrapped, err := wrapDEK(dek, pub) + if err != nil { + return SealResult{}, err + } + + block, err := aes.NewCipher(dek) + if err != nil { + return SealResult{}, fmt.Errorf("aes cipher: %w", err) + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return SealResult{}, fmt.Errorf("gcm: %w", err) + } + + hdr := Header{ + Version: 1, + KeyName: keyName, + KeyFingerprint: pub.Fingerprint, + WrappedDEKLen: len(wrapped), + NoncePrefix: noncePrefix, + FrameSize: frameSize, + Compression: "zstd", + Cipher: "AES-256-GCM", + } + hdrJSON, err := json.Marshal(hdr) + if err != nil { + return SealResult{}, fmt.Errorf("marshal header: %w", err) + } + + cw := &countingWriter{w: w} + + // magic + if _, err := cw.Write(ContainerMagic); err != nil { + return SealResult{}, err + } + // header length + header + if err := writeUint32(cw, uint32(len(hdrJSON))); err != nil { + return SealResult{}, err + } + if _, err := cw.Write(hdrJSON); err != nil { + return SealResult{}, err + } + // wrapped DEK + if _, err := cw.Write(wrapped); err != nil { + return SealResult{}, err + } + + // frames + for i, off := 0, 0; off < len(compressed); i++ { + end := off + frameSize + if end > len(compressed) { + end = len(compressed) + } + nonce := frameNonce(noncePrefix, uint64(i)) + aad := aadFor(uint64(i)) + ct := gcm.Seal(nil, nonce, compressed[off:end], aad) + if err := writeUint32(cw, uint32(len(ct))); err != nil { + return SealResult{}, err + } + if _, err := cw.Write(ct); err != nil { + return SealResult{}, err + } + off = end + } + // terminating zero-length frame + if err := writeUint32(cw, 0); err != nil { + return SealResult{}, err + } + + return SealResult{Header: hdr, RawBytes: int64(len(plaintext)), StoredBytes: cw.n}, nil +} + +// wrapDEK OpenPGP-encrypts the DEK to pub, producing a compact binary message. +func wrapDEK(dek []byte, pub *PublicKey) ([]byte, error) { + var buf bytes.Buffer + cfg := &packet.Config{ + DefaultCipher: packet.CipherAES256, + } + wc, err := openpgp.Encrypt(&buf, []*openpgp.Entity{pub.entity}, nil, nil, cfg) + if err != nil { + return nil, fmt.Errorf("openpgp encrypt dek: %w", err) + } + if _, err := wc.Write(dek); err != nil { + return nil, fmt.Errorf("write dek: %w", err) + } + if err := wc.Close(); err != nil { + return nil, fmt.Errorf("close openpgp: %w", err) + } + return buf.Bytes(), nil +} + +// UnwrapFunc recovers the DEK from the wrapped OpenPGP blob. In production this +// calls the Vault GPG engine decrypt endpoint; tests supply a local one. +type UnwrapFunc func(wrappedDEK []byte) (dek []byte, err error) + +// ReadHeader reads and validates the LARC1 magic + header and the wrapped DEK, +// leaving r positioned at the first frame. It does not require decryption keys, +// so it is cheap for `search`/metadata inspection. +func ReadHeader(r io.Reader) (Header, []byte, error) { + magic := make([]byte, len(ContainerMagic)) + if _, err := io.ReadFull(r, magic); err != nil { + return Header{}, nil, fmt.Errorf("read magic: %w", err) + } + if !bytes.Equal(magic, ContainerMagic) { + return Header{}, nil, fmt.Errorf("bad magic: not a logarchiver (LARC1) object") + } + hlen, err := readUint32(r) + if err != nil { + return Header{}, nil, fmt.Errorf("read header len: %w", err) + } + if hlen == 0 || hlen > 1<<20 { + return Header{}, nil, fmt.Errorf("implausible header length %d", hlen) + } + hdrJSON := make([]byte, hlen) + if _, err := io.ReadFull(r, hdrJSON); err != nil { + return Header{}, nil, fmt.Errorf("read header: %w", err) + } + var hdr Header + if err := json.Unmarshal(hdrJSON, &hdr); err != nil { + return Header{}, nil, fmt.Errorf("parse header: %w", err) + } + if hdr.Version != 1 { + return Header{}, nil, fmt.Errorf("unsupported container version %d", hdr.Version) + } + if hdr.WrappedDEKLen <= 0 || hdr.WrappedDEKLen > 1<<20 { + return Header{}, nil, fmt.Errorf("implausible wrapped dek length %d", hdr.WrappedDEKLen) + } + wrapped := make([]byte, hdr.WrappedDEKLen) + if _, err := io.ReadFull(r, wrapped); err != nil { + return Header{}, nil, fmt.Errorf("read wrapped dek: %w", err) + } + return hdr, wrapped, nil +} + +// Open reads a LARC1 container from r, recovers the DEK via unwrap, and streams +// the decrypted, decompressed NDJSON to w. +func Open(r io.Reader, w io.Writer, unwrap UnwrapFunc) error { + hdr, wrapped, err := ReadHeader(r) + if err != nil { + return err + } + dek, err := unwrap(wrapped) + if err != nil { + return fmt.Errorf("unwrap dek: %w", err) + } + if len(dek) != dekSize { + return fmt.Errorf("unwrapped dek has wrong length %d", len(dek)) + } + block, err := aes.NewCipher(dek) + if err != nil { + return fmt.Errorf("aes cipher: %w", err) + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return fmt.Errorf("gcm: %w", err) + } + + fr := &frameReader{r: r, gcm: gcm, noncePrefix: hdr.NoncePrefix} + zr, err := zstd.NewReader(fr) + if err != nil { + return fmt.Errorf("zstd reader: %w", err) + } + defer zr.Close() + if _, err := io.Copy(w, zr); err != nil { + return fmt.Errorf("decompress: %w", err) + } + return nil +} + +// frameReader decrypts LARC1 frames on demand, presenting the decrypted +// (compressed) bytes as an io.Reader for the zstd decoder. +type frameReader struct { + r io.Reader + gcm cipher.AEAD + noncePrefix []byte + counter uint64 + buf []byte // leftover decrypted plaintext not yet consumed + done bool +} + +func (f *frameReader) Read(p []byte) (int, error) { + if len(f.buf) == 0 && !f.done { + if err := f.next(); err != nil { + return 0, err + } + } + if len(f.buf) == 0 { + return 0, io.EOF + } + n := copy(p, f.buf) + f.buf = f.buf[n:] + return n, nil +} + +func (f *frameReader) next() error { + ln, err := readUint32(f.r) + if err != nil { + return fmt.Errorf("read frame len: %w", err) + } + if ln == 0 { // terminator + f.done = true + return nil + } + if ln > maxFrameCiphertext { + return fmt.Errorf("frame length %d exceeds max", ln) + } + ct := make([]byte, ln) + if _, err := io.ReadFull(f.r, ct); err != nil { + return fmt.Errorf("read frame: %w", err) + } + nonce := frameNonce(f.noncePrefix, f.counter) + aad := aadFor(f.counter) + pt, err := f.gcm.Open(nil, nonce, ct, aad) + if err != nil { + return fmt.Errorf("decrypt frame %d: %w", f.counter, err) + } + f.counter++ + f.buf = pt + return nil +} + +func frameNonce(prefix []byte, counter uint64) []byte { + nonce := make([]byte, noncePrefixSize+counterSize) + copy(nonce, prefix) + binary.BigEndian.PutUint64(nonce[noncePrefixSize:], counter) + return nonce +} + +func aadFor(counter uint64) []byte { + aad := make([]byte, counterSize) + binary.BigEndian.PutUint64(aad, counter) + return aad +} + +func writeUint32(w io.Writer, v uint32) error { + var b [4]byte + binary.BigEndian.PutUint32(b[:], v) + _, err := w.Write(b[:]) + return err +} + +func readUint32(r io.Reader) (uint32, error) { + var b [4]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return 0, err + } + return binary.BigEndian.Uint32(b[:]), nil +} + +type countingWriter struct { + w io.Writer + n int64 +} + +func (c *countingWriter) Write(p []byte) (int, error) { + n, err := c.w.Write(p) + c.n += int64(n) + return n, err +} diff --git a/internal/crypto/pubkey.go b/internal/crypto/pubkey.go new file mode 100644 index 0000000..760ea64 --- /dev/null +++ b/internal/crypto/pubkey.go @@ -0,0 +1,81 @@ +// Package crypto implements logarchiver's object encryption. +// +// # Why not plain OpenPGP-encrypt the whole object? +// +// The private key lives only in Ben's Vault GPG secrets engine +// (vault-plugin-secrets-gpg). That engine's decrypt endpoint does WHOLE-payload +// inline decryption only: you POST the entire OpenPGP message (base64 in a JSON +// body) and it returns the entire plaintext (base64). There is no session-key / +// PKESK extraction and no streaming, so a multi-hundred-MiB archive could not be +// retrieved without blowing Vault's request-size limit and buffering everything +// twice in the server. +// +// # The wrapped-DEK envelope (container "LARC1") +// +// logarchiver therefore does hybrid encryption itself: +// +// - a fresh random 256-bit Data Encryption Key (DEK) per object; +// - the bulk (zstd-compressed NDJSON) is encrypted locally with AES-256-GCM in +// independent frames, so decryption streams frame-by-frame; +// - only the 32-byte DEK is OpenPGP-encrypted to the engine's PUBLIC key, +// producing a small (~hundreds of bytes) standard OpenPGP message. +// +// On retrieval the CLI sends ONLY that small wrapped-DEK blob to the engine's +// decrypt endpoint, recovers the DEK, and streams the bulk locally. The Vault +// round-trip is tiny and constant regardless of object size, and the private key +// never leaves Vault. The trade-off vs. a single standard OpenPGP object: these +// objects are a logarchiver-specific container, not decryptable by a bare `gpg` +// even with the private key. The retrieval runbook documents the format. +package crypto + +import ( + "crypto/sha256" + "fmt" + "strings" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" +) + +// PublicKey is a parsed OpenPGP public key plus its fingerprint (uppercase hex, +// no spaces — matching the Vault GPG engine's `%X` fingerprint format). +type PublicKey struct { + entity *openpgp.Entity + Fingerprint string +} + +// LoadPublicKey parses an ASCII-armored (or binary) OpenPGP public key. +func LoadPublicKey(data []byte) (*PublicKey, error) { + var keyring openpgp.EntityList + var err error + if strings.Contains(string(data), "BEGIN PGP") { + block, berr := armor.Decode(strings.NewReader(string(data))) + if berr != nil { + return nil, fmt.Errorf("decode armor: %w", berr) + } + keyring, err = openpgp.ReadKeyRing(block.Body) + } else { + keyring, err = openpgp.ReadKeyRing(strings.NewReader(string(data))) + } + if err != nil { + return nil, fmt.Errorf("read public key: %w", err) + } + if len(keyring) == 0 { + return nil, fmt.Errorf("no public key found") + } + ent := keyring[0] + if ent.PrimaryKey == nil { + return nil, fmt.Errorf("key has no primary public key") + } + return &PublicKey{ + entity: ent, + Fingerprint: fmt.Sprintf("%X", ent.PrimaryKey.Fingerprint), + }, nil +} + +// digestArmored is used by tests to sanity check key identity independent of +// go-crypto internals. +func digestArmored(data []byte) string { + sum := sha256.Sum256(data) + return fmt.Sprintf("%x", sum[:8]) +} diff --git a/internal/event/event.go b/internal/event/event.go new file mode 100644 index 0000000..f21da86 --- /dev/null +++ b/internal/event/event.go @@ -0,0 +1,182 @@ +// Package event extracts the fields logarchiver needs (host, timestamp) from a +// raw log event as it flows through the centralized logging JetStream stream +// (argocd-apps #296). Events are one JSON object per NATS message and are NOT +// normalized — logarchiver persists them raw, so extraction must be tolerant of +// the two shapes that share the logs.> subject space: +// +// - k8s pod logs (Vector kubernetes_logs): host lives at .kubernetes.pod_node_name, +// timestamp at .timestamp (RFC3339). +// - VM logs (vm-ingest): host at .host (fallback .hostname), timestamp at +// .timestamp (fallback .ts). +// +// The NATS subject itself is authoritative for partitioning and is supplied by +// the consumer, not read from the payload. +package event + +import ( + "encoding/json" + "strings" + "time" +) + +// Meta is the minimal, index-relevant projection of a raw log event. +type Meta struct { + // Host is the best-effort source host/node for the event, or "" if none + // could be determined. + Host string + // Timestamp is the event time. Ok reports whether a timestamp field was + // found and parsed; when false callers should fall back to ingest time. + Timestamp time.Time + Ok bool +} + +// hostPaths and tsPaths are tried in order. Dotted paths descend into nested +// objects (only .kubernetes.pod_node_name is nested today). +var ( + hostPaths = [][]string{ + {"host"}, + {"hostname"}, + {"kubernetes", "pod_node_name"}, + } + tsPaths = [][]string{ + {"timestamp"}, + {"ts"}, + {"@timestamp"}, + } +) + +// Extract parses raw (a single JSON log event) and returns its host/timestamp +// projection. It never errors: malformed or field-less events yield a zero-value +// Meta (Host=="", Ok==false) so the archiver still stores the raw bytes and the +// caller can fall back to ingest time. Only the fields of interest are decoded. +func Extract(raw []byte) Meta { + var doc map[string]json.RawMessage + if err := json.Unmarshal(raw, &doc); err != nil { + return Meta{} + } + m := Meta{} + m.Host = firstString(doc, hostPaths) + if ts, ok := firstTime(doc, tsPaths); ok { + m.Timestamp = ts + m.Ok = true + } + return m +} + +// firstString walks each path and returns the first value that decodes to a +// non-empty string. +func firstString(doc map[string]json.RawMessage, paths [][]string) string { + for _, p := range paths { + if v, ok := lookup(doc, p); ok { + var s string + if json.Unmarshal(v, &s) == nil && s != "" { + return s + } + } + } + return "" +} + +// firstTime walks each path and returns the first value that parses as a +// timestamp (RFC3339/RFC3339Nano string, or a numeric unix seconds/millis). +func firstTime(doc map[string]json.RawMessage, paths [][]string) (time.Time, bool) { + for _, p := range paths { + v, ok := lookup(doc, p) + if !ok { + continue + } + var s string + if json.Unmarshal(v, &s) == nil && s != "" { + if t, err := parseTimeString(s); err == nil { + return t, true + } + } + var n json.Number + if json.Unmarshal(v, &n) == nil { + if t, ok := parseNumericTime(n); ok { + return t, true + } + } + } + return time.Time{}, false +} + +// lookup descends doc following path. Intermediate elements must be JSON objects. +func lookup(doc map[string]json.RawMessage, path []string) (json.RawMessage, bool) { + cur := doc + for i, key := range path { + v, ok := cur[key] + if !ok { + return nil, false + } + if i == len(path)-1 { + return v, true + } + var next map[string]json.RawMessage + if json.Unmarshal(v, &next) != nil { + return nil, false + } + cur = next + } + return nil, false +} + +var timeLayouts = []string{ + time.RFC3339Nano, + time.RFC3339, + "2006-01-02T15:04:05.999999999Z0700", + "2006-01-02 15:04:05.999999999Z07:00", + "2006-01-02 15:04:05", +} + +func parseTimeString(s string) (time.Time, error) { + var lastErr error + for _, l := range timeLayouts { + t, err := time.Parse(l, s) + if err == nil { + return t.UTC(), nil + } + lastErr = err + } + return time.Time{}, lastErr +} + +// parseNumericTime interprets n as unix seconds, milliseconds, microseconds, or +// nanoseconds based on magnitude. Fractional seconds are supported. +func parseNumericTime(n json.Number) (time.Time, bool) { + f, err := n.Float64() + if err != nil || f <= 0 { + return time.Time{}, false + } + switch { + case f >= 1e18: // nanoseconds + return time.Unix(0, int64(f)).UTC(), true + case f >= 1e15: // microseconds + return time.Unix(0, int64(f*1e3)).UTC(), true + case f >= 1e12: // milliseconds + return time.Unix(0, int64(f*1e6)).UTC(), true + default: // seconds (possibly fractional) + sec := int64(f) + nsec := int64((f - float64(sec)) * 1e9) + return time.Unix(sec, nsec).UTC(), true + } +} + +// SubjectToken sanitizes a NATS subject into a filesystem/object-key-safe token, +// matching the logging stack's convention of replacing [^a-zA-Z0-9_.-] with '_'. +// Dots are preserved because subjects are dot-delimited. +func SubjectToken(subject string) string { + if subject == "" { + return "_" + } + var b strings.Builder + for _, r := range subject { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '_', r == '.', r == '-': + b.WriteRune(r) + default: + b.WriteRune('_') + } + } + return b.String() +} diff --git a/internal/event/event_test.go b/internal/event/event_test.go new file mode 100644 index 0000000..4bbf86f --- /dev/null +++ b/internal/event/event_test.go @@ -0,0 +1,88 @@ +package event + +import ( + "testing" + "time" +) + +func TestExtractK8s(t *testing.T) { + raw := []byte(`{"message":"hello from pod","stream":"stdout","timestamp":"2026-07-27T00:00:00Z","kubernetes":{"pod_name":"web-abc","pod_namespace":"shop","container_name":"web","pod_node_name":"node-1"},"ns_token":"shop","cont_token":"web"}`) + m := Extract(raw) + if m.Host != "node-1" { + t.Errorf("host = %q, want node-1 (pod_node_name)", m.Host) + } + if !m.Ok { + t.Fatalf("timestamp not parsed") + } + if !m.Timestamp.Equal(time.Date(2026, 7, 27, 0, 0, 0, 0, time.UTC)) { + t.Errorf("timestamp = %v", m.Timestamp) + } +} + +func TestExtractVMHostAndFallbacks(t *testing.T) { + raw := []byte(`{"message":"sshd started","host":"vm-db-1","severity":"info","role":"database","host_token":"vm-db-1"}`) + m := Extract(raw) + if m.Host != "vm-db-1" { + t.Errorf("host = %q, want vm-db-1", m.Host) + } + if m.Ok { + t.Errorf("no timestamp field present; Ok should be false") + } +} + +func TestExtractHostnameFallback(t *testing.T) { + m := Extract([]byte(`{"hostname":"legacy-box","ts":"2026-01-02T03:04:05Z"}`)) + if m.Host != "legacy-box" { + t.Errorf("host = %q, want legacy-box (hostname fallback)", m.Host) + } + if !m.Ok || !m.Timestamp.Equal(time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) { + t.Errorf("ts fallback failed: ok=%v ts=%v", m.Ok, m.Timestamp) + } +} + +func TestExtractHostPrecedence(t *testing.T) { + // .host wins over .kubernetes.pod_node_name when both present. + m := Extract([]byte(`{"host":"explicit","kubernetes":{"pod_node_name":"node-x"}}`)) + if m.Host != "explicit" { + t.Errorf("host precedence wrong: %q", m.Host) + } +} + +func TestExtractNumericTimestamp(t *testing.T) { + // unix millis + m := Extract([]byte(`{"host":"h","timestamp":1769472000000}`)) + if !m.Ok { + t.Fatalf("numeric millis not parsed") + } + if !m.Timestamp.Equal(time.Date(2026, 1, 27, 0, 0, 0, 0, time.UTC)) { + t.Errorf("numeric ts = %v", m.Timestamp.UTC()) + } +} + +func TestExtractMalformed(t *testing.T) { + m := Extract([]byte(`not json`)) + if m.Host != "" || m.Ok { + t.Errorf("malformed event should yield zero Meta, got %+v", m) + } +} + +func TestExtractEmptyHost(t *testing.T) { + m := Extract([]byte(`{"host":"","hostname":"backup"}`)) + if m.Host != "backup" { + t.Errorf("empty host should fall through to hostname, got %q", m.Host) + } +} + +func TestSubjectToken(t *testing.T) { + cases := map[string]string{ + "logs.k8s.vault.audit": "logs.k8s.vault.audit", + "logs.vm.vm-db-1": "logs.vm.vm-db-1", + "logs.k8s.a/b.c": "logs.k8s.a_b.c", + "": "_", + } + for in, want := range cases { + if got := SubjectToken(in); got != want { + t.Errorf("SubjectToken(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/index/ddl.go b/internal/index/ddl.go new file mode 100644 index 0000000..395f924 --- /dev/null +++ b/internal/index/ddl.go @@ -0,0 +1,42 @@ +package index + +import "fmt" + +// CreateDatabaseSQL creates the index database if absent. +func CreateDatabaseSQL(database string) string { + return fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", database) +} + +// CreateTableSQL returns the DDL for the archive index table. One row is written +// per archived S3 object. In-cluster the argocd bootstrap Job owns table +// creation (like the logging stack's clickhouse-schema PostSync hook); this DDL +// is also shipped as schema/archive_index.sql and applied by `logarchiver +// init-schema`. +// +// PARTITION BY month of min_ts keeps partitions coarse (few objects/day). +// ORDER BY (subject, min_ts) matches the primary search axes. A bloom_filter +// skip index on hosts accelerates host lookups without a per-host column. +func CreateTableSQL(database, table string) string { + return fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.%s +( + object_key String, + bucket LowCardinality(String), + subject LowCardinality(String), + hosts Array(LowCardinality(String)), + min_ts DateTime64(3), + max_ts DateTime64(3), + event_count UInt64, + raw_bytes UInt64, + stored_bytes UInt64, + compression LowCardinality(String), + cipher LowCardinality(String), + container_format LowCardinality(String), + key_name LowCardinality(String), + key_fingerprint String, + created_at DateTime64(3) DEFAULT now64(3), + INDEX idx_hosts hosts TYPE bloom_filter GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(min_ts) +ORDER BY (subject, min_ts, object_key)`, database, table) +} diff --git a/internal/index/index.go b/internal/index/index.go new file mode 100644 index 0000000..efade24 --- /dev/null +++ b/internal/index/index.go @@ -0,0 +1,159 @@ +// Package index writes and queries the ClickHouse archive index — one row per +// stored S3 object — so operators can answer "which objects hold vault logs +// from host X between Y and Z" without scanning S3. The concrete store is +// behind the Index interface so the archiver and CLI test against a fake. +package index + +import ( + "context" + "crypto/tls" + "fmt" + "time" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" +) + +// Row is one archive-index record. +type Row struct { + ObjectKey string + Bucket string + Subject string + Hosts []string + MinTS time.Time + MaxTS time.Time + EventCount uint64 + RawBytes uint64 + StoredBytes uint64 + Compression string + Cipher string + ContainerFormat string + KeyName string + KeyFingerprint string +} + +// Result is one row returned by Search (a subset relevant to retrieval). +type Result struct { + ObjectKey string + Bucket string + Subject string + Hosts []string + MinTS time.Time + MaxTS time.Time + EventCount uint64 + RawBytes uint64 + StoredBytes uint64 + KeyName string + KeyFingerprint string +} + +// Index is the archive-index surface. +type Index interface { + Insert(ctx context.Context, row Row) error + Search(ctx context.Context, q SearchQuery) ([]Result, error) + InitSchema(ctx context.Context) error + Ping(ctx context.Context) error + Close() error +} + +// Config configures the ClickHouse client. +type Config struct { + Address string // host:port (native protocol, 9000) + Database string + Table string + Username string + Password string + TLS bool +} + +// ClickHouse is the ClickHouse-backed Index. +type ClickHouse struct { + conn driver.Conn + database string + table string +} + +// NewClickHouse connects to ClickHouse. +func NewClickHouse(ctx context.Context, cfg Config) (*ClickHouse, error) { + opts := &clickhouse.Options{ + Addr: []string{cfg.Address}, + Auth: clickhouse.Auth{ + Database: cfg.Database, + Username: cfg.Username, + Password: cfg.Password, + }, + } + if cfg.TLS { + opts.TLS = &tls.Config{MinVersion: tls.VersionTLS12} + } + conn, err := clickhouse.Open(opts) + if err != nil { + return nil, fmt.Errorf("open clickhouse: %w", err) + } + ch := &ClickHouse{conn: conn, database: cfg.Database, table: cfg.Table} + return ch, nil +} + +// Ping verifies connectivity. +func (c *ClickHouse) Ping(ctx context.Context) error { + return c.conn.Ping(ctx) +} + +// Close closes the connection. +func (c *ClickHouse) Close() error { + return c.conn.Close() +} + +// InitSchema creates the database and table if absent (idempotent). +func (c *ClickHouse) InitSchema(ctx context.Context) error { + if err := c.conn.Exec(ctx, CreateDatabaseSQL(c.database)); err != nil { + return fmt.Errorf("create database: %w", err) + } + if err := c.conn.Exec(ctx, CreateTableSQL(c.database, c.table)); err != nil { + return fmt.Errorf("create table: %w", err) + } + return nil +} + +// Insert writes one row. +func (c *ClickHouse) Insert(ctx context.Context, row Row) error { + sql := fmt.Sprintf( + "INSERT INTO %s.%s (object_key, bucket, subject, hosts, min_ts, max_ts, event_count, raw_bytes, stored_bytes, compression, cipher, container_format, key_name, key_fingerprint) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + c.database, c.table) + err := c.conn.Exec(ctx, sql, + row.ObjectKey, row.Bucket, row.Subject, row.Hosts, + row.MinTS, row.MaxTS, row.EventCount, row.RawBytes, row.StoredBytes, + row.Compression, row.Cipher, row.ContainerFormat, row.KeyName, row.KeyFingerprint, + ) + if err != nil { + return fmt.Errorf("insert index row: %w", err) + } + return nil +} + +// Search runs the parameterized query built from q. +func (c *ClickHouse) Search(ctx context.Context, q SearchQuery) ([]Result, error) { + sql, args := buildSearchSQL(c.database, c.table, q) + rows, err := c.conn.Query(ctx, sql, args...) + if err != nil { + return nil, fmt.Errorf("search index: %w", err) + } + defer func() { _ = rows.Close() }() + + var out []Result + for rows.Next() { + var r Result + if err := rows.Scan( + &r.ObjectKey, &r.Bucket, &r.Subject, &r.Hosts, + &r.MinTS, &r.MaxTS, &r.EventCount, &r.RawBytes, &r.StoredBytes, + &r.KeyName, &r.KeyFingerprint, + ); err != nil { + return nil, fmt.Errorf("scan result: %w", err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate results: %w", err) + } + return out, nil +} diff --git a/internal/index/query.go b/internal/index/query.go new file mode 100644 index 0000000..56e2de0 --- /dev/null +++ b/internal/index/query.go @@ -0,0 +1,111 @@ +package index + +import ( + "fmt" + "strings" + "time" +) + +// SearchQuery describes an index search. Zero-valued fields are omitted. +type SearchQuery struct { + Subject string // NATS-style glob: '*' = one token, '>' = rest. Empty = any. + Host string // exact, or a glob containing '*'. Empty = any. + From time.Time // objects whose range overlaps [From,To] + To time.Time + Limit int +} + +// buildSearchSQL renders q into a parameterized ClickHouse SELECT and its args. +// It is pure so it can be unit-tested without a database. Placeholders use the +// clickhouse-go positional style (?), matching Query(ctx, sql, args...). +func buildSearchSQL(database, table string, q SearchQuery) (string, []any) { + var ( + where []string + args []any + ) + if q.Subject != "" { + where = append(where, "match(subject, ?)") + args = append(args, subjectToRegex(q.Subject)) + } + if q.Host != "" { + if strings.Contains(q.Host, "*") { + where = append(where, "arrayExists(h -> match(h, ?), hosts)") + args = append(args, hostGlobToRegex(q.Host)) + } else { + where = append(where, "has(hosts, ?)") + args = append(args, q.Host) + } + } + if !q.From.IsZero() { + // object overlaps the window if its max_ts is at/after From. + where = append(where, "max_ts >= ?") + args = append(args, q.From.UTC()) + } + if !q.To.IsZero() { + where = append(where, "min_ts <= ?") + args = append(args, q.To.UTC()) + } + + sql := fmt.Sprintf( + "SELECT object_key, bucket, subject, hosts, min_ts, max_ts, event_count, raw_bytes, stored_bytes, key_name, key_fingerprint FROM %s.%s", + database, table) + if len(where) > 0 { + sql += " WHERE " + strings.Join(where, " AND ") + } + sql += " ORDER BY min_ts, object_key" + if q.Limit > 0 { + sql += " LIMIT ?" + args = append(args, q.Limit) + } + return sql, args +} + +// subjectToRegex converts a NATS-style subject glob into an anchored regex for +// ClickHouse match(). '*' matches exactly one dot-delimited token; '>' (only +// meaningful as the final token) matches one or more trailing tokens. Literal +// dots and regex metacharacters are escaped. +func subjectToRegex(glob string) string { + tokens := strings.Split(glob, ".") + var parts []string + for i, tok := range tokens { + switch tok { + case "*": + parts = append(parts, `[^.]+`) + case ">": + // '>' consumes the rest; emit and stop. + if i == 0 { + return "^.+$" + } + return "^" + strings.Join(parts[:i], `\.`) + `(\..+)?$` + default: + parts = append(parts, regexEscape(tok)) + } + } + return "^" + strings.Join(parts, `\.`) + "$" +} + +// hostGlobToRegex converts a host glob (where '*' matches any run of +// characters, including dots in an FQDN) into an anchored regex for match(). +func hostGlobToRegex(glob string) string { + var b strings.Builder + b.WriteByte('^') + for _, seg := range strings.Split(glob, "*") { + b.WriteString(regexEscape(seg)) + b.WriteString(".*") + } + // Trim the trailing ".*" added after the last segment, then anchor. + out := strings.TrimSuffix(b.String(), ".*") + return out + "$" +} + +func regexEscape(s string) string { + const meta = `\.+*?()|[]{}^$` + var b strings.Builder + for _, r := range s { + if strings.ContainsRune(meta, r) { + b.WriteByte('\\') + } + b.WriteRune(r) + } + return b.String() +} diff --git a/internal/index/query_test.go b/internal/index/query_test.go new file mode 100644 index 0000000..bb0bc9b --- /dev/null +++ b/internal/index/query_test.go @@ -0,0 +1,110 @@ +package index + +import ( + "regexp" + "strings" + "testing" + "time" +) + +func TestSubjectToRegex(t *testing.T) { + cases := []struct { + glob string + match []string + nomatch []string + }{ + { + glob: "logs.vm.*", + match: []string{"logs.vm.db-1", "logs.vm.web"}, + nomatch: []string{"logs.vm", "logs.vm.db.1", "logs.k8s.x"}, + }, + { + glob: "logs.k8s.vault.>", + match: []string{"logs.k8s.vault.audit", "logs.k8s.vault.a.b", "logs.k8s.vault"}, + nomatch: []string{"logs.k8s.shop.web", "logs.vm.x"}, + }, + { + glob: "logs.vm.db-1", + match: []string{"logs.vm.db-1"}, + nomatch: []string{"logs.vm.db-2", "logs.vm.db-1.x"}, + }, + } + for _, c := range cases { + re := regexp.MustCompile(subjectToRegex(c.glob)) + for _, s := range c.match { + if !re.MatchString(s) { + t.Errorf("%q -> %q should match %q", c.glob, re.String(), s) + } + } + for _, s := range c.nomatch { + if re.MatchString(s) { + t.Errorf("%q -> %q should NOT match %q", c.glob, re.String(), s) + } + } + } +} + +func TestBuildSearchSQLFull(t *testing.T) { + from := time.Date(2026, 7, 26, 0, 0, 0, 0, time.UTC) + to := time.Date(2026, 7, 27, 0, 0, 0, 0, time.UTC) + sql, args := buildSearchSQL("logs", "archive_index", SearchQuery{ + Subject: "logs.k8s.vault.>", + Host: "node-1", + From: from, + To: to, + Limit: 50, + }) + if !strings.Contains(sql, "FROM logs.archive_index") { + t.Errorf("missing table: %s", sql) + } + for _, want := range []string{"match(subject, ?)", "has(hosts, ?)", "max_ts >= ?", "min_ts <= ?", "ORDER BY min_ts", "LIMIT ?"} { + if !strings.Contains(sql, want) { + t.Errorf("sql missing %q: %s", want, sql) + } + } + if len(args) != 5 { + t.Fatalf("args = %d, want 5: %v", len(args), args) + } + if args[1] != "node-1" { + t.Errorf("host arg = %v", args[1]) + } + if args[4] != 50 { + t.Errorf("limit arg = %v", args[4]) + } +} + +func TestBuildSearchSQLEmpty(t *testing.T) { + sql, args := buildSearchSQL("logs", "archive_index", SearchQuery{}) + if strings.Contains(sql, "WHERE") { + t.Errorf("empty query should have no WHERE: %s", sql) + } + if len(args) != 0 { + t.Errorf("args = %v, want none", args) + } +} + +func TestBuildSearchSQLHostGlob(t *testing.T) { + sql, args := buildSearchSQL("logs", "archive_index", SearchQuery{Host: "db-*"}) + if !strings.Contains(sql, "arrayExists(h -> match(h, ?), hosts)") { + t.Errorf("host glob should use arrayExists/match: %s", sql) + } + if len(args) != 1 { + t.Fatalf("args = %v", args) + } + re := regexp.MustCompile(args[0].(string)) + if !re.MatchString("db-1") || re.MatchString("web-1") { + t.Errorf("host glob regex wrong: %q", args[0]) + } +} + +func TestDDLContainsKeyColumns(t *testing.T) { + ddl := CreateTableSQL("logs", "archive_index") + for _, col := range []string{"object_key", "subject", "hosts", "min_ts", "max_ts", "event_count", "key_fingerprint", "bloom_filter"} { + if !strings.Contains(ddl, col) { + t.Errorf("DDL missing %q", col) + } + } + if !strings.Contains(ddl, "IF NOT EXISTS") { + t.Errorf("DDL should be idempotent") + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..e9a1548 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,97 @@ +// Package metrics exposes Prometheus metrics for the logarchiver service. The +// logging stack ships no metrics convention today, so this is the first +// /metrics endpoint there; it is opt-in via config. +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// Metrics holds the collectors. +type Metrics struct { + objectsStored *prometheus.CounterVec + eventsArchived *prometheus.CounterVec + rawBytes *prometheus.CounterVec + storedBytes *prometheus.CounterVec + storeFailures *prometheus.CounterVec + indexFailures *prometheus.CounterVec + messagesFetched prometheus.Counter + acksSent prometheus.Counter + batchesFlushed *prometheus.CounterVec + pendingEvents prometheus.Gauge +} + +// New registers the collectors on reg (use prometheus.DefaultRegisterer for the +// default /metrics handler). +func New(reg prometheus.Registerer) *Metrics { + f := promauto.With(reg) + return &Metrics{ + objectsStored: f.NewCounterVec(prometheus.CounterOpts{ + Name: "logarchiver_objects_stored_total", + Help: "Objects successfully sealed, uploaded and indexed.", + }, []string{"subject"}), + eventsArchived: f.NewCounterVec(prometheus.CounterOpts{ + Name: "logarchiver_events_archived_total", + Help: "Log events archived.", + }, []string{"subject"}), + rawBytes: f.NewCounterVec(prometheus.CounterOpts{ + Name: "logarchiver_raw_bytes_total", + Help: "Raw NDJSON bytes archived (pre-compression).", + }, []string{"subject"}), + storedBytes: f.NewCounterVec(prometheus.CounterOpts{ + Name: "logarchiver_stored_bytes_total", + Help: "Stored object bytes written to S3 (post-compression/encryption).", + }, []string{"subject"}), + storeFailures: f.NewCounterVec(prometheus.CounterOpts{ + Name: "logarchiver_store_failures_total", + Help: "Failed store attempts (seal/upload); batch not acked.", + }, []string{"subject"}), + indexFailures: f.NewCounterVec(prometheus.CounterOpts{ + Name: "logarchiver_index_failures_total", + Help: "Failed index writes after a successful S3 upload; batch not acked.", + }, []string{"subject"}), + messagesFetched: f.NewCounter(prometheus.CounterOpts{ + Name: "logarchiver_messages_fetched_total", + Help: "JetStream messages fetched.", + }), + acksSent: f.NewCounter(prometheus.CounterOpts{ + Name: "logarchiver_acks_total", + Help: "JetStream acknowledgements sent (after successful persist).", + }), + batchesFlushed: f.NewCounterVec(prometheus.CounterOpts{ + Name: "logarchiver_batches_flushed_total", + Help: "Batches flushed, labelled by trigger (full|age|shutdown).", + }, []string{"trigger"}), + pendingEvents: f.NewGauge(prometheus.GaugeOpts{ + Name: "logarchiver_pending_events", + Help: "Events currently buffered in open batches (unacked).", + }), + } +} + +// ObjectStored records a successful persist. +func (m *Metrics) ObjectStored(subject string, events int, rawBytes, storedBytes int64) { + m.objectsStored.WithLabelValues(subject).Inc() + m.eventsArchived.WithLabelValues(subject).Add(float64(events)) + m.rawBytes.WithLabelValues(subject).Add(float64(rawBytes)) + m.storedBytes.WithLabelValues(subject).Add(float64(storedBytes)) +} + +// StoreFailed records a seal/upload failure. +func (m *Metrics) StoreFailed(subject string) { m.storeFailures.WithLabelValues(subject).Inc() } + +// IndexFailed records an index-write failure. +func (m *Metrics) IndexFailed(subject string) { m.indexFailures.WithLabelValues(subject).Inc() } + +// MessagesFetched records fetched messages. +func (m *Metrics) MessagesFetched(n int) { m.messagesFetched.Add(float64(n)) } + +// Acked records sent acknowledgements. +func (m *Metrics) Acked(n int) { m.acksSent.Add(float64(n)) } + +// BatchFlushed records a flush by trigger. +func (m *Metrics) BatchFlushed(trigger string) { m.batchesFlushed.WithLabelValues(trigger).Inc() } + +// SetPending sets the pending-events gauge. +func (m *Metrics) SetPending(n int) { m.pendingEvents.Set(float64(n)) } diff --git a/internal/s3store/s3store.go b/internal/s3store/s3store.go new file mode 100644 index 0000000..7f32126 --- /dev/null +++ b/internal/s3store/s3store.go @@ -0,0 +1,140 @@ +// Package s3store wraps object storage (Ceph RGW via the S3 API) behind an +// interface so the archiver and CLI can be tested with a fake. Credentials come +// from the standard AWS_* environment (the cephrgw BucketAccess secret +// logs-archive-s3); this package only wires the custom endpoint, path-style +// addressing, and the internal Vault-PKI CA needed for s3.ceph.unkin.net. +package s3store + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net/http" + "os" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// ObjectStore is the minimal object-storage surface logarchiver needs. +type ObjectStore interface { + Put(ctx context.Context, key string, body io.Reader, size int64) error + Get(ctx context.Context, key string) (io.ReadCloser, error) + List(ctx context.Context, prefix string) ([]string, error) + Bucket() string +} + +// Config configures the S3 client. +type Config struct { + Endpoint string + Bucket string + Region string + PathStyle bool + CAFile string +} + +// Store is the S3-backed ObjectStore. +type Store struct { + client *s3.Client + bucket string +} + +// New builds a Store, trusting CAFile (in addition to the system roots) when set. +func New(ctx context.Context, cfg Config) (*Store, error) { + httpClient, err := httpClientWithCA(cfg.CAFile) + if err != nil { + return nil, err + } + region := cfg.Region + if region == "" { + region = "us-east-1" + } + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, + awsconfig.WithRegion(region), + awsconfig.WithHTTPClient(httpClient), + ) + if err != nil { + return nil, fmt.Errorf("load aws config: %w", err) + } + client := s3.NewFromConfig(awsCfg, func(o *s3.Options) { + if cfg.Endpoint != "" { + o.BaseEndpoint = &cfg.Endpoint + } + o.UsePathStyle = cfg.PathStyle + }) + return &Store{client: client, bucket: cfg.Bucket}, nil +} + +// Bucket returns the configured bucket name. +func (s *Store) Bucket() string { return s.bucket } + +// Put uploads body of the given size under key. +func (s *Store) Put(ctx context.Context, key string, body io.Reader, size int64) error { + _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &s.bucket, + Key: &key, + Body: body, + ContentLength: &size, + }) + if err != nil { + return fmt.Errorf("put s3://%s/%s: %w", s.bucket, key, err) + } + return nil +} + +// Get streams the object at key. +func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) { + out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &s.bucket, + Key: &key, + }) + if err != nil { + return nil, fmt.Errorf("get s3://%s/%s: %w", s.bucket, key, err) + } + return out.Body, nil +} + +// List returns object keys under prefix (paginated). +func (s *Store) List(ctx context.Context, prefix string) ([]string, error) { + var keys []string + p := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{ + Bucket: &s.bucket, + Prefix: &prefix, + }) + for p.HasMorePages() { + page, err := p.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("list s3://%s/%s: %w", s.bucket, prefix, err) + } + for _, obj := range page.Contents { + if obj.Key != nil { + keys = append(keys, *obj.Key) + } + } + } + return keys, nil +} + +func httpClientWithCA(caFile string) (*http.Client, error) { + if caFile == "" { + return http.DefaultClient, nil + } + pem, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("read s3 ca file %s: %w", caFile, err) + } + pool, err := x509.SystemCertPool() + if err != nil || pool == nil { + pool = x509.NewCertPool() + } + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("no certificates parsed from %s", caFile) + } + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + }, + }, nil +} diff --git a/internal/vaultgpg/client.go b/internal/vaultgpg/client.go new file mode 100644 index 0000000..c83f8db --- /dev/null +++ b/internal/vaultgpg/client.go @@ -0,0 +1,195 @@ +// Package vaultgpg is a thin client for Ben's Vault GPG secrets engine +// (vault-plugin-secrets-gpg), used two ways: +// +// - the service reads the ARMORED PUBLIC key (GET /keys/) to +// encrypt objects locally; and +// - the CLI decrypts a wrapped DEK (POST /decrypt/) to retrieve +// objects. The engine does whole-payload decrypt only, but logarchiver only +// ever sends it the tiny wrapped-DEK blob, so that is a non-issue. +// +// Auth mirrors passv: ambient VAULT_* env (token or ~/.vault-token) for humans, +// or the kubernetes auth method for the in-cluster service. +package vaultgpg + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + vault "github.com/hashicorp/vault/api" +) + +// Config configures the Vault client. +type Config struct { + Address string + Mount string // e.g. "gpg" + AuthMethod string // "token" | "kubernetes" + K8sRole string + K8sMount string // e.g. "k8s/au/syd1" + K8sJWTPath string + CAFile string +} + +// Client wraps the Vault API client for GPG-engine operations. +type Client struct { + api *vault.Client + mount string +} + +// New builds a Client and authenticates per cfg.AuthMethod. +func New(ctx context.Context, cfg Config) (*Client, error) { + vc := vault.DefaultConfig() + if err := vc.ReadEnvironment(); err != nil { + return nil, fmt.Errorf("read vault env: %w", err) + } + if cfg.Address != "" { + vc.Address = cfg.Address + } + if cfg.CAFile != "" { + if err := vc.ConfigureTLS(&vault.TLSConfig{CACert: cfg.CAFile}); err != nil { + return nil, fmt.Errorf("configure vault tls: %w", err) + } + } + api, err := vault.NewClient(vc) + if err != nil { + return nil, fmt.Errorf("new vault client: %w", err) + } + mount := cfg.Mount + if mount == "" { + mount = "gpg" + } + c := &Client{api: api, mount: strings.Trim(mount, "/")} + + switch cfg.AuthMethod { + case "", "token": + if api.Token() == "" { + tok, err := resolveToken() + if err != nil { + return nil, err + } + api.SetToken(tok) + } + case "kubernetes": + if err := c.k8sLogin(ctx, cfg); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unknown vault auth_method %q", cfg.AuthMethod) + } + return c, nil +} + +func (c *Client) k8sLogin(ctx context.Context, cfg Config) error { + jwtPath := cfg.K8sJWTPath + if jwtPath == "" { + jwtPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" + } + jwt, err := os.ReadFile(jwtPath) + if err != nil { + return fmt.Errorf("read service account token: %w", err) + } + mount := cfg.K8sMount + if mount == "" { + mount = "kubernetes" + } + path := fmt.Sprintf("auth/%s/login", strings.Trim(mount, "/")) + secret, err := c.api.Logical().WriteWithContext(ctx, path, map[string]any{ + "role": cfg.K8sRole, + "jwt": strings.TrimSpace(string(jwt)), + }) + if err != nil { + return fmt.Errorf("kubernetes login: %w", err) + } + if secret == nil || secret.Auth == nil || secret.Auth.ClientToken == "" { + return fmt.Errorf("kubernetes login returned no token") + } + c.api.SetToken(secret.Auth.ClientToken) + return nil +} + +// resolveToken mirrors passv: VAULT_TOKEN, else ~/.vault-token. +func resolveToken() (string, error) { + if t := os.Getenv("VAULT_TOKEN"); t != "" { + return t, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("no VAULT_TOKEN and cannot find home dir: %w", err) + } + data, err := os.ReadFile(filepath.Join(home, ".vault-token")) + if err != nil { + return "", fmt.Errorf("no VAULT_TOKEN and no ~/.vault-token: %w", err) + } + tok := strings.TrimSpace(string(data)) + if tok == "" { + return "", fmt.Errorf("~/.vault-token is empty") + } + return tok, nil +} + +// PublicKey is the result of reading a GPG engine key. +type PublicKey struct { + Armored string + Fingerprint string +} + +// FetchPublicKey reads /keys/ and returns the latest armored public +// key and its fingerprint. +func (c *Client) FetchPublicKey(ctx context.Context, name string) (PublicKey, error) { + path := fmt.Sprintf("%s/keys/%s", c.mount, name) + secret, err := c.api.Logical().ReadWithContext(ctx, path) + if err != nil { + return PublicKey{}, fmt.Errorf("read %s: %w", path, err) + } + if secret == nil || secret.Data == nil { + return PublicKey{}, fmt.Errorf("key %q not found at %s", name, path) + } + pub, _ := secret.Data["public_key"].(string) + if pub == "" { + return PublicKey{}, fmt.Errorf("key %q has no public_key field", name) + } + fpr, _ := secret.Data["fingerprint"].(string) + return PublicKey{Armored: pub, Fingerprint: fpr}, nil +} + +// Decrypt sends ciphertext (raw binary OpenPGP) to /decrypt/ and +// returns the plaintext. The engine auto-detects binary vs armored; we send +// base64 of the raw bytes, as passv does. +func (c *Client) Decrypt(ctx context.Context, name string, ciphertext []byte) ([]byte, error) { + path := fmt.Sprintf("%s/decrypt/%s", c.mount, name) + secret, err := c.api.Logical().WriteWithContext(ctx, path, map[string]any{ + "ciphertext": base64.StdEncoding.EncodeToString(ciphertext), + }) + if err != nil { + return nil, fmt.Errorf("decrypt via %s: %w", path, err) + } + if secret == nil || secret.Data == nil { + return nil, fmt.Errorf("decrypt returned no data") + } + b64, _ := secret.Data["plaintext"].(string) + if b64 == "" { + return nil, fmt.Errorf("decrypt returned no plaintext") + } + plain, err := base64.StdEncoding.DecodeString(b64) + if err != nil { + return nil, fmt.Errorf("decode plaintext: %w", err) + } + return plain, nil +} + +// TokenTTL returns the remaining lease TTL of the current token, for diagnostics. +func (c *Client) TokenTTL(ctx context.Context) (time.Duration, error) { + secret, err := c.api.Auth().Token().LookupSelfWithContext(ctx) + if err != nil { + return 0, err + } + ttl, err := secret.TokenTTL() + if err != nil { + return 0, err + } + return ttl, nil +} diff --git a/packaging/nfpm.yaml b/packaging/nfpm.yaml new file mode 100644 index 0000000..4ddd9d3 --- /dev/null +++ b/packaging/nfpm.yaml @@ -0,0 +1,52 @@ +--- +# nfpm config for building the logarchiver RPM (CLI use-case). +# Rendered through envsubst (see scripts/build-rpm.sh) then fed to `nfpm pkg`. + +name: ${PACKAGE_NAME} +version: ${PACKAGE_VERSION} +release: ${PACKAGE_RELEASE} +arch: ${PACKAGE_ARCH} +platform: ${PACKAGE_PLATFORM} +section: default +priority: extra +description: "${PACKAGE_DESCRIPTION}" + +maintainer: ${PACKAGE_MAINTAINER} +homepage: ${PACKAGE_HOMEPAGE} +license: ${PACKAGE_LICENSE} + +disable_globbing: false + +replaces: + - logarchiver +provides: + - logarchiver + +contents: + - src: dist/logarchiver + dst: /usr/bin/logarchiver + file_info: + mode: 0755 + owner: root + group: root + + # Shell completions (generated by scripts/build-rpm.sh before packaging). + - src: dist/completions/logarchiver.bash + dst: /usr/share/bash-completion/completions/logarchiver + file_info: + mode: 0644 + - src: dist/completions/_logarchiver + dst: /usr/share/zsh/site-functions/_logarchiver + file_info: + mode: 0644 + - src: dist/completions/logarchiver.fish + dst: /usr/share/fish/vendor_completions.d/logarchiver.fish + file_info: + mode: 0644 + + # Example config (documentation; not wired into any service by default). + - src: config.example.yaml + dst: /usr/share/doc/logarchiver/config.example.yaml + type: config + file_info: + mode: 0644 diff --git a/schema/archive_index.sql b/schema/archive_index.sql new file mode 100644 index 0000000..56dd0d1 --- /dev/null +++ b/schema/archive_index.sql @@ -0,0 +1,35 @@ +-- logarchiver archive index schema. +-- +-- One row is written per stored S3 object. In-cluster this DDL is owned by the +-- argocd bootstrap Job (a ClickHouse PostSync hook, like the logging stack's +-- clickhouse-schema job); `logarchiver init-schema` applies the same statements +-- for local/dev use, and `logarchiver init-schema --print` emits them. +-- +-- Keep this file in sync with internal/index/ddl.go (the source of truth used by +-- init-schema). The database/table names below match the config defaults +-- (database `logs`, table `archive_index`). + +CREATE DATABASE IF NOT EXISTS logs; + +CREATE TABLE IF NOT EXISTS logs.archive_index +( + object_key String, -- S3 key of the stored object + bucket LowCardinality(String), -- S3 bucket (e.g. logs-archive) + subject LowCardinality(String), -- NATS subject the batch came from + hosts Array(LowCardinality(String)),-- distinct source hosts in the object + min_ts DateTime64(3), -- earliest event time in the object + max_ts DateTime64(3), -- latest event time in the object + event_count UInt64, -- number of events + raw_bytes UInt64, -- pre-compression NDJSON bytes + stored_bytes UInt64, -- stored object bytes (post zstd+encrypt) + compression LowCardinality(String), -- 'zstd' + cipher LowCardinality(String), -- 'AES-256-GCM' + container_format LowCardinality(String), -- 'LARC1' + key_name LowCardinality(String), -- Vault GPG engine key name + key_fingerprint String, -- 40-hex OpenPGP fingerprint (engine %X) + created_at DateTime64(3) DEFAULT now64(3), + INDEX idx_hosts hosts TYPE bloom_filter GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(min_ts) +ORDER BY (subject, min_ts, object_key); diff --git a/scripts/build-rpm.sh b/scripts/build-rpm.sh new file mode 100755 index 0000000..e2ae157 --- /dev/null +++ b/scripts/build-rpm.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Package the (already built) logarchiver binary into an RPM with nfpm, +# bundling generated bash/zsh/fish shell completions. +# Usage: scripts/build-rpm.sh [version] (version defaults to $CI_COMMIT_TAG) +# +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${ROOT_DIR}" + +VERSION="${1:-${CI_COMMIT_TAG:-0.0.0-dev}}" +VERSION="${VERSION#v}" # strip a leading v +BINARY="logarchiver" +DIST="dist" + +if [ ! -f "${DIST}/${BINARY}" ]; then + echo "ERROR: ${DIST}/${BINARY} not found; run 'make build' first" >&2 + exit 1 +fi + +# Generate shell completions from the freshly built binary so they always match +# the shipped flags/subcommands. +COMP_DIR="${DIST}/completions" +mkdir -p "${COMP_DIR}" +"./${DIST}/${BINARY}" completion bash >"${COMP_DIR}/${BINARY}.bash" +"./${DIST}/${BINARY}" completion zsh >"${COMP_DIR}/_${BINARY}" +"./${DIST}/${BINARY}" completion fish >"${COMP_DIR}/${BINARY}.fish" + +export PACKAGE_NAME="${BINARY}" +export PACKAGE_VERSION="${VERSION}" +export PACKAGE_RELEASE="1" +export PACKAGE_ARCH="amd64" +export PACKAGE_PLATFORM="linux" +export PACKAGE_DESCRIPTION="Archive NATS JetStream logs to S3 (zstd + OpenPGP, indexed) and search/retrieve them. Service + operator CLI." +export PACKAGE_MAINTAINER="Ben Vincent " +export PACKAGE_HOMEPAGE="https://git.unkin.net/unkin/logarchiver" +export PACKAGE_LICENSE="MIT" + +envsubst "${DIST}/nfpm.yaml" +nfpm pkg --config "${DIST}/nfpm.yaml" --target "${DIST}" --packager rpm + +echo "Built:" +ls -1 "${DIST}"/*.rpm